T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enable copy-on-write heap initialization by default (#3825)

Browse Source 
* Enable copy-on-write heap initialization by default

This commit enables the `Config::memfd` feature by default now that it's
been fuzzed for a few weeks on oss-fuzz, and will continue to be fuzzed
leading up to the next release of Wasmtime in early March. The
documentation of the `Config` option has been updated as well as adding
a CLI flag to disable the feature.

* Remove ubiquitous "memfd" terminology

Switch instead to forms of "memory image" or "cow" or some combination
thereof.

* Update new option names
This commit is contained in:
Alex Crichton
2022-02-22 17:12:18 -06:00
committed by GitHub
parent 593f8d96aa
commit bbd4a4a500
16 changed files with 294 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
crates/fuzzing/src/generators.rs
View File

@@ -256,8 +256,8 @@ pub struct WasmtimeConfig {
/// The Wasmtime memory configuration to use.
pub memory_config: MemoryConfig,
force_jump_veneers: bool,
memfd: bool,
memfd_guaranteed_dense_image_size: u64,
memory_init_cow: bool,
memory_guaranteed_dense_image_size: u64,
use_precompiled_cwasm: bool,
/// Configuration for the instance allocation strategy to use.
pub strategy: InstanceAllocationStrategy,
@@ -441,12 +441,12 @@ impl Config {
.cranelift_opt_level(self.wasmtime.opt_level.to_wasmtime())
.interruptable(self.wasmtime.interruptable)
.consume_fuel(self.wasmtime.consume_fuel)
.memfd(self.wasmtime.memfd)
.memfd_guaranteed_dense_image_size(std::cmp::min(
.memory_init_cow(self.wasmtime.memory_init_cow)
.memory_guaranteed_dense_image_size(std::cmp::min(
// Clamp this at 16MiB so we don't get huge in-memory
// images during fuzzing.
16 << 20,
self.wasmtime.memfd_guaranteed_dense_image_size,
self.wasmtime.memory_guaranteed_dense_image_size,
))
.allocation_strategy(self.wasmtime.strategy.to_wasmtime());

1
crates/runtime/Cargo.toml
View File

@@ -47,6 +47,7 @@ maintenance = { status = "actively-developed" }
[features]
default = []
memory-init-cow = ['memfd']
async = ["wasmtime-fiber"]

16
crates/runtime/build.rs
View File

@@ -11,17 +11,13 @@ fn main() {
.file("src/helpers.c")
.compile("wasmtime-helpers");
// Check to see if we are on Unix and the `memfd` feature is
// active. If so, enable the `memfd` rustc cfg so `#[cfg(memfd)]`
// will work.
//
// Note that while this is called memfd it only actually uses the `memfd`
// crate on Linux and on other Unix platforms this tries to reuse mmap'd
// `*.cwasm` files.
// Check to see if we are on Unix and the `memory-init-cow` feature is
// active. If so, enable the `memory_init_cow` rustc cfg so
// `#[cfg(memory_init_cow)]` will work.
let family = env::var("CARGO_CFG_TARGET_FAMILY").unwrap();
let is_memfd = env::var("CARGO_FEATURE_MEMFD").is_ok();
let memory_init_cow = env::var("CARGO_FEATURE_MEMORY_INIT_COW").is_ok();
let is_uffd = env::var("CARGO_FEATURE_UFFD").is_ok();
if &family == "unix" && is_memfd && !is_uffd {
println!("cargo:rustc-cfg=memfd");
if &family == "unix" && memory_init_cow && !is_uffd {
println!("cargo:rustc-cfg=memory_init_cow");
}
}

120
crates/runtime/src/memfd.rs → crates/runtime/src/cow.rs
View File

@@ -1,5 +1,5 @@
//! memfd support: creation of backing images for modules, and logic
//! to support mapping these backing images into memory.
//! Copy-on-write initialization support: creation of backing images for
//! modules, and logic to support mapping these backing images into memory.
use crate::InstantiationError;
use crate::MmapVec;
@@ -11,30 +11,30 @@ use std::sync::Arc;
use std::{convert::TryFrom, ops::Range};
use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, Module, PrimaryMap};
/// MemFDs containing backing images for certain memories in a module.
/// Backing images for memories in a module.
///
/// This is meant to be built once, when a module is first
/// loaded/constructed, and then used many times for instantiation.
pub struct ModuleMemFds {
memories: PrimaryMap<DefinedMemoryIndex, Option<Arc<MemoryMemFd>>>,
/// This is meant to be built once, when a module is first loaded/constructed,
/// and then used many times for instantiation.
pub struct ModuleMemoryImages {
memories: PrimaryMap<DefinedMemoryIndex, Option<Arc<MemoryImage>>>,
}
impl ModuleMemFds {
/// Get the MemoryMemFd for a given memory.
pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc<MemoryMemFd>> {
impl ModuleMemoryImages {
/// Get the MemoryImage for a given memory.
pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc<MemoryImage>> {
self.memories[defined_index].as_ref()
}
}
/// One backing image for one memory.
#[derive(Debug)]
pub struct MemoryMemFd {
pub struct MemoryImage {
/// The file descriptor source of this image.
///
/// This might be an mmaped `*.cwasm` file or on Linux it could also be a
/// `Memfd` as an anonymous file in memory. In either case this is used as
/// the backing-source for the CoW image.
fd: MemFdSource,
fd: FdSource,
/// Length of image, in bytes.
///
@@ -59,29 +59,29 @@ pub struct MemoryMemFd {
}
#[derive(Debug)]
enum MemFdSource {
enum FdSource {
Mmap(Arc<File>),
#[cfg(target_os = "linux")]
Memfd(memfd::Memfd),
}
impl MemFdSource {
impl FdSource {
fn as_file(&self) -> &File {
match self {
MemFdSource::Mmap(file) => file,
FdSource::Mmap(file) => file,
#[cfg(target_os = "linux")]
MemFdSource::Memfd(memfd) => memfd.as_file(),
FdSource::Memfd(memfd) => memfd.as_file(),
}
}
}
impl MemoryMemFd {
impl MemoryImage {
fn new(
page_size: u32,
offset: u64,
data: &[u8],
mmap: Option<&MmapVec>,
) -> Result<Option<MemoryMemFd>> {
) -> Result<Option<MemoryImage>> {
// Sanity-check that various parameters are page-aligned.
let len = data.len();
let offset = u32::try_from(offset).unwrap();
@@ -115,8 +115,8 @@ impl MemoryMemFd {
assert_eq!((mmap.original_offset() as u32) % page_size, 0);
if let Some(file) = mmap.original_file() {
return Ok(Some(MemoryMemFd {
fd: MemFdSource::Mmap(file.clone()),
return Ok(Some(MemoryImage {
fd: FdSource::Mmap(file.clone()),
fd_offset: u64::try_from(mmap.original_offset() + (data_start - start))
.unwrap(),
linear_memory_offset,
@@ -159,8 +159,8 @@ impl MemoryMemFd {
memfd.add_seal(memfd::FileSeal::SealWrite)?;
memfd.add_seal(memfd::FileSeal::SealSeal)?;
Ok(Some(MemoryMemFd {
fd: MemFdSource::Memfd(memfd),
Ok(Some(MemoryImage {
fd: FdSource::Memfd(memfd),
fd_offset: 0,
linear_memory_offset,
len,
@@ -188,15 +188,15 @@ fn create_memfd() -> Result<memfd::Memfd> {
.map_err(|e| e.into())
}
impl ModuleMemFds {
/// Create a new `ModuleMemFds` for the given module. This can be
impl ModuleMemoryImages {
/// Create a new `ModuleMemoryImages` for the given module. This can be
/// passed in as part of a `InstanceAllocationRequest` to speed up
/// instantiation and execution by using memfd-backed memories.
/// instantiation and execution by using copy-on-write-backed memories.
pub fn new(
module: &Module,
wasm_data: &[u8],
mmap: Option<&MmapVec>,
) -> Result<Option<ModuleMemFds>> {
) -> Result<Option<ModuleMemoryImages>> {
let map = match &module.memory_initialization {
MemoryInitialization::Static { map } => map,
_ => return Ok(None),
@@ -223,30 +223,30 @@ impl ModuleMemFds {
};
// Get the image for this wasm module as a subslice of `wasm_data`,
// and then use that to try to create the `MemoryMemFd`. If this
// creation files then we fail creating `ModuleMemFds` since this
// and then use that to try to create the `MemoryImage`. If this
// creation files then we fail creating `ModuleMemoryImages` since this
// memory couldn't be represented.
let data = &wasm_data[init.data.start as usize..init.data.end as usize];
let memfd = match MemoryMemFd::new(page_size, init.offset, data, mmap)? {
Some(memfd) => memfd,
let image = match MemoryImage::new(page_size, init.offset, data, mmap)? {
Some(image) => image,
None => return Ok(None),
};
let idx = memories.push(Some(Arc::new(memfd)));
let idx = memories.push(Some(Arc::new(image)));
assert_eq!(idx, defined_memory);
}
Ok(Some(ModuleMemFds { memories }))
Ok(Some(ModuleMemoryImages { memories }))
}
}
/// A single slot handled by the memfd instance-heap mechanism.
/// A single slot handled by the copy-on-write memory initialization mechanism.
///
/// The mmap scheme is:
///
/// base ==> (points here)
/// - (image.offset bytes) anonymous zero memory, pre-image
/// - (image.len bytes) CoW mapping of memfd heap image
/// - (image.len bytes) CoW mapping of memory image
/// - (up to static_size) anonymous zero memory, post-image
///
/// The ordering of mmaps to set this up is:
@@ -257,7 +257,7 @@ impl ModuleMemFds {
/// - per instantiation of new image in a slot:
/// - mmap of anonymous zero memory, from 0 to max heap size
/// (static_size)
/// - mmap of CoW'd memfd image, from `image.offset` to
/// - mmap of CoW'd image, from `image.offset` to
/// `image.offset + image.len`. This overwrites part of the
/// anonymous zero memory, potentially splitting it into a pre-
/// and post-region.
@@ -265,15 +265,15 @@ impl ModuleMemFds {
/// heap size; we re-mprotect it with R+W bits when the heap is
/// grown.
#[derive(Debug)]
pub struct MemFdSlot {
pub struct MemoryImageSlot {
/// The base of the actual heap memory. Bytes at this address are
/// what is seen by the Wasm guest code.
base: usize,
/// The maximum static memory size, plus post-guard.
static_size: usize,
/// The memfd image that backs this memory. May be `None`, in
/// The image that backs this memory. May be `None`, in
/// which case the memory is all zeroes.
pub(crate) image: Option<Arc<MemoryMemFd>>,
pub(crate) image: Option<Arc<MemoryImage>>,
/// The initial heap size.
initial_size: usize,
/// The current heap size. All memory above `base + cur_size`
@@ -290,19 +290,19 @@ pub struct MemFdSlot {
/// from offset 0 to `initial_size` are accessible R+W and the
/// rest of the slot is inaccessible.
dirty: bool,
/// Whether this MemFdSlot is responsible for mapping anonymous
/// Whether this MemoryImageSlot is responsible for mapping anonymous
/// memory (to hold the reservation while overwriting mappings
/// specific to this slot) in place when it is dropped. Default
/// on, unless the caller knows what they are doing.
clear_on_drop: bool,
}
impl MemFdSlot {
/// Create a new MemFdSlot. Assumes that there is an anonymous
impl MemoryImageSlot {
/// Create a new MemoryImageSlot. Assumes that there is an anonymous
/// mmap backing in the given range to start.
pub(crate) fn create(base_addr: *mut c_void, initial_size: usize, static_size: usize) -> Self {
let base = base_addr as usize;
MemFdSlot {
MemoryImageSlot {
base,
static_size,
initial_size,
@@ -313,7 +313,7 @@ impl MemFdSlot {
}
}
/// Inform the MemFdSlot that it should *not* clear the underlying
/// Inform the MemoryImageSlot that it should *not* clear the underlying
/// address space when dropped. This should be used only when the
/// caller will clear or reuse the address space in some other
/// way.
@@ -335,7 +335,7 @@ impl MemFdSlot {
pub(crate) fn instantiate(
&mut self,
initial_size_bytes: usize,
maybe_image: Option<&Arc<MemoryMemFd>>,
maybe_image: Option<&Arc<MemoryImage>>,
) -> Result<(), InstantiationError> {
assert!(!self.dirty);
assert_eq!(self.cur_size, self.initial_size);
@@ -543,17 +543,17 @@ impl MemFdSlot {
}
}
impl Drop for MemFdSlot {
impl Drop for MemoryImageSlot {
fn drop(&mut self) {
// The MemFdSlot may be dropped if there is an error during
// The MemoryImageSlot may be dropped if there is an error during
// instantiation: for example, if a memory-growth limiter
// disallows a guest from having a memory of a certain size,
// after we've already initialized the MemFdSlot.
// after we've already initialized the MemoryImageSlot.
//
// We need to return this region of the large pool mmap to a
// safe state (with no module-specific mappings). The
// MemFdSlot will not be returned to the MemoryPool, so a new
// MemFdSlot will be created and overwrite the mappings anyway
// MemoryImageSlot will not be returned to the MemoryPool, so a new
// MemoryImageSlot will be created and overwrite the mappings anyway
// on the slot's next use; but for safety and to avoid
// resource leaks it's better not to have stale mappings to a
// possibly-otherwise-dead module's image.
@@ -563,17 +563,17 @@ impl Drop for MemFdSlot {
// *can't* simply munmap, because that leaves a hole in the
// middle of the pooling allocator's big memory area that some
// other random mmap may swoop in and take, to be trampled
// over by the next MemFdSlot later.
// over by the next MemoryImageSlot later.
//
// Since we're in drop(), we can't sanely return an error if
// this mmap fails. Let's ignore the failure if so; the next
// MemFdSlot to be created for this slot will try to overwrite
// MemoryImageSlot to be created for this slot will try to overwrite
// the existing stale mappings, and return a failure properly
// if we still cannot map new memory.
//
// The exception to all of this is if the `unmap_on_drop` flag
// (which is set by default) is false. If so, the owner of
// this MemFdSlot has indicated that it will clean up in some
// this MemoryImageSlot has indicated that it will clean up in some
// other way.
if self.clear_on_drop {
let _ = self.reset_with_anon_memory();
@@ -585,12 +585,12 @@ impl Drop for MemFdSlot {
mod test {
use std::sync::Arc;
use super::{create_memfd, MemFdSlot, MemFdSource, MemoryMemFd};
use super::{create_memfd, FdSource, MemoryImage, MemoryImageSlot};
use crate::mmap::Mmap;
use anyhow::Result;
use std::io::Write;
fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryMemFd> {
fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryImage> {
// Offset must be page-aligned.
let page_size = region::page::size();
assert_eq!(offset & (page_size - 1), 0);
@@ -601,8 +601,8 @@ mod test {
let image_len = (data.len() + page_size - 1) & !(page_size - 1);
memfd.as_file().set_len(image_len as u64)?;
Ok(MemoryMemFd {
fd: MemFdSource::Memfd(memfd),
Ok(MemoryImage {
fd: FdSource::Memfd(memfd),
len: image_len,
fd_offset: 0,
linear_memory_offset: offset,
@@ -613,8 +613,8 @@ mod test {
fn instantiate_no_image() {
// 4 MiB mmap'd area, not accessible
let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap();
// Create a MemFdSlot on top of it
let mut memfd = MemFdSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
// Create a MemoryImageSlot on top of it
let mut memfd = MemoryImageSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
memfd.no_clear_on_drop();
assert!(!memfd.is_dirty());
// instantiate with 64 KiB initial size
@@ -645,8 +645,8 @@ mod test {
fn instantiate_image() {
// 4 MiB mmap'd area, not accessible
let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap();
// Create a MemFdSlot on top of it
let mut memfd = MemFdSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
// Create a MemoryImageSlot on top of it
let mut memfd = MemoryImageSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
memfd.no_clear_on_drop();
// Create an image with some data.
let image = Arc::new(create_memfd_with_data(4096, &[1, 2, 3, 4]).unwrap());

39
crates/runtime/src/memfd_disabled.rs → crates/runtime/src/cow_disabled.rs
View File

@@ -1,5 +1,5 @@
//! Shims for MemFdSlot when the memfd allocator is not
//! included. Enables unconditional use of the type and its methods
//! Shims for MemoryImageSlot when the copy-on-write memory initialization is
//! not included. Enables unconditional use of the type and its methods
//! throughout higher-level code.
use crate::{InstantiationError, MmapVec};
@@ -7,49 +7,46 @@ use anyhow::Result;
use std::sync::Arc;
use wasmtime_environ::{DefinedMemoryIndex, Module};
/// A shim for the memfd image container when memfd support is not
/// included.
pub enum ModuleMemFds {}
/// A shim for the memory image container when support is not included.
pub enum ModuleMemoryImages {}
/// A shim for an individual memory image.
#[allow(dead_code)]
pub enum MemoryMemFd {}
pub enum MemoryImage {}
impl ModuleMemFds {
/// Construct a new set of memfd images. This variant is used
/// when memfd support is not included; it always returns no
impl ModuleMemoryImages {
/// Construct a new set of memory images. This variant is used
/// when cow support is not included; it always returns no
/// images.
pub fn new(_: &Module, _: &[u8], _: Option<&MmapVec>) -> Result<Option<ModuleMemFds>> {
pub fn new(_: &Module, _: &[u8], _: Option<&MmapVec>) -> Result<Option<ModuleMemoryImages>> {
Ok(None)
}
/// Get the memfd image for a particular memory.
pub fn get_memory_image(&self, _: DefinedMemoryIndex) -> Option<&Arc<MemoryMemFd>> {
// Should be unreachable because the `Self` type is
// uninhabitable.
/// Get the memory image for a particular memory.
pub fn get_memory_image(&self, _: DefinedMemoryIndex) -> Option<&Arc<MemoryImage>> {
match *self {}
}
}
/// A placeholder for MemFdSlot when we have not included the pooling
/// A placeholder for MemoryImageSlot when we have not included the pooling
/// allocator.
///
/// To allow MemFdSlot to be unconditionally passed around in various
/// places (e.g. a `Memory`), we define a zero-sized type when memfd is
/// To allow MemoryImageSlot to be unconditionally passed around in various
/// places (e.g. a `Memory`), we define a zero-sized type when memory is
/// not included in the build.
#[derive(Debug)]
pub enum MemFdSlot {}
pub enum MemoryImageSlot {}
#[allow(dead_code)]
impl MemFdSlot {
impl MemoryImageSlot {
pub(crate) fn create(_: *mut libc::c_void, _: usize, _: usize) -> Self {
panic!("create() on invalid MemFdSlot");
panic!("create() on invalid MemoryImageSlot");
}
pub(crate) fn instantiate(
&mut self,
_: usize,
_: Option<&Arc<MemoryMemFd>>,
_: Option<&Arc<MemoryImage>>,
) -> Result<Self, InstantiationError> {
match *self {}
}

13
crates/runtime/src/instance/allocator.rs
View File

@@ -365,9 +365,9 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<(), I
},
&mut |memory_index, init| {
// If this initializer applies to a defined memory but that memory
// doesn't need initialization, due to something like uffd or memfd
// pre-initializing it via mmap magic, then this initializer can be
// skipped entirely.
// doesn't need initialization, due to something like uffd or
// copy-on-write pre-initializing it via mmap magic, then this
// initializer can be skipped entirely.
if let Some(memory_index) = module.defined_memory_index(memory_index) {
if !instance.memories[memory_index].needs_init() {
return true;
@@ -480,12 +480,11 @@ impl OnDemandInstanceAllocator {
let mut memories: PrimaryMap<DefinedMemoryIndex, _> =
PrimaryMap::with_capacity(module.memory_plans.len() - num_imports);
for (memory_idx, plan) in module.memory_plans.iter().skip(num_imports) {
// Create a MemFdSlot if there is an image for this memory.
let defined_memory_idx = module
.defined_memory_index(memory_idx)
.expect("Skipped imports, should never be None");
let memfd_image = runtime_info
.memfd_image(defined_memory_idx)
let image = runtime_info
.memory_image(defined_memory_idx)
.map_err(|err| InstantiationError::Resource(err.into()))?;
memories.push(
@@ -497,7 +496,7 @@ impl OnDemandInstanceAllocator {
.get()
.expect("if module has memory plans, store is not empty")
},
memfd_image,
image,
)
.map_err(InstantiationError::Resource)?,
);

68
crates/runtime/src/instance/allocator/pooling.rs
View File

@@ -12,7 +12,7 @@ use super::{
InstantiationError,
};
use crate::{instance::Instance, Memory, Mmap, Table};
use crate::{MemFdSlot, ModuleRuntimeInfo, Store};
use crate::{MemoryImageSlot, ModuleRuntimeInfo, Store};
use anyhow::{anyhow, bail, Context, Result};
use libc::c_void;
use std::convert::TryFrom;
@@ -259,7 +259,7 @@ pub enum PoolingAllocationStrategy {
impl Default for PoolingAllocationStrategy {
fn default() -> Self {
if cfg!(memfd) {
if cfg!(memory_init_cow) {
Self::ReuseAffinity
} else {
Self::NextAvailable
@@ -476,10 +476,12 @@ impl InstancePool {
};
if let Some(image) = runtime_info
.memfd_image(defined_index)
.memory_image(defined_index)
.map_err(|err| InstantiationError::Resource(err.into()))?
{
let mut slot = self.memories.take_memfd_slot(instance_index, defined_index);
let mut slot = self
.memories
.take_memory_image_slot(instance_index, defined_index);
let initial_size = plan.memory.minimum * WASM_PAGE_SIZE as u64;
// If instantiation fails, we can propagate the error
@@ -487,7 +489,7 @@ impl InstancePool {
// handler to attempt to map the range with PROT_NONE
// memory, to reserve the space while releasing any
// stale mappings. The next use of this slot will then
// create a new MemFdSlot that will try to map over
// create a new slot that will try to map over
// this, returning errors as well if the mapping
// errors persist. The unmap-on-drop is best effort;
// if it fails, then we can still soundly continue
@@ -531,16 +533,16 @@ impl InstancePool {
match memory {
Memory::Static {
memfd_slot: Some(mut memfd_slot),
memory_image: Some(mut image),
..
} => {
// If there was any error clearing the memfd, just
// If there was any error clearing the image, just
// drop it here, and let the drop handler for the
// MemFdSlot unmap in a way that retains the
// slot unmap in a way that retains the
// address space reservation.
if memfd_slot.clear_and_remain_ready().is_ok() {
if image.clear_and_remain_ready().is_ok() {
self.memories
.return_memfd_slot(instance_index, def_mem_idx, memfd_slot);
.return_memory_image_slot(instance_index, def_mem_idx, image);
}
}
@@ -630,10 +632,10 @@ impl InstancePool {
#[derive(Debug)]
struct MemoryPool {
mapping: Mmap,
// If using the memfd allocation scheme, the MemFd slots. We
// If using a copy-on-write allocation scheme, the slot management. We
// dynamically transfer ownership of a slot to a Memory when in
// use.
memfd_slots: Vec<Mutex<Option<MemFdSlot>>>,
image_slots: Vec<Mutex<Option<MemoryImageSlot>>>,
// The size, in bytes, of each linear memory's reservation plus the guard
// region allocated for it.
memory_size: usize,
@@ -718,18 +720,18 @@ impl MemoryPool {
let mapping = Mmap::accessible_reserved(0, allocation_size)
.context("failed to create memory pool mapping")?;
let num_memfd_slots = if cfg!(memfd) {
let num_image_slots = if cfg!(memory_init_cow) {
max_instances * max_memories
} else {
0
};
let memfd_slots: Vec<_> = std::iter::repeat_with(|| Mutex::new(None))
.take(num_memfd_slots)
let image_slots: Vec<_> = std::iter::repeat_with(|| Mutex::new(None))
.take(num_image_slots)
.collect();
let pool = Self {
mapping,
memfd_slots,
image_slots,
memory_size,
initial_memory_offset,
max_memories,
@@ -758,18 +760,18 @@ impl MemoryPool {
.map(move |i| self.get_base(instance_index, DefinedMemoryIndex::from_u32(i as u32)))
}
/// Take ownership of the given memfd slot. Must be returned via
/// `return_memfd_slot` when the instance is done using it.
fn take_memfd_slot(
/// Take ownership of the given image slot. Must be returned via
/// `return_memory_image_slot` when the instance is done using it.
fn take_memory_image_slot(
&self,
instance_index: usize,
memory_index: DefinedMemoryIndex,
) -> MemFdSlot {
) -> MemoryImageSlot {
let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize);
let maybe_slot = self.memfd_slots[idx].lock().unwrap().take();
let maybe_slot = self.image_slots[idx].lock().unwrap().take();
maybe_slot.unwrap_or_else(|| {
MemFdSlot::create(
MemoryImageSlot::create(
self.get_base(instance_index, memory_index) as *mut c_void,
0,
self.memory_size,
@@ -777,28 +779,28 @@ impl MemoryPool {
})
}
/// Return ownership of the given memfd slot.
fn return_memfd_slot(
/// Return ownership of the given image slot.
fn return_memory_image_slot(
&self,
instance_index: usize,
memory_index: DefinedMemoryIndex,
slot: MemFdSlot,
slot: MemoryImageSlot,
) {
assert!(!slot.is_dirty());
let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize);
*self.memfd_slots[idx].lock().unwrap() = Some(slot);
*self.image_slots[idx].lock().unwrap() = Some(slot);
}
}
impl Drop for MemoryPool {
fn drop(&mut self) {
// Clear the `clear_no_drop` flag (i.e., ask to *not* clear on
// drop) for all MemFdSlots, and then drop them here. This is
// drop) for all slots, and then drop them here. This is
// valid because the one `Mmap` that covers the whole region
// can just do its one munmap.
for mut memfd in std::mem::take(&mut self.memfd_slots) {
if let Some(memfd_slot) = memfd.get_mut().unwrap() {
memfd_slot.no_clear_on_drop();
for mut slot in std::mem::take(&mut self.image_slots) {
if let Some(slot) = slot.get_mut().unwrap() {
slot.no_clear_on_drop();
}
}
}
@@ -1145,7 +1147,7 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator {
#[cfg(test)]
mod test {
use super::*;
use crate::{CompiledModuleId, Imports, MemoryMemFd, StorePtr, VMSharedSignatureIndex};
use crate::{CompiledModuleId, Imports, MemoryImage, StorePtr, VMSharedSignatureIndex};
use std::sync::Arc;
use wasmtime_environ::{
DefinedFuncIndex, DefinedMemoryIndex, EntityRef, FunctionInfo, Global, GlobalInit, Memory,
@@ -1456,10 +1458,10 @@ mod test {
fn signature(&self, _: SignatureIndex) -> VMSharedSignatureIndex {
unimplemented!()
}
fn memfd_image(
fn memory_image(
&self,
_: DefinedMemoryIndex,
) -> anyhow::Result<Option<&Arc<MemoryMemFd>>> {
) -> anyhow::Result<Option<&Arc<MemoryImage>>> {
Ok(None)
}

26
crates/runtime/src/lib.rs
View File

@@ -19,7 +19,7 @@
clippy::use_self
)
)]
#![cfg_attr(not(memfd), allow(unused_variables, unreachable_code))]
#![cfg_attr(not(memory_init_cow), allow(unused_variables, unreachable_code))]
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
@@ -74,15 +74,15 @@ pub use crate::vmcontext::{
mod module_id;
pub use module_id::{CompiledModuleId, CompiledModuleIdAllocator};
#[cfg(memfd)]
mod memfd;
#[cfg(memfd)]
pub use crate::memfd::{MemFdSlot, MemoryMemFd, ModuleMemFds};
#[cfg(memory_init_cow)]
mod cow;
#[cfg(memory_init_cow)]
pub use crate::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
#[cfg(not(memfd))]
mod memfd_disabled;
#[cfg(not(memfd))]
pub use crate::memfd_disabled::{MemFdSlot, MemoryMemFd, ModuleMemFds};
#[cfg(not(memory_init_cow))]
mod cow_disabled;
#[cfg(not(memory_init_cow))]
pub use crate::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
/// Version number of this crate.
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -158,7 +158,7 @@ pub unsafe trait Store {
/// instance state.
///
/// When an instance is created, it holds an Arc<dyn ModuleRuntimeInfo>
/// so that it can get to signatures, metadata on functions, memfd and
/// so that it can get to signatures, metadata on functions, memory and
/// funcref-table images, etc. All of these things are ordinarily known
/// by the higher-level layers of Wasmtime. Specifically, the main
/// implementation of this trait is provided by
@@ -180,8 +180,10 @@ pub trait ModuleRuntimeInfo: Send + Sync + 'static {
/// `image_base`.
fn function_info(&self, func_index: DefinedFuncIndex) -> &FunctionInfo;
/// memfd images, if any, for this module.
fn memfd_image(&self, memory: DefinedMemoryIndex) -> anyhow::Result<Option<&Arc<MemoryMemFd>>>;
/// Returns the `MemoryImage` structure used for copy-on-write
/// initialization of the memory, if it's applicable.
fn memory_image(&self, memory: DefinedMemoryIndex)
-> anyhow::Result<Option<&Arc<MemoryImage>>>;
/// A unique ID for this particular module. This can be used to
/// allow for fastpaths to optimize a "re-instantiate the same

89
crates/runtime/src/memory.rs
View File

@@ -4,8 +4,8 @@
use crate::mmap::Mmap;
use crate::vmcontext::VMMemoryDefinition;
use crate::MemFdSlot;
use crate::MemoryMemFd;
use crate::MemoryImage;
use crate::MemoryImageSlot;
use crate::Store;
use anyhow::Error;
use anyhow::{bail, format_err, Result};
@@ -25,8 +25,8 @@ pub trait RuntimeMemoryCreator: Send + Sync {
plan: &MemoryPlan,
minimum: usize,
maximum: Option<usize>,
// Optionally, a memfd image for CoW backing.
memfd_image: Option<&Arc<MemoryMemFd>>,
// Optionally, a memory image for CoW backing.
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Box<dyn RuntimeLinearMemory>>;
}
@@ -40,13 +40,13 @@ impl RuntimeMemoryCreator for DefaultMemoryCreator {
plan: &MemoryPlan,
minimum: usize,
maximum: Option<usize>,
memfd_image: Option<&Arc<MemoryMemFd>>,
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Box<dyn RuntimeLinearMemory>> {
Ok(Box::new(MmapMemory::new(
plan,
minimum,
maximum,
memfd_image,
memory_image,
)?))
}
}
@@ -71,7 +71,7 @@ pub trait RuntimeLinearMemory: Send + Sync {
fn vmmemory(&self) -> VMMemoryDefinition;
/// Does this memory need initialization? It may not if it already
/// has initial contents courtesy of the `MemoryMemFd` passed to
/// has initial contents courtesy of the `MemoryImage` passed to
/// `RuntimeMemoryCreator::new_memory()`.
fn needs_init(&self) -> bool;
}
@@ -103,13 +103,9 @@ pub struct MmapMemory {
pre_guard_size: usize,
offset_guard_size: usize,
// A MemFd CoW mapping that provides the initial content of this
// An optional CoW mapping that provides the initial content of this
// MmapMemory, if mapped.
//
// N.B.: this comes after the `mmap` field above because it must
// be destructed first. It puts a placeholder mapping in place on
// drop, then the `mmap` above completely unmaps the region.
memfd: Option<MemFdSlot>,
memory_image: Option<MemoryImageSlot>,
}
impl MmapMemory {
@@ -118,7 +114,7 @@ impl MmapMemory {
plan: &MemoryPlan,
minimum: usize,
mut maximum: Option<usize>,
memfd_image: Option<&Arc<MemoryMemFd>>,
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Self> {
// It's a programmer error for these two configuration values to exceed
// the host available address space, so panic if such a configuration is
@@ -155,22 +151,22 @@ impl MmapMemory {
mmap.make_accessible(pre_guard_bytes, minimum)?;
}
// If a memfd image was specified, try to create the MemFdSlot on top of our mmap.
let memfd = match memfd_image {
// If a memory image was specified, try to create the MemoryImageSlot on
// top of our mmap.
let memory_image = match memory_image {
Some(image) => {
let base = unsafe { mmap.as_mut_ptr().add(pre_guard_bytes) };
let mut memfd_slot = MemFdSlot::create(
let mut slot = MemoryImageSlot::create(
base.cast(),
minimum,
alloc_bytes + extra_to_reserve_on_growth,
);
memfd_slot.instantiate(minimum, Some(image))?;
// On drop, we will unmap our mmap'd range that this
// memfd_slot was mapped on top of, so there is no
// need for the memfd_slot to wipe it with an
// anonymous mapping first.
memfd_slot.no_clear_on_drop();
Some(memfd_slot)
slot.instantiate(minimum, Some(image))?;
// On drop, we will unmap our mmap'd range that this slot was
// mapped on top of, so there is no need for the slot to wipe
// it with an anonymous mapping first.
slot.no_clear_on_drop();
Some(slot)
}
None => None,
};
@@ -182,7 +178,7 @@ impl MmapMemory {
pre_guard_size: pre_guard_bytes,
offset_guard_size: offset_guard_bytes,
extra_to_reserve_on_growth,
memfd,
memory_image,
})
}
}
@@ -215,19 +211,18 @@ impl RuntimeLinearMemory for MmapMemory {
new_mmap.as_mut_slice()[self.pre_guard_size..][..self.accessible]
.copy_from_slice(&self.mmap.as_slice()[self.pre_guard_size..][..self.accessible]);
// Now drop the MemFdSlot, if any. We've lost the CoW
// Now drop the MemoryImageSlot, if any. We've lost the CoW
// advantages by explicitly copying all data, but we have
// preserved all of its content; so we no longer need the
// memfd mapping. We need to do this before we
// (implicitly) drop the `mmap` field by overwriting it
// below.
let _ = self.memfd.take();
// mapping. We need to do this before we (implicitly) drop the
// `mmap` field by overwriting it below.
drop(self.memory_image.take());
self.mmap = new_mmap;
} else if let Some(memfd) = self.memfd.as_mut() {
// MemFdSlot has its own growth mechanisms; defer to its
} else if let Some(image) = self.memory_image.as_mut() {
// MemoryImageSlot has its own growth mechanisms; defer to its
// implementation.
memfd.set_heap_limit(new_size)?;
image.set_heap_limit(new_size)?;
} else {
// If the new size of this heap fits within the existing allocation
// then all we need to do is to make the new pages accessible. This
@@ -255,9 +250,9 @@ impl RuntimeLinearMemory for MmapMemory {
}
fn needs_init(&self) -> bool {
// If we're using a memfd CoW mapping, then no initialization
// If we're using a CoW mapping, then no initialization
// is needed.
self.memfd.is_none()
self.memory_image.is_none()
}
}
@@ -278,9 +273,9 @@ pub enum Memory {
/// fault.
make_accessible: Option<fn(*mut u8, usize) -> Result<()>>,
/// The MemFdSlot, if any, for this memory. Owned here and
/// The image management, if any, for this memory. Owned here and
/// returned to the pooling allocator when termination occurs.
memfd_slot: Option<MemFdSlot>,
memory_image: Option<MemoryImageSlot>,
/// Stores the pages in the linear memory that have faulted as guard pages when using the `uffd` feature.
/// These pages need their protection level reset before the memory can grow.
@@ -299,14 +294,14 @@ impl Memory {
plan: &MemoryPlan,
creator: &dyn RuntimeMemoryCreator,
store: &mut dyn Store,
memfd_image: Option<&Arc<MemoryMemFd>>,
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Self> {
let (minimum, maximum) = Self::limit_new(plan, store)?;
Ok(Memory::Dynamic(creator.new_memory(
plan,
minimum,
maximum,
memfd_image,
memory_image,
)?))
}
@@ -315,7 +310,7 @@ impl Memory {
plan: &MemoryPlan,
base: &'static mut [u8],
make_accessible: Option<fn(*mut u8, usize) -> Result<()>>,
memfd_slot: Option<MemFdSlot>,
memory_image: Option<MemoryImageSlot>,
store: &mut dyn Store,
) -> Result<Self> {
let (minimum, maximum) = Self::limit_new(plan, store)?;
@@ -335,7 +330,7 @@ impl Memory {
base,
size: minimum,
make_accessible,
memfd_slot,
memory_image,
#[cfg(all(feature = "uffd", target_os = "linux"))]
guard_page_faults: Vec::new(),
})
@@ -457,11 +452,11 @@ impl Memory {
/// Returns whether or not this memory needs initialization. It
/// may not if it already has initial content thanks to a CoW
/// mechanism like memfd.
/// mechanism.
pub(crate) fn needs_init(&self) -> bool {
match self {
Memory::Static {
memfd_slot: Some(ref slot),
memory_image: Some(slot),
..
} => !slot.has_image(),
Memory::Dynamic(mem) => mem.needs_init(),
@@ -542,7 +537,7 @@ impl Memory {
Memory::Static {
base,
size,
memfd_slot: Some(ref mut memfd_slot),
memory_image: Some(image),
..
} => {
// Never exceed static memory size
@@ -551,7 +546,7 @@ impl Memory {
return Ok(None);
}
if let Err(e) = memfd_slot.set_heap_limit(new_byte_size) {
if let Err(e) = image.set_heap_limit(new_byte_size) {
store.memory_grow_failed(&e);
return Ok(None);
}
@@ -564,7 +559,7 @@ impl Memory {
..
} => {
let make_accessible = make_accessible
.expect("make_accessible must be Some if this is not a MemFD memory");
.expect("make_accessible must be Some if this is not a CoW memory");
// Never exceed static memory size
if new_byte_size > base.len() {
@@ -658,7 +653,7 @@ impl Default for Memory {
base: &mut [],
size: 0,
make_accessible: Some(|_, _| unreachable!()),
memfd_slot: None,
memory_image: None,
#[cfg(all(feature = "uffd", target_os = "linux"))]
guard_page_faults: Vec::new(),
}

14
crates/wasmtime/Cargo.toml
View File

@@ -59,7 +59,7 @@ default = [
'parallel-compilation',
'cranelift',
'pooling-allocator',
'memfd',
'memory-init-cow',
'vtune',
]
@@ -101,10 +101,10 @@ all-arch = ["wasmtime-cranelift/all-arch"]
# need portable signal handling.
posix-signals-on-macos = ["wasmtime-runtime/posix-signals-on-macos"]
# Enables, on Linux, the usage of memfd mappings to enable instantiation to use
# copy-on-write to initialize linear memory for wasm modules which have
# compatible linear memories.
# Enables, on supported platforms, the usage of copy-on-write initialization of
# compatible linear memories. For more information see the documentation of
# `Config::memory_init_cow`.
#
# Enabling this feature has no effect on non-Linux platforms or when the `uffd`
# feature is enabled.
memfd = ["wasmtime-runtime/memfd"]
# Enabling this feature has no effect on unsupported platforms or when the
# `uffd` feature is enabled.
memory-init-cow = ["wasmtime-runtime/memory-init-cow"]

110
crates/wasmtime/src/config.rs
View File

@@ -104,8 +104,8 @@ pub struct Config {
pub(crate) module_version: ModuleVersionStrategy,
pub(crate) parallel_compilation: bool,
pub(crate) paged_memory_initialization: bool,
pub(crate) memfd: bool,
pub(crate) memfd_guaranteed_dense_image_size: u64,
pub(crate) memory_init_cow: bool,
pub(crate) memory_guaranteed_dense_image_size: u64,
}
impl Config {
@@ -131,8 +131,8 @@ impl Config {
parallel_compilation: true,
// Default to paged memory initialization when using uffd on linux
paged_memory_initialization: cfg!(all(target_os = "linux", feature = "uffd")),
memfd: false,
memfd_guaranteed_dense_image_size: 16 << 20,
memory_init_cow: true,
memory_guaranteed_dense_image_size: 16 << 20,
};
#[cfg(compiler)]
{
@@ -1178,71 +1178,91 @@ impl Config {
self
}
/// Configures whether `memfd`, if supported, will be used to initialize
/// applicable module memories.
/// Configures whether copy-on-write memory-mapped data is used to
/// initialize a linear memory.
///
/// This is a Linux-specific feature since `memfd` is only supported on
/// Linux. Support for this is also enabled by default at compile time but
/// is otherwise disabled at runtime by default. This feature needs to be
/// enabled to `true` for support to be used.
/// Initializing linear memory via a copy-on-write mapping can drastically
/// improve instantiation costs of a WebAssembly module because copying
/// memory is deferred. Additionally if a page of memory is only ever read
/// from WebAssembly and never written too then the same underlying page of
/// data will be reused between all instantiations of a module meaning that
/// if a module is instantiated many times this can lower the overall memory
/// required needed to run that module.
///
/// Also note that even if this feature is enabled it may not be applicable
/// to all memories in all wasm modules. At this time memories must meet
/// specific criteria to be memfd-initialized:
/// This feature is only applicable when a WebAssembly module meets specific
/// criteria to be initialized in this fashion, such as:
///
/// * Only memories defined in the module can be initialized this way.
/// * Data segments for memory must use statically known offsets.
/// * Data segments for memory must all be in-bounds.
///
/// If all of the above applies, this setting is enabled, and the current
/// platform is Linux the `memfd` will be used to efficiently initialize
/// linear memories with `mmap` to avoid copying data from initializers into
/// linear memory.
#[cfg(feature = "memfd")]
#[cfg_attr(nightlydoc, doc(cfg(feature = "memfd")))]
pub fn memfd(&mut self, memfd: bool) -> &mut Self {
self.memfd = memfd;
/// Modules which do not meet these criteria will fall back to
/// initialization of linear memory based on copying memory.
///
/// This feature of Wasmtime is also platform-specific:
///
/// * Linux - this feature is supported for all instances of [`Module`].
/// Modules backed by an existing mmap (such as those created by
/// [`Module::deserialize_file`]) will reuse that mmap to cow-initialize
/// memory. Other instance of [`Module`] may use the `memfd_create`
/// syscall to create an initialization image to `mmap`.
/// * Unix (not Linux) - this feature is only supported when loading modules
/// from a precompiled file via [`Module::deserialize_file`] where there
/// is a file descriptor to use to map data into the process. Note that
/// the module must have been compiled with this setting enabled as well.
/// * Windows - there is no support for this feature at this time. Memory
/// initialization will always copy bytes.
///
/// By default this option is enabled.
///
/// [`Module::deserialize_file`]: crate::Module::deserialize_file
/// [`Module`]: crate::Module
#[cfg(feature = "memory-init-cow")]
#[cfg_attr(nightlydoc, doc(cfg(feature = "memory-init-cow")))]
pub fn memory_init_cow(&mut self, enable: bool) -> &mut Self {
self.memory_init_cow = enable;
self
}
/// Configures the "guaranteed dense image size" for memfd.
/// Configures the "guaranteed dense image size" for copy-on-write
/// initialized memories.
///
/// When using the memfd feature to initialize memory efficiently,
/// compiled modules contain an image of the module's initial
/// heap. If the module has a fairly sparse initial heap, with
/// just a few data segments at very different offsets, this could
/// result in a large region of zero bytes in the image. In other
/// words, it's not very memory-efficient.
/// When using the [`Config::memory_init_cow`] feature to initialize memory
/// efficiently (which is enabled by default), compiled modules contain an
/// image of the module's initial heap. If the module has a fairly sparse
/// initial heap, with just a few data segments at very different offsets,
/// this could result in a large region of zero bytes in the image. In
/// other words, it's not very memory-efficient.
///
/// We normally use a heuristic to avoid this: if less than half
/// of the initialized range (first non-zero to last non-zero
/// byte) of any memory in the module has pages with nonzero
/// bytes, then we avoid memfd for the entire module.
/// bytes, then we avoid creating a memory image for the entire module.
///
/// However, if the embedder always needs the instantiation-time
/// efficiency of memfd, and is otherwise carefully controlling
/// parameters of the modules (for example, by limiting the
/// maximum heap size of the modules), then it may be desirable to
/// ensure memfd is used even if this could go against the
/// heuristic above. Thus, we add another condition: there is a
/// size of initialized data region up to which we *always* allow
/// memfd. The embedder can set this to a known maximum heap size
/// if they desire to always get the benefits of memfd.
/// However, if the embedder always needs the instantiation-time efficiency
/// of copy-on-write initialization, and is otherwise carefully controlling
/// parameters of the modules (for example, by limiting the maximum heap
/// size of the modules), then it may be desirable to ensure a memory image
/// is created even if this could go against the heuristic above. Thus, we
/// add another condition: there is a size of initialized data region up to
/// which we *always* allow a memory image. The embedder can set this to a
/// known maximum heap size if they desire to always get the benefits of
/// copy-on-write images.
///
/// In the future we may implement a "best of both worlds"
/// solution where we have a dense image up to some limit, and
/// then support a sparse list of initializers beyond that; this
/// would get most of the benefit of memfd and pay the incremental
/// would get most of the benefit of copy-on-write and pay the incremental
/// cost of eager initialization only for those bits of memory
/// that are out-of-bounds. However, for now, an embedder desiring
/// fast instantiation should ensure that this setting is as large
/// as the maximum module initial memory content size.
///
/// By default this value is 16 MiB.
#[cfg(feature = "memfd")]
#[cfg_attr(nightlydoc, doc(cfg(feature = "memfd")))]
pub fn memfd_guaranteed_dense_image_size(&mut self, size_in_bytes: u64) -> &mut Self {
self.memfd_guaranteed_dense_image_size = size_in_bytes;
#[cfg(feature = "memory-init-cow")]
#[cfg_attr(nightlydoc, doc(cfg(feature = "memory-init-cow")))]
pub fn memory_guaranteed_dense_image_size(&mut self, size_in_bytes: u64) -> &mut Self {
self.memory_guaranteed_dense_image_size = size_in_bytes;
self
}
@@ -1315,8 +1335,8 @@ impl Clone for Config {
module_version: self.module_version.clone(),
parallel_compilation: self.parallel_compilation,
paged_memory_initialization: self.paged_memory_initialization,
memfd: self.memfd,
memfd_guaranteed_dense_image_size: self.memfd_guaranteed_dense_image_size,
memory_init_cow: self.memory_init_cow,
memory_guaranteed_dense_image_size: self.memory_guaranteed_dense_image_size,
}
}
}

12
crates/wasmtime/src/lib.rs
View File

@@ -281,14 +281,14 @@
//! efficient reuse of resources for high-concurrency and
//! high-instantiation-count scenarios.
//!
//! * `memfd` - Enabled by default, this feature builds in support for a
//! Linux-specific feature of creating a `memfd` where applicable for a
//! [`Module`]'s initial memory. This makes instantiation much faster by
//! * `memory-init-cow` - Enabled by default, this feature builds in support
//! for, on supported platforms, initializing wasm linear memories with
//! copy-on-write heap mappings. This makes instantiation much faster by
//! `mmap`-ing the initial memory image into place instead of copying memory
//! into place, allowing sharing pages that end up only getting read and
//! otherwise using copy-on-write for efficient initialization of memory. Note
//! into place, allowing sharing pages that end up only getting read. Note
//! that this is simply compile-time support and this must also be enabled at
//! run-time via [`Config::memfd`].
//! run-time via [`Config::memory_init_cow`] (which is also enabled by
//! default).
//!
//! ## Examples
//!

35
crates/wasmtime/src/module.rs
View File

@@ -16,7 +16,7 @@ use wasmtime_environ::{
};
use wasmtime_jit::{CompiledModule, CompiledModuleInfo, TypeTables};
use wasmtime_runtime::{
CompiledModuleId, MemoryMemFd, MmapVec, ModuleMemFds, VMSharedSignatureIndex,
CompiledModuleId, MemoryImage, MmapVec, ModuleMemoryImages, VMSharedSignatureIndex,
};
mod registry;
@@ -114,11 +114,10 @@ struct ModuleInner {
types: Arc<TypeTables>,
/// Registered shared signature for the module.
signatures: Arc<SignatureCollection>,
/// A set of memfd images for memories, if any. Note that module
/// instantiation (hence the need for lazy init) may happen for
/// the same module concurrently in multiple Stores, so we use a
/// OnceCell.
memfds: OnceCell<Option<ModuleMemFds>>,
/// A set of initialization images for memories, if any. Note that module
/// instantiation (hence the need for lazy init) may happen for the same
/// module concurrently in multiple Stores, so we use a OnceCell.
memory_images: OnceCell<Option<ModuleMemoryImages>>,
}
impl Module {
@@ -430,9 +429,9 @@ impl Module {
// can either at runtime be implemented as a single memcpy to
// initialize memory or otherwise enabling virtual-memory-tricks
// such as mmap'ing from a file to get copy-on-write.
if engine.config().memfd {
if engine.config().memory_init_cow {
let align = engine.compiler().page_size_align();
let max_always_allowed = engine.config().memfd_guaranteed_dense_image_size;
let max_always_allowed = engine.config().memory_guaranteed_dense_image_size;
translation.try_static_init(align, max_always_allowed);
}
@@ -575,7 +574,7 @@ impl Module {
artifact_upvars: modules,
module_upvars,
signatures,
memfds: OnceCell::new(),
memory_images: OnceCell::new(),
}),
});
@@ -594,7 +593,7 @@ impl Module {
engine: engine.clone(),
types: types.clone(),
module,
memfds: OnceCell::new(),
memory_images: OnceCell::new(),
artifact_upvars: artifact_upvars
.iter()
.map(|i| artifacts[*i].clone())
@@ -720,7 +719,7 @@ impl Module {
types: self.inner.types.clone(),
engine: self.inner.engine.clone(),
module,
memfds: OnceCell::new(),
memory_images: OnceCell::new(),
artifact_upvars: artifact_upvars
.iter()
.map(|i| self.inner.artifact_upvars[*i].clone())
@@ -1032,21 +1031,21 @@ impl wasmtime_runtime::ModuleRuntimeInfo for ModuleInner {
self.module.func_info(index)
}
fn memfd_image(&self, memory: DefinedMemoryIndex) -> Result<Option<&Arc<MemoryMemFd>>> {
if !self.engine.config().memfd {
fn memory_image(&self, memory: DefinedMemoryIndex) -> Result<Option<&Arc<MemoryImage>>> {
if !self.engine.config().memory_init_cow {
return Ok(None);
}
let memfds = self.memfds.get_or_try_init(|| {
ModuleMemFds::new(
let images = self.memory_images.get_or_try_init(|| {
ModuleMemoryImages::new(
self.module.module(),
self.module.wasm_data(),
Some(self.module.mmap()),
)
})?;
Ok(memfds
Ok(images
.as_ref()
.and_then(|memfds| memfds.get_memory_image(memory)))
.and_then(|images| images.get_memory_image(memory)))
}
fn unique_id(&self) -> Option<CompiledModuleId> {
@@ -1138,7 +1137,7 @@ impl wasmtime_runtime::ModuleRuntimeInfo for BareModuleInfo {
&self.function_info[index]
}
fn memfd_image(&self, _memory: DefinedMemoryIndex) -> Result<Option<&Arc<MemoryMemFd>>> {
fn memory_image(&self, _memory: DefinedMemoryIndex) -> Result<Option<&Arc<MemoryImage>>> {
Ok(None)
}

4
crates/wasmtime/src/trampoline/memory.rs
View File

@@ -7,7 +7,7 @@ use std::convert::TryFrom;
use std::sync::Arc;
use wasmtime_environ::{EntityIndex, MemoryPlan, MemoryStyle, Module, WASM_PAGE_SIZE};
use wasmtime_runtime::{
MemoryMemFd, RuntimeLinearMemory, RuntimeMemoryCreator, VMMemoryDefinition,
MemoryImage, RuntimeLinearMemory, RuntimeMemoryCreator, VMMemoryDefinition,
};
pub fn create_memory(store: &mut StoreOpaque, memory: &MemoryType) -> Result<InstanceId> {
@@ -63,7 +63,7 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy {
plan: &MemoryPlan,
minimum: usize,
maximum: Option<usize>,
_: Option<&Arc<MemoryMemFd>>,
_: Option<&Arc<MemoryImage>>,
) -> Result<Box<dyn RuntimeLinearMemory>> {
let ty = MemoryType::from_wasmtime_memory(&plan.memory);
let reserved_size_in_bytes = match plan.style {