diff --git a/Cargo.toml b/Cargo.toml index 06184baafc..803626376a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,14 +98,14 @@ default = [ "vtune", "wasi-nn", "pooling-allocator", - "memfd", + "memory-init-cow", ] jitdump = ["wasmtime/jitdump"] vtune = ["wasmtime/vtune"] wasi-crypto = ["wasmtime-wasi-crypto"] wasi-nn = ["wasmtime-wasi-nn"] uffd = ["wasmtime/uffd"] -memfd = ["wasmtime/memfd"] +memory-init-cow = ["wasmtime/memory-init-cow"] pooling-allocator = ["wasmtime/pooling-allocator"] all-arch = ["wasmtime/all-arch"] posix-signals-on-macos = ["wasmtime/posix-signals-on-macos"] diff --git a/crates/fuzzing/src/generators.rs b/crates/fuzzing/src/generators.rs index 4393522486..cc1059d098 100644 --- a/crates/fuzzing/src/generators.rs +++ b/crates/fuzzing/src/generators.rs @@ -256,8 +256,8 @@ pub struct WasmtimeConfig { /// The Wasmtime memory configuration to use. pub memory_config: MemoryConfig, force_jump_veneers: bool, - memfd: bool, - memfd_guaranteed_dense_image_size: u64, + memory_init_cow: bool, + memory_guaranteed_dense_image_size: u64, use_precompiled_cwasm: bool, /// Configuration for the instance allocation strategy to use. pub strategy: InstanceAllocationStrategy, @@ -441,12 +441,12 @@ impl Config { .cranelift_opt_level(self.wasmtime.opt_level.to_wasmtime()) .interruptable(self.wasmtime.interruptable) .consume_fuel(self.wasmtime.consume_fuel) - .memfd(self.wasmtime.memfd) - .memfd_guaranteed_dense_image_size(std::cmp::min( + .memory_init_cow(self.wasmtime.memory_init_cow) + .memory_guaranteed_dense_image_size(std::cmp::min( // Clamp this at 16MiB so we don't get huge in-memory // images during fuzzing. 16 << 20, - self.wasmtime.memfd_guaranteed_dense_image_size, + self.wasmtime.memory_guaranteed_dense_image_size, )) .allocation_strategy(self.wasmtime.strategy.to_wasmtime()); diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index f3e8c90da2..50f94fac65 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -47,6 +47,7 @@ maintenance = { status = "actively-developed" } [features] default = [] +memory-init-cow = ['memfd'] async = ["wasmtime-fiber"] diff --git a/crates/runtime/build.rs b/crates/runtime/build.rs index 2cb442131c..58d04617ef 100644 --- a/crates/runtime/build.rs +++ b/crates/runtime/build.rs @@ -11,17 +11,13 @@ fn main() { .file("src/helpers.c") .compile("wasmtime-helpers"); - // Check to see if we are on Unix and the `memfd` feature is - // active. If so, enable the `memfd` rustc cfg so `#[cfg(memfd)]` - // will work. - // - // Note that while this is called memfd it only actually uses the `memfd` - // crate on Linux and on other Unix platforms this tries to reuse mmap'd - // `*.cwasm` files. + // Check to see if we are on Unix and the `memory-init-cow` feature is + // active. If so, enable the `memory_init_cow` rustc cfg so + // `#[cfg(memory_init_cow)]` will work. let family = env::var("CARGO_CFG_TARGET_FAMILY").unwrap(); - let is_memfd = env::var("CARGO_FEATURE_MEMFD").is_ok(); + let memory_init_cow = env::var("CARGO_FEATURE_MEMORY_INIT_COW").is_ok(); let is_uffd = env::var("CARGO_FEATURE_UFFD").is_ok(); - if &family == "unix" && is_memfd && !is_uffd { - println!("cargo:rustc-cfg=memfd"); + if &family == "unix" && memory_init_cow && !is_uffd { + println!("cargo:rustc-cfg=memory_init_cow"); } } diff --git a/crates/runtime/src/memfd.rs b/crates/runtime/src/cow.rs similarity index 89% rename from crates/runtime/src/memfd.rs rename to crates/runtime/src/cow.rs index f54d18f6c1..4a08ff662f 100644 --- a/crates/runtime/src/memfd.rs +++ b/crates/runtime/src/cow.rs @@ -1,5 +1,5 @@ -//! memfd support: creation of backing images for modules, and logic -//! to support mapping these backing images into memory. +//! Copy-on-write initialization support: creation of backing images for +//! modules, and logic to support mapping these backing images into memory. use crate::InstantiationError; use crate::MmapVec; @@ -11,30 +11,30 @@ use std::sync::Arc; use std::{convert::TryFrom, ops::Range}; use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, Module, PrimaryMap}; -/// MemFDs containing backing images for certain memories in a module. +/// Backing images for memories in a module. /// -/// This is meant to be built once, when a module is first -/// loaded/constructed, and then used many times for instantiation. -pub struct ModuleMemFds { - memories: PrimaryMap>>, +/// This is meant to be built once, when a module is first loaded/constructed, +/// and then used many times for instantiation. +pub struct ModuleMemoryImages { + memories: PrimaryMap>>, } -impl ModuleMemFds { - /// Get the MemoryMemFd for a given memory. - pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc> { +impl ModuleMemoryImages { + /// Get the MemoryImage for a given memory. + pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc> { self.memories[defined_index].as_ref() } } /// One backing image for one memory. #[derive(Debug)] -pub struct MemoryMemFd { +pub struct MemoryImage { /// The file descriptor source of this image. /// /// This might be an mmaped `*.cwasm` file or on Linux it could also be a /// `Memfd` as an anonymous file in memory. In either case this is used as /// the backing-source for the CoW image. - fd: MemFdSource, + fd: FdSource, /// Length of image, in bytes. /// @@ -59,29 +59,29 @@ pub struct MemoryMemFd { } #[derive(Debug)] -enum MemFdSource { +enum FdSource { Mmap(Arc), #[cfg(target_os = "linux")] Memfd(memfd::Memfd), } -impl MemFdSource { +impl FdSource { fn as_file(&self) -> &File { match self { - MemFdSource::Mmap(file) => file, + FdSource::Mmap(file) => file, #[cfg(target_os = "linux")] - MemFdSource::Memfd(memfd) => memfd.as_file(), + FdSource::Memfd(memfd) => memfd.as_file(), } } } -impl MemoryMemFd { +impl MemoryImage { fn new( page_size: u32, offset: u64, data: &[u8], mmap: Option<&MmapVec>, - ) -> Result> { + ) -> Result> { // Sanity-check that various parameters are page-aligned. let len = data.len(); let offset = u32::try_from(offset).unwrap(); @@ -115,8 +115,8 @@ impl MemoryMemFd { assert_eq!((mmap.original_offset() as u32) % page_size, 0); if let Some(file) = mmap.original_file() { - return Ok(Some(MemoryMemFd { - fd: MemFdSource::Mmap(file.clone()), + return Ok(Some(MemoryImage { + fd: FdSource::Mmap(file.clone()), fd_offset: u64::try_from(mmap.original_offset() + (data_start - start)) .unwrap(), linear_memory_offset, @@ -159,8 +159,8 @@ impl MemoryMemFd { memfd.add_seal(memfd::FileSeal::SealWrite)?; memfd.add_seal(memfd::FileSeal::SealSeal)?; - Ok(Some(MemoryMemFd { - fd: MemFdSource::Memfd(memfd), + Ok(Some(MemoryImage { + fd: FdSource::Memfd(memfd), fd_offset: 0, linear_memory_offset, len, @@ -188,15 +188,15 @@ fn create_memfd() -> Result { .map_err(|e| e.into()) } -impl ModuleMemFds { - /// Create a new `ModuleMemFds` for the given module. This can be +impl ModuleMemoryImages { + /// Create a new `ModuleMemoryImages` for the given module. This can be /// passed in as part of a `InstanceAllocationRequest` to speed up - /// instantiation and execution by using memfd-backed memories. + /// instantiation and execution by using copy-on-write-backed memories. pub fn new( module: &Module, wasm_data: &[u8], mmap: Option<&MmapVec>, - ) -> Result> { + ) -> Result> { let map = match &module.memory_initialization { MemoryInitialization::Static { map } => map, _ => return Ok(None), @@ -223,30 +223,30 @@ impl ModuleMemFds { }; // Get the image for this wasm module as a subslice of `wasm_data`, - // and then use that to try to create the `MemoryMemFd`. If this - // creation files then we fail creating `ModuleMemFds` since this + // and then use that to try to create the `MemoryImage`. If this + // creation files then we fail creating `ModuleMemoryImages` since this // memory couldn't be represented. let data = &wasm_data[init.data.start as usize..init.data.end as usize]; - let memfd = match MemoryMemFd::new(page_size, init.offset, data, mmap)? { - Some(memfd) => memfd, + let image = match MemoryImage::new(page_size, init.offset, data, mmap)? { + Some(image) => image, None => return Ok(None), }; - let idx = memories.push(Some(Arc::new(memfd))); + let idx = memories.push(Some(Arc::new(image))); assert_eq!(idx, defined_memory); } - Ok(Some(ModuleMemFds { memories })) + Ok(Some(ModuleMemoryImages { memories })) } } -/// A single slot handled by the memfd instance-heap mechanism. +/// A single slot handled by the copy-on-write memory initialization mechanism. /// /// The mmap scheme is: /// /// base ==> (points here) /// - (image.offset bytes) anonymous zero memory, pre-image -/// - (image.len bytes) CoW mapping of memfd heap image +/// - (image.len bytes) CoW mapping of memory image /// - (up to static_size) anonymous zero memory, post-image /// /// The ordering of mmaps to set this up is: @@ -257,7 +257,7 @@ impl ModuleMemFds { /// - per instantiation of new image in a slot: /// - mmap of anonymous zero memory, from 0 to max heap size /// (static_size) -/// - mmap of CoW'd memfd image, from `image.offset` to +/// - mmap of CoW'd image, from `image.offset` to /// `image.offset + image.len`. This overwrites part of the /// anonymous zero memory, potentially splitting it into a pre- /// and post-region. @@ -265,15 +265,15 @@ impl ModuleMemFds { /// heap size; we re-mprotect it with R+W bits when the heap is /// grown. #[derive(Debug)] -pub struct MemFdSlot { +pub struct MemoryImageSlot { /// The base of the actual heap memory. Bytes at this address are /// what is seen by the Wasm guest code. base: usize, /// The maximum static memory size, plus post-guard. static_size: usize, - /// The memfd image that backs this memory. May be `None`, in + /// The image that backs this memory. May be `None`, in /// which case the memory is all zeroes. - pub(crate) image: Option>, + pub(crate) image: Option>, /// The initial heap size. initial_size: usize, /// The current heap size. All memory above `base + cur_size` @@ -290,19 +290,19 @@ pub struct MemFdSlot { /// from offset 0 to `initial_size` are accessible R+W and the /// rest of the slot is inaccessible. dirty: bool, - /// Whether this MemFdSlot is responsible for mapping anonymous + /// Whether this MemoryImageSlot is responsible for mapping anonymous /// memory (to hold the reservation while overwriting mappings /// specific to this slot) in place when it is dropped. Default /// on, unless the caller knows what they are doing. clear_on_drop: bool, } -impl MemFdSlot { - /// Create a new MemFdSlot. Assumes that there is an anonymous +impl MemoryImageSlot { + /// Create a new MemoryImageSlot. Assumes that there is an anonymous /// mmap backing in the given range to start. pub(crate) fn create(base_addr: *mut c_void, initial_size: usize, static_size: usize) -> Self { let base = base_addr as usize; - MemFdSlot { + MemoryImageSlot { base, static_size, initial_size, @@ -313,7 +313,7 @@ impl MemFdSlot { } } - /// Inform the MemFdSlot that it should *not* clear the underlying + /// Inform the MemoryImageSlot that it should *not* clear the underlying /// address space when dropped. This should be used only when the /// caller will clear or reuse the address space in some other /// way. @@ -335,7 +335,7 @@ impl MemFdSlot { pub(crate) fn instantiate( &mut self, initial_size_bytes: usize, - maybe_image: Option<&Arc>, + maybe_image: Option<&Arc>, ) -> Result<(), InstantiationError> { assert!(!self.dirty); assert_eq!(self.cur_size, self.initial_size); @@ -543,17 +543,17 @@ impl MemFdSlot { } } -impl Drop for MemFdSlot { +impl Drop for MemoryImageSlot { fn drop(&mut self) { - // The MemFdSlot may be dropped if there is an error during + // The MemoryImageSlot may be dropped if there is an error during // instantiation: for example, if a memory-growth limiter // disallows a guest from having a memory of a certain size, - // after we've already initialized the MemFdSlot. + // after we've already initialized the MemoryImageSlot. // // We need to return this region of the large pool mmap to a // safe state (with no module-specific mappings). The - // MemFdSlot will not be returned to the MemoryPool, so a new - // MemFdSlot will be created and overwrite the mappings anyway + // MemoryImageSlot will not be returned to the MemoryPool, so a new + // MemoryImageSlot will be created and overwrite the mappings anyway // on the slot's next use; but for safety and to avoid // resource leaks it's better not to have stale mappings to a // possibly-otherwise-dead module's image. @@ -563,17 +563,17 @@ impl Drop for MemFdSlot { // *can't* simply munmap, because that leaves a hole in the // middle of the pooling allocator's big memory area that some // other random mmap may swoop in and take, to be trampled - // over by the next MemFdSlot later. + // over by the next MemoryImageSlot later. // // Since we're in drop(), we can't sanely return an error if // this mmap fails. Let's ignore the failure if so; the next - // MemFdSlot to be created for this slot will try to overwrite + // MemoryImageSlot to be created for this slot will try to overwrite // the existing stale mappings, and return a failure properly // if we still cannot map new memory. // // The exception to all of this is if the `unmap_on_drop` flag // (which is set by default) is false. If so, the owner of - // this MemFdSlot has indicated that it will clean up in some + // this MemoryImageSlot has indicated that it will clean up in some // other way. if self.clear_on_drop { let _ = self.reset_with_anon_memory(); @@ -585,12 +585,12 @@ impl Drop for MemFdSlot { mod test { use std::sync::Arc; - use super::{create_memfd, MemFdSlot, MemFdSource, MemoryMemFd}; + use super::{create_memfd, FdSource, MemoryImage, MemoryImageSlot}; use crate::mmap::Mmap; use anyhow::Result; use std::io::Write; - fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result { + fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result { // Offset must be page-aligned. let page_size = region::page::size(); assert_eq!(offset & (page_size - 1), 0); @@ -601,8 +601,8 @@ mod test { let image_len = (data.len() + page_size - 1) & !(page_size - 1); memfd.as_file().set_len(image_len as u64)?; - Ok(MemoryMemFd { - fd: MemFdSource::Memfd(memfd), + Ok(MemoryImage { + fd: FdSource::Memfd(memfd), len: image_len, fd_offset: 0, linear_memory_offset: offset, @@ -613,8 +613,8 @@ mod test { fn instantiate_no_image() { // 4 MiB mmap'd area, not accessible let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap(); - // Create a MemFdSlot on top of it - let mut memfd = MemFdSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20); + // Create a MemoryImageSlot on top of it + let mut memfd = MemoryImageSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20); memfd.no_clear_on_drop(); assert!(!memfd.is_dirty()); // instantiate with 64 KiB initial size @@ -645,8 +645,8 @@ mod test { fn instantiate_image() { // 4 MiB mmap'd area, not accessible let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap(); - // Create a MemFdSlot on top of it - let mut memfd = MemFdSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20); + // Create a MemoryImageSlot on top of it + let mut memfd = MemoryImageSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20); memfd.no_clear_on_drop(); // Create an image with some data. let image = Arc::new(create_memfd_with_data(4096, &[1, 2, 3, 4]).unwrap()); diff --git a/crates/runtime/src/memfd_disabled.rs b/crates/runtime/src/cow_disabled.rs similarity index 58% rename from crates/runtime/src/memfd_disabled.rs rename to crates/runtime/src/cow_disabled.rs index 161f4fef4b..a62ba7fca9 100644 --- a/crates/runtime/src/memfd_disabled.rs +++ b/crates/runtime/src/cow_disabled.rs @@ -1,5 +1,5 @@ -//! Shims for MemFdSlot when the memfd allocator is not -//! included. Enables unconditional use of the type and its methods +//! Shims for MemoryImageSlot when the copy-on-write memory initialization is +//! not included. Enables unconditional use of the type and its methods //! throughout higher-level code. use crate::{InstantiationError, MmapVec}; @@ -7,49 +7,46 @@ use anyhow::Result; use std::sync::Arc; use wasmtime_environ::{DefinedMemoryIndex, Module}; -/// A shim for the memfd image container when memfd support is not -/// included. -pub enum ModuleMemFds {} +/// A shim for the memory image container when support is not included. +pub enum ModuleMemoryImages {} /// A shim for an individual memory image. #[allow(dead_code)] -pub enum MemoryMemFd {} +pub enum MemoryImage {} -impl ModuleMemFds { - /// Construct a new set of memfd images. This variant is used - /// when memfd support is not included; it always returns no +impl ModuleMemoryImages { + /// Construct a new set of memory images. This variant is used + /// when cow support is not included; it always returns no /// images. - pub fn new(_: &Module, _: &[u8], _: Option<&MmapVec>) -> Result> { + pub fn new(_: &Module, _: &[u8], _: Option<&MmapVec>) -> Result> { Ok(None) } - /// Get the memfd image for a particular memory. - pub fn get_memory_image(&self, _: DefinedMemoryIndex) -> Option<&Arc> { - // Should be unreachable because the `Self` type is - // uninhabitable. + /// Get the memory image for a particular memory. + pub fn get_memory_image(&self, _: DefinedMemoryIndex) -> Option<&Arc> { match *self {} } } -/// A placeholder for MemFdSlot when we have not included the pooling +/// A placeholder for MemoryImageSlot when we have not included the pooling /// allocator. /// -/// To allow MemFdSlot to be unconditionally passed around in various -/// places (e.g. a `Memory`), we define a zero-sized type when memfd is +/// To allow MemoryImageSlot to be unconditionally passed around in various +/// places (e.g. a `Memory`), we define a zero-sized type when memory is /// not included in the build. #[derive(Debug)] -pub enum MemFdSlot {} +pub enum MemoryImageSlot {} #[allow(dead_code)] -impl MemFdSlot { +impl MemoryImageSlot { pub(crate) fn create(_: *mut libc::c_void, _: usize, _: usize) -> Self { - panic!("create() on invalid MemFdSlot"); + panic!("create() on invalid MemoryImageSlot"); } pub(crate) fn instantiate( &mut self, _: usize, - _: Option<&Arc>, + _: Option<&Arc>, ) -> Result { match *self {} } diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index ca2b21404a..3428d76c8d 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -365,9 +365,9 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<(), I }, &mut |memory_index, init| { // If this initializer applies to a defined memory but that memory - // doesn't need initialization, due to something like uffd or memfd - // pre-initializing it via mmap magic, then this initializer can be - // skipped entirely. + // doesn't need initialization, due to something like uffd or + // copy-on-write pre-initializing it via mmap magic, then this + // initializer can be skipped entirely. if let Some(memory_index) = module.defined_memory_index(memory_index) { if !instance.memories[memory_index].needs_init() { return true; @@ -480,12 +480,11 @@ impl OnDemandInstanceAllocator { let mut memories: PrimaryMap = PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); for (memory_idx, plan) in module.memory_plans.iter().skip(num_imports) { - // Create a MemFdSlot if there is an image for this memory. let defined_memory_idx = module .defined_memory_index(memory_idx) .expect("Skipped imports, should never be None"); - let memfd_image = runtime_info - .memfd_image(defined_memory_idx) + let image = runtime_info + .memory_image(defined_memory_idx) .map_err(|err| InstantiationError::Resource(err.into()))?; memories.push( @@ -497,7 +496,7 @@ impl OnDemandInstanceAllocator { .get() .expect("if module has memory plans, store is not empty") }, - memfd_image, + image, ) .map_err(InstantiationError::Resource)?, ); diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 75073982b1..ef9bb33455 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -12,7 +12,7 @@ use super::{ InstantiationError, }; use crate::{instance::Instance, Memory, Mmap, Table}; -use crate::{MemFdSlot, ModuleRuntimeInfo, Store}; +use crate::{MemoryImageSlot, ModuleRuntimeInfo, Store}; use anyhow::{anyhow, bail, Context, Result}; use libc::c_void; use std::convert::TryFrom; @@ -259,7 +259,7 @@ pub enum PoolingAllocationStrategy { impl Default for PoolingAllocationStrategy { fn default() -> Self { - if cfg!(memfd) { + if cfg!(memory_init_cow) { Self::ReuseAffinity } else { Self::NextAvailable @@ -476,10 +476,12 @@ impl InstancePool { }; if let Some(image) = runtime_info - .memfd_image(defined_index) + .memory_image(defined_index) .map_err(|err| InstantiationError::Resource(err.into()))? { - let mut slot = self.memories.take_memfd_slot(instance_index, defined_index); + let mut slot = self + .memories + .take_memory_image_slot(instance_index, defined_index); let initial_size = plan.memory.minimum * WASM_PAGE_SIZE as u64; // If instantiation fails, we can propagate the error @@ -487,7 +489,7 @@ impl InstancePool { // handler to attempt to map the range with PROT_NONE // memory, to reserve the space while releasing any // stale mappings. The next use of this slot will then - // create a new MemFdSlot that will try to map over + // create a new slot that will try to map over // this, returning errors as well if the mapping // errors persist. The unmap-on-drop is best effort; // if it fails, then we can still soundly continue @@ -531,16 +533,16 @@ impl InstancePool { match memory { Memory::Static { - memfd_slot: Some(mut memfd_slot), + memory_image: Some(mut image), .. } => { - // If there was any error clearing the memfd, just + // If there was any error clearing the image, just // drop it here, and let the drop handler for the - // MemFdSlot unmap in a way that retains the + // slot unmap in a way that retains the // address space reservation. - if memfd_slot.clear_and_remain_ready().is_ok() { + if image.clear_and_remain_ready().is_ok() { self.memories - .return_memfd_slot(instance_index, def_mem_idx, memfd_slot); + .return_memory_image_slot(instance_index, def_mem_idx, image); } } @@ -630,10 +632,10 @@ impl InstancePool { #[derive(Debug)] struct MemoryPool { mapping: Mmap, - // If using the memfd allocation scheme, the MemFd slots. We + // If using a copy-on-write allocation scheme, the slot management. We // dynamically transfer ownership of a slot to a Memory when in // use. - memfd_slots: Vec>>, + image_slots: Vec>>, // The size, in bytes, of each linear memory's reservation plus the guard // region allocated for it. memory_size: usize, @@ -718,18 +720,18 @@ impl MemoryPool { let mapping = Mmap::accessible_reserved(0, allocation_size) .context("failed to create memory pool mapping")?; - let num_memfd_slots = if cfg!(memfd) { + let num_image_slots = if cfg!(memory_init_cow) { max_instances * max_memories } else { 0 }; - let memfd_slots: Vec<_> = std::iter::repeat_with(|| Mutex::new(None)) - .take(num_memfd_slots) + let image_slots: Vec<_> = std::iter::repeat_with(|| Mutex::new(None)) + .take(num_image_slots) .collect(); let pool = Self { mapping, - memfd_slots, + image_slots, memory_size, initial_memory_offset, max_memories, @@ -758,18 +760,18 @@ impl MemoryPool { .map(move |i| self.get_base(instance_index, DefinedMemoryIndex::from_u32(i as u32))) } - /// Take ownership of the given memfd slot. Must be returned via - /// `return_memfd_slot` when the instance is done using it. - fn take_memfd_slot( + /// Take ownership of the given image slot. Must be returned via + /// `return_memory_image_slot` when the instance is done using it. + fn take_memory_image_slot( &self, instance_index: usize, memory_index: DefinedMemoryIndex, - ) -> MemFdSlot { + ) -> MemoryImageSlot { let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize); - let maybe_slot = self.memfd_slots[idx].lock().unwrap().take(); + let maybe_slot = self.image_slots[idx].lock().unwrap().take(); maybe_slot.unwrap_or_else(|| { - MemFdSlot::create( + MemoryImageSlot::create( self.get_base(instance_index, memory_index) as *mut c_void, 0, self.memory_size, @@ -777,28 +779,28 @@ impl MemoryPool { }) } - /// Return ownership of the given memfd slot. - fn return_memfd_slot( + /// Return ownership of the given image slot. + fn return_memory_image_slot( &self, instance_index: usize, memory_index: DefinedMemoryIndex, - slot: MemFdSlot, + slot: MemoryImageSlot, ) { assert!(!slot.is_dirty()); let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize); - *self.memfd_slots[idx].lock().unwrap() = Some(slot); + *self.image_slots[idx].lock().unwrap() = Some(slot); } } impl Drop for MemoryPool { fn drop(&mut self) { // Clear the `clear_no_drop` flag (i.e., ask to *not* clear on - // drop) for all MemFdSlots, and then drop them here. This is + // drop) for all slots, and then drop them here. This is // valid because the one `Mmap` that covers the whole region // can just do its one munmap. - for mut memfd in std::mem::take(&mut self.memfd_slots) { - if let Some(memfd_slot) = memfd.get_mut().unwrap() { - memfd_slot.no_clear_on_drop(); + for mut slot in std::mem::take(&mut self.image_slots) { + if let Some(slot) = slot.get_mut().unwrap() { + slot.no_clear_on_drop(); } } } @@ -1145,7 +1147,7 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator { #[cfg(test)] mod test { use super::*; - use crate::{CompiledModuleId, Imports, MemoryMemFd, StorePtr, VMSharedSignatureIndex}; + use crate::{CompiledModuleId, Imports, MemoryImage, StorePtr, VMSharedSignatureIndex}; use std::sync::Arc; use wasmtime_environ::{ DefinedFuncIndex, DefinedMemoryIndex, EntityRef, FunctionInfo, Global, GlobalInit, Memory, @@ -1456,10 +1458,10 @@ mod test { fn signature(&self, _: SignatureIndex) -> VMSharedSignatureIndex { unimplemented!() } - fn memfd_image( + fn memory_image( &self, _: DefinedMemoryIndex, - ) -> anyhow::Result>> { + ) -> anyhow::Result>> { Ok(None) } diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 7ff5474538..e279d91d8c 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -19,7 +19,7 @@ clippy::use_self ) )] -#![cfg_attr(not(memfd), allow(unused_variables, unreachable_code))] +#![cfg_attr(not(memory_init_cow), allow(unused_variables, unreachable_code))] use std::sync::atomic::AtomicU64; use std::sync::Arc; @@ -74,15 +74,15 @@ pub use crate::vmcontext::{ mod module_id; pub use module_id::{CompiledModuleId, CompiledModuleIdAllocator}; -#[cfg(memfd)] -mod memfd; -#[cfg(memfd)] -pub use crate::memfd::{MemFdSlot, MemoryMemFd, ModuleMemFds}; +#[cfg(memory_init_cow)] +mod cow; +#[cfg(memory_init_cow)] +pub use crate::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages}; -#[cfg(not(memfd))] -mod memfd_disabled; -#[cfg(not(memfd))] -pub use crate::memfd_disabled::{MemFdSlot, MemoryMemFd, ModuleMemFds}; +#[cfg(not(memory_init_cow))] +mod cow_disabled; +#[cfg(not(memory_init_cow))] +pub use crate::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages}; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -158,7 +158,7 @@ pub unsafe trait Store { /// instance state. /// /// When an instance is created, it holds an Arc -/// so that it can get to signatures, metadata on functions, memfd and +/// so that it can get to signatures, metadata on functions, memory and /// funcref-table images, etc. All of these things are ordinarily known /// by the higher-level layers of Wasmtime. Specifically, the main /// implementation of this trait is provided by @@ -180,8 +180,10 @@ pub trait ModuleRuntimeInfo: Send + Sync + 'static { /// `image_base`. fn function_info(&self, func_index: DefinedFuncIndex) -> &FunctionInfo; - /// memfd images, if any, for this module. - fn memfd_image(&self, memory: DefinedMemoryIndex) -> anyhow::Result>>; + /// Returns the `MemoryImage` structure used for copy-on-write + /// initialization of the memory, if it's applicable. + fn memory_image(&self, memory: DefinedMemoryIndex) + -> anyhow::Result>>; /// A unique ID for this particular module. This can be used to /// allow for fastpaths to optimize a "re-instantiate the same diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index 932ecff431..b310ad8a86 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -4,8 +4,8 @@ use crate::mmap::Mmap; use crate::vmcontext::VMMemoryDefinition; -use crate::MemFdSlot; -use crate::MemoryMemFd; +use crate::MemoryImage; +use crate::MemoryImageSlot; use crate::Store; use anyhow::Error; use anyhow::{bail, format_err, Result}; @@ -25,8 +25,8 @@ pub trait RuntimeMemoryCreator: Send + Sync { plan: &MemoryPlan, minimum: usize, maximum: Option, - // Optionally, a memfd image for CoW backing. - memfd_image: Option<&Arc>, + // Optionally, a memory image for CoW backing. + memory_image: Option<&Arc>, ) -> Result>; } @@ -40,13 +40,13 @@ impl RuntimeMemoryCreator for DefaultMemoryCreator { plan: &MemoryPlan, minimum: usize, maximum: Option, - memfd_image: Option<&Arc>, + memory_image: Option<&Arc>, ) -> Result> { Ok(Box::new(MmapMemory::new( plan, minimum, maximum, - memfd_image, + memory_image, )?)) } } @@ -71,7 +71,7 @@ pub trait RuntimeLinearMemory: Send + Sync { fn vmmemory(&self) -> VMMemoryDefinition; /// Does this memory need initialization? It may not if it already - /// has initial contents courtesy of the `MemoryMemFd` passed to + /// has initial contents courtesy of the `MemoryImage` passed to /// `RuntimeMemoryCreator::new_memory()`. fn needs_init(&self) -> bool; } @@ -103,13 +103,9 @@ pub struct MmapMemory { pre_guard_size: usize, offset_guard_size: usize, - // A MemFd CoW mapping that provides the initial content of this + // An optional CoW mapping that provides the initial content of this // MmapMemory, if mapped. - // - // N.B.: this comes after the `mmap` field above because it must - // be destructed first. It puts a placeholder mapping in place on - // drop, then the `mmap` above completely unmaps the region. - memfd: Option, + memory_image: Option, } impl MmapMemory { @@ -118,7 +114,7 @@ impl MmapMemory { plan: &MemoryPlan, minimum: usize, mut maximum: Option, - memfd_image: Option<&Arc>, + memory_image: Option<&Arc>, ) -> Result { // It's a programmer error for these two configuration values to exceed // the host available address space, so panic if such a configuration is @@ -155,22 +151,22 @@ impl MmapMemory { mmap.make_accessible(pre_guard_bytes, minimum)?; } - // If a memfd image was specified, try to create the MemFdSlot on top of our mmap. - let memfd = match memfd_image { + // If a memory image was specified, try to create the MemoryImageSlot on + // top of our mmap. + let memory_image = match memory_image { Some(image) => { let base = unsafe { mmap.as_mut_ptr().add(pre_guard_bytes) }; - let mut memfd_slot = MemFdSlot::create( + let mut slot = MemoryImageSlot::create( base.cast(), minimum, alloc_bytes + extra_to_reserve_on_growth, ); - memfd_slot.instantiate(minimum, Some(image))?; - // On drop, we will unmap our mmap'd range that this - // memfd_slot was mapped on top of, so there is no - // need for the memfd_slot to wipe it with an - // anonymous mapping first. - memfd_slot.no_clear_on_drop(); - Some(memfd_slot) + slot.instantiate(minimum, Some(image))?; + // On drop, we will unmap our mmap'd range that this slot was + // mapped on top of, so there is no need for the slot to wipe + // it with an anonymous mapping first. + slot.no_clear_on_drop(); + Some(slot) } None => None, }; @@ -182,7 +178,7 @@ impl MmapMemory { pre_guard_size: pre_guard_bytes, offset_guard_size: offset_guard_bytes, extra_to_reserve_on_growth, - memfd, + memory_image, }) } } @@ -215,19 +211,18 @@ impl RuntimeLinearMemory for MmapMemory { new_mmap.as_mut_slice()[self.pre_guard_size..][..self.accessible] .copy_from_slice(&self.mmap.as_slice()[self.pre_guard_size..][..self.accessible]); - // Now drop the MemFdSlot, if any. We've lost the CoW + // Now drop the MemoryImageSlot, if any. We've lost the CoW // advantages by explicitly copying all data, but we have // preserved all of its content; so we no longer need the - // memfd mapping. We need to do this before we - // (implicitly) drop the `mmap` field by overwriting it - // below. - let _ = self.memfd.take(); + // mapping. We need to do this before we (implicitly) drop the + // `mmap` field by overwriting it below. + drop(self.memory_image.take()); self.mmap = new_mmap; - } else if let Some(memfd) = self.memfd.as_mut() { - // MemFdSlot has its own growth mechanisms; defer to its + } else if let Some(image) = self.memory_image.as_mut() { + // MemoryImageSlot has its own growth mechanisms; defer to its // implementation. - memfd.set_heap_limit(new_size)?; + image.set_heap_limit(new_size)?; } else { // If the new size of this heap fits within the existing allocation // then all we need to do is to make the new pages accessible. This @@ -255,9 +250,9 @@ impl RuntimeLinearMemory for MmapMemory { } fn needs_init(&self) -> bool { - // If we're using a memfd CoW mapping, then no initialization + // If we're using a CoW mapping, then no initialization // is needed. - self.memfd.is_none() + self.memory_image.is_none() } } @@ -278,9 +273,9 @@ pub enum Memory { /// fault. make_accessible: Option Result<()>>, - /// The MemFdSlot, if any, for this memory. Owned here and + /// The image management, if any, for this memory. Owned here and /// returned to the pooling allocator when termination occurs. - memfd_slot: Option, + memory_image: Option, /// Stores the pages in the linear memory that have faulted as guard pages when using the `uffd` feature. /// These pages need their protection level reset before the memory can grow. @@ -299,14 +294,14 @@ impl Memory { plan: &MemoryPlan, creator: &dyn RuntimeMemoryCreator, store: &mut dyn Store, - memfd_image: Option<&Arc>, + memory_image: Option<&Arc>, ) -> Result { let (minimum, maximum) = Self::limit_new(plan, store)?; Ok(Memory::Dynamic(creator.new_memory( plan, minimum, maximum, - memfd_image, + memory_image, )?)) } @@ -315,7 +310,7 @@ impl Memory { plan: &MemoryPlan, base: &'static mut [u8], make_accessible: Option Result<()>>, - memfd_slot: Option, + memory_image: Option, store: &mut dyn Store, ) -> Result { let (minimum, maximum) = Self::limit_new(plan, store)?; @@ -335,7 +330,7 @@ impl Memory { base, size: minimum, make_accessible, - memfd_slot, + memory_image, #[cfg(all(feature = "uffd", target_os = "linux"))] guard_page_faults: Vec::new(), }) @@ -457,11 +452,11 @@ impl Memory { /// Returns whether or not this memory needs initialization. It /// may not if it already has initial content thanks to a CoW - /// mechanism like memfd. + /// mechanism. pub(crate) fn needs_init(&self) -> bool { match self { Memory::Static { - memfd_slot: Some(ref slot), + memory_image: Some(slot), .. } => !slot.has_image(), Memory::Dynamic(mem) => mem.needs_init(), @@ -542,7 +537,7 @@ impl Memory { Memory::Static { base, size, - memfd_slot: Some(ref mut memfd_slot), + memory_image: Some(image), .. } => { // Never exceed static memory size @@ -551,7 +546,7 @@ impl Memory { return Ok(None); } - if let Err(e) = memfd_slot.set_heap_limit(new_byte_size) { + if let Err(e) = image.set_heap_limit(new_byte_size) { store.memory_grow_failed(&e); return Ok(None); } @@ -564,7 +559,7 @@ impl Memory { .. } => { let make_accessible = make_accessible - .expect("make_accessible must be Some if this is not a MemFD memory"); + .expect("make_accessible must be Some if this is not a CoW memory"); // Never exceed static memory size if new_byte_size > base.len() { @@ -658,7 +653,7 @@ impl Default for Memory { base: &mut [], size: 0, make_accessible: Some(|_, _| unreachable!()), - memfd_slot: None, + memory_image: None, #[cfg(all(feature = "uffd", target_os = "linux"))] guard_page_faults: Vec::new(), } diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 51ef1cdfde..784c7dbbbc 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -59,7 +59,7 @@ default = [ 'parallel-compilation', 'cranelift', 'pooling-allocator', - 'memfd', + 'memory-init-cow', 'vtune', ] @@ -101,10 +101,10 @@ all-arch = ["wasmtime-cranelift/all-arch"] # need portable signal handling. posix-signals-on-macos = ["wasmtime-runtime/posix-signals-on-macos"] -# Enables, on Linux, the usage of memfd mappings to enable instantiation to use -# copy-on-write to initialize linear memory for wasm modules which have -# compatible linear memories. +# Enables, on supported platforms, the usage of copy-on-write initialization of +# compatible linear memories. For more information see the documentation of +# `Config::memory_init_cow`. # -# Enabling this feature has no effect on non-Linux platforms or when the `uffd` -# feature is enabled. -memfd = ["wasmtime-runtime/memfd"] +# Enabling this feature has no effect on unsupported platforms or when the +# `uffd` feature is enabled. +memory-init-cow = ["wasmtime-runtime/memory-init-cow"] diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index f8144cc172..00689904e9 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -104,8 +104,8 @@ pub struct Config { pub(crate) module_version: ModuleVersionStrategy, pub(crate) parallel_compilation: bool, pub(crate) paged_memory_initialization: bool, - pub(crate) memfd: bool, - pub(crate) memfd_guaranteed_dense_image_size: u64, + pub(crate) memory_init_cow: bool, + pub(crate) memory_guaranteed_dense_image_size: u64, } impl Config { @@ -131,8 +131,8 @@ impl Config { parallel_compilation: true, // Default to paged memory initialization when using uffd on linux paged_memory_initialization: cfg!(all(target_os = "linux", feature = "uffd")), - memfd: false, - memfd_guaranteed_dense_image_size: 16 << 20, + memory_init_cow: true, + memory_guaranteed_dense_image_size: 16 << 20, }; #[cfg(compiler)] { @@ -1178,71 +1178,91 @@ impl Config { self } - /// Configures whether `memfd`, if supported, will be used to initialize - /// applicable module memories. + /// Configures whether copy-on-write memory-mapped data is used to + /// initialize a linear memory. /// - /// This is a Linux-specific feature since `memfd` is only supported on - /// Linux. Support for this is also enabled by default at compile time but - /// is otherwise disabled at runtime by default. This feature needs to be - /// enabled to `true` for support to be used. + /// Initializing linear memory via a copy-on-write mapping can drastically + /// improve instantiation costs of a WebAssembly module because copying + /// memory is deferred. Additionally if a page of memory is only ever read + /// from WebAssembly and never written too then the same underlying page of + /// data will be reused between all instantiations of a module meaning that + /// if a module is instantiated many times this can lower the overall memory + /// required needed to run that module. /// - /// Also note that even if this feature is enabled it may not be applicable - /// to all memories in all wasm modules. At this time memories must meet - /// specific criteria to be memfd-initialized: + /// This feature is only applicable when a WebAssembly module meets specific + /// criteria to be initialized in this fashion, such as: /// /// * Only memories defined in the module can be initialized this way. /// * Data segments for memory must use statically known offsets. /// * Data segments for memory must all be in-bounds. /// - /// If all of the above applies, this setting is enabled, and the current - /// platform is Linux the `memfd` will be used to efficiently initialize - /// linear memories with `mmap` to avoid copying data from initializers into - /// linear memory. - #[cfg(feature = "memfd")] - #[cfg_attr(nightlydoc, doc(cfg(feature = "memfd")))] - pub fn memfd(&mut self, memfd: bool) -> &mut Self { - self.memfd = memfd; + /// Modules which do not meet these criteria will fall back to + /// initialization of linear memory based on copying memory. + /// + /// This feature of Wasmtime is also platform-specific: + /// + /// * Linux - this feature is supported for all instances of [`Module`]. + /// Modules backed by an existing mmap (such as those created by + /// [`Module::deserialize_file`]) will reuse that mmap to cow-initialize + /// memory. Other instance of [`Module`] may use the `memfd_create` + /// syscall to create an initialization image to `mmap`. + /// * Unix (not Linux) - this feature is only supported when loading modules + /// from a precompiled file via [`Module::deserialize_file`] where there + /// is a file descriptor to use to map data into the process. Note that + /// the module must have been compiled with this setting enabled as well. + /// * Windows - there is no support for this feature at this time. Memory + /// initialization will always copy bytes. + /// + /// By default this option is enabled. + /// + /// [`Module::deserialize_file`]: crate::Module::deserialize_file + /// [`Module`]: crate::Module + #[cfg(feature = "memory-init-cow")] + #[cfg_attr(nightlydoc, doc(cfg(feature = "memory-init-cow")))] + pub fn memory_init_cow(&mut self, enable: bool) -> &mut Self { + self.memory_init_cow = enable; self } - /// Configures the "guaranteed dense image size" for memfd. + /// Configures the "guaranteed dense image size" for copy-on-write + /// initialized memories. /// - /// When using the memfd feature to initialize memory efficiently, - /// compiled modules contain an image of the module's initial - /// heap. If the module has a fairly sparse initial heap, with - /// just a few data segments at very different offsets, this could - /// result in a large region of zero bytes in the image. In other - /// words, it's not very memory-efficient. + /// When using the [`Config::memory_init_cow`] feature to initialize memory + /// efficiently (which is enabled by default), compiled modules contain an + /// image of the module's initial heap. If the module has a fairly sparse + /// initial heap, with just a few data segments at very different offsets, + /// this could result in a large region of zero bytes in the image. In + /// other words, it's not very memory-efficient. /// /// We normally use a heuristic to avoid this: if less than half /// of the initialized range (first non-zero to last non-zero /// byte) of any memory in the module has pages with nonzero - /// bytes, then we avoid memfd for the entire module. + /// bytes, then we avoid creating a memory image for the entire module. /// - /// However, if the embedder always needs the instantiation-time - /// efficiency of memfd, and is otherwise carefully controlling - /// parameters of the modules (for example, by limiting the - /// maximum heap size of the modules), then it may be desirable to - /// ensure memfd is used even if this could go against the - /// heuristic above. Thus, we add another condition: there is a - /// size of initialized data region up to which we *always* allow - /// memfd. The embedder can set this to a known maximum heap size - /// if they desire to always get the benefits of memfd. + /// However, if the embedder always needs the instantiation-time efficiency + /// of copy-on-write initialization, and is otherwise carefully controlling + /// parameters of the modules (for example, by limiting the maximum heap + /// size of the modules), then it may be desirable to ensure a memory image + /// is created even if this could go against the heuristic above. Thus, we + /// add another condition: there is a size of initialized data region up to + /// which we *always* allow a memory image. The embedder can set this to a + /// known maximum heap size if they desire to always get the benefits of + /// copy-on-write images. /// /// In the future we may implement a "best of both worlds" /// solution where we have a dense image up to some limit, and /// then support a sparse list of initializers beyond that; this - /// would get most of the benefit of memfd and pay the incremental + /// would get most of the benefit of copy-on-write and pay the incremental /// cost of eager initialization only for those bits of memory /// that are out-of-bounds. However, for now, an embedder desiring /// fast instantiation should ensure that this setting is as large /// as the maximum module initial memory content size. /// /// By default this value is 16 MiB. - #[cfg(feature = "memfd")] - #[cfg_attr(nightlydoc, doc(cfg(feature = "memfd")))] - pub fn memfd_guaranteed_dense_image_size(&mut self, size_in_bytes: u64) -> &mut Self { - self.memfd_guaranteed_dense_image_size = size_in_bytes; + #[cfg(feature = "memory-init-cow")] + #[cfg_attr(nightlydoc, doc(cfg(feature = "memory-init-cow")))] + pub fn memory_guaranteed_dense_image_size(&mut self, size_in_bytes: u64) -> &mut Self { + self.memory_guaranteed_dense_image_size = size_in_bytes; self } @@ -1315,8 +1335,8 @@ impl Clone for Config { module_version: self.module_version.clone(), parallel_compilation: self.parallel_compilation, paged_memory_initialization: self.paged_memory_initialization, - memfd: self.memfd, - memfd_guaranteed_dense_image_size: self.memfd_guaranteed_dense_image_size, + memory_init_cow: self.memory_init_cow, + memory_guaranteed_dense_image_size: self.memory_guaranteed_dense_image_size, } } } diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index 7e8cd20e7e..9fb6233741 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -281,14 +281,14 @@ //! efficient reuse of resources for high-concurrency and //! high-instantiation-count scenarios. //! -//! * `memfd` - Enabled by default, this feature builds in support for a -//! Linux-specific feature of creating a `memfd` where applicable for a -//! [`Module`]'s initial memory. This makes instantiation much faster by +//! * `memory-init-cow` - Enabled by default, this feature builds in support +//! for, on supported platforms, initializing wasm linear memories with +//! copy-on-write heap mappings. This makes instantiation much faster by //! `mmap`-ing the initial memory image into place instead of copying memory -//! into place, allowing sharing pages that end up only getting read and -//! otherwise using copy-on-write for efficient initialization of memory. Note +//! into place, allowing sharing pages that end up only getting read. Note //! that this is simply compile-time support and this must also be enabled at -//! run-time via [`Config::memfd`]. +//! run-time via [`Config::memory_init_cow`] (which is also enabled by +//! default). //! //! ## Examples //! diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 3d0bc8bead..e71d3e28d2 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -16,7 +16,7 @@ use wasmtime_environ::{ }; use wasmtime_jit::{CompiledModule, CompiledModuleInfo, TypeTables}; use wasmtime_runtime::{ - CompiledModuleId, MemoryMemFd, MmapVec, ModuleMemFds, VMSharedSignatureIndex, + CompiledModuleId, MemoryImage, MmapVec, ModuleMemoryImages, VMSharedSignatureIndex, }; mod registry; @@ -114,11 +114,10 @@ struct ModuleInner { types: Arc, /// Registered shared signature for the module. signatures: Arc, - /// A set of memfd images for memories, if any. Note that module - /// instantiation (hence the need for lazy init) may happen for - /// the same module concurrently in multiple Stores, so we use a - /// OnceCell. - memfds: OnceCell>, + /// A set of initialization images for memories, if any. Note that module + /// instantiation (hence the need for lazy init) may happen for the same + /// module concurrently in multiple Stores, so we use a OnceCell. + memory_images: OnceCell>, } impl Module { @@ -430,9 +429,9 @@ impl Module { // can either at runtime be implemented as a single memcpy to // initialize memory or otherwise enabling virtual-memory-tricks // such as mmap'ing from a file to get copy-on-write. - if engine.config().memfd { + if engine.config().memory_init_cow { let align = engine.compiler().page_size_align(); - let max_always_allowed = engine.config().memfd_guaranteed_dense_image_size; + let max_always_allowed = engine.config().memory_guaranteed_dense_image_size; translation.try_static_init(align, max_always_allowed); } @@ -575,7 +574,7 @@ impl Module { artifact_upvars: modules, module_upvars, signatures, - memfds: OnceCell::new(), + memory_images: OnceCell::new(), }), }); @@ -594,7 +593,7 @@ impl Module { engine: engine.clone(), types: types.clone(), module, - memfds: OnceCell::new(), + memory_images: OnceCell::new(), artifact_upvars: artifact_upvars .iter() .map(|i| artifacts[*i].clone()) @@ -720,7 +719,7 @@ impl Module { types: self.inner.types.clone(), engine: self.inner.engine.clone(), module, - memfds: OnceCell::new(), + memory_images: OnceCell::new(), artifact_upvars: artifact_upvars .iter() .map(|i| self.inner.artifact_upvars[*i].clone()) @@ -1032,21 +1031,21 @@ impl wasmtime_runtime::ModuleRuntimeInfo for ModuleInner { self.module.func_info(index) } - fn memfd_image(&self, memory: DefinedMemoryIndex) -> Result>> { - if !self.engine.config().memfd { + fn memory_image(&self, memory: DefinedMemoryIndex) -> Result>> { + if !self.engine.config().memory_init_cow { return Ok(None); } - let memfds = self.memfds.get_or_try_init(|| { - ModuleMemFds::new( + let images = self.memory_images.get_or_try_init(|| { + ModuleMemoryImages::new( self.module.module(), self.module.wasm_data(), Some(self.module.mmap()), ) })?; - Ok(memfds + Ok(images .as_ref() - .and_then(|memfds| memfds.get_memory_image(memory))) + .and_then(|images| images.get_memory_image(memory))) } fn unique_id(&self) -> Option { @@ -1138,7 +1137,7 @@ impl wasmtime_runtime::ModuleRuntimeInfo for BareModuleInfo { &self.function_info[index] } - fn memfd_image(&self, _memory: DefinedMemoryIndex) -> Result>> { + fn memory_image(&self, _memory: DefinedMemoryIndex) -> Result>> { Ok(None) } diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs index bd47e45144..2bb91d644e 100644 --- a/crates/wasmtime/src/trampoline/memory.rs +++ b/crates/wasmtime/src/trampoline/memory.rs @@ -7,7 +7,7 @@ use std::convert::TryFrom; use std::sync::Arc; use wasmtime_environ::{EntityIndex, MemoryPlan, MemoryStyle, Module, WASM_PAGE_SIZE}; use wasmtime_runtime::{ - MemoryMemFd, RuntimeLinearMemory, RuntimeMemoryCreator, VMMemoryDefinition, + MemoryImage, RuntimeLinearMemory, RuntimeMemoryCreator, VMMemoryDefinition, }; pub fn create_memory(store: &mut StoreOpaque, memory: &MemoryType) -> Result { @@ -63,7 +63,7 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy { plan: &MemoryPlan, minimum: usize, maximum: Option, - _: Option<&Arc>, + _: Option<&Arc>, ) -> Result> { let ty = MemoryType::from_wasmtime_memory(&plan.memory); let reserved_size_in_bytes = match plan.style { diff --git a/src/lib.rs b/src/lib.rs index 1a9b20c54f..fc40d9d0d4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -252,6 +252,12 @@ struct CommonOptions { #[structopt(long)] paged_memory_initialization: bool, + /// Disables the default of attempting to initialize linear memory via a + /// copy-on-write mapping. + #[cfg(feature = "memory-init-cow")] + #[structopt(long)] + disable_memory_init_cow: bool, + /// Enables the pooling allocator, in place of the on-demand /// allocator. #[cfg(feature = "pooling-allocator")] @@ -335,6 +341,8 @@ impl CommonOptions { config.epoch_interruption(self.epoch_interruption); config.generate_address_map(!self.disable_address_map); config.paged_memory_initialization(self.paged_memory_initialization); + #[cfg(feature = "memory-init-cow")] + config.memory_init_cow(!self.disable_memory_init_cow); #[cfg(feature = "pooling-allocator")] {