T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enable copy-on-write heap initialization by default (#3825)

Browse Source 
* Enable copy-on-write heap initialization by default

This commit enables the `Config::memfd` feature by default now that it's
been fuzzed for a few weeks on oss-fuzz, and will continue to be fuzzed
leading up to the next release of Wasmtime in early March. The
documentation of the `Config` option has been updated as well as adding
a CLI flag to disable the feature.

* Remove ubiquitous "memfd" terminology

Switch instead to forms of "memory image" or "cow" or some combination
thereof.

* Update new option names
This commit is contained in:
Alex Crichton
2022-02-22 17:12:18 -06:00
committed by GitHub
parent 593f8d96aa
commit bbd4a4a500
16 changed files with 294 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.toml
View File

@@ -98,14 +98,14 @@ default = [
"vtune", "vtune",
"wasi-nn", "wasi-nn",
"pooling-allocator", "pooling-allocator",
"memfd", "memory-init-cow",
] ]
jitdump = ["wasmtime/jitdump"] jitdump = ["wasmtime/jitdump"]
vtune = ["wasmtime/vtune"] vtune = ["wasmtime/vtune"]
wasi-crypto = ["wasmtime-wasi-crypto"] wasi-crypto = ["wasmtime-wasi-crypto"]
wasi-nn = ["wasmtime-wasi-nn"] wasi-nn = ["wasmtime-wasi-nn"]
uffd = ["wasmtime/uffd"] uffd = ["wasmtime/uffd"]
memfd = ["wasmtime/memfd"] memory-init-cow = ["wasmtime/memory-init-cow"]
pooling-allocator = ["wasmtime/pooling-allocator"] pooling-allocator = ["wasmtime/pooling-allocator"]
all-arch = ["wasmtime/all-arch"] all-arch = ["wasmtime/all-arch"]
posix-signals-on-macos = ["wasmtime/posix-signals-on-macos"] posix-signals-on-macos = ["wasmtime/posix-signals-on-macos"]

10
crates/fuzzing/src/generators.rs
View File

@@ -256,8 +256,8 @@ pub struct WasmtimeConfig {
/// The Wasmtime memory configuration to use. /// The Wasmtime memory configuration to use.
pub memory_config: MemoryConfig, pub memory_config: MemoryConfig,
force_jump_veneers: bool, force_jump_veneers: bool,
memfd: bool, memory_init_cow: bool,
memfd_guaranteed_dense_image_size: u64, memory_guaranteed_dense_image_size: u64,
use_precompiled_cwasm: bool, use_precompiled_cwasm: bool,
/// Configuration for the instance allocation strategy to use. /// Configuration for the instance allocation strategy to use.
pub strategy: InstanceAllocationStrategy, pub strategy: InstanceAllocationStrategy,
@@ -441,12 +441,12 @@ impl Config {
.cranelift_opt_level(self.wasmtime.opt_level.to_wasmtime()) .cranelift_opt_level(self.wasmtime.opt_level.to_wasmtime())
.interruptable(self.wasmtime.interruptable) .interruptable(self.wasmtime.interruptable)
.consume_fuel(self.wasmtime.consume_fuel) .consume_fuel(self.wasmtime.consume_fuel)
.memfd(self.wasmtime.memfd) .memory_init_cow(self.wasmtime.memory_init_cow)
.memfd_guaranteed_dense_image_size(std::cmp::min( .memory_guaranteed_dense_image_size(std::cmp::min(
// Clamp this at 16MiB so we don't get huge in-memory // Clamp this at 16MiB so we don't get huge in-memory
// images during fuzzing. // images during fuzzing.
16 << 20, 16 << 20,
self.wasmtime.memfd_guaranteed_dense_image_size, self.wasmtime.memory_guaranteed_dense_image_size,
)) ))
.allocation_strategy(self.wasmtime.strategy.to_wasmtime()); .allocation_strategy(self.wasmtime.strategy.to_wasmtime());

1
crates/runtime/Cargo.toml
View File

@@ -47,6 +47,7 @@ maintenance = { status = "actively-developed" }
[features] [features]
default = [] default = []
memory-init-cow = ['memfd']
async = ["wasmtime-fiber"] async = ["wasmtime-fiber"]

16
crates/runtime/build.rs
View File

@@ -11,17 +11,13 @@ fn main() {
.file("src/helpers.c") .file("src/helpers.c")
.compile("wasmtime-helpers"); .compile("wasmtime-helpers");
// Check to see if we are on Unix and the `memfd` feature is // Check to see if we are on Unix and the `memory-init-cow` feature is
// active. If so, enable the `memfd` rustc cfg so `#[cfg(memfd)]` // active. If so, enable the `memory_init_cow` rustc cfg so
// will work. // `#[cfg(memory_init_cow)]` will work.
//
// Note that while this is called memfd it only actually uses the `memfd`
// crate on Linux and on other Unix platforms this tries to reuse mmap'd
// `*.cwasm` files.
let family = env::var("CARGO_CFG_TARGET_FAMILY").unwrap(); let family = env::var("CARGO_CFG_TARGET_FAMILY").unwrap();
let is_memfd = env::var("CARGO_FEATURE_MEMFD").is_ok(); let memory_init_cow = env::var("CARGO_FEATURE_MEMORY_INIT_COW").is_ok();
let is_uffd = env::var("CARGO_FEATURE_UFFD").is_ok(); let is_uffd = env::var("CARGO_FEATURE_UFFD").is_ok();
if &family == "unix" && is_memfd && !is_uffd { if &family == "unix" && memory_init_cow && !is_uffd {
println!("cargo:rustc-cfg=memfd"); println!("cargo:rustc-cfg=memory_init_cow");
} }
} }

120
crates/runtime/src/memfd.rs → crates/runtime/src/cow.rs
View File

@@ -1,5 +1,5 @@
//! memfd support: creation of backing images for modules, and logic //! Copy-on-write initialization support: creation of backing images for
//! to support mapping these backing images into memory. //! modules, and logic to support mapping these backing images into memory.
use crate::InstantiationError; use crate::InstantiationError;
use crate::MmapVec; use crate::MmapVec;
@@ -11,30 +11,30 @@ use std::sync::Arc;
use std::{convert::TryFrom, ops::Range}; use std::{convert::TryFrom, ops::Range};
use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, Module, PrimaryMap}; use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, Module, PrimaryMap};
/// MemFDs containing backing images for certain memories in a module. /// Backing images for memories in a module.
/// ///
/// This is meant to be built once, when a module is first /// This is meant to be built once, when a module is first loaded/constructed,
/// loaded/constructed, and then used many times for instantiation. /// and then used many times for instantiation.
pub struct ModuleMemFds { pub struct ModuleMemoryImages {
memories: PrimaryMap<DefinedMemoryIndex, Option<Arc<MemoryMemFd>>>, memories: PrimaryMap<DefinedMemoryIndex, Option<Arc<MemoryImage>>>,
} }
impl ModuleMemFds { impl ModuleMemoryImages {
/// Get the MemoryMemFd for a given memory. /// Get the MemoryImage for a given memory.
pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc<MemoryMemFd>> { pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc<MemoryImage>> {
self.memories[defined_index].as_ref() self.memories[defined_index].as_ref()
} }
} }
/// One backing image for one memory. /// One backing image for one memory.
#[derive(Debug)] #[derive(Debug)]
pub struct MemoryMemFd { pub struct MemoryImage {
/// The file descriptor source of this image. /// The file descriptor source of this image.
/// ///
/// This might be an mmaped `*.cwasm` file or on Linux it could also be a /// This might be an mmaped `*.cwasm` file or on Linux it could also be a
/// `Memfd` as an anonymous file in memory. In either case this is used as /// `Memfd` as an anonymous file in memory. In either case this is used as
/// the backing-source for the CoW image. /// the backing-source for the CoW image.
fd: MemFdSource, fd: FdSource,
/// Length of image, in bytes. /// Length of image, in bytes.
/// ///
@@ -59,29 +59,29 @@ pub struct MemoryMemFd {
} }
#[derive(Debug)] #[derive(Debug)]
enum MemFdSource { enum FdSource {
Mmap(Arc<File>), Mmap(Arc<File>),
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
Memfd(memfd::Memfd), Memfd(memfd::Memfd),
} }
impl MemFdSource { impl FdSource {
fn as_file(&self) -> &File { fn as_file(&self) -> &File {
match self { match self {
MemFdSource::Mmap(file) => file, FdSource::Mmap(file) => file,
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
MemFdSource::Memfd(memfd) => memfd.as_file(), FdSource::Memfd(memfd) => memfd.as_file(),
} }
} }
} }
impl MemoryMemFd { impl MemoryImage {
fn new( fn new(
page_size: u32, page_size: u32,
offset: u64, offset: u64,
data: &[u8], data: &[u8],
mmap: Option<&MmapVec>, mmap: Option<&MmapVec>,
) -> Result<Option<MemoryMemFd>> { ) -> Result<Option<MemoryImage>> {
// Sanity-check that various parameters are page-aligned. // Sanity-check that various parameters are page-aligned.
let len = data.len(); let len = data.len();
let offset = u32::try_from(offset).unwrap(); let offset = u32::try_from(offset).unwrap();
@@ -115,8 +115,8 @@ impl MemoryMemFd {
assert_eq!((mmap.original_offset() as u32) % page_size, 0); assert_eq!((mmap.original_offset() as u32) % page_size, 0);
if let Some(file) = mmap.original_file() { if let Some(file) = mmap.original_file() {
return Ok(Some(MemoryMemFd { return Ok(Some(MemoryImage {
fd: MemFdSource::Mmap(file.clone()), fd: FdSource::Mmap(file.clone()),
fd_offset: u64::try_from(mmap.original_offset() + (data_start - start)) fd_offset: u64::try_from(mmap.original_offset() + (data_start - start))
.unwrap(), .unwrap(),
linear_memory_offset, linear_memory_offset,
@@ -159,8 +159,8 @@ impl MemoryMemFd {
memfd.add_seal(memfd::FileSeal::SealWrite)?; memfd.add_seal(memfd::FileSeal::SealWrite)?;
memfd.add_seal(memfd::FileSeal::SealSeal)?; memfd.add_seal(memfd::FileSeal::SealSeal)?;
Ok(Some(MemoryMemFd { Ok(Some(MemoryImage {
fd: MemFdSource::Memfd(memfd), fd: FdSource::Memfd(memfd),
fd_offset: 0, fd_offset: 0,
linear_memory_offset, linear_memory_offset,
len, len,
@@ -188,15 +188,15 @@ fn create_memfd() -> Result<memfd::Memfd> {
.map_err(|e| e.into()) .map_err(|e| e.into())
} }
impl ModuleMemFds { impl ModuleMemoryImages {
/// Create a new `ModuleMemFds` for the given module. This can be /// Create a new `ModuleMemoryImages` for the given module. This can be
/// passed in as part of a `InstanceAllocationRequest` to speed up /// passed in as part of a `InstanceAllocationRequest` to speed up
/// instantiation and execution by using memfd-backed memories. /// instantiation and execution by using copy-on-write-backed memories.
pub fn new( pub fn new(
module: &Module, module: &Module,
wasm_data: &[u8], wasm_data: &[u8],
mmap: Option<&MmapVec>, mmap: Option<&MmapVec>,
) -> Result<Option<ModuleMemFds>> { ) -> Result<Option<ModuleMemoryImages>> {
let map = match &module.memory_initialization { let map = match &module.memory_initialization {
MemoryInitialization::Static { map } => map, MemoryInitialization::Static { map } => map,
_ => return Ok(None), _ => return Ok(None),
@@ -223,30 +223,30 @@ impl ModuleMemFds {
}; };
// Get the image for this wasm module as a subslice of `wasm_data`, // Get the image for this wasm module as a subslice of `wasm_data`,
// and then use that to try to create the `MemoryMemFd`. If this // and then use that to try to create the `MemoryImage`. If this
// creation files then we fail creating `ModuleMemFds` since this // creation files then we fail creating `ModuleMemoryImages` since this
// memory couldn't be represented. // memory couldn't be represented.
let data = &wasm_data[init.data.start as usize..init.data.end as usize]; let data = &wasm_data[init.data.start as usize..init.data.end as usize];
let memfd = match MemoryMemFd::new(page_size, init.offset, data, mmap)? { let image = match MemoryImage::new(page_size, init.offset, data, mmap)? {
Some(memfd) => memfd, Some(image) => image,
None => return Ok(None), None => return Ok(None),
}; };
let idx = memories.push(Some(Arc::new(memfd))); let idx = memories.push(Some(Arc::new(image)));
assert_eq!(idx, defined_memory); assert_eq!(idx, defined_memory);
} }
Ok(Some(ModuleMemFds { memories })) Ok(Some(ModuleMemoryImages { memories }))
} }
} }
/// A single slot handled by the memfd instance-heap mechanism. /// A single slot handled by the copy-on-write memory initialization mechanism.
/// ///
/// The mmap scheme is: /// The mmap scheme is:
/// ///
/// base ==> (points here) /// base ==> (points here)
/// - (image.offset bytes) anonymous zero memory, pre-image /// - (image.offset bytes) anonymous zero memory, pre-image
/// - (image.len bytes) CoW mapping of memfd heap image /// - (image.len bytes) CoW mapping of memory image
/// - (up to static_size) anonymous zero memory, post-image /// - (up to static_size) anonymous zero memory, post-image
/// ///
/// The ordering of mmaps to set this up is: /// The ordering of mmaps to set this up is:
@@ -257,7 +257,7 @@ impl ModuleMemFds {
/// - per instantiation of new image in a slot: /// - per instantiation of new image in a slot:
/// - mmap of anonymous zero memory, from 0 to max heap size /// - mmap of anonymous zero memory, from 0 to max heap size
/// (static_size) /// (static_size)
/// - mmap of CoW'd memfd image, from `image.offset` to /// - mmap of CoW'd image, from `image.offset` to
/// `image.offset + image.len`. This overwrites part of the /// `image.offset + image.len`. This overwrites part of the
/// anonymous zero memory, potentially splitting it into a pre- /// anonymous zero memory, potentially splitting it into a pre-
/// and post-region. /// and post-region.
@@ -265,15 +265,15 @@ impl ModuleMemFds {
/// heap size; we re-mprotect it with R+W bits when the heap is /// heap size; we re-mprotect it with R+W bits when the heap is
/// grown. /// grown.
#[derive(Debug)] #[derive(Debug)]
pub struct MemFdSlot { pub struct MemoryImageSlot {
/// The base of the actual heap memory. Bytes at this address are /// The base of the actual heap memory. Bytes at this address are
/// what is seen by the Wasm guest code. /// what is seen by the Wasm guest code.
base: usize, base: usize,
/// The maximum static memory size, plus post-guard. /// The maximum static memory size, plus post-guard.
static_size: usize, static_size: usize,
/// The memfd image that backs this memory. May be `None`, in /// The image that backs this memory. May be `None`, in
/// which case the memory is all zeroes. /// which case the memory is all zeroes.
pub(crate) image: Option<Arc<MemoryMemFd>>, pub(crate) image: Option<Arc<MemoryImage>>,
/// The initial heap size. /// The initial heap size.
initial_size: usize, initial_size: usize,
/// The current heap size. All memory above `base + cur_size` /// The current heap size. All memory above `base + cur_size`
@@ -290,19 +290,19 @@ pub struct MemFdSlot {
/// from offset 0 to `initial_size` are accessible R+W and the /// from offset 0 to `initial_size` are accessible R+W and the
/// rest of the slot is inaccessible. /// rest of the slot is inaccessible.
dirty: bool, dirty: bool,
/// Whether this MemFdSlot is responsible for mapping anonymous /// Whether this MemoryImageSlot is responsible for mapping anonymous
/// memory (to hold the reservation while overwriting mappings /// memory (to hold the reservation while overwriting mappings
/// specific to this slot) in place when it is dropped. Default /// specific to this slot) in place when it is dropped. Default
/// on, unless the caller knows what they are doing. /// on, unless the caller knows what they are doing.
clear_on_drop: bool, clear_on_drop: bool,
} }
impl MemFdSlot { impl MemoryImageSlot {
/// Create a new MemFdSlot. Assumes that there is an anonymous /// Create a new MemoryImageSlot. Assumes that there is an anonymous
/// mmap backing in the given range to start. /// mmap backing in the given range to start.
pub(crate) fn create(base_addr: *mut c_void, initial_size: usize, static_size: usize) -> Self { pub(crate) fn create(base_addr: *mut c_void, initial_size: usize, static_size: usize) -> Self {
let base = base_addr as usize; let base = base_addr as usize;
MemFdSlot { MemoryImageSlot {
base, base,
static_size, static_size,
initial_size, initial_size,
@@ -313,7 +313,7 @@ impl MemFdSlot {
} }
} }
/// Inform the MemFdSlot that it should *not* clear the underlying /// Inform the MemoryImageSlot that it should *not* clear the underlying
/// address space when dropped. This should be used only when the /// address space when dropped. This should be used only when the
/// caller will clear or reuse the address space in some other /// caller will clear or reuse the address space in some other
/// way. /// way.
@@ -335,7 +335,7 @@ impl MemFdSlot {
pub(crate) fn instantiate( pub(crate) fn instantiate(
&mut self, &mut self,
initial_size_bytes: usize, initial_size_bytes: usize,
maybe_image: Option<&Arc<MemoryMemFd>>, maybe_image: Option<&Arc<MemoryImage>>,
) -> Result<(), InstantiationError> { ) -> Result<(), InstantiationError> {
assert!(!self.dirty); assert!(!self.dirty);
assert_eq!(self.cur_size, self.initial_size); assert_eq!(self.cur_size, self.initial_size);
@@ -543,17 +543,17 @@ impl MemFdSlot {
} }
} }
impl Drop for MemFdSlot { impl Drop for MemoryImageSlot {
fn drop(&mut self) { fn drop(&mut self) {
// The MemFdSlot may be dropped if there is an error during // The MemoryImageSlot may be dropped if there is an error during
// instantiation: for example, if a memory-growth limiter // instantiation: for example, if a memory-growth limiter
// disallows a guest from having a memory of a certain size, // disallows a guest from having a memory of a certain size,
// after we've already initialized the MemFdSlot. // after we've already initialized the MemoryImageSlot.
// //
// We need to return this region of the large pool mmap to a // We need to return this region of the large pool mmap to a
// safe state (with no module-specific mappings). The // safe state (with no module-specific mappings). The
// MemFdSlot will not be returned to the MemoryPool, so a new // MemoryImageSlot will not be returned to the MemoryPool, so a new
// MemFdSlot will be created and overwrite the mappings anyway // MemoryImageSlot will be created and overwrite the mappings anyway
// on the slot's next use; but for safety and to avoid // on the slot's next use; but for safety and to avoid
// resource leaks it's better not to have stale mappings to a // resource leaks it's better not to have stale mappings to a
// possibly-otherwise-dead module's image. // possibly-otherwise-dead module's image.
@@ -563,17 +563,17 @@ impl Drop for MemFdSlot {
// *can't* simply munmap, because that leaves a hole in the // *can't* simply munmap, because that leaves a hole in the
// middle of the pooling allocator's big memory area that some // middle of the pooling allocator's big memory area that some
// other random mmap may swoop in and take, to be trampled // other random mmap may swoop in and take, to be trampled
// over by the next MemFdSlot later. // over by the next MemoryImageSlot later.
// //
// Since we're in drop(), we can't sanely return an error if // Since we're in drop(), we can't sanely return an error if
// this mmap fails. Let's ignore the failure if so; the next // this mmap fails. Let's ignore the failure if so; the next
// MemFdSlot to be created for this slot will try to overwrite // MemoryImageSlot to be created for this slot will try to overwrite
// the existing stale mappings, and return a failure properly // the existing stale mappings, and return a failure properly
// if we still cannot map new memory. // if we still cannot map new memory.
// //
// The exception to all of this is if the `unmap_on_drop` flag // The exception to all of this is if the `unmap_on_drop` flag
// (which is set by default) is false. If so, the owner of // (which is set by default) is false. If so, the owner of
// this MemFdSlot has indicated that it will clean up in some // this MemoryImageSlot has indicated that it will clean up in some
// other way. // other way.
if self.clear_on_drop { if self.clear_on_drop {
let _ = self.reset_with_anon_memory(); let _ = self.reset_with_anon_memory();
@@ -585,12 +585,12 @@ impl Drop for MemFdSlot {
mod test { mod test {
use std::sync::Arc; use std::sync::Arc;
use super::{create_memfd, MemFdSlot, MemFdSource, MemoryMemFd}; use super::{create_memfd, FdSource, MemoryImage, MemoryImageSlot};
use crate::mmap::Mmap; use crate::mmap::Mmap;
use anyhow::Result; use anyhow::Result;
use std::io::Write; use std::io::Write;
fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryMemFd> { fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryImage> {
// Offset must be page-aligned. // Offset must be page-aligned.
let page_size = region::page::size(); let page_size = region::page::size();
assert_eq!(offset & (page_size - 1), 0); assert_eq!(offset & (page_size - 1), 0);
@@ -601,8 +601,8 @@ mod test {
let image_len = (data.len() + page_size - 1) & !(page_size - 1); let image_len = (data.len() + page_size - 1) & !(page_size - 1);
memfd.as_file().set_len(image_len as u64)?; memfd.as_file().set_len(image_len as u64)?;
Ok(MemoryMemFd { Ok(MemoryImage {
fd: MemFdSource::Memfd(memfd), fd: FdSource::Memfd(memfd),
len: image_len, len: image_len,
fd_offset: 0, fd_offset: 0,
linear_memory_offset: offset, linear_memory_offset: offset,
@@ -613,8 +613,8 @@ mod test {
fn instantiate_no_image() { fn instantiate_no_image() {
// 4 MiB mmap'd area, not accessible // 4 MiB mmap'd area, not accessible
let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap(); let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap();
// Create a MemFdSlot on top of it // Create a MemoryImageSlot on top of it
let mut memfd = MemFdSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20); let mut memfd = MemoryImageSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
memfd.no_clear_on_drop(); memfd.no_clear_on_drop();
assert!(!memfd.is_dirty()); assert!(!memfd.is_dirty());
// instantiate with 64 KiB initial size // instantiate with 64 KiB initial size
@@ -645,8 +645,8 @@ mod test {
fn instantiate_image() { fn instantiate_image() {
// 4 MiB mmap'd area, not accessible // 4 MiB mmap'd area, not accessible
let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap(); let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap();
// Create a MemFdSlot on top of it // Create a MemoryImageSlot on top of it
let mut memfd = MemFdSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20); let mut memfd = MemoryImageSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
memfd.no_clear_on_drop(); memfd.no_clear_on_drop();
// Create an image with some data. // Create an image with some data.
let image = Arc::new(create_memfd_with_data(4096, &[1, 2, 3, 4]).unwrap()); let image = Arc::new(create_memfd_with_data(4096, &[1, 2, 3, 4]).unwrap());

39
crates/runtime/src/memfd_disabled.rs → crates/runtime/src/cow_disabled.rs
View File

@@ -1,5 +1,5 @@
//! Shims for MemFdSlot when the memfd allocator is not //! Shims for MemoryImageSlot when the copy-on-write memory initialization is
//! included. Enables unconditional use of the type and its methods //! not included. Enables unconditional use of the type and its methods
//! throughout higher-level code. //! throughout higher-level code.
use crate::{InstantiationError, MmapVec}; use crate::{InstantiationError, MmapVec};
@@ -7,49 +7,46 @@ use anyhow::Result;
use std::sync::Arc; use std::sync::Arc;
use wasmtime_environ::{DefinedMemoryIndex, Module}; use wasmtime_environ::{DefinedMemoryIndex, Module};
/// A shim for the memfd image container when memfd support is not /// A shim for the memory image container when support is not included.
/// included. pub enum ModuleMemoryImages {}
pub enum ModuleMemFds {}
/// A shim for an individual memory image. /// A shim for an individual memory image.
#[allow(dead_code)] #[allow(dead_code)]
pub enum MemoryMemFd {} pub enum MemoryImage {}
impl ModuleMemFds { impl ModuleMemoryImages {
/// Construct a new set of memfd images. This variant is used /// Construct a new set of memory images. This variant is used
/// when memfd support is not included; it always returns no /// when cow support is not included; it always returns no
/// images. /// images.
pub fn new(_: &Module, _: &[u8], _: Option<&MmapVec>) -> Result<Option<ModuleMemFds>> { pub fn new(_: &Module, _: &[u8], _: Option<&MmapVec>) -> Result<Option<ModuleMemoryImages>> {
Ok(None) Ok(None)
} }
/// Get the memfd image for a particular memory. /// Get the memory image for a particular memory.
pub fn get_memory_image(&self, _: DefinedMemoryIndex) -> Option<&Arc<MemoryMemFd>> { pub fn get_memory_image(&self, _: DefinedMemoryIndex) -> Option<&Arc<MemoryImage>> {
// Should be unreachable because the `Self` type is
// uninhabitable.
match *self {} match *self {}
} }
} }
/// A placeholder for MemFdSlot when we have not included the pooling /// A placeholder for MemoryImageSlot when we have not included the pooling
/// allocator. /// allocator.
/// ///
/// To allow MemFdSlot to be unconditionally passed around in various /// To allow MemoryImageSlot to be unconditionally passed around in various
/// places (e.g. a `Memory`), we define a zero-sized type when memfd is /// places (e.g. a `Memory`), we define a zero-sized type when memory is
/// not included in the build. /// not included in the build.
#[derive(Debug)] #[derive(Debug)]
pub enum MemFdSlot {} pub enum MemoryImageSlot {}
#[allow(dead_code)] #[allow(dead_code)]
impl MemFdSlot { impl MemoryImageSlot {
pub(crate) fn create(_: *mut libc::c_void, _: usize, _: usize) -> Self { pub(crate) fn create(_: *mut libc::c_void, _: usize, _: usize) -> Self {
panic!("create() on invalid MemFdSlot"); panic!("create() on invalid MemoryImageSlot");
} }
pub(crate) fn instantiate( pub(crate) fn instantiate(
&mut self, &mut self,
_: usize, _: usize,
_: Option<&Arc<MemoryMemFd>>, _: Option<&Arc<MemoryImage>>,
) -> Result<Self, InstantiationError> { ) -> Result<Self, InstantiationError> {
match *self {} match *self {}
} }

13
crates/runtime/src/instance/allocator.rs
View File

@@ -365,9 +365,9 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<(), I
}, },
&mut |memory_index, init| { &mut |memory_index, init| {
// If this initializer applies to a defined memory but that memory // If this initializer applies to a defined memory but that memory
// doesn't need initialization, due to something like uffd or memfd // doesn't need initialization, due to something like uffd or
// pre-initializing it via mmap magic, then this initializer can be // copy-on-write pre-initializing it via mmap magic, then this
// skipped entirely. // initializer can be skipped entirely.
if let Some(memory_index) = module.defined_memory_index(memory_index) { if let Some(memory_index) = module.defined_memory_index(memory_index) {
if !instance.memories[memory_index].needs_init() { if !instance.memories[memory_index].needs_init() {
return true; return true;
@@ -480,12 +480,11 @@ impl OnDemandInstanceAllocator {
let mut memories: PrimaryMap<DefinedMemoryIndex, _> = let mut memories: PrimaryMap<DefinedMemoryIndex, _> =
PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); PrimaryMap::with_capacity(module.memory_plans.len() - num_imports);
for (memory_idx, plan) in module.memory_plans.iter().skip(num_imports) { for (memory_idx, plan) in module.memory_plans.iter().skip(num_imports) {
// Create a MemFdSlot if there is an image for this memory.
let defined_memory_idx = module let defined_memory_idx = module
.defined_memory_index(memory_idx) .defined_memory_index(memory_idx)
.expect("Skipped imports, should never be None"); .expect("Skipped imports, should never be None");
let memfd_image = runtime_info let image = runtime_info
.memfd_image(defined_memory_idx) .memory_image(defined_memory_idx)
.map_err(|err| InstantiationError::Resource(err.into()))?; .map_err(|err| InstantiationError::Resource(err.into()))?;
memories.push( memories.push(
@@ -497,7 +496,7 @@ impl OnDemandInstanceAllocator {
.get() .get()
.expect("if module has memory plans, store is not empty") .expect("if module has memory plans, store is not empty")
}, },
memfd_image, image,
) )
.map_err(InstantiationError::Resource)?, .map_err(InstantiationError::Resource)?,
); );

68
crates/runtime/src/instance/allocator/pooling.rs
View File

@@ -12,7 +12,7 @@ use super::{
InstantiationError, InstantiationError,
}; };
use crate::{instance::Instance, Memory, Mmap, Table}; use crate::{instance::Instance, Memory, Mmap, Table};
use crate::{MemFdSlot, ModuleRuntimeInfo, Store}; use crate::{MemoryImageSlot, ModuleRuntimeInfo, Store};
use anyhow::{anyhow, bail, Context, Result}; use anyhow::{anyhow, bail, Context, Result};
use libc::c_void; use libc::c_void;
use std::convert::TryFrom; use std::convert::TryFrom;
@@ -259,7 +259,7 @@ pub enum PoolingAllocationStrategy {
impl Default for PoolingAllocationStrategy { impl Default for PoolingAllocationStrategy {
fn default() -> Self { fn default() -> Self {
if cfg!(memfd) { if cfg!(memory_init_cow) {
Self::ReuseAffinity Self::ReuseAffinity
} else { } else {
Self::NextAvailable Self::NextAvailable
@@ -476,10 +476,12 @@ impl InstancePool {
}; };
if let Some(image) = runtime_info if let Some(image) = runtime_info
.memfd_image(defined_index) .memory_image(defined_index)
.map_err(|err| InstantiationError::Resource(err.into()))? .map_err(|err| InstantiationError::Resource(err.into()))?
{ {
let mut slot = self.memories.take_memfd_slot(instance_index, defined_index); let mut slot = self
.memories
.take_memory_image_slot(instance_index, defined_index);
let initial_size = plan.memory.minimum * WASM_PAGE_SIZE as u64; let initial_size = plan.memory.minimum * WASM_PAGE_SIZE as u64;
// If instantiation fails, we can propagate the error // If instantiation fails, we can propagate the error
@@ -487,7 +489,7 @@ impl InstancePool {
// handler to attempt to map the range with PROT_NONE // handler to attempt to map the range with PROT_NONE
// memory, to reserve the space while releasing any // memory, to reserve the space while releasing any
// stale mappings. The next use of this slot will then // stale mappings. The next use of this slot will then
// create a new MemFdSlot that will try to map over // create a new slot that will try to map over
// this, returning errors as well if the mapping // this, returning errors as well if the mapping
// errors persist. The unmap-on-drop is best effort; // errors persist. The unmap-on-drop is best effort;
// if it fails, then we can still soundly continue // if it fails, then we can still soundly continue
@@ -531,16 +533,16 @@ impl InstancePool {
match memory { match memory {
Memory::Static { Memory::Static {
memfd_slot: Some(mut memfd_slot), memory_image: Some(mut image),
.. ..
} => { } => {
// If there was any error clearing the memfd, just // If there was any error clearing the image, just
// drop it here, and let the drop handler for the // drop it here, and let the drop handler for the
// MemFdSlot unmap in a way that retains the // slot unmap in a way that retains the
// address space reservation. // address space reservation.
if memfd_slot.clear_and_remain_ready().is_ok() { if image.clear_and_remain_ready().is_ok() {
self.memories self.memories
.return_memfd_slot(instance_index, def_mem_idx, memfd_slot); .return_memory_image_slot(instance_index, def_mem_idx, image);
} }
} }
@@ -630,10 +632,10 @@ impl InstancePool {
#[derive(Debug)] #[derive(Debug)]
struct MemoryPool { struct MemoryPool {
mapping: Mmap, mapping: Mmap,
// If using the memfd allocation scheme, the MemFd slots. We // If using a copy-on-write allocation scheme, the slot management. We
// dynamically transfer ownership of a slot to a Memory when in // dynamically transfer ownership of a slot to a Memory when in
// use. // use.
memfd_slots: Vec<Mutex<Option<MemFdSlot>>>, image_slots: Vec<Mutex<Option<MemoryImageSlot>>>,
// The size, in bytes, of each linear memory's reservation plus the guard // The size, in bytes, of each linear memory's reservation plus the guard
// region allocated for it. // region allocated for it.
memory_size: usize, memory_size: usize,
@@ -718,18 +720,18 @@ impl MemoryPool {
let mapping = Mmap::accessible_reserved(0, allocation_size) let mapping = Mmap::accessible_reserved(0, allocation_size)
.context("failed to create memory pool mapping")?; .context("failed to create memory pool mapping")?;
let num_memfd_slots = if cfg!(memfd) { let num_image_slots = if cfg!(memory_init_cow) {
max_instances * max_memories max_instances * max_memories
} else { } else {
0 0
}; };
let memfd_slots: Vec<_> = std::iter::repeat_with(|| Mutex::new(None)) let image_slots: Vec<_> = std::iter::repeat_with(|| Mutex::new(None))
.take(num_memfd_slots) .take(num_image_slots)
.collect(); .collect();
let pool = Self { let pool = Self {
mapping, mapping,
memfd_slots, image_slots,
memory_size, memory_size,
initial_memory_offset, initial_memory_offset,
max_memories, max_memories,
@@ -758,18 +760,18 @@ impl MemoryPool {
.map(move |i| self.get_base(instance_index, DefinedMemoryIndex::from_u32(i as u32))) .map(move |i| self.get_base(instance_index, DefinedMemoryIndex::from_u32(i as u32)))
} }
/// Take ownership of the given memfd slot. Must be returned via /// Take ownership of the given image slot. Must be returned via
/// `return_memfd_slot` when the instance is done using it. /// `return_memory_image_slot` when the instance is done using it.
fn take_memfd_slot( fn take_memory_image_slot(
&self, &self,
instance_index: usize, instance_index: usize,
memory_index: DefinedMemoryIndex, memory_index: DefinedMemoryIndex,
) -> MemFdSlot { ) -> MemoryImageSlot {
let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize); let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize);
let maybe_slot = self.memfd_slots[idx].lock().unwrap().take(); let maybe_slot = self.image_slots[idx].lock().unwrap().take();
maybe_slot.unwrap_or_else(|| { maybe_slot.unwrap_or_else(|| {
MemFdSlot::create( MemoryImageSlot::create(
self.get_base(instance_index, memory_index) as *mut c_void, self.get_base(instance_index, memory_index) as *mut c_void,
0, 0,
self.memory_size, self.memory_size,
@@ -777,28 +779,28 @@ impl MemoryPool {
}) })
} }
/// Return ownership of the given memfd slot. /// Return ownership of the given image slot.
fn return_memfd_slot( fn return_memory_image_slot(
&self, &self,
instance_index: usize, instance_index: usize,
memory_index: DefinedMemoryIndex, memory_index: DefinedMemoryIndex,
slot: MemFdSlot, slot: MemoryImageSlot,
) { ) {
assert!(!slot.is_dirty()); assert!(!slot.is_dirty());
let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize); let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize);
*self.memfd_slots[idx].lock().unwrap() = Some(slot); *self.image_slots[idx].lock().unwrap() = Some(slot);
} }
} }
impl Drop for MemoryPool { impl Drop for MemoryPool {
fn drop(&mut self) { fn drop(&mut self) {
// Clear the `clear_no_drop` flag (i.e., ask to *not* clear on // Clear the `clear_no_drop` flag (i.e., ask to *not* clear on
// drop) for all MemFdSlots, and then drop them here. This is // drop) for all slots, and then drop them here. This is
// valid because the one `Mmap` that covers the whole region // valid because the one `Mmap` that covers the whole region
// can just do its one munmap. // can just do its one munmap.
for mut memfd in std::mem::take(&mut self.memfd_slots) { for mut slot in std::mem::take(&mut self.image_slots) {
if let Some(memfd_slot) = memfd.get_mut().unwrap() { if let Some(slot) = slot.get_mut().unwrap() {
memfd_slot.no_clear_on_drop(); slot.no_clear_on_drop();
} }
} }
} }
@@ -1145,7 +1147,7 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
use crate::{CompiledModuleId, Imports, MemoryMemFd, StorePtr, VMSharedSignatureIndex}; use crate::{CompiledModuleId, Imports, MemoryImage, StorePtr, VMSharedSignatureIndex};
use std::sync::Arc; use std::sync::Arc;
use wasmtime_environ::{ use wasmtime_environ::{
DefinedFuncIndex, DefinedMemoryIndex, EntityRef, FunctionInfo, Global, GlobalInit, Memory, DefinedFuncIndex, DefinedMemoryIndex, EntityRef, FunctionInfo, Global, GlobalInit, Memory,
@@ -1456,10 +1458,10 @@ mod test {
fn signature(&self, _: SignatureIndex) -> VMSharedSignatureIndex { fn signature(&self, _: SignatureIndex) -> VMSharedSignatureIndex {
unimplemented!() unimplemented!()
} }
fn memfd_image( fn memory_image(
&self, &self,
_: DefinedMemoryIndex, _: DefinedMemoryIndex,
) -> anyhow::Result<Option<&Arc<MemoryMemFd>>> { ) -> anyhow::Result<Option<&Arc<MemoryImage>>> {
Ok(None) Ok(None)
} }

26
crates/runtime/src/lib.rs
View File

@@ -19,7 +19,7 @@
clippy::use_self clippy::use_self
) )
)] )]
#![cfg_attr(not(memfd), allow(unused_variables, unreachable_code))] #![cfg_attr(not(memory_init_cow), allow(unused_variables, unreachable_code))]
use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicU64;
use std::sync::Arc; use std::sync::Arc;
@@ -74,15 +74,15 @@ pub use crate::vmcontext::{
mod module_id; mod module_id;
pub use module_id::{CompiledModuleId, CompiledModuleIdAllocator}; pub use module_id::{CompiledModuleId, CompiledModuleIdAllocator};
#[cfg(memfd)] #[cfg(memory_init_cow)]
mod memfd; mod cow;
#[cfg(memfd)] #[cfg(memory_init_cow)]
pub use crate::memfd::{MemFdSlot, MemoryMemFd, ModuleMemFds}; pub use crate::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
#[cfg(not(memfd))] #[cfg(not(memory_init_cow))]
mod memfd_disabled; mod cow_disabled;
#[cfg(not(memfd))] #[cfg(not(memory_init_cow))]
pub use crate::memfd_disabled::{MemFdSlot, MemoryMemFd, ModuleMemFds}; pub use crate::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
/// Version number of this crate. /// Version number of this crate.
pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -158,7 +158,7 @@ pub unsafe trait Store {
/// instance state. /// instance state.
/// ///
/// When an instance is created, it holds an Arc<dyn ModuleRuntimeInfo> /// When an instance is created, it holds an Arc<dyn ModuleRuntimeInfo>
/// so that it can get to signatures, metadata on functions, memfd and /// so that it can get to signatures, metadata on functions, memory and
/// funcref-table images, etc. All of these things are ordinarily known /// funcref-table images, etc. All of these things are ordinarily known
/// by the higher-level layers of Wasmtime. Specifically, the main /// by the higher-level layers of Wasmtime. Specifically, the main
/// implementation of this trait is provided by /// implementation of this trait is provided by
@@ -180,8 +180,10 @@ pub trait ModuleRuntimeInfo: Send + Sync + 'static {
/// `image_base`. /// `image_base`.
fn function_info(&self, func_index: DefinedFuncIndex) -> &FunctionInfo; fn function_info(&self, func_index: DefinedFuncIndex) -> &FunctionInfo;
/// memfd images, if any, for this module. /// Returns the `MemoryImage` structure used for copy-on-write
fn memfd_image(&self, memory: DefinedMemoryIndex) -> anyhow::Result<Option<&Arc<MemoryMemFd>>>; /// initialization of the memory, if it's applicable.
fn memory_image(&self, memory: DefinedMemoryIndex)
-> anyhow::Result<Option<&Arc<MemoryImage>>>;
/// A unique ID for this particular module. This can be used to /// A unique ID for this particular module. This can be used to
/// allow for fastpaths to optimize a "re-instantiate the same /// allow for fastpaths to optimize a "re-instantiate the same

89
crates/runtime/src/memory.rs
View File

@@ -4,8 +4,8 @@
use crate::mmap::Mmap; use crate::mmap::Mmap;
use crate::vmcontext::VMMemoryDefinition; use crate::vmcontext::VMMemoryDefinition;
use crate::MemFdSlot; use crate::MemoryImage;
use crate::MemoryMemFd; use crate::MemoryImageSlot;
use crate::Store; use crate::Store;
use anyhow::Error; use anyhow::Error;
use anyhow::{bail, format_err, Result}; use anyhow::{bail, format_err, Result};
@@ -25,8 +25,8 @@ pub trait RuntimeMemoryCreator: Send + Sync {
plan: &MemoryPlan, plan: &MemoryPlan,
minimum: usize, minimum: usize,
maximum: Option<usize>, maximum: Option<usize>,
// Optionally, a memfd image for CoW backing. // Optionally, a memory image for CoW backing.
memfd_image: Option<&Arc<MemoryMemFd>>, memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Box<dyn RuntimeLinearMemory>>; ) -> Result<Box<dyn RuntimeLinearMemory>>;
} }
@@ -40,13 +40,13 @@ impl RuntimeMemoryCreator for DefaultMemoryCreator {
plan: &MemoryPlan, plan: &MemoryPlan,
minimum: usize, minimum: usize,
maximum: Option<usize>, maximum: Option<usize>,
memfd_image: Option<&Arc<MemoryMemFd>>, memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Box<dyn RuntimeLinearMemory>> { ) -> Result<Box<dyn RuntimeLinearMemory>> {
Ok(Box::new(MmapMemory::new( Ok(Box::new(MmapMemory::new(
plan, plan,
minimum, minimum,
maximum, maximum,
memfd_image, memory_image,
)?)) )?))
} }
} }
@@ -71,7 +71,7 @@ pub trait RuntimeLinearMemory: Send + Sync {
fn vmmemory(&self) -> VMMemoryDefinition; fn vmmemory(&self) -> VMMemoryDefinition;
/// Does this memory need initialization? It may not if it already /// Does this memory need initialization? It may not if it already
/// has initial contents courtesy of the `MemoryMemFd` passed to /// has initial contents courtesy of the `MemoryImage` passed to
/// `RuntimeMemoryCreator::new_memory()`. /// `RuntimeMemoryCreator::new_memory()`.
fn needs_init(&self) -> bool; fn needs_init(&self) -> bool;
} }
@@ -103,13 +103,9 @@ pub struct MmapMemory {
pre_guard_size: usize, pre_guard_size: usize,
offset_guard_size: usize, offset_guard_size: usize,
// A MemFd CoW mapping that provides the initial content of this // An optional CoW mapping that provides the initial content of this
// MmapMemory, if mapped. // MmapMemory, if mapped.
// memory_image: Option<MemoryImageSlot>,
// N.B.: this comes after the `mmap` field above because it must
// be destructed first. It puts a placeholder mapping in place on
// drop, then the `mmap` above completely unmaps the region.
memfd: Option<MemFdSlot>,
} }
impl MmapMemory { impl MmapMemory {
@@ -118,7 +114,7 @@ impl MmapMemory {
plan: &MemoryPlan, plan: &MemoryPlan,
minimum: usize, minimum: usize,
mut maximum: Option<usize>, mut maximum: Option<usize>,
memfd_image: Option<&Arc<MemoryMemFd>>, memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Self> { ) -> Result<Self> {
// It's a programmer error for these two configuration values to exceed // It's a programmer error for these two configuration values to exceed
// the host available address space, so panic if such a configuration is // the host available address space, so panic if such a configuration is
@@ -155,22 +151,22 @@ impl MmapMemory {
mmap.make_accessible(pre_guard_bytes, minimum)?; mmap.make_accessible(pre_guard_bytes, minimum)?;
} }
// If a memfd image was specified, try to create the MemFdSlot on top of our mmap. // If a memory image was specified, try to create the MemoryImageSlot on
let memfd = match memfd_image { // top of our mmap.
let memory_image = match memory_image {
Some(image) => { Some(image) => {
let base = unsafe { mmap.as_mut_ptr().add(pre_guard_bytes) }; let base = unsafe { mmap.as_mut_ptr().add(pre_guard_bytes) };
let mut memfd_slot = MemFdSlot::create( let mut slot = MemoryImageSlot::create(
base.cast(), base.cast(),
minimum, minimum,
alloc_bytes + extra_to_reserve_on_growth, alloc_bytes + extra_to_reserve_on_growth,
); );
memfd_slot.instantiate(minimum, Some(image))?; slot.instantiate(minimum, Some(image))?;
// On drop, we will unmap our mmap'd range that this // On drop, we will unmap our mmap'd range that this slot was
// memfd_slot was mapped on top of, so there is no // mapped on top of, so there is no need for the slot to wipe
// need for the memfd_slot to wipe it with an // it with an anonymous mapping first.
// anonymous mapping first. slot.no_clear_on_drop();
memfd_slot.no_clear_on_drop(); Some(slot)
Some(memfd_slot)
} }
None => None, None => None,
}; };
@@ -182,7 +178,7 @@ impl MmapMemory {
pre_guard_size: pre_guard_bytes, pre_guard_size: pre_guard_bytes,
offset_guard_size: offset_guard_bytes, offset_guard_size: offset_guard_bytes,
extra_to_reserve_on_growth, extra_to_reserve_on_growth,
memfd, memory_image,
}) })
} }
} }
@@ -215,19 +211,18 @@ impl RuntimeLinearMemory for MmapMemory {
new_mmap.as_mut_slice()[self.pre_guard_size..][..self.accessible] new_mmap.as_mut_slice()[self.pre_guard_size..][..self.accessible]
.copy_from_slice(&self.mmap.as_slice()[self.pre_guard_size..][..self.accessible]); .copy_from_slice(&self.mmap.as_slice()[self.pre_guard_size..][..self.accessible]);
// Now drop the MemFdSlot, if any. We've lost the CoW // Now drop the MemoryImageSlot, if any. We've lost the CoW
// advantages by explicitly copying all data, but we have // advantages by explicitly copying all data, but we have
// preserved all of its content; so we no longer need the // preserved all of its content; so we no longer need the
// memfd mapping. We need to do this before we // mapping. We need to do this before we (implicitly) drop the
// (implicitly) drop the `mmap` field by overwriting it // `mmap` field by overwriting it below.
// below. drop(self.memory_image.take());
let _ = self.memfd.take();
self.mmap = new_mmap; self.mmap = new_mmap;
} else if let Some(memfd) = self.memfd.as_mut() { } else if let Some(image) = self.memory_image.as_mut() {
// MemFdSlot has its own growth mechanisms; defer to its // MemoryImageSlot has its own growth mechanisms; defer to its
// implementation. // implementation.
memfd.set_heap_limit(new_size)?; image.set_heap_limit(new_size)?;
} else { } else {
// If the new size of this heap fits within the existing allocation // If the new size of this heap fits within the existing allocation
// then all we need to do is to make the new pages accessible. This // then all we need to do is to make the new pages accessible. This
@@ -255,9 +250,9 @@ impl RuntimeLinearMemory for MmapMemory {
} }
fn needs_init(&self) -> bool { fn needs_init(&self) -> bool {
// If we're using a memfd CoW mapping, then no initialization // If we're using a CoW mapping, then no initialization
// is needed. // is needed.
self.memfd.is_none() self.memory_image.is_none()
} }
} }
@@ -278,9 +273,9 @@ pub enum Memory {
/// fault. /// fault.
make_accessible: Option<fn(*mut u8, usize) -> Result<()>>, make_accessible: Option<fn(*mut u8, usize) -> Result<()>>,
/// The MemFdSlot, if any, for this memory. Owned here and /// The image management, if any, for this memory. Owned here and
/// returned to the pooling allocator when termination occurs. /// returned to the pooling allocator when termination occurs.
memfd_slot: Option<MemFdSlot>, memory_image: Option<MemoryImageSlot>,
/// Stores the pages in the linear memory that have faulted as guard pages when using the `uffd` feature. /// Stores the pages in the linear memory that have faulted as guard pages when using the `uffd` feature.
/// These pages need their protection level reset before the memory can grow. /// These pages need their protection level reset before the memory can grow.
@@ -299,14 +294,14 @@ impl Memory {
plan: &MemoryPlan, plan: &MemoryPlan,
creator: &dyn RuntimeMemoryCreator, creator: &dyn RuntimeMemoryCreator,
store: &mut dyn Store, store: &mut dyn Store,
memfd_image: Option<&Arc<MemoryMemFd>>, memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Self> { ) -> Result<Self> {
let (minimum, maximum) = Self::limit_new(plan, store)?; let (minimum, maximum) = Self::limit_new(plan, store)?;
Ok(Memory::Dynamic(creator.new_memory( Ok(Memory::Dynamic(creator.new_memory(
plan, plan,
minimum, minimum,
maximum, maximum,
memfd_image, memory_image,
)?)) )?))
} }
@@ -315,7 +310,7 @@ impl Memory {
plan: &MemoryPlan, plan: &MemoryPlan,
base: &'static mut [u8], base: &'static mut [u8],
make_accessible: Option<fn(*mut u8, usize) -> Result<()>>, make_accessible: Option<fn(*mut u8, usize) -> Result<()>>,
memfd_slot: Option<MemFdSlot>, memory_image: Option<MemoryImageSlot>,
store: &mut dyn Store, store: &mut dyn Store,
) -> Result<Self> { ) -> Result<Self> {
let (minimum, maximum) = Self::limit_new(plan, store)?; let (minimum, maximum) = Self::limit_new(plan, store)?;
@@ -335,7 +330,7 @@ impl Memory {
base, base,
size: minimum, size: minimum,
make_accessible, make_accessible,
memfd_slot, memory_image,
#[cfg(all(feature = "uffd", target_os = "linux"))] #[cfg(all(feature = "uffd", target_os = "linux"))]
guard_page_faults: Vec::new(), guard_page_faults: Vec::new(),
}) })
@@ -457,11 +452,11 @@ impl Memory {
/// Returns whether or not this memory needs initialization. It /// Returns whether or not this memory needs initialization. It
/// may not if it already has initial content thanks to a CoW /// may not if it already has initial content thanks to a CoW
/// mechanism like memfd. /// mechanism.
pub(crate) fn needs_init(&self) -> bool { pub(crate) fn needs_init(&self) -> bool {
match self { match self {
Memory::Static { Memory::Static {
memfd_slot: Some(ref slot), memory_image: Some(slot),
.. ..
} => !slot.has_image(), } => !slot.has_image(),
Memory::Dynamic(mem) => mem.needs_init(), Memory::Dynamic(mem) => mem.needs_init(),
@@ -542,7 +537,7 @@ impl Memory {
Memory::Static { Memory::Static {
base, base,
size, size,
memfd_slot: Some(ref mut memfd_slot), memory_image: Some(image),
.. ..
} => { } => {
// Never exceed static memory size // Never exceed static memory size
@@ -551,7 +546,7 @@ impl Memory {
return Ok(None); return Ok(None);
} }
if let Err(e) = memfd_slot.set_heap_limit(new_byte_size) { if let Err(e) = image.set_heap_limit(new_byte_size) {
store.memory_grow_failed(&e); store.memory_grow_failed(&e);
return Ok(None); return Ok(None);
} }
@@ -564,7 +559,7 @@ impl Memory {
.. ..
} => { } => {
let make_accessible = make_accessible let make_accessible = make_accessible
.expect("make_accessible must be Some if this is not a MemFD memory"); .expect("make_accessible must be Some if this is not a CoW memory");
// Never exceed static memory size // Never exceed static memory size
if new_byte_size > base.len() { if new_byte_size > base.len() {
@@ -658,7 +653,7 @@ impl Default for Memory {
base: &mut [], base: &mut [],
size: 0, size: 0,
make_accessible: Some(|_, _| unreachable!()), make_accessible: Some(|_, _| unreachable!()),
memfd_slot: None, memory_image: None,
#[cfg(all(feature = "uffd", target_os = "linux"))] #[cfg(all(feature = "uffd", target_os = "linux"))]
guard_page_faults: Vec::new(), guard_page_faults: Vec::new(),
} }

14
crates/wasmtime/Cargo.toml
View File

@@ -59,7 +59,7 @@ default = [
'parallel-compilation', 'parallel-compilation',
'cranelift', 'cranelift',
'pooling-allocator', 'pooling-allocator',
'memfd', 'memory-init-cow',
'vtune', 'vtune',
] ]
@@ -101,10 +101,10 @@ all-arch = ["wasmtime-cranelift/all-arch"]
# need portable signal handling. # need portable signal handling.
posix-signals-on-macos = ["wasmtime-runtime/posix-signals-on-macos"] posix-signals-on-macos = ["wasmtime-runtime/posix-signals-on-macos"]
# Enables, on Linux, the usage of memfd mappings to enable instantiation to use # Enables, on supported platforms, the usage of copy-on-write initialization of
# copy-on-write to initialize linear memory for wasm modules which have # compatible linear memories. For more information see the documentation of
# compatible linear memories. # `Config::memory_init_cow`.
# #
# Enabling this feature has no effect on non-Linux platforms or when the `uffd` # Enabling this feature has no effect on unsupported platforms or when the
# feature is enabled. # `uffd` feature is enabled.
memfd = ["wasmtime-runtime/memfd"] memory-init-cow = ["wasmtime-runtime/memory-init-cow"]

110
crates/wasmtime/src/config.rs
View File

@@ -104,8 +104,8 @@ pub struct Config {
pub(crate) module_version: ModuleVersionStrategy, pub(crate) module_version: ModuleVersionStrategy,
pub(crate) parallel_compilation: bool, pub(crate) parallel_compilation: bool,
pub(crate) paged_memory_initialization: bool, pub(crate) paged_memory_initialization: bool,
pub(crate) memfd: bool, pub(crate) memory_init_cow: bool,
pub(crate) memfd_guaranteed_dense_image_size: u64, pub(crate) memory_guaranteed_dense_image_size: u64,
} }
impl Config { impl Config {
@@ -131,8 +131,8 @@ impl Config {
parallel_compilation: true, parallel_compilation: true,
// Default to paged memory initialization when using uffd on linux // Default to paged memory initialization when using uffd on linux
paged_memory_initialization: cfg!(all(target_os = "linux", feature = "uffd")), paged_memory_initialization: cfg!(all(target_os = "linux", feature = "uffd")),
memfd: false, memory_init_cow: true,
memfd_guaranteed_dense_image_size: 16 << 20, memory_guaranteed_dense_image_size: 16 << 20,
}; };
#[cfg(compiler)] #[cfg(compiler)]
{ {
@@ -1178,71 +1178,91 @@ impl Config {
self self
} }
/// Configures whether `memfd`, if supported, will be used to initialize /// Configures whether copy-on-write memory-mapped data is used to
/// applicable module memories. /// initialize a linear memory.
/// ///
/// This is a Linux-specific feature since `memfd` is only supported on /// Initializing linear memory via a copy-on-write mapping can drastically
/// Linux. Support for this is also enabled by default at compile time but /// improve instantiation costs of a WebAssembly module because copying
/// is otherwise disabled at runtime by default. This feature needs to be /// memory is deferred. Additionally if a page of memory is only ever read
/// enabled to `true` for support to be used. /// from WebAssembly and never written too then the same underlying page of
/// data will be reused between all instantiations of a module meaning that
/// if a module is instantiated many times this can lower the overall memory
/// required needed to run that module.
/// ///
/// Also note that even if this feature is enabled it may not be applicable /// This feature is only applicable when a WebAssembly module meets specific
/// to all memories in all wasm modules. At this time memories must meet /// criteria to be initialized in this fashion, such as:
/// specific criteria to be memfd-initialized:
/// ///
/// * Only memories defined in the module can be initialized this way. /// * Only memories defined in the module can be initialized this way.
/// * Data segments for memory must use statically known offsets. /// * Data segments for memory must use statically known offsets.
/// * Data segments for memory must all be in-bounds. /// * Data segments for memory must all be in-bounds.
/// ///
/// If all of the above applies, this setting is enabled, and the current /// Modules which do not meet these criteria will fall back to
/// platform is Linux the `memfd` will be used to efficiently initialize /// initialization of linear memory based on copying memory.
/// linear memories with `mmap` to avoid copying data from initializers into ///
/// linear memory. /// This feature of Wasmtime is also platform-specific:
#[cfg(feature = "memfd")] ///
#[cfg_attr(nightlydoc, doc(cfg(feature = "memfd")))] /// * Linux - this feature is supported for all instances of [`Module`].
pub fn memfd(&mut self, memfd: bool) -> &mut Self { /// Modules backed by an existing mmap (such as those created by
self.memfd = memfd; /// [`Module::deserialize_file`]) will reuse that mmap to cow-initialize
/// memory. Other instance of [`Module`] may use the `memfd_create`
/// syscall to create an initialization image to `mmap`.
/// * Unix (not Linux) - this feature is only supported when loading modules
/// from a precompiled file via [`Module::deserialize_file`] where there
/// is a file descriptor to use to map data into the process. Note that
/// the module must have been compiled with this setting enabled as well.
/// * Windows - there is no support for this feature at this time. Memory
/// initialization will always copy bytes.
///
/// By default this option is enabled.
///
/// [`Module::deserialize_file`]: crate::Module::deserialize_file
/// [`Module`]: crate::Module
#[cfg(feature = "memory-init-cow")]
#[cfg_attr(nightlydoc, doc(cfg(feature = "memory-init-cow")))]
pub fn memory_init_cow(&mut self, enable: bool) -> &mut Self {
self.memory_init_cow = enable;
self self
} }
/// Configures the "guaranteed dense image size" for memfd. /// Configures the "guaranteed dense image size" for copy-on-write
/// initialized memories.
/// ///
/// When using the memfd feature to initialize memory efficiently, /// When using the [`Config::memory_init_cow`] feature to initialize memory
/// compiled modules contain an image of the module's initial /// efficiently (which is enabled by default), compiled modules contain an
/// heap. If the module has a fairly sparse initial heap, with /// image of the module's initial heap. If the module has a fairly sparse
/// just a few data segments at very different offsets, this could /// initial heap, with just a few data segments at very different offsets,
/// result in a large region of zero bytes in the image. In other /// this could result in a large region of zero bytes in the image. In
/// words, it's not very memory-efficient. /// other words, it's not very memory-efficient.
/// ///
/// We normally use a heuristic to avoid this: if less than half /// We normally use a heuristic to avoid this: if less than half
/// of the initialized range (first non-zero to last non-zero /// of the initialized range (first non-zero to last non-zero
/// byte) of any memory in the module has pages with nonzero /// byte) of any memory in the module has pages with nonzero
/// bytes, then we avoid memfd for the entire module. /// bytes, then we avoid creating a memory image for the entire module.
/// ///
/// However, if the embedder always needs the instantiation-time /// However, if the embedder always needs the instantiation-time efficiency
/// efficiency of memfd, and is otherwise carefully controlling /// of copy-on-write initialization, and is otherwise carefully controlling
/// parameters of the modules (for example, by limiting the /// parameters of the modules (for example, by limiting the maximum heap
/// maximum heap size of the modules), then it may be desirable to /// size of the modules), then it may be desirable to ensure a memory image
/// ensure memfd is used even if this could go against the /// is created even if this could go against the heuristic above. Thus, we
/// heuristic above. Thus, we add another condition: there is a /// add another condition: there is a size of initialized data region up to
/// size of initialized data region up to which we *always* allow /// which we *always* allow a memory image. The embedder can set this to a
/// memfd. The embedder can set this to a known maximum heap size /// known maximum heap size if they desire to always get the benefits of
/// if they desire to always get the benefits of memfd. /// copy-on-write images.
/// ///
/// In the future we may implement a "best of both worlds" /// In the future we may implement a "best of both worlds"
/// solution where we have a dense image up to some limit, and /// solution where we have a dense image up to some limit, and
/// then support a sparse list of initializers beyond that; this /// then support a sparse list of initializers beyond that; this
/// would get most of the benefit of memfd and pay the incremental /// would get most of the benefit of copy-on-write and pay the incremental
/// cost of eager initialization only for those bits of memory /// cost of eager initialization only for those bits of memory
/// that are out-of-bounds. However, for now, an embedder desiring /// that are out-of-bounds. However, for now, an embedder desiring
/// fast instantiation should ensure that this setting is as large /// fast instantiation should ensure that this setting is as large
/// as the maximum module initial memory content size. /// as the maximum module initial memory content size.
/// ///
/// By default this value is 16 MiB. /// By default this value is 16 MiB.
#[cfg(feature = "memfd")] #[cfg(feature = "memory-init-cow")]
#[cfg_attr(nightlydoc, doc(cfg(feature = "memfd")))] #[cfg_attr(nightlydoc, doc(cfg(feature = "memory-init-cow")))]
pub fn memfd_guaranteed_dense_image_size(&mut self, size_in_bytes: u64) -> &mut Self { pub fn memory_guaranteed_dense_image_size(&mut self, size_in_bytes: u64) -> &mut Self {
self.memfd_guaranteed_dense_image_size = size_in_bytes; self.memory_guaranteed_dense_image_size = size_in_bytes;
self self
} }
@@ -1315,8 +1335,8 @@ impl Clone for Config {
module_version: self.module_version.clone(), module_version: self.module_version.clone(),
parallel_compilation: self.parallel_compilation, parallel_compilation: self.parallel_compilation,
paged_memory_initialization: self.paged_memory_initialization, paged_memory_initialization: self.paged_memory_initialization,
memfd: self.memfd, memory_init_cow: self.memory_init_cow,
memfd_guaranteed_dense_image_size: self.memfd_guaranteed_dense_image_size, memory_guaranteed_dense_image_size: self.memory_guaranteed_dense_image_size,
} }
} }
} }

12
crates/wasmtime/src/lib.rs
View File

@@ -281,14 +281,14 @@
//! efficient reuse of resources for high-concurrency and //! efficient reuse of resources for high-concurrency and
//! high-instantiation-count scenarios. //! high-instantiation-count scenarios.
//! //!
//! * `memfd` - Enabled by default, this feature builds in support for a //! * `memory-init-cow` - Enabled by default, this feature builds in support
//! Linux-specific feature of creating a `memfd` where applicable for a //! for, on supported platforms, initializing wasm linear memories with
//! [`Module`]'s initial memory. This makes instantiation much faster by //! copy-on-write heap mappings. This makes instantiation much faster by
//! `mmap`-ing the initial memory image into place instead of copying memory //! `mmap`-ing the initial memory image into place instead of copying memory
//! into place, allowing sharing pages that end up only getting read and //! into place, allowing sharing pages that end up only getting read. Note
//! otherwise using copy-on-write for efficient initialization of memory. Note
//! that this is simply compile-time support and this must also be enabled at //! that this is simply compile-time support and this must also be enabled at
//! run-time via [`Config::memfd`]. //! run-time via [`Config::memory_init_cow`] (which is also enabled by
//! default).
//! //!
//! ## Examples //! ## Examples
//! //!

35
crates/wasmtime/src/module.rs
View File

@@ -16,7 +16,7 @@ use wasmtime_environ::{
}; };
use wasmtime_jit::{CompiledModule, CompiledModuleInfo, TypeTables}; use wasmtime_jit::{CompiledModule, CompiledModuleInfo, TypeTables};
use wasmtime_runtime::{ use wasmtime_runtime::{
CompiledModuleId, MemoryMemFd, MmapVec, ModuleMemFds, VMSharedSignatureIndex, CompiledModuleId, MemoryImage, MmapVec, ModuleMemoryImages, VMSharedSignatureIndex,
}; };
mod registry; mod registry;
@@ -114,11 +114,10 @@ struct ModuleInner {
types: Arc<TypeTables>, types: Arc<TypeTables>,
/// Registered shared signature for the module. /// Registered shared signature for the module.
signatures: Arc<SignatureCollection>, signatures: Arc<SignatureCollection>,
/// A set of memfd images for memories, if any. Note that module /// A set of initialization images for memories, if any. Note that module
/// instantiation (hence the need for lazy init) may happen for /// instantiation (hence the need for lazy init) may happen for the same
/// the same module concurrently in multiple Stores, so we use a /// module concurrently in multiple Stores, so we use a OnceCell.
/// OnceCell. memory_images: OnceCell<Option<ModuleMemoryImages>>,
memfds: OnceCell<Option<ModuleMemFds>>,
} }
impl Module { impl Module {
@@ -430,9 +429,9 @@ impl Module {
// can either at runtime be implemented as a single memcpy to // can either at runtime be implemented as a single memcpy to
// initialize memory or otherwise enabling virtual-memory-tricks // initialize memory or otherwise enabling virtual-memory-tricks
// such as mmap'ing from a file to get copy-on-write. // such as mmap'ing from a file to get copy-on-write.
if engine.config().memfd { if engine.config().memory_init_cow {
let align = engine.compiler().page_size_align(); let align = engine.compiler().page_size_align();
let max_always_allowed = engine.config().memfd_guaranteed_dense_image_size; let max_always_allowed = engine.config().memory_guaranteed_dense_image_size;
translation.try_static_init(align, max_always_allowed); translation.try_static_init(align, max_always_allowed);
} }
@@ -575,7 +574,7 @@ impl Module {
artifact_upvars: modules, artifact_upvars: modules,
module_upvars, module_upvars,
signatures, signatures,
memfds: OnceCell::new(), memory_images: OnceCell::new(),
}), }),
}); });
@@ -594,7 +593,7 @@ impl Module {
engine: engine.clone(), engine: engine.clone(),
types: types.clone(), types: types.clone(),
module, module,
memfds: OnceCell::new(), memory_images: OnceCell::new(),
artifact_upvars: artifact_upvars artifact_upvars: artifact_upvars
.iter() .iter()
.map(|i| artifacts[*i].clone()) .map(|i| artifacts[*i].clone())
@@ -720,7 +719,7 @@ impl Module {
types: self.inner.types.clone(), types: self.inner.types.clone(),
engine: self.inner.engine.clone(), engine: self.inner.engine.clone(),
module, module,
memfds: OnceCell::new(), memory_images: OnceCell::new(),
artifact_upvars: artifact_upvars artifact_upvars: artifact_upvars
.iter() .iter()
.map(|i| self.inner.artifact_upvars[*i].clone()) .map(|i| self.inner.artifact_upvars[*i].clone())
@@ -1032,21 +1031,21 @@ impl wasmtime_runtime::ModuleRuntimeInfo for ModuleInner {
self.module.func_info(index) self.module.func_info(index)
} }
fn memfd_image(&self, memory: DefinedMemoryIndex) -> Result<Option<&Arc<MemoryMemFd>>> { fn memory_image(&self, memory: DefinedMemoryIndex) -> Result<Option<&Arc<MemoryImage>>> {
if !self.engine.config().memfd { if !self.engine.config().memory_init_cow {
return Ok(None); return Ok(None);
} }
let memfds = self.memfds.get_or_try_init(|| { let images = self.memory_images.get_or_try_init(|| {
ModuleMemFds::new( ModuleMemoryImages::new(
self.module.module(), self.module.module(),
self.module.wasm_data(), self.module.wasm_data(),
Some(self.module.mmap()), Some(self.module.mmap()),
) )
})?; })?;
Ok(memfds Ok(images
.as_ref() .as_ref()
.and_then(|memfds| memfds.get_memory_image(memory))) .and_then(|images| images.get_memory_image(memory)))
} }
fn unique_id(&self) -> Option<CompiledModuleId> { fn unique_id(&self) -> Option<CompiledModuleId> {
@@ -1138,7 +1137,7 @@ impl wasmtime_runtime::ModuleRuntimeInfo for BareModuleInfo {
&self.function_info[index] &self.function_info[index]
} }
fn memfd_image(&self, _memory: DefinedMemoryIndex) -> Result<Option<&Arc<MemoryMemFd>>> { fn memory_image(&self, _memory: DefinedMemoryIndex) -> Result<Option<&Arc<MemoryImage>>> {
Ok(None) Ok(None)
} }

4
crates/wasmtime/src/trampoline/memory.rs
View File

@@ -7,7 +7,7 @@ use std::convert::TryFrom;
use std::sync::Arc; use std::sync::Arc;
use wasmtime_environ::{EntityIndex, MemoryPlan, MemoryStyle, Module, WASM_PAGE_SIZE}; use wasmtime_environ::{EntityIndex, MemoryPlan, MemoryStyle, Module, WASM_PAGE_SIZE};
use wasmtime_runtime::{ use wasmtime_runtime::{
MemoryMemFd, RuntimeLinearMemory, RuntimeMemoryCreator, VMMemoryDefinition, MemoryImage, RuntimeLinearMemory, RuntimeMemoryCreator, VMMemoryDefinition,
}; };
pub fn create_memory(store: &mut StoreOpaque, memory: &MemoryType) -> Result<InstanceId> { pub fn create_memory(store: &mut StoreOpaque, memory: &MemoryType) -> Result<InstanceId> {
@@ -63,7 +63,7 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy {
plan: &MemoryPlan, plan: &MemoryPlan,
minimum: usize, minimum: usize,
maximum: Option<usize>, maximum: Option<usize>,
_: Option<&Arc<MemoryMemFd>>, _: Option<&Arc<MemoryImage>>,
) -> Result<Box<dyn RuntimeLinearMemory>> { ) -> Result<Box<dyn RuntimeLinearMemory>> {
let ty = MemoryType::from_wasmtime_memory(&plan.memory); let ty = MemoryType::from_wasmtime_memory(&plan.memory);
let reserved_size_in_bytes = match plan.style { let reserved_size_in_bytes = match plan.style {

8
src/lib.rs
View File

@@ -252,6 +252,12 @@ struct CommonOptions {
#[structopt(long)] #[structopt(long)]
paged_memory_initialization: bool, paged_memory_initialization: bool,
/// Disables the default of attempting to initialize linear memory via a
/// copy-on-write mapping.
#[cfg(feature = "memory-init-cow")]
#[structopt(long)]
disable_memory_init_cow: bool,
/// Enables the pooling allocator, in place of the on-demand /// Enables the pooling allocator, in place of the on-demand
/// allocator. /// allocator.
#[cfg(feature = "pooling-allocator")] #[cfg(feature = "pooling-allocator")]
@@ -335,6 +341,8 @@ impl CommonOptions {
config.epoch_interruption(self.epoch_interruption); config.epoch_interruption(self.epoch_interruption);
config.generate_address_map(!self.disable_address_map); config.generate_address_map(!self.disable_address_map);
config.paged_memory_initialization(self.paged_memory_initialization); config.paged_memory_initialization(self.paged_memory_initialization);
#[cfg(feature = "memory-init-cow")]
config.memory_init_cow(!self.disable_memory_init_cow);
#[cfg(feature = "pooling-allocator")] #[cfg(feature = "pooling-allocator")]
{ {