T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enable copy-on-write heap initialization by default (#3825)

Browse Source 
* Enable copy-on-write heap initialization by default

This commit enables the `Config::memfd` feature by default now that it's
been fuzzed for a few weeks on oss-fuzz, and will continue to be fuzzed
leading up to the next release of Wasmtime in early March. The
documentation of the `Config` option has been updated as well as adding
a CLI flag to disable the feature.

* Remove ubiquitous "memfd" terminology

Switch instead to forms of "memory image" or "cow" or some combination
thereof.

* Update new option names
This commit is contained in:
Alex Crichton
2022-02-22 17:12:18 -06:00
committed by GitHub
parent 593f8d96aa
commit bbd4a4a500
16 changed files with 294 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
crates/runtime/src/memfd.rs → crates/runtime/src/cow.rs
View File

@@ -1,5 +1,5 @@
//! memfd support: creation of backing images for modules, and logic
//! to support mapping these backing images into memory.
//! Copy-on-write initialization support: creation of backing images for
//! modules, and logic to support mapping these backing images into memory.
use crate::InstantiationError;
use crate::MmapVec;
@@ -11,30 +11,30 @@ use std::sync::Arc;
use std::{convert::TryFrom, ops::Range};
use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, Module, PrimaryMap};
/// MemFDs containing backing images for certain memories in a module.
/// Backing images for memories in a module.
///
/// This is meant to be built once, when a module is first
/// loaded/constructed, and then used many times for instantiation.
pub struct ModuleMemFds {
memories: PrimaryMap<DefinedMemoryIndex, Option<Arc<MemoryMemFd>>>,
/// This is meant to be built once, when a module is first loaded/constructed,
/// and then used many times for instantiation.
pub struct ModuleMemoryImages {
memories: PrimaryMap<DefinedMemoryIndex, Option<Arc<MemoryImage>>>,
}
impl ModuleMemFds {
/// Get the MemoryMemFd for a given memory.
pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc<MemoryMemFd>> {
impl ModuleMemoryImages {
/// Get the MemoryImage for a given memory.
pub fn get_memory_image(&self, defined_index: DefinedMemoryIndex) -> Option<&Arc<MemoryImage>> {
self.memories[defined_index].as_ref()
}
}
/// One backing image for one memory.
#[derive(Debug)]
pub struct MemoryMemFd {
pub struct MemoryImage {
/// The file descriptor source of this image.
///
/// This might be an mmaped `*.cwasm` file or on Linux it could also be a
/// `Memfd` as an anonymous file in memory. In either case this is used as
/// the backing-source for the CoW image.
fd: MemFdSource,
fd: FdSource,
/// Length of image, in bytes.
///
@@ -59,29 +59,29 @@ pub struct MemoryMemFd {
}
#[derive(Debug)]
enum MemFdSource {
enum FdSource {
Mmap(Arc<File>),
#[cfg(target_os = "linux")]
Memfd(memfd::Memfd),
}
impl MemFdSource {
impl FdSource {
fn as_file(&self) -> &File {
match self {
MemFdSource::Mmap(file) => file,
FdSource::Mmap(file) => file,
#[cfg(target_os = "linux")]
MemFdSource::Memfd(memfd) => memfd.as_file(),
FdSource::Memfd(memfd) => memfd.as_file(),
}
}
}
impl MemoryMemFd {
impl MemoryImage {
fn new(
page_size: u32,
offset: u64,
data: &[u8],
mmap: Option<&MmapVec>,
) -> Result<Option<MemoryMemFd>> {
) -> Result<Option<MemoryImage>> {
// Sanity-check that various parameters are page-aligned.
let len = data.len();
let offset = u32::try_from(offset).unwrap();
@@ -115,8 +115,8 @@ impl MemoryMemFd {
assert_eq!((mmap.original_offset() as u32) % page_size, 0);
if let Some(file) = mmap.original_file() {
return Ok(Some(MemoryMemFd {
fd: MemFdSource::Mmap(file.clone()),
return Ok(Some(MemoryImage {
fd: FdSource::Mmap(file.clone()),
fd_offset: u64::try_from(mmap.original_offset() + (data_start - start))
.unwrap(),
linear_memory_offset,
@@ -159,8 +159,8 @@ impl MemoryMemFd {
memfd.add_seal(memfd::FileSeal::SealWrite)?;
memfd.add_seal(memfd::FileSeal::SealSeal)?;
Ok(Some(MemoryMemFd {
fd: MemFdSource::Memfd(memfd),
Ok(Some(MemoryImage {
fd: FdSource::Memfd(memfd),
fd_offset: 0,
linear_memory_offset,
len,
@@ -188,15 +188,15 @@ fn create_memfd() -> Result<memfd::Memfd> {
.map_err(|e| e.into())
}
impl ModuleMemFds {
/// Create a new `ModuleMemFds` for the given module. This can be
impl ModuleMemoryImages {
/// Create a new `ModuleMemoryImages` for the given module. This can be
/// passed in as part of a `InstanceAllocationRequest` to speed up
/// instantiation and execution by using memfd-backed memories.
/// instantiation and execution by using copy-on-write-backed memories.
pub fn new(
module: &Module,
wasm_data: &[u8],
mmap: Option<&MmapVec>,
) -> Result<Option<ModuleMemFds>> {
) -> Result<Option<ModuleMemoryImages>> {
let map = match &module.memory_initialization {
MemoryInitialization::Static { map } => map,
_ => return Ok(None),
@@ -223,30 +223,30 @@ impl ModuleMemFds {
};
// Get the image for this wasm module as a subslice of `wasm_data`,
// and then use that to try to create the `MemoryMemFd`. If this
// creation files then we fail creating `ModuleMemFds` since this
// and then use that to try to create the `MemoryImage`. If this
// creation files then we fail creating `ModuleMemoryImages` since this
// memory couldn't be represented.
let data = &wasm_data[init.data.start as usize..init.data.end as usize];
let memfd = match MemoryMemFd::new(page_size, init.offset, data, mmap)? {
Some(memfd) => memfd,
let image = match MemoryImage::new(page_size, init.offset, data, mmap)? {
Some(image) => image,
None => return Ok(None),
};
let idx = memories.push(Some(Arc::new(memfd)));
let idx = memories.push(Some(Arc::new(image)));
assert_eq!(idx, defined_memory);
}
Ok(Some(ModuleMemFds { memories }))
Ok(Some(ModuleMemoryImages { memories }))
}
}
/// A single slot handled by the memfd instance-heap mechanism.
/// A single slot handled by the copy-on-write memory initialization mechanism.
///
/// The mmap scheme is:
///
/// base ==> (points here)
/// - (image.offset bytes) anonymous zero memory, pre-image
/// - (image.len bytes) CoW mapping of memfd heap image
/// - (image.len bytes) CoW mapping of memory image
/// - (up to static_size) anonymous zero memory, post-image
///
/// The ordering of mmaps to set this up is:
@@ -257,7 +257,7 @@ impl ModuleMemFds {
/// - per instantiation of new image in a slot:
/// - mmap of anonymous zero memory, from 0 to max heap size
/// (static_size)
/// - mmap of CoW'd memfd image, from `image.offset` to
/// - mmap of CoW'd image, from `image.offset` to
/// `image.offset + image.len`. This overwrites part of the
/// anonymous zero memory, potentially splitting it into a pre-
/// and post-region.
@@ -265,15 +265,15 @@ impl ModuleMemFds {
/// heap size; we re-mprotect it with R+W bits when the heap is
/// grown.
#[derive(Debug)]
pub struct MemFdSlot {
pub struct MemoryImageSlot {
/// The base of the actual heap memory. Bytes at this address are
/// what is seen by the Wasm guest code.
base: usize,
/// The maximum static memory size, plus post-guard.
static_size: usize,
/// The memfd image that backs this memory. May be `None`, in
/// The image that backs this memory. May be `None`, in
/// which case the memory is all zeroes.
pub(crate) image: Option<Arc<MemoryMemFd>>,
pub(crate) image: Option<Arc<MemoryImage>>,
/// The initial heap size.
initial_size: usize,
/// The current heap size. All memory above `base + cur_size`
@@ -290,19 +290,19 @@ pub struct MemFdSlot {
/// from offset 0 to `initial_size` are accessible R+W and the
/// rest of the slot is inaccessible.
dirty: bool,
/// Whether this MemFdSlot is responsible for mapping anonymous
/// Whether this MemoryImageSlot is responsible for mapping anonymous
/// memory (to hold the reservation while overwriting mappings
/// specific to this slot) in place when it is dropped. Default
/// on, unless the caller knows what they are doing.
clear_on_drop: bool,
}
impl MemFdSlot {
/// Create a new MemFdSlot. Assumes that there is an anonymous
impl MemoryImageSlot {
/// Create a new MemoryImageSlot. Assumes that there is an anonymous
/// mmap backing in the given range to start.
pub(crate) fn create(base_addr: *mut c_void, initial_size: usize, static_size: usize) -> Self {
let base = base_addr as usize;
MemFdSlot {
MemoryImageSlot {
base,
static_size,
initial_size,
@@ -313,7 +313,7 @@ impl MemFdSlot {
}
}
/// Inform the MemFdSlot that it should *not* clear the underlying
/// Inform the MemoryImageSlot that it should *not* clear the underlying
/// address space when dropped. This should be used only when the
/// caller will clear or reuse the address space in some other
/// way.
@@ -335,7 +335,7 @@ impl MemFdSlot {
pub(crate) fn instantiate(
&mut self,
initial_size_bytes: usize,
maybe_image: Option<&Arc<MemoryMemFd>>,
maybe_image: Option<&Arc<MemoryImage>>,
) -> Result<(), InstantiationError> {
assert!(!self.dirty);
assert_eq!(self.cur_size, self.initial_size);
@@ -543,17 +543,17 @@ impl MemFdSlot {
}
}
impl Drop for MemFdSlot {
impl Drop for MemoryImageSlot {
fn drop(&mut self) {
// The MemFdSlot may be dropped if there is an error during
// The MemoryImageSlot may be dropped if there is an error during
// instantiation: for example, if a memory-growth limiter
// disallows a guest from having a memory of a certain size,
// after we've already initialized the MemFdSlot.
// after we've already initialized the MemoryImageSlot.
//
// We need to return this region of the large pool mmap to a
// safe state (with no module-specific mappings). The
// MemFdSlot will not be returned to the MemoryPool, so a new
// MemFdSlot will be created and overwrite the mappings anyway
// MemoryImageSlot will not be returned to the MemoryPool, so a new
// MemoryImageSlot will be created and overwrite the mappings anyway
// on the slot's next use; but for safety and to avoid
// resource leaks it's better not to have stale mappings to a
// possibly-otherwise-dead module's image.
@@ -563,17 +563,17 @@ impl Drop for MemFdSlot {
// *can't* simply munmap, because that leaves a hole in the
// middle of the pooling allocator's big memory area that some
// other random mmap may swoop in and take, to be trampled
// over by the next MemFdSlot later.
// over by the next MemoryImageSlot later.
//
// Since we're in drop(), we can't sanely return an error if
// this mmap fails. Let's ignore the failure if so; the next
// MemFdSlot to be created for this slot will try to overwrite
// MemoryImageSlot to be created for this slot will try to overwrite
// the existing stale mappings, and return a failure properly
// if we still cannot map new memory.
//
// The exception to all of this is if the `unmap_on_drop` flag
// (which is set by default) is false. If so, the owner of
// this MemFdSlot has indicated that it will clean up in some
// this MemoryImageSlot has indicated that it will clean up in some
// other way.
if self.clear_on_drop {
let _ = self.reset_with_anon_memory();
@@ -585,12 +585,12 @@ impl Drop for MemFdSlot {
mod test {
use std::sync::Arc;
use super::{create_memfd, MemFdSlot, MemFdSource, MemoryMemFd};
use super::{create_memfd, FdSource, MemoryImage, MemoryImageSlot};
use crate::mmap::Mmap;
use anyhow::Result;
use std::io::Write;
fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryMemFd> {
fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryImage> {
// Offset must be page-aligned.
let page_size = region::page::size();
assert_eq!(offset & (page_size - 1), 0);
@@ -601,8 +601,8 @@ mod test {
let image_len = (data.len() + page_size - 1) & !(page_size - 1);
memfd.as_file().set_len(image_len as u64)?;
Ok(MemoryMemFd {
fd: MemFdSource::Memfd(memfd),
Ok(MemoryImage {
fd: FdSource::Memfd(memfd),
len: image_len,
fd_offset: 0,
linear_memory_offset: offset,
@@ -613,8 +613,8 @@ mod test {
fn instantiate_no_image() {
// 4 MiB mmap'd area, not accessible
let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap();
// Create a MemFdSlot on top of it
let mut memfd = MemFdSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
// Create a MemoryImageSlot on top of it
let mut memfd = MemoryImageSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
memfd.no_clear_on_drop();
assert!(!memfd.is_dirty());
// instantiate with 64 KiB initial size
@@ -645,8 +645,8 @@ mod test {
fn instantiate_image() {
// 4 MiB mmap'd area, not accessible
let mut mmap = Mmap::accessible_reserved(0, 4 << 20).unwrap();
// Create a MemFdSlot on top of it
let mut memfd = MemFdSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
// Create a MemoryImageSlot on top of it
let mut memfd = MemoryImageSlot::create(mmap.as_mut_ptr() as *mut _, 0, 4 << 20);
memfd.no_clear_on_drop();
// Create an image with some data.
let image = Arc::new(create_memfd_with_data(4096, &[1, 2, 3, 4]).unwrap());

39
crates/runtime/src/memfd_disabled.rs → crates/runtime/src/cow_disabled.rs
View File

@@ -1,5 +1,5 @@
//! Shims for MemFdSlot when the memfd allocator is not
//! included. Enables unconditional use of the type and its methods
//! Shims for MemoryImageSlot when the copy-on-write memory initialization is
//! not included. Enables unconditional use of the type and its methods
//! throughout higher-level code.
use crate::{InstantiationError, MmapVec};
@@ -7,49 +7,46 @@ use anyhow::Result;
use std::sync::Arc;
use wasmtime_environ::{DefinedMemoryIndex, Module};
/// A shim for the memfd image container when memfd support is not
/// included.
pub enum ModuleMemFds {}
/// A shim for the memory image container when support is not included.
pub enum ModuleMemoryImages {}
/// A shim for an individual memory image.
#[allow(dead_code)]
pub enum MemoryMemFd {}
pub enum MemoryImage {}
impl ModuleMemFds {
/// Construct a new set of memfd images. This variant is used
/// when memfd support is not included; it always returns no
impl ModuleMemoryImages {
/// Construct a new set of memory images. This variant is used
/// when cow support is not included; it always returns no
/// images.
pub fn new(_: &Module, _: &[u8], _: Option<&MmapVec>) -> Result<Option<ModuleMemFds>> {
pub fn new(_: &Module, _: &[u8], _: Option<&MmapVec>) -> Result<Option<ModuleMemoryImages>> {
Ok(None)
}
/// Get the memfd image for a particular memory.
pub fn get_memory_image(&self, _: DefinedMemoryIndex) -> Option<&Arc<MemoryMemFd>> {
// Should be unreachable because the `Self` type is
// uninhabitable.
/// Get the memory image for a particular memory.
pub fn get_memory_image(&self, _: DefinedMemoryIndex) -> Option<&Arc<MemoryImage>> {
match *self {}
}
}
/// A placeholder for MemFdSlot when we have not included the pooling
/// A placeholder for MemoryImageSlot when we have not included the pooling
/// allocator.
///
/// To allow MemFdSlot to be unconditionally passed around in various
/// places (e.g. a `Memory`), we define a zero-sized type when memfd is
/// To allow MemoryImageSlot to be unconditionally passed around in various
/// places (e.g. a `Memory`), we define a zero-sized type when memory is
/// not included in the build.
#[derive(Debug)]
pub enum MemFdSlot {}
pub enum MemoryImageSlot {}
#[allow(dead_code)]
impl MemFdSlot {
impl MemoryImageSlot {
pub(crate) fn create(_: *mut libc::c_void, _: usize, _: usize) -> Self {
panic!("create() on invalid MemFdSlot");
panic!("create() on invalid MemoryImageSlot");
}
pub(crate) fn instantiate(
&mut self,
_: usize,
_: Option<&Arc<MemoryMemFd>>,
_: Option<&Arc<MemoryImage>>,
) -> Result<Self, InstantiationError> {
match *self {}
}

13
crates/runtime/src/instance/allocator.rs
View File

@@ -365,9 +365,9 @@ fn initialize_memories(instance: &mut Instance, module: &Module) -> Result<(), I
},
&mut |memory_index, init| {
// If this initializer applies to a defined memory but that memory
// doesn't need initialization, due to something like uffd or memfd
// pre-initializing it via mmap magic, then this initializer can be
// skipped entirely.
// doesn't need initialization, due to something like uffd or
// copy-on-write pre-initializing it via mmap magic, then this
// initializer can be skipped entirely.
if let Some(memory_index) = module.defined_memory_index(memory_index) {
if !instance.memories[memory_index].needs_init() {
return true;
@@ -480,12 +480,11 @@ impl OnDemandInstanceAllocator {
let mut memories: PrimaryMap<DefinedMemoryIndex, _> =
PrimaryMap::with_capacity(module.memory_plans.len() - num_imports);
for (memory_idx, plan) in module.memory_plans.iter().skip(num_imports) {
// Create a MemFdSlot if there is an image for this memory.
let defined_memory_idx = module
.defined_memory_index(memory_idx)
.expect("Skipped imports, should never be None");
let memfd_image = runtime_info
.memfd_image(defined_memory_idx)
let image = runtime_info
.memory_image(defined_memory_idx)
.map_err(|err| InstantiationError::Resource(err.into()))?;
memories.push(
@@ -497,7 +496,7 @@ impl OnDemandInstanceAllocator {
.get()
.expect("if module has memory plans, store is not empty")
},
memfd_image,
image,
)
.map_err(InstantiationError::Resource)?,
);

68
crates/runtime/src/instance/allocator/pooling.rs
View File

@@ -12,7 +12,7 @@ use super::{
InstantiationError,
};
use crate::{instance::Instance, Memory, Mmap, Table};
use crate::{MemFdSlot, ModuleRuntimeInfo, Store};
use crate::{MemoryImageSlot, ModuleRuntimeInfo, Store};
use anyhow::{anyhow, bail, Context, Result};
use libc::c_void;
use std::convert::TryFrom;
@@ -259,7 +259,7 @@ pub enum PoolingAllocationStrategy {
impl Default for PoolingAllocationStrategy {
fn default() -> Self {
if cfg!(memfd) {
if cfg!(memory_init_cow) {
Self::ReuseAffinity
} else {
Self::NextAvailable
@@ -476,10 +476,12 @@ impl InstancePool {
};
if let Some(image) = runtime_info
.memfd_image(defined_index)
.memory_image(defined_index)
.map_err(|err| InstantiationError::Resource(err.into()))?
{
let mut slot = self.memories.take_memfd_slot(instance_index, defined_index);
let mut slot = self
.memories
.take_memory_image_slot(instance_index, defined_index);
let initial_size = plan.memory.minimum * WASM_PAGE_SIZE as u64;
// If instantiation fails, we can propagate the error
@@ -487,7 +489,7 @@ impl InstancePool {
// handler to attempt to map the range with PROT_NONE
// memory, to reserve the space while releasing any
// stale mappings. The next use of this slot will then
// create a new MemFdSlot that will try to map over
// create a new slot that will try to map over
// this, returning errors as well if the mapping
// errors persist. The unmap-on-drop is best effort;
// if it fails, then we can still soundly continue
@@ -531,16 +533,16 @@ impl InstancePool {
match memory {
Memory::Static {
memfd_slot: Some(mut memfd_slot),
memory_image: Some(mut image),
..
} => {
// If there was any error clearing the memfd, just
// If there was any error clearing the image, just
// drop it here, and let the drop handler for the
// MemFdSlot unmap in a way that retains the
// slot unmap in a way that retains the
// address space reservation.
if memfd_slot.clear_and_remain_ready().is_ok() {
if image.clear_and_remain_ready().is_ok() {
self.memories
.return_memfd_slot(instance_index, def_mem_idx, memfd_slot);
.return_memory_image_slot(instance_index, def_mem_idx, image);
}
}
@@ -630,10 +632,10 @@ impl InstancePool {
#[derive(Debug)]
struct MemoryPool {
mapping: Mmap,
// If using the memfd allocation scheme, the MemFd slots. We
// If using a copy-on-write allocation scheme, the slot management. We
// dynamically transfer ownership of a slot to a Memory when in
// use.
memfd_slots: Vec<Mutex<Option<MemFdSlot>>>,
image_slots: Vec<Mutex<Option<MemoryImageSlot>>>,
// The size, in bytes, of each linear memory's reservation plus the guard
// region allocated for it.
memory_size: usize,
@@ -718,18 +720,18 @@ impl MemoryPool {
let mapping = Mmap::accessible_reserved(0, allocation_size)
.context("failed to create memory pool mapping")?;
let num_memfd_slots = if cfg!(memfd) {
let num_image_slots = if cfg!(memory_init_cow) {
max_instances * max_memories
} else {
0
};
let memfd_slots: Vec<_> = std::iter::repeat_with(|| Mutex::new(None))
.take(num_memfd_slots)
let image_slots: Vec<_> = std::iter::repeat_with(|| Mutex::new(None))
.take(num_image_slots)
.collect();
let pool = Self {
mapping,
memfd_slots,
image_slots,
memory_size,
initial_memory_offset,
max_memories,
@@ -758,18 +760,18 @@ impl MemoryPool {
.map(move |i| self.get_base(instance_index, DefinedMemoryIndex::from_u32(i as u32)))
}
/// Take ownership of the given memfd slot. Must be returned via
/// `return_memfd_slot` when the instance is done using it.
fn take_memfd_slot(
/// Take ownership of the given image slot. Must be returned via
/// `return_memory_image_slot` when the instance is done using it.
fn take_memory_image_slot(
&self,
instance_index: usize,
memory_index: DefinedMemoryIndex,
) -> MemFdSlot {
) -> MemoryImageSlot {
let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize);
let maybe_slot = self.memfd_slots[idx].lock().unwrap().take();
let maybe_slot = self.image_slots[idx].lock().unwrap().take();
maybe_slot.unwrap_or_else(|| {
MemFdSlot::create(
MemoryImageSlot::create(
self.get_base(instance_index, memory_index) as *mut c_void,
0,
self.memory_size,
@@ -777,28 +779,28 @@ impl MemoryPool {
})
}
/// Return ownership of the given memfd slot.
fn return_memfd_slot(
/// Return ownership of the given image slot.
fn return_memory_image_slot(
&self,
instance_index: usize,
memory_index: DefinedMemoryIndex,
slot: MemFdSlot,
slot: MemoryImageSlot,
) {
assert!(!slot.is_dirty());
let idx = instance_index * self.max_memories + (memory_index.as_u32() as usize);
*self.memfd_slots[idx].lock().unwrap() = Some(slot);
*self.image_slots[idx].lock().unwrap() = Some(slot);
}
}
impl Drop for MemoryPool {
fn drop(&mut self) {
// Clear the `clear_no_drop` flag (i.e., ask to *not* clear on
// drop) for all MemFdSlots, and then drop them here. This is
// drop) for all slots, and then drop them here. This is
// valid because the one `Mmap` that covers the whole region
// can just do its one munmap.
for mut memfd in std::mem::take(&mut self.memfd_slots) {
if let Some(memfd_slot) = memfd.get_mut().unwrap() {
memfd_slot.no_clear_on_drop();
for mut slot in std::mem::take(&mut self.image_slots) {
if let Some(slot) = slot.get_mut().unwrap() {
slot.no_clear_on_drop();
}
}
}
@@ -1145,7 +1147,7 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator {
#[cfg(test)]
mod test {
use super::*;
use crate::{CompiledModuleId, Imports, MemoryMemFd, StorePtr, VMSharedSignatureIndex};
use crate::{CompiledModuleId, Imports, MemoryImage, StorePtr, VMSharedSignatureIndex};
use std::sync::Arc;
use wasmtime_environ::{
DefinedFuncIndex, DefinedMemoryIndex, EntityRef, FunctionInfo, Global, GlobalInit, Memory,
@@ -1456,10 +1458,10 @@ mod test {
fn signature(&self, _: SignatureIndex) -> VMSharedSignatureIndex {
unimplemented!()
}
fn memfd_image(
fn memory_image(
&self,
_: DefinedMemoryIndex,
) -> anyhow::Result<Option<&Arc<MemoryMemFd>>> {
) -> anyhow::Result<Option<&Arc<MemoryImage>>> {
Ok(None)
}

26
crates/runtime/src/lib.rs
View File

@@ -19,7 +19,7 @@
clippy::use_self
)
)]
#![cfg_attr(not(memfd), allow(unused_variables, unreachable_code))]
#![cfg_attr(not(memory_init_cow), allow(unused_variables, unreachable_code))]
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
@@ -74,15 +74,15 @@ pub use crate::vmcontext::{
mod module_id;
pub use module_id::{CompiledModuleId, CompiledModuleIdAllocator};
#[cfg(memfd)]
mod memfd;
#[cfg(memfd)]
pub use crate::memfd::{MemFdSlot, MemoryMemFd, ModuleMemFds};
#[cfg(memory_init_cow)]
mod cow;
#[cfg(memory_init_cow)]
pub use crate::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
#[cfg(not(memfd))]
mod memfd_disabled;
#[cfg(not(memfd))]
pub use crate::memfd_disabled::{MemFdSlot, MemoryMemFd, ModuleMemFds};
#[cfg(not(memory_init_cow))]
mod cow_disabled;
#[cfg(not(memory_init_cow))]
pub use crate::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
/// Version number of this crate.
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -158,7 +158,7 @@ pub unsafe trait Store {
/// instance state.
///
/// When an instance is created, it holds an Arc<dyn ModuleRuntimeInfo>
/// so that it can get to signatures, metadata on functions, memfd and
/// so that it can get to signatures, metadata on functions, memory and
/// funcref-table images, etc. All of these things are ordinarily known
/// by the higher-level layers of Wasmtime. Specifically, the main
/// implementation of this trait is provided by
@@ -180,8 +180,10 @@ pub trait ModuleRuntimeInfo: Send + Sync + 'static {
/// `image_base`.
fn function_info(&self, func_index: DefinedFuncIndex) -> &FunctionInfo;
/// memfd images, if any, for this module.
fn memfd_image(&self, memory: DefinedMemoryIndex) -> anyhow::Result<Option<&Arc<MemoryMemFd>>>;
/// Returns the `MemoryImage` structure used for copy-on-write
/// initialization of the memory, if it's applicable.
fn memory_image(&self, memory: DefinedMemoryIndex)
-> anyhow::Result<Option<&Arc<MemoryImage>>>;
/// A unique ID for this particular module. This can be used to
/// allow for fastpaths to optimize a "re-instantiate the same

89
crates/runtime/src/memory.rs
View File

@@ -4,8 +4,8 @@
use crate::mmap::Mmap;
use crate::vmcontext::VMMemoryDefinition;
use crate::MemFdSlot;
use crate::MemoryMemFd;
use crate::MemoryImage;
use crate::MemoryImageSlot;
use crate::Store;
use anyhow::Error;
use anyhow::{bail, format_err, Result};
@@ -25,8 +25,8 @@ pub trait RuntimeMemoryCreator: Send + Sync {
plan: &MemoryPlan,
minimum: usize,
maximum: Option<usize>,
// Optionally, a memfd image for CoW backing.
memfd_image: Option<&Arc<MemoryMemFd>>,
// Optionally, a memory image for CoW backing.
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Box<dyn RuntimeLinearMemory>>;
}
@@ -40,13 +40,13 @@ impl RuntimeMemoryCreator for DefaultMemoryCreator {
plan: &MemoryPlan,
minimum: usize,
maximum: Option<usize>,
memfd_image: Option<&Arc<MemoryMemFd>>,
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Box<dyn RuntimeLinearMemory>> {
Ok(Box::new(MmapMemory::new(
plan,
minimum,
maximum,
memfd_image,
memory_image,
)?))
}
}
@@ -71,7 +71,7 @@ pub trait RuntimeLinearMemory: Send + Sync {
fn vmmemory(&self) -> VMMemoryDefinition;
/// Does this memory need initialization? It may not if it already
/// has initial contents courtesy of the `MemoryMemFd` passed to
/// has initial contents courtesy of the `MemoryImage` passed to
/// `RuntimeMemoryCreator::new_memory()`.
fn needs_init(&self) -> bool;
}
@@ -103,13 +103,9 @@ pub struct MmapMemory {
pre_guard_size: usize,
offset_guard_size: usize,
// A MemFd CoW mapping that provides the initial content of this
// An optional CoW mapping that provides the initial content of this
// MmapMemory, if mapped.
//
// N.B.: this comes after the `mmap` field above because it must
// be destructed first. It puts a placeholder mapping in place on
// drop, then the `mmap` above completely unmaps the region.
memfd: Option<MemFdSlot>,
memory_image: Option<MemoryImageSlot>,
}
impl MmapMemory {
@@ -118,7 +114,7 @@ impl MmapMemory {
plan: &MemoryPlan,
minimum: usize,
mut maximum: Option<usize>,
memfd_image: Option<&Arc<MemoryMemFd>>,
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Self> {
// It's a programmer error for these two configuration values to exceed
// the host available address space, so panic if such a configuration is
@@ -155,22 +151,22 @@ impl MmapMemory {
mmap.make_accessible(pre_guard_bytes, minimum)?;
}
// If a memfd image was specified, try to create the MemFdSlot on top of our mmap.
let memfd = match memfd_image {
// If a memory image was specified, try to create the MemoryImageSlot on
// top of our mmap.
let memory_image = match memory_image {
Some(image) => {
let base = unsafe { mmap.as_mut_ptr().add(pre_guard_bytes) };
let mut memfd_slot = MemFdSlot::create(
let mut slot = MemoryImageSlot::create(
base.cast(),
minimum,
alloc_bytes + extra_to_reserve_on_growth,
);
memfd_slot.instantiate(minimum, Some(image))?;
// On drop, we will unmap our mmap'd range that this
// memfd_slot was mapped on top of, so there is no
// need for the memfd_slot to wipe it with an
// anonymous mapping first.
memfd_slot.no_clear_on_drop();
Some(memfd_slot)
slot.instantiate(minimum, Some(image))?;
// On drop, we will unmap our mmap'd range that this slot was
// mapped on top of, so there is no need for the slot to wipe
// it with an anonymous mapping first.
slot.no_clear_on_drop();
Some(slot)
}
None => None,
};
@@ -182,7 +178,7 @@ impl MmapMemory {
pre_guard_size: pre_guard_bytes,
offset_guard_size: offset_guard_bytes,
extra_to_reserve_on_growth,
memfd,
memory_image,
})
}
}
@@ -215,19 +211,18 @@ impl RuntimeLinearMemory for MmapMemory {
new_mmap.as_mut_slice()[self.pre_guard_size..][..self.accessible]
.copy_from_slice(&self.mmap.as_slice()[self.pre_guard_size..][..self.accessible]);
// Now drop the MemFdSlot, if any. We've lost the CoW
// Now drop the MemoryImageSlot, if any. We've lost the CoW
// advantages by explicitly copying all data, but we have
// preserved all of its content; so we no longer need the
// memfd mapping. We need to do this before we
// (implicitly) drop the `mmap` field by overwriting it
// below.
let _ = self.memfd.take();
// mapping. We need to do this before we (implicitly) drop the
// `mmap` field by overwriting it below.
drop(self.memory_image.take());
self.mmap = new_mmap;
} else if let Some(memfd) = self.memfd.as_mut() {
// MemFdSlot has its own growth mechanisms; defer to its
} else if let Some(image) = self.memory_image.as_mut() {
// MemoryImageSlot has its own growth mechanisms; defer to its
// implementation.
memfd.set_heap_limit(new_size)?;
image.set_heap_limit(new_size)?;
} else {
// If the new size of this heap fits within the existing allocation
// then all we need to do is to make the new pages accessible. This
@@ -255,9 +250,9 @@ impl RuntimeLinearMemory for MmapMemory {
}
fn needs_init(&self) -> bool {
// If we're using a memfd CoW mapping, then no initialization
// If we're using a CoW mapping, then no initialization
// is needed.
self.memfd.is_none()
self.memory_image.is_none()
}
}
@@ -278,9 +273,9 @@ pub enum Memory {
/// fault.
make_accessible: Option<fn(*mut u8, usize) -> Result<()>>,
/// The MemFdSlot, if any, for this memory. Owned here and
/// The image management, if any, for this memory. Owned here and
/// returned to the pooling allocator when termination occurs.
memfd_slot: Option<MemFdSlot>,
memory_image: Option<MemoryImageSlot>,
/// Stores the pages in the linear memory that have faulted as guard pages when using the `uffd` feature.
/// These pages need their protection level reset before the memory can grow.
@@ -299,14 +294,14 @@ impl Memory {
plan: &MemoryPlan,
creator: &dyn RuntimeMemoryCreator,
store: &mut dyn Store,
memfd_image: Option<&Arc<MemoryMemFd>>,
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<Self> {
let (minimum, maximum) = Self::limit_new(plan, store)?;
Ok(Memory::Dynamic(creator.new_memory(
plan,
minimum,
maximum,
memfd_image,
memory_image,
)?))
}
@@ -315,7 +310,7 @@ impl Memory {
plan: &MemoryPlan,
base: &'static mut [u8],
make_accessible: Option<fn(*mut u8, usize) -> Result<()>>,
memfd_slot: Option<MemFdSlot>,
memory_image: Option<MemoryImageSlot>,
store: &mut dyn Store,
) -> Result<Self> {
let (minimum, maximum) = Self::limit_new(plan, store)?;
@@ -335,7 +330,7 @@ impl Memory {
base,
size: minimum,
make_accessible,
memfd_slot,
memory_image,
#[cfg(all(feature = "uffd", target_os = "linux"))]
guard_page_faults: Vec::new(),
})
@@ -457,11 +452,11 @@ impl Memory {
/// Returns whether or not this memory needs initialization. It
/// may not if it already has initial content thanks to a CoW
/// mechanism like memfd.
/// mechanism.
pub(crate) fn needs_init(&self) -> bool {
match self {
Memory::Static {
memfd_slot: Some(ref slot),
memory_image: Some(slot),
..
} => !slot.has_image(),
Memory::Dynamic(mem) => mem.needs_init(),
@@ -542,7 +537,7 @@ impl Memory {
Memory::Static {
base,
size,
memfd_slot: Some(ref mut memfd_slot),
memory_image: Some(image),
..
} => {
// Never exceed static memory size
@@ -551,7 +546,7 @@ impl Memory {
return Ok(None);
}
if let Err(e) = memfd_slot.set_heap_limit(new_byte_size) {
if let Err(e) = image.set_heap_limit(new_byte_size) {
store.memory_grow_failed(&e);
return Ok(None);
}
@@ -564,7 +559,7 @@ impl Memory {
..
} => {
let make_accessible = make_accessible
.expect("make_accessible must be Some if this is not a MemFD memory");
.expect("make_accessible must be Some if this is not a CoW memory");
// Never exceed static memory size
if new_byte_size > base.len() {
@@ -658,7 +653,7 @@ impl Default for Memory {
base: &mut [],
size: 0,
make_accessible: Some(|_, _| unreachable!()),
memfd_slot: None,
memory_image: None,
#[cfg(all(feature = "uffd", target_os = "linux"))]
guard_page_faults: Vec::new(),
}