diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02185746bc..38cb9584f0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -122,6 +122,7 @@ jobs: - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features jitdump - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features cache - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features async + - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features uffd # Check some feature combinations of the `wasmtime-c-api` crate - run: cargo check --manifest-path crates/c-api/Cargo.toml --no-default-features diff --git a/Cargo.toml b/Cargo.toml index a0c5c3f6e2..1262115d19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,7 @@ jitdump = ["wasmtime/jitdump"] vtune = ["wasmtime/vtune"] wasi-crypto = ["wasmtime-wasi-crypto"] wasi-nn = ["wasmtime-wasi-nn"] +uffd = ["wasmtime/uffd"] # Try the experimental, work-in-progress new x86_64 backend. This is not stable # as of June 2020. diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index bdcb99755a..af73a7d102 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -37,3 +37,9 @@ cc = "1.0" [badges] maintenance = { status = "actively-developed" } + +[features] +default = [] + +# Enables support for userfaultfd in the pooling allocator when building on Linux +uffd = ["userfaultfd"] diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 91a0dd5c46..c90c267f6b 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -67,6 +67,11 @@ pub(crate) struct Instance { /// Hosts can store arbitrary per-instance information here. host_state: Box, + /// Stores guard page faults in memory relating to the instance. + /// This is used for the pooling allocator with uffd enabled on Linux. + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell bool)>>, + /// Additional context used by compiled wasm code. This field is last, and /// represents a dynamically-sized array that extends beyond the nominal /// end of the struct (similar to a flexible array member). @@ -376,6 +381,10 @@ impl Instance { /// Returns `None` if memory can't be grown by the specified amount /// of pages. pub(crate) fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option { + // Reset all guard pages before growing any memory + #[cfg(all(feature = "uffd", target_os = "linux"))] + self.reset_guard_pages().ok()?; + let result = self .memories .get(memory_index) @@ -803,6 +812,40 @@ impl Instance { (foreign_table_index, foreign_instance) } } + + /// Records a faulted guard page. + /// + /// This is used to track faulted guard pages that need to be reset. + #[cfg(all(feature = "uffd", target_os = "linux"))] + pub(crate) fn record_guard_page_fault( + &self, + page_addr: *mut u8, + size: usize, + reset: unsafe fn(*mut u8, usize) -> bool, + ) { + self.guard_page_faults + .borrow_mut() + .push((page_addr, size, reset)); + } + + /// Resets previously faulted guard pages. + /// + /// This is used to reset the protection of any guard pages that were previously faulted. + /// + /// Resetting the guard pages is required before growing memory. + #[cfg(all(feature = "uffd", target_os = "linux"))] + pub(crate) fn reset_guard_pages(&self) -> Result<(), String> { + let mut faults = self.guard_page_faults.borrow_mut(); + for (addr, len, reset) in faults.drain(..) { + unsafe { + if !reset(addr, len) { + return Err("failed to reset previously faulted memory guard page".into()); + } + } + } + + Ok(()) + } } /// A handle holding an `Instance` of a WebAssembly module. diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index a1d806e4a1..927afd9e41 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -525,6 +525,8 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { )), dropped_data: RefCell::new(EntitySet::with_capacity(req.module.passive_data.len())), host_state: req.host_state, + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), vmctx: VMContext {}, }; let layout = instance.alloc_layout(); diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 5bf139555a..e401988312 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -31,6 +31,11 @@ cfg_if::cfg_if! { if #[cfg(windows)] { mod windows; use windows as imp; + } else if #[cfg(all(feature = "uffd", target_os = "linux"))] { + mod uffd; + use uffd as imp; + use imp::{PageFaultHandler, reset_guard_page}; + use std::sync::atomic::{AtomicBool, Ordering}; } else if #[cfg(target_os = "linux")] { mod linux; use linux as imp; @@ -335,6 +340,9 @@ impl Iterator for BasePointerIterator { /// structure depending on the limits used to create the pool. /// /// The pool maintains a free list for fast instance allocation. +/// +/// The userfault handler relies on how instances are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct InstancePool { mapping: Mmap, @@ -413,6 +421,8 @@ impl InstancePool { dropped_elements: RefCell::new(EntitySet::new()), dropped_data: RefCell::new(EntitySet::new()), host_state: Box::new(()), + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), vmctx: VMContext {}, }, ); @@ -523,6 +533,12 @@ impl InstancePool { ) -> Result<(), InstantiationError> { let module = instance.module.as_ref(); + // Reset all guard pages before reusing the instance + #[cfg(all(feature = "uffd", target_os = "linux"))] + instance + .reset_guard_pages() + .map_err(InstantiationError::Resource)?; + instance.memories.clear(); for plan in @@ -590,6 +606,10 @@ impl Drop for InstancePool { /// /// Each instance index into the pool returns an iterator over the base addresses /// of the instance's linear memories. +/// +/// +/// The userfault handler relies on how memories are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct MemoryPool { mapping: Mmap, @@ -646,6 +666,9 @@ impl MemoryPool { /// /// Each instance index into the pool returns an iterator over the base addresses /// of the instance's tables. +/// +/// The userfault handler relies on how tables are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct TablePool { mapping: Mmap, @@ -710,6 +733,9 @@ impl TablePool { /// /// The top of the stack (starting stack pointer) is returned when a stack is allocated /// from the pool. +/// +/// The userfault handler relies on how stacks are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. #[derive(Debug)] struct StackPool { mapping: Mmap, @@ -717,6 +743,8 @@ struct StackPool { max_instances: usize, page_size: usize, free_list: Mutex>, + #[cfg(all(feature = "uffd", target_os = "linux"))] + faulted_guard_pages: Arc<[AtomicBool]>, } impl StackPool { @@ -745,6 +773,11 @@ impl StackPool { max_instances, page_size, free_list: Mutex::new((0..max_instances).collect()), + #[cfg(all(feature = "uffd", target_os = "linux"))] + faulted_guard_pages: std::iter::repeat_with(|| false.into()) + .take(max_instances) + .collect::>() + .into(), }) } @@ -774,11 +807,25 @@ impl StackPool { .as_mut_ptr() .add((index * self.stack_size) + self.page_size); - // Make the stack accessible (excluding the guard page) - if !make_accessible(bottom_of_stack, size_without_guard) { - return Err(FiberStackError::Resource( - "failed to make instance memory accessible".into(), - )); + cfg_if::cfg_if! { + if #[cfg(all(feature = "uffd", target_os = "linux"))] { + // Check to see if a guard page needs to be reset + if self.faulted_guard_pages[index].swap(false, Ordering::SeqCst) { + if !reset_guard_page(bottom_of_stack.sub(self.page_size), self.page_size) { + return Err(FiberStackError::Resource( + "failed to reset stack guard page".into(), + )); + } + } + + } else { + // Make the stack accessible (excluding the guard page) + if !make_accessible(bottom_of_stack, size_without_guard) { + return Err(FiberStackError::Resource( + "failed to make instance memory accessible".into(), + )); + } + } } // The top of the stack should be returned @@ -824,6 +871,8 @@ pub struct PoolingInstanceAllocator { instance_limits: InstanceLimits, instances: mem::ManuallyDrop, stacks: mem::ManuallyDrop, + #[cfg(all(feature = "uffd", target_os = "linux"))] + _fault_handler: PageFaultHandler, } impl PoolingInstanceAllocator { @@ -866,19 +915,28 @@ impl PoolingInstanceAllocator { )); } + let instances = InstancePool::new(&module_limits, &instance_limits)?; + let stacks = StackPool::new(&instance_limits, stack_size)?; + + #[cfg(all(feature = "uffd", target_os = "linux"))] + let _fault_handler = PageFaultHandler::new(&instances, &stacks)?; + Ok(Self { strategy, module_limits, instance_limits, - instances: mem::ManuallyDrop::new(InstancePool::new(&module_limits, &instance_limits)?), - stacks: mem::ManuallyDrop::new(StackPool::new(&instance_limits, stack_size)?), + instances: mem::ManuallyDrop::new(instances), + stacks: mem::ManuallyDrop::new(stacks), + #[cfg(all(feature = "uffd", target_os = "linux"))] + _fault_handler, }) } } impl Drop for PoolingInstanceAllocator { fn drop(&mut self) { - // There are manually dropped for the future uffd implementation + // Manually drop the pools before the fault handler (if uffd is enabled) + // This ensures that any fault handler thread monitoring the pool memory terminates unsafe { mem::ManuallyDrop::drop(&mut self.instances); mem::ManuallyDrop::drop(&mut self.stacks); diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs new file mode 100644 index 0000000000..ed236a8d43 --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -0,0 +1,752 @@ +//! Implements user-mode page fault handling with the `userfaultfd` ("uffd") system call on Linux. +//! +//! Handling page faults for memory accesses in regions relating to WebAssembly instances +//! enables the implementation of guard pages in user space rather than kernel space. +//! +//! This reduces the number of system calls and kernel locks needed to provide correct +//! WebAssembly memory semantics. +//! +//! Additionally, linear memories and WebAssembly tables can be lazy-initialized upon access. +//! +//! This feature requires a Linux kernel 4.11 or newer to use. + +use super::{InstancePool, StackPool}; +use crate::{instance::Instance, Mmap}; +use std::convert::TryInto; +use std::ptr; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use std::thread; +use userfaultfd::{Event, FeatureFlags, IoctlFlags, Uffd, UffdBuilder}; +use wasmtime_environ::{wasm::DefinedMemoryIndex, WASM_PAGE_SIZE}; + +pub unsafe fn make_accessible(_addr: *mut u8, _len: usize) -> bool { + // A no-op when userfaultfd is used + true +} + +pub unsafe fn reset_guard_page(addr: *mut u8, len: usize) -> bool { + // Guard pages are READ_WRITE with uffd until faulted + region::protect(addr, len, region::Protection::READ_WRITE).is_ok() +} + +pub unsafe fn decommit(addr: *mut u8, len: usize) { + // Use MADV_DONTNEED to mark the pages as missing + // This will cause a missing page fault for next access on any page in the given range + assert_eq!( + libc::madvise(addr as _, len, libc::MADV_DONTNEED), + 0, + "madvise failed to mark pages as missing: {}", + std::io::Error::last_os_error() + ); +} + +pub fn create_memory_map(_accessible_size: usize, mapping_size: usize) -> Result { + // Allocate a single read-write region at once + // As writable pages need to count towards commit charge, use MAP_NORESERVE to override. + // This implies that the kernel is configured to allow overcommit or else + // this allocation will almost certainly fail without a plethora of physical memory to back the alloction. + // The consequence of not reserving is that our process may segfault on any write to a memory + // page that cannot be backed (i.e. out of memory conditions). + + if mapping_size == 0 { + return Ok(Mmap::new()); + } + + unsafe { + let ptr = libc::mmap( + ptr::null_mut(), + mapping_size, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANON | libc::MAP_NORESERVE, + -1, + 0, + ); + + if ptr as isize == -1_isize { + return Err(format!( + "failed to allocate pool memory: {}", + std::io::Error::last_os_error() + )); + } + + Ok(Mmap::from_raw(ptr as usize, mapping_size)) + } +} + +/// Represents a location of a page fault within monitored regions of memory. +enum AddressLocation<'a> { + /// The address location is in a WebAssembly table page. + /// The fault handler will zero the page as tables are initialized at instantiation-time. + TablePage { + /// The address of the page being accessed. + page_addr: *mut u8, + /// The length of the page being accessed. + len: usize, + }, + /// The address location is in a WebAssembly linear memory page. + /// The fault handler will copy the pages from initialization data if necessary. + MemoryPage { + /// The address of the page being accessed. + page_addr: *mut u8, + /// The length of the page being accessed. + len: usize, + /// The instance related to the memory page that was accessed. + instance: &'a Instance, + /// The index of the memory that was accessed. + memory_index: usize, + /// The Wasm page index to initialize if the access was not a guard page. + page_index: Option, + }, + /// The address location is in an execution stack. + /// The fault handler will zero the page. + StackPage { + /// The address of the page being accessed. + page_addr: *mut u8, + /// The length of the page being accessed. + len: usize, + /// The index of the stack that was accessed. + index: usize, + /// Whether or not the access was to a guard page. + guard_page: bool, + }, +} + +/// Used to resolve fault addresses to address locations. +/// +/// This implementation relies heavily on how the various resource pools utilize their memory. +/// +/// `usize` is used here instead of pointers to keep this `Send` as it gets sent to the handler thread. +struct AddressLocator { + instances_start: usize, + instance_size: usize, + max_instances: usize, + memories_start: usize, + memories_end: usize, + memory_size: usize, + max_memories: usize, + tables_start: usize, + tables_end: usize, + table_size: usize, + stacks_start: usize, + stacks_end: usize, + stack_size: usize, + page_size: usize, +} + +impl AddressLocator { + fn new(instances: &InstancePool, stacks: &StackPool) -> Self { + let instances_start = instances.mapping.as_ptr() as usize; + let memories_start = instances.memories.mapping.as_ptr() as usize; + let memories_end = memories_start + instances.memories.mapping.len(); + let tables_start = instances.tables.mapping.as_ptr() as usize; + let tables_end = tables_start + instances.tables.mapping.len(); + let stacks_start = stacks.mapping.as_ptr() as usize; + let stacks_end = stacks_start + stacks.mapping.len(); + let stack_size = stacks.stack_size; + + // Should always have instances + debug_assert!(instances_start != 0); + + Self { + instances_start, + instance_size: instances.instance_size, + max_instances: instances.max_instances, + memories_start, + memories_end, + memory_size: instances.memories.memory_size, + max_memories: instances.memories.max_memories, + tables_start, + tables_end, + table_size: instances.tables.table_size, + stacks_start, + stacks_end, + stack_size, + page_size: instances.tables.page_size, + } + } + + // This is super-duper unsafe as it is used from the handler thread + // to access instance data without any locking primitives. + /// + /// It is assumed that the thread that owns the instance being accessed is + /// currently suspended waiting on a fault to be handled. + /// + /// Of course a stray faulting memory access from a thread that does not own + /// the instance might introduce a race, but this implementation considers + /// such to be a serious bug. + /// + /// If the assumption holds true, accessing the instance data from the handler thread + /// should, in theory, be safe. + unsafe fn get_instance(&self, index: usize) -> &mut Instance { + debug_assert!(index < self.max_instances); + &mut *((self.instances_start + (index * self.instance_size)) as *mut Instance) + } + + unsafe fn get_location(&self, addr: usize) -> Option { + // Check for a memory location + if addr >= self.memories_start && addr < self.memories_end { + let index = (addr - self.memories_start) / self.memory_size; + let memory_index = index % self.max_memories; + let memory_start = self.memories_start + (index * self.memory_size); + let page_index = (addr - memory_start) / (WASM_PAGE_SIZE as usize); + let instance = self.get_instance(index / self.max_memories); + + let init_page_index = instance + .memories + .get( + DefinedMemoryIndex::from_u32(memory_index as u32) + .try_into() + .unwrap(), + ) + .and_then(|m| { + if page_index < m.size() as usize { + Some(page_index) + } else { + None + } + }); + + return Some(AddressLocation::MemoryPage { + page_addr: (memory_start + page_index * (WASM_PAGE_SIZE as usize)) as _, + len: WASM_PAGE_SIZE as usize, + instance, + memory_index, + page_index: init_page_index, + }); + } + + // Check for a table location + if addr >= self.tables_start && addr < self.tables_end { + let index = (addr - self.tables_start) / self.table_size; + let table_start = self.tables_start + (index * self.table_size); + let table_offset = addr - table_start; + let page_index = table_offset / self.page_size; + + return Some(AddressLocation::TablePage { + page_addr: (table_start + (page_index * self.page_size)) as _, + len: self.page_size, + }); + } + + // Check for a stack location + if addr >= self.stacks_start && addr < self.stacks_end { + let index = (addr - self.stacks_start) / self.stack_size; + let stack_start = self.stacks_start + (index * self.stack_size); + let stack_offset = addr - stack_start; + let page_offset = (stack_offset / self.page_size) * self.page_size; + + return Some(AddressLocation::StackPage { + page_addr: (stack_start + page_offset) as _, + len: self.page_size, + index, + guard_page: stack_offset < self.page_size, + }); + } + + None + } +} + +fn wake_guard_page_access(uffd: &Uffd, page_addr: *const u8, len: usize) -> Result<(), String> { + unsafe { + // Set the page to NONE to induce a SIGSEV for the access on the next retry + region::protect(page_addr, len, region::Protection::NONE) + .map_err(|e| format!("failed to change guard page protection: {}", e))?; + + uffd.wake(page_addr as _, len).map_err(|e| { + format!( + "failed to wake page at {:p} with length {}: {}", + page_addr, len, e + ) + })?; + + Ok(()) + } +} + +fn handler_thread( + uffd: Uffd, + locator: AddressLocator, + mut registrations: usize, + faulted_stack_guard_pages: Arc<[AtomicBool]>, +) -> Result<(), String> { + loop { + match uffd.read_event().expect("failed to read event") { + Some(Event::Unmap { start, end }) => { + log::trace!("memory region unmapped: {:p}-{:p}", start, end); + + let (start, end) = (start as usize, end as usize); + + if (start == locator.memories_start && end == locator.memories_end) + || (start == locator.tables_start && end == locator.tables_end) + || (start == locator.stacks_start && end == locator.stacks_end) + { + registrations -= 1; + if registrations == 0 { + break; + } + } else { + panic!("unexpected memory region unmapped"); + } + } + Some(Event::Pagefault { + addr: access_addr, .. + }) => { + unsafe { + match locator.get_location(access_addr as usize) { + Some(AddressLocation::TablePage { page_addr, len }) => { + log::trace!( + "handling fault in table at address {:p} on page {:p}", + access_addr, + page_addr, + ); + + // Tables are always initialized upon instantiation, so zero the page + uffd.zeropage(page_addr as _, len, true).map_err(|e| { + format!( + "failed to zero page at {:p} with length {}: {}", + page_addr, len, e + ) + })?; + } + Some(AddressLocation::MemoryPage { + page_addr, + len, + instance, + memory_index, + page_index, + }) => { + log::trace!( + "handling fault in linear memory at address {:p} on page {:p}", + access_addr, + page_addr + ); + + match page_index { + Some(page_index) => { + // TODO: copy the memory initialization data rather than zero the page + uffd.zeropage(page_addr as _, len, true).map_err(|e| { + format!( + "failed to zero page at {:p} with length {}: {}", + page_addr, len, e + ) + })?; + } + None => { + log::trace!("out of bounds memory access at {:p}", access_addr); + + // Record the guard page fault with the instance so it can be reset later. + instance.record_guard_page_fault( + page_addr, + len, + reset_guard_page, + ); + wake_guard_page_access(&uffd, page_addr, len)?; + } + } + } + Some(AddressLocation::StackPage { + page_addr, + len, + index, + guard_page, + }) => { + log::trace!( + "handling fault in stack {} at address {:p}", + index, + access_addr, + ); + + if guard_page { + // Logging as trace as stack guard pages might be a trap condition in the future + log::trace!("stack overflow fault at {:p}", access_addr); + + // Mark the stack as having a faulted guard page + // The next time the stack is used the guard page will be reset + faulted_stack_guard_pages[index].store(true, Ordering::SeqCst); + wake_guard_page_access(&uffd, page_addr, len)?; + continue; + } + + // Always zero stack pages + uffd.zeropage(page_addr as _, len, true).map_err(|e| { + format!( + "failed to zero page at {:p} with length {}: {}", + page_addr, len, e + ) + })?; + } + None => { + return Err(format!( + "failed to locate fault address {:p} in registered memory regions", + access_addr + )); + } + } + } + } + Some(_) => continue, + None => break, + } + } + + Ok(()) +} + +#[derive(Debug)] +pub struct PageFaultHandler { + thread: Option>>, +} + +impl PageFaultHandler { + pub(super) fn new(instances: &InstancePool, stacks: &StackPool) -> Result { + let uffd = UffdBuilder::new() + .close_on_exec(true) + .require_features(FeatureFlags::EVENT_UNMAP) + .create() + .map_err(|e| format!("failed to create user fault descriptor: {}", e))?; + + // Register the ranges with the userfault fd + let mut registrations = 0; + for (start, len) in &[ + ( + instances.memories.mapping.as_ptr() as usize, + instances.memories.mapping.len(), + ), + ( + instances.tables.mapping.as_ptr() as usize, + instances.tables.mapping.len(), + ), + (stacks.mapping.as_ptr() as usize, stacks.mapping.len()), + ] { + if *start == 0 || *len == 0 { + continue; + } + + let ioctls = uffd + .register(*start as _, *len) + .map_err(|e| format!("failed to register user fault range: {}", e))?; + + if !ioctls.contains(IoctlFlags::WAKE | IoctlFlags::COPY | IoctlFlags::ZEROPAGE) { + return Err(format!( + "required user fault ioctls not supported; found: {:?}", + ioctls, + )); + } + + registrations += 1; + } + + let thread = if registrations == 0 { + log::trace!("user fault handling disabled as there are no regions to monitor"); + None + } else { + log::trace!( + "user fault handling enabled on {} memory regions", + registrations + ); + + let locator = AddressLocator::new(&instances, &stacks); + + let faulted_stack_guard_pages = stacks.faulted_guard_pages.clone(); + + Some( + thread::Builder::new() + .name("page fault handler".into()) + .spawn(move || { + handler_thread(uffd, locator, registrations, faulted_stack_guard_pages) + }) + .map_err(|e| format!("failed to spawn page fault handler thread: {}", e))?, + ) + }; + + Ok(Self { thread }) + } +} + +impl Drop for PageFaultHandler { + fn drop(&mut self) { + if let Some(thread) = self.thread.take() { + thread + .join() + .expect("failed to join page fault handler thread") + .expect("fault handler thread failed"); + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{ + table::max_table_element_size, Imports, InstanceAllocationRequest, InstanceLimits, + ModuleLimits, PoolingAllocationStrategy, VMSharedSignatureIndex, + }; + use wasmtime_environ::{ + entity::PrimaryMap, + wasm::{Memory, Table, TableElementType, WasmType}, + MemoryPlan, MemoryStyle, Module, TablePlan, TableStyle, + }; + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_address_locator() { + let module_limits = ModuleLimits { + imported_functions: 0, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 0, + functions: 0, + tables: 3, + memories: 2, + globals: 0, + table_elements: 1000, + memory_pages: 2, + }; + let instance_limits = InstanceLimits { + count: 3, + address_space_size: (WASM_PAGE_SIZE * 10) as u64, + }; + + let instances = + InstancePool::new(&module_limits, &instance_limits).expect("should allocate"); + let stacks = StackPool::new(&instance_limits, 8192).expect("should allocate"); + + let locator = AddressLocator::new(&instances, &stacks); + + assert_eq!(locator.instances_start, instances.mapping.as_ptr() as usize); + assert_eq!(locator.instance_size, 4096); + assert_eq!(locator.max_instances, 3); + assert_eq!( + locator.memories_start, + instances.memories.mapping.as_ptr() as usize + ); + assert_eq!( + locator.memories_end, + locator.memories_start + instances.memories.mapping.len() + ); + assert_eq!(locator.memory_size, (WASM_PAGE_SIZE * 10) as usize); + assert_eq!(locator.max_memories, 2); + assert_eq!( + locator.tables_start, + instances.tables.mapping.as_ptr() as usize + ); + assert_eq!( + locator.tables_end, + locator.tables_start + instances.tables.mapping.len() + ); + assert_eq!(locator.table_size, 8192); + + assert_eq!(locator.stacks_start, stacks.mapping.as_ptr() as usize); + assert_eq!( + locator.stacks_end, + locator.stacks_start + stacks.mapping.len() + ); + assert_eq!(locator.stack_size, 12288); + + unsafe { + assert!(locator.get_location(0).is_none()); + assert!(locator + .get_location(std::cmp::max( + locator.memories_end, + std::cmp::max(locator.tables_end, locator.stacks_end) + )) + .is_none()); + + let mut module = Module::new(); + + for _ in 0..module_limits.memories { + module.memory_plans.push(MemoryPlan { + memory: Memory { + minimum: 2, + maximum: Some(2), + shared: false, + }, + style: MemoryStyle::Static { bound: 1 }, + offset_guard_size: 0, + }); + } + + for _ in 0..module_limits.tables { + module.table_plans.push(TablePlan { + table: Table { + wasm_ty: WasmType::FuncRef, + ty: TableElementType::Func, + minimum: 800, + maximum: Some(900), + }, + style: TableStyle::CallerChecksSignature, + }); + } + + module_limits + .validate_module(&module) + .expect("should validate"); + + let mut handles = Vec::new(); + let module = Arc::new(module); + let finished_functions = &PrimaryMap::new(); + + // Allocate the maximum number of instances with the maxmimum number of memories and tables + for _ in 0..instances.max_instances { + handles.push( + instances + .allocate( + PoolingAllocationStrategy::Random, + InstanceAllocationRequest { + module: module.clone(), + finished_functions, + imports: Imports { + functions: &[], + tables: &[], + memories: &[], + globals: &[], + }, + lookup_shared_signature: &|_| VMSharedSignatureIndex::default(), + host_state: Box::new(()), + interrupts: std::ptr::null(), + externref_activations_table: std::ptr::null_mut(), + stack_map_registry: std::ptr::null_mut(), + }, + ) + .expect("instance should allocate"), + ); + } + + // Validate memory locations + for instance_index in 0..instances.max_instances { + for memory_index in 0..instances.memories.max_memories { + let memory_start = locator.memories_start + + (instance_index * locator.memory_size * locator.max_memories) + + (memory_index * locator.memory_size); + + // Test for access to first page + match locator.get_location(memory_start + 10000) { + Some(AddressLocation::MemoryPage { + page_addr, + len, + instance: _, + memory_index: mem_index, + page_index, + }) => { + assert_eq!(page_addr, memory_start as _); + assert_eq!(len, WASM_PAGE_SIZE as usize); + assert_eq!(mem_index, memory_index); + assert_eq!(page_index, Some(0)); + } + _ => panic!("expected a memory page location"), + } + + // Test for access to second page + match locator.get_location(memory_start + 1024 + WASM_PAGE_SIZE as usize) { + Some(AddressLocation::MemoryPage { + page_addr, + len, + instance: _, + memory_index: mem_index, + page_index, + }) => { + assert_eq!(page_addr, (memory_start + WASM_PAGE_SIZE as usize) as _); + assert_eq!(len, WASM_PAGE_SIZE as usize); + assert_eq!(mem_index, memory_index); + assert_eq!(page_index, Some(1)); + } + _ => panic!("expected a memory page location"), + } + + // Test for guard page + match locator.get_location(memory_start + 10 + 9 * WASM_PAGE_SIZE as usize) { + Some(AddressLocation::MemoryPage { + page_addr, + len, + instance: _, + memory_index: mem_index, + page_index, + }) => { + assert_eq!( + page_addr, + (memory_start + (9 * WASM_PAGE_SIZE as usize)) as _ + ); + assert_eq!(len, WASM_PAGE_SIZE as usize); + assert_eq!(mem_index, memory_index); + assert_eq!(page_index, None); + } + _ => panic!("expected a memory page location"), + } + } + } + + // Validate table locations + for instance_index in 0..instances.max_instances { + for table_index in 0..instances.tables.max_tables { + let table_start = locator.tables_start + + (instance_index * locator.table_size * instances.tables.max_tables) + + (table_index * locator.table_size); + + // Check for an access of index 107 (first page) + match locator.get_location(table_start + (107 * max_table_element_size())) { + Some(AddressLocation::TablePage { page_addr, len }) => { + assert_eq!(page_addr, table_start as _); + assert_eq!(len, locator.page_size); + } + _ => panic!("expected a table page location"), + } + + // Check for an access of index 799 (second page) + match locator.get_location(table_start + (799 * max_table_element_size())) { + Some(AddressLocation::TablePage { page_addr, len }) => { + assert_eq!(page_addr, (table_start + locator.page_size) as _); + assert_eq!(len, locator.page_size); + } + _ => panic!("expected a table page location"), + } + } + } + + // Validate stack locations + for stack_index in 0..instances.max_instances { + let stack_start = locator.stacks_start + (stack_index * locator.stack_size); + + // Check for stack page location + match locator.get_location(stack_start + locator.page_size * 2) { + Some(AddressLocation::StackPage { + page_addr, + len, + index, + guard_page, + }) => { + assert_eq!(page_addr, (stack_start + locator.page_size * 2) as _); + assert_eq!(len, locator.page_size); + assert_eq!(index, stack_index); + assert!(!guard_page); + } + _ => panic!("expected a stack page location"), + } + + // Check for guard page + match locator.get_location(stack_start) { + Some(AddressLocation::StackPage { + page_addr, + len, + index, + guard_page, + }) => { + assert_eq!(page_addr, stack_start as _); + assert_eq!(len, locator.page_size); + assert_eq!(index, stack_index); + assert!(guard_page); + } + _ => panic!("expected a stack page location"), + } + } + + for handle in handles.drain(..) { + instances.deallocate(&handle); + } + } + } +} diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 752b3c7ff8..466e3cb61d 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -73,3 +73,6 @@ experimental_x64 = ["wasmtime-jit/experimental_x64"] # Enables support for "async stores" as well as defining host functions as # `async fn` and calling functions asynchronously. async = ["wasmtime-fiber"] + +# Enables userfaultfd support in the runtime's pooling allocator when building on Linux +uffd = ["wasmtime-runtime/uffd"]