Remove support for userfaultfd (#4040)
This commit removes support for the `userfaultfd` or "uffd" syscall on Linux. This support was originally added for users migrating from Lucet to Wasmtime, but the recent developments of kernel-supported copy-on-write support for memory initialization wound up being more appropriate for these use cases than usefaultfd. The main reason for moving to copy-on-write initialization are: * The `userfaultfd` feature was never necessarily intended for this style of use case with wasm and was susceptible to subtle and rare bugs that were extremely difficult to track down. We were never 100% certain that there were kernel bugs related to userfaultfd but the suspicion never went away. * Handling faults with userfaultfd was always slow and single-threaded. Only one thread could handle faults and traveling to user-space to handle faults is inherently slower than handling them all in the kernel. The single-threaded aspect in particular presented a significant scaling bottleneck for embeddings that want to run many wasm instances in parallel. * One of the major benefits of userfaultfd was lazy initialization of wasm linear memory which is also achieved with the copy-on-write initialization support we have right now. * One of the suspected benefits of userfaultfd was less frobbing of the kernel vma structures when wasm modules are instantiated. Currently the copy-on-write support has a mitigation where we attempt to reuse the memory images where possible to avoid changing vma structures. When comparing this to userfaultfd's performance it was found that kernel modifications of vmas aren't a worrisome bottleneck so copy-on-write is suitable for this as well. Overall there are no remaining benefits that userfaultfd gives that copy-on-write doesn't, and copy-on-write solves a major downsides of userfaultfd, the scaling issue with a single faulting thread. Additionally copy-on-write support seems much more robust in terms of kernel implementation since it's only using standard memory-management syscalls which are heavily exercised. Finally copy-on-write support provides a new bonus where read-only memory in WebAssembly can be mapped directly to the same kernel cache page, even amongst many wasm instances of the same module, which was never possible with userfaultfd. In light of all this it's expected that all users of userfaultfd should migrate to the copy-on-write initialization of Wasmtime (which is enabled by default).
This commit is contained in:
Alex Crichton
@@ -30,10 +30,6 @@ cfg_if::cfg_if! {
|
||||
if #[cfg(windows)] {
|
||||
mod windows;
|
||||
use windows as imp;
|
||||
} else if #[cfg(all(feature = "uffd", target_os = "linux"))] {
|
||||
mod uffd;
|
||||
use uffd as imp;
|
||||
use imp::initialize_memory_pool;
|
||||
} else if #[cfg(target_os = "linux")] {
|
||||
mod linux;
|
||||
use linux as imp;
|
||||
@@ -205,9 +201,6 @@ impl Default for PoolingAllocationStrategy {
|
||||
/// structure depending on the limits used to create the pool.
|
||||
///
|
||||
/// The pool maintains a free list for fast instance allocation.
|
||||
///
|
||||
/// The userfault handler relies on how instances are stored in the mapping,
|
||||
/// so make sure the uffd implementation is kept up-to-date.
|
||||
#[derive(Debug)]
|
||||
struct InstancePool {
|
||||
mapping: Mmap,
|
||||
@@ -456,7 +449,7 @@ impl InstancePool {
|
||||
for ((def_mem_idx, memory), base) in
|
||||
memories.iter_mut().zip(self.memories.get(instance_index))
|
||||
{
|
||||
let mut memory = mem::take(memory);
|
||||
let memory = mem::take(memory);
|
||||
assert!(memory.is_static());
|
||||
|
||||
match memory {
|
||||
@@ -475,16 +468,6 @@ impl InstancePool {
|
||||
}
|
||||
|
||||
_ => {
|
||||
// Reset any faulted guard pages as the physical
|
||||
// memory may be reused for another instance in
|
||||
// the future.
|
||||
#[cfg(all(feature = "uffd", target_os = "linux"))]
|
||||
memory
|
||||
.reset_guard_pages()
|
||||
.expect("failed to reset guard pages");
|
||||
// require mutable on all platforms, not just uffd
|
||||
drop(&mut memory);
|
||||
|
||||
let size = memory.byte_size();
|
||||
drop(memory);
|
||||
decommit_memory_pages(base, size)
|
||||
@@ -667,9 +650,6 @@ impl InstancePool {
|
||||
///
|
||||
/// Each instance index into the pool returns an iterator over the base addresses
|
||||
/// of the instance's linear memories.
|
||||
///
|
||||
/// The userfault handler relies on how memories are stored in the mapping,
|
||||
/// so make sure the uffd implementation is kept up-to-date.
|
||||
#[derive(Debug)]
|
||||
struct MemoryPool {
|
||||
mapping: Mmap,
|
||||
@@ -778,10 +758,6 @@ impl MemoryPool {
|
||||
max_memory_size: (instance_limits.memory_pages as usize) * (WASM_PAGE_SIZE as usize),
|
||||
};
|
||||
|
||||
// uffd support requires some special setup for the memory pool
|
||||
#[cfg(all(feature = "uffd", target_os = "linux"))]
|
||||
initialize_memory_pool(&pool)?;
|
||||
|
||||
Ok(pool)
|
||||
}
|
||||
|
||||
@@ -1044,14 +1020,11 @@ impl StackPool {
|
||||
/// Note: the resource pools are manually dropped so that the fault handler terminates correctly.
|
||||
#[derive(Debug)]
|
||||
pub struct PoolingInstanceAllocator {
|
||||
// This is manually drop so that the pools unmap their memory before the page fault handler drops.
|
||||
instances: mem::ManuallyDrop<InstancePool>,
|
||||
instances: InstancePool,
|
||||
#[cfg(all(feature = "async", unix))]
|
||||
stacks: StackPool,
|
||||
#[cfg(all(feature = "async", windows))]
|
||||
stack_size: usize,
|
||||
#[cfg(all(feature = "uffd", target_os = "linux"))]
|
||||
_fault_handler: imp::PageFaultHandler,
|
||||
}
|
||||
|
||||
impl PoolingInstanceAllocator {
|
||||
@@ -1068,33 +1041,18 @@ impl PoolingInstanceAllocator {
|
||||
|
||||
let instances = InstancePool::new(strategy, &instance_limits, tunables)?;
|
||||
|
||||
#[cfg(all(feature = "uffd", target_os = "linux"))]
|
||||
let _fault_handler = imp::PageFaultHandler::new(&instances)?;
|
||||
|
||||
drop(stack_size); // suppress unused warnings w/o async feature
|
||||
|
||||
Ok(Self {
|
||||
instances: mem::ManuallyDrop::new(instances),
|
||||
instances: instances,
|
||||
#[cfg(all(feature = "async", unix))]
|
||||
stacks: StackPool::new(&instance_limits, stack_size)?,
|
||||
#[cfg(all(feature = "async", windows))]
|
||||
stack_size,
|
||||
#[cfg(all(feature = "uffd", target_os = "linux"))]
|
||||
_fault_handler,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PoolingInstanceAllocator {
|
||||
fn drop(&mut self) {
|
||||
// Manually drop the pools before the fault handler (if uffd is enabled)
|
||||
// This ensures that any fault handler thread monitoring the pool memory terminates
|
||||
unsafe {
|
||||
mem::ManuallyDrop::drop(&mut self.instances);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl InstanceAllocator for PoolingInstanceAllocator {
|
||||
fn validate(&self, module: &Module) -> Result<()> {
|
||||
self.instances.validate_memory_plans(module)?;
|
||||
@@ -1132,28 +1090,7 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator {
|
||||
is_bulk_memory: bool,
|
||||
) -> Result<(), InstantiationError> {
|
||||
let instance = handle.instance_mut();
|
||||
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(all(feature = "uffd", target_os = "linux"))] {
|
||||
match &module.memory_initialization {
|
||||
wasmtime_environ::MemoryInitialization::Paged { .. } => {
|
||||
if !is_bulk_memory {
|
||||
super::check_init_bounds(instance, module)?;
|
||||
}
|
||||
|
||||
// Initialize the tables
|
||||
super::initialize_tables(instance, module)?;
|
||||
|
||||
// Don't initialize the memory; the fault handler will back the pages when accessed
|
||||
|
||||
Ok(())
|
||||
},
|
||||
_ => initialize_instance(instance, module, is_bulk_memory)
|
||||
}
|
||||
} else {
|
||||
initialize_instance(instance, module, is_bulk_memory)
|
||||
}
|
||||
}
|
||||
initialize_instance(instance, module, is_bulk_memory)
|
||||
}
|
||||
|
||||
unsafe fn deallocate(&self, handle: &InstanceHandle) {
|
||||
|
||||
Reference in New Issue
Block a user