From 03715dda9d536cc8f81d4ed9d5d152139fcd2eb0 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 1 Dec 2022 16:22:08 -0600 Subject: [PATCH] Tidy up some internals of instance allocation (#5346) * Simplify the `ModuleRuntimeInfo` trait slightly Fold two functions into one as they're only called from one location anyway. * Remove ModuleRuntimeInfo::signature This is redundant as the array mapping is already stored within the `VMContext` so that can be consulted rather than having a separate trait function for it. This required altering the `Global` creation slightly to work correctly in this situation. * Remove a now-dead constant * Shared `VMOffsets` across instances This commit removes the computation of `VMOffsets` to being per-module instead of per-instance. The `VMOffsets` structure is also quite large so this shaves off 112 bytes per instance which isn't a huge impact but should help lower the cost of instantiating small modules. * Remove `InstanceAllocator::adjust_tunables` This is no longer needed or necessary with the pooling allocator. * Fix compile warning * Fix a vtune warning * Fix pooling tests * Fix another test warning --- crates/environ/src/vmoffsets.rs | 5 -- crates/jit/src/instantiate.rs | 30 +++---- crates/jit/src/profiling/jitdump_linux.rs | 3 +- crates/jit/src/profiling/vtune.rs | 3 +- crates/runtime/src/instance.rs | 85 ++++++++++--------- crates/runtime/src/instance/allocator.rs | 25 ++---- .../runtime/src/instance/allocator/pooling.rs | 51 ++++------- crates/runtime/src/lib.rs | 19 ++--- crates/wasmtime/src/engine.rs | 2 +- crates/wasmtime/src/module.rs | 68 +++++++-------- crates/wasmtime/src/module/registry.rs | 8 +- crates/wasmtime/src/trampoline.rs | 4 +- crates/wasmtime/src/trampoline/global.rs | 4 +- tests/all/pooling_allocator.rs | 11 +-- 14 files changed, 142 insertions(+), 176 deletions(-) diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index de131eab4a..5207870101 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -31,11 +31,6 @@ use cranelift_entity::packed_option::ReservedValue; use std::convert::TryFrom; use wasmtime_types::OwnedMemoryIndex; -/// Sentinel value indicating that wasm has been interrupted. -// Note that this has a bit of an odd definition. See the `insert_stack_check` -// function in `cranelift/codegen/src/isa/x86/abi.rs` for more information -pub const INTERRUPTED: usize = usize::max_value() - 32 * 1024; - #[cfg(target_pointer_width = "32")] fn cast_to_u32(sz: usize) -> u32 { u32::try_from(sz).unwrap() diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 52f7297b32..db3137d329 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -20,8 +20,7 @@ use wasmtime_environ::{ PrimaryMap, SignatureIndex, StackMapInformation, Tunables, WasmFunctionInfo, }; use wasmtime_runtime::{ - CompiledModuleId, CompiledModuleIdAllocator, GdbJitImageRegistration, MmapVec, VMFunctionBody, - VMTrampoline, + CompiledModuleId, CompiledModuleIdAllocator, GdbJitImageRegistration, MmapVec, VMTrampoline, }; /// Secondary in-memory results of compilation. @@ -482,19 +481,22 @@ impl CompiledModule { Arc::get_mut(&mut self.module) } - /// Returns the map of all finished JIT functions compiled for this module + /// Returns an iterator over all functions defined within this module with + /// their index and their body in memory. #[inline] pub fn finished_functions( &self, - ) -> impl ExactSizeIterator + '_ { - let text = self.text(); - self.funcs.iter().map(move |(i, (_, loc))| { - let func = &text[loc.start as usize..][..loc.length as usize]; - ( - i, - std::ptr::slice_from_raw_parts(func.as_ptr().cast::(), func.len()), - ) - }) + ) -> impl ExactSizeIterator + '_ { + self.funcs + .iter() + .map(move |(i, _)| (i, self.finished_function(i))) + } + + /// Returns the body of the function that `index` points to. + #[inline] + pub fn finished_function(&self, index: DefinedFuncIndex) -> &[u8] { + let (_, loc) = &self.funcs[index]; + &self.text()[loc.start as usize..][..loc.length as usize] } /// Returns the per-signature trampolines for this module. @@ -517,9 +519,7 @@ impl CompiledModule { /// /// The iterator returned iterates over the span of the compiled function in /// memory with the stack maps associated with those bytes. - pub fn stack_maps( - &self, - ) -> impl Iterator { + pub fn stack_maps(&self) -> impl Iterator { self.finished_functions() .map(|(_, f)| f) .zip(self.funcs.values().map(|f| &f.0.stack_maps[..])) diff --git a/crates/jit/src/profiling/jitdump_linux.rs b/crates/jit/src/profiling/jitdump_linux.rs index 8ff1c3a11f..1336bf215d 100644 --- a/crates/jit/src/profiling/jitdump_linux.rs +++ b/crates/jit/src/profiling/jitdump_linux.rs @@ -83,7 +83,8 @@ impl State { let tid = pid; // ThreadId does appear to track underlying thread. Using PID. for (idx, func) in module.finished_functions() { - let (addr, len) = unsafe { ((*func).as_ptr().cast::(), (*func).len()) }; + let addr = func.as_ptr(); + let len = func.len(); if let Some(img) = &dbg_image { if let Err(err) = self.dump_from_debug_image(img, "wasm", addr, len, pid, tid) { println!( diff --git a/crates/jit/src/profiling/vtune.rs b/crates/jit/src/profiling/vtune.rs index 3e6b6415a5..b99511110b 100644 --- a/crates/jit/src/profiling/vtune.rs +++ b/crates/jit/src/profiling/vtune.rs @@ -93,7 +93,8 @@ impl State { .unwrap_or_else(|| format!("wasm_module_{}", global_module_id)); for (idx, func) in module.finished_functions() { - let (addr, len) = unsafe { ((*func).as_ptr().cast::(), (*func).len()) }; + let addr = func.as_ptr(); + let len = func.len(); let method_name = super::debug_name(module, idx); log::trace!( "new function {:?}::{:?} @ {:?}\n", diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 9e4f88ecb4..84acc6294e 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -13,7 +13,7 @@ use crate::vmcontext::{ }; use crate::{ ExportFunction, ExportGlobal, ExportMemory, ExportTable, Imports, ModuleRuntimeInfo, Store, - VMFunctionBody, + VMFunctionBody, VMSharedSignatureIndex, }; use anyhow::Error; use memoffset::offset_of; @@ -61,9 +61,6 @@ pub(crate) struct Instance { /// functions, lazy initialization state, etc. runtime_info: Arc, - /// Offsets in the `vmctx` region, precomputed from the `module` above. - offsets: VMOffsets, - /// WebAssembly linear memory data. /// /// This is where all runtime information about defined linear memories in @@ -105,13 +102,12 @@ impl Instance { unsafe fn new_at( ptr: *mut Instance, alloc_size: usize, - offsets: VMOffsets, req: InstanceAllocationRequest, memories: PrimaryMap, tables: PrimaryMap, ) { // The allocation must be *at least* the size required of `Instance`. - assert!(alloc_size >= Self::alloc_layout(&offsets).size()); + assert!(alloc_size >= Self::alloc_layout(req.runtime_info.offsets()).size()); let module = req.runtime_info.module(); let dropped_elements = EntitySet::with_capacity(module.passive_elements.len()); @@ -121,7 +117,6 @@ impl Instance { ptr, Instance { runtime_info: req.runtime_info.clone(), - offsets, memories, tables, dropped_elements, @@ -133,7 +128,7 @@ impl Instance { }, ); - (*ptr).initialize_vmctx(module, req.store, req.imports); + (*ptr).initialize_vmctx(module, req.runtime_info.offsets(), req.store, req.imports); } /// Helper function to access various locations offset from our `*mut @@ -148,24 +143,28 @@ impl Instance { self.runtime_info.module() } + fn offsets(&self) -> &VMOffsets { + self.runtime_info.offsets() + } + /// Return the indexed `VMFunctionImport`. fn imported_function(&self, index: FuncIndex) -> &VMFunctionImport { - unsafe { &*self.vmctx_plus_offset(self.offsets.vmctx_vmfunction_import(index)) } + unsafe { &*self.vmctx_plus_offset(self.offsets().vmctx_vmfunction_import(index)) } } /// Return the index `VMTableImport`. fn imported_table(&self, index: TableIndex) -> &VMTableImport { - unsafe { &*self.vmctx_plus_offset(self.offsets.vmctx_vmtable_import(index)) } + unsafe { &*self.vmctx_plus_offset(self.offsets().vmctx_vmtable_import(index)) } } /// Return the indexed `VMMemoryImport`. fn imported_memory(&self, index: MemoryIndex) -> &VMMemoryImport { - unsafe { &*self.vmctx_plus_offset(self.offsets.vmctx_vmmemory_import(index)) } + unsafe { &*self.vmctx_plus_offset(self.offsets().vmctx_vmmemory_import(index)) } } /// Return the indexed `VMGlobalImport`. fn imported_global(&self, index: GlobalIndex) -> &VMGlobalImport { - unsafe { &*self.vmctx_plus_offset(self.offsets.vmctx_vmglobal_import(index)) } + unsafe { &*self.vmctx_plus_offset(self.offsets().vmctx_vmglobal_import(index)) } } /// Return the indexed `VMTableDefinition`. @@ -183,7 +182,7 @@ impl Instance { /// Return the indexed `VMTableDefinition`. fn table_ptr(&self, index: DefinedTableIndex) -> *mut VMTableDefinition { - unsafe { self.vmctx_plus_offset(self.offsets.vmctx_vmtable_definition(index)) } + unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmtable_definition(index)) } } /// Get a locally defined or imported memory. @@ -221,7 +220,7 @@ impl Instance { /// Return the indexed `VMMemoryDefinition`. fn memory_ptr(&self, index: DefinedMemoryIndex) -> *mut VMMemoryDefinition { - unsafe { *self.vmctx_plus_offset(self.offsets.vmctx_vmmemory_pointer(index)) } + unsafe { *self.vmctx_plus_offset(self.offsets().vmctx_vmmemory_pointer(index)) } } /// Return the indexed `VMGlobalDefinition`. @@ -231,7 +230,7 @@ impl Instance { /// Return the indexed `VMGlobalDefinition`. fn global_ptr(&self, index: DefinedGlobalIndex) -> *mut VMGlobalDefinition { - unsafe { self.vmctx_plus_offset(self.offsets.vmctx_vmglobal_definition(index)) } + unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmglobal_definition(index)) } } /// Get a raw pointer to the global at the given index regardless whether it @@ -251,17 +250,17 @@ impl Instance { /// Return a pointer to the interrupts structure pub fn runtime_limits(&self) -> *mut *const VMRuntimeLimits { - unsafe { self.vmctx_plus_offset(self.offsets.vmctx_runtime_limits()) } + unsafe { self.vmctx_plus_offset(self.offsets().vmctx_runtime_limits()) } } /// Return a pointer to the global epoch counter used by this instance. pub fn epoch_ptr(&self) -> *mut *const AtomicU64 { - unsafe { self.vmctx_plus_offset(self.offsets.vmctx_epoch_ptr()) } + unsafe { self.vmctx_plus_offset(self.offsets().vmctx_epoch_ptr()) } } /// Return a pointer to the `VMExternRefActivationsTable`. pub fn externref_activations_table(&self) -> *mut *mut VMExternRefActivationsTable { - unsafe { self.vmctx_plus_offset(self.offsets.vmctx_externref_activations_table()) } + unsafe { self.vmctx_plus_offset(self.offsets().vmctx_externref_activations_table()) } } /// Gets a pointer to this instance's `Store` which was originally @@ -276,14 +275,15 @@ impl Instance { /// store). #[inline] pub fn store(&self) -> *mut dyn Store { - let ptr = unsafe { *self.vmctx_plus_offset::<*mut dyn Store>(self.offsets.vmctx_store()) }; + let ptr = + unsafe { *self.vmctx_plus_offset::<*mut dyn Store>(self.offsets().vmctx_store()) }; assert!(!ptr.is_null()); ptr } pub unsafe fn set_store(&mut self, store: Option<*mut dyn Store>) { if let Some(store) = store { - *self.vmctx_plus_offset(self.offsets.vmctx_store()) = store; + *self.vmctx_plus_offset(self.offsets().vmctx_store()) = store; *self.runtime_limits() = (*store).vmruntime_limits(); *self.epoch_ptr() = (*store).epoch_ptr(); *self.externref_activations_table() = (*store).externref_activations_table().0; @@ -292,7 +292,7 @@ impl Instance { mem::size_of::<*mut dyn Store>(), mem::size_of::<[*mut (); 2]>() ); - *self.vmctx_plus_offset::<[*mut (); 2]>(self.offsets.vmctx_store()) = + *self.vmctx_plus_offset::<[*mut (); 2]>(self.offsets().vmctx_store()) = [ptr::null_mut(), ptr::null_mut()]; *self.runtime_limits() = ptr::null_mut(); @@ -302,7 +302,7 @@ impl Instance { } pub(crate) unsafe fn set_callee(&mut self, callee: Option>) { - *self.vmctx_plus_offset(self.offsets.vmctx_callee()) = + *self.vmctx_plus_offset(self.offsets().vmctx_callee()) = callee.map_or(ptr::null_mut(), |c| c.as_ptr()); } @@ -499,13 +499,15 @@ impl Instance { sig: SignatureIndex, into: *mut VMCallerCheckedAnyfunc, ) { - let type_index = self.runtime_info.signature(sig); + let type_index = unsafe { + let base: *const VMSharedSignatureIndex = + *self.vmctx_plus_offset(self.offsets().vmctx_signature_ids_array()); + *base.add(sig.index()) + }; let (func_ptr, vmctx) = if let Some(def_index) = self.module().defined_func_index(index) { ( - (self.runtime_info.image_base() - + self.runtime_info.function_loc(def_index).start as usize) - as *mut _, + self.runtime_info.function(def_index), VMOpaqueContext::from_vmcontext(self.vmctx_ptr()), ) } else { @@ -569,7 +571,7 @@ impl Instance { let sig = func.signature; let anyfunc: *mut VMCallerCheckedAnyfunc = self .vmctx_plus_offset::( - self.offsets.vmctx_anyfunc(func.anyfunc), + self.offsets().vmctx_anyfunc(func.anyfunc), ); self.construct_anyfunc(index, sig, anyfunc); @@ -898,44 +900,49 @@ impl Instance { /// The `VMContext` memory is assumed to be uninitialized; any field /// that we need in a certain state will be explicitly written by this /// function. - unsafe fn initialize_vmctx(&mut self, module: &Module, store: StorePtr, imports: Imports) { + unsafe fn initialize_vmctx( + &mut self, + module: &Module, + offsets: &VMOffsets, + store: StorePtr, + imports: Imports, + ) { assert!(std::ptr::eq(module, self.module().as_ref())); - *self.vmctx_plus_offset(self.offsets.vmctx_magic()) = VMCONTEXT_MAGIC; + *self.vmctx_plus_offset(offsets.vmctx_magic()) = VMCONTEXT_MAGIC; self.set_callee(None); self.set_store(store.as_raw()); // Initialize shared signatures let signatures = self.runtime_info.signature_ids(); - *self.vmctx_plus_offset(self.offsets.vmctx_signature_ids_array()) = signatures.as_ptr(); + *self.vmctx_plus_offset(offsets.vmctx_signature_ids_array()) = signatures.as_ptr(); // Initialize the built-in functions - *self.vmctx_plus_offset(self.offsets.vmctx_builtin_functions()) = - &VMBuiltinFunctionsArray::INIT; + *self.vmctx_plus_offset(offsets.vmctx_builtin_functions()) = &VMBuiltinFunctionsArray::INIT; // Initialize the imports debug_assert_eq!(imports.functions.len(), module.num_imported_funcs); ptr::copy_nonoverlapping( imports.functions.as_ptr(), - self.vmctx_plus_offset(self.offsets.vmctx_imported_functions_begin()), + self.vmctx_plus_offset(offsets.vmctx_imported_functions_begin()), imports.functions.len(), ); debug_assert_eq!(imports.tables.len(), module.num_imported_tables); ptr::copy_nonoverlapping( imports.tables.as_ptr(), - self.vmctx_plus_offset(self.offsets.vmctx_imported_tables_begin()), + self.vmctx_plus_offset(offsets.vmctx_imported_tables_begin()), imports.tables.len(), ); debug_assert_eq!(imports.memories.len(), module.num_imported_memories); ptr::copy_nonoverlapping( imports.memories.as_ptr(), - self.vmctx_plus_offset(self.offsets.vmctx_imported_memories_begin()), + self.vmctx_plus_offset(offsets.vmctx_imported_memories_begin()), imports.memories.len(), ); debug_assert_eq!(imports.globals.len(), module.num_imported_globals); ptr::copy_nonoverlapping( imports.globals.as_ptr(), - self.vmctx_plus_offset(self.offsets.vmctx_imported_globals_begin()), + self.vmctx_plus_offset(offsets.vmctx_imported_globals_begin()), imports.globals.len(), ); @@ -946,7 +953,7 @@ impl Instance { // any state now. // Initialize the defined tables - let mut ptr = self.vmctx_plus_offset(self.offsets.vmctx_tables_begin()); + let mut ptr = self.vmctx_plus_offset(offsets.vmctx_tables_begin()); for i in 0..module.table_plans.len() - module.num_imported_tables { ptr::write(ptr, self.tables[DefinedTableIndex::new(i)].vmtable()); ptr = ptr.add(1); @@ -957,8 +964,8 @@ impl Instance { // time. Entries in `defined_memories` hold a pointer to a definition // (all memories) whereas the `owned_memories` hold the actual // definitions of memories owned (not shared) in the module. - let mut ptr = self.vmctx_plus_offset(self.offsets.vmctx_memories_begin()); - let mut owned_ptr = self.vmctx_plus_offset(self.offsets.vmctx_owned_memories_begin()); + let mut ptr = self.vmctx_plus_offset(offsets.vmctx_memories_begin()); + let mut owned_ptr = self.vmctx_plus_offset(offsets.vmctx_owned_memories_begin()); for i in 0..module.memory_plans.len() - module.num_imported_memories { let defined_memory_index = DefinedMemoryIndex::new(i); let memory_index = module.memory_index(defined_memory_index); diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 9ea6d95011..e8f8e75f0d 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -92,18 +92,11 @@ impl StorePtr { /// This trait is unsafe as it requires knowledge of Wasmtime's runtime internals to implement correctly. pub unsafe trait InstanceAllocator: Send + Sync { /// Validates that a module is supported by the allocator. - fn validate(&self, module: &Module) -> Result<()> { - drop(module); + fn validate(&self, module: &Module, offsets: &VMOffsets) -> Result<()> { + drop((module, offsets)); Ok(()) } - /// Adjusts the tunables prior to creation of any JIT compiler. - /// - /// This method allows the instance allocator control over tunables passed to a `wasmtime_jit::Compiler`. - fn adjust_tunables(&self, tunables: &mut wasmtime_environ::Tunables) { - drop(tunables); - } - /// Allocates an instance for the given allocation request. /// /// # Safety @@ -464,11 +457,9 @@ pub unsafe fn allocate_single_memory_instance( let mut memories = PrimaryMap::default(); memories.push(memory); let tables = PrimaryMap::default(); - let module = req.runtime_info.module(); - let offsets = VMOffsets::new(HostPtr, module); - let layout = Instance::alloc_layout(&offsets); + let layout = Instance::alloc_layout(req.runtime_info.offsets()); let instance = alloc::alloc(layout) as *mut Instance; - Instance::new_at(instance, layout.size(), offsets, req, memories, tables); + Instance::new_at(instance, layout.size(), req, memories, tables); Ok(InstanceHandle { instance }) } @@ -476,7 +467,7 @@ pub unsafe fn allocate_single_memory_instance( /// /// See [`InstanceAllocator::deallocate()`] for more details. pub unsafe fn deallocate(handle: &InstanceHandle) { - let layout = Instance::alloc_layout(&handle.instance().offsets); + let layout = Instance::alloc_layout(handle.instance().offsets()); ptr::drop_in_place(handle.instance); alloc::dealloc(handle.instance.cast(), layout); } @@ -485,12 +476,10 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { unsafe fn allocate(&self, mut req: InstanceAllocationRequest) -> Result { let memories = self.create_memories(&mut req.store, &req.runtime_info)?; let tables = Self::create_tables(&mut req.store, &req.runtime_info)?; - let module = req.runtime_info.module(); - let offsets = VMOffsets::new(HostPtr, module); - let layout = Instance::alloc_layout(&offsets); + let layout = Instance::alloc_layout(req.runtime_info.offsets()); let instance_ptr = alloc::alloc(layout) as *mut Instance; - Instance::new_at(instance_ptr, layout.size(), offsets, req, memories, tables); + Instance::new_at(instance_ptr, layout.size(), req, memories, tables); Ok(InstanceHandle { instance: instance_ptr, diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index b40ab8b2ab..efd5b57fc5 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -169,7 +169,7 @@ impl InstancePool { // If this fails then it's a configuration error at the `Engine` level // from when this pooling allocator was created and that needs updating // if this is to succeed. - let offsets = self.validate_instance_size(module)?; + self.validate_instance_size(req.runtime_info.offsets())?; let mut memories = PrimaryMap::with_capacity(module.memory_plans.len() - module.num_imported_memories); @@ -192,14 +192,7 @@ impl InstancePool { let instance_ptr = self.instance(instance_index) as _; - Instance::new_at( - instance_ptr, - self.instance_size, - offsets, - req, - memories, - tables, - ); + Instance::new_at(instance_ptr, self.instance_size, req, memories, tables); Ok(InstanceHandle { instance: instance_ptr, @@ -485,11 +478,10 @@ impl InstancePool { Ok(()) } - fn validate_instance_size(&self, module: &Module) -> Result> { - let offsets = VMOffsets::new(HostPtr, module); - let layout = Instance::alloc_layout(&offsets); + fn validate_instance_size(&self, offsets: &VMOffsets) -> Result<()> { + let layout = Instance::alloc_layout(offsets); if layout.size() <= self.instance_size { - return Ok(offsets); + return Ok(()); } // If this `module` exceeds the allocation size allotted to it then an @@ -1078,17 +1070,10 @@ impl PoolingInstanceAllocator { } unsafe impl InstanceAllocator for PoolingInstanceAllocator { - fn validate(&self, module: &Module) -> Result<()> { + fn validate(&self, module: &Module, offsets: &VMOffsets) -> Result<()> { self.instances.validate_memory_plans(module)?; self.instances.validate_table_plans(module)?; - - // Note that this check is not 100% accurate for cross-compiled systems - // where the pointer size may change since this check is often performed - // at compile time instead of runtime. Given that Wasmtime is almost - // always on a 64-bit platform though this is generally ok, and - // otherwise this check also happens during instantiation to - // double-check at that point. - self.instances.validate_instance_size(module)?; + self.instances.validate_instance_size(offsets)?; Ok(()) } @@ -1145,26 +1130,22 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator { #[cfg(test)] mod test { use super::*; - use crate::{CompiledModuleId, Imports, MemoryImage, StorePtr, VMSharedSignatureIndex}; + use crate::{ + CompiledModuleId, Imports, MemoryImage, StorePtr, VMFunctionBody, VMSharedSignatureIndex, + }; use std::sync::Arc; - use wasmtime_environ::{DefinedFuncIndex, DefinedMemoryIndex, FunctionLoc, SignatureIndex}; + use wasmtime_environ::{DefinedFuncIndex, DefinedMemoryIndex}; pub(crate) fn empty_runtime_info( module: Arc, ) -> Arc { - struct RuntimeInfo(Arc); + struct RuntimeInfo(Arc, VMOffsets); impl ModuleRuntimeInfo for RuntimeInfo { fn module(&self) -> &Arc { &self.0 } - fn image_base(&self) -> usize { - 0 - } - fn function_loc(&self, _: DefinedFuncIndex) -> &FunctionLoc { - unimplemented!() - } - fn signature(&self, _: SignatureIndex) -> VMSharedSignatureIndex { + fn function(&self, _: DefinedFuncIndex) -> *mut VMFunctionBody { unimplemented!() } fn memory_image( @@ -1183,9 +1164,13 @@ mod test { fn signature_ids(&self) -> &[VMSharedSignatureIndex] { &[] } + fn offsets(&self) -> &VMOffsets { + &self.1 + } } - Arc::new(RuntimeInfo(module)) + let offsets = VMOffsets::new(HostPtr, &module); + Arc::new(RuntimeInfo(module, offsets)) } #[cfg(target_pointer_width = "64")] diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 373081b1a5..d74363d132 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -23,10 +23,7 @@ use anyhow::Error; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; -use wasmtime_environ::DefinedFuncIndex; -use wasmtime_environ::DefinedMemoryIndex; -use wasmtime_environ::FunctionLoc; -use wasmtime_environ::SignatureIndex; +use wasmtime_environ::{DefinedFuncIndex, DefinedMemoryIndex, HostPtr, VMOffsets}; #[macro_use] mod trampolines; @@ -172,15 +169,8 @@ pub trait ModuleRuntimeInfo: Send + Sync + 'static { /// The underlying Module. fn module(&self) -> &Arc; - /// The signatures. - fn signature(&self, index: SignatureIndex) -> VMSharedSignatureIndex; - - /// The base address of where JIT functions are located. - fn image_base(&self) -> usize; - - /// Descriptors about each compiled function, such as the offset from - /// `image_base`. - fn function_loc(&self, func_index: DefinedFuncIndex) -> &FunctionLoc; + /// Returns the address, in memory, that the function `index` resides at. + fn function(&self, index: DefinedFuncIndex) -> *mut VMFunctionBody; /// Returns the `MemoryImage` structure used for copy-on-write /// initialization of the memory, if it's applicable. @@ -198,6 +188,9 @@ pub trait ModuleRuntimeInfo: Send + Sync + 'static { /// Returns an array, indexed by `SignatureIndex` of all /// `VMSharedSignatureIndex` entries corresponding to the `SignatureIndex`. fn signature_ids(&self) -> &[VMSharedSignatureIndex]; + + /// Offset information for the current host. + fn offsets(&self) -> &VMOffsets; } /// Returns the host OS page size, in bytes. diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index 9c4a67f252..65fb029f32 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -86,9 +86,9 @@ impl Engine { #[cfg(compiler)] let compiler = config.build_compiler()?; + drop(&mut config); // silence warnings without `cfg(compiler)` let allocator = config.build_allocator()?; - allocator.adjust_tunables(&mut config.tunables); let profiler = config.build_profiler()?; Ok(Engine { diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 97efc56a0e..d528321956 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -14,12 +14,13 @@ use std::path::Path; use std::sync::Arc; use wasmparser::{Parser, ValidPayload, Validator}; use wasmtime_environ::{ - DefinedFuncIndex, DefinedMemoryIndex, FunctionLoc, ModuleEnvironment, ModuleTranslation, - ModuleTypes, ObjectKind, PrimaryMap, SignatureIndex, WasmFunctionInfo, + DefinedFuncIndex, DefinedMemoryIndex, HostPtr, ModuleEnvironment, ModuleTranslation, + ModuleTypes, ObjectKind, PrimaryMap, VMOffsets, WasmFunctionInfo, }; use wasmtime_jit::{CodeMemory, CompiledModule, CompiledModuleInfo}; use wasmtime_runtime::{ - CompiledModuleId, MemoryImage, MmapVec, ModuleMemoryImages, VMSharedSignatureIndex, + CompiledModuleId, MemoryImage, MmapVec, ModuleMemoryImages, VMFunctionBody, + VMSharedSignatureIndex, }; mod registry; @@ -123,6 +124,9 @@ struct ModuleInner { /// Flag indicating whether this module can be serialized or not. serializable: bool, + + /// Runtime offset information for `VMContext`. + offsets: VMOffsets, } impl Module { @@ -664,7 +668,8 @@ impl Module { )?; // Validate the module can be used with the current allocator - engine.allocator().validate(module.module())?; + let offsets = VMOffsets::new(HostPtr, module.module()); + engine.allocator().validate(module.module(), &offsets)?; Ok(Self { inner: Arc::new(ModuleInner { @@ -673,6 +678,7 @@ impl Module { memory_images: OnceCell::new(), module, serializable, + offsets, }), }) } @@ -1098,16 +1104,12 @@ impl wasmtime_runtime::ModuleRuntimeInfo for ModuleInner { self.module.module() } - fn signature(&self, index: SignatureIndex) -> VMSharedSignatureIndex { - self.code.signatures().as_module_map()[index] - } - - fn image_base(&self) -> usize { - self.module.text().as_ptr() as usize - } - - fn function_loc(&self, index: DefinedFuncIndex) -> &FunctionLoc { - self.module.func_loc(index) + fn function(&self, index: DefinedFuncIndex) -> *mut VMFunctionBody { + self.module + .finished_function(index) + .as_ptr() + .cast::() + .cast_mut() } fn memory_image(&self, memory: DefinedMemoryIndex) -> Result>> { @@ -1126,6 +1128,10 @@ impl wasmtime_runtime::ModuleRuntimeInfo for ModuleInner { fn signature_ids(&self) -> &[VMSharedSignatureIndex] { self.code.signatures().as_module_map().values().as_slice() } + + fn offsets(&self) -> &VMOffsets { + &self.offsets + } } impl wasmtime_runtime::ModuleInfo for ModuleInner { @@ -1160,26 +1166,22 @@ impl wasmtime_runtime::ModuleInfo for ModuleInner { /// default-callee instance). pub(crate) struct BareModuleInfo { module: Arc, - image_base: usize, - one_signature: Option<(SignatureIndex, VMSharedSignatureIndex)>, + one_signature: Option, + offsets: VMOffsets, } impl BareModuleInfo { pub(crate) fn empty(module: Arc) -> Self { - BareModuleInfo { - module, - image_base: 0, - one_signature: None, - } + BareModuleInfo::maybe_imported_func(module, None) } pub(crate) fn maybe_imported_func( module: Arc, - one_signature: Option<(SignatureIndex, VMSharedSignatureIndex)>, + one_signature: Option, ) -> Self { BareModuleInfo { + offsets: VMOffsets::new(HostPtr, &module), module, - image_base: 0, one_signature, } } @@ -1194,19 +1196,7 @@ impl wasmtime_runtime::ModuleRuntimeInfo for BareModuleInfo { &self.module } - fn signature(&self, index: SignatureIndex) -> VMSharedSignatureIndex { - let (signature_id, signature) = self - .one_signature - .expect("Signature for one function should be present if queried"); - assert_eq!(index, signature_id); - signature - } - - fn image_base(&self) -> usize { - self.image_base - } - - fn function_loc(&self, _index: DefinedFuncIndex) -> &FunctionLoc { + fn function(&self, _index: DefinedFuncIndex) -> *mut VMFunctionBody { unreachable!() } @@ -1224,10 +1214,14 @@ impl wasmtime_runtime::ModuleRuntimeInfo for BareModuleInfo { fn signature_ids(&self) -> &[VMSharedSignatureIndex] { match &self.one_signature { - Some((_, id)) => std::slice::from_ref(id), + Some(id) => std::slice::from_ref(id), None => &[], } } + + fn offsets(&self) -> &VMOffsets { + &self.offsets + } } /// Helper method to construct a `ModuleMemoryImages` for an associated diff --git a/crates/wasmtime/src/module/registry.rs b/crates/wasmtime/src/module/registry.rs index 21bae8c5a8..7784d4cb1a 100644 --- a/crates/wasmtime/src/module/registry.rs +++ b/crates/wasmtime/src/module/registry.rs @@ -161,7 +161,7 @@ impl LoadedCode { // functions. None => return, }; - let start = unsafe { (*func).as_ptr() as usize }; + let start = func.as_ptr() as usize; match self.modules.entry(start) { // This module is already present, and it should be the same as @@ -279,9 +279,9 @@ fn test_frame_info() -> Result<(), anyhow::Error> { Instance::new(&mut store, &module, &[])?; for (i, alloc) in module.compiled_module().finished_functions() { - let (start, end) = unsafe { - let ptr = (*alloc).as_ptr(); - let len = (*alloc).len(); + let (start, end) = { + let ptr = alloc.as_ptr(); + let len = alloc.len(); (ptr as usize, ptr as usize + len) }; for pc in start..end { diff --git a/crates/wasmtime/src/trampoline.rs b/crates/wasmtime/src/trampoline.rs index 46c9fa202e..59e62cb976 100644 --- a/crates/wasmtime/src/trampoline.rs +++ b/crates/wasmtime/src/trampoline.rs @@ -17,7 +17,7 @@ use crate::{GlobalType, MemoryType, TableType, Val}; use anyhow::Result; use std::any::Any; use std::sync::Arc; -use wasmtime_environ::{GlobalIndex, MemoryIndex, Module, SignatureIndex, TableIndex}; +use wasmtime_environ::{GlobalIndex, MemoryIndex, Module, TableIndex}; use wasmtime_runtime::{ Imports, InstanceAllocationRequest, InstanceAllocator, OnDemandInstanceAllocator, SharedMemory, StorePtr, VMFunctionImport, VMSharedSignatureIndex, @@ -28,7 +28,7 @@ fn create_handle( store: &mut StoreOpaque, host_state: Box, func_imports: &[VMFunctionImport], - one_signature: Option<(SignatureIndex, VMSharedSignatureIndex)>, + one_signature: Option, ) -> Result { let mut imports = Imports::default(); imports.functions = func_imports; diff --git a/crates/wasmtime/src/trampoline/global.rs b/crates/wasmtime/src/trampoline/global.rs index ccc00c142b..e84601bd81 100644 --- a/crates/wasmtime/src/trampoline/global.rs +++ b/crates/wasmtime/src/trampoline/global.rs @@ -39,8 +39,8 @@ pub fn create_global(store: &mut StoreOpaque, gt: &GlobalType, val: Val) -> Resu // our global with a `ref.func` to grab that imported function. let f = f.caller_checked_anyfunc(store); let f = unsafe { f.as_ref() }; - let sig_id = SignatureIndex::from_u32(u32::max_value() - 1); - one_signature = Some((sig_id, f.type_index)); + let sig_id = SignatureIndex::from_u32(0); + one_signature = Some(f.type_index); module.types.push(ModuleType::Function(sig_id)); let func_index = module.push_escaped_function(sig_id, AnyfuncIndex::from_u32(0)); module.num_imported_funcs = 1; diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs index 0354a9c743..dcce9bfb3f 100644 --- a/tests/all/pooling_allocator.rs +++ b/tests/all/pooling_allocator.rs @@ -646,10 +646,11 @@ fn instance_too_large() -> Result<()> { let engine = Engine::new(&config)?; let expected = "\ -instance allocation for this module requires 336 bytes which exceeds the \ +instance allocation for this module requires 224 bytes which exceeds the \ configured maximum of 16 bytes; breakdown of allocation requirement: - * 76.19% - 256 bytes - instance state management + * 64.29% - 144 bytes - instance state management + * 7.14% - 16 bytes - jit store state "; match Module::new(&engine, "(module)") { Ok(_) => panic!("should have failed to compile"), @@ -663,11 +664,11 @@ configured maximum of 16 bytes; breakdown of allocation requirement: lots_of_globals.push_str(")"); let expected = "\ -instance allocation for this module requires 1936 bytes which exceeds the \ +instance allocation for this module requires 1824 bytes which exceeds the \ configured maximum of 16 bytes; breakdown of allocation requirement: - * 13.22% - 256 bytes - instance state management - * 82.64% - 1600 bytes - defined globals + * 7.89% - 144 bytes - instance state management + * 87.72% - 1600 bytes - defined globals "; match Module::new(&engine, &lots_of_globals) { Ok(_) => panic!("should have failed to compile"),