diff --git a/crates/api/src/callable.rs b/crates/api/src/callable.rs index c4983df8c8..39791c810f 100644 --- a/crates/api/src/callable.rs +++ b/crates/api/src/callable.rs @@ -3,10 +3,10 @@ use crate::trampoline::generate_func_export; use crate::trap::Trap; use crate::types::FuncType; use crate::values::Val; +use std::cmp::max; use std::ptr; use std::rc::Rc; -use wasmtime_environ::ir; -use wasmtime_runtime::{Export, InstanceHandle}; +use wasmtime_runtime::{ExportFunction, InstanceHandle, VMTrampoline}; /// A trait representing a function that can be imported and called from inside /// WebAssembly. @@ -87,45 +87,42 @@ pub trait Callable { pub(crate) trait WrappedCallable { fn call(&self, params: &[Val], results: &mut [Val]) -> Result<(), Trap>; - fn signature(&self) -> &ir::Signature { - match self.wasmtime_export() { - Export::Function { signature, .. } => signature, - _ => panic!("unexpected export type in Callable"), - } - } fn wasmtime_handle(&self) -> &InstanceHandle; - fn wasmtime_export(&self) -> &Export; + fn wasmtime_function(&self) -> &ExportFunction; } pub(crate) struct WasmtimeFn { store: Store, instance: InstanceHandle, - export: Export, + export: ExportFunction, + trampoline: VMTrampoline, } impl WasmtimeFn { - pub fn new(store: &Store, instance: InstanceHandle, export: Export) -> WasmtimeFn { + pub fn new( + store: &Store, + instance: InstanceHandle, + export: ExportFunction, + trampoline: VMTrampoline, + ) -> WasmtimeFn { WasmtimeFn { store: store.clone(), instance, export, + trampoline, } } } impl WrappedCallable for WasmtimeFn { fn call(&self, params: &[Val], results: &mut [Val]) -> Result<(), Trap> { - use std::cmp::max; - use std::mem; - - let (vmctx, body, signature) = match self.wasmtime_export() { - Export::Function { - vmctx, - address, - signature, - } => (*vmctx, *address, signature.clone()), - _ => panic!("unexpected export type in Callable"), - }; + let f = self.wasmtime_function(); + let signature = self + .store + .compiler() + .signatures() + .lookup(f.signature) + .expect("missing signature"); if signature.params.len() - 2 != params.len() { return Err(Trap::new(format!( "expected {} arguments, got {}", @@ -141,7 +138,6 @@ impl WrappedCallable for WasmtimeFn { ))); } - let value_size = mem::size_of::(); let mut values_vec = vec![0; max(params.len(), results.len())]; // Store the argument values into `values_vec`. @@ -155,20 +151,13 @@ impl WrappedCallable for WasmtimeFn { } } - // Get the trampoline to call for this function. - let exec_code_buf = self - .store - .compiler_mut() - .get_published_trampoline(&signature, value_size) - .map_err(|e| Trap::new(format!("trampoline error: {:?}", e)))?; - // Call the trampoline. if let Err(error) = unsafe { wasmtime_runtime::wasmtime_call_trampoline( - vmctx, + f.vmctx, ptr::null_mut(), - exec_code_buf, - body, + self.trampoline, + f.address, values_vec.as_mut_ptr() as *mut u8, ) } { @@ -189,7 +178,7 @@ impl WrappedCallable for WasmtimeFn { fn wasmtime_handle(&self) -> &InstanceHandle { &self.instance } - fn wasmtime_export(&self) -> &Export { + fn wasmtime_function(&self) -> &ExportFunction { &self.export } } @@ -197,7 +186,7 @@ impl WrappedCallable for WasmtimeFn { pub struct NativeCallable { callable: Rc, instance: InstanceHandle, - export: Export, + export: ExportFunction, } impl NativeCallable { @@ -219,7 +208,7 @@ impl WrappedCallable for NativeCallable { fn wasmtime_handle(&self) -> &InstanceHandle { &self.instance } - fn wasmtime_export(&self) -> &Export { + fn wasmtime_function(&self) -> &ExportFunction { &self.export } } diff --git a/crates/api/src/externals.rs b/crates/api/src/externals.rs index 8e081cff46..fb5efa91c6 100644 --- a/crates/api/src/externals.rs +++ b/crates/api/src/externals.rs @@ -83,10 +83,10 @@ impl Extern { pub(crate) fn get_wasmtime_export(&self) -> wasmtime_runtime::Export { match self { - Extern::Func(f) => f.wasmtime_export().clone(), - Extern::Global(g) => g.wasmtime_export().clone(), - Extern::Memory(m) => m.wasmtime_export().clone(), - Extern::Table(t) => t.wasmtime_export().clone(), + Extern::Func(f) => f.wasmtime_function().clone().into(), + Extern::Global(g) => g.wasmtime_export.clone().into(), + Extern::Memory(m) => m.wasmtime_export.clone().into(), + Extern::Table(t) => t.wasmtime_export.clone().into(), } } @@ -96,17 +96,17 @@ impl Extern { export: wasmtime_runtime::Export, ) -> Extern { match export { - wasmtime_runtime::Export::Function { .. } => { - Extern::Func(Func::from_wasmtime_function(export, store, instance_handle)) + wasmtime_runtime::Export::Function(f) => { + Extern::Func(Func::from_wasmtime_function(f, store, instance_handle)) } - wasmtime_runtime::Export::Memory { .. } => { - Extern::Memory(Memory::from_wasmtime_memory(export, store, instance_handle)) + wasmtime_runtime::Export::Memory(m) => { + Extern::Memory(Memory::from_wasmtime_memory(m, store, instance_handle)) } - wasmtime_runtime::Export::Global { .. } => { - Extern::Global(Global::from_wasmtime_global(export, store, instance_handle)) + wasmtime_runtime::Export::Global(g) => { + Extern::Global(Global::from_wasmtime_global(g, store, instance_handle)) } - wasmtime_runtime::Export::Table { .. } => { - Extern::Table(Table::from_wasmtime_table(export, store, instance_handle)) + wasmtime_runtime::Export::Table(t) => { + Extern::Table(Table::from_wasmtime_table(t, store, instance_handle)) } } } @@ -165,7 +165,7 @@ impl From for Extern { pub struct Global { store: Store, ty: GlobalType, - wasmtime_export: wasmtime_runtime::Export, + wasmtime_export: wasmtime_runtime::ExportGlobal, wasmtime_handle: InstanceHandle, } @@ -202,17 +202,10 @@ impl Global { &self.ty } - fn wasmtime_global_definition(&self) -> *mut wasmtime_runtime::VMGlobalDefinition { - match self.wasmtime_export { - wasmtime_runtime::Export::Global { definition, .. } => definition, - _ => panic!("global definition not found"), - } - } - /// Returns the current [`Val`] of this global. pub fn get(&self) -> Val { - let definition = unsafe { &mut *self.wasmtime_global_definition() }; unsafe { + let definition = &mut *self.wasmtime_export.definition; match self.ty().content() { ValType::I32 => Val::from(*definition.as_i32()), ValType::I64 => Val::from(*definition.as_i64()), @@ -243,8 +236,8 @@ impl Global { if !val.comes_from_same_store(&self.store) { bail!("cross-`Store` values are not supported"); } - let definition = unsafe { &mut *self.wasmtime_global_definition() }; unsafe { + let definition = &mut *self.wasmtime_export.definition; match val { Val::I32(i) => *definition.as_i32_mut() = i, Val::I64(i) => *definition.as_i64_mut() = i, @@ -256,28 +249,19 @@ impl Global { Ok(()) } - pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::Export { - &self.wasmtime_export - } - pub(crate) fn from_wasmtime_global( - export: wasmtime_runtime::Export, + wasmtime_export: wasmtime_runtime::ExportGlobal, store: &Store, wasmtime_handle: InstanceHandle, ) -> Global { - let global = if let wasmtime_runtime::Export::Global { ref global, .. } = export { - global - } else { - panic!("wasmtime export is not global") - }; // The original export is coming from wasmtime_runtime itself we should // support all the types coming out of it, so assert such here. - let ty = GlobalType::from_wasmtime_global(&global) + let ty = GlobalType::from_wasmtime_global(&wasmtime_export.global) .expect("core wasm global type should be supported"); Global { store: store.clone(), ty: ty, - wasmtime_export: export, + wasmtime_export, wasmtime_handle, } } @@ -303,11 +287,11 @@ pub struct Table { store: Store, ty: TableType, wasmtime_handle: InstanceHandle, - wasmtime_export: wasmtime_runtime::Export, + wasmtime_export: wasmtime_runtime::ExportTable, } fn set_table_item( - handle: &mut InstanceHandle, + handle: &InstanceHandle, table_index: wasm::DefinedTableIndex, item_index: u32, item: wasmtime_runtime::VMCallerCheckedAnyfunc, @@ -331,18 +315,13 @@ impl Table { /// Returns an error if `init` does not match the element type of the table. pub fn new(store: &Store, ty: TableType, init: Val) -> Result
{ let item = into_checked_anyfunc(init, store)?; - let (mut wasmtime_handle, wasmtime_export) = generate_table_export(store, &ty)?; + let (wasmtime_handle, wasmtime_export) = generate_table_export(store, &ty)?; // Initialize entries with the init value. - match wasmtime_export { - wasmtime_runtime::Export::Table { definition, .. } => { - let index = wasmtime_handle.table_index(unsafe { &*definition }); - let len = unsafe { (*definition).current_elements }; - for i in 0..len { - set_table_item(&mut wasmtime_handle, index, i, item.clone())?; - } - } - _ => unreachable!("export should be a table"), + let definition = unsafe { &*wasmtime_export.definition }; + let index = wasmtime_handle.table_index(definition); + for i in 0..definition.current_elements { + set_table_item(&wasmtime_handle, index, i, item.clone())?; } Ok(Table { @@ -360,11 +339,9 @@ impl Table { } fn wasmtime_table_index(&self) -> wasm::DefinedTableIndex { - match self.wasmtime_export { - wasmtime_runtime::Export::Table { definition, .. } => { - self.wasmtime_handle.table_index(unsafe { &*definition }) - } - _ => panic!("global definition not found"), + unsafe { + self.wasmtime_handle + .table_index(&*self.wasmtime_export.definition) } } @@ -385,19 +362,13 @@ impl Table { /// the right type to be stored in this table. pub fn set(&self, index: u32, val: Val) -> Result<()> { let table_index = self.wasmtime_table_index(); - let mut wasmtime_handle = self.wasmtime_handle.clone(); let item = into_checked_anyfunc(val, &self.store)?; - set_table_item(&mut wasmtime_handle, table_index, index, item) + set_table_item(&self.wasmtime_handle, table_index, index, item) } /// Returns the current size of this table. pub fn size(&self) -> u32 { - match self.wasmtime_export { - wasmtime_runtime::Export::Table { definition, .. } => unsafe { - (*definition).current_elements - }, - _ => panic!("global definition not found"), - } + unsafe { (&*self.wasmtime_export.definition).current_elements } } /// Grows the size of this table by `delta` more elements, initialization @@ -462,26 +433,17 @@ impl Table { Ok(()) } - pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::Export { - &self.wasmtime_export - } - pub(crate) fn from_wasmtime_table( - export: wasmtime_runtime::Export, + wasmtime_export: wasmtime_runtime::ExportTable, store: &Store, - instance_handle: wasmtime_runtime::InstanceHandle, + wasmtime_handle: wasmtime_runtime::InstanceHandle, ) -> Table { - let table = if let wasmtime_runtime::Export::Table { ref table, .. } = export { - table - } else { - panic!("wasmtime export is not table") - }; - let ty = TableType::from_wasmtime_table(&table.table); + let ty = TableType::from_wasmtime_table(&wasmtime_export.table.table); Table { store: store.clone(), - ty: ty, - wasmtime_handle: instance_handle, - wasmtime_export: export, + ty, + wasmtime_handle, + wasmtime_export, } } } @@ -511,7 +473,7 @@ pub struct Memory { store: Store, ty: MemoryType, wasmtime_handle: InstanceHandle, - wasmtime_export: wasmtime_runtime::Export, + wasmtime_export: wasmtime_runtime::ExportMemory, } impl Memory { @@ -536,13 +498,6 @@ impl Memory { &self.ty } - fn wasmtime_memory_definition(&self) -> *mut wasmtime_runtime::VMMemoryDefinition { - match self.wasmtime_export { - wasmtime_runtime::Export::Memory { definition, .. } => definition, - _ => panic!("memory definition not found"), - } - } - /// Returns this memory as a slice view that can be read natively in Rust. /// /// # Safety @@ -584,7 +539,7 @@ impl Memory { /// and in general you probably want to result to unsafe accessors and the /// `data` methods below. pub unsafe fn data_unchecked_mut(&self) -> &mut [u8] { - let definition = &*self.wasmtime_memory_definition(); + let definition = &*self.wasmtime_export.definition; slice::from_raw_parts_mut(definition.base, definition.current_length) } @@ -595,14 +550,14 @@ impl Memory { /// of [`Memory::data_unchecked`] to make sure that you can safely /// read/write the memory. pub fn data_ptr(&self) -> *mut u8 { - unsafe { (*self.wasmtime_memory_definition()).base } + unsafe { (*self.wasmtime_export.definition).base } } /// Returns the byte length of this memory. /// /// The returned value will be a multiple of the wasm page size, 64k. pub fn data_size(&self) -> usize { - unsafe { (*self.wasmtime_memory_definition()).current_length } + unsafe { (*self.wasmtime_export.definition).current_length } } /// Returns the size, in pages, of this wasm memory. @@ -625,39 +580,26 @@ impl Memory { /// Returns an error if memory could not be grown, for example if it exceeds /// the maximum limits of this memory. pub fn grow(&self, delta: u32) -> Result { - match self.wasmtime_export { - wasmtime_runtime::Export::Memory { definition, .. } => { - let definition = unsafe { &(*definition) }; - let index = self.wasmtime_handle.memory_index(definition); - self.wasmtime_handle - .clone() - .memory_grow(index, delta) - .ok_or_else(|| anyhow!("failed to grow memory")) - } - _ => panic!("memory definition not found"), - } - } - - pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::Export { - &self.wasmtime_export + let index = self + .wasmtime_handle + .memory_index(unsafe { &*self.wasmtime_export.definition }); + self.wasmtime_handle + .clone() + .memory_grow(index, delta) + .ok_or_else(|| anyhow!("failed to grow memory")) } pub(crate) fn from_wasmtime_memory( - export: wasmtime_runtime::Export, + wasmtime_export: wasmtime_runtime::ExportMemory, store: &Store, - instance_handle: wasmtime_runtime::InstanceHandle, + wasmtime_handle: wasmtime_runtime::InstanceHandle, ) -> Memory { - let memory = if let wasmtime_runtime::Export::Memory { ref memory, .. } = export { - memory - } else { - panic!("wasmtime export is not memory") - }; - let ty = MemoryType::from_wasmtime_memory(&memory.memory); + let ty = MemoryType::from_wasmtime_memory(&wasmtime_export.memory.memory); Memory { store: store.clone(), ty: ty, - wasmtime_handle: instance_handle, - wasmtime_export: export, + wasmtime_handle, + wasmtime_export, } } } diff --git a/crates/api/src/func.rs b/crates/api/src/func.rs index 69c0848a61..b8559a3d0c 100644 --- a/crates/api/src/func.rs +++ b/crates/api/src/func.rs @@ -184,23 +184,53 @@ macro_rules! wrappers { } } + #[allow(non_snake_case)] + unsafe extern "C" fn trampoline( + callee_vmctx: *mut VMContext, + caller_vmctx: *mut VMContext, + ptr: *const VMFunctionBody, + args: *mut u128, + ) + where + F: Fn($($args),*) -> R + 'static, + $($args: WasmTy,)* + R: WasmRet, + { + let ptr = mem::transmute::< + *const VMFunctionBody, + unsafe extern "C" fn( + *mut VMContext, + *mut VMContext, + $($args::Abi,)* + ) -> R::Abi, + >(ptr); + + let mut _next = args as *const u128; + $(let $args = $args::load(&mut _next);)* + + let ret = ptr(callee_vmctx, caller_vmctx, $($args),*); + R::store(ret, args); + } + let mut _args = Vec::new(); $($args::push(&mut _args);)* let mut ret = Vec::new(); R::push(&mut ret); let ty = FuncType::new(_args.into(), ret.into()); unsafe { + let trampoline = trampoline::; let (instance, export) = crate::trampoline::generate_raw_func_export( &ty, std::slice::from_raw_parts_mut( shim:: as *mut _, 0, ), + trampoline, store, Box::new(func), ) .expect("failed to generate export"); - let callable = Rc::new(WasmtimeFn::new(store, instance, export)); + let callable = Rc::new(WasmtimeFn::new(store, instance, export, trampoline)); Func::from_wrapped(store, ty, callable) } } @@ -214,8 +244,8 @@ macro_rules! getters { )*) => ($( $(#[$doc])* #[allow(non_snake_case)] - pub fn $name<$($args,)* R>(&self) - -> anyhow::Result Result> + pub fn $name<'a, $($args,)* R>(&'a self) + -> anyhow::Result Result + 'a> where $($args: WasmTy,)* R: WasmTy, @@ -239,28 +269,23 @@ macro_rules! getters { // ... and then once we've passed the typechecks we can hand out our // object since our `transmute` below should be safe! - let (address, vmctx) = match self.wasmtime_export() { - wasmtime_runtime::Export::Function { address, vmctx, signature: _} => { - (*address, *vmctx) - } - _ => panic!("expected function export"), - }; + let f = self.wasmtime_function(); Ok(move |$($args: $args),*| -> Result { unsafe { - let f = mem::transmute::< + let fnptr = mem::transmute::< *const VMFunctionBody, unsafe extern "C" fn( *mut VMContext, *mut VMContext, $($args::Abi,)* ) -> R::Abi, - >(address); + >(f.address); let mut ret = None; $(let $args = $args.into_abi();)* - wasmtime_runtime::catch_traps(vmctx, || { - ret = Some(f(vmctx, ptr::null_mut(), $($args,)*)); + wasmtime_runtime::catch_traps(f.vmctx, || { + ret = Some(fnptr(f.vmctx, ptr::null_mut(), $($args,)*)); }).map_err(Trap::from_jit)?; - Ok(R::from_abi(vmctx, ret.unwrap())) + Ok(R::from_abi(f.vmctx, ret.unwrap())) } }) } @@ -553,25 +578,37 @@ impl Func { Ok(results.into_boxed_slice()) } - pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::Export { - self.callable.wasmtime_export() + pub(crate) fn wasmtime_function(&self) -> &wasmtime_runtime::ExportFunction { + self.callable.wasmtime_function() } pub(crate) fn from_wasmtime_function( - export: wasmtime_runtime::Export, + export: wasmtime_runtime::ExportFunction, store: &Store, instance_handle: InstanceHandle, ) -> Self { + // Signatures should always be registered in the store's registry of + // shared signatures, so we should be able to unwrap safely here. + let sig = store + .compiler() + .signatures() + .lookup(export.signature) + .expect("failed to lookup signature"); + // This is only called with `Export::Function`, and since it's coming // from wasmtime_runtime itself we should support all the types coming // out of it, so assert such here. - let ty = if let wasmtime_runtime::Export::Function { signature, .. } = &export { - FuncType::from_wasmtime_signature(signature.clone()) - .expect("core wasm signature should be supported") - } else { - panic!("expected function export") - }; - let callable = WasmtimeFn::new(store, instance_handle, export); + let ty = FuncType::from_wasmtime_signature(sig) + .expect("core wasm signature should be supported"); + + // Each function signature in a module should have a trampoline stored + // on that module as well, so unwrap the result here since otherwise + // it's a bug in wasmtime. + let trampoline = instance_handle + .trampoline(export.signature) + .expect("failed to retrieve trampoline from module"); + + let callable = WasmtimeFn::new(store, instance_handle, export, trampoline); Func::from_wrapped(store, ty, Rc::new(callable)) } @@ -727,6 +764,10 @@ pub trait WasmTy { fn from_abi(vmctx: *mut VMContext, abi: Self::Abi) -> Self; #[doc(hidden)] fn into_abi(self) -> Self::Abi; + #[doc(hidden)] + unsafe fn load(ptr: &mut *const u128) -> Self::Abi; + #[doc(hidden)] + unsafe fn store(abi: Self::Abi, ptr: *mut u128); } impl WasmTy for () { @@ -743,6 +784,10 @@ impl WasmTy for () { fn into_abi(self) -> Self::Abi { self } + #[inline] + unsafe fn load(_ptr: &mut *const u128) -> Self::Abi {} + #[inline] + unsafe fn store(_abi: Self::Abi, _ptr: *mut u128) {} } impl WasmTy for i32 { @@ -767,6 +812,16 @@ impl WasmTy for i32 { fn into_abi(self) -> Self::Abi { self } + #[inline] + unsafe fn load(ptr: &mut *const u128) -> Self::Abi { + let ret = **ptr as Self; + *ptr = (*ptr).add(1); + return ret; + } + #[inline] + unsafe fn store(abi: Self::Abi, ptr: *mut u128) { + *ptr = abi as u128; + } } impl WasmTy for i64 { @@ -791,6 +846,16 @@ impl WasmTy for i64 { fn into_abi(self) -> Self::Abi { self } + #[inline] + unsafe fn load(ptr: &mut *const u128) -> Self::Abi { + let ret = **ptr as Self; + *ptr = (*ptr).add(1); + return ret; + } + #[inline] + unsafe fn store(abi: Self::Abi, ptr: *mut u128) { + *ptr = abi as u128; + } } impl WasmTy for f32 { @@ -815,6 +880,16 @@ impl WasmTy for f32 { fn into_abi(self) -> Self::Abi { self } + #[inline] + unsafe fn load(ptr: &mut *const u128) -> Self::Abi { + let ret = f32::from_bits(**ptr as u32); + *ptr = (*ptr).add(1); + return ret; + } + #[inline] + unsafe fn store(abi: Self::Abi, ptr: *mut u128) { + *ptr = abi.to_bits() as u128; + } } impl WasmTy for f64 { @@ -839,6 +914,16 @@ impl WasmTy for f64 { fn into_abi(self) -> Self::Abi { self } + #[inline] + unsafe fn load(ptr: &mut *const u128) -> Self::Abi { + let ret = f64::from_bits(**ptr as u64); + *ptr = (*ptr).add(1); + return ret; + } + #[inline] + unsafe fn store(abi: Self::Abi, ptr: *mut u128) { + *ptr = abi.to_bits() as u128; + } } /// A trait implemented for types which can be returned from closures passed to @@ -858,6 +943,8 @@ pub trait WasmRet { fn matches(tys: impl Iterator) -> anyhow::Result<()>; #[doc(hidden)] fn into_abi(self) -> Self::Abi; + #[doc(hidden)] + unsafe fn store(abi: Self::Abi, ptr: *mut u128); } impl WasmRet for T { @@ -874,6 +961,11 @@ impl WasmRet for T { fn into_abi(self) -> Self::Abi { T::into_abi(self) } + + #[inline] + unsafe fn store(abi: Self::Abi, ptr: *mut u128) { + T::store(abi, ptr); + } } impl WasmRet for Result { @@ -889,7 +981,7 @@ impl WasmRet for Result { #[inline] fn into_abi(self) -> Self::Abi { match self { - Ok(val) => return val.into_abi(), + Ok(val) => return T::into_abi(val), Err(trap) => handle_trap(trap), } @@ -897,4 +989,9 @@ impl WasmRet for Result { unsafe { wasmtime_runtime::raise_user_trap(Box::new(trap)) } } } + + #[inline] + unsafe fn store(abi: Self::Abi, ptr: *mut u128) { + T::store(abi, ptr); + } } diff --git a/crates/api/src/instance.rs b/crates/api/src/instance.rs index 46aff4e4d2..a1ff8a2087 100644 --- a/crates/api/src/instance.rs +++ b/crates/api/src/instance.rs @@ -4,7 +4,7 @@ use crate::runtime::{Config, Store}; use crate::trap::Trap; use anyhow::{bail, Error, Result}; use wasmtime_jit::{CompiledModule, Resolver}; -use wasmtime_runtime::{Export, InstanceHandle, InstantiationError}; +use wasmtime_runtime::{Export, InstanceHandle, InstantiationError, SignatureRegistry}; struct SimpleResolver<'a> { imports: &'a [Extern], @@ -22,6 +22,7 @@ fn instantiate( config: &Config, compiled_module: &CompiledModule, imports: &[Extern], + sig_registry: &SignatureRegistry, ) -> Result { let mut resolver = SimpleResolver { imports }; unsafe { @@ -29,6 +30,7 @@ fn instantiate( .instantiate( config.validating_config.operator_config.enable_bulk_memory, &mut resolver, + sig_registry, ) .map_err(|e| -> Error { match e { @@ -122,26 +124,28 @@ impl Instance { } let config = store.engine().config(); - let instance_handle = instantiate(config, module.compiled_module(), imports)?; + let instance_handle = instantiate( + config, + module.compiled_module(), + imports, + store.compiler().signatures(), + )?; - let exports = { - let mut exports = Vec::with_capacity(module.exports().len()); - for export in module.exports() { - let name = export.name().to_string(); - let export = instance_handle.lookup(&name).expect("export"); - exports.push(Extern::from_wasmtime_export( - store, - instance_handle.clone(), - export, - )); - } - exports.into_boxed_slice() - }; + let mut exports = Vec::with_capacity(module.exports().len()); + for export in module.exports() { + let name = export.name().to_string(); + let export = instance_handle.lookup(&name).expect("export"); + exports.push(Extern::from_wasmtime_export( + store, + instance_handle.clone(), + export, + )); + } module.register_frame_info(); Ok(Instance { instance_handle, module: module.clone(), - exports, + exports: exports.into_boxed_slice(), }) } diff --git a/crates/api/src/trampoline/create_handle.rs b/crates/api/src/trampoline/create_handle.rs index d3d295de9d..d83ef7a3e1 100644 --- a/crates/api/src/trampoline/create_handle.rs +++ b/crates/api/src/trampoline/create_handle.rs @@ -3,17 +3,20 @@ use crate::runtime::Store; use anyhow::Result; use std::any::Any; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::Module; -use wasmtime_runtime::{Imports, InstanceHandle, VMFunctionBody}; +use wasmtime_runtime::{ + Imports, InstanceHandle, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, +}; pub(crate) fn create_handle( module: Module, store: &Store, finished_functions: PrimaryMap, + trampolines: HashMap, state: Box, ) -> Result { let imports = Imports::new( @@ -38,6 +41,7 @@ pub(crate) fn create_handle( Arc::new(module), store.compiler().trap_registry().register_traps(Vec::new()), finished_functions.into_boxed_slice(), + trampolines, imports, &data_initializers, signatures.into_boxed_slice(), diff --git a/crates/api/src/trampoline/func.rs b/crates/api/src/trampoline/func.rs index 83753be403..76d29d2e8d 100644 --- a/crates/api/src/trampoline/func.rs +++ b/crates/api/src/trampoline/func.rs @@ -5,12 +5,14 @@ use crate::{Callable, FuncType, Store, Trap, Val}; use anyhow::{bail, Result}; use std::any::Any; use std::cmp; +use std::collections::HashMap; +use std::mem; use std::panic::{self, AssertUnwindSafe}; use std::rc::Rc; use wasmtime_environ::entity::{EntityRef, PrimaryMap}; use wasmtime_environ::ir::types; use wasmtime_environ::isa::TargetIsa; -use wasmtime_environ::wasm::{DefinedFuncIndex, FuncIndex}; +use wasmtime_environ::wasm::FuncIndex; use wasmtime_environ::{ ir, settings, CompiledFunction, CompiledFunctionUnwindInfo, Export, Module, }; @@ -21,7 +23,7 @@ use wasmtime_jit::trampoline::{ binemit, pretty_error, Context, FunctionBuilder, FunctionBuilderContext, }; use wasmtime_jit::{native, CodeMemory}; -use wasmtime_runtime::{InstanceHandle, VMContext, VMFunctionBody}; +use wasmtime_runtime::{InstanceHandle, VMContext, VMFunctionBody, VMTrampoline}; struct TrampolineState { func: Rc, @@ -145,7 +147,7 @@ fn make_trampoline( stub_sig.params.push(ir::AbiParam::new(pointer_type)); // Compute the size of the values vector. The vmctx and caller vmctx are passed separately. - let value_size = 16; + let value_size = mem::size_of::(); let values_vec_len = ((value_size as usize) * cmp::max(signature.params.len() - 2, signature.returns.len())) as u32; @@ -264,10 +266,12 @@ pub fn create_handle_with_function( let mut fn_builder_ctx = FunctionBuilderContext::new(); let mut module = Module::new(); - let mut finished_functions: PrimaryMap = - PrimaryMap::new(); + let mut finished_functions = PrimaryMap::new(); + let mut trampolines = HashMap::new(); let mut code_memory = CodeMemory::new(); + // First up we manufacture a trampoline which has the ABI specified by `ft` + // and calls into `stub_fn`... let sig_id = module.local.signatures.push(sig.clone()); let func_id = module.local.functions.push(sig_id); module @@ -280,16 +284,31 @@ pub fn create_handle_with_function( func_id.index() as u32, &sig, ); - code_memory.publish(); - finished_functions.push(trampoline); - let trampoline_state = TrampolineState::new(func.clone(), code_memory); + // ... and then we also need a trampoline with the standard "trampoline ABI" + // which enters into the ABI specified by `ft`. Note that this is only used + // if `Func::call` is called on an object created by `Func::new`. + let trampoline = wasmtime_jit::make_trampoline( + &*isa, + &mut code_memory, + &mut fn_builder_ctx, + &sig, + mem::size_of::(), + )?; + let sig_id = store.compiler().signatures().register(&sig); + trampolines.insert(sig_id, trampoline); + // Next up we wrap everything up into an `InstanceHandle` by publishing our + // code memory (makes it executable) and ensuring all our various bits of + // state make it into the instance constructors. + code_memory.publish(); + let trampoline_state = TrampolineState::new(func.clone(), code_memory); create_handle( module, store, finished_functions, + trampolines, Box::new(trampoline_state), ) } @@ -297,6 +316,7 @@ pub fn create_handle_with_function( pub unsafe fn create_handle_with_raw_function( ft: &FuncType, func: *mut [VMFunctionBody], + trampoline: VMTrampoline, store: &Store, state: Box, ) -> Result { @@ -314,6 +334,7 @@ pub unsafe fn create_handle_with_raw_function( let mut module = Module::new(); let mut finished_functions = PrimaryMap::new(); + let mut trampolines = HashMap::new(); let sig_id = module.local.signatures.push(sig.clone()); let func_id = module.local.functions.push(sig_id); @@ -321,6 +342,8 @@ pub unsafe fn create_handle_with_raw_function( .exports .insert("trampoline".to_string(), Export::Function(func_id)); finished_functions.push(func); + let sig_id = store.compiler().signatures().register(&sig); + trampolines.insert(sig_id, trampoline); - create_handle(module, store, finished_functions, state) + create_handle(module, store, finished_functions, trampolines, state) } diff --git a/crates/api/src/trampoline/global.rs b/crates/api/src/trampoline/global.rs index ad87386a3e..1499d18bee 100644 --- a/crates/api/src/trampoline/global.rs +++ b/crates/api/src/trampoline/global.rs @@ -30,6 +30,12 @@ pub fn create_global(store: &Store, gt: &GlobalType, val: Val) -> Result Result, store: &Store, -) -> Result<(wasmtime_runtime::InstanceHandle, wasmtime_runtime::Export)> { +) -> Result<( + wasmtime_runtime::InstanceHandle, + wasmtime_runtime::ExportFunction, +)> { let instance = create_handle_with_function(ft, func, store)?; - let export = instance.lookup("trampoline").expect("trampoline export"); - Ok((instance, export)) + match instance.lookup("trampoline").expect("trampoline export") { + wasmtime_runtime::Export::Function(f) => Ok((instance, f)), + _ => unreachable!(), + } } /// Note that this is `unsafe` since `func` must be a valid function pointer and @@ -32,38 +37,59 @@ pub fn generate_func_export( pub unsafe fn generate_raw_func_export( ft: &FuncType, func: *mut [VMFunctionBody], + trampoline: VMTrampoline, store: &Store, state: Box, -) -> Result<(wasmtime_runtime::InstanceHandle, wasmtime_runtime::Export)> { - let instance = func::create_handle_with_raw_function(ft, func, store, state)?; - let export = instance.lookup("trampoline").expect("trampoline export"); - Ok((instance, export)) +) -> Result<( + wasmtime_runtime::InstanceHandle, + wasmtime_runtime::ExportFunction, +)> { + let instance = func::create_handle_with_raw_function(ft, func, trampoline, store, state)?; + match instance.lookup("trampoline").expect("trampoline export") { + wasmtime_runtime::Export::Function(f) => Ok((instance, f)), + _ => unreachable!(), + } } pub fn generate_global_export( store: &Store, gt: &GlobalType, val: Val, -) -> Result<(wasmtime_runtime::InstanceHandle, wasmtime_runtime::Export)> { +) -> Result<( + wasmtime_runtime::InstanceHandle, + wasmtime_runtime::ExportGlobal, +)> { let instance = create_global(store, gt, val)?; - let export = instance.lookup("global").expect("global export"); - Ok((instance, export)) + match instance.lookup("global").expect("global export") { + wasmtime_runtime::Export::Global(g) => Ok((instance, g)), + _ => unreachable!(), + } } pub fn generate_memory_export( store: &Store, m: &MemoryType, -) -> Result<(wasmtime_runtime::InstanceHandle, wasmtime_runtime::Export)> { +) -> Result<( + wasmtime_runtime::InstanceHandle, + wasmtime_runtime::ExportMemory, +)> { let instance = create_handle_with_memory(store, m)?; - let export = instance.lookup("memory").expect("memory export"); - Ok((instance, export)) + match instance.lookup("memory").expect("memory export") { + wasmtime_runtime::Export::Memory(m) => Ok((instance, m)), + _ => unreachable!(), + } } pub fn generate_table_export( store: &Store, t: &TableType, -) -> Result<(wasmtime_runtime::InstanceHandle, wasmtime_runtime::Export)> { +) -> Result<( + wasmtime_runtime::InstanceHandle, + wasmtime_runtime::ExportTable, +)> { let instance = create_handle_with_table(store, t)?; - let export = instance.lookup("table").expect("table export"); - Ok((instance, export)) + match instance.lookup("table").expect("table export") { + wasmtime_runtime::Export::Table(t) => Ok((instance, t)), + _ => unreachable!(), + } } diff --git a/crates/api/src/trampoline/table.rs b/crates/api/src/trampoline/table.rs index 7f067b6393..3de2f771f4 100644 --- a/crates/api/src/trampoline/table.rs +++ b/crates/api/src/trampoline/table.rs @@ -26,5 +26,11 @@ pub fn create_handle_with_table(store: &Store, table: &TableType) -> Result { - let (vmctx, func_ptr, signature) = match f.wasmtime_export() { - wasmtime_runtime::Export::Function { - vmctx, - address, - signature, - } => (*vmctx, *address, signature), - _ => panic!("expected function export"), - }; - let type_index = store.compiler().signatures().register(signature); + let f = f.wasmtime_function(); wasmtime_runtime::VMCallerCheckedAnyfunc { - func_ptr, - type_index, - vmctx, + func_ptr: f.address, + type_index: f.signature, + vmctx: f.vmctx, } } _ => bail!("val is not funcref"), @@ -227,15 +219,10 @@ pub(crate) fn from_checked_anyfunc( if item.type_index == wasmtime_runtime::VMSharedSignatureIndex::default() { Val::AnyRef(AnyRef::Null); } - let signature = store - .compiler() - .signatures() - .lookup(item.type_index) - .expect("signature"); let instance_handle = unsafe { wasmtime_runtime::InstanceHandle::from_vmctx(item.vmctx) }; - let export = wasmtime_runtime::Export::Function { + let export = wasmtime_runtime::ExportFunction { address: item.func_ptr, - signature, + signature: item.type_index, vmctx: item.vmctx, }; let f = Func::from_wasmtime_function(export, store, instance_handle); diff --git a/crates/api/tests/func.rs b/crates/api/tests/func.rs index dcb1a1d8c9..60a5b6f596 100644 --- a/crates/api/tests/func.rs +++ b/crates/api/tests/func.rs @@ -289,3 +289,41 @@ fn get_from_module() -> anyhow::Result<()> { assert!(f2.get1::().is_err()); Ok(()) } + +#[test] +fn call_wrapped_func() -> Result<()> { + let store = Store::default(); + let f = Func::wrap4(&store, |a: i32, b: i64, c: f32, d: f64| { + assert_eq!(a, 1); + assert_eq!(b, 2); + assert_eq!(c, 3.0); + assert_eq!(d, 4.0); + }); + f.call(&[Val::I32(1), Val::I64(2), 3.0f32.into(), 4.0f64.into()])?; + f.get4::()?(1, 2, 3.0, 4.0)?; + + let f = Func::wrap0(&store, || 1i32); + let results = f.call(&[])?; + assert_eq!(results.len(), 1); + assert_eq!(results[0].unwrap_i32(), 1); + assert_eq!(f.get0::()?()?, 1); + + let f = Func::wrap0(&store, || 2i64); + let results = f.call(&[])?; + assert_eq!(results.len(), 1); + assert_eq!(results[0].unwrap_i64(), 2); + assert_eq!(f.get0::()?()?, 2); + + let f = Func::wrap0(&store, || 3.0f32); + let results = f.call(&[])?; + assert_eq!(results.len(), 1); + assert_eq!(results[0].unwrap_f32(), 3.0); + assert_eq!(f.get0::()?()?, 3.0); + + let f = Func::wrap0(&store, || 4.0f64); + let results = f.call(&[])?; + assert_eq!(results.len(), 1); + assert_eq!(results[0].unwrap_f64(), 4.0); + assert_eq!(f.get0::()?()?, 4.0); + Ok(()) +} diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index 4670f40dfc..968b1cb62e 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -71,10 +71,9 @@ impl CodeMemory { ) -> Result<&mut [VMFunctionBody], String> { let size = Self::function_allocation_size(func); - let start = self.position as u32; - let (buf, table) = self.allocate(size)?; + let (buf, table, start) = self.allocate(size)?; - let (_, _, _, vmfunc) = Self::copy_function(func, start, buf, table); + let (_, _, _, vmfunc) = Self::copy_function(func, start as u32, buf, table); Ok(vmfunc) } @@ -90,9 +89,9 @@ impl CodeMemory { .into_iter() .fold(0, |acc, func| acc + Self::function_allocation_size(func)); - let mut start = self.position as u32; - let (mut buf, mut table) = self.allocate(total_len)?; + let (mut buf, mut table, start) = self.allocate(total_len)?; let mut result = Vec::with_capacity(compilation.len()); + let mut start = start as u32; for func in compilation.into_iter() { let (next_start, next_buf, next_table, vmfunc) = @@ -134,8 +133,14 @@ impl CodeMemory { /// that it can be written to and patched, though we make it readonly before /// actually executing from it. /// + /// A few values are returned: + /// + /// * A mutable slice which references the allocated memory + /// * A function table instance where unwind information is registered + /// * The offset within the current mmap that the slice starts at + /// /// TODO: Add an alignment flag. - fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut FunctionTable), String> { + fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut FunctionTable, usize), String> { if self.current.mmap.len() - self.position < size { self.push_current(cmp::max(0x10000, size))?; } @@ -146,6 +151,7 @@ impl CodeMemory { Ok(( &mut self.current.mmap.as_mut_slice()[old_position..self.position], &mut self.current.table, + old_position, )) } diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index f33baa874e..4e1c617514 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -23,7 +23,7 @@ use wasmtime_environ::{ use wasmtime_profiling::ProfilingAgent; use wasmtime_runtime::{ InstantiationError, SignatureRegistry, TrapRegistration, TrapRegistry, VMFunctionBody, - VMSharedSignatureIndex, + VMSharedSignatureIndex, VMTrampoline, }; /// Select which kind of compilation to use. @@ -53,13 +53,9 @@ pub struct Compiler { code_memory: CodeMemory, trap_registry: TrapRegistry, - trampoline_park: HashMap, signatures: SignatureRegistry, strategy: CompilationStrategy, cache_config: CacheConfig, - - /// The `FunctionBuilderContext`, shared between trampline function compilations. - fn_builder_ctx: FunctionBuilderContext, } impl Compiler { @@ -72,9 +68,7 @@ impl Compiler { Self { isa, code_memory: CodeMemory::new(), - trampoline_park: HashMap::new(), signatures: SignatureRegistry::new(), - fn_builder_ctx: FunctionBuilderContext::new(), strategy, trap_registry: TrapRegistry::default(), cache_config, @@ -103,6 +97,7 @@ impl Compiler { ) -> Result< ( PrimaryMap, + HashMap, PrimaryMap, Relocations, Option>, @@ -145,6 +140,8 @@ impl Compiler { } .map_err(SetupError::Compile)?; + // Allocate all of the compiled functions into executable memory, + // copying over their contents. let allocated_functions = allocate_functions(&mut self.code_memory, &compilation).map_err(|message| { SetupError::Instantiate(InstantiationError::Resource(format!( @@ -153,8 +150,45 @@ impl Compiler { ))) })?; + // Create a registration value for all traps in our allocated + // functions. This registration will allow us to map a trapping PC + // value to what the trap actually means if it came from JIT code. let trap_registration = register_traps(&allocated_functions, &traps, &self.trap_registry); + // Eagerly generate a entry trampoline for every type signature in the + // module. This should be "relatively lightweight" for most modules and + // guarantees that all functions (including indirect ones through + // tables) have a trampoline when invoked through the wasmtime API. + let mut cx = FunctionBuilderContext::new(); + let mut trampolines = HashMap::new(); + for sig in module.local.signatures.values() { + let index = self.signatures.register(sig); + if trampolines.contains_key(&index) { + continue; + } + // FIXME(#1303) we should be generating a trampoline for all + // functions in a module, not just those with less than 40 + // arguments. Currently though cranelift dies in spec tests when one + // function has 100 arguments. This looks to be a cranelift bug, so + // let's work around it for now by skipping generating a trampoline + // for that massive function. The trampoline isn't actually needed + // at this time, and we'll hopefully get the cranelift bug fixed + // soon enough to remove this condition. + if sig.params.len() > 40 { + continue; + } + trampolines.insert( + index, + make_trampoline( + &*self.isa, + &mut self.code_memory, + &mut cx, + sig, + std::mem::size_of::(), + )?, + ); + } + // Translate debug info (DWARF) only if at least one function is present. let dbg = if debug_data.is_some() && !allocated_functions.is_empty() { let target_config = self.isa.frontend_config(); @@ -199,6 +233,7 @@ impl Compiler { Ok(( allocated_functions, + trampolines, jt_offsets, relocations, dbg, @@ -206,38 +241,6 @@ impl Compiler { )) } - /// Create a trampoline for invoking a function. - pub(crate) fn get_trampoline( - &mut self, - signature: &ir::Signature, - value_size: usize, - ) -> Result<*const VMFunctionBody, SetupError> { - let index = self.signatures.register(signature); - if let Some(trampoline) = self.trampoline_park.get(&index) { - return Ok(*trampoline); - } - let body = make_trampoline( - &*self.isa, - &mut self.code_memory, - &mut self.fn_builder_ctx, - signature, - value_size, - )?; - self.trampoline_park.insert(index, body); - return Ok(body); - } - - /// Create and publish a trampoline for invoking a function. - pub fn get_published_trampoline( - &mut self, - signature: &ir::Signature, - value_size: usize, - ) -> Result<*const VMFunctionBody, SetupError> { - let result = self.get_trampoline(signature, value_size)?; - self.publish_compiled_code(); - Ok(result) - } - /// Make memory containing compiled code executable. pub(crate) fn publish_compiled_code(&mut self) { self.code_memory.publish(); @@ -265,13 +268,13 @@ impl Compiler { } /// Create a trampoline for invoking a function. -fn make_trampoline( +pub fn make_trampoline( isa: &dyn TargetIsa, code_memory: &mut CodeMemory, fn_builder_ctx: &mut FunctionBuilderContext, signature: &ir::Signature, value_size: usize, -) -> Result<*const VMFunctionBody, SetupError> { +) -> Result { let pointer_type = isa.pointer_type(); let mut wrapper_sig = ir::Signature::new(isa.frontend_config().default_call_conv); @@ -373,14 +376,15 @@ fn make_trampoline( let unwind_info = CompiledFunctionUnwindInfo::new(isa, &context); - Ok(code_memory + let ptr = code_memory .allocate_for_function(&CompiledFunction { body: code_buf, jt_offsets: context.func.jt_offsets, unwind_info, }) .map_err(|message| SetupError::Instantiate(InstantiationError::Resource(message)))? - .as_ptr()) + .as_ptr(); + Ok(unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) }) } fn allocate_functions( diff --git a/crates/jit/src/imports.rs b/crates/jit/src/imports.rs index c2151ec94e..adc76ccffc 100644 --- a/crates/jit/src/imports.rs +++ b/crates/jit/src/imports.rs @@ -7,27 +7,28 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::{Global, GlobalInit, Memory, Table, TableElementType}; use wasmtime_environ::{MemoryPlan, MemoryStyle, Module, TablePlan}; use wasmtime_runtime::{ - Export, Imports, InstanceHandle, LinkError, VMFunctionImport, VMGlobalImport, VMMemoryImport, - VMTableImport, + Export, Imports, InstanceHandle, LinkError, SignatureRegistry, VMFunctionImport, + VMGlobalImport, VMMemoryImport, VMTableImport, }; /// This function allows to match all imports of a `Module` with concrete definitions provided by /// a `Resolver`. /// /// If all imports are satisfied returns an `Imports` instance required for a module instantiation. -pub fn resolve_imports(module: &Module, resolver: &mut dyn Resolver) -> Result { +pub fn resolve_imports( + module: &Module, + signatures: &SignatureRegistry, + resolver: &mut dyn Resolver, +) -> Result { let mut dependencies = HashSet::new(); let mut function_imports = PrimaryMap::with_capacity(module.imported_funcs.len()); for (index, (module_name, field, import_idx)) in module.imported_funcs.iter() { match resolver.resolve(*import_idx, module_name, field) { Some(export_value) => match export_value { - Export::Function { - address, - signature, - vmctx, - } => { + Export::Function(f) => { let import_signature = &module.local.signatures[module.local.functions[index]]; + let signature = signatures.lookup(f.signature).unwrap(); if signature != *import_signature { // TODO: If the difference is in the calling convention, // we could emit a wrapper function to fix it up. @@ -37,13 +38,13 @@ pub fn resolve_imports(module: &Module, resolver: &mut dyn Resolver) -> Result { + Export::Table(_) | Export::Memory(_) | Export::Global(_) => { return Err(LinkError(format!( "{}/{}: incompatible import type: export incompatible with function import", module_name, field @@ -63,26 +64,22 @@ pub fn resolve_imports(module: &Module, resolver: &mut dyn Resolver) -> Result match export_value { - Export::Table { - definition, - vmctx, - table, - } => { + Export::Table(t) => { let import_table = &module.local.table_plans[index]; - if !is_table_compatible(&table, import_table) { + if !is_table_compatible(&t.table, import_table) { return Err(LinkError(format!( "{}/{}: incompatible import type: exported table incompatible with \ table import", module_name, field, ))); } - dependencies.insert(unsafe { InstanceHandle::from_vmctx(vmctx) }); + dependencies.insert(unsafe { InstanceHandle::from_vmctx(t.vmctx) }); table_imports.push(VMTableImport { - from: definition, - vmctx, + from: t.definition, + vmctx: t.vmctx, }); } - Export::Global { .. } | Export::Memory { .. } | Export::Function { .. } => { + Export::Global(_) | Export::Memory(_) | Export::Function(_) => { return Err(LinkError(format!( "{}/{}: incompatible import type: export incompatible with table import", module_name, field @@ -102,13 +99,9 @@ pub fn resolve_imports(module: &Module, resolver: &mut dyn Resolver) -> Result match export_value { - Export::Memory { - definition, - vmctx, - memory, - } => { + Export::Memory(m) => { let import_memory = &module.local.memory_plans[index]; - if !is_memory_compatible(&memory, import_memory) { + if !is_memory_compatible(&m.memory, import_memory) { return Err(LinkError(format!( "{}/{}: incompatible import type: exported memory incompatible with \ memory import", @@ -123,19 +116,19 @@ pub fn resolve_imports(module: &Module, resolver: &mut dyn Resolver) -> Result { + Export::Table(_) | Export::Global(_) | Export::Function(_) => { return Err(LinkError(format!( "{}/{}: incompatible import type: export incompatible with memory import", module_name, field @@ -155,28 +148,24 @@ pub fn resolve_imports(module: &Module, resolver: &mut dyn Resolver) -> Result match export_value { - Export::Table { .. } | Export::Memory { .. } | Export::Function { .. } => { + Export::Table(_) | Export::Memory(_) | Export::Function(_) => { return Err(LinkError(format!( "{}/{}: incompatible import type: exported global incompatible with \ global import", module_name, field ))); } - Export::Global { - definition, - vmctx, - global, - } => { + Export::Global(g) => { let imported_global = module.local.globals[index]; - if !is_global_compatible(&global, &imported_global) { + if !is_global_compatible(&g.global, &imported_global) { return Err(LinkError(format!( "{}/{}: incompatible import type: exported global incompatible with \ global import", module_name, field ))); } - dependencies.insert(unsafe { InstanceHandle::from_vmctx(vmctx) }); - global_imports.push(VMGlobalImport { from: definition }); + dependencies.insert(unsafe { InstanceHandle::from_vmctx(g.vmctx) }); + global_imports.push(VMGlobalImport { from: g.definition }); } }, None => { diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 3c44dff4fb..b93eafa1dd 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -7,6 +7,7 @@ use crate::compiler::Compiler; use crate::imports::resolve_imports; use crate::link::link_module; use crate::resolver::Resolver; +use std::collections::HashMap; use std::io::Write; use std::rc::Rc; use std::sync::{Arc, Mutex}; @@ -19,8 +20,8 @@ use wasmtime_environ::{ }; use wasmtime_profiling::ProfilingAgent; use wasmtime_runtime::{ - GdbJitImageRegistration, InstanceHandle, InstantiationError, TrapRegistration, VMFunctionBody, - VMSharedSignatureIndex, + GdbJitImageRegistration, InstanceHandle, InstantiationError, SignatureRegistry, + TrapRegistration, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, }; /// An error condition while setting up a wasm instance, be it validation, @@ -50,6 +51,7 @@ pub enum SetupError { struct RawCompiledModule<'data> { module: Module, finished_functions: BoxedSlice, + trampolines: HashMap, data_initializers: Box<[DataInitializer<'data>]>, signatures: BoxedSlice, dbg_jit_registration: Option, @@ -78,13 +80,19 @@ impl<'data> RawCompiledModule<'data> { None }; - let (finished_functions, jt_offsets, relocations, dbg_image, trap_registration) = compiler - .compile( - &translation.module, - translation.module_translation.as_ref().unwrap(), - translation.function_body_inputs, - debug_data, - )?; + let ( + finished_functions, + trampolines, + jt_offsets, + relocations, + dbg_image, + trap_registration, + ) = compiler.compile( + &translation.module, + translation.module_translation.as_ref().unwrap(), + translation.function_body_inputs, + debug_data, + )?; link_module( &translation.module, @@ -135,6 +143,7 @@ impl<'data> RawCompiledModule<'data> { Ok(Self { module: translation.module, finished_functions: finished_functions.into_boxed_slice(), + trampolines, data_initializers: translation.data_initializers.into_boxed_slice(), signatures: signatures.into_boxed_slice(), dbg_jit_registration, @@ -147,6 +156,7 @@ impl<'data> RawCompiledModule<'data> { pub struct CompiledModule { module: Arc, finished_functions: BoxedSlice, + trampolines: HashMap, data_initializers: Box<[OwnedDataInitializer]>, signatures: BoxedSlice, dbg_jit_registration: Option>, @@ -166,6 +176,7 @@ impl CompiledModule { Ok(Self::from_parts( raw.module, raw.finished_functions, + raw.trampolines, raw.data_initializers .iter() .map(OwnedDataInitializer::new) @@ -181,6 +192,7 @@ impl CompiledModule { pub fn from_parts( module: Module, finished_functions: BoxedSlice, + trampolines: HashMap, data_initializers: Box<[OwnedDataInitializer]>, signatures: BoxedSlice, dbg_jit_registration: Option, @@ -189,6 +201,7 @@ impl CompiledModule { Self { module: Arc::new(module), finished_functions, + trampolines, data_initializers, signatures, dbg_jit_registration: dbg_jit_registration.map(Rc::new), @@ -209,6 +222,7 @@ impl CompiledModule { &self, is_bulk_memory: bool, resolver: &mut dyn Resolver, + sig_registry: &SignatureRegistry, ) -> Result { let data_initializers = self .data_initializers @@ -218,11 +232,12 @@ impl CompiledModule { data: &*init.data, }) .collect::>(); - let imports = resolve_imports(&self.module, resolver)?; + let imports = resolve_imports(&self.module, &sig_registry, resolver)?; InstanceHandle::new( Arc::clone(&self.module), self.trap_registration.clone(), self.finished_functions.clone(), + self.trampolines.clone(), imports, &data_initializers, self.signatures.clone(), @@ -284,7 +299,10 @@ pub unsafe fn instantiate( is_bulk_memory: bool, profiler: Option<&Arc>>>, ) -> Result { - let instance = CompiledModule::new(compiler, data, debug_info, profiler)? - .instantiate(is_bulk_memory, resolver)?; + let instance = CompiledModule::new(compiler, data, debug_info, profiler)?.instantiate( + is_bulk_memory, + resolver, + compiler.signatures(), + )?; Ok(instance) } diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 0c56c58a6f..4f672cfbd0 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -34,7 +34,7 @@ pub mod native; pub mod trampoline; pub use crate::code_memory::CodeMemory; -pub use crate::compiler::{CompilationStrategy, Compiler}; +pub use crate::compiler::{make_trampoline, CompilationStrategy, Compiler}; pub use crate::instantiate::{instantiate, CompiledModule, SetupError}; pub use crate::link::link_module; pub use crate::resolver::{NullResolver, Resolver}; diff --git a/crates/runtime/src/export.rs b/crates/runtime/src/export.rs index 547d9593cd..8cf42b8f8f 100644 --- a/crates/runtime/src/export.rs +++ b/crates/runtime/src/export.rs @@ -1,7 +1,7 @@ use crate::vmcontext::{ - VMContext, VMFunctionBody, VMGlobalDefinition, VMMemoryDefinition, VMTableDefinition, + VMContext, VMFunctionBody, VMGlobalDefinition, VMMemoryDefinition, VMSharedSignatureIndex, + VMTableDefinition, }; -use wasmtime_environ::ir; use wasmtime_environ::wasm::Global; use wasmtime_environ::{MemoryPlan, TablePlan}; @@ -9,96 +9,84 @@ use wasmtime_environ::{MemoryPlan, TablePlan}; #[derive(Debug, Clone)] pub enum Export { /// A function export value. - Function { - /// The address of the native-code function. - address: *const VMFunctionBody, - /// Pointer to the containing `VMContext`. - vmctx: *mut VMContext, - /// The function signature declaration, used for compatibilty checking. - signature: ir::Signature, - }, + Function(ExportFunction), /// A table export value. - Table { - /// The address of the table descriptor. - definition: *mut VMTableDefinition, - /// Pointer to the containing `VMContext`. - vmctx: *mut VMContext, - /// The table declaration, used for compatibilty checking. - table: TablePlan, - }, + Table(ExportTable), /// A memory export value. - Memory { - /// The address of the memory descriptor. - definition: *mut VMMemoryDefinition, - /// Pointer to the containing `VMContext`. - vmctx: *mut VMContext, - /// The memory declaration, used for compatibilty checking. - memory: MemoryPlan, - }, + Memory(ExportMemory), /// A global export value. - Global { - /// The address of the global storage. - definition: *mut VMGlobalDefinition, - /// Pointer to the containing `VMContext`. - vmctx: *mut VMContext, - /// The global declaration, used for compatibilty checking. - global: Global, - }, + Global(ExportGlobal), } -impl Export { - /// Construct a function export value. - pub fn function( - address: *const VMFunctionBody, - vmctx: *mut VMContext, - signature: ir::Signature, - ) -> Self { - Self::Function { - address, - vmctx, - signature, - } - } +/// A function export value. +#[derive(Debug, Clone)] +pub struct ExportFunction { + /// The address of the native-code function. + pub address: *const VMFunctionBody, + /// Pointer to the containing `VMContext`. + pub vmctx: *mut VMContext, + /// The function signature declaration, used for compatibilty checking. + /// + /// Note that this indexes within the module associated with `vmctx`. + pub signature: VMSharedSignatureIndex, +} - /// Construct a table export value. - pub fn table( - definition: *mut VMTableDefinition, - vmctx: *mut VMContext, - table: TablePlan, - ) -> Self { - Self::Table { - definition, - vmctx, - table, - } - } - - /// Construct a memory export value. - pub fn memory( - definition: *mut VMMemoryDefinition, - vmctx: *mut VMContext, - memory: MemoryPlan, - ) -> Self { - Self::Memory { - definition, - vmctx, - memory, - } - } - - /// Construct a global export value. - pub fn global( - definition: *mut VMGlobalDefinition, - vmctx: *mut VMContext, - global: Global, - ) -> Self { - Self::Global { - definition, - vmctx, - global, - } +impl From for Export { + fn from(func: ExportFunction) -> Export { + Export::Function(func) + } +} + +/// A table export value. +#[derive(Debug, Clone)] +pub struct ExportTable { + /// The address of the table descriptor. + pub definition: *mut VMTableDefinition, + /// Pointer to the containing `VMContext`. + pub vmctx: *mut VMContext, + /// The table declaration, used for compatibilty checking. + pub table: TablePlan, +} + +impl From for Export { + fn from(func: ExportTable) -> Export { + Export::Table(func) + } +} + +/// A memory export value. +#[derive(Debug, Clone)] +pub struct ExportMemory { + /// The address of the memory descriptor. + pub definition: *mut VMMemoryDefinition, + /// Pointer to the containing `VMContext`. + pub vmctx: *mut VMContext, + /// The memory declaration, used for compatibilty checking. + pub memory: MemoryPlan, +} + +impl From for Export { + fn from(func: ExportMemory) -> Export { + Export::Memory(func) + } +} + +/// A global export value. +#[derive(Debug, Clone)] +pub struct ExportGlobal { + /// The address of the global storage. + pub definition: *mut VMGlobalDefinition, + /// Pointer to the containing `VMContext`. + pub vmctx: *mut VMContext, + /// The global declaration, used for compatibilty checking. + pub global: Global, +} + +impl From for Export { + fn from(func: ExportGlobal) -> Export { + Export::Global(func) } } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index e912ce350b..a6eb25fda5 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -12,16 +12,16 @@ use crate::traphandlers::{catch_traps, Trap}; use crate::vmcontext::{ VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, - VMTableDefinition, VMTableImport, + VMTableDefinition, VMTableImport, VMTrampoline, }; use crate::TrapRegistration; +use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable}; use memoffset::offset_of; use more_asserts::assert_lt; use std::alloc::{self, Layout}; use std::any::Any; use std::cell::{Cell, RefCell}; -use std::collections::HashMap; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; use std::rc::Rc; use std::sync::Arc; @@ -99,6 +99,9 @@ pub(crate) struct Instance { /// Pointers to functions in executable memory. finished_functions: BoxedSlice, + /// Pointers to trampoline functions used to enter particular signatures + trampolines: HashMap, + /// Hosts can store arbitrary per-instance information here. host_state: Box, @@ -301,8 +304,7 @@ impl Instance { pub fn lookup_by_declaration(&self, export: &wasmtime_environ::Export) -> Export { match export { wasmtime_environ::Export::Function(index) => { - let signature = - self.module.local.signatures[self.module.local.functions[*index]].clone(); + let signature = self.signature_id(self.module.local.functions[*index]); let (address, vmctx) = if let Some(def_index) = self.module.local.defined_func_index(*index) { ( @@ -313,11 +315,12 @@ impl Instance { let import = self.imported_function(*index); (import.body, import.vmctx) }; - Export::Function { + ExportFunction { address, signature, vmctx, } + .into() } wasmtime_environ::Export::Table(index) => { let (definition, vmctx) = @@ -327,11 +330,12 @@ impl Instance { let import = self.imported_table(*index); (import.from, import.vmctx) }; - Export::Table { + ExportTable { definition, vmctx, table: self.module.local.table_plans[*index].clone(), } + .into() } wasmtime_environ::Export::Memory(index) => { let (definition, vmctx) = @@ -341,13 +345,14 @@ impl Instance { let import = self.imported_memory(*index); (import.from, import.vmctx) }; - Export::Memory { + ExportMemory { definition, vmctx, memory: self.module.local.memory_plans[*index].clone(), } + .into() } - wasmtime_environ::Export::Global(index) => Export::Global { + wasmtime_environ::Export::Global(index) => ExportGlobal { definition: if let Some(def_index) = self.module.local.defined_global_index(*index) { self.global_ptr(def_index) @@ -356,7 +361,8 @@ impl Instance { }, vmctx: self.vmctx_ptr(), global: self.module.local.globals[*index], - }, + } + .into(), } } @@ -853,6 +859,7 @@ impl InstanceHandle { module: Arc, trap_registration: TrapRegistration, finished_functions: BoxedSlice, + trampolines: HashMap, imports: Imports, data_initializers: &[DataInitializer<'_>], vmshared_signatures: BoxedSlice, @@ -892,6 +899,7 @@ impl InstanceHandle { passive_elements: Default::default(), passive_data, finished_functions, + trampolines, dbg_jit_registration, host_state, signal_handler: Cell::new(None), @@ -1093,6 +1101,11 @@ impl InstanceHandle { self.instance().get_defined_table(index) } + /// Gets the trampoline pre-registered for a particular signature + pub fn trampoline(&self, sig: VMSharedSignatureIndex) -> Option { + self.instance().trampolines.get(&sig).cloned() + } + /// Return a reference to the contained `Instance`. pub(crate) fn instance(&self) -> &Instance { unsafe { &*(self.instance as *const Instance) } diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 736c40e2df..212defdff8 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -36,7 +36,7 @@ mod vmcontext; pub mod libcalls; -pub use crate::export::Export; +pub use crate::export::*; pub use crate::imports::Imports; pub use crate::instance::{InstanceHandle, InstantiationError, LinkError}; pub use crate::jit_int::GdbJitImageRegistration; @@ -51,7 +51,7 @@ pub use crate::traphandlers::{ pub use crate::vmcontext::{ VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMInvokeArgument, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, - VMTableDefinition, VMTableImport, + VMTableDefinition, VMTableImport, VMTrampoline, }; /// Version number of this crate. diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 7182a6506d..351d71ce4e 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -3,7 +3,7 @@ use crate::instance::{InstanceHandle, SignalHandler}; use crate::trap_registry::TrapDescription; -use crate::vmcontext::{VMContext, VMFunctionBody}; +use crate::vmcontext::{VMContext, VMFunctionBody, VMTrampoline}; use backtrace::Backtrace; use std::any::Any; use std::cell::Cell; @@ -170,7 +170,7 @@ impl Trap { pub unsafe fn wasmtime_call_trampoline( vmctx: *mut VMContext, caller_vmctx: *mut VMContext, - trampoline: *const VMFunctionBody, + trampoline: VMTrampoline, callee: *const VMFunctionBody, values_vec: *mut u8, ) -> Result<(), Trap> { diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index 53ca19d21c..23e6384bfc 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -645,3 +645,11 @@ impl VMContext { self.instance().host_state() } } + +/// +pub type VMTrampoline = unsafe extern "C" fn( + *mut VMContext, // callee vmctx + *mut VMContext, // caller vmctx + *const VMFunctionBody, // function we're actually calling + *mut u128, // space for arguments and return values +); diff --git a/crates/wasi/src/lib.rs b/crates/wasi/src/lib.rs index 5a6d29c199..3b0f65c38a 100644 --- a/crates/wasi/src/lib.rs +++ b/crates/wasi/src/lib.rs @@ -45,13 +45,9 @@ impl wasmtime::WasmTy for WasiCallerMemory { fn from_abi(vmctx: *mut wasmtime_runtime::VMContext, _abi: ()) -> Self { unsafe { match wasmtime_runtime::InstanceHandle::from_vmctx(vmctx).lookup("memory") { - Some(wasmtime_runtime::Export::Memory { - definition, - vmctx: _, - memory: _, - }) => WasiCallerMemory { - base: (*definition).base, - len: (*definition).current_length, + Some(wasmtime_runtime::Export::Memory(m)) => WasiCallerMemory { + base: (*m.definition).base, + len: (*m.definition).current_length, }, _ => WasiCallerMemory { base: std::ptr::null_mut(), @@ -62,6 +58,8 @@ impl wasmtime::WasmTy for WasiCallerMemory { } fn into_abi(self) {} + unsafe fn load(_ptr: &mut *const u128) {} + unsafe fn store(_abi: Self::Abi, _ptr: *mut u128) {} } impl WasiCallerMemory {