T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move the signature registry into Engine.

Browse Source 
This commit moves the shared signature registry out of `Store` and into
`Engine`.

This helps eliminate work that was performed whenever a `Module` was
instantiated into a `Store`.

Now a `Module` is registered with the shared signature registry upon creation,
storing the mapping from the module's signature index space to the shared index
space.

This also refactors the "frame info" registry into a general purpose "module
registry" that is used to look up trap information, signature information, and
(soon) stack map information.
This commit is contained in:
Peter Huene
2021-04-11 00:37:27 -07:00
parent f26449f03d
commit a2466b3c23
17 changed files with 768 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/runtime/src/externref.rs
View File

@@ -753,7 +753,7 @@ pub struct StackMapRegistry {
struct StackMapRegistryInner {
/// A map from the highest pc in a module, to its stack maps.
///
/// For details, see the comment above `GlobalFrameInfo::ranges`.
/// For details, see the comment above `GlobalModuleRegistry`.
ranges: BTreeMap<usize, ModuleStackMaps>,
}

2
crates/runtime/src/traphandlers/macos.rs
View File

@@ -25,7 +25,7 @@
//! use a thread-local to store information about how to unwind. Additionally
//! this requires that the check of whether a pc is a wasm trap or not is a
//! global check rather than a per-thread check. This necessitates the existence
//! of `GlobalFrameInfo` in the `wasmtime` crate.
//! of `GlobalModuleRegistry` in the `wasmtime` crate.
//!
//! Otherwise this file heavily uses the `mach` Rust crate for type and
//! function declarations. Many bits and pieces are copied or translated from

8
crates/wasmtime/src/config.rs
View File

@@ -319,6 +319,10 @@ impl HostFuncMap {
fn async_required(&self) -> bool {
self.funcs.values().any(|f| f.1)
}
fn iter(&self) -> impl Iterator<Item = &HostFunc> {
self.funcs.values().map(|v| &*v.0)
}
}
macro_rules! generate_wrap_async_host_func {
@@ -1318,6 +1322,10 @@ impl Config {
for_each_function_signature!(generate_wrap_async_host_func);
pub(crate) fn host_funcs(&self) -> impl Iterator<Item = &HostFunc> {
self.host_funcs.iter()
}
pub(crate) fn get_host_func(&self, module: &str, name: &str) -> Option<&HostFunc> {
self.host_funcs.get(module, name)
}

100
crates/wasmtime/src/engine.rs
View File

@@ -1,10 +1,31 @@
use crate::signatures::{SharedSignatures, SignatureRegistry, TrampolineMap};
use crate::Config;
use anyhow::Result;
use std::sync::Arc;
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
#[cfg(feature = "cache")]
use wasmtime_cache::CacheConfig;
use wasmtime_environ::{
entity::PrimaryMap,
wasm::{SignatureIndex, WasmFuncType},
};
use wasmtime_jit::Compiler;
use wasmtime_runtime::{debug_builtins, InstanceAllocator};
use wasmtime_runtime::{
debug_builtins, InstanceAllocator, InstanceHandle, VMCallerCheckedAnyfunc,
VMSharedSignatureIndex, VMTrampoline,
};
#[derive(Default)]
struct EngineHostFuncs {
anyfuncs: HashMap<InstanceHandle, Box<VMCallerCheckedAnyfunc>>,
trampolines: TrampolineMap,
}
// This is safe for send and sync as it is read-only once the
// engine is constructed and the host functions live with the config,
// which the engine keeps a strong reference to.
unsafe impl Send for EngineHostFuncs {}
unsafe impl Sync for EngineHostFuncs {}
/// An `Engine` which is a global context for compilation and management of wasm
/// modules.
@@ -37,6 +58,16 @@ struct EngineInner {
config: Config,
compiler: Compiler,
allocator: Box<dyn InstanceAllocator>,
signatures: RwLock<SignatureRegistry>,
host_funcs: EngineHostFuncs,
}
impl Drop for EngineInner {
fn drop(&mut self) {
let mut signatures = self.signatures.write().unwrap();
signatures.unregister(self.host_funcs.trampolines.indexes());
assert!(signatures.is_empty());
}
}
impl Engine {
@@ -46,11 +77,29 @@ impl Engine {
debug_builtins::ensure_exported();
config.validate()?;
let allocator = config.build_allocator()?;
let mut signatures = SignatureRegistry::default();
let mut host_funcs = EngineHostFuncs::default();
// Register all the host function signatures
for func in config.host_funcs() {
let sig = signatures.register(func.ty.as_wasm_func_type());
// Cloning the instance handle is safe as host functions outlive the engine
host_funcs.anyfuncs.insert(
unsafe { func.instance.clone() },
Box::new(func.anyfunc(sig)),
);
host_funcs.trampolines.insert(sig, func.trampoline);
}
Ok(Engine {
inner: Arc::new(EngineInner {
config: config.clone(),
compiler: config.build_compiler(allocator.as_ref()),
allocator,
signatures: RwLock::new(signatures),
host_funcs,
}),
})
}
@@ -79,6 +128,53 @@ impl Engine {
Arc::ptr_eq(&a.inner, &b.inner)
}
pub(crate) fn register_module_signatures(
&self,
signatures: &PrimaryMap<SignatureIndex, WasmFuncType>,
trampolines: impl Iterator<Item = (SignatureIndex, VMTrampoline)>,
) -> (SharedSignatures, TrampolineMap) {
self.inner
.signatures
.write()
.unwrap()
.register_module(signatures, trampolines)
}
pub(crate) fn register_signature(&self, ty: &WasmFuncType) -> VMSharedSignatureIndex {
self.inner.signatures.write().unwrap().register(ty)
}
pub(crate) fn unregister_signatures(
&self,
indexes: impl Iterator<Item = VMSharedSignatureIndex>,
) {
self.inner.signatures.write().unwrap().unregister(indexes);
}
pub(crate) fn lookup_func_type(&self, index: VMSharedSignatureIndex) -> Option<WasmFuncType> {
self.inner
.signatures
.read()
.unwrap()
.lookup_type(index)
.cloned()
}
pub(crate) fn host_func_trampolines(&self) -> &TrampolineMap {
&self.inner.host_funcs.trampolines
}
pub(crate) fn host_func_anyfunc(
&self,
instance: &InstanceHandle,
) -> Option<&VMCallerCheckedAnyfunc> {
self.inner
.host_funcs
.anyfuncs
.get(instance)
.map(AsRef::as_ref)
}
/// Ahead-of-time (AOT) compiles a WebAssembly module.
///
/// The `bytes` provided must be in one of two formats:

109
crates/wasmtime/src/func.rs
View File

@@ -1,4 +1,4 @@
use crate::{sig_registry::SignatureRegistry, trampoline::StoreInstanceHandle};
use crate::trampoline::StoreInstanceHandle;
use crate::{Config, Extern, FuncType, Store, Trap, Val, ValType};
use anyhow::{bail, Context as _, Result};
use smallvec::{smallvec, SmallVec};
@@ -22,9 +22,9 @@ use wasmtime_runtime::{
/// This differs from `Func` in that it is not associated with a `Store`.
/// Host functions are associated with a `Config`.
pub(crate) struct HostFunc {
ty: FuncType,
instance: InstanceHandle,
trampoline: VMTrampoline,
pub ty: FuncType,
pub instance: InstanceHandle,
pub trampoline: VMTrampoline,
}
impl HostFunc {
@@ -73,6 +73,23 @@ impl HostFunc {
}
}
/// Gets a caller-checked anyfunc for this host function given a shared signature index.
///
/// The shared signature index must have been registered for the signature of
/// this host function.
pub fn anyfunc(&self, sig: VMSharedSignatureIndex) -> VMCallerCheckedAnyfunc {
let mut anyfunc = match self
.instance
.lookup_by_declaration(&EntityIndex::Function(FuncIndex::from_u32(0)))
{
wasmtime_runtime::Export::Function(f) => unsafe { f.anyfunc.as_ref() }.clone(),
_ => unreachable!(),
};
anyfunc.type_index = sig;
anyfunc
}
/// Converts a `HostFunc` to a `Func`.
///
/// # Safety
@@ -88,11 +105,11 @@ impl HostFunc {
};
let export = ExportFunction {
anyfunc: std::ptr::NonNull::new_unchecked(store.get_host_anyfunc(
&self.instance,
&self.ty,
self.trampoline,
)),
anyfunc: store
.engine()
.host_func_anyfunc(&self.instance)
.unwrap()
.into(),
};
Func {
@@ -408,12 +425,8 @@ impl Func {
Func::invoke(&store, &ty_clone, caller_vmctx, values_vec, &func)
});
let (instance, trampoline) = crate::trampoline::create_function(
&ty,
func,
store.engine().config(),
Some(&mut store.signatures().borrow_mut()),
)
let (instance, trampoline) =
crate::trampoline::create_function(&ty, func, store.engine().config(), Some(store))
.expect("failed to create function");
let idx = EntityIndex::Function(FuncIndex::from_u32(0));
@@ -734,7 +747,7 @@ impl Func {
/// # }
/// ```
pub fn wrap<Params, Results>(store: &Store, func: impl IntoFunc<Params, Results>) -> Func {
let (_, instance, trampoline) = func.into_func(Some(&mut store.signatures().borrow_mut()));
let (_, instance, trampoline) = func.into_func(Some(store));
let (instance, export) = unsafe {
let idx = EntityIndex::Function(FuncIndex::from_u32(0));
@@ -759,33 +772,35 @@ impl Func {
/// Returns the underlying wasm type that this `Func` has.
pub fn ty(&self) -> FuncType {
// Signatures should always be registered in the store's registry of
// Signatures should always be registered in the engine's registry of
// shared signatures, so we should be able to unwrap safely here.
let signatures = self.instance.store.signatures().borrow();
let (wft, _) = signatures
.lookup_shared(self.sig_index())
.expect("signature should be registered");
// This is only called with `Export::Function`, and since it's coming
// from wasmtime_runtime itself we should support all the types coming
// out of it, so assert such here.
FuncType::from_wasm_func_type(&wft)
FuncType::from_wasm_func_type(
self.instance
.store
.engine()
.lookup_func_type(self.sig_index())
.expect("signature should be registered"),
)
}
/// Returns the number of parameters that this function takes.
pub fn param_arity(&self) -> usize {
let signatures = self.instance.store.signatures().borrow();
let (sig, _) = signatures
.lookup_shared(self.sig_index())
let sig = self
.instance
.store
.engine()
.lookup_func_type(self.sig_index())
.expect("signature should be registered");
sig.params.len()
}
/// Returns the number of results this function produces.
pub fn result_arity(&self) -> usize {
let signatures = self.instance.store.signatures().borrow();
let (sig, _) = signatures
.lookup_shared(self.sig_index())
let sig = self
.instance
.store
.engine()
.lookup_func_type(self.sig_index())
.expect("signature should be registered");
sig.returns.len()
}
@@ -907,21 +922,12 @@ impl Func {
}
pub(crate) unsafe fn from_wasmtime_function(export: &ExportFunction, store: &Store) -> Self {
// Each function signature in a module should have a trampoline stored
// on that module as well, so unwrap the result here since otherwise
// it's a bug in wasmtime.
let anyfunc = export.anyfunc.as_ref();
let trampoline = store
.signatures()
.borrow()
.lookup_shared(anyfunc.type_index)
.expect("failed to retrieve trampoline from module")
.1;
Func {
instance: store.existing_vmctx(anyfunc.vmctx),
export: export.clone(),
trampoline,
trampoline: store.lookup_trampoline(anyfunc.type_index),
}
}
@@ -1542,10 +1548,7 @@ for_each_function_signature!(impl_host_abi);
/// as an implementation detail of this crate.
pub trait IntoFunc<Params, Results> {
#[doc(hidden)]
fn into_func(
self,
registry: Option<&mut SignatureRegistry>,
) -> (FuncType, InstanceHandle, VMTrampoline);
fn into_func(self, store: Option<&Store>) -> (FuncType, InstanceHandle, VMTrampoline);
}
/// A structure representing the *caller's* context when creating a function
@@ -1658,12 +1661,12 @@ macro_rules! impl_into_func {
$($args: WasmTy,)*
R: WasmRet,
{
fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) {
fn into_func(self, store: Option<&Store>) -> (FuncType, InstanceHandle, VMTrampoline) {
let f = move |_: Caller<'_>, $($args:$args),*| {
self($($args),*)
};
f.into_func(registry)
f.into_func(store)
}
}
@@ -1674,7 +1677,7 @@ macro_rules! impl_into_func {
$($args: WasmTy,)*
R: WasmRet,
{
fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) {
fn into_func(self, store: Option<&Store>) -> (FuncType, InstanceHandle, VMTrampoline) {
/// This shim is called by Wasm code, constructs a `Caller`,
/// calls the wrapped host function, and returns the translated
/// result back to Wasm.
@@ -1807,10 +1810,10 @@ macro_rules! impl_into_func {
let trampoline = host_trampoline::<$($args,)* R>;
// If not given a registry, use a default signature index that is guaranteed to trap
// if the function is called indirectly without first being associated with a store (a bug condition).
let shared_signature_id = registry
.map(|r| r.register(ty.as_wasm_func_type(), trampoline))
// If not given a store, use a default signature index that is guaranteed to trap.
// If the function is called indirectly without first being associated with a store (a bug condition).
let shared_signature_id = store
.map(|s| s.register_signature(ty.as_wasm_func_type(), trampoline))
.unwrap_or(VMSharedSignatureIndex::default());
let instance = unsafe {

5
crates/wasmtime/src/instance.rs
View File

@@ -362,6 +362,7 @@ impl<'a> Instantiator<'a> {
let expected_ty =
self.cur.module.compiled_module().module().type_of(*index);
matching::MatchCx {
signatures: self.cur.module.signatures(),
types: self.cur.module.types(),
store: self.store,
}
@@ -513,14 +514,12 @@ impl<'a> Instantiator<'a> {
unsafe {
let engine = self.store.engine();
let allocator = engine.allocator();
let signatures = self.store.signatures().borrow();
let signatures = signatures.lookup_table(&self.cur.module);
let instance = allocator.allocate(InstanceAllocationRequest {
module: compiled_module.module().clone(),
finished_functions: compiled_module.finished_functions(),
imports: self.cur.build(),
shared_signatures: (&signatures).into(),
shared_signatures: self.cur.module.signatures().into(),
host_state: Box::new(()),
interrupts: self.store.interrupts(),
externref_activations_table: self.store.externref_activations_table()

6
crates/wasmtime/src/lib.rs
View File

@@ -282,13 +282,12 @@ mod func;
mod config;
mod engine;
mod externals;
mod frame_info;
mod instance;
mod linker;
mod memory;
mod module;
mod r#ref;
mod sig_registry;
mod signatures;
mod store;
mod trampoline;
mod trap;
@@ -298,12 +297,11 @@ mod values;
pub use crate::config::*;
pub use crate::engine::*;
pub use crate::externals::*;
pub use crate::frame_info::{FrameInfo, FrameSymbol};
pub use crate::func::*;
pub use crate::instance::Instance;
pub use crate::linker::*;
pub use crate::memory::*;
pub use crate::module::Module;
pub use crate::module::{FrameInfo, FrameSymbol, Module};
pub use crate::r#ref::ExternRef;
pub use crate::store::*;
pub use crate::trap::*;

60
crates/wasmtime/src/module.rs
View File

@@ -1,4 +1,7 @@
use crate::types::{ExportType, ExternType, ImportType};
use crate::{
signatures::{SharedSignatures, TrampolineMap},
types::{ExportType, ExternType, ImportType},
};
use crate::{Engine, ModuleType};
use anyhow::{bail, Context, Result};
use std::fs;
@@ -8,13 +11,36 @@ use wasmparser::Validator;
#[cfg(feature = "cache")]
use wasmtime_cache::ModuleCacheEntry;
use wasmtime_environ::entity::PrimaryMap;
use wasmtime_environ::wasm::ModuleIndex;
use wasmtime_environ::wasm::{ModuleIndex, SignatureIndex};
use wasmtime_jit::{CompilationArtifacts, CompiledModule, TypeTables};
use wasmtime_runtime::VMSharedSignatureIndex;
mod registry;
mod serialization;
pub use registry::{FrameInfo, FrameSymbol, GlobalModuleRegistry, ModuleRegistry};
pub use serialization::SerializedModule;
// A wrapper around registered signatures and trampolines that will automatically
/// unregister the signatures when dropped.
pub(crate) struct ModuleSharedSignatures {
engine: Engine,
signatures: SharedSignatures,
trampolines: TrampolineMap,
}
impl Drop for ModuleSharedSignatures {
fn drop(&mut self) {
if !self.signatures.is_empty() {
// Use the shared signatures map to unregister as not every registered
// signature will have a trampoline, but every index in the trampoline map
// will be present in the shared signatures map.
self.engine
.unregister_signatures(self.signatures.values().cloned());
}
}
}
/// A compiled WebAssembly module, ready to be instantiated.
///
/// A `Module` is a compiled in-memory representation of an input WebAssembly
@@ -102,6 +128,8 @@ struct ModuleInner {
/// Type information of this module and all `artifact_upvars` compiled
/// modules.
types: Arc<TypeTables>,
/// Registered shared signature for the module.
signatures: Arc<ModuleSharedSignatures>,
}
impl Module {
@@ -318,10 +346,16 @@ impl Module {
engine.compiler().isa(),
&*engine.config().profiler,
)?;
let module = modules.remove(main_module);
// Validate the module can be used with the current allocator
engine.allocator().validate(module.module())?;
engine.allocator().validate(modules[main_module].module())?;
let (signatures, trampolines) = engine.register_module_signatures(
&types.wasm_signatures,
modules.iter().flat_map(|m| m.trampolines().iter().cloned()),
);
let module = modules.remove(main_module);
Ok(Module {
inner: Arc::new(ModuleInner {
@@ -330,6 +364,11 @@ impl Module {
types: Arc::new(types),
artifact_upvars: modules,
module_upvars: Vec::new(),
signatures: Arc::new(ModuleSharedSignatures {
engine: engine.clone(),
signatures,
trampolines,
}),
}),
})
}
@@ -416,8 +455,8 @@ impl Module {
) -> Module {
Module {
inner: Arc::new(ModuleInner {
types: self.types().clone(),
engine: self.engine().clone(),
types: self.inner.types.clone(),
engine: self.inner.engine.clone(),
module: self.inner.artifact_upvars[artifact_index].clone(),
artifact_upvars: artifact_upvars
.iter()
@@ -432,6 +471,7 @@ impl Module {
wasmtime_environ::ModuleUpvar::Local(i) => modules[i].clone(),
})
.collect(),
signatures: self.inner.signatures.clone(),
}),
}
}
@@ -448,6 +488,14 @@ impl Module {
&self.inner.types
}
pub(crate) fn signatures(&self) -> &PrimaryMap<SignatureIndex, VMSharedSignatureIndex> {
&self.inner.signatures.signatures
}
pub(crate) fn shared_signatures(&self) -> &Arc<ModuleSharedSignatures> {
&self.inner.signatures
}
/// Looks up the module upvar value at the `index` specified.
///
/// Note that this panics if `index` is out of bounds since this should

217
crates/wasmtime/src/frame_info.rs → crates/wasmtime/src/module/registry.rs
View File

@@ -1,35 +1,30 @@
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::Mutex;
use wasmtime_environ::entity::EntityRef;
use wasmtime_environ::ir;
use wasmtime_environ::wasm::DefinedFuncIndex;
use wasmtime_environ::{FunctionAddressMap, TrapInformation};
use wasmtime_jit::CompiledModule;
//! Implements a registry of modules for a store.
/// This is a structure that lives within a `Store` and retains information
/// about all modules registered with the `Store` via instantiation.
///
/// "frame information" here refers to things like determining whether a
/// program counter is a wasm program counter, and additionally mapping program
/// counters to wasm filenames, modules, line numbers, etc. This store of
/// information lives as long as a `Store` lives since modules are never
/// unloaded today.
#[derive(Default)]
pub struct StoreFrameInfo {
/// An internal map that keeps track of backtrace frame information for
/// each module.
///
/// This map is morally a map of ranges to a map of information for that
/// module. Each module is expected to reside in a disjoint section of
/// contiguous memory. No modules can overlap.
///
/// The key of this map is the highest address in the module and the value
/// is the module's information, which also contains the start address.
ranges: BTreeMap<usize, ModuleFrameInfo>,
use crate::{module::ModuleSharedSignatures, Module};
use std::{
collections::BTreeMap,
sync::{Arc, Mutex},
};
use wasmtime_environ::{
entity::EntityRef, ir, wasm::DefinedFuncIndex, FunctionAddressMap, TrapInformation,
};
use wasmtime_jit::CompiledModule;
use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline};
lazy_static::lazy_static! {
static ref GLOBAL_MODULES: Mutex<GlobalModuleRegistry> = Default::default();
}
impl StoreFrameInfo {
/// Used for registering modules with a store.
///
/// The map is from the ending (exclusive) address for the module code to
/// the registered module.
///
/// The `BTreeMap` is used to quickly locate a module based on a program counter value.
#[derive(Default)]
pub struct ModuleRegistry(BTreeMap<usize, RegisteredModule>);
impl ModuleRegistry {
/// Fetches frame information about a program counter in a backtrace.
///
/// Returns an object if this `pc` is known to some previously registered
@@ -48,8 +43,8 @@ impl StoreFrameInfo {
self.module(pc)?.lookup_trap_info(pc)
}
fn module(&self, pc: usize) -> Option<&ModuleFrameInfo> {
let (end, info) = self.ranges.range(pc..).next()?;
fn module(&self, pc: usize) -> Option<&RegisteredModule> {
let (end, info) = self.0.range(pc..).next()?;
if pc < info.start || *end < pc {
return None;
}
@@ -57,57 +52,78 @@ impl StoreFrameInfo {
Some(info)
}
/// Registers a new compiled module's frame information.
pub fn register(&mut self, module: &Arc<CompiledModule>) {
let (start, end) = module.code().range();
/// Registers a new module with the registry.
pub fn register(&mut self, module: &Module) -> bool {
let compiled_module = module.compiled_module();
let (start, end) = compiled_module.code().range();
// Ignore modules with no code or finished functions
if start == end || module.finished_functions().is_empty() {
return;
// Ignore modules with no code, finished functions, or if the module is already registered
if start == end || compiled_module.finished_functions().is_empty() {
return false;
}
// The module code range is exclusive for end, so make it inclusive as it
// may be a valid PC value
let end = end - 1;
if self.0.get(&end).is_some() {
return false;
}
// Assert that this module's code doesn't collide with any other registered modules
if let Some((_, prev)) = self.ranges.range(end..).next() {
if let Some((_, prev)) = self.0.range(end..).next() {
assert!(prev.start > end);
}
if let Some((prev_end, _)) = self.ranges.range(..=start).next_back() {
if let Some((prev_end, _)) = self.0.range(..=start).next_back() {
assert!(*prev_end < start);
}
let prev = self.ranges.insert(
let prev = self.0.insert(
end,
ModuleFrameInfo {
RegisteredModule {
start,
module: module.clone(),
module: compiled_module.clone(),
signatures: module.shared_signatures().clone(),
},
);
assert!(prev.is_none());
GLOBAL_INFO.lock().unwrap().register(start, end, module);
GLOBAL_MODULES.lock().unwrap().register(start, end, module);
true
}
/// Looks up a trampoline from a shared signature index.
///
/// This will search all modules associated with the store for a suitable trampoline
/// given the shared signature index.
pub fn lookup_trampoline(&self, index: VMSharedSignatureIndex) -> Option<VMTrampoline> {
for (_, m) in &self.0 {
if let Some(trampoline) = m.signatures.trampolines.get(index) {
return Some(trampoline);
}
}
None
}
}
impl Drop for StoreFrameInfo {
impl Drop for ModuleRegistry {
fn drop(&mut self) {
let mut info = GLOBAL_INFO.lock().unwrap();
for end in self.ranges.keys() {
let mut info = GLOBAL_MODULES.lock().unwrap();
for end in self.0.keys() {
info.unregister(*end);
}
}
}
/// Represents a module's frame information.
#[derive(Clone)]
pub struct ModuleFrameInfo {
struct RegisteredModule {
start: usize,
module: Arc<CompiledModule>,
signatures: Arc<ModuleSharedSignatures>,
}
impl ModuleFrameInfo {
impl RegisteredModule {
/// Determines if the related module has unparsed debug information.
pub fn has_unparsed_debuginfo(&self) -> bool {
self.module.has_unparsed_debuginfo()
@@ -214,47 +230,29 @@ impl ModuleFrameInfo {
}
}
/// This is the dual of `StoreFrameInfo` and is stored globally (as the name
/// implies) rather than simply in one `Store`.
/// This is the global module registry that stores information for all modules
/// that are currently in use by any `Store`.
///
/// The purpose of this map is to be called from signal handlers to determine
/// whether a program counter is a wasm trap or not. Specifically macOS has
/// no contextual information about the thread available, hence the necessity
/// for global state rather than using thread local state.
///
/// This is similar to `StoreFrameInfo` except that it has less information and
/// supports removal. Any time anything is registered with a `StoreFrameInfo`
/// it is also automatically registered with the singleton global frame
/// information. When a `StoreFrameInfo` is destroyed then all of its entries
/// are removed from the global frame information.
/// This is similar to `ModuleRegistry` except that it has less information and
/// supports removal. Any time anything is registered with a `ModuleRegistry`
/// it is also automatically registered with the singleton global module
/// registry. When a `ModuleRegistry` is destroyed then all of its entries
/// are removed from the global module registry.
#[derive(Default)]
pub struct GlobalFrameInfo {
// The map here behaves the same way as `StoreFrameInfo`.
ranges: BTreeMap<usize, GlobalModuleFrameInfo>,
}
pub struct GlobalModuleRegistry(BTreeMap<usize, GlobalRegisteredModule>);
/// This is the equivalent of `ModuleFrameInfo` except it keeps a reference count.
struct GlobalModuleFrameInfo {
module: ModuleFrameInfo,
/// Note that modules can be instantiated in many stores, so the purpose of
/// this field is to keep track of how many stores have registered a
/// module. Information is only removed from the global store when this
/// reference count reaches 0.
references: usize,
}
lazy_static::lazy_static! {
static ref GLOBAL_INFO: Mutex<GlobalFrameInfo> = Default::default();
}
impl GlobalFrameInfo {
impl GlobalModuleRegistry {
/// Returns whether the `pc`, according to globally registered information,
/// is a wasm trap or not.
pub(crate) fn is_wasm_pc(pc: usize) -> bool {
let info = GLOBAL_INFO.lock().unwrap();
let info = GLOBAL_MODULES.lock().unwrap();
match info.ranges.range(pc..).next() {
match info.0.range(pc..).next() {
Some((end, info)) => {
if pc < info.module.start || *end < pc {
return false;
@@ -263,7 +261,7 @@ impl GlobalFrameInfo {
match info.module.func(pc) {
Some((index, offset)) => {
let (addr_map, _) = info.module.module.func_info(index);
ModuleFrameInfo::instr_pos(offset, addr_map).is_some()
RegisteredModule::instr_pos(offset, addr_map).is_some()
}
None => false,
}
@@ -274,14 +272,12 @@ impl GlobalFrameInfo {
/// Registers a new region of code, described by `(start, end)` and with
/// the given function information, with the global information.
fn register(&mut self, start: usize, end: usize, module: &Arc<CompiledModule>) {
let info = self
.ranges
.entry(end)
.or_insert_with(|| GlobalModuleFrameInfo {
module: ModuleFrameInfo {
fn register(&mut self, start: usize, end: usize, module: &Module) {
let info = self.0.entry(end).or_insert_with(|| GlobalRegisteredModule {
module: RegisteredModule {
start,
module: module.clone(),
module: module.compiled_module().clone(),
signatures: module.shared_signatures().clone(),
},
references: 0,
});
@@ -293,17 +289,28 @@ impl GlobalFrameInfo {
info.references += 1;
}
/// Unregisters a region of code (keyed by the `end` address) from this
/// Unregisters a region of code (keyed by the `end` address) from the
/// global information.
fn unregister(&mut self, end: usize) {
let info = self.ranges.get_mut(&end).unwrap();
let info = self.0.get_mut(&end).unwrap();
info.references -= 1;
if info.references == 0 {
self.ranges.remove(&end);
self.0.remove(&end);
}
}
}
/// This is the equivalent of `RegisteredModule` except it keeps a reference count.
struct GlobalRegisteredModule {
module: RegisteredModule,
/// Note that modules can be instantiated in many stores, so the purpose of
/// this field is to keep track of how many stores have registered a
/// module. Information is only removed from the global registry when this
/// reference count reaches 0.
references: usize,
}
/// Description of a frame in a backtrace for a [`Trap`].
///
/// Whenever a WebAssembly trap occurs an instance of [`Trap`] is created. Each
@@ -321,19 +328,6 @@ pub struct FrameInfo {
symbols: Vec<FrameSymbol>,
}
/// Debug information for a symbol that is attached to a [`FrameInfo`].
///
/// When DWARF debug information is present in a wasm file then this structure
/// can be found on a [`FrameInfo`] and can be used to learn about filenames,
/// line numbers, etc, which are the origin of a function in a stack trace.
#[derive(Debug)]
pub struct FrameSymbol {
name: Option<String>,
file: Option<String>,
line: Option<u32>,
column: Option<u32>,
}
impl FrameInfo {
/// Returns the WebAssembly function index for this frame.
///
@@ -405,6 +399,19 @@ impl FrameInfo {
}
}
/// Debug information for a symbol that is attached to a [`FrameInfo`].
///
/// When DWARF debug information is present in a wasm file then this structure
/// can be found on a [`FrameInfo`] and can be used to learn about filenames,
/// line numbers, etc, which are the origin of a function in a stack trace.
#[derive(Debug)]
pub struct FrameSymbol {
name: Option<String>,
file: Option<String>,
line: Option<u32>,
column: Option<u32>,
}
impl FrameSymbol {
/// Returns the function name associated with this symbol.
///
@@ -463,7 +470,7 @@ fn test_frame_info() -> Result<(), anyhow::Error> {
)?;
// Create an instance to ensure the frame information is registered.
Instance::new(&store, &module, &[])?;
let info = store.frame_info().borrow();
let modules = store.modules().borrow();
for (i, alloc) in module.compiled_module().finished_functions() {
let (start, end) = unsafe {
let ptr = (**alloc).as_ptr();
@@ -471,7 +478,7 @@ fn test_frame_info() -> Result<(), anyhow::Error> {
(ptr as usize, ptr as usize + len)
};
for pc in start..end {
let (frame, _) = info.lookup_frame_info(pc).unwrap();
let (frame, _) = modules.lookup_frame_info(pc).unwrap();
assert!(frame.func_index() == i.as_u32());
}
}

227
crates/wasmtime/src/module/serialization.rs
View File

@@ -1,6 +1,6 @@
//! Implements module serialization.
use super::ModuleInner;
use super::{ModuleInner, ModuleSharedSignatures};
use crate::{Engine, Module, OptLevel};
use anyhow::{anyhow, bail, Context, Result};
use bincode::Options;
@@ -10,8 +10,7 @@ use std::fmt;
use std::str::FromStr;
use std::sync::Arc;
use std::{collections::HashMap, fmt::Display};
use wasmtime_environ::Tunables;
use wasmtime_environ::{isa::TargetIsa, settings};
use wasmtime_environ::{isa::TargetIsa, settings, Tunables};
use wasmtime_jit::{
CompilationArtifacts, CompilationStrategy, CompiledModule, Compiler, TypeTables,
};
@@ -123,55 +122,44 @@ impl From<settings::OptLevel> for OptLevel {
}
}
/// A small helper struct which defines modules are serialized.
/// A small helper struct for serialized module upvars.
#[derive(Serialize, Deserialize)]
struct SerializedModuleData<'a> {
/// All compiled artifacts needed by this module, where the last entry in
/// this list is the artifacts for the module itself.
artifacts: Vec<MyCow<'a, CompilationArtifacts>>,
struct SerializedModuleUpvar {
/// The module's index into the compilation artifact.
index: usize,
/// Indexes into the list of all compilation artifacts for this module.
artifact_upvars: Vec<usize>,
/// Closed-over module values that are also needed for this module.
modules: Vec<SerializedModuleData<'a>>,
/// The index into the list of type tables that are used for this module's
/// type tables.
type_tables: usize,
module_upvars: Vec<SerializedModuleUpvar>,
}
impl<'a> SerializedModuleData<'a> {
pub fn new(module: &'a Module) -> (Self, Vec<MyCow<'a, TypeTables>>) {
let mut pushed = HashMap::new();
let mut tables = Vec::new();
return (module_data(module, &mut pushed, &mut tables), tables);
impl SerializedModuleUpvar {
pub fn new(module: &Module, artifacts: &[Arc<CompiledModule>]) -> Self {
// TODO: improve upon the linear searches in the artifact list
let index = artifacts
.iter()
.position(|a| Arc::as_ptr(a) == Arc::as_ptr(&module.inner.module))
.expect("module should be in artifacts list");
fn module_data<'a>(
module: &'a Module,
type_tables_pushed: &mut HashMap<usize, usize>,
type_tables: &mut Vec<MyCow<'a, TypeTables>>,
) -> SerializedModuleData<'a> {
// Deduplicate `Arc<TypeTables>` using our two parameters to ensure we
// serialize type tables as little as possible.
let ptr = Arc::as_ptr(module.types());
let type_tables_idx = *type_tables_pushed.entry(ptr as usize).or_insert_with(|| {
type_tables.push(MyCow::Borrowed(module.types()));
type_tables.len() - 1
});
SerializedModuleData {
artifacts: module
SerializedModuleUpvar {
index,
artifact_upvars: module
.inner
.artifact_upvars
.iter()
.map(|i| MyCow::Borrowed(i.compilation_artifacts()))
.chain(Some(MyCow::Borrowed(
module.compiled_module().compilation_artifacts(),
)))
.map(|m| {
artifacts
.iter()
.position(|a| Arc::as_ptr(a) == Arc::as_ptr(m))
.expect("artifact should be in artifacts list")
})
.collect(),
modules: module
module_upvars: module
.inner
.module_upvars
.iter()
.map(|i| module_data(i, type_tables_pushed, type_tables))
.map(|m| SerializedModuleUpvar::new(m, artifacts))
.collect(),
type_tables: type_tables_idx,
}
}
}
}
@@ -212,14 +200,36 @@ pub struct SerializedModule<'a> {
strategy: CompilationStrategy,
tunables: Tunables,
features: WasmFeatures,
data: SerializedModuleData<'a>,
tables: Vec<MyCow<'a, TypeTables>>,
artifacts: Vec<MyCow<'a, CompilationArtifacts>>,
module_upvars: Vec<SerializedModuleUpvar>,
types: MyCow<'a, TypeTables>,
}
impl<'a> SerializedModule<'a> {
pub fn new(module: &'a Module) -> Self {
let (data, tables) = SerializedModuleData::new(module);
Self::with_data(module.engine().compiler(), data, tables)
let compiler = module.engine().compiler();
let artifacts = module
.inner
.artifact_upvars
.iter()
.map(|m| MyCow::Borrowed(m.compilation_artifacts()))
.chain(Some(MyCow::Borrowed(
module.inner.module.compilation_artifacts(),
)))
.collect::<Vec<_>>();
let module_upvars = module
.inner
.module_upvars
.iter()
.map(|m| SerializedModuleUpvar::new(m, &module.inner.artifact_upvars))
.collect::<Vec<_>>();
Self::with_data(
compiler,
artifacts,
module_upvars,
MyCow::Borrowed(module.types()),
)
}
pub fn from_artifacts(
@@ -229,19 +239,17 @@ impl<'a> SerializedModule<'a> {
) -> Self {
Self::with_data(
compiler,
SerializedModuleData {
artifacts: artifacts.iter().map(MyCow::Borrowed).collect(),
modules: Vec::new(),
type_tables: 0,
},
vec![MyCow::Borrowed(types)],
artifacts.iter().map(MyCow::Borrowed).collect(),
Vec::new(),
MyCow::Borrowed(types),
)
}
fn with_data(
compiler: &Compiler,
data: SerializedModuleData<'a>,
tables: Vec<MyCow<'a, TypeTables>>,
artifacts: Vec<MyCow<'a, CompilationArtifacts>>,
module_upvars: Vec<SerializedModuleUpvar>,
types: MyCow<'a, TypeTables>,
) -> Self {
let isa = compiler.isa();
@@ -260,8 +268,9 @@ impl<'a> SerializedModule<'a> {
strategy: compiler.strategy(),
tunables: compiler.tunables().clone(),
features: compiler.features().into(),
data,
tables,
artifacts,
module_upvars,
types,
}
}
@@ -276,45 +285,95 @@ impl<'a> SerializedModule<'a> {
self.check_tunables(compiler)?;
self.check_features(compiler)?;
let types = self
.tables
.into_iter()
.map(|t| Arc::new(t.unwrap_owned()))
.collect::<Vec<_>>();
let module = mk(engine, &types, self.data)?;
// Validate the module can be used with the current allocator
engine.allocator().validate(module.inner.module.module())?;
return Ok(module);
fn mk(
engine: &Engine,
types: &Vec<Arc<TypeTables>>,
data: SerializedModuleData<'_>,
) -> Result<Module> {
let mut artifacts = CompiledModule::from_artifacts_list(
data.artifacts
let types = Arc::new(self.types.unwrap_owned());
let mut modules = CompiledModule::from_artifacts_list(
self.artifacts
.into_iter()
.map(|i| i.unwrap_owned())
.collect(),
engine.compiler().isa(),
&*engine.config().profiler,
)?;
let inner = ModuleInner {
engine: engine.clone(),
types: types[data.type_tables].clone(),
module: artifacts.pop().unwrap(),
artifact_upvars: artifacts,
module_upvars: data
.modules
.into_iter()
.map(|m| mk(engine, types, m))
.collect::<Result<Vec<_>>>()?,
};
// Validate the module can be used with the current allocator
engine
.allocator()
.validate(modules.last().unwrap().module())?;
let (signatures, trampolines) = engine.register_module_signatures(
&types.wasm_signatures,
modules.iter().flat_map(|m| m.trampolines().iter().cloned()),
);
let signatures = Arc::new(ModuleSharedSignatures {
engine: engine.clone(),
signatures,
trampolines,
});
let module = modules.pop().unwrap();
let module_upvars = self
.module_upvars
.iter()
.map(|m| {
mk(
engine,
&modules,
&types,
m.index,
&m.artifact_upvars,
&m.module_upvars,
&signatures,
)
})
.collect::<Result<Vec<_>>>()?;
return Ok(Module {
inner: Arc::new(ModuleInner {
engine: engine.clone(),
types,
module,
artifact_upvars: modules,
module_upvars,
signatures,
}),
});
fn mk(
engine: &Engine,
artifacts: &[Arc<CompiledModule>],
types: &Arc<TypeTables>,
module_index: usize,
artifact_upvars: &[usize],
module_upvars: &[SerializedModuleUpvar],
signatures: &Arc<ModuleSharedSignatures>,
) -> Result<Module> {
Ok(Module {
inner: Arc::new(inner),
inner: Arc::new(ModuleInner {
engine: engine.clone(),
types: types.clone(),
module: artifacts[module_index].clone(),
artifact_upvars: artifact_upvars
.iter()
.map(|i| artifacts[*i].clone())
.collect(),
module_upvars: module_upvars
.into_iter()
.map(|m| {
mk(
engine,
artifacts,
types,
m.index,
&m.artifact_upvars,
&m.module_upvars,
signatures,
)
})
.collect::<Result<Vec<_>>>()?,
signatures: signatures.clone(),
}),
})
}
}

155
crates/wasmtime/src/sig_registry.rs
View File

@@ -1,155 +0,0 @@
//! Implement a registry of function signatures, for fast indirect call
//! signature checking.
use crate::Module;
use std::collections::{hash_map, HashMap};
use std::convert::TryFrom;
use wasmtime_environ::entity::PrimaryMap;
use wasmtime_environ::wasm::{SignatureIndex, WasmFuncType};
use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline};
/// WebAssembly requires that the caller and callee signatures in an indirect
/// call must match. To implement this efficiently, keep a registry of all
/// signatures, shared by all instances, so that call sites can just do an
/// index comparison.
#[derive(Debug, Default)]
pub struct SignatureRegistry {
// Map from a wasm actual function type to the index that it is assigned,
// shared amongst all wasm modules.
wasm2index: HashMap<WasmFuncType, VMSharedSignatureIndex>,
// Map of all known wasm function signatures in this registry. This is
// keyed by `VMSharedSignatureIndex` above.
index_map: Vec<Entry>,
}
#[derive(Debug)]
struct Entry {
// The WebAssembly type signature, using wasm types.
wasm: WasmFuncType,
// The native trampoline used to invoke this type signature from `Func`.
// Note that the code memory for this trampoline is not owned by this
// type, but instead it's expected to be owned by the store that this
// registry lives within.
trampoline: Option<VMTrampoline>,
}
impl SignatureRegistry {
/// Registers all signatures within a module into this registry all at once.
///
/// This will also internally register trampolines compiled in the module.
pub fn register_module(&mut self, module: &Module) {
// Register a unique index for all types in this module, even if they
// don't have a trampoline.
let signatures = &module.types().wasm_signatures;
for ty in module.compiled_module().module().types.values() {
if let wasmtime_environ::ModuleType::Function(index) = ty {
self.register_one(&signatures[*index], None);
}
}
// Once we've got a shared index for all types used then also fill in
// any trampolines that the module has compiled as well.
for (index, trampoline) in module.compiled_module().trampolines() {
let shared = self.wasm2index[&signatures[*index]];
let entry = &mut self.index_map[shared.bits() as usize];
if entry.trampoline.is_none() {
entry.trampoline = Some(*trampoline);
}
}
}
/// Register a signature and return its unique index.
pub fn register(
&mut self,
wasm: &WasmFuncType,
trampoline: VMTrampoline,
) -> VMSharedSignatureIndex {
self.register_one(wasm, Some(trampoline))
}
fn register_one(
&mut self,
wasm: &WasmFuncType,
trampoline: Option<VMTrampoline>,
) -> VMSharedSignatureIndex {
let len = self.wasm2index.len();
match self.wasm2index.entry(wasm.clone()) {
hash_map::Entry::Occupied(entry) => {
let ret = *entry.get();
let entry = &mut self.index_map[ret.bits() as usize];
// If the entry does not previously have a trampoline, then
// overwrite it with whatever was specified by this function.
if entry.trampoline.is_none() {
entry.trampoline = trampoline;
}
ret
}
hash_map::Entry::Vacant(entry) => {
// Keep `signature_hash` len under 2**32 -- VMSharedSignatureIndex::new(std::u32::MAX)
// is reserved for VMSharedSignatureIndex::default().
assert!(
len < std::u32::MAX as usize,
"Invariant check: signature_hash.len() < std::u32::MAX"
);
debug_assert_eq!(len, self.index_map.len());
let index = VMSharedSignatureIndex::new(u32::try_from(len).unwrap());
self.index_map.push(Entry {
wasm: wasm.clone(),
trampoline,
});
entry.insert(index);
index
}
}
}
/// Looks up a shared index from the wasm signature itself.
pub fn lookup(&self, wasm: &WasmFuncType) -> Option<VMSharedSignatureIndex> {
self.wasm2index.get(wasm).cloned()
}
/// Builds a lookup table for a module from the possible module's signature
/// indices to the shared signature index within this registry.
pub fn lookup_table(
&self,
module: &Module,
) -> PrimaryMap<SignatureIndex, VMSharedSignatureIndex> {
// For module-linking using modules this builds up a map that is
// too large. This builds up a map for everything in `TypeTables` but
// that's all the types for all modules in a whole module linking graph,
// which our `module` may not be using.
//
// For all non-module-linking-using modules, though, this is not an
// issue. This is optimizing for the non-module-linking case right now
// and it seems like module linking will likely change to the point that
// this will no longer be an issue in the future.
let signatures = &module.types().wasm_signatures;
let mut map = PrimaryMap::with_capacity(signatures.len());
for wasm in signatures.values() {
map.push(
self.wasm2index
.get(wasm)
.cloned()
.unwrap_or(VMSharedSignatureIndex::new(u32::MAX)),
);
}
map
}
/// Looks up information known about a shared signature index.
///
/// Note that for this operation to be semantically correct the `idx` must
/// have previously come from a call to `register` of this same object.
pub fn lookup_shared(
&self,
idx: VMSharedSignatureIndex,
) -> Option<(&WasmFuncType, VMTrampoline)> {
let (wasm, trampoline) = self
.index_map
.get(idx.bits() as usize)
.map(|e| (&e.wasm, e.trampoline))?;
Some((wasm, trampoline?))
}
}

185
crates/wasmtime/src/signatures.rs Normal file
View File

@@ -0,0 +1,185 @@
//! Implement a registry of function signatures, for fast indirect call
//! signature checking.
use std::collections::{hash_map::Entry, HashMap};
use std::convert::TryFrom;
use wasmtime_environ::entity::PrimaryMap;
use wasmtime_environ::wasm::{SignatureIndex, WasmFuncType};
use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline};
/// Represents a mapping of shared signature index to trampolines.
///
/// This is used in various places to store trampolines associated with shared
/// signature indexes.
///
/// As multiple trampolines may exist for a single signature, the map entries
/// are internally reference counted.
#[derive(Default)]
pub struct TrampolineMap(HashMap<VMSharedSignatureIndex, (usize, VMTrampoline)>);
impl TrampolineMap {
/// Inserts a trampoline into the map.
pub fn insert(&mut self, index: VMSharedSignatureIndex, trampoline: VMTrampoline) {
let entry = match self.0.entry(index) {
Entry::Occupied(e) => e.into_mut(),
Entry::Vacant(e) => e.insert((0, trampoline)),
};
// Increment the ref count
entry.0 += 1;
}
/// Gets a trampoline from the map.
pub fn get(&self, index: VMSharedSignatureIndex) -> Option<VMTrampoline> {
self.0.get(&index).map(|(_, trampoline)| *trampoline)
}
/// Iterates the shared signature indexes stored in the map.
///
/// A shared signature index will be returned by the iterator for every
/// trampoline registered for that index, so duplicates may be present.
///
/// This iterator can be used for deregistering signatures with the
/// signature registry.
pub fn indexes<'a>(&'a self) -> impl Iterator<Item = VMSharedSignatureIndex> + 'a {
self.0
.iter()
.flat_map(|(index, (count, _))| std::iter::repeat(*index).take(*count))
}
/// Determines if the trampoline map is empty.
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
}
/// Represents a map between module signature indexes and
/// shared signature indexes.
pub type SharedSignatures = PrimaryMap<SignatureIndex, VMSharedSignatureIndex>;
#[derive(Debug)]
struct RegistryEntry {
references: usize,
ty: WasmFuncType,
}
/// WebAssembly requires that the caller and callee signatures in an indirect
/// call must match. To implement this efficiently, keep a registry of all
/// signatures, shared by all instances, so that call sites can just do an
/// index comparison.
#[derive(Debug, Default)]
pub struct SignatureRegistry {
map: HashMap<WasmFuncType, VMSharedSignatureIndex>,
entries: Vec<Option<RegistryEntry>>,
free: Vec<VMSharedSignatureIndex>,
}
impl SignatureRegistry {
/// Registers a module with the signature registry from the collection of
/// all signatures and trampolines in the module.
pub fn register_module(
&mut self,
signatures: &PrimaryMap<SignatureIndex, WasmFuncType>,
trampolines: impl Iterator<Item = (SignatureIndex, VMTrampoline)>,
) -> (SharedSignatures, TrampolineMap) {
let mut sigs = SharedSignatures::default();
let mut map = TrampolineMap::default();
for (_, ty) in signatures.iter() {
sigs.push(self.register(ty));
}
for (index, trampoline) in trampolines {
let index = self.map[&signatures[index]];
map.insert(index, trampoline);
}
(sigs, map)
}
/// Registers a single signature with the registry.
///
/// This is used for registering host functions created with the Wasmtime API.
pub fn register(&mut self, ty: &WasmFuncType) -> VMSharedSignatureIndex {
let len = self.map.len();
let index = match self.map.entry(ty.clone()) {
Entry::Occupied(e) => *e.get(),
Entry::Vacant(e) => {
let (index, entry) = match self.free.pop() {
Some(index) => (index, &mut self.entries[index.bits() as usize]),
None => {
// Keep `index_map` len under 2**32 -- VMSharedSignatureIndex::new(std::u32::MAX)
// is reserved for VMSharedSignatureIndex::default().
assert!(
len < std::u32::MAX as usize,
"Invariant check: index_map.len() < std::u32::MAX"
);
debug_assert_eq!(len, self.entries.len());
let index = VMSharedSignatureIndex::new(u32::try_from(len).unwrap());
self.entries.push(None);
(index, self.entries.last_mut().unwrap())
}
};
*entry = Some(RegistryEntry {
references: 0,
ty: ty.clone(),
});
*e.insert(index)
}
};
self.entries[index.bits() as usize]
.as_mut()
.unwrap()
.references += 1;
index
}
/// Unregisters a collection of shared indexes from the registry.
pub fn unregister(&mut self, indexes: impl Iterator<Item = VMSharedSignatureIndex>) {
for index in indexes {
let removed = {
let entry = self.entries[index.bits() as usize].as_mut().unwrap();
debug_assert!(entry.references > 0);
entry.references -= 1;
if entry.references == 0 {
self.map.remove(&entry.ty);
self.free.push(index);
true
} else {
false
}
};
if removed {
self.entries[index.bits() as usize] = None;
}
}
}
/// Looks up a function type from a shared signature index.
pub fn lookup_type(&self, index: VMSharedSignatureIndex) -> Option<&WasmFuncType> {
self.entries
.get(index.bits() as usize)
.and_then(|e| e.as_ref().map(|e| &e.ty))
}
/// Determines if the registry is semantically empty.
pub fn is_empty(&self) -> bool {
// If the map is empty, assert that all remaining entries are "free"
if self.map.is_empty() {
assert!(self.free.len() == self.entries.len());
true
} else {
false
}
}
}

152
crates/wasmtime/src/store.rs
View File

@@ -1,12 +1,11 @@
use crate::frame_info;
use crate::frame_info::StoreFrameInfo;
use crate::sig_registry::SignatureRegistry;
use crate::trampoline::StoreInstanceHandle;
use crate::{Engine, Func, FuncType, Module, Trap};
use crate::{
module::ModuleRegistry, signatures::TrampolineMap, trampoline::StoreInstanceHandle, Engine,
Func, Module, Trap,
};
use anyhow::{bail, Result};
use std::any::{Any, TypeId};
use std::cell::{Cell, RefCell};
use std::collections::{hash_map::Entry, HashMap, HashSet};
use std::collections::{hash_map::Entry, HashMap};
use std::convert::TryFrom;
use std::fmt;
use std::future::Future;
@@ -16,12 +15,12 @@ use std::ptr;
use std::rc::Rc;
use std::sync::Arc;
use std::task::{Context, Poll};
use wasmtime_environ::wasm;
use wasmtime_environ::wasm::WasmFuncType;
use wasmtime_jit::{CompiledModule, ModuleCode};
use wasmtime_runtime::{
Export, InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler,
StackMapRegistry, TrapInfo, VMCallerCheckedAnyfunc, VMContext, VMExternRef,
VMExternRefActivationsTable, VMInterrupts, VMTrampoline,
InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler, StackMapRegistry,
TrapInfo, VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts,
VMSharedSignatureIndex, VMTrampoline,
};
/// Used to associate instances with the store.
@@ -72,20 +71,13 @@ pub struct Store {
pub(crate) struct StoreInner {
engine: Engine,
/// The map of all host functions registered with this store's signature registry
host_funcs: RefCell<HashMap<InstanceHandle, Box<VMCallerCheckedAnyfunc>>>,
interrupts: Arc<VMInterrupts>,
signatures: RefCell<SignatureRegistry>,
instances: RefCell<Vec<StoreInstance>>,
signal_handler: RefCell<Option<Box<SignalHandler<'static>>>>,
externref_activations_table: VMExternRefActivationsTable,
stack_map_registry: StackMapRegistry,
/// Information about JIT code which allows us to test if a program counter
/// is in JIT code, lookup trap information, etc.
frame_info: RefCell<StoreFrameInfo>,
/// Set of all compiled modules that we're holding a strong reference to
/// the module's code for. This includes JIT functions, trampolines, etc.
modules: RefCell<HashSet<ArcModuleCode>>,
modules: RefCell<ModuleRegistry>,
trampolines: RefCell<TrampolineMap>,
// Numbers of resources instantiated in this store.
instance_count: Cell<usize>,
memory_count: Cell<usize>,
@@ -137,21 +129,19 @@ impl Store {
// once-per-thread. Platforms like Unix, however, only require this
// once-per-program. In any case this is safe to call many times and
// each one that's not relevant just won't do anything.
wasmtime_runtime::init_traps(frame_info::GlobalFrameInfo::is_wasm_pc)
wasmtime_runtime::init_traps(crate::module::GlobalModuleRegistry::is_wasm_pc)
.expect("failed to initialize trap handling");
Store {
inner: Rc::new(StoreInner {
engine: engine.clone(),
host_funcs: RefCell::new(HashMap::new()),
interrupts: Arc::new(Default::default()),
signatures: RefCell::new(Default::default()),
instances: RefCell::new(Vec::new()),
signal_handler: RefCell::new(None),
externref_activations_table: VMExternRefActivationsTable::new(),
stack_map_registry: StackMapRegistry::default(),
frame_info: Default::default(),
modules: Default::default(),
modules: RefCell::new(ModuleRegistry::default()),
trampolines: RefCell::new(TrampolineMap::default()),
instance_count: Default::default(),
memory_count: Default::default(),
table_count: Default::default(),
@@ -181,35 +171,6 @@ impl Store {
})
}
pub(crate) fn get_host_anyfunc(
&self,
instance: &InstanceHandle,
ty: &FuncType,
trampoline: VMTrampoline,
) -> *mut VMCallerCheckedAnyfunc {
let mut funcs = self.inner.host_funcs.borrow_mut();
let anyfunc = funcs.entry(unsafe { instance.clone() }).or_insert_with(|| {
let mut anyfunc = match instance
.lookup_by_declaration(&wasm::EntityIndex::Function(wasm::FuncIndex::from_u32(0)))
{
Export::Function(f) => unsafe { f.anyfunc.as_ref() }.clone(),
_ => unreachable!(),
};
// Register the function with this store's signature registry
anyfunc.type_index = self
.inner
.signatures
.borrow_mut()
.register(ty.as_wasm_func_type(), trampoline);
Box::new(anyfunc)
});
&mut **anyfunc
}
/// Returns the [`Engine`] that this store is associated with.
#[inline]
pub fn engine(&self) -> &Engine {
@@ -244,52 +205,16 @@ impl Store {
}
}
pub(crate) fn signatures(&self) -> &RefCell<SignatureRegistry> {
&self.inner.signatures
}
pub(crate) fn register_module(&self, module: &Module) {
// With a module being instantiated into this `Store` we need to
// preserve its jit-code. References to this module's code and
// trampolines are not owning-references so it's our responsibility to
// keep it all alive within the `Store`.
//
// If this module is already present in the store then we skip all
// further registration steps.
let first = self
.inner
.modules
.borrow_mut()
.insert(ArcModuleCode(module.compiled_module().code().clone()));
if !first {
// Register the module with the registry
if !self.inner.modules.borrow_mut().register(module) {
return;
}
// All modules register their JIT code in a store for two reasons
// currently:
//
// * First we only catch signals/traps if the program counter falls
// within the jit code of an instantiated wasm module. This ensures
// we don't catch accidental Rust/host segfaults.
//
// * Second when generating a backtrace we'll use this mapping to
// only generate wasm frames for instruction pointers that fall
// within jit code.
self.inner
.frame_info
.borrow_mut()
.register(module.compiled_module());
// We need to know about all the stack maps of all instantiated modules
// so when performing a GC we know about all wasm frames that we find
// on the stack.
self.register_stack_maps(module.compiled_module());
// Signatures are loaded into our `SignatureRegistry` here
// once-per-module (and once-per-signature). This allows us to create
// a `Func` wrapper for any function in the module, which requires that
// we know about the signature and trampoline for all instances.
self.signatures().borrow_mut().register_module(module);
}
fn register_stack_maps(&self, module: &CompiledModule) {
@@ -304,6 +229,39 @@ impl Store {
}));
}
// This is used to register a `Func` with the store
pub(crate) fn register_signature(
&self,
ty: &WasmFuncType,
trampoline: VMTrampoline,
) -> VMSharedSignatureIndex {
let index = self.inner.engine.register_signature(ty);
self.inner
.trampolines
.borrow_mut()
.insert(index, trampoline);
index
}
pub(crate) fn lookup_trampoline(&self, index: VMSharedSignatureIndex) -> VMTrampoline {
// Look up the trampoline with the store's trampolines (from `Func`).
if let Some(trampoline) = self.inner.trampolines.borrow().get(index) {
return trampoline;
}
// Look up the trampoline with the registered modules
if let Some(trampoline) = self.inner.modules.borrow().lookup_trampoline(index) {
return trampoline;
}
// Lastly, check with the engine (for `HostFunc`)
self.inner
.engine
.host_func_trampolines()
.get(index)
.expect("trampoline missing")
}
pub(crate) fn bump_resource_counts(&self, module: &Module) -> Result<()> {
let config = self.engine().config();
@@ -363,7 +321,7 @@ impl Store {
.borrow()
.iter()
.any(|i| i.handle.vmctx_ptr() == handle.vmctx_ptr())
|| self.inner.host_funcs.borrow().get(&handle).is_some()
|| self.inner.engine.host_func_anyfunc(&handle).is_some()
);
StoreInstanceHandle {
store: self.clone(),
@@ -494,8 +452,8 @@ impl Store {
&self.inner.stack_map_registry
}
pub(crate) fn frame_info(&self) -> &RefCell<StoreFrameInfo> {
&self.inner.frame_info
pub(crate) fn modules(&self) -> &RefCell<ModuleRegistry> {
&self.inner.modules
}
/// Notifies that the current Store (and all referenced entities) has been moved over to a
@@ -984,6 +942,12 @@ impl Drop for StoreInner {
}
}
}
let trampolines = self.trampolines.borrow();
if !trampolines.is_empty() {
self.engine.unregister_signatures(trampolines.indexes());
}
}
}

10
crates/wasmtime/src/trampoline/func.rs
View File

@@ -1,6 +1,6 @@
//! Support for a calling of an imported function.
use crate::{sig_registry::SignatureRegistry, Config, FuncType, Trap};
use crate::{Config, FuncType, Store, Trap};
use anyhow::Result;
use std::any::Any;
use std::cmp;
@@ -262,15 +262,15 @@ pub fn create_function(
ft: &FuncType,
func: Box<dyn Fn(*mut VMContext, *mut u128) -> Result<(), Trap>>,
config: &Config,
registry: Option<&mut SignatureRegistry>,
store: Option<&Store>,
) -> Result<(InstanceHandle, VMTrampoline)> {
let (module, finished_functions, trampoline, trampoline_state) =
create_function_trampoline(config, ft, func)?;
// If there is no signature registry, use the default signature index which is
// If there is no store, use the default signature index which is
// guaranteed to trap if there is ever an indirect call on the function (should not happen)
let shared_signature_id = registry
.map(|r| r.register(ft.as_wasm_func_type(), trampoline))
let shared_signature_id = store
.map(|s| s.register_signature(ft.as_wasm_func_type(), trampoline))
.unwrap_or(VMSharedSignatureIndex::default());
unsafe {

4
crates/wasmtime/src/trap.rs
View File

@@ -161,7 +161,7 @@ impl Trap {
maybe_interrupted,
} => {
let mut code = store
.frame_info()
.modules()
.borrow()
.lookup_trap_info(pc)
.map(|info| info.trap_code)
@@ -239,7 +239,7 @@ impl Trap {
// (the call instruction) so we subtract one as the lookup.
let pc_to_lookup = if Some(pc) == trap_pc { pc } else { pc - 1 };
if let Some((info, has_unparsed_debuginfo)) =
store.frame_info().borrow().lookup_frame_info(pc_to_lookup)
store.modules().borrow().lookup_frame_info(pc_to_lookup)
{
wasm_trace.push(info);

7
crates/wasmtime/src/types.rs
View File

@@ -204,8 +204,7 @@ impl ExternType {
) -> ExternType {
match ty {
EntityType::Function(idx) => {
let sig = &types.wasm_signatures[*idx];
FuncType::from_wasm_func_type(sig).into()
FuncType::from_wasm_func_type(types.wasm_signatures[*idx].clone()).into()
}
EntityType::Global(ty) => GlobalType::from_wasmtime_global(ty).into(),
EntityType::Memory(ty) => MemoryType::from_wasmtime_memory(ty).into(),
@@ -298,8 +297,8 @@ impl FuncType {
&self.sig
}
pub(crate) fn from_wasm_func_type(sig: &wasm::WasmFuncType) -> FuncType {
FuncType { sig: sig.clone() }
pub(crate) fn from_wasm_func_type(sig: wasm::WasmFuncType) -> FuncType {
Self { sig }
}
}

42
crates/wasmtime/src/types/matching.rs
View File

@@ -1,4 +1,4 @@
use crate::{Extern, Store};
use crate::{signatures::SharedSignatures, Extern, Store};
use anyhow::{bail, Context, Result};
use wasmtime_environ::wasm::{
EntityType, Global, InstanceTypeIndex, Memory, ModuleTypeIndex, SignatureIndex, Table,
@@ -6,6 +6,7 @@ use wasmtime_environ::wasm::{
use wasmtime_jit::TypeTables;
pub struct MatchCx<'a> {
pub signatures: &'a SharedSignatures,
pub types: &'a TypeTables,
pub store: &'a Store,
}
@@ -70,13 +71,8 @@ impl MatchCx<'_> {
}
pub fn func(&self, expected: SignatureIndex, actual: &crate::Func) -> Result<()> {
let matches = match self
.store
.signatures()
.borrow()
.lookup(&self.types.wasm_signatures[expected])
{
Some(idx) => actual.sig_index() == idx,
let matches = match self.signatures.get(expected) {
Some(idx) => actual.sig_index() == *idx,
// If our expected signature isn't registered, then there's no way
// that `actual` can match it.
None => false,
@@ -114,15 +110,19 @@ impl MatchCx<'_> {
let module = actual.compiled_module().module();
self.imports_match(
expected,
actual.signatures(),
actual.types(),
module.imports().map(|(name, field, ty)| {
assert!(field.is_none()); // should be true if module linking is enabled
(name, ty)
}),
)?;
self.exports_match(expected_sig.exports, actual.types(), |name| {
module.exports.get(name).map(|idx| module.type_of(*idx))
})?;
self.exports_match(
expected_sig.exports,
actual.signatures(),
actual.types(),
|name| module.exports.get(name).map(|idx| module.type_of(*idx)),
)?;
Ok(())
}
@@ -133,6 +133,7 @@ impl MatchCx<'_> {
fn imports_match<'a>(
&self,
expected: ModuleTypeIndex,
actual_signatures: &SharedSignatures,
actual_types: &TypeTables,
actual_imports: impl Iterator<Item = (&'a str, EntityType)>,
) -> Result<()> {
@@ -146,10 +147,11 @@ impl MatchCx<'_> {
None => bail!("expected type doesn't import {:?}", name),
};
MatchCx {
signatures: actual_signatures,
types: actual_types,
store: self.store,
}
.extern_ty_matches(&actual_ty, expected_ty, self.types)
.extern_ty_matches(&actual_ty, expected_ty, self.signatures, self.types)
.with_context(|| format!("module import {:?} incompatible", name))?;
}
Ok(())
@@ -160,6 +162,7 @@ impl MatchCx<'_> {
fn exports_match(
&self,
expected: InstanceTypeIndex,
actual_signatures: &SharedSignatures,
actual_types: &TypeTables,
lookup: impl Fn(&str) -> Option<EntityType>,
) -> Result<()> {
@@ -169,7 +172,7 @@ impl MatchCx<'_> {
for (name, expected) in self.types.instance_signatures[expected].exports.iter() {
match lookup(name) {
Some(ty) => self
.extern_ty_matches(expected, &ty, actual_types)
.extern_ty_matches(expected, &ty, actual_signatures, actual_types)
.with_context(|| format!("export {:?} incompatible", name))?,
None => bail!("failed to find export {:?}", name),
}
@@ -183,6 +186,7 @@ impl MatchCx<'_> {
&self,
expected: &EntityType,
actual_ty: &EntityType,
actual_signatures: &SharedSignatures,
actual_types: &TypeTables,
) -> Result<()> {
let actual_desc = match actual_ty {
@@ -221,7 +225,7 @@ impl MatchCx<'_> {
EntityType::Instance(expected) => match actual_ty {
EntityType::Instance(actual) => {
let sig = &actual_types.instance_signatures[*actual];
self.exports_match(*expected, actual_types, |name| {
self.exports_match(*expected, actual_signatures, actual_types, |name| {
sig.exports.get(name).cloned()
})?;
Ok(())
@@ -237,15 +241,19 @@ impl MatchCx<'_> {
self.imports_match(
*expected,
actual_signatures,
actual_types,
actual_module_sig
.imports
.iter()
.map(|(module, ty)| (module.as_str(), ty.clone())),
)?;
self.exports_match(expected_module_sig.exports, actual_types, |name| {
actual_instance_sig.exports.get(name).cloned()
})?;
self.exports_match(
expected_module_sig.exports,
actual_signatures,
actual_types,
|name| actual_instance_sig.exports.get(name).cloned(),
)?;
Ok(())
}
_ => bail!("expected module, but found {}", actual_desc),