diff --git a/Cargo.lock b/Cargo.lock index 2f560c2df1..5ef2a1aec4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3514,7 +3514,9 @@ dependencies = [ "serde", "target-lexicon", "thiserror", + "wasm-encoder", "wasmparser", + "wasmprinter", "wasmtime-types", ] diff --git a/cranelift/entity/src/primary.rs b/cranelift/entity/src/primary.rs index f35c6f44a6..55b923a0e3 100644 --- a/cranelift/entity/src/primary.rs +++ b/cranelift/entity/src/primary.rs @@ -125,8 +125,17 @@ where } /// Returns the last element that was inserted in the map. - pub fn last(&self) -> Option<&V> { - self.elems.last() + pub fn last(&self) -> Option<(K, &V)> { + let len = self.elems.len(); + let last = self.elems.last()?; + Some((K::new(len - 1), last)) + } + + /// Returns the last element that was inserted in the map. + pub fn last_mut(&mut self) -> Option<(K, &mut V)> { + let len = self.elems.len(); + let last = self.elems.last_mut()?; + Some((K::new(len - 1), last)) } /// Reserves capacity for at least `additional` more elements to be inserted. diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs index a55867e766..f45862f0c8 100644 --- a/crates/cranelift/src/compiler/component.rs +++ b/crates/cranelift/src/compiler/component.rs @@ -72,12 +72,12 @@ impl ComponentCompiler for Compiler { i32::try_from(offsets.lowering_data(lowering.index)).unwrap(), )); - // flags: *mut VMComponentFlags + // flags: *mut VMGlobalDefinition host_sig.params.push(ir::AbiParam::new(pointer_type)); callee_args.push( builder .ins() - .iadd_imm(vmctx, i64::from(offsets.flags(instance))), + .iadd_imm(vmctx, i64::from(offsets.instance_flags(instance))), ); // memory: *mut VMMemoryDefinition diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 7d5131fa24..dd9a5df7a7 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -23,9 +23,11 @@ more-asserts = "0.2.1" gimli = { version = "0.26.0", default-features = false, features = ['read'] } object = { version = "0.29.0", default-features = false, features = ['read_core', 'write_core', 'elf'] } target-lexicon = "0.12" +wasm-encoder = { version = "0.14.0", optional = true } +wasmprinter = { version = "0.2.37", optional = true } [badges] maintenance = { status = "actively-developed" } [features] -component-model = [] +component-model = ["dep:wasm-encoder", "dep:wasmprinter"] diff --git a/crates/environ/src/component.rs b/crates/environ/src/component.rs index 20bb2ec993..e130ce8206 100644 --- a/crates/environ/src/component.rs +++ b/crates/environ/src/component.rs @@ -26,6 +26,17 @@ //! any time. Some comments may reflect historical rather than current state as //! well (sorry). +/// Canonical ABI-defined constant for the maximum number of "flat" parameters +/// to a wasm function, or the maximum number of parameters a core wasm function +/// will take for just the parameters used. Over this number the heap is used +/// for transferring parameters. +pub const MAX_FLAT_PARAMS: usize = 16; + +/// Canonical ABI-defined constant for the maximum number of "flat" results. +/// This number of results are returned directly from wasm and otherwise results +/// are transferred through memory. +pub const MAX_FLAT_RESULTS: usize = 1; + mod compiler; mod info; mod translate; diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rs index b84bd3bebb..85eac6814a 100644 --- a/crates/environ/src/component/info.rs +++ b/crates/environ/src/component/info.rs @@ -313,6 +313,20 @@ pub enum CoreDef { /// function is immediately `canon lower`'d in the same instance. Such a /// function always traps at runtime. AlwaysTrap(RuntimeAlwaysTrapIndex), + /// This refers to a core wasm function which is a synthesized fused adapter + /// between two other core wasm functions. + /// + /// The adapter's information is identified by `AdapterIndex` which is + /// available through an auxiliary map created during compilation of a + /// component. For more information see `adapt.rs`. + /// + /// Note that this is an intermediate variant which is replaced by the time + /// a component is fully compiled. This will be replaced with the `Export` + /// variant which refers to the export of an adapter module. + Adapter(AdapterIndex), + /// This is a reference to a wasm global which represents the + /// runtime-managed flags for a wasm instance. + InstanceFlags(RuntimeComponentInstanceIndex), } impl From> for CoreDef diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rs index f4ebc98492..a2ba465b88 100644 --- a/crates/environ/src/component/translate.rs +++ b/crates/environ/src/component/translate.rs @@ -1,4 +1,5 @@ use crate::component::*; +use crate::ScopeVec; use crate::{ EntityIndex, ModuleEnvironment, ModuleTranslation, PrimaryMap, SignatureIndex, Tunables, }; @@ -8,6 +9,8 @@ use std::collections::HashMap; use std::mem; use wasmparser::{Chunk, Encoding, Parser, Payload, Validator}; +mod adapt; +pub use self::adapt::*; mod inline; /// Structure used to translate a component and parse it. @@ -44,6 +47,9 @@ pub struct Translator<'a, 'data> { /// The compiler configuration provided by the embedder. tunables: &'a Tunables, + /// Auxiliary location to push generated adapter modules onto. + scope_vec: &'data ScopeVec, + /// Completely translated core wasm modules that have been found so far. /// /// Note that this translation only involves learning about type @@ -275,6 +281,7 @@ impl<'a, 'data> Translator<'a, 'data> { tunables: &'a Tunables, validator: &'a mut Validator, types: &'a mut ComponentTypesBuilder, + scope_vec: &'data ScopeVec, ) -> Self { Self { result: Translation::default(), @@ -286,6 +293,7 @@ impl<'a, 'data> Translator<'a, 'data> { static_components: Default::default(), static_modules: Default::default(), synthetic_instance_types: Default::default(), + scope_vec, } } @@ -353,12 +361,13 @@ impl<'a, 'data> Translator<'a, 'data> { // much simpler than the original component and more efficient for // Wasmtime to process at runtime as well (e.g. no string lookups as // most everything is done through indices instead). - let component = inline::run( + let (mut component, mut adapters) = inline::run( &self.types, &self.result, &self.static_modules, &self.static_components, )?; + self.insert_adapter_module_initializers(&mut component, &mut adapters); Ok((component, self.static_modules)) } diff --git a/crates/environ/src/component/translate/adapt.rs b/crates/environ/src/component/translate/adapt.rs new file mode 100644 index 0000000000..e097e05fc2 --- /dev/null +++ b/crates/environ/src/component/translate/adapt.rs @@ -0,0 +1,657 @@ +//! Identification and creation of fused adapter modules in Wasmtime. +//! +//! A major piece of the component model is the ability for core wasm modules to +//! talk to each other through the use of lifted and lowered functions. For +//! example one core wasm module can export a function which is lifted. Another +//! component could import that lifted function, lower it, and pass it as the +//! import to another core wasm module. This is what Wasmtime calls "adapter +//! fusion" where two core wasm functions are coming together through the +//! component model. +//! +//! There are a few ingredients during adapter fusion: +//! +//! * A core wasm function which is "lifted". +//! * A "lift type" which is the type that the component model function had in +//! the original component +//! * A "lower type" which is the type that the component model function has +//! in the destination component (the one the uses `canon lower`) +//! * Configuration options for both the lift and the lower operations such as +//! memories, reallocs, etc. +//! +//! With these ingredients combined Wasmtime must produce a function which +//! connects the two components through the options specified. The fused adapter +//! performs tasks such as validation of passed values, copying data between +//! linear memories, etc. +//! +//! Wasmtime's current implementation of fused adapters is designed to reduce +//! complexity elsewhere as much as possible while also being suitable for being +//! used as a polyfill for the component model in JS environments as well. To +//! that end Wasmtime implements a fused adapter with another wasm module that +//! it itself generates on the fly. The usage of WebAssembly for fused adapters +//! has a number of advantages: +//! +//! * There is no need to create a raw Cranelift-based compiler. This is where +//! majority of "unsafety" lives in Wasmtime so reducing the need to lean on +//! this or audit another compiler is predicted to weed out a whole class of +//! bugs in the fused adapter compiler. +//! +//! * As mentioned above generation of WebAssembly modules means that this is +//! suitable for use in JS environments. For example a hypothetical tool which +//! polyfills a component onto the web today would need to do something for +//! adapter modules, and ideally the adapters themselves are speedy. While +//! this could all be written in JS the adapting process is quite nontrivial +//! so sharing code with Wasmtime would be ideal. +//! +//! * Using WebAssembly insulates the implementation to bugs to a certain +//! degree. While logic bugs are still possible it should be much more +//! difficult to have segfaults or things like that. With adapters exclusively +//! executing inside a WebAssembly sandbox like everything else the failure +//! modes to the host at least should be minimized. +//! +//! * Integration into the runtime is relatively simple, the adapter modules are +//! just another kind of wasm module to instantiate and wire up at runtime. +//! The goal is that the `GlobalInitializer` list that is processed at runtime +//! will have all of its `Adapter`-using variants erased by the time it makes +//! its way all the way up to Wasmtime. This means that the support in +//! Wasmtime prior to adapter modules is actually the same as the support +//! after adapter modules are added, keeping the runtime fiddly bits quite +//! minimal. +//! +//! This isn't to say that this approach isn't without its disadvantages of +//! course. For now though this seems to be a reasonable set of tradeoffs for +//! the development stage of the component model proposal. +//! +//! ## Creating adapter modules +//! +//! With WebAssembly itself being used to implement fused adapters, Wasmtime +//! still has the question of how to organize the adapter functions into actual +//! wasm modules. +//! +//! The first thing you might reach for is to put all the adapters into the same +//! wasm module. This cannot be done, however, because some adapters may depend +//! on other adapters (transitively) to be created. This means that if +//! everything were in the same module there would be no way to instantiate the +//! module. An example of this dependency is an adapter (A) used to create a +//! core wasm instance (M) whose exported memory is then referenced by another +//! adapter (B). In this situation the adapter B cannot be in the same module +//! as adapter A because B needs the memory of M but M is created with A which +//! would otherwise create a circular dependency. +//! +//! The second possibility of organizing adapter modules would be to place each +//! fused adapter into its own module. Each `canon lower` would effectively +//! become a core wasm module instantiation at that point. While this works it's +//! currently believed to be a bit too fine-grained. For example it would mean +//! that importing a dozen lowered functions into a module could possibly result +//! in up to a dozen different adapter modules. While this possibility could +//! work it has been ruled out as "probably too expensive at runtime". +//! +//! Thus the purpose and existence of this module is now evident -- this module +//! exists to identify what exactly goes into which adapter module. This will +//! evaluate the `GlobalInitializer` lists coming out of the `inline` pass and +//! insert `InstantiateModule` entries for where adapter modules should be +//! created. +//! +//! ## Partitioning adapter modules +//! +//! Currently this module does not attempt to be really all that fancy about +//! grouping adapters into adapter modules. The main idea is that most items +//! within an adapter module are likely to be close together since they're +//! theoretically going to be used for an instantiation of a core wasm module +//! just after the fused adapter was declared. With that in mind the current +//! algorithm is a one-pass approach to partitioning everything into adapter +//! modules. +//! +//! As the `GlobalInitializer` list is iterated over the last adapter module +//! created is recorded. Each adapter module, when created, records the index +//! space limits at the time of its creation. If a new adapter is found which +//! depends on an item after the original adapter module was created then the +//! prior adapter module is finished and a new one is started. Adapters only +//! ever attempt to get inserted into the most recent adapter module, no +//! searching is currently done to try to fit adapters into a prior adapter +//! module. +//! +//! During this remapping process the `RuntimeInstanceIndex` for all instances +//! is also updated. Insertion of an adapter module will increase all further +//! instance indices by one so this must be accounted for in various +//! references. + +use crate::component::translate::*; +use crate::fact::Module; +use wasmparser::WasmFeatures; + +/// Information about fused adapters within a component. +#[derive(Default)] +pub struct Adapters { + /// List of all fused adapters identified which are assigned an index and + /// contain various metadata about them as well. + pub adapters: PrimaryMap, +} + +/// Metadata information about a fused adapter. +pub struct Adapter { + /// The type used when the original core wasm function was lifted. + /// + /// Note that this could be different than `lower_ty` (but still matches + /// according to subtyping rules). + pub lift_ty: TypeFuncIndex, + /// Canonical ABI options used when the function was lifted. + pub lift_options: AdapterOptions, + /// The type used when the function was lowered back into a core wasm + /// function. + /// + /// Note that this could be different than `lift_ty` (but still matches + /// according to subtyping rules). + pub lower_ty: TypeFuncIndex, + /// Canonical ABI options used when the function was lowered. + pub lower_options: AdapterOptions, + /// The original core wasm function which was lifted. + pub func: CoreDef, +} + +/// Configuration options which can be specified as part of the canonical ABI +/// in the component model. +#[derive(Clone)] +pub struct AdapterOptions { + /// The Wasmtime-assigned component instance index where the options were + /// originally specified. + pub instance: RuntimeComponentInstanceIndex, + /// How strings are encoded. + pub string_encoding: StringEncoding, + /// An optional memory definition supplied. + pub memory: Option>, + /// An optional definition of `realloc` to used. + pub realloc: Option, + /// An optional definition of a `post-return` to use. + pub post_return: Option, +} + +impl<'data> Translator<'_, 'data> { + /// Modifies the list of `GlobalInitializer` entries within a + /// `Component`with `InstantiateModule::Adapter` entries where necessary. + /// + /// This is the entrypoint of functionality within this module which + /// performs all the work of identifying adapter usages and organizing + /// everything into adapter modules. + pub(super) fn insert_adapter_module_initializers( + &mut self, + component: &mut Component, + adapters: &mut Adapters, + ) { + let mut state = PartitionAdapterModules { + to_process: Vec::new(), + cur_idx: 0, + adapter_modules: PrimaryMap::new(), + items: DefinedItems::default(), + instance_map: PrimaryMap::with_capacity(component.num_runtime_instances as usize), + }; + state.run(component, adapters); + + // Next, in reverse, insert all of the adapter modules into the actual + // initializer list. Note that the iteration order is important here to + // ensure that all the `at_initializer_index` listed is valid for each + // entry. + let mut adapter_map = PrimaryMap::with_capacity(adapters.adapters.len()); + for _ in adapters.adapters.iter() { + adapter_map.push(None); + } + for (_, module) in state.adapter_modules.into_iter().rev() { + let index = module.at_initializer_index; + let instantiate = self.compile_adapter_module(module, adapters, &mut adapter_map); + let init = GlobalInitializer::InstantiateModule(instantiate); + component.initializers.insert(index, init); + } + + // Finally all references to `CoreDef::Adapter` are rewritten to their + // corresponding `CoreDef::Export` as identified within `adapter_map`. + for init in component.initializers.iter_mut() { + map_adapter_references(init, &adapter_map); + } + } + + fn compile_adapter_module( + &mut self, + module_parts: AdapterModuleParts, + adapters: &Adapters, + adapter_map: &mut PrimaryMap>>, + ) -> InstantiateModule { + // Use the `fact::Module` builder to create a new wasm module which + // represents all of the adapters specified here. + let mut module = Module::new( + self.types.component_types(), + self.tunables.debug_adapter_modules, + ); + let mut names = Vec::with_capacity(module_parts.adapters.len()); + for adapter in module_parts.adapters.iter() { + let name = format!("adapter{}", adapter.as_u32()); + module.adapt(&name, &adapters.adapters[*adapter]); + names.push(name); + } + let wasm = module.encode(); + let args = module.imports().to_vec(); + + // Extend the lifetime of the owned `wasm: Vec` on the stack to a + // higher scope defined by our original caller. That allows to transform + // `wasm` into `&'data [u8]` which is much easier to work with here. + let wasm = &*self.scope_vec.push(wasm); + if log::log_enabled!(log::Level::Trace) { + match wasmprinter::print_bytes(wasm) { + Ok(s) => log::trace!("generated adapter module:\n{}", s), + Err(e) => log::trace!("failed to print adapter module: {}", e), + } + } + + // With the wasm binary this is then pushed through general translation, + // validation, etc. Note that multi-memory is specifically enabled here + // since the adapter module is highly likely to use that if anything is + // actually indirected through memory. + let mut validator = Validator::new_with_features(WasmFeatures { + multi_memory: true, + ..*self.validator.features() + }); + let translation = ModuleEnvironment::new( + self.tunables, + &mut validator, + self.types.module_types_builder(), + ) + .translate(Parser::new(0), wasm) + .expect("invalid adapter module generated"); + + // And with all metadata available about the generated module a map can + // be built from adapter index to the precise export in the module that + // was generated. + for (adapter, name) in module_parts.adapters.iter().zip(&names) { + assert!(adapter_map[*adapter].is_none()); + let index = translation.module.exports[name]; + adapter_map[*adapter] = Some(CoreExport { + instance: module_parts.index, + item: ExportItem::Index(index), + }); + } + + // Finally the module translation is saved in the list of static + // modules to get fully compiled later and the `InstantiateModule` + // representation of this adapter module is returned. + let static_index = self.static_modules.push(translation); + InstantiateModule::Static(static_index, args.into()) + } +} + +struct PartitionAdapterModules { + /// Stack of remaining elements to process + to_process: Vec, + + /// Index of the current `GlobalInitializer` being processed. + cur_idx: usize, + + /// Information about all fused adapter modules that have been created so + /// far. + /// + /// This is modified whenever a fused adapter is used. + adapter_modules: PrimaryMap, + + /// Map from "old runtime instance index" to "new runtime instance index". + /// + /// This map is populated when instances are created to account for prior + /// adapter modules having been created. This effectively tracks an offset + /// for each index. + instance_map: PrimaryMap, + + /// Current limits of index spaces. + items: DefinedItems, +} + +/// Entries in the `PartitionAdapterModules::to_process` array. +enum ToProcess { + /// An adapter needs its own dependencies processed. This will map the + /// fields of `Adapter` above for the specified index. + Adapter(AdapterIndex), + /// An adapter has had its dependencies fully processed (transitively) and + /// the adapter now needs to be inserted into a module. + AddAdapterToModule(AdapterIndex), + /// A global initializer needs to be remapped. + GlobalInitializer(usize), + /// An export needs to be remapped. + Export(usize), + /// A global initializer which creates an instance has had all of its + /// arguments processed and now the instance number needs to be recorded. + PushInstance, +} + +/// Custom index type used exclusively for the `adapter_modules` map above. +#[derive(Copy, Clone, PartialEq, Eq)] +struct AdapterModuleIndex(u32); +cranelift_entity::entity_impl!(AdapterModuleIndex); + +struct AdapterModuleParts { + /// The runtime index that will be assigned to this adapter module when it's + /// instantiated. + index: RuntimeInstanceIndex, + /// The index in the `GlobalInitializer` list that this adapter module will + /// get inserted at. + at_initializer_index: usize, + /// Items that were available when this adapter module was created. + items_at_initializer: DefinedItems, + /// Adapters that have been inserted into this module, guaranteed to be + /// non-empty. + adapters: Vec, +} + +#[derive(Default, Clone)] +struct DefinedItems { + /// Number of core wasm instances created so far. + /// + /// Note that this does not count adapter modules created, only the + /// instance index space before adapter modules were inserted. + instances: u32, + /// Number of host-lowered functions seen so far. + lowerings: u32, + /// Number of "always trap" functions seen so far. + always_trap: u32, + /// Map of whether adapters have been inserted into an adapter module yet. + adapter_to_module: PrimaryMap>, +} + +impl PartitionAdapterModules { + /// Process the list of global `initializers` and partitions adapters into + /// adapter modules which will get inserted into the provided list in a + /// later pass. + fn run(&mut self, component: &mut Component, adapters: &mut Adapters) { + // This function is designed to be an iterative loop which models + // recursion in the `self.to_process` array instead of on the host call + // stack. The reason for this is that adapters need recursive processing + // since the argument to an adapter can hypothetically be an adapter + // itself (albeit silly but still valid). This recursive nature of + // adapters means that a component could be crafted to have an + // arbitrarily deep recursive dependeny chain for any one adapter. To + // avoid consuming host stack space the storage for this dependency + // chain is placed on the heap. + // + // The `self.to_process` list is a FIFO queue of what to process next. + // Initially seeded with all the global initializer indexes this is + // pushed to during processing to recursively handle adapters and + // similar. + assert!(self.to_process.is_empty()); + assert!(self.items.adapter_to_module.is_empty()); + + // Initially record all adapters as having no module which will get + // filled in over time. + for _ in adapters.adapters.iter() { + self.items.adapter_to_module.push(None); + } + + // Seed the worklist of what to process with the list of global + // initializers and exports, but in reverse order since this is a LIFO + // queue. Afterwards all of the items to process are handled in a loop. + for i in (0..component.exports.len()).rev() { + self.to_process.push(ToProcess::Export(i)); + } + for i in (0..component.initializers.len()).rev() { + self.to_process.push(ToProcess::GlobalInitializer(i)); + } + + while let Some(to_process) = self.to_process.pop() { + match to_process { + ToProcess::GlobalInitializer(i) => { + assert!(i <= self.cur_idx + 1); + self.cur_idx = i; + self.global_initializer(&mut component.initializers[i]); + } + + ToProcess::Export(i) => { + self.cur_idx = component.initializers.len(); + self.export(&mut component.exports[i]); + } + + ToProcess::PushInstance => { + // A new runtime instance is being created here so insert an + // entry into the remapping map for instance indexes. This + // instance's index is offset by the number of adapter modules + // created prior. + self.instance_map + .push(RuntimeInstanceIndex::from_u32(self.items.instances)); + self.items.instances += 1; + } + + ToProcess::Adapter(idx) => { + let info = &mut adapters.adapters[idx]; + self.process_core_def(&mut info.func); + self.process_options(&mut info.lift_options); + self.process_options(&mut info.lower_options); + } + + ToProcess::AddAdapterToModule(idx) => { + // If this adapter has already been assigned to a module + // then there's no need to do anything else here. + // + // This can happen when a core wasm instance is created with + // an adapter as the argument multiple times for example. + if self.items.adapter_to_module[idx].is_some() { + continue; + } + + // If an adapter module is already in progress and + // everything this adapter depends on was available at the + // time of creation of that adapter module, then this + // adapter can go in that module. + if let Some((module_idx, module)) = self.adapter_modules.last_mut() { + let info = &adapters.adapters[idx]; + if module.items_at_initializer.contains(info) { + self.items.adapter_to_module[idx] = Some(module_idx); + module.adapters.push(idx); + continue; + } + } + + // ... otherwise a new adapter module is started. Note that + // the instance count is bumped here to model the + // instantiation of the adapter module. + let module = AdapterModuleParts { + index: RuntimeInstanceIndex::from_u32(self.items.instances), + at_initializer_index: self.cur_idx, + items_at_initializer: self.items.clone(), + adapters: vec![idx], + }; + let index = self.adapter_modules.push(module); + self.items.adapter_to_module[idx] = Some(index); + self.items.instances += 1; + } + } + } + } + + fn global_initializer(&mut self, init: &mut GlobalInitializer) { + match init { + GlobalInitializer::InstantiateModule(module) => { + // Enqueue a bump of the instance count, but this only happens + // after all the arguments have been processed below. Given the + // LIFO nature of `self.to_process` this will be handled after + // all arguments are recursively processed. + self.to_process.push(ToProcess::PushInstance); + + match module { + InstantiateModule::Static(_, args) => { + for def in args.iter_mut() { + self.process_core_def(def); + } + } + InstantiateModule::Import(_, args) => { + for (_, map) in args { + for (_, def) in map { + self.process_core_def(def); + } + } + } + } + } + + GlobalInitializer::ExtractRealloc(e) => self.process_core_def(&mut e.def), + GlobalInitializer::ExtractPostReturn(e) => self.process_core_def(&mut e.def), + + // Update items available as they're defined + GlobalInitializer::LowerImport(_) => self.items.lowerings += 1, + GlobalInitializer::AlwaysTrap(_) => self.items.always_trap += 1, + + // Nothing is defined or referenced by these initializers that we + // need to worry about here. + GlobalInitializer::ExtractMemory(_) => {} + GlobalInitializer::SaveStaticModule(_) => {} + GlobalInitializer::SaveModuleImport(_) => {} + } + } + + fn export(&mut self, export: &mut Export) { + match export { + Export::LiftedFunction { func, .. } => { + self.process_core_def(func); + } + Export::Instance(exports) => { + for (_, export) in exports { + self.export(export); + } + } + Export::Module(_) => {} + } + } + + fn process_options(&mut self, opts: &mut AdapterOptions) { + if let Some(memory) = &mut opts.memory { + self.process_core_export(memory); + } + if let Some(def) = &mut opts.realloc { + self.process_core_def(def); + } + if let Some(def) = &mut opts.post_return { + self.process_core_def(def); + } + } + + fn process_core_def(&mut self, def: &mut CoreDef) { + match def { + CoreDef::Adapter(idx) => { + // The `to_process` queue is a LIFO queue so first enqueue the + // addition of this adapter into a module followed by the + // processing of the adapter itself. This means that the + // adapter's own dependencies will be processed before the + // adapter is added to a module. + self.to_process.push(ToProcess::AddAdapterToModule(*idx)); + self.to_process.push(ToProcess::Adapter(*idx)); + } + + CoreDef::Export(e) => self.process_core_export(e), + + // These are ignored since they don't contain a reference to an + // adapter which may need to be inserted into a module. + CoreDef::Lowered(_) | CoreDef::AlwaysTrap(_) | CoreDef::InstanceFlags(_) => {} + } + } + + fn process_core_export(&mut self, export: &mut CoreExport) { + // Remap the instance index referenced here as necessary to account + // for any adapter modules that needed creating in the meantime. + export.instance = self.instance_map[export.instance]; + } +} + +impl DefinedItems { + fn contains(&self, info: &Adapter) -> bool { + self.contains_options(&info.lift_options) + && self.contains_options(&info.lower_options) + && self.contains_def(&info.func) + } + + fn contains_options(&self, options: &AdapterOptions) -> bool { + let AdapterOptions { + instance: _, + string_encoding: _, + memory, + realloc, + post_return, + } = options; + + if let Some(mem) = memory { + if !self.contains_export(mem) { + return false; + } + } + + if let Some(def) = realloc { + if !self.contains_def(def) { + return false; + } + } + + if let Some(def) = post_return { + if !self.contains_def(def) { + return false; + } + } + + true + } + + fn contains_def(&self, options: &CoreDef) -> bool { + match options { + CoreDef::Export(e) => self.contains_export(e), + CoreDef::AlwaysTrap(i) => i.as_u32() < self.always_trap, + CoreDef::Lowered(i) => i.as_u32() < self.lowerings, + CoreDef::Adapter(idx) => self.adapter_to_module[*idx].is_some(), + CoreDef::InstanceFlags(_) => true, + } + } + + fn contains_export(&self, export: &CoreExport) -> bool { + // This `DefinedItems` index space will contain `export` if the + // instance referenced has already been instantiated. The actual item + // that `export` points to doesn't need to be tested since it comes + // from the instance regardless. + export.instance.as_u32() < self.instances + } +} + +/// Rewrites all instances of `CoreDef::Adapter` within the `init` initializer +/// provided to `CoreExport` according to the `map` provided. +/// +/// This is called after all adapter modules have been constructed and the +/// core wasm function for each adapter has been identified. +fn map_adapter_references( + init: &mut GlobalInitializer, + map: &PrimaryMap>>, +) { + let map_core_def = |def: &mut CoreDef| { + let adapter = match def { + CoreDef::Adapter(idx) => *idx, + _ => return, + }; + *def = CoreDef::Export( + map[adapter] + .clone() + .expect("adapter should have been instantiated"), + ); + }; + match init { + GlobalInitializer::InstantiateModule(module) => match module { + InstantiateModule::Static(_, args) => { + for def in args.iter_mut() { + map_core_def(def); + } + } + InstantiateModule::Import(_, args) => { + for (_, map) in args { + for (_, def) in map { + map_core_def(def); + } + } + } + }, + + GlobalInitializer::ExtractRealloc(e) => map_core_def(&mut e.def), + GlobalInitializer::ExtractPostReturn(e) => map_core_def(&mut e.def), + + // Nothing to map here + GlobalInitializer::LowerImport(_) + | GlobalInitializer::AlwaysTrap(_) + | GlobalInitializer::ExtractMemory(_) => {} + GlobalInitializer::SaveStaticModule(_) => {} + GlobalInitializer::SaveModuleImport(_) => {} + } +} diff --git a/crates/environ/src/component/translate/inline.rs b/crates/environ/src/component/translate/inline.rs index 29f04342e0..0cae7a624e 100644 --- a/crates/environ/src/component/translate/inline.rs +++ b/crates/environ/src/component/translate/inline.rs @@ -45,8 +45,9 @@ //! side-effectful initializers are emitted to the `GlobalInitializer` list in the //! final `Component`. +use crate::component::translate::adapt::{Adapter, AdapterOptions, Adapters}; use crate::component::translate::*; -use crate::{ModuleTranslation, PrimaryMap, SignatureIndex}; +use crate::{PrimaryMap, SignatureIndex}; use indexmap::IndexMap; pub(super) fn run( @@ -54,12 +55,13 @@ pub(super) fn run( result: &Translation<'_>, nested_modules: &PrimaryMap>, nested_components: &PrimaryMap>, -) -> Result { +) -> Result<(Component, Adapters)> { let mut inliner = Inliner { types, nested_modules, nested_components, result: Component::default(), + adapters: Adapters::default(), import_path_interner: Default::default(), runtime_realloc_interner: Default::default(), runtime_post_return_interner: Default::default(), @@ -106,7 +108,7 @@ pub(super) fn run( } inliner.result.exports = export_map; - Ok(inliner.result) + Ok((inliner.result, inliner.adapters)) } struct Inliner<'a> { @@ -133,6 +135,9 @@ struct Inliner<'a> { /// inliner. result: Component, + /// Metadata about fused adapters identified throughout inlining. + adapters: Adapters, + // Maps used to "intern" various runtime items to only save them once at // runtime instead of multiple times. import_path_interner: HashMap, RuntimeImportIndex>, @@ -270,7 +275,7 @@ enum ComponentFuncDef<'a> { Lifted { ty: TypeFuncIndex, func: CoreDef, - options: CanonicalOptions, + options: AdapterOptions, }, } @@ -392,7 +397,7 @@ impl<'a> Inliner<'a> { let canonical_abi = frame.translation.funcs[frame.funcs.next_key()]; let lower_ty = frame.translation.component_funcs[*func]; - let options_lower = self.canonical_options(frame, options); + let options_lower = self.adapter_options(frame, options); let func = match &frame.component_funcs[*func] { // If this component function was originally a host import // then this is a lowered host function which needs a @@ -402,13 +407,14 @@ impl<'a> Inliner<'a> { let index = LoweredIndex::from_u32(self.result.num_lowerings); self.result.num_lowerings += 1; let import = self.runtime_import(path); + let options = self.canonical_options(options_lower); self.result .initializers .push(GlobalInitializer::LowerImport(LowerImport { canonical_abi, import, index, - options: options_lower, + options, })); CoreDef::Lowered(index) } @@ -464,33 +470,43 @@ impl<'a> Inliner<'a> { // component is different than the source component means // that a "fused adapter" was just identified. // - // This is the location where, when this is actually - // implemented, we'll record metadata about this fused - // adapter to get compiled later during the compilation - // process. The fused adapter here will be generated by - // cranelift and will perfom argument validation when - // called, copy the arguments from `options_lower` to - // `options_lift` and then call the `func` specified for the - // lifted options. + // Metadata about this fused adapter is recorded in the + // `Adapters` output of this compilation pass. Currently the + // implementation of fused adapters is to generate a core + // wasm module which is instantiated with relevant imports + // and the exports are used as the fused adapters. At this + // time we don't know when precisely the instance will be + // created but we do know that the result of this will be an + // export from a previously-created instance. // - // When the `func` returns the canonical adapter will verify - // the return values, copy them from `options_lift` to - // `options_lower`, and then return. + // To model this the result of this arm is a + // `CoreDef::Export`. The actual indices listed within the + // export are "fake indices" in the sense of they're not + // resolved yet. This resolution will happen at a later + // compilation phase. Any usages of the `CoreDef::Export` + // here will be detected and rewritten to an actual runtime + // instance created. + // + // The `instance` field of the `CoreExport` has a marker + // which indicates that it's a fused adapter. The `item` is + // a function where the function index corresponds to the + // `adapter_idx` which contains the metadata about this + // adapter being created. The metadata is used to learn + // about the dependencies and when the adapter module can + // be instantiated. ComponentFuncDef::Lifted { ty: lift_ty, func, options: options_lift, } => { - // These are the various compilation options for lifting - // and lowering. - drop(lift_ty); // type used when lifting the core function - drop(lower_ty); // type used when lowering the core function - drop(func); // original core wasm function that was lifted - drop(options_lift); // options during `canon lift` - drop(options_lower); // options during `canon lower` - drop(canonical_abi); // type signature of created core wasm function - - unimplemented!("lowering a lifted function") + let adapter_idx = self.adapters.adapters.push(Adapter { + lift_ty: *lift_ty, + lift_options: options_lift.clone(), + lower_ty, + lower_options: options_lower, + func: func.clone(), + }); + CoreDef::Adapter(adapter_idx) } }; frame.funcs.push(func); @@ -500,7 +516,7 @@ impl<'a> Inliner<'a> { // some metadata about the lifting is simply recorded. This'll get // plumbed through to exports or a fused adapter later on. Lift(ty, func, options) => { - let options = self.canonical_options(frame, options); + let options = self.adapter_options(frame, options); frame.component_funcs.push(ComponentFuncDef::Lifted { ty: *ty, func: frame.funcs[*func].clone(), @@ -639,7 +655,7 @@ impl<'a> Inliner<'a> { frame.tables.push( match self.core_def_of_module_instance_export(frame, *instance, *name) { CoreDef::Export(e) => e, - CoreDef::Lowered(_) | CoreDef::AlwaysTrap(_) => unreachable!(), + _ => unreachable!(), }, ); } @@ -648,7 +664,7 @@ impl<'a> Inliner<'a> { frame.globals.push( match self.core_def_of_module_instance_export(frame, *instance, *name) { CoreDef::Export(e) => e, - CoreDef::Lowered(_) | CoreDef::AlwaysTrap(_) => unreachable!(), + _ => unreachable!(), }, ); } @@ -657,7 +673,7 @@ impl<'a> Inliner<'a> { frame.memories.push( match self.core_def_of_module_instance_export(frame, *instance, *name) { CoreDef::Export(e) => e, - CoreDef::Lowered(_) | CoreDef::AlwaysTrap(_) => unreachable!(), + _ => unreachable!(), }, ); } @@ -795,19 +811,34 @@ impl<'a> Inliner<'a> { /// Translates a `LocalCanonicalOptions` which indexes into the `frame` /// specified into a runtime representation. - /// - /// This will "intern" repeatedly reused memories or functions to avoid - /// storing them in multiple locations at runtime. - fn canonical_options( + fn adapter_options( &mut self, frame: &InlinerFrame<'a>, options: &LocalCanonicalOptions, - ) -> CanonicalOptions { + ) -> AdapterOptions { let memory = options.memory.map(|i| { - let export = frame.memories[i].clone().map_index(|i| match i { + frame.memories[i].clone().map_index(|i| match i { EntityIndex::Memory(i) => i, _ => unreachable!(), - }); + }) + }); + let realloc = options.realloc.map(|i| frame.funcs[i].clone()); + let post_return = options.post_return.map(|i| frame.funcs[i].clone()); + AdapterOptions { + instance: frame.instance, + string_encoding: options.string_encoding, + memory, + realloc, + post_return, + } + } + + /// Translatees an `AdapterOptions` into a `CanonicalOptions` where + /// memories/functions are inserted into the global initializer list for + /// use at runtime. This is only used for lowered host functions and lifted + /// functions exported to the host. + fn canonical_options(&mut self, options: AdapterOptions) -> CanonicalOptions { + let memory = options.memory.map(|export| { *self .runtime_memory_interner .entry(export.clone()) @@ -823,8 +854,7 @@ impl<'a> Inliner<'a> { index }) }); - let realloc = options.realloc.map(|i| { - let def = frame.funcs[i].clone(); + let realloc = options.realloc.map(|def| { *self .runtime_realloc_interner .entry(def.clone()) @@ -840,8 +870,7 @@ impl<'a> Inliner<'a> { index }) }); - let post_return = options.post_return.map(|i| { - let def = frame.funcs[i].clone(); + let post_return = options.post_return.map(|def| { *self .runtime_post_return_interner .entry(def.clone()) @@ -859,7 +888,7 @@ impl<'a> Inliner<'a> { }) }); CanonicalOptions { - instance: frame.instance, + instance: options.instance, string_encoding: options.string_encoding, memory, realloc, @@ -896,6 +925,7 @@ impl<'a> Inliner<'a> { // component then the configured options are plumbed through // here. ComponentFuncDef::Lifted { ty, func, options } => { + let options = self.canonical_options(options); Export::LiftedFunction { ty, func, options } } diff --git a/crates/environ/src/component/types.rs b/crates/environ/src/component/types.rs index bdb3438def..040c24318f 100644 --- a/crates/environ/src/component/types.rs +++ b/crates/environ/src/component/types.rs @@ -166,6 +166,10 @@ indices! { /// Index that represents an exported module from a component since that's /// currently the only use for saving the entire module state at runtime. pub struct RuntimeModuleIndex(u32); + + /// Index into the list of fused adapters identified during compilation. + /// Used in conjuction with the `Adapters` type. + pub struct AdapterIndex(u32); } // Reexport for convenience some core-wasm indices which are also used in the @@ -286,6 +290,11 @@ impl ComponentTypesBuilder { self.component_types } + /// Returns the `ComponentTypes`-in-progress. + pub fn component_types(&self) -> &ComponentTypes { + &self.component_types + } + /// Returns the underlying builder used to build up core wasm module types. /// /// Note that this is shared across all modules found within a component to diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rs index 1748a7c2e3..9a26c1475c 100644 --- a/crates/environ/src/component/vmcomponent_offsets.rs +++ b/crates/environ/src/component/vmcomponent_offsets.rs @@ -3,7 +3,7 @@ // struct VMComponentContext { // magic: u32, // store: *mut dyn Store, -// flags: [VMComponentFlags; component.num_runtime_component_instances], +// flags: [VMGlobalDefinition; component.num_runtime_component_instances], // lowering_anyfuncs: [VMCallerCheckedAnyfunc; component.num_lowerings], // always_trap_anyfuncs: [VMCallerCheckedAnyfunc; component.num_always_trap], // lowerings: [VMLowering; component.num_lowerings], @@ -26,15 +26,15 @@ pub const VMCOMPONENT_MAGIC: u32 = u32::from_le_bytes(*b"comp"); /// Flag for the `VMComponentContext::flags` field which corresponds to the /// canonical ABI flag `may_leave` -pub const VMCOMPONENT_FLAG_MAY_LEAVE: u8 = 1 << 0; +pub const FLAG_MAY_LEAVE: i32 = 1 << 0; /// Flag for the `VMComponentContext::flags` field which corresponds to the /// canonical ABI flag `may_enter` -pub const VMCOMPONENT_FLAG_MAY_ENTER: u8 = 1 << 1; +pub const FLAG_MAY_ENTER: i32 = 1 << 1; /// Flag for the `VMComponentContext::flags` field which is set whenever a /// function is called to indicate that `post_return` must be called next. -pub const VMCOMPONENT_FLAG_NEEDS_POST_RETURN: u8 = 1 << 2; +pub const FLAG_NEEDS_POST_RETURN: i32 = 1 << 2; /// Runtime offsets within a `VMComponentContext` for a specific component. #[derive(Debug, Clone, Copy)] @@ -131,7 +131,8 @@ impl VMComponentOffsets

{ size(magic) = 4u32, align(u32::from(ret.ptr.size())), size(store) = cmul(2, ret.ptr.size()), - size(flags) = cmul(ret.num_runtime_component_instances, ret.size_of_vmcomponent_flags()), + align(16), + size(flags) = cmul(ret.num_runtime_component_instances, ret.ptr.size_of_vmglobal_definition()), align(u32::from(ret.ptr.size())), size(lowering_anyfuncs) = cmul(ret.num_lowerings, ret.ptr.size_of_vmcaller_checked_anyfunc()), size(always_trap_anyfuncs) = cmul(ret.num_always_trap, ret.ptr.size_of_vmcaller_checked_anyfunc()), @@ -163,17 +164,11 @@ impl VMComponentOffsets

{ self.magic } - /// The size of the `VMComponentFlags` type. - #[inline] - pub fn size_of_vmcomponent_flags(&self) -> u8 { - 1 - } - /// The offset of the `flags` field. #[inline] - pub fn flags(&self, index: RuntimeComponentInstanceIndex) -> u32 { + pub fn instance_flags(&self, index: RuntimeComponentInstanceIndex) -> u32 { assert!(index.as_u32() < self.num_runtime_component_instances); - self.flags + index.as_u32() + self.flags + index.as_u32() * u32::from(self.ptr.size_of_vmglobal_definition()) } /// The offset of the `store` field. diff --git a/crates/environ/src/fact.rs b/crates/environ/src/fact.rs new file mode 100644 index 0000000000..bb21075021 --- /dev/null +++ b/crates/environ/src/fact.rs @@ -0,0 +1,290 @@ +//! Wasmtime's Fused Adapter Compiler of Trampolines (FACT) +//! +//! This module contains a compiler which emits trampolines to implement fused +//! adatpers for the component model. A fused adapter is when a core wasm +//! function is lifted from one component instance and then lowered into another +//! component instance. This communication between components is well-defined by +//! the spec and ends up creating what's called a "fused adapter". +//! +//! Adapters are currently implemented with WebAssembly modules. This submodule +//! will generate a core wasm binary which contains the adapters specified +//! during compilation. The actual wasm is then later processed by standard +//! paths in Wasmtime to create native machine code and runtime representations +//! of modules. +//! +//! Note that identification of precisely what goes into an adapter module is +//! not handled in this file, instead that's all done in `translate/adapt.rs`. +//! Otherwise this module is only reponsible for taking a set of adapters and +//! their imports and then generating a core wasm module to implement all of +//! that. + +use crate::component::{ + Adapter, AdapterOptions, ComponentTypes, CoreDef, StringEncoding, TypeFuncIndex, +}; +use crate::{FuncIndex, GlobalIndex, MemoryIndex}; +use std::collections::HashMap; +use wasm_encoder::*; + +mod core_types; +mod signature; +mod trampoline; +mod traps; + +/// Representation of an adapter module. +pub struct Module<'a> { + /// Whether or not debug code is inserted into the adapters themselves. + debug: bool, + /// Type information from the creator of this `Module` + types: &'a ComponentTypes, + + /// Core wasm type section that's incrementally built + core_types: core_types::CoreTypes, + + /// Core wasm import section which is built as adapters are inserted. Note + /// that imports here are intern'd to avoid duplicate imports of the same + /// item. + core_imports: ImportSection, + /// Final list of imports that this module ended up using, in the same order + /// as the imports in the import section. + imports: Vec, + /// Intern'd imports and what index they were assigned. + imported: HashMap, + + // Current status of index spaces from the imports generated so far. + core_funcs: u32, + core_memories: u32, + core_globals: u32, + + /// Adapters which will be compiled once they're all registered. + adapters: Vec, +} + +struct AdapterData { + /// Export name of this adapter + name: String, + /// Options specified during the `canon lift` operation + lift: Options, + /// Options specified during the `canon lower` operation + lower: Options, + /// The core wasm function that this adapter will be calling (the original + /// function that was `canon lift`'d) + callee: FuncIndex, + /// FIXME(#4185) should be plumbed and handled as part of the new reentrance + /// rules not yet implemented here. + called_as_export: bool, +} + +struct Options { + ty: TypeFuncIndex, + string_encoding: StringEncoding, + flags: GlobalIndex, + memory64: bool, + memory: Option, + realloc: Option, + post_return: Option, +} + +enum Context { + Lift, + Lower, +} + +impl<'a> Module<'a> { + pub fn new(types: &'a ComponentTypes, debug: bool) -> Module<'a> { + Module { + debug, + types, + core_types: Default::default(), + core_imports: Default::default(), + imported: Default::default(), + adapters: Default::default(), + imports: Default::default(), + core_funcs: 0, + core_memories: 0, + core_globals: 0, + } + } + + /// Registers a new adapter within this adapter module. + /// + /// The `name` provided is the export name of the adapter from the final + /// module, and `adapter` contains all metadata necessary for compilation. + pub fn adapt(&mut self, name: &str, adapter: &Adapter) { + // Import core wasm function which was lifted using its appropriate + // signature since the exported function this adapter generates will + // call the lifted function. + let signature = self.signature(adapter.lift_ty, Context::Lift); + let ty = self + .core_types + .function(&signature.params, &signature.results); + let callee = self.import_func("callee", name, ty, adapter.func.clone()); + + // Next import any items required by the various canonical options + // (memories, reallocs, etc) + let mut lift = self.import_options(adapter.lift_ty, &adapter.lift_options); + let lower = self.import_options(adapter.lower_ty, &adapter.lower_options); + + // Handle post-return specifically here where we have `core_ty` and the + // results of `core_ty` are the parameters to the post-return function. + lift.post_return = adapter.lift_options.post_return.as_ref().map(|func| { + let ty = self.core_types.function(&signature.results, &[]); + self.import_func("post_return", name, ty, func.clone()) + }); + + // Lowering options are not allowed to specify post-return as per the + // current canonical abi specification. + assert!(adapter.lower_options.post_return.is_none()); + + self.adapters.push(AdapterData { + name: name.to_string(), + lift, + lower, + callee, + // FIXME(#4185) should be plumbed and handled as part of the new + // reentrance rules not yet implemented here. + called_as_export: true, + }); + } + + fn import_options(&mut self, ty: TypeFuncIndex, options: &AdapterOptions) -> Options { + let AdapterOptions { + instance, + string_encoding, + memory, + realloc, + post_return: _, // handled above + } = options; + let memory64 = false; // FIXME(#4311) should be plumbed from somewhere + let flags = self.import_global( + "flags", + &format!("instance{}", instance.as_u32()), + GlobalType { + val_type: ValType::I32, + mutable: true, + }, + CoreDef::InstanceFlags(*instance), + ); + let memory = memory.as_ref().map(|memory| { + self.import_memory( + "memory", + "", + MemoryType { + minimum: 0, + maximum: None, + shared: false, + memory64, + }, + memory.clone().into(), + ) + }); + let realloc = realloc.as_ref().map(|func| { + let ptr = if memory64 { ValType::I64 } else { ValType::I32 }; + let ty = self.core_types.function(&[ptr, ptr, ptr, ptr], &[ptr]); + self.import_func("realloc", "", ty, func.clone()) + }); + Options { + ty, + string_encoding: *string_encoding, + flags, + memory64, + memory, + realloc, + post_return: None, + } + } + + fn import_func(&mut self, module: &str, name: &str, ty: u32, def: CoreDef) -> FuncIndex { + FuncIndex::from_u32( + self.import(module, name, EntityType::Function(ty), def, |m| { + &mut m.core_funcs + }), + ) + } + + fn import_global( + &mut self, + module: &str, + name: &str, + ty: GlobalType, + def: CoreDef, + ) -> GlobalIndex { + GlobalIndex::from_u32(self.import(module, name, EntityType::Global(ty), def, |m| { + &mut m.core_globals + })) + } + + fn import_memory( + &mut self, + module: &str, + name: &str, + ty: MemoryType, + def: CoreDef, + ) -> MemoryIndex { + MemoryIndex::from_u32(self.import(module, name, EntityType::Memory(ty), def, |m| { + &mut m.core_memories + })) + } + + fn import( + &mut self, + module: &str, + name: &str, + ty: EntityType, + def: CoreDef, + new: impl FnOnce(&mut Self) -> &mut u32, + ) -> u32 { + if let Some(prev) = self.imported.get(&def) { + return *prev; + } + let cnt = new(self); + *cnt += 1; + let ret = *cnt - 1; + self.core_imports.import(module, name, ty); + self.imported.insert(def.clone(), ret); + self.imports.push(def); + ret + } + + pub fn encode(&mut self) -> Vec { + let mut funcs = FunctionSection::new(); + let mut code = CodeSection::new(); + let mut exports = ExportSection::new(); + let mut traps = traps::TrapSection::default(); + + for adapter in self.adapters.iter() { + let idx = self.core_funcs + funcs.len(); + exports.export(&adapter.name, ExportKind::Func, idx); + + let signature = self.signature(adapter.lower.ty, Context::Lower); + let ty = self + .core_types + .function(&signature.params, &signature.results); + funcs.function(ty); + + let (function, func_traps) = trampoline::compile(self, adapter); + code.raw(&function); + traps.append(idx, func_traps); + } + let traps = traps.finish(); + + let mut result = wasm_encoder::Module::new(); + result.section(&self.core_types.section); + result.section(&self.core_imports); + result.section(&funcs); + result.section(&exports); + result.section(&code); + if self.debug { + result.section(&CustomSection { + name: "wasmtime-trampoline-traps", + data: &traps, + }); + } + result.finish() + } + + /// Returns the imports that were used, in order, to create this adapter + /// module. + pub fn imports(&self) -> &[CoreDef] { + &self.imports + } +} diff --git a/crates/environ/src/fact/core_types.rs b/crates/environ/src/fact/core_types.rs new file mode 100644 index 0000000000..b7699ad23c --- /dev/null +++ b/crates/environ/src/fact/core_types.rs @@ -0,0 +1,24 @@ +use std::collections::HashMap; +use wasm_encoder::{TypeSection, ValType}; + +/// A simple representation of the type section which automatically intern's +/// types and ensures they're only defined once. +#[derive(Default)] +pub struct CoreTypes { + pub section: TypeSection, + intern: HashMap<(Vec, Vec), u32>, +} + +impl CoreTypes { + pub fn function(&mut self, params: &[ValType], results: &[ValType]) -> u32 { + *self + .intern + .entry((params.to_vec(), results.to_vec())) + .or_insert_with(|| { + let idx = self.section.len(); + self.section + .function(params.iter().copied(), results.iter().copied()); + idx + }) + } +} diff --git a/crates/environ/src/fact/signature.rs b/crates/environ/src/fact/signature.rs new file mode 100644 index 0000000000..e289103da1 --- /dev/null +++ b/crates/environ/src/fact/signature.rs @@ -0,0 +1,265 @@ +//! Size, align, and flattening information about component model types. + +use crate::component::{InterfaceType, TypeFuncIndex, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS}; +use crate::fact::{Context, Module}; +use wasm_encoder::ValType; + +/// Metadata about a core wasm signature which is created for a component model +/// signature. +pub struct Signature { + /// Core wasm parameters. + pub params: Vec, + /// Core wasm results. + pub results: Vec, + /// Indicator to whether parameters are indirect, meaning that the first + /// entry of `params` is a pointer type which all parameters are loaded + /// through. + pub params_indirect: bool, + /// Indicator whether results are passed indirectly. This may mean that + /// `results` is an `i32` or that `params` ends with an `i32` depending on + /// the `Context`. + pub results_indirect: bool, +} + +pub(crate) fn align_to(n: usize, align: usize) -> usize { + assert!(align.is_power_of_two()); + (n + (align - 1)) & !(align - 1) +} + +impl Module<'_> { + /// Calculates the core wasm function signature for the component function + /// type specified within `Context`. + /// + /// This is used to generate the core wasm signatures for functions that are + /// imported (matching whatever was `canon lift`'d) and functions that are + /// exported (matching the generated function from `canon lower`). + pub(super) fn signature(&self, ty: TypeFuncIndex, context: Context) -> Signature { + let ty = &self.types[ty]; + + let mut params = self.flatten_types(ty.params.iter().map(|(_, ty)| *ty)); + let mut params_indirect = false; + if params.len() > MAX_FLAT_PARAMS { + params = vec![ValType::I32]; + params_indirect = true; + } + + let mut results = self.flatten_types([ty.result]); + let mut results_indirect = false; + if results.len() > MAX_FLAT_RESULTS { + results_indirect = true; + match context { + // For a lifted function too-many-results gets translated to a + // returned pointer where results are read from. The callee + // allocates space here. + Context::Lift => results = vec![ValType::I32], + // For a lowered function too-many-results becomes a return + // pointer which is passed as the last argument. The caller + // allocates space here. + Context::Lower => { + results.truncate(0); + params.push(ValType::I32); + } + } + } + Signature { + params, + results, + params_indirect, + results_indirect, + } + } + + /// Pushes the flat version of a list of component types into a final result + /// list. + pub(crate) fn flatten_types( + &self, + tys: impl IntoIterator, + ) -> Vec { + let mut result = Vec::new(); + for ty in tys { + self.push_flat(&ty, &mut result); + } + result + } + + fn push_flat(&self, ty: &InterfaceType, dst: &mut Vec) { + match ty { + InterfaceType::Unit => {} + + InterfaceType::Bool + | InterfaceType::S8 + | InterfaceType::U8 + | InterfaceType::S16 + | InterfaceType::U16 + | InterfaceType::S32 + | InterfaceType::U32 + | InterfaceType::Char => dst.push(ValType::I32), + + InterfaceType::S64 | InterfaceType::U64 => dst.push(ValType::I64), + + InterfaceType::Float32 => dst.push(ValType::F32), + InterfaceType::Float64 => dst.push(ValType::F64), + + InterfaceType::String | InterfaceType::List(_) => { + dst.push(ValType::I32); + dst.push(ValType::I32); + } + InterfaceType::Record(r) => { + for field in self.types[*r].fields.iter() { + self.push_flat(&field.ty, dst); + } + } + InterfaceType::Tuple(t) => { + for ty in self.types[*t].types.iter() { + self.push_flat(ty, dst); + } + } + InterfaceType::Flags(f) => { + let flags = &self.types[*f]; + let nflags = align_to(flags.names.len(), 32) / 32; + for _ in 0..nflags { + dst.push(ValType::I32); + } + } + InterfaceType::Enum(_) => dst.push(ValType::I32), + InterfaceType::Option(t) => { + dst.push(ValType::I32); + self.push_flat(&self.types[*t], dst); + } + InterfaceType::Variant(t) => { + dst.push(ValType::I32); + let pos = dst.len(); + let mut tmp = Vec::new(); + for case in self.types[*t].cases.iter() { + self.push_flat_variant(&case.ty, pos, &mut tmp, dst); + } + } + InterfaceType::Union(t) => { + dst.push(ValType::I32); + let pos = dst.len(); + let mut tmp = Vec::new(); + for ty in self.types[*t].types.iter() { + self.push_flat_variant(ty, pos, &mut tmp, dst); + } + } + InterfaceType::Expected(t) => { + dst.push(ValType::I32); + let e = &self.types[*t]; + let pos = dst.len(); + let mut tmp = Vec::new(); + self.push_flat_variant(&e.ok, pos, &mut tmp, dst); + self.push_flat_variant(&e.err, pos, &mut tmp, dst); + } + } + } + + fn push_flat_variant( + &self, + ty: &InterfaceType, + pos: usize, + tmp: &mut Vec, + dst: &mut Vec, + ) { + tmp.truncate(0); + self.push_flat(ty, tmp); + for (i, a) in tmp.iter().enumerate() { + match dst.get_mut(pos + i) { + Some(b) => join(*a, b), + None => dst.push(*a), + } + } + + fn join(a: ValType, b: &mut ValType) { + if a == *b { + return; + } + match (a, *b) { + (ValType::I32, ValType::F32) | (ValType::F32, ValType::I32) => *b = ValType::I32, + _ => *b = ValType::I64, + } + } + } + + pub(crate) fn align(&self, ty: &InterfaceType) -> usize { + self.size_align(ty).1 + } + + /// Returns a (size, align) pair corresponding to the byte-size and + /// byte-alignment of the type specified. + // + // TODO: this is probably inefficient to entire recalculate at all phases, + // seems like it would be best to intern this in some sort of map somewhere. + pub(crate) fn size_align(&self, ty: &InterfaceType) -> (usize, usize) { + match ty { + InterfaceType::Unit => (0, 1), + InterfaceType::Bool | InterfaceType::S8 | InterfaceType::U8 => (1, 1), + InterfaceType::S16 | InterfaceType::U16 => (2, 2), + InterfaceType::S32 + | InterfaceType::U32 + | InterfaceType::Char + | InterfaceType::Float32 => (4, 4), + InterfaceType::S64 | InterfaceType::U64 | InterfaceType::Float64 => (8, 8), + InterfaceType::String | InterfaceType::List(_) => (8, 4), + + InterfaceType::Record(r) => { + self.record_size_align(self.types[*r].fields.iter().map(|f| &f.ty)) + } + InterfaceType::Tuple(t) => self.record_size_align(self.types[*t].types.iter()), + InterfaceType::Flags(f) => match self.types[*f].names.len() { + n if n <= 8 => (1, 1), + n if n <= 16 => (2, 2), + n if n <= 32 => (4, 4), + n => (4 * (align_to(n, 32) / 32), 4), + }, + InterfaceType::Enum(t) => self.discrim_size_align(self.types[*t].names.len()), + InterfaceType::Option(t) => { + let ty = &self.types[*t]; + self.variant_size_align([&InterfaceType::Unit, ty].into_iter()) + } + InterfaceType::Variant(t) => { + self.variant_size_align(self.types[*t].cases.iter().map(|c| &c.ty)) + } + InterfaceType::Union(t) => self.variant_size_align(self.types[*t].types.iter()), + InterfaceType::Expected(t) => { + let e = &self.types[*t]; + self.variant_size_align([&e.ok, &e.err].into_iter()) + } + } + } + + pub(crate) fn record_size_align<'a>( + &self, + fields: impl Iterator, + ) -> (usize, usize) { + let mut size = 0; + let mut align = 1; + for ty in fields { + let (fsize, falign) = self.size_align(ty); + size = align_to(size, falign) + fsize; + align = align.max(falign); + } + (align_to(size, align), align) + } + + fn variant_size_align<'a>( + &self, + cases: impl ExactSizeIterator, + ) -> (usize, usize) { + let (discrim_size, mut align) = self.discrim_size_align(cases.len()); + let mut payload_size = 0; + for ty in cases { + let (csize, calign) = self.size_align(ty); + payload_size = payload_size.max(csize); + align = align.max(calign); + } + (align_to(discrim_size, align) + payload_size, align) + } + + fn discrim_size_align<'a>(&self, cases: usize) -> (usize, usize) { + match cases { + n if n <= u8::MAX as usize => (1, 1), + n if n <= u16::MAX as usize => (2, 2), + _ => (4, 4), + } + } +} diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rs new file mode 100644 index 0000000000..40694493d4 --- /dev/null +++ b/crates/environ/src/fact/trampoline.rs @@ -0,0 +1,763 @@ +//! Low-level compilation of an fused adapter function. +//! +//! This module is tasked with the top-level `compile` function which creates a +//! single WebAssembly function which will perform the steps of the fused +//! adapter for an `AdapterData` provided. This is the "meat" of compilation +//! where the validation of the canonical ABI or similar all happens to +//! translate arguments from one module to another. +//! +//! ## Traps and their ordering +//! +//! Currently this compiler is pretty "loose" about the ordering of precisely +//! what trap happens where. The main reason for this is that to core wasm all +//! traps are the same and for fused adapters if a trap happens no intermediate +//! side effects are visible (as designed by the canonical ABI itself). For this +//! it's important to note that some of the precise choices of control flow here +//! can be somewhat arbitrary, an intentional decision. + +use crate::component::{ + InterfaceType, TypeRecordIndex, TypeTupleIndex, FLAG_MAY_ENTER, FLAG_MAY_LEAVE, + MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, +}; +use crate::fact::signature::{align_to, Signature}; +use crate::fact::traps::Trap; +use crate::fact::{AdapterData, Context, Module, Options}; +use crate::GlobalIndex; +use std::collections::HashMap; +use std::mem; +use std::ops::Range; +use wasm_encoder::{BlockType, Encode, Instruction, Instruction::*, MemArg, ValType}; + +struct Compiler<'a> { + /// The module that the adapter will eventually be inserted into. + module: &'a Module<'a>, + + /// Metadata about the adapter that is being compiled. + adapter: &'a AdapterData, + + /// The encoded WebAssembly function body so far, not including locals. + code: Vec, + + /// Generated locals that this function will use. + /// + /// The first entry in the tuple is the number of locals and the second + /// entry is the type of those locals. This is pushed during compilation as + /// locals become necessary. + locals: Vec<(u32, ValType)>, + + /// Total number of locals generated so far. + nlocals: u32, + + /// Metadata about all `unreachable` trap instructions in this function and + /// what the trap represents. The offset within `self.code` is recorded as + /// well. + traps: Vec<(usize, Trap)>, + + /// The function signature of the lowered half of this trampoline, or the + /// signature of the function that's being generated. + lower_sig: &'a Signature, + + /// The function signature of the lifted half of this trampoline, or the + /// signature of the function that's imported the trampoline will call. + lift_sig: &'a Signature, +} + +pub(super) fn compile(module: &Module<'_>, adapter: &AdapterData) -> (Vec, Vec<(usize, Trap)>) { + let lower_sig = &module.signature(adapter.lower.ty, Context::Lower); + let lift_sig = &module.signature(adapter.lift.ty, Context::Lift); + Compiler { + module, + adapter, + code: Vec::new(), + locals: Vec::new(), + nlocals: lower_sig.params.len() as u32, + traps: Vec::new(), + lower_sig, + lift_sig, + } + .compile() +} + +/// Possible ways that a interface value is represented in the core wasm +/// canonical ABI. +enum Source<'a> { + /// This value is stored on the "stack" in wasm locals. + /// + /// This could mean that it's inline from the parameters to the function or + /// that after a function call the results were stored in locals and the + /// locals are the inline results. + Stack(Stack<'a>), + + /// This value is stored in linear memory described by the `Memory` + /// structure. + Memory(Memory), +} + +/// Same as `Source` but for where values are translated into. +enum Destination { + /// This value is destined for the WebAssembly stack which means that + /// results are simply pushed as we go along. + Stack, + + /// This value is to be placed in linear memory described by `Memory`. + Memory(Memory), +} + +struct Stack<'a> { + /// The locals that comprise a particular value. + /// + /// The length of this list represents the flattened list of types that make + /// up the component value. Each list has the index of the local being + /// accessed as well as the type of the local itself. + locals: &'a [(u32, ValType)], +} + +/// Representation of where a value is going to be stored in linear memory. +struct Memory { + /// The index of the local that contains the base address of where the + /// storage is happening. + addr_local: u32, + /// A "static" offset that will be baked into wasm instructions for where + /// memory loads/stores happen. + offset: u32, + /// The index of memory in the wasm module memory index space that this + /// memory is referring to. + memory_idx: u32, +} + +impl Compiler<'_> { + fn compile(&mut self) -> (Vec, Vec<(usize, Trap)>) { + // Check the instance flags required for this trampoline. + // + // This inserts the initial check required by `canon_lower` that the + // caller instance can be left and additionally checks the + // flags on the callee if necessary whether it can be entered. + self.trap_if_not_flag(self.adapter.lower.flags, FLAG_MAY_LEAVE, Trap::CannotLeave); + if self.adapter.called_as_export { + self.trap_if_not_flag(self.adapter.lift.flags, FLAG_MAY_ENTER, Trap::CannotEnter); + self.set_flag(self.adapter.lift.flags, FLAG_MAY_ENTER, false); + } else if self.module.debug { + self.assert_not_flag( + self.adapter.lift.flags, + FLAG_MAY_ENTER, + "may_enter should be unset", + ); + } + + // Perform the translation of arguments. Note that `FLAG_MAY_LEAVE` is + // cleared around this invocation for the callee as per the + // `canon_lift` definition in the spec. Additionally note that the + // precise ordering of traps here is not required since internal state + // is not visible to either instance and a trap will "lock down" both + // instances to no longer be visible. This means that we're free to + // reorder lifts/lowers and flags and such as is necessary and + // convenient here. + // + // TODO: if translation doesn't actually call any functions in either + // instance then there's no need to set/clear the flag here and that can + // be optimized away. + self.set_flag(self.adapter.lift.flags, FLAG_MAY_LEAVE, false); + let param_locals = self + .lower_sig + .params + .iter() + .enumerate() + .map(|(i, ty)| (i as u32, *ty)) + .collect::>(); + self.translate_params(¶m_locals); + self.set_flag(self.adapter.lift.flags, FLAG_MAY_LEAVE, true); + + // With all the arguments on the stack the actual target function is + // now invoked. The core wasm results of the function are then placed + // into locals for result translation afterwards. + self.instruction(Call(self.adapter.callee.as_u32())); + let mut result_locals = Vec::with_capacity(self.lift_sig.results.len()); + for ty in self.lift_sig.results.iter().rev() { + let local = self.gen_local(*ty); + self.instruction(LocalSet(local)); + result_locals.push((local, *ty)); + } + result_locals.reverse(); + + // Like above during the translation of results the caller cannot be + // left (as we might invoke things like `realloc`). Again the precise + // order of everything doesn't matter since intermediate states cannot + // be witnessed, hence the setting of flags here to encapsulate both + // liftings and lowerings. + // + // TODO: like above the management of the `MAY_LEAVE` flag can probably + // be elided here for "simple" results. + self.set_flag(self.adapter.lower.flags, FLAG_MAY_LEAVE, false); + self.translate_results(¶m_locals, &result_locals); + self.set_flag(self.adapter.lower.flags, FLAG_MAY_LEAVE, true); + + // And finally post-return state is handled here once all results/etc + // are all translated. + if let Some(func) = self.adapter.lift.post_return { + for (result, _) in result_locals.iter() { + self.instruction(LocalGet(*result)); + } + self.instruction(Call(func.as_u32())); + } + if self.adapter.called_as_export { + self.set_flag(self.adapter.lift.flags, FLAG_MAY_ENTER, true); + } + + self.finish() + } + + fn translate_params(&mut self, param_locals: &[(u32, ValType)]) { + let src_tys = &self.module.types[self.adapter.lower.ty].params; + let src_tys = src_tys.iter().map(|(_, ty)| *ty).collect::>(); + let dst_tys = &self.module.types[self.adapter.lift.ty].params; + let dst_tys = dst_tys.iter().map(|(_, ty)| *ty).collect::>(); + + // TODO: handle subtyping + assert_eq!(src_tys.len(), dst_tys.len()); + + let src_flat = self.module.flatten_types(src_tys.iter().copied()); + let dst_flat = self.module.flatten_types(dst_tys.iter().copied()); + + let src = if src_flat.len() <= MAX_FLAT_PARAMS { + Source::Stack(Stack { + locals: ¶m_locals[..src_flat.len()], + }) + } else { + // If there are too many parameters then that means the parameters + // are actually a tuple stored in linear memory addressed by the + // first parameter local. + let (addr, ty) = param_locals[0]; + assert_eq!(ty, self.adapter.lower.ptr()); + let align = src_tys + .iter() + .map(|t| self.module.align(t)) + .max() + .unwrap_or(1); + Source::Memory(self.memory_operand(&self.adapter.lower, addr, align)) + }; + + let dst = if dst_flat.len() <= MAX_FLAT_PARAMS { + Destination::Stack + } else { + // If there are too many parameters then space is allocated in the + // destination module for the parameters via its `realloc` function. + let (size, align) = self.module.record_size_align(dst_tys.iter()); + Destination::Memory(self.malloc(&self.adapter.lift, size, align)) + }; + + let srcs = src + .record_field_sources(self.module, src_tys.iter().copied()) + .zip(src_tys.iter()); + let dsts = dst + .record_field_sources(self.module, dst_tys.iter().copied()) + .zip(dst_tys.iter()); + for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) { + self.translate(&src_ty, &src, &dst_ty, &dst); + } + + // If the destination was linear memory instead of the stack then the + // actual parameter that we're passing is the address of the values + // stored, so ensure that's happening in the wasm body here. + if let Destination::Memory(mem) = dst { + self.instruction(LocalGet(mem.addr_local)); + } + } + + fn translate_results( + &mut self, + param_locals: &[(u32, ValType)], + result_locals: &[(u32, ValType)], + ) { + let src_ty = self.module.types[self.adapter.lift.ty].result; + let dst_ty = self.module.types[self.adapter.lower.ty].result; + + let src_flat = self.module.flatten_types([src_ty]); + let dst_flat = self.module.flatten_types([dst_ty]); + + let src = if src_flat.len() <= MAX_FLAT_RESULTS { + Source::Stack(Stack { + locals: result_locals, + }) + } else { + // The original results to read from in this case come from the + // return value of the function itself. The imported function will + // return a linear memory address at which the values can be read + // from. + let align = self.module.align(&src_ty); + assert_eq!(result_locals.len(), 1); + let (addr, ty) = result_locals[0]; + assert_eq!(ty, self.adapter.lift.ptr()); + Source::Memory(self.memory_operand(&self.adapter.lift, addr, align)) + }; + + let dst = if dst_flat.len() <= MAX_FLAT_RESULTS { + Destination::Stack + } else { + // This is slightly different than `translate_params` where the + // return pointer was provided by the caller of this function + // meaning the last parameter local is a pointer into linear memory. + let align = self.module.align(&dst_ty); + let (addr, ty) = *param_locals.last().expect("no retptr"); + assert_eq!(ty, self.adapter.lower.ptr()); + Destination::Memory(self.memory_operand(&self.adapter.lower, addr, align)) + }; + + self.translate(&src_ty, &src, &dst_ty, &dst); + } + + fn translate( + &mut self, + src_ty: &InterfaceType, + src: &Source<'_>, + dst_ty: &InterfaceType, + dst: &Destination, + ) { + if let Source::Memory(mem) = src { + self.assert_aligned(src_ty, mem); + } + if let Destination::Memory(mem) = dst { + self.assert_aligned(dst_ty, mem); + } + match src_ty { + InterfaceType::Unit => self.translate_unit(src, dst_ty, dst), + InterfaceType::Bool => self.translate_bool(src, dst_ty, dst), + InterfaceType::U8 => self.translate_u8(src, dst_ty, dst), + InterfaceType::U32 => self.translate_u32(src, dst_ty, dst), + InterfaceType::Record(t) => self.translate_record(*t, src, dst_ty, dst), + InterfaceType::Tuple(t) => self.translate_tuple(*t, src, dst_ty, dst), + + InterfaceType::String => { + // consider this field used for now until this is fully + // implemented. + drop(&self.adapter.lift.string_encoding); + unimplemented!("don't know how to translate strings") + } + + // TODO: this needs to be filled out for all the other interface + // types. + ty => unimplemented!("don't know how to translate {ty:?}"), + } + } + + fn translate_unit(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { + // TODO: subtyping + assert!(matches!(dst_ty, InterfaceType::Unit)); + drop((src, dst)); + } + + fn translate_bool(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { + // TODO: subtyping + assert!(matches!(dst_ty, InterfaceType::Bool)); + self.push_dst_addr(dst); + + // Booleans are canonicalized to 0 or 1 as they pass through the + // component boundary, so use a `select` instruction to do so. + self.instruction(I32Const(1)); + self.instruction(I32Const(0)); + match src { + Source::Memory(mem) => self.i32_load8u(mem), + Source::Stack(stack) => self.stack_get(stack, ValType::I32), + } + self.instruction(Select); + + match dst { + Destination::Memory(mem) => self.i32_store8(mem), + Destination::Stack => {} + } + } + + fn translate_u8(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { + // TODO: subtyping + assert!(matches!(dst_ty, InterfaceType::U8)); + self.push_dst_addr(dst); + match src { + Source::Memory(mem) => self.i32_load8u(mem), + Source::Stack(stack) => self.stack_get(stack, ValType::I32), + } + match dst { + Destination::Memory(mem) => self.i32_store8(mem), + Destination::Stack => {} + } + } + + fn translate_u32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) { + // TODO: subtyping + assert!(matches!(dst_ty, InterfaceType::U32)); + self.push_dst_addr(dst); + match src { + Source::Memory(mem) => self.i32_load(mem), + Source::Stack(stack) => self.stack_get(stack, ValType::I32), + } + match dst { + Destination::Memory(mem) => self.i32_store(mem), + Destination::Stack => {} + } + } + + fn translate_record( + &mut self, + src_ty: TypeRecordIndex, + src: &Source<'_>, + dst_ty: &InterfaceType, + dst: &Destination, + ) { + let src_ty = &self.module.types[src_ty]; + let dst_ty = match dst_ty { + InterfaceType::Record(r) => &self.module.types[*r], + _ => panic!("expected a record"), + }; + + // TODO: subtyping + assert_eq!(src_ty.fields.len(), dst_ty.fields.len()); + + // First a map is made of the source fields to where they're coming + // from (e.g. which offset or which locals). This map is keyed by the + // fields' names + let mut src_fields = HashMap::new(); + for (i, src) in src + .record_field_sources(self.module, src_ty.fields.iter().map(|f| f.ty)) + .enumerate() + { + let field = &src_ty.fields[i]; + src_fields.insert(&field.name, (src, &field.ty)); + } + + // .. and next translation is performed in the order of the destination + // fields in case the destination is the stack to ensure that the stack + // has the fields all in the right order. + // + // Note that the lookup in `src_fields` is an infallible lookup which + // will panic if the field isn't found. + // + // TODO: should that lookup be fallible with subtyping? + for (i, dst) in dst + .record_field_sources(self.module, dst_ty.fields.iter().map(|f| f.ty)) + .enumerate() + { + let field = &dst_ty.fields[i]; + let (src, src_ty) = &src_fields[&field.name]; + self.translate(src_ty, src, &field.ty, &dst); + } + } + + fn translate_tuple( + &mut self, + src_ty: TypeTupleIndex, + src: &Source<'_>, + dst_ty: &InterfaceType, + dst: &Destination, + ) { + let src_ty = &self.module.types[src_ty]; + let dst_ty = match dst_ty { + InterfaceType::Tuple(t) => &self.module.types[*t], + _ => panic!("expected a tuple"), + }; + + // TODO: subtyping + assert_eq!(src_ty.types.len(), dst_ty.types.len()); + + let srcs = src + .record_field_sources(self.module, src_ty.types.iter().copied()) + .zip(src_ty.types.iter()); + let dsts = dst + .record_field_sources(self.module, dst_ty.types.iter().copied()) + .zip(dst_ty.types.iter()); + for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) { + self.translate(src_ty, &src, dst_ty, &dst); + } + } + + fn trap_if_not_flag(&mut self, flags_global: GlobalIndex, flag_to_test: i32, trap: Trap) { + self.instruction(GlobalGet(flags_global.as_u32())); + self.instruction(I32Const(flag_to_test)); + self.instruction(I32And); + self.instruction(I32Eqz); + self.instruction(If(BlockType::Empty)); + self.trap(trap); + self.instruction(End); + } + + fn assert_not_flag(&mut self, flags_global: GlobalIndex, flag_to_test: i32, msg: &'static str) { + self.instruction(GlobalGet(flags_global.as_u32())); + self.instruction(I32Const(flag_to_test)); + self.instruction(I32And); + self.instruction(If(BlockType::Empty)); + self.trap(Trap::AssertFailed(msg)); + self.instruction(End); + } + + fn set_flag(&mut self, flags_global: GlobalIndex, flag_to_set: i32, value: bool) { + self.instruction(GlobalGet(flags_global.as_u32())); + if value { + self.instruction(I32Const(flag_to_set)); + self.instruction(I32Or); + } else { + self.instruction(I32Const(!flag_to_set)); + self.instruction(I32And); + } + self.instruction(GlobalSet(flags_global.as_u32())); + } + + fn verify_aligned(&mut self, local: u32, align: usize) { + // If the alignment is 1 then everything is trivially aligned and the + // check can be omitted. + if align == 1 { + return; + } + self.instruction(LocalGet(local)); + assert!(align.is_power_of_two()); + let mask = i32::try_from(align - 1).unwrap(); + self.instruction(I32Const(mask)); + self.instruction(I32And); + self.instruction(If(BlockType::Empty)); + self.trap(Trap::UnalignedPointer); + self.instruction(End); + } + + fn assert_aligned(&mut self, ty: &InterfaceType, mem: &Memory) { + if !self.module.debug { + return; + } + let align = self.module.align(ty); + if align == 1 { + return; + } + assert!(align.is_power_of_two()); + self.instruction(LocalGet(mem.addr_local)); + self.instruction(I32Const(mem.i32_offset())); + self.instruction(I32Add); + let mask = i32::try_from(align - 1).unwrap(); + self.instruction(I32Const(mask)); + self.instruction(I32And); + self.instruction(If(BlockType::Empty)); + self.trap(Trap::AssertFailed("pointer not aligned")); + self.instruction(End); + } + + fn malloc(&mut self, opts: &Options, size: usize, align: usize) -> Memory { + let addr_local = self.gen_local(opts.ptr()); + let realloc = opts.realloc.unwrap(); + if opts.memory64 { + self.instruction(I64Const(0)); + self.instruction(I64Const(0)); + self.instruction(I64Const(i64::try_from(align).unwrap())); + self.instruction(I64Const(i64::try_from(size).unwrap())); + } else { + self.instruction(I32Const(0)); + self.instruction(I32Const(0)); + self.instruction(I32Const(i32::try_from(align).unwrap())); + self.instruction(I32Const(i32::try_from(size).unwrap())); + } + self.instruction(Call(realloc.as_u32())); + self.instruction(LocalSet(addr_local)); + self.memory_operand(opts, addr_local, align) + } + + fn memory_operand(&mut self, opts: &Options, addr_local: u32, align: usize) -> Memory { + let memory = opts.memory.unwrap(); + self.verify_aligned(addr_local, align); + Memory { + addr_local, + offset: 0, + memory_idx: memory.as_u32(), + } + } + + fn gen_local(&mut self, ty: ValType) -> u32 { + // TODO: see if local reuse is necessary, right now this always + // generates a new local. + match self.locals.last_mut() { + Some((cnt, prev_ty)) if ty == *prev_ty => *cnt += 1, + _ => self.locals.push((1, ty)), + } + self.nlocals += 1; + self.nlocals - 1 + } + + fn instruction(&mut self, instr: Instruction) { + instr.encode(&mut self.code); + } + + fn trap(&mut self, trap: Trap) { + self.traps.push((self.code.len(), trap)); + self.instruction(Unreachable); + } + + fn finish(&mut self) -> (Vec, Vec<(usize, Trap)>) { + self.instruction(End); + + let mut bytes = Vec::new(); + + // Encode all locals used for this function + self.locals.len().encode(&mut bytes); + for (count, ty) in self.locals.iter() { + count.encode(&mut bytes); + ty.encode(&mut bytes); + } + + // Factor in the size of the encodings of locals into the offsets of + // traps. + for (offset, _) in self.traps.iter_mut() { + *offset += bytes.len(); + } + + // Then append the function we built and return + bytes.extend_from_slice(&self.code); + (bytes, mem::take(&mut self.traps)) + } + + fn stack_get(&mut self, stack: &Stack<'_>, dst_ty: ValType) { + assert_eq!(stack.locals.len(), 1); + let (idx, src_ty) = stack.locals[0]; + self.instruction(LocalGet(idx)); + match (src_ty, dst_ty) { + (ValType::I32, ValType::I32) + | (ValType::I64, ValType::I64) + | (ValType::F32, ValType::F32) + | (ValType::F64, ValType::F64) => {} + + (ValType::I32, ValType::F32) => self.instruction(F32ReinterpretI32), + (ValType::I64, ValType::I32) => self.instruction(I32WrapI64), + (ValType::I64, ValType::F64) => self.instruction(F64ReinterpretI64), + (ValType::F64, ValType::F32) => self.instruction(F32DemoteF64), + (ValType::I64, ValType::F32) => { + self.instruction(F64ReinterpretI64); + self.instruction(F32DemoteF64); + } + + // should not be possible given the `join` function for variants + (ValType::I32, ValType::I64) + | (ValType::I32, ValType::F64) + | (ValType::F32, ValType::I32) + | (ValType::F32, ValType::I64) + | (ValType::F32, ValType::F64) + | (ValType::F64, ValType::I32) + | (ValType::F64, ValType::I64) + + // not used in the component model + | (ValType::ExternRef, _) + | (_, ValType::ExternRef) + | (ValType::FuncRef, _) + | (_, ValType::FuncRef) + | (ValType::V128, _) + | (_, ValType::V128) => { + panic!("cannot get {dst_ty:?} from {src_ty:?} local"); + } + } + } + + fn i32_load8u(&mut self, mem: &Memory) { + self.instruction(LocalGet(mem.addr_local)); + self.instruction(I32Load8_U(mem.memarg(0))); + } + + fn i32_load(&mut self, mem: &Memory) { + self.instruction(LocalGet(mem.addr_local)); + self.instruction(I32Load(mem.memarg(2))); + } + + fn push_dst_addr(&mut self, dst: &Destination) { + if let Destination::Memory(mem) = dst { + self.instruction(LocalGet(mem.addr_local)); + } + } + + fn i32_store8(&mut self, mem: &Memory) { + self.instruction(I32Store8(mem.memarg(0))); + } + + fn i32_store(&mut self, mem: &Memory) { + self.instruction(I32Store(mem.memarg(2))); + } +} + +impl<'a> Source<'a> { + /// Given this `Source` returns an iterator over the `Source` for each of + /// the component `fields` specified. + /// + /// This will automatically slice stack-based locals to the appropriate + /// width for each component type and additionally calculate the appropriate + /// offset for each memory-based type. + fn record_field_sources<'b>( + &'b self, + module: &'b Module, + fields: impl IntoIterator + 'b, + ) -> impl Iterator> + 'b + where + 'a: 'b, + { + let mut offset = 0; + fields.into_iter().map(move |ty| match self { + Source::Memory(mem) => { + let (size, align) = module.size_align(&ty); + offset = align_to(offset, align) + size; + Source::Memory(mem.bump(offset - size)) + } + Source::Stack(stack) => { + let cnt = module.flatten_types([ty]).len(); + offset += cnt; + Source::Stack(stack.slice(offset - cnt..offset)) + } + }) + } +} + +impl Destination { + /// Same as `Source::record_field_sources` but for destinations. + fn record_field_sources<'a>( + &'a self, + module: &'a Module, + fields: impl IntoIterator + 'a, + ) -> impl Iterator + 'a { + let mut offset = 0; + fields.into_iter().map(move |ty| match self { + // TODO: dedupe with above? + Destination::Memory(mem) => { + let (size, align) = module.size_align(&ty); + offset = align_to(offset, align) + size; + Destination::Memory(mem.bump(offset - size)) + } + Destination::Stack => Destination::Stack, + }) + } +} + +impl Memory { + fn i32_offset(&self) -> i32 { + self.offset as i32 + } + + fn memarg(&self, align: u32) -> MemArg { + MemArg { + offset: u64::from(self.offset), + align, + memory_index: self.memory_idx, + } + } + + fn bump(&self, offset: usize) -> Memory { + Memory { + addr_local: self.addr_local, + memory_idx: self.memory_idx, + offset: self.offset + u32::try_from(offset).unwrap(), + } + } +} + +impl<'a> Stack<'a> { + fn slice(&self, range: Range) -> Stack<'a> { + Stack { + locals: &self.locals[range], + } + } +} + +impl Options { + fn ptr(&self) -> ValType { + if self.memory64 { + ValType::I64 + } else { + ValType::I32 + } + } +} diff --git a/crates/environ/src/fact/traps.rs b/crates/environ/src/fact/traps.rs new file mode 100644 index 0000000000..6b5410ef1b --- /dev/null +++ b/crates/environ/src/fact/traps.rs @@ -0,0 +1,105 @@ +//! Module used to encode failure messages associated with traps in an adapter +//! module. +//! +//! This module is a bit forward-looking in an attempt to help assist with +//! debugging issues with adapter modules and their implementation. This isn't +//! actually wired up to any decoder at this time and may end up getting deleted +//! entirely depending on how things go. +//! +//! Currently in core wasm the `unreachable` instruction and other traps have no +//! ability to assign failure messages with traps. This means that if an adapter +//! fails all you have is the program counter into the wasm function, but +//! there's not actually any source corresponding to wasm adapters either. This +//! module is an attempt to assign optional string messages to `unreachable` +//! trap instructions so, when sufficient debugging options are enabled, these +//! trap messages could be displayed instead of a bland "unreachable" trap +//! message. +//! +//! This information is currently encoded as a custom section in the wasm +//! module. + +use std::collections::HashMap; +use std::fmt; +use wasm_encoder::Encode; + +#[derive(Hash, PartialEq, Eq, Copy, Clone)] +pub enum Trap { + CannotLeave, + CannotEnter, + UnalignedPointer, + AssertFailed(&'static str), +} + +#[derive(Default)] +pub struct TrapSection { + trap_to_index: HashMap, + trap_list: Vec, + function_traps: Vec<(u32, Vec<(usize, usize)>)>, +} + +impl TrapSection { + /// Appends a list of traps found within a function. + /// + /// The `func` is the core wasm function index that is being described. The + /// `traps` is a list of `(offset, trap)` where `offset` is the offset + /// within the function body itself and `trap` is the description of the + /// trap of the opcode at that offset. + pub fn append(&mut self, func: u32, traps: Vec<(usize, Trap)>) { + if traps.is_empty() { + return; + } + + // Deduplicate `Trap` annotations to avoid repeating the trap string + // internally within the custom section. + let traps = traps + .into_iter() + .map(|(offset, trap)| { + let trap = *self.trap_to_index.entry(trap).or_insert_with(|| { + let idx = self.trap_list.len(); + self.trap_list.push(trap); + idx + }); + (offset, trap) + }) + .collect(); + self.function_traps.push((func, traps)); + } + + /// Creates the custom section payload of this section to be encoded into a + /// core wasm module. + pub fn finish(self) -> Vec { + let mut data = Vec::new(); + + // First append all trap messages which will be indexed below. + self.trap_list.len().encode(&mut data); + for trap in self.trap_list.iter() { + trap.to_string().encode(&mut data); + } + + // Afterwards encode trap information for all known functions where + // offsets are relative to the body of the function index specified and + // the trap message is a pointer into the table built above this. + self.function_traps.len().encode(&mut data); + for (func, traps) in self.function_traps.iter() { + func.encode(&mut data); + traps.len().encode(&mut data); + for (func_offset, trap_message) in traps { + func_offset.encode(&mut data); + trap_message.encode(&mut data); + } + } + + data + } +} + +impl fmt::Display for Trap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Trap::CannotLeave => "cannot leave instance".fmt(f), + Trap::CannotEnter => "cannot enter instance".fmt(f), + Trap::UnalignedPointer => "pointer not aligned correctly".fmt(f), + Trap::AssertFailed(s) => write!(f, "assertion failure: {}", s), + } + } +} diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index b0d414564b..f74794a291 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -31,6 +31,7 @@ mod module_environ; mod module_types; pub mod obj; mod ref_bits; +mod scopevec; mod stack_map; mod trap_encoding; mod tunables; @@ -43,6 +44,7 @@ pub use crate::module::*; pub use crate::module_environ::*; pub use crate::module_types::*; pub use crate::ref_bits::*; +pub use crate::scopevec::ScopeVec; pub use crate::stack_map::StackMap; pub use crate::trap_encoding::*; pub use crate::tunables::Tunables; @@ -51,6 +53,8 @@ pub use object; #[cfg(feature = "component-model")] pub mod component; +#[cfg(feature = "component-model")] +mod fact; // Reexport all of these type-level since they're quite commonly used and it's // much easier to refer to everything through one crate rather than importing diff --git a/crates/environ/src/scopevec.rs b/crates/environ/src/scopevec.rs new file mode 100644 index 0000000000..6c7e58a163 --- /dev/null +++ b/crates/environ/src/scopevec.rs @@ -0,0 +1,57 @@ +use std::cell::RefCell; + +/// Small data structure to help extend the lifetime of a slice to a higher +/// scope. +/// +/// This is currently used during component translation where translation in +/// general works on a borrowed slice which contains all input modules, but +/// generated adapter modules for components don't live within the original +/// slice but the data structures are much easier if the dynamically generated +/// adapter modules live for the same lifetime as the original input slice. To +/// solve this problem this `ScopeVec` helper is used to move ownership of a +/// `Vec` to a higher scope in the program, then borrowing the slice from +/// that scope. +pub struct ScopeVec { + data: RefCell>>, +} + +impl ScopeVec { + /// Creates a new blank scope. + pub fn new() -> ScopeVec { + ScopeVec { + data: Default::default(), + } + } + + /// Transfers ownership of `data` into this scope and then yields the slice + /// back to the caller. + /// + /// The original data will be deallocated when `self` is dropped. + pub fn push(&self, data: Vec) -> &mut [T] { + let mut data: Box<[T]> = data.into(); + let ptr = data.as_mut_ptr(); + let len = data.len(); + self.data.borrow_mut().push(data); + + // This should be safe for a few reasons: + // + // * The returned pointer on the heap that `data` owns. Despite moving + // `data` around it doesn't actually move the slice itself around, so + // the pointer returned should be valid (and length). + // + // * The lifetime of the returned pointer is connected to the lifetime + // of `self`. This reflects how when `self` is destroyed the `data` is + // destroyed as well, or otherwise the returned slice will be valid + // for as long as `self` is valid since `self` owns the original data + // at that point. + // + // * This function was given ownership of `data` so it should be safe to + // hand back a mutable reference. Once placed within a `ScopeVec` the + // data is never mutated so the caller will enjoy exclusive access to + // the slice of the original vec. + // + // This all means that it should be safe to return a mutable slice of + // all of `data` after the data has been pushed onto our internal list. + unsafe { std::slice::from_raw_parts_mut(ptr, len) } + } +} diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs index 01d333810a..59992f60d6 100644 --- a/crates/environ/src/tunables.rs +++ b/crates/environ/src/tunables.rs @@ -41,6 +41,10 @@ pub struct Tunables { /// Indicates whether an address map from compiled native code back to wasm /// offsets in the original file is generated. pub generate_address_map: bool, + + /// Flag for the component module whether adapter modules have debug + /// assertions baked into them. + pub debug_adapter_modules: bool, } impl Default for Tunables { @@ -86,6 +90,7 @@ impl Default for Tunables { static_memory_bound_is_maximum: false, guard_before_linear_memory: true, generate_address_map: true, + debug_adapter_modules: false, } } } diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 347bec97a6..96535b2465 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -126,6 +126,13 @@ pub trait PtrSize { fn size_of_vmcaller_checked_anyfunc(&self) -> u8 { 3 * self.size() } + + /// Return the size of `VMGlobalDefinition`; this is the size of the largest value type (i.e. a + /// V128). + #[inline] + fn size_of_vmglobal_definition(&self) -> u8 { + 16 + } } /// Type representing the size of a pointer for the current compilation host @@ -355,7 +362,7 @@ impl From> for VMOffsets

{ = cmul(ret.num_owned_memories, ret.size_of_vmmemory_definition()), align(16), size(defined_globals) - = cmul(ret.num_defined_globals, ret.size_of_vmglobal_definition()), + = cmul(ret.num_defined_globals, ret.ptr.size_of_vmglobal_definition()), size(defined_anyfuncs) = cmul( ret.num_escaped_funcs, ret.ptr.size_of_vmcaller_checked_anyfunc(), @@ -524,16 +531,6 @@ impl VMOffsets

{ } } -/// Offsets for `VMGlobalDefinition`. -impl VMOffsets

{ - /// Return the size of `VMGlobalDefinition`; this is the size of the largest value type (i.e. a - /// V128). - #[inline] - pub fn size_of_vmglobal_definition(&self) -> u8 { - 16 - } -} - /// Offsets for `VMSharedSignatureIndex`. impl VMOffsets

{ /// Return the size of `VMSharedSignatureIndex`. @@ -729,7 +726,8 @@ impl VMOffsets

{ #[inline] pub fn vmctx_vmglobal_definition(&self, index: DefinedGlobalIndex) -> u32 { assert_lt!(index.as_u32(), self.num_defined_globals); - self.vmctx_globals_begin() + index.as_u32() * u32::from(self.size_of_vmglobal_definition()) + self.vmctx_globals_begin() + + index.as_u32() * u32::from(self.ptr.size_of_vmglobal_definition()) } /// Return the offset to the `VMCallerCheckedAnyfunc` for the given function diff --git a/crates/runtime/src/component.rs b/crates/runtime/src/component.rs index ddc939d0c8..a8cf3309de 100644 --- a/crates/runtime/src/component.rs +++ b/crates/runtime/src/component.rs @@ -7,8 +7,8 @@ //! cranelift-compiled adapters, will use this `VMComponentContext` as well. use crate::{ - Store, VMCallerCheckedAnyfunc, VMFunctionBody, VMMemoryDefinition, VMOpaqueContext, - VMSharedSignatureIndex, ValRaw, + Store, VMCallerCheckedAnyfunc, VMFunctionBody, VMGlobalDefinition, VMMemoryDefinition, + VMOpaqueContext, VMSharedSignatureIndex, ValRaw, }; use memoffset::offset_of; use std::alloc::{self, Layout}; @@ -19,8 +19,7 @@ use std::ptr::{self, NonNull}; use wasmtime_environ::component::{ Component, LoweredIndex, RuntimeAlwaysTrapIndex, RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex, RuntimeReallocIndex, StringEncoding, - VMComponentOffsets, VMCOMPONENT_FLAG_MAY_ENTER, VMCOMPONENT_FLAG_MAY_LEAVE, - VMCOMPONENT_FLAG_NEEDS_POST_RETURN, VMCOMPONENT_MAGIC, + VMComponentOffsets, FLAG_MAY_ENTER, FLAG_MAY_LEAVE, FLAG_NEEDS_POST_RETURN, VMCOMPONENT_MAGIC, }; use wasmtime_environ::HostPtr; @@ -75,7 +74,7 @@ pub struct ComponentInstance { pub type VMLoweringCallee = extern "C" fn( vmctx: *mut VMOpaqueContext, data: *mut u8, - flags: *mut VMComponentFlags, + flags: InstanceFlags, opt_memory: *mut VMMemoryDefinition, opt_realloc: *mut VMCallerCheckedAnyfunc, string_encoding: StringEncoding, @@ -114,11 +113,6 @@ pub struct VMComponentContext { _marker: marker::PhantomPinned, } -/// Flags stored in a `VMComponentContext` with values defined by -/// `VMCOMPONENT_FLAG_*` -#[repr(transparent)] -pub struct VMComponentFlags(u8); - impl ComponentInstance { /// Returns the layout corresponding to what would be an allocation of a /// `ComponentInstance` for the `offsets` provided. @@ -174,8 +168,8 @@ impl ComponentInstance { /// Returns a pointer to the "may leave" flag for this instance specified /// for canonical lowering and lifting operations. - pub fn flags(&self, instance: RuntimeComponentInstanceIndex) -> *mut VMComponentFlags { - unsafe { self.vmctx_plus_offset(self.offsets.flags(instance)) } + pub fn instance_flags(&self, instance: RuntimeComponentInstanceIndex) -> InstanceFlags { + unsafe { InstanceFlags(self.vmctx_plus_offset(self.offsets.instance_flags(instance))) } } /// Returns the store that this component was created with. @@ -381,7 +375,9 @@ impl ComponentInstance { *self.vmctx_plus_offset(self.offsets.store()) = store; for i in 0..self.offsets.num_runtime_component_instances { let i = RuntimeComponentInstanceIndex::from_u32(i); - *self.flags(i) = VMComponentFlags::new(); + let mut def = VMGlobalDefinition::new(); + *def.as_i32_mut() = FLAG_MAY_ENTER | FLAG_MAY_LEAVE; + *self.instance_flags(i).0 = def; } // In debug mode set non-null bad values to all "pointer looking" bits @@ -569,52 +565,57 @@ impl VMOpaqueContext { } #[allow(missing_docs)] -impl VMComponentFlags { - fn new() -> VMComponentFlags { - VMComponentFlags(VMCOMPONENT_FLAG_MAY_LEAVE | VMCOMPONENT_FLAG_MAY_ENTER) +#[repr(transparent)] +pub struct InstanceFlags(*mut VMGlobalDefinition); + +#[allow(missing_docs)] +impl InstanceFlags { + #[inline] + pub unsafe fn may_leave(&self) -> bool { + *(*self.0).as_i32() & FLAG_MAY_LEAVE != 0 } #[inline] - pub fn may_leave(&self) -> bool { - self.0 & VMCOMPONENT_FLAG_MAY_LEAVE != 0 - } - - #[inline] - pub fn set_may_leave(&mut self, val: bool) { + pub unsafe fn set_may_leave(&mut self, val: bool) { if val { - self.0 |= VMCOMPONENT_FLAG_MAY_LEAVE; + *(*self.0).as_i32_mut() |= FLAG_MAY_LEAVE; } else { - self.0 &= !VMCOMPONENT_FLAG_MAY_LEAVE; + *(*self.0).as_i32_mut() &= !FLAG_MAY_LEAVE; } } #[inline] - pub fn may_enter(&self) -> bool { - self.0 & VMCOMPONENT_FLAG_MAY_ENTER != 0 + pub unsafe fn may_enter(&self) -> bool { + *(*self.0).as_i32() & FLAG_MAY_ENTER != 0 } #[inline] - pub fn set_may_enter(&mut self, val: bool) { + pub unsafe fn set_may_enter(&mut self, val: bool) { if val { - self.0 |= VMCOMPONENT_FLAG_MAY_ENTER; + *(*self.0).as_i32_mut() |= FLAG_MAY_ENTER; } else { - self.0 &= !VMCOMPONENT_FLAG_MAY_ENTER; + *(*self.0).as_i32_mut() &= !FLAG_MAY_ENTER; } } #[inline] - pub fn needs_post_return(&self) -> bool { - self.0 & VMCOMPONENT_FLAG_NEEDS_POST_RETURN != 0 + pub unsafe fn needs_post_return(&self) -> bool { + *(*self.0).as_i32() & FLAG_NEEDS_POST_RETURN != 0 } #[inline] - pub fn set_needs_post_return(&mut self, val: bool) { + pub unsafe fn set_needs_post_return(&mut self, val: bool) { if val { - self.0 |= VMCOMPONENT_FLAG_NEEDS_POST_RETURN; + *(*self.0).as_i32_mut() |= FLAG_NEEDS_POST_RETURN; } else { - self.0 &= !VMCOMPONENT_FLAG_NEEDS_POST_RETURN; + *(*self.0).as_i32_mut() &= !FLAG_NEEDS_POST_RETURN; } } + + #[inline] + pub fn as_raw(&self) -> *mut VMGlobalDefinition { + self.0 + } } #[cfg(test)] diff --git a/crates/runtime/src/export.rs b/crates/runtime/src/export.rs index a96e77f12c..1aafb97848 100644 --- a/crates/runtime/src/export.rs +++ b/crates/runtime/src/export.rs @@ -90,9 +90,6 @@ impl From for Export { pub struct ExportGlobal { /// The address of the global storage. pub definition: *mut VMGlobalDefinition, - /// Pointer to a `VMContext` which has a lifetime at least as long as the - /// global. This may not be the `VMContext` which defines the global. - pub vmctx: *mut VMContext, /// The global declaration, used for compatibilty checking. pub global: Global, } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 08f4f4c634..aeebe249a1 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -330,7 +330,6 @@ impl Instance { } else { self.imported_global(index).from }, - vmctx: self.vmctx_ptr(), global: self.module().globals[index], } } diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index 12e4a41304..b7d343cc1b 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -334,7 +334,7 @@ mod test_vmglobal_definition { use crate::externref::VMExternRef; use more_asserts::assert_ge; use std::mem::{align_of, size_of}; - use wasmtime_environ::{Module, VMOffsets}; + use wasmtime_environ::{Module, PtrSize, VMOffsets}; #[test] fn check_vmglobal_definition_alignment() { @@ -351,7 +351,7 @@ mod test_vmglobal_definition { let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); assert_eq!( size_of::(), - usize::from(offsets.size_of_vmglobal_definition()) + usize::from(offsets.ptr.size_of_vmglobal_definition()) ); } @@ -676,7 +676,7 @@ pub struct VMInvokeArgument([u8; 16]); mod test_vm_invoke_argument { use super::VMInvokeArgument; use std::mem::{align_of, size_of}; - use wasmtime_environ::{Module, VMOffsets}; + use wasmtime_environ::{Module, PtrSize, VMOffsets}; #[test] fn check_vm_invoke_argument_alignment() { @@ -689,7 +689,7 @@ mod test_vm_invoke_argument { let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); assert_eq!( size_of::(), - usize::from(offsets.size_of_vmglobal_definition()) + usize::from(offsets.ptr.size_of_vmglobal_definition()) ); } } diff --git a/crates/wasmtime/src/component/component.rs b/crates/wasmtime/src/component/component.rs index 49071de636..c60713f26d 100644 --- a/crates/wasmtime/src/component/component.rs +++ b/crates/wasmtime/src/component/component.rs @@ -13,7 +13,7 @@ use wasmtime_environ::component::{ AlwaysTrapInfo, ComponentTypes, GlobalInitializer, LoweredIndex, LoweringInfo, RuntimeAlwaysTrapIndex, StaticModuleIndex, Translator, }; -use wasmtime_environ::{PrimaryMap, SignatureIndex, Trampoline, TrapCode}; +use wasmtime_environ::{PrimaryMap, ScopeVec, SignatureIndex, Trampoline, TrapCode}; use wasmtime_jit::CodeMemory; use wasmtime_runtime::VMFunctionBody; @@ -119,10 +119,11 @@ impl Component { let tunables = &engine.config().tunables; + let scope = ScopeVec::new(); let mut validator = wasmparser::Validator::new_with_features(engine.config().features.clone()); let mut types = Default::default(); - let (component, modules) = Translator::new(tunables, &mut validator, &mut types) + let (component, modules) = Translator::new(tunables, &mut validator, &mut types, &scope) .translate(binary) .context("failed to parse WebAssembly module")?; let types = Arc::new(types.finish()); diff --git a/crates/wasmtime/src/component/func.rs b/crates/wasmtime/src/component/func.rs index 39de8b05d1..58b0d0624c 100644 --- a/crates/wasmtime/src/component/func.rs +++ b/crates/wasmtime/src/component/func.rs @@ -9,12 +9,10 @@ use std::ptr::NonNull; use std::sync::Arc; use wasmtime_environ::component::{ CanonicalOptions, ComponentTypes, CoreDef, RuntimeComponentInstanceIndex, TypeFuncIndex, + MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, }; use wasmtime_runtime::{Export, ExportFunction, VMTrampoline}; -const MAX_STACK_PARAMS: usize = 16; -const MAX_STACK_RESULTS: usize = 1; - /// A helper macro to safely map `MaybeUninit` to `MaybeUninit` where `U` /// is a field projection within `T`. /// @@ -305,20 +303,20 @@ impl Func { self.call_raw( store, args, - |store, options, args, dst: &mut MaybeUninit<[ValRaw; MAX_STACK_PARAMS]>| { - if param_count > MAX_STACK_PARAMS { + |store, options, args, dst: &mut MaybeUninit<[ValRaw; MAX_FLAT_PARAMS]>| { + if param_count > MAX_FLAT_PARAMS { self.store_args(store, &options, ¶ms, args, dst) } else { - dst.write([ValRaw::u64(0); MAX_STACK_PARAMS]); + dst.write([ValRaw::u64(0); MAX_FLAT_PARAMS]); let dst = unsafe { - mem::transmute::<_, &mut [MaybeUninit; MAX_STACK_PARAMS]>(dst) + mem::transmute::<_, &mut [MaybeUninit; MAX_FLAT_PARAMS]>(dst) }; args.iter() .try_for_each(|arg| arg.lower(store, &options, &mut dst.iter_mut())) } }, - |store, options, src: &[ValRaw; MAX_STACK_RESULTS]| { - if result_count > MAX_STACK_RESULTS { + |store, options, src: &[ValRaw; MAX_FLAT_RESULTS]| { + if result_count > MAX_FLAT_RESULTS { Self::load_result(&Memory::new(store, &options), &result, &mut src.iter()) } else { Val::lift(&result, store, &options, &mut src.iter()) @@ -378,7 +376,7 @@ impl Func { assert!(mem::align_of_val(map_maybe_uninit!(space.ret)) == val_align); let instance = store.0[instance.0].as_ref().unwrap().instance(); - let flags = instance.flags(component_instance); + let mut flags = instance.instance_flags(component_instance); unsafe { // Test the "may enter" flag which is a "lock" on this instance. @@ -388,15 +386,15 @@ impl Func { // from this point on the instance is considered "poisoned" and can // never be entered again. The only time this flag is set to `true` // again is after post-return logic has completed successfully. - if !(*flags).may_enter() { + if !flags.may_enter() { bail!("cannot reenter component instance"); } - (*flags).set_may_enter(false); + flags.set_may_enter(false); - debug_assert!((*flags).may_leave()); - (*flags).set_may_leave(false); + debug_assert!(flags.may_leave()); + flags.set_may_leave(false); let result = lower(store, &options, params, map_maybe_uninit!(space.params)); - (*flags).set_may_leave(true); + flags.set_may_leave(true); result?; // This is unsafe as we are providing the guarantee that all the @@ -430,7 +428,7 @@ impl Func { // is currently required to be 0 or 1 values according to the // canonical ABI, is saved within the `Store`'s `FuncData`. This'll // later get used in post-return. - (*flags).set_needs_post_return(true); + flags.set_needs_post_return(true); let val = lift(store.0, &options, ret)?; let ret_slice = cast_storage(ret); let data = &mut store.0[self.0]; @@ -488,7 +486,7 @@ impl Func { let component_instance = data.component_instance; let post_return_arg = data.post_return_arg.take(); let instance = store.0[instance.0].as_ref().unwrap().instance(); - let flags = instance.flags(component_instance); + let mut flags = instance.instance_flags(component_instance); unsafe { // First assert that the instance is in a "needs post return" state. @@ -508,18 +506,18 @@ impl Func { // `post_return` on the right function despite the call being a // separate step in the API. assert!( - (*flags).needs_post_return(), + flags.needs_post_return(), "post_return can only be called after a function has previously been called", ); let post_return_arg = post_return_arg.expect("calling post_return on wrong function"); // This is a sanity-check assert which shouldn't ever trip. - assert!(!(*flags).may_enter()); + assert!(!flags.may_enter()); // Unset the "needs post return" flag now that post-return is being // processed. This will cause future invocations of this method to // panic, even if the function call below traps. - (*flags).set_needs_post_return(false); + flags.set_needs_post_return(false); // If the function actually had a `post-return` configured in its // canonical options that's executed here. @@ -539,7 +537,7 @@ impl Func { // And finally if everything completed successfully then the "may // enter" flag is set to `true` again here which enables further use // of the component. - (*flags).set_may_enter(true); + flags.set_may_enter(true); } Ok(()) } @@ -550,7 +548,7 @@ impl Func { options: &Options, params: &[Type], args: &[Val], - dst: &mut MaybeUninit<[ValRaw; MAX_STACK_PARAMS]>, + dst: &mut MaybeUninit<[ValRaw; MAX_FLAT_PARAMS]>, ) -> Result<()> { let mut size = 0; let mut alignment = 1; diff --git a/crates/wasmtime/src/component/func/host.rs b/crates/wasmtime/src/component/func/host.rs index 5f073ffc61..b76daeb494 100644 --- a/crates/wasmtime/src/component/func/host.rs +++ b/crates/wasmtime/src/component/func/host.rs @@ -1,4 +1,4 @@ -use crate::component::func::{Memory, MemoryMut, Options, MAX_STACK_PARAMS, MAX_STACK_RESULTS}; +use crate::component::func::{Memory, MemoryMut, Options}; use crate::component::{ComponentParams, ComponentType, Lift, Lower}; use crate::{AsContextMut, StoreContextMut, ValRaw}; use anyhow::{bail, Context, Result}; @@ -7,9 +7,11 @@ use std::mem::MaybeUninit; use std::panic::{self, AssertUnwindSafe}; use std::ptr::NonNull; use std::sync::Arc; -use wasmtime_environ::component::{ComponentTypes, StringEncoding, TypeFuncIndex}; +use wasmtime_environ::component::{ + ComponentTypes, StringEncoding, TypeFuncIndex, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, +}; use wasmtime_runtime::component::{ - VMComponentContext, VMComponentFlags, VMLowering, VMLoweringCallee, + InstanceFlags, VMComponentContext, VMLowering, VMLoweringCallee, }; use wasmtime_runtime::{VMCallerCheckedAnyfunc, VMMemoryDefinition, VMOpaqueContext}; @@ -27,7 +29,7 @@ pub trait IntoComponentFunc { extern "C" fn entrypoint( cx: *mut VMOpaqueContext, data: *mut u8, - flags: *mut VMComponentFlags, + flags: InstanceFlags, memory: *mut VMMemoryDefinition, realloc: *mut VMCallerCheckedAnyfunc, string_encoding: StringEncoding, @@ -106,7 +108,7 @@ where /// the select few places it's intended to be called from. unsafe fn call_host( cx: *mut VMOpaqueContext, - flags: *mut VMComponentFlags, + mut flags: InstanceFlags, memory: *mut VMMemoryDefinition, realloc: *mut VMCallerCheckedAnyfunc, string_encoding: StringEncoding, @@ -150,7 +152,7 @@ where // Perform a dynamic check that this instance can indeed be left. Exiting // the component is disallowed, for example, when the `realloc` function // calls a canonical import. - if !(*flags).may_leave() { + if !flags.may_leave() { bail!("cannot leave component instance"); } @@ -164,12 +166,12 @@ where // trivially DCE'd by LLVM. Perhaps one day with enough const programming in // Rust we can make monomorphizations of this function codegen only one // branch, but today is not that day. - if Params::flatten_count() <= MAX_STACK_PARAMS { - if Return::flatten_count() <= MAX_STACK_RESULTS { + if Params::flatten_count() <= MAX_FLAT_PARAMS { + if Return::flatten_count() <= MAX_FLAT_RESULTS { let storage = cast_storage::>(storage); let params = Params::lift(cx.0, &options, &storage.assume_init_ref().args)?; let ret = closure(cx.as_context_mut(), params)?; - (*flags).set_may_leave(false); + flags.set_may_leave(false); ret.lower(&mut cx, &options, map_maybe_uninit!(storage.ret))?; } else { let storage = cast_storage::>(storage).assume_init_ref(); @@ -177,18 +179,18 @@ where let ret = closure(cx.as_context_mut(), params)?; let mut memory = MemoryMut::new(cx.as_context_mut(), &options); let ptr = validate_inbounds::(memory.as_slice_mut(), &storage.retptr)?; - (*flags).set_may_leave(false); + flags.set_may_leave(false); ret.store(&mut memory, ptr)?; } } else { let memory = Memory::new(cx.0, &options); - if Return::flatten_count() <= MAX_STACK_RESULTS { + if Return::flatten_count() <= MAX_FLAT_RESULTS { let storage = cast_storage::>(storage); let ptr = validate_inbounds::(memory.as_slice(), &storage.assume_init_ref().args)?; let params = Params::load(&memory, &memory.as_slice()[ptr..][..Params::SIZE32])?; let ret = closure(cx.as_context_mut(), params)?; - (*flags).set_may_leave(false); + flags.set_may_leave(false); ret.lower(&mut cx, &options, map_maybe_uninit!(storage.ret))?; } else { let storage = cast_storage::>(storage).assume_init_ref(); @@ -197,12 +199,12 @@ where let ret = closure(cx.as_context_mut(), params)?; let mut memory = MemoryMut::new(cx.as_context_mut(), &options); let ptr = validate_inbounds::(memory.as_slice_mut(), &storage.retptr)?; - (*flags).set_may_leave(false); + flags.set_may_leave(false); ret.store(&mut memory, ptr)?; } } - (*flags).set_may_leave(true); + flags.set_may_leave(true); return Ok(()); } @@ -260,7 +262,7 @@ macro_rules! impl_into_component_func { extern "C" fn entrypoint( cx: *mut VMOpaqueContext, data: *mut u8, - flags: *mut VMComponentFlags, + flags: InstanceFlags, memory: *mut VMMemoryDefinition, realloc: *mut VMCallerCheckedAnyfunc, string_encoding: StringEncoding, @@ -298,7 +300,7 @@ macro_rules! impl_into_component_func { extern "C" fn entrypoint( cx: *mut VMOpaqueContext, data: *mut u8, - flags: *mut VMComponentFlags, + flags: InstanceFlags, memory: *mut VMMemoryDefinition, realloc: *mut VMCallerCheckedAnyfunc, string_encoding: StringEncoding, diff --git a/crates/wasmtime/src/component/func/typed.rs b/crates/wasmtime/src/component/func/typed.rs index 098e028003..6e93f36d7d 100644 --- a/crates/wasmtime/src/component/func/typed.rs +++ b/crates/wasmtime/src/component/func/typed.rs @@ -1,6 +1,4 @@ -use crate::component::func::{ - Func, Memory, MemoryMut, Options, MAX_STACK_PARAMS, MAX_STACK_RESULTS, -}; +use crate::component::func::{Func, Memory, MemoryMut, Options}; use crate::store::StoreOpaque; use crate::{AsContext, AsContextMut, StoreContext, StoreContextMut, ValRaw}; use anyhow::{bail, Context, Result}; @@ -9,7 +7,9 @@ use std::fmt; use std::marker; use std::mem::{self, MaybeUninit}; use std::str; -use wasmtime_environ::component::{ComponentTypes, InterfaceType, StringEncoding}; +use wasmtime_environ::component::{ + ComponentTypes, InterfaceType, StringEncoding, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, +}; /// A statically-typed version of [`Func`] which takes `Params` as input and /// returns `Return`. @@ -156,8 +156,8 @@ where // space reserved for the params/results is always of the appropriate // size (as the params/results needed differ depending on the "flatten" // count) - if Params::flatten_count() <= MAX_STACK_PARAMS { - if Return::flatten_count() <= MAX_STACK_RESULTS { + if Params::flatten_count() <= MAX_FLAT_PARAMS { + if Return::flatten_count() <= MAX_FLAT_RESULTS { self.func.call_raw( store, ¶ms, @@ -173,7 +173,7 @@ where ) } } else { - if Return::flatten_count() <= MAX_STACK_RESULTS { + if Return::flatten_count() <= MAX_FLAT_RESULTS { self.func.call_raw( store, ¶ms, @@ -203,7 +203,7 @@ where params: &Params, dst: &mut MaybeUninit, ) -> Result<()> { - assert!(Params::flatten_count() <= MAX_STACK_PARAMS); + assert!(Params::flatten_count() <= MAX_FLAT_PARAMS); params.lower(store, options, dst)?; Ok(()) } @@ -211,7 +211,7 @@ where /// Lower parameters onto a heap-allocated location. /// /// This is used when the stack space to be used for the arguments is above - /// the `MAX_STACK_PARAMS` threshold. Here the wasm's `realloc` function is + /// the `MAX_FLAT_PARAMS` threshold. Here the wasm's `realloc` function is /// invoked to allocate space and then parameters are stored at that heap /// pointer location. fn lower_heap_args( @@ -220,7 +220,7 @@ where params: &Params, dst: &mut MaybeUninit, ) -> Result<()> { - assert!(Params::flatten_count() > MAX_STACK_PARAMS); + assert!(Params::flatten_count() > MAX_FLAT_PARAMS); // Memory must exist via validation if the arguments are stored on the // heap, so we can create a `MemoryMut` at this point. Afterwards @@ -257,14 +257,14 @@ where options: &Options, dst: &Return::Lower, ) -> Result { - assert!(Return::flatten_count() <= MAX_STACK_RESULTS); + assert!(Return::flatten_count() <= MAX_FLAT_RESULTS); Return::lift(store, options, dst) } /// Lift the result of a function where the result is stored indirectly on /// the heap. fn lift_heap_result(store: &StoreOpaque, options: &Options, dst: &ValRaw) -> Result { - assert!(Return::flatten_count() > MAX_STACK_RESULTS); + assert!(Return::flatten_count() > MAX_FLAT_RESULTS); // FIXME: needs to read an i64 for memory64 let ptr = usize::try_from(dst.get_u32())?; if ptr % usize::try_from(Return::ALIGN32)? != 0 { diff --git a/crates/wasmtime/src/component/instance.rs b/crates/wasmtime/src/component/instance.rs index f0711f75a6..fd0a98cb02 100644 --- a/crates/wasmtime/src/component/instance.rs +++ b/crates/wasmtime/src/component/instance.rs @@ -12,7 +12,7 @@ use wasmtime_environ::component::{ ExtractPostReturn, ExtractRealloc, GlobalInitializer, InstantiateModule, LowerImport, RuntimeImportIndex, RuntimeInstanceIndex, RuntimeModuleIndex, }; -use wasmtime_environ::{EntityIndex, PrimaryMap}; +use wasmtime_environ::{EntityIndex, Global, GlobalInit, PrimaryMap, WasmType}; use wasmtime_runtime::component::{ComponentInstance, OwnedComponentInstance}; /// An instantiated component. @@ -132,6 +132,18 @@ impl InstanceData { anyfunc: self.state.always_trap_anyfunc(*idx), }) } + CoreDef::InstanceFlags(idx) => { + wasmtime_runtime::Export::Global(wasmtime_runtime::ExportGlobal { + definition: self.state.instance_flags(*idx).as_raw(), + global: Global { + wasm_ty: WasmType::I32, + mutability: true, + initializer: GlobalInit::I32Const(0), + }, + }) + } + // This should have been processed away during compilation. + CoreDef::Adapter(_) => unreachable!(), } } @@ -354,10 +366,28 @@ impl<'a> Instantiator<'a> { ) -> &OwnedImports { self.core_imports.clear(); self.core_imports.reserve(module); + let mut imports = module.compiled_module().module().imports(); for arg in args { - let export = self.data.lookup_def(store, arg); + // The general idea of Wasmtime is that at runtime type-checks for + // core wasm instantiations internally within a component are + // unnecessary and superfluous. Naturally though mistakes may be + // made, so double-check this property of wasmtime in debug mode. + if cfg!(debug_assertions) { + let export = self.data.lookup_def(store, arg); + let (_, _, expected) = imports.next().unwrap(); + let val = unsafe { crate::Extern::from_wasmtime_export(export, store) }; + crate::types::matching::MatchCx { + store, + engine: store.engine(), + signatures: module.signatures(), + types: module.types(), + } + .extern_(&expected, &val) + .expect("unexpected typecheck failure"); + } + let export = self.data.lookup_def(store, arg); // The unsafety here should be ok since the `export` is loaded // directly from an instance which should only give us valid export // items. @@ -365,6 +395,7 @@ impl<'a> Instantiator<'a> { self.core_imports.push_export(&export); } } + debug_assert!(imports.next().is_none()); &self.core_imports } diff --git a/crates/wasmtime/src/module/serialization.rs b/crates/wasmtime/src/module/serialization.rs index cb45b6ed79..4666763ea4 100644 --- a/crates/wasmtime/src/module/serialization.rs +++ b/crates/wasmtime/src/module/serialization.rs @@ -426,6 +426,9 @@ impl<'a> SerializedModule<'a> { // setting just fine (it's just a section in the compiled file and // whether it's present or not) generate_address_map: _, + + // Just a debugging aid, doesn't affect functionality at all. + debug_adapter_modules: _, } = self.metadata.tunables; Self::check_int( diff --git a/tests/all/component_model/func.rs b/tests/all/component_model/func.rs index 0425c38bf3..322d7a1c56 100644 --- a/tests/all/component_model/func.rs +++ b/tests/all/component_model/func.rs @@ -2363,3 +2363,50 @@ fn errors_that_poison_instance() -> Result<()> { ); } } + +#[test] +fn run_export_with_internal_adapter() -> Result<()> { + let component = r#" +(component + (type $t (func (param u32) (result u32))) + (component $a + (core module $m + (func (export "add-five") (param i32) (result i32) + local.get 0 + i32.const 5 + i32.add) + ) + (core instance $m (instantiate $m)) + (func (export "add-five") (type $t) (canon lift (core func $m "add-five"))) + ) + (component $b + (import "interface-0.1.0" (instance $i + (export "add-five" (func (type $t))))) + (core module $m + (func $add-five (import "interface-0.1.0" "add-five") (param i32) (result i32)) + (func) ;; causes index out of bounds + (func (export "run") (result i32) i32.const 0 call $add-five) + ) + (core func $add-five (canon lower (func $i "add-five"))) + (core instance $i (instantiate 0 + (with "interface-0.1.0" (instance + (export "add-five" (func $add-five)) + )) + )) + (func (result u32) (canon lift (core func $i "run"))) + (export "run" (func 1)) + ) + (instance $a (instantiate $a)) + (instance $b (instantiate $b (with "interface-0.1.0" (instance $a)))) + (export "run" (func $b "run")) +) +"#; + let engine = super::engine(); + let component = Component::new(&engine, component)?; + let mut store = Store::new(&engine, ()); + let linker = Linker::new(&engine); + let instance = linker.instantiate(&mut store, &component)?; + let run = instance.get_typed_func::<(), u32, _>(&mut store, "run")?; + assert_eq!(run.call(&mut store, ())?, 5); + Ok(()) +} diff --git a/tests/all/wast.rs b/tests/all/wast.rs index 3a34dee980..1ef00c6f3d 100644 --- a/tests/all/wast.rs +++ b/tests/all/wast.rs @@ -13,6 +13,8 @@ include!(concat!(env!("OUT_DIR"), "/wast_testsuite_tests.rs")); // function which actually executes the `wast` test suite given the `strategy` // to compile it. fn run_wast(wast: &str, strategy: Strategy, pooling: bool) -> anyhow::Result<()> { + drop(env_logger::try_init()); + match strategy { Strategy::Cranelift => {} _ => unimplemented!(), diff --git a/tests/misc_testsuite/component-model/fused.wast b/tests/misc_testsuite/component-model/fused.wast new file mode 100644 index 0000000000..c2af88ed83 --- /dev/null +++ b/tests/misc_testsuite/component-model/fused.wast @@ -0,0 +1,682 @@ +;; smoke test with no arguments and no results +(component + (core module $m + (func (export "")) + ) + (core instance $m (instantiate $m)) + (func $foo (canon lift (core func $m ""))) + + (component $c + (import "" (func $foo)) + + (core func $foo (canon lower (func $foo))) + (core module $m2 + (import "" "" (func)) + (start 0) + ) + (core instance $m2 (instantiate $m2 (with "" (instance (export "" (func $foo)))))) + ) + + (instance $c (instantiate $c (with "" (func $foo)))) +) + +;; boolean parameters +(component + (core module $m + (func (export "assert_true") (param i32) + local.get 0 + i32.const 1 + i32.eq + i32.eqz + if unreachable end + ) + (func (export "assert_false") (param i32) + local.get 0 + if unreachable end + ) + (func (export "ret-bool") (param i32) (result i32) + local.get 0 + ) + ) + (core instance $m (instantiate $m)) + (func $assert_true (param bool) (canon lift (core func $m "assert_true"))) + (func $assert_false (param bool) (canon lift (core func $m "assert_false"))) + (func $ret_bool (param u32) (result bool) (canon lift (core func $m "ret-bool"))) + + (component $c + (import "assert-true" (func $assert_true (param bool))) + (import "assert-false" (func $assert_false (param bool))) + (import "ret-bool" (func $ret_bool (param u32) (result bool))) + + (core func $assert_true (canon lower (func $assert_true))) + (core func $assert_false (canon lower (func $assert_false))) + (core func $ret_bool (canon lower (func $ret_bool))) + + (core module $m2 + (import "" "assert-true" (func $assert_true (param i32))) + (import "" "assert-false" (func $assert_false (param i32))) + (import "" "ret-bool" (func $ret_bool (param i32) (result i32))) + + (func $start + (call $assert_true (i32.const 1)) + (call $assert_true (i32.const 2)) + (call $assert_true (i32.const -1)) + (call $assert_false (i32.const 0)) + + (if (i32.ne (call $ret_bool (i32.const 1)) (i32.const 1)) + (unreachable)) + (if (i32.ne (call $ret_bool (i32.const 2)) (i32.const 1)) + (unreachable)) + (if (i32.ne (call $ret_bool (i32.const -1)) (i32.const 1)) + (unreachable)) + (if (i32.ne (call $ret_bool (i32.const 0)) (i32.const 0)) + (unreachable)) + ) + (start $start) + ) + (core instance $m2 (instantiate $m2 + (with "" (instance + (export "assert-true" (func $assert_true)) + (export "assert-false" (func $assert_false)) + (export "ret-bool" (func $ret_bool)) + )) + )) + ) + + (instance $c (instantiate $c + (with "assert-true" (func $assert_true)) + (with "assert-false" (func $assert_false)) + (with "ret-bool" (func $ret_bool)) + )) +) + +;; lots of parameters and results +(component + (type $roundtrip (func + ;; 20 u32 params + (param u32) (param u32) (param u32) (param u32) (param u32) + (param u32) (param u32) (param u32) (param u32) (param u32) + (param u32) (param u32) (param u32) (param u32) (param u32) + (param u32) (param u32) (param u32) (param u32) (param u32) + + ;; 10 u32 results + (result (tuple u32 u32 u32 u32 u32 u32 u32 u32 u32 u32)) + )) + + (core module $m + (memory (export "memory") 1) + (func (export "roundtrip") (param $src i32) (result i32) + (local $dst i32) + (if (i32.ne (local.get $src) (i32.const 16)) + (unreachable)) + + (if (i32.ne (i32.load offset=0 (local.get $src)) (i32.const 1)) (unreachable)) + (if (i32.ne (i32.load offset=4 (local.get $src)) (i32.const 2)) (unreachable)) + (if (i32.ne (i32.load offset=8 (local.get $src)) (i32.const 3)) (unreachable)) + (if (i32.ne (i32.load offset=12 (local.get $src)) (i32.const 4)) (unreachable)) + (if (i32.ne (i32.load offset=16 (local.get $src)) (i32.const 5)) (unreachable)) + (if (i32.ne (i32.load offset=20 (local.get $src)) (i32.const 6)) (unreachable)) + (if (i32.ne (i32.load offset=24 (local.get $src)) (i32.const 7)) (unreachable)) + (if (i32.ne (i32.load offset=28 (local.get $src)) (i32.const 8)) (unreachable)) + (if (i32.ne (i32.load offset=32 (local.get $src)) (i32.const 9)) (unreachable)) + (if (i32.ne (i32.load offset=36 (local.get $src)) (i32.const 10)) (unreachable)) + (if (i32.ne (i32.load offset=40 (local.get $src)) (i32.const 11)) (unreachable)) + (if (i32.ne (i32.load offset=44 (local.get $src)) (i32.const 12)) (unreachable)) + (if (i32.ne (i32.load offset=48 (local.get $src)) (i32.const 13)) (unreachable)) + (if (i32.ne (i32.load offset=52 (local.get $src)) (i32.const 14)) (unreachable)) + (if (i32.ne (i32.load offset=56 (local.get $src)) (i32.const 15)) (unreachable)) + (if (i32.ne (i32.load offset=60 (local.get $src)) (i32.const 16)) (unreachable)) + (if (i32.ne (i32.load offset=64 (local.get $src)) (i32.const 17)) (unreachable)) + (if (i32.ne (i32.load offset=68 (local.get $src)) (i32.const 18)) (unreachable)) + (if (i32.ne (i32.load offset=72 (local.get $src)) (i32.const 19)) (unreachable)) + (if (i32.ne (i32.load offset=76 (local.get $src)) (i32.const 20)) (unreachable)) + + (local.set $dst (i32.const 500)) + + (i32.store offset=0 (local.get $dst) (i32.const 21)) + (i32.store offset=4 (local.get $dst) (i32.const 22)) + (i32.store offset=8 (local.get $dst) (i32.const 23)) + (i32.store offset=12 (local.get $dst) (i32.const 24)) + (i32.store offset=16 (local.get $dst) (i32.const 25)) + (i32.store offset=20 (local.get $dst) (i32.const 26)) + (i32.store offset=24 (local.get $dst) (i32.const 27)) + (i32.store offset=28 (local.get $dst) (i32.const 28)) + (i32.store offset=32 (local.get $dst) (i32.const 29)) + (i32.store offset=36 (local.get $dst) (i32.const 30)) + + local.get $dst + ) + + (func (export "realloc") (param i32 i32 i32 i32) (result i32) + i32.const 16) + ) + (core instance $m (instantiate $m)) + + (func $roundtrip (type $roundtrip) + (canon lift (core func $m "roundtrip") (memory $m "memory") + (realloc (func $m "realloc"))) + ) + + (component $c + (import "roundtrip" (func $roundtrip (type $roundtrip))) + + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $roundtrip (canon lower (func $roundtrip) (memory $libc "memory"))) + + (core module $m2 + (import "libc" "memory" (memory 1)) + (import "" "roundtrip" (func $roundtrip (param i32 i32))) + + (func $start + (local $addr i32) + (local $retaddr i32) + + (local.set $addr (i32.const 100)) + (call $store_many (i32.const 20) (local.get $addr)) + + (local.set $retaddr (i32.const 200)) + (call $roundtrip (local.get $addr) (local.get $retaddr)) + + (if (i32.ne (i32.load offset=0 (local.get $retaddr)) (i32.const 21)) (unreachable)) + (if (i32.ne (i32.load offset=4 (local.get $retaddr)) (i32.const 22)) (unreachable)) + (if (i32.ne (i32.load offset=8 (local.get $retaddr)) (i32.const 23)) (unreachable)) + (if (i32.ne (i32.load offset=12 (local.get $retaddr)) (i32.const 24)) (unreachable)) + (if (i32.ne (i32.load offset=16 (local.get $retaddr)) (i32.const 25)) (unreachable)) + (if (i32.ne (i32.load offset=20 (local.get $retaddr)) (i32.const 26)) (unreachable)) + (if (i32.ne (i32.load offset=24 (local.get $retaddr)) (i32.const 27)) (unreachable)) + (if (i32.ne (i32.load offset=28 (local.get $retaddr)) (i32.const 28)) (unreachable)) + (if (i32.ne (i32.load offset=32 (local.get $retaddr)) (i32.const 29)) (unreachable)) + (if (i32.ne (i32.load offset=36 (local.get $retaddr)) (i32.const 30)) (unreachable)) + ) + + (func $store_many (param $amt i32) (param $addr i32) + (local $c i32) + (loop $loop + (local.set $c (i32.add (local.get $c) (i32.const 1))) + (i32.store (local.get $addr) (local.get $c)) + (local.set $addr (i32.add (local.get $addr) (i32.const 4))) + + (if (i32.ne (local.get $amt) (local.get $c)) (br $loop)) + ) + ) + (start $start) + ) + (core instance $m2 (instantiate $m2 + (with "libc" (instance $libc)) + (with "" (instance (export "roundtrip" (func $roundtrip)))) + )) + ) + + (instance $c (instantiate $c + (with "roundtrip" (func $roundtrip)) + )) +) + +;; this will require multiple adapter modules to get generated +(component + (core module $root (func (export "") (result i32) + i32.const 0 + )) + (core instance $root (instantiate $root)) + (func $root (result u32) (canon lift (core func $root ""))) + + (component $c + (import "thunk" (func $import (result u32))) + (core func $import (canon lower (func $import))) + (core module $reexport + (import "" "" (func $thunk (result i32))) + (func (export "thunk") (result i32) + call $thunk + i32.const 1 + i32.add) + ) + (core instance $reexport (instantiate $reexport + (with "" (instance + (export "" (func $import)) + )) + )) + (func $export (export "thunk") (result u32) + (canon lift (core func $reexport "thunk")) + ) + ) + + (instance $c1 (instantiate $c (with "thunk" (func $root)))) + (instance $c2 (instantiate $c (with "thunk" (func $c1 "thunk")))) + (instance $c3 (instantiate $c (with "thunk" (func $c2 "thunk")))) + (instance $c4 (instantiate $c (with "thunk" (func $c3 "thunk")))) + (instance $c5 (instantiate $c (with "thunk" (func $c4 "thunk")))) + (instance $c6 (instantiate $c (with "thunk" (func $c5 "thunk")))) + + (component $verify + (import "thunk" (func $thunk (result u32))) + (core func $thunk (canon lower (func $thunk))) + (core module $verify + (import "" "" (func $thunk (result i32))) + + (func $start + call $thunk + i32.const 6 + i32.ne + if unreachable end + ) + (start $start) + ) + (core instance (instantiate $verify + (with "" (instance + (export "" (func $thunk)) + )) + )) + ) + (instance (instantiate $verify (with "thunk" (func $c6 "thunk")))) +) + +;; Fancy case of an adapter using an adapter. Note that this is silly and +;; doesn't actually make any sense at runtime, we just shouldn't panic on a +;; valid component. +(component + (type $tuple20 (tuple + u32 u32 u32 u32 u32 + u32 u32 u32 u32 u32 + u32 u32 u32 u32 u32 + u32 u32 u32 u32 u32)) + + (component $realloc + (core module $realloc + (memory (export "memory") 1) + (func (export "realloc") (param i32 i32 i32 i32) (result i32) + unreachable) + ) + (core instance $realloc (instantiate $realloc)) + (func $realloc (param (tuple u32 u32 u32 u32)) (result u32) + (canon lift (core func $realloc "realloc")) + ) + (export "realloc" (func $realloc)) + ) + (instance $realloc (instantiate $realloc)) + (core func $realloc (canon lower (func $realloc "realloc"))) + + (core module $m + (memory (export "memory") 1) + (func (export "foo") (param i32)) + ) + (core instance $m (instantiate $m)) + (func $foo (param $tuple20) + (canon lift + (core func $m "foo") + (memory $m "memory") + (realloc (func $realloc)) + ) + ) + + (component $c + (import "foo" (func $foo (param $tuple20))) + + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $foo (canon lower (func $foo) (memory $libc "memory"))) + (core module $something + (import "" "foo" (func (param i32))) + ) + (core instance (instantiate $something + (with "" (instance + (export "foo" (func $foo)) + )) + )) + ) + (instance (instantiate $c + (with "foo" (func $foo)) + )) +) + +;; Don't panic or otherwise create extraneous adapter modules when the same +;; adapter is used twice for a module's argument. +(component + (core module $m + (func (export "foo") (param)) + ) + (core instance $m (instantiate $m)) + (func $foo (canon lift (core func $m "foo"))) + + (component $c + (import "foo" (func $foo)) + (core func $foo (canon lower (func $foo))) + + (core module $something + (import "" "a" (func)) + (import "" "b" (func)) + ) + (core instance (instantiate $something + (with "" (instance + (export "a" (func $foo)) + (export "b" (func $foo)) + )) + )) + ) + (instance (instantiate $c (with "foo" (func $foo)))) +) + +;; post-return should get invoked by the generated adapter, if specified +(component + (core module $m + (global $post_called (mut i32) (i32.const 0)) + (func (export "foo") + ;; assert `foo-post` not called yet + global.get $post_called + i32.const 1 + i32.eq + if unreachable end + ) + (func (export "foo-post") + ;; assert `foo-post` not called before + global.get $post_called + i32.const 1 + i32.eq + if unreachable end + ;; ... then flag as called + i32.const 1 + global.set $post_called + ) + (func (export "assert-post") + global.get $post_called + i32.const 1 + i32.ne + if unreachable end + ) + ) + (core instance $m (instantiate $m)) + (func $foo (canon lift (core func $m "foo") (post-return (func $m "foo-post")))) + (func $assert_post (canon lift (core func $m "assert-post"))) + + (component $c + (import "foo" (func $foo)) + (import "assert-post" (func $assert_post)) + (core func $foo (canon lower (func $foo))) + (core func $assert_post (canon lower (func $assert_post))) + + (core module $something + (import "" "foo" (func $foo)) + (import "" "assert-post" (func $assert_post)) + + (func $start + call $foo + call $assert_post + ) + (start $start) + ) + (core instance (instantiate $something + (with "" (instance + (export "foo" (func $foo)) + (export "assert-post" (func $assert_post)) + )) + )) + ) + (instance (instantiate $c + (with "foo" (func $foo)) + (with "assert-post" (func $assert_post)) + )) +) + +;; post-return passes the results +(component + (core module $m + (func (export "foo") (result i32) i32.const 100) + (func (export "foo-post") (param i32) + (if (i32.ne (local.get 0) (i32.const 100)) (unreachable))) + ) + (core instance $m (instantiate $m)) + (func $foo (result u32) + (canon lift (core func $m "foo") (post-return (func $m "foo-post")))) + + (component $c + (import "foo" (func $foo (result u32))) + (core func $foo (canon lower (func $foo))) + + (core module $something + (import "" "foo" (func $foo (result i32))) + (func $start + (if (i32.ne (call $foo) (i32.const 100)) (unreachable))) + (start $start) + ) + (core instance (instantiate $something + (with "" (instance + (export "foo" (func $foo)) + )) + )) + ) + (instance (instantiate $c + (with "foo" (func $foo)) + )) +) + +;; struct field reordering +(component + (component $c1 + (type $in (record + (field "a" u32) + (field "b" bool) + (field "c" u8) + )) + (type $out (record + (field "x" u8) + (field "y" u32) + (field "z" bool) + )) + + (core module $m + (memory (export "memory") 1) + (func (export "r") (param i32 i32 i32) (result i32) + (if (i32.ne (local.get 0) (i32.const 3)) (unreachable)) ;; a == 3 + (if (i32.ne (local.get 1) (i32.const 1)) (unreachable)) ;; b == true + (if (i32.ne (local.get 2) (i32.const 2)) (unreachable)) ;; c == 2 + + + (i32.store8 offset=0 (i32.const 200) (i32.const 0xab)) ;; x == 0xab + (i32.store offset=4 (i32.const 200) (i32.const 200)) ;; y == 200 + (i32.store8 offset=8 (i32.const 200) (i32.const 0)) ;; z == false + i32.const 200 + ) + ) + (core instance $m (instantiate $m)) + (func (export "r") (param $in) (result $out) + (canon lift (core func $m "r") (memory $m "memory")) + ) + ) + (component $c2 + ;; note the different field orderings than the records specified above + (type $in (record + (field "b" bool) + (field "c" u8) + (field "a" u32) + )) + (type $out (record + (field "z" bool) + (field "x" u8) + (field "y" u32) + )) + (import "r" (func $r (param $in) (result $out))) + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $r (canon lower (func $r) (memory $libc "memory"))) + + (core module $m + (import "" "r" (func $r (param i32 i32 i32 i32))) + (import "libc" "memory" (memory 0)) + (func $start + i32.const 100 ;; b: bool + i32.const 2 ;; c: u8 + i32.const 3 ;; a: u32 + i32.const 100 ;; retptr + call $r + + ;; z == false + (if (i32.ne (i32.load8_u offset=0 (i32.const 100)) (i32.const 0)) (unreachable)) + ;; x == 0xab + (if (i32.ne (i32.load8_u offset=1 (i32.const 100)) (i32.const 0xab)) (unreachable)) + ;; y == 200 + (if (i32.ne (i32.load offset=4 (i32.const 100)) (i32.const 200)) (unreachable)) + ) + (start $start) + ) + (core instance (instantiate $m + (with "libc" (instance $libc)) + (with "" (instance + (export "r" (func $r)) + )) + )) + ) + (instance $c1 (instantiate $c1)) + (instance $c2 (instantiate $c2 (with "r" (func $c1 "r")))) +) + +;; callee retptr misaligned +(assert_trap + (component + (component $c1 + (core module $m + (memory (export "memory") 1) + (func (export "r") (result i32) i32.const 1) + ) + (core instance $m (instantiate $m)) + (func (export "r") (result (tuple u32 u32)) + (canon lift (core func $m "r") (memory $m "memory")) + ) + ) + (component $c2 + (import "r" (func $r (result (tuple u32 u32)))) + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $r (canon lower (func $r) (memory $libc "memory"))) + + (core module $m + (import "" "r" (func $r (param i32))) + (func $start + i32.const 4 + call $r + ) + (start $start) + ) + (core instance (instantiate $m + (with "" (instance (export "r" (func $r)))) + )) + ) + (instance $c1 (instantiate $c1)) + (instance $c2 (instantiate $c2 (with "r" (func $c1 "r")))) + ) + "unreachable") + +;; caller retptr misaligned +(assert_trap + (component + (component $c1 + (core module $m + (memory (export "memory") 1) + (func (export "r") (result i32) i32.const 0) + ) + (core instance $m (instantiate $m)) + (func (export "r") (result (tuple u32 u32)) + (canon lift (core func $m "r") (memory $m "memory")) + ) + ) + (component $c2 + (import "r" (func $r (result (tuple u32 u32)))) + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $r (canon lower (func $r) (memory $libc "memory"))) + + (core module $m + (import "" "r" (func $r (param i32))) + (func $start + i32.const 1 + call $r + ) + (start $start) + ) + (core instance (instantiate $m + (with "" (instance (export "r" (func $r)))) + )) + ) + (instance $c1 (instantiate $c1)) + (instance $c2 (instantiate $c2 (with "r" (func $c1 "r")))) + ) + "unreachable") + +;; callee argptr misaligned +(assert_trap + (component + (type $big (tuple u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32)) + + (component $c1 + (core module $m + (memory (export "memory") 1) + (func (export "r") (param i32)) + (func (export "realloc") (param i32 i32 i32 i32) (result i32) + i32.const 1) + ) + (core instance $m (instantiate $m)) + (func (export "r") (param $big) + (canon lift (core func $m "r") (memory $m "memory") (realloc (func $m "realloc"))) + ) + ) + (component $c2 + (import "r" (func $r (param $big))) + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $r (canon lower (func $r) (memory $libc "memory"))) + + (core module $m + (import "" "r" (func $r (param i32))) + (func $start + i32.const 4 + call $r + ) + (start $start) + ) + (core instance (instantiate $m + (with "" (instance (export "r" (func $r)))) + )) + ) + (instance $c1 (instantiate $c1)) + (instance $c2 (instantiate $c2 (with "r" (func $c1 "r")))) + ) + "unreachable") + +;; caller argptr misaligned +(assert_trap + (component + (type $big (tuple u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32 u32)) + + (component $c1 + (core module $m + (memory (export "memory") 1) + (func (export "r") (param i32)) + (func (export "realloc") (param i32 i32 i32 i32) (result i32) + i32.const 4) + ) + (core instance $m (instantiate $m)) + (func (export "r") (param $big) + (canon lift (core func $m "r") (memory $m "memory") (realloc (func $m "realloc"))) + ) + ) + (component $c2 + (import "r" (func $r (param $big))) + (core module $libc (memory (export "memory") 1)) + (core instance $libc (instantiate $libc)) + (core func $r (canon lower (func $r) (memory $libc "memory"))) + + (core module $m + (import "" "r" (func $r (param i32))) + (func $start + i32.const 1 + call $r + ) + (start $start) + ) + (core instance (instantiate $m + (with "" (instance (export "r" (func $r)))) + )) + ) + (instance $c1 (instantiate $c1)) + (instance $c2 (instantiate $c2 (with "r" (func $c1 "r")))) + ) + "unreachable")