Instantiate nested modules for module linking (#2447)

This commit implements the interpretation necessary of the instance
section of the module linking proposal. Instantiating a module which
itself has nested instantiated instances will now instantiate the nested
instances properly. This isn't all that useful without the ability to
alias exports off the result, but we can at least observe the side
effects of instantiation through the `start` function.

cc #2094
This commit is contained in:
Alex Crichton
2020-12-01 14:01:31 -06:00
committed by GitHub
parent 40ad39fee9
commit 88a8a8993a
11 changed files with 433 additions and 105 deletions

View File

@@ -31,6 +31,7 @@ fn main() -> anyhow::Result<()> {
test_directory_module(out, "tests/misc_testsuite/bulk-memory-operations", strategy)?;
test_directory_module(out, "tests/misc_testsuite/reference-types", strategy)?;
test_directory_module(out, "tests/misc_testsuite/multi-memory", strategy)?;
test_directory_module(out, "tests/misc_testsuite/module-linking", strategy)?;
Ok(())
})?;

View File

@@ -8,8 +8,8 @@
use crate::state::FuncTranslationState;
use crate::translation_utils::{
DataIndex, ElemIndex, EntityType, Event, EventIndex, FuncIndex, Global, GlobalIndex, Memory,
MemoryIndex, Table, TableIndex, TypeIndex,
DataIndex, ElemIndex, EntityIndex, EntityType, Event, EventIndex, FuncIndex, Global,
GlobalIndex, Memory, MemoryIndex, ModuleIndex, Table, TableIndex, TypeIndex,
};
use core::convert::From;
use core::convert::TryFrom;
@@ -22,6 +22,7 @@ use cranelift_frontend::FunctionBuilder;
use serde::{Deserialize, Serialize};
use std::boxed::Box;
use std::string::ToString;
use std::vec::Vec;
use thiserror::Error;
use wasmparser::ValidatorResources;
use wasmparser::{BinaryReaderError, FuncValidator, FunctionBody, Operator, WasmFeatures};
@@ -969,4 +970,16 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
fn module_end(&mut self, index: usize) {
drop(index);
}
/// Indicates that this module will have `amount` instances.
fn reserve_instances(&mut self, amount: u32) {
drop(amount);
}
/// Declares a new instance which this module will instantiate before it's
/// instantiated.
fn declare_instance(&mut self, module: ModuleIndex, args: Vec<EntityIndex>) -> WasmResult<()> {
drop((module, args));
Err(WasmError::Unsupported("wasm instance".to_string()))
}
}

View File

@@ -3,8 +3,9 @@
use crate::environ::{ModuleEnvironment, WasmResult};
use crate::sections_translator::{
parse_data_section, parse_element_section, parse_event_section, parse_export_section,
parse_function_section, parse_global_section, parse_import_section, parse_memory_section,
parse_name_section, parse_start_section, parse_table_section, parse_type_section,
parse_function_section, parse_global_section, parse_import_section, parse_instance_section,
parse_memory_section, parse_name_section, parse_start_section, parse_table_section,
parse_type_section,
};
use crate::state::ModuleTranslationState;
use cranelift_codegen::timing;
@@ -116,7 +117,7 @@ pub fn translate_module<'data>(
}
Payload::InstanceSection(s) => {
validator.instance_section(&s)?;
unimplemented!("module linking not implemented yet")
parse_instance_section(s, environ)?;
}
Payload::AliasSection(s) => {
validator.alias_section(&s)?;

View File

@@ -10,9 +10,9 @@
use crate::environ::{ModuleEnvironment, WasmError, WasmResult};
use crate::state::ModuleTranslationState;
use crate::translation_utils::{
tabletype_to_type, type_to_type, DataIndex, ElemIndex, EntityType, Event, EventIndex,
FuncIndex, Global, GlobalIndex, GlobalInit, Memory, MemoryIndex, Table, TableElementType,
TableIndex, TypeIndex,
tabletype_to_type, type_to_type, DataIndex, ElemIndex, EntityIndex, EntityType, Event,
EventIndex, FuncIndex, Global, GlobalIndex, GlobalInit, InstanceIndex, Memory, MemoryIndex,
ModuleIndex, Table, TableElementType, TableIndex, TypeIndex,
};
use crate::wasm_unsupported;
use core::convert::TryFrom;
@@ -475,3 +475,37 @@ pub fn parse_name_section<'data>(
}
Ok(())
}
/// Parses the Instance section of the wasm module.
pub fn parse_instance_section<'data>(
section: wasmparser::InstanceSectionReader<'data>,
environ: &mut dyn ModuleEnvironment<'data>,
) -> WasmResult<()> {
environ.reserve_types(section.get_count())?;
for instance in section {
let instance = instance?;
let module = ModuleIndex::from_u32(instance.module());
let args = instance
.args()?
.into_iter()
.map(|result| {
let (kind, idx) = result?;
Ok(match kind {
ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(idx)),
ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(idx)),
ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(idx)),
ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(idx)),
ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(idx)),
ExternalKind::Instance => EntityIndex::Instance(InstanceIndex::from_u32(idx)),
ExternalKind::Event => unimplemented!(),
// this won't pass validation
ExternalKind::Type => unreachable!(),
})
})
.collect::<WasmResult<Vec<_>>>()?;
environ.declare_instance(module, args)?;
}
Ok(())
}

View File

@@ -1,4 +1,4 @@
use crate::module::{MemoryPlan, Module, ModuleType, TableElements, TablePlan};
use crate::module::{Instance, MemoryPlan, Module, ModuleType, TableElements, TablePlan};
use crate::tunables::Tunables;
use cranelift_codegen::ir;
use cranelift_codegen::ir::{AbiParam, ArgumentPurpose};
@@ -6,8 +6,8 @@ use cranelift_codegen::isa::TargetFrontendConfig;
use cranelift_entity::PrimaryMap;
use cranelift_wasm::{
self, translate_module, DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType,
FuncIndex, Global, GlobalIndex, Memory, MemoryIndex, SignatureIndex, Table, TableIndex,
TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult,
FuncIndex, Global, GlobalIndex, Memory, MemoryIndex, ModuleIndex, SignatureIndex, Table,
TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult,
};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -60,7 +60,7 @@ pub struct ModuleTranslation<'data> {
/// Indexes into the returned list of translations that are submodules of
/// this module.
pub submodules: Vec<usize>,
pub submodules: PrimaryMap<ModuleIndex, usize>,
code_index: u32,
}
@@ -649,6 +649,18 @@ and for re-adding support for interface types you can see this issue:
self.result.submodules.push(self.results.len());
self.results.push(finished);
}
fn reserve_instances(&mut self, amt: u32) {
self.result.module.instances.reserve(amt as usize);
}
fn declare_instance(&mut self, module: ModuleIndex, args: Vec<EntityIndex>) -> WasmResult<()> {
self.result
.module
.instances
.push(Instance::Instantiate { module, args });
Ok(())
}
}
/// Add environment-specific function parameters.

View File

@@ -17,7 +17,7 @@ use thiserror::Error;
use wasmtime_debug::create_gdbjit_image;
use wasmtime_environ::entity::PrimaryMap;
use wasmtime_environ::isa::TargetIsa;
use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex};
use wasmtime_environ::wasm::{DefinedFuncIndex, ModuleIndex, SignatureIndex};
use wasmtime_environ::{
CompileError, DataInitializer, DataInitializerLocation, FunctionAddressMap, Module,
ModuleEnvironment, ModuleTranslation, StackMapInformation, TrapInformation,
@@ -71,6 +71,10 @@ pub struct CompilationArtifacts {
/// Debug info presence flags.
debug_info: bool,
/// Where to find this module's submodule code in the top-level list of
/// modules.
submodules: PrimaryMap<ModuleIndex, usize>,
}
impl CompilationArtifacts {
@@ -98,6 +102,7 @@ impl CompilationArtifacts {
let ModuleTranslation {
module,
data_initializers,
submodules,
..
} = translation;
@@ -118,6 +123,7 @@ impl CompilationArtifacts {
obj: obj.into_boxed_slice(),
unwind_info: unwind_info.into_boxed_slice(),
data_initializers,
submodules,
funcs: funcs
.into_iter()
.map(|(_, func)| FunctionInfo {
@@ -336,6 +342,12 @@ impl CompiledModule {
pub fn code(&self) -> &Arc<ModuleCode> {
&self.code
}
/// Returns where the specified submodule lives in this module's
/// array-of-modules (store at the top-level)
pub fn submodule_idx(&self, idx: ModuleIndex) -> usize {
self.artifacts.submodules[idx]
}
}
/// Similar to `DataInitializer`, but owns its own copy of the data rather

View File

@@ -115,15 +115,6 @@ impl Extern {
};
Store::same(my_store, store)
}
pub(crate) fn desc(&self) -> &'static str {
match self {
Extern::Func(_) => "function",
Extern::Table(_) => "table",
Extern::Memory(_) => "memory",
Extern::Global(_) => "global",
}
}
}
impl From<Func> for Extern {

View File

@@ -1,21 +1,64 @@
use crate::trampoline::StoreInstanceHandle;
use crate::{Engine, Export, Extern, Func, Global, Memory, Module, Store, Table, Trap};
use anyhow::{anyhow, bail, Context, Error, Result};
use std::any::Any;
use anyhow::{bail, Error, Result};
use std::mem;
use wasmtime_environ::wasm::EntityIndex;
use wasmtime_environ::entity::PrimaryMap;
use wasmtime_environ::wasm::{EntityIndex, FuncIndex, GlobalIndex, MemoryIndex, TableIndex};
use wasmtime_jit::CompiledModule;
use wasmtime_runtime::{
Imports, InstantiationError, StackMapRegistry, VMContext, VMExternRefActivationsTable,
VMFunctionBody,
VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport,
};
fn instantiate(
store: &Store,
compiled_module: &CompiledModule,
imports: Imports<'_>,
host: Box<dyn Any>,
all_modules: &[CompiledModule],
imports: &mut ImportsBuilder<'_>,
) -> Result<StoreInstanceHandle, Error> {
let env_module = compiled_module.module();
// The first part of instantiating any module is to first follow any
// `instantiate` instructions it has as part of the module linking
// proposal. Here we iterate overall those instructions and create the
// instances as necessary.
for instance in env_module.instances.values() {
let (module_idx, args) = match instance {
wasmtime_environ::Instance::Instantiate { module, args } => (*module, args),
wasmtime_environ::Instance::Import(_) => continue,
};
// Translate the `module_idx` to a top-level module `usize` and then
// use that to extract the child `&CompiledModule` itself. Then we can
// iterate over each of the arguments provided to satisfy its imports.
//
// Note that we directly reach into `imports` below based on indexes
// and push raw value into how to instantiate our submodule. This should
// be safe due to wasm validation ensuring that all our indices are
// in-bounds and all the expected types and such line up.
let module_idx = compiled_module.submodule_idx(module_idx);
let compiled_module = &all_modules[module_idx];
let mut builder = ImportsBuilder::new(compiled_module.module(), store);
for arg in args {
match *arg {
EntityIndex::Global(i) => {
builder.globals.push(imports.globals[i]);
}
EntityIndex::Table(i) => {
builder.tables.push(imports.tables[i]);
}
EntityIndex::Function(i) => {
builder.functions.push(imports.functions[i]);
}
EntityIndex::Memory(i) => {
builder.memories.push(imports.memories[i]);
}
EntityIndex::Module(_) => unimplemented!(),
EntityIndex::Instance(_) => unimplemented!(),
}
}
instantiate(store, compiled_module, all_modules, &mut builder)?;
}
// Register the module just before instantiation to ensure we have a
// trampoline registered for every signature and to preserve the module's
// compiled JIT code within the `Store`.
@@ -24,11 +67,11 @@ fn instantiate(
let config = store.engine().config();
let instance = unsafe {
let instance = compiled_module.instantiate(
imports,
imports.imports(),
&store.lookup_shared_signature(compiled_module.module()),
config.memory_creator.as_ref().map(|a| a as _),
store.interrupts(),
host,
Box::new(()),
store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _,
store.stack_map_registry() as *const StackMapRegistry as *mut _,
)?;
@@ -165,9 +208,27 @@ impl Instance {
bail!("cross-`Engine` instantiation is not currently supported");
}
let handle = with_imports(store, module.compiled_module(), imports, |imports| {
instantiate(store, module.compiled_module(), imports, Box::new(()))
})?;
let mut builder = ImportsBuilder::new(module.compiled_module().module(), store);
for import in imports {
// For now we have a restriction that the `Store` that we're working
// with is the same for everything involved here.
if !import.comes_from_same_store(store) {
bail!("cross-`Store` instantiation is not currently supported");
}
match import {
Extern::Global(e) => builder.global(e)?,
Extern::Func(e) => builder.func(e)?,
Extern::Table(e) => builder.table(e)?,
Extern::Memory(e) => builder.memory(e)?,
}
}
builder.validate_all_imports_provided()?;
let handle = instantiate(
store,
module.compiled_module(),
&module.compiled,
&mut builder,
)?;
Ok(Instance {
handle,
@@ -238,87 +299,126 @@ impl Instance {
}
}
fn with_imports<R>(
store: &Store,
module: &CompiledModule,
externs: &[Extern],
f: impl FnOnce(Imports<'_>) -> Result<R>,
) -> Result<R> {
let m = module.module();
if externs.len() != m.imports.len() {
bail!(
"wrong number of imports provided, {} != {}",
externs.len(),
m.imports.len()
);
struct ImportsBuilder<'a> {
functions: PrimaryMap<FuncIndex, VMFunctionImport>,
tables: PrimaryMap<TableIndex, VMTableImport>,
memories: PrimaryMap<MemoryIndex, VMMemoryImport>,
globals: PrimaryMap<GlobalIndex, VMGlobalImport>,
module: &'a wasmtime_environ::Module,
imports: std::slice::Iter<'a, (String, Option<String>, EntityIndex)>,
store: &'a Store,
}
let mut tables = Vec::new();
let mut functions = Vec::new();
let mut globals = Vec::new();
let mut memories = Vec::new();
let mut process = |expected: &EntityIndex, actual: &Extern| {
// For now we have a restriction that the `Store` that we're working
// with is the same for everything involved here.
if !actual.comes_from_same_store(store) {
bail!("cross-`Store` instantiation is not currently supported");
impl<'a> ImportsBuilder<'a> {
fn new(module: &'a wasmtime_environ::Module, store: &'a Store) -> ImportsBuilder<'a> {
ImportsBuilder {
imports: module.imports.iter(),
module,
store,
functions: PrimaryMap::with_capacity(module.num_imported_funcs),
tables: PrimaryMap::with_capacity(module.num_imported_tables),
memories: PrimaryMap::with_capacity(module.num_imported_memories),
globals: PrimaryMap::with_capacity(module.num_imported_globals),
}
}
match *expected {
EntityIndex::Table(i) => tables.push(match actual {
Extern::Table(e) if e.matches_expected(&m.table_plans[i]) => e.vmimport(),
Extern::Table(_) => bail!("table types incompatible"),
_ => bail!("expected table, but found {}", actual.desc()),
}),
EntityIndex::Memory(i) => memories.push(match actual {
Extern::Memory(e) if e.matches_expected(&m.memory_plans[i]) => e.vmimport(),
Extern::Memory(_) => bail!("memory types incompatible"),
_ => bail!("expected memory, but found {}", actual.desc()),
}),
EntityIndex::Global(i) => globals.push(match actual {
Extern::Global(e) if e.matches_expected(&m.globals[i]) => e.vmimport(),
Extern::Global(_) => bail!("global types incompatible"),
_ => bail!("expected global, but found {}", actual.desc()),
}),
EntityIndex::Function(i) => {
let func = match actual {
Extern::Func(e) => e,
_ => bail!("expected function, but found {}", actual.desc()),
fn next_import(
&mut self,
found: &str,
get: impl FnOnce(&wasmtime_environ::Module, &EntityIndex) -> Option<bool>,
) -> Result<()> {
match self.imports.next() {
Some((module, field, idx)) => {
let error = match get(self.module, idx) {
Some(true) => return Ok(()),
Some(false) => {
anyhow::anyhow!("{} types incompatible", found)
}
None => {
let desc = match idx {
EntityIndex::Table(_) => "table",
EntityIndex::Function(_) => "func",
EntityIndex::Memory(_) => "memory",
EntityIndex::Global(_) => "global",
EntityIndex::Instance(_) => "instance",
EntityIndex::Module(_) => "module",
};
anyhow::anyhow!("expected {}, but found {}", desc, found)
}
};
let import_name = match field {
Some(name) => format!("{}/{}", module, name),
None => module.to_string(),
};
Err(error.context(format!("incompatible import type for {}", import_name)))
}
None => bail!("too many imports provided"),
}
}
fn global(&mut self, global: &Global) -> Result<()> {
self.next_import("global", |m, e| match e {
EntityIndex::Global(i) => Some(global.matches_expected(&m.globals[*i])),
_ => None,
})?;
self.globals.push(global.vmimport());
Ok(())
}
fn memory(&mut self, mem: &Memory) -> Result<()> {
self.next_import("memory", |m, e| match e {
EntityIndex::Memory(i) => Some(mem.matches_expected(&m.memory_plans[*i])),
_ => None,
})?;
self.memories.push(mem.vmimport());
Ok(())
}
fn table(&mut self, table: &Table) -> Result<()> {
self.next_import("table", |m, e| match e {
EntityIndex::Table(i) => Some(table.matches_expected(&m.table_plans[*i])),
_ => None,
})?;
self.tables.push(table.vmimport());
Ok(())
}
fn func(&mut self, func: &Func) -> Result<()> {
let store = self.store;
self.next_import("func", |m, e| match e {
EntityIndex::Function(i) => Some(
// Look up the `i`th function's type from the module in our
// signature registry. If it's not present then we have no
// functions registered with that type, so `func` is guaranteed
// to not match.
let ty = store
match store
.signatures()
.borrow()
.lookup(&m.signatures[m.functions[i]])
.ok_or_else(|| anyhow!("function types incompatible"))?;
if !func.matches_expected(ty) {
bail!("function types incompatible");
}
functions.push(func.vmimport());
.lookup(&m.signatures[m.functions[*i]])
{
Some(ty) => func.matches_expected(ty),
None => false,
},
),
_ => None,
})?;
self.functions.push(func.vmimport());
Ok(())
}
// FIXME(#2094)
EntityIndex::Module(_i) => unimplemented!(),
EntityIndex::Instance(_i) => unimplemented!(),
fn validate_all_imports_provided(&mut self) -> Result<()> {
if self.imports.next().is_some() {
bail!("not enough imports provided");
}
Ok(())
};
for (expected, actual) in m.imports.iter().zip(externs) {
process(&expected.2, actual).with_context(|| match &expected.1 {
Some(name) => format!("incompatible import type for {}/{}", expected.0, name),
None => format!("incompatible import type for {}", expected.0),
})?;
}
return f(Imports {
tables: &tables,
functions: &functions,
globals: &globals,
memories: &memories,
});
fn imports(&self) -> Imports<'_> {
Imports {
tables: self.tables.values().as_slice(),
globals: self.globals.values().as_slice(),
memories: self.memories.values().as_slice(),
functions: self.functions.values().as_slice(),
}
}
}

View File

@@ -81,7 +81,7 @@ use wasmtime_jit::{CompilationArtifacts, CompiledModule};
#[derive(Clone)]
pub struct Module {
engine: Engine,
compiled: Arc<[CompiledModule]>,
pub(crate) compiled: Arc<[CompiledModule]>,
index: usize,
}

View File

@@ -13,6 +13,7 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> {
let simd = wast.iter().any(|s| s == "simd");
let multi_memory = wast.iter().any(|s| s == "multi-memory");
let module_linking = wast.iter().any(|s| s == "module-linking");
let bulk_mem = multi_memory || wast.iter().any(|s| s == "bulk-memory-operations");
// Some simd tests assume support for multiple tables, which are introduced
@@ -24,6 +25,7 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> {
.wasm_bulk_memory(bulk_mem)
.wasm_reference_types(reftypes)
.wasm_multi_memory(multi_memory)
.wasm_module_linking(module_linking)
.strategy(strategy)?
.cranelift_debug_verifier(true);

View File

@@ -0,0 +1,162 @@
(module
(module)
(instance $a (instantiate 0))
)
(module $a
(global (export "global") (mut i32) (i32.const 0))
(func (export "reset")
i32.const 0
global.set 0)
(func $set (export "inc")
i32.const 1
global.get 0
i32.add
global.set 0)
(func (export "get") (result i32)
global.get 0)
(func (export "load") (result i32)
i32.const 0
i32.load)
(memory (export "memory") 1)
(table (export "table") 1 funcref)
(elem (i32.const 0) $set)
)
;; Imported functions work
(module
(import "a" "inc" (func $set))
(module
(import "" (func))
(start 0))
(instance $a (instantiate 0 (func $set)))
)
(assert_return (invoke $a "get") (i32.const 1))
;; Imported globals work
(module
(import "a" "global" (global $g (mut i32)))
(module
(import "" (global (mut i32)))
(func
i32.const 2
global.set 0)
(start 0))
(instance $a (instantiate 0 (global $g)))
)
(assert_return (invoke $a "get") (i32.const 2))
;; Imported tables work
(module
(import "a" "table" (table $t 1 funcref))
(module
(import "" (table 1 funcref))
(func
i32.const 0
call_indirect)
(start 0))
(instance $a (instantiate 0 (table $t)))
)
(assert_return (invoke $a "get") (i32.const 3))
;; Imported memories work
(module
(import "a" "memory" (memory $m 1))
(module
(import "" (memory 1))
(func
i32.const 0
i32.const 4
i32.store)
(start 0))
(instance $a (instantiate 0 (memory $m)))
)
(assert_return (invoke $a "load") (i32.const 4))
;; all at once
(module
(import "a" "inc" (func $f))
(import "a" "global" (global $g (mut i32)))
(import "a" "table" (table $t 1 funcref))
(import "a" "memory" (memory $m 1))
(module
(import "" (memory 1))
(import "" (global (mut i32)))
(import "" (table 1 funcref))
(import "" (func))
(func $start
call 0
i32.const 0
i32.const 4
i32.store
i32.const 0
call_indirect
global.get 0
global.set 0)
(start $start))
(instance $a
(instantiate 0
(memory $m)
(global $g)
(table $t)
(func $f)
)
)
)
;; instantiate lots
(module
(import "a" "inc" (func $f))
(import "a" "global" (global $g (mut i32)))
(import "a" "table" (table $t 1 funcref))
(import "a" "memory" (memory $m 1))
(module $mm (import "" (memory 1)))
(module $mf (import "" (func)))
(module $mt (import "" (table 1 funcref)))
(module $mg (import "" (global (mut i32))))
(instance (instantiate $mm (memory $m)))
(instance (instantiate $mf (func $f)))
(instance (instantiate $mt (table $t)))
(instance (instantiate $mg (global $g)))
)
;; instantiate nested
(assert_return (invoke $a "reset"))
(assert_return (invoke $a "get") (i32.const 0))
(module
(import "a" "inc" (func))
(module
(import "" (func))
(module
(import "" (func))
(module
(import "" (func))
(module
(import "" (func))
(start 0)
)
(instance (instantiate 0 (func 0)))
)
(instance (instantiate 0 (func 0)))
)
(instance (instantiate 0 (func 0)))
)
(instance (instantiate 0 (func 0)))
)
(assert_return (invoke $a "get") (i32.const 1))