Merge pull request #976 from fitzgen/implement-most-of-bulk-memory

Implement most of bulk memory

Cargo.toml

@@ -39,7 +39,7 @@ anyhow = "1.0.19"
 target-lexicon = { version = "0.10.0", default-features = false }
 pretty_env_logger = "0.3.0"
 file-per-thread-logger = "0.1.1"
-wat = "1.0.2"
+wat = "1.0.10"
 libc = "0.2.60"
 rayon = "1.2.1"
 wasm-webidl-bindings = "0.8"

build.rs

@@ -26,40 +26,38 @@ fn main() -> anyhow::Result<()> {
         writeln!(out, "#[allow(non_snake_case)]")?;
         writeln!(out, "mod {} {{", strategy)?;
 
-        test_directory(&mut out, "tests/misc_testsuite", strategy)?;
+        with_test_module(&mut out, "misc", |out| {
-        let spec_tests = test_directory(&mut out, "tests/spec_testsuite", strategy)?;
+            test_directory(out, "tests/misc_testsuite", strategy)?;
-        // Skip running spec_testsuite tests if the submodule isn't checked
+            test_directory_module(out, "tests/misc_testsuite/bulk-memory-operations", strategy)?;
-        // out.
+            test_directory_module(out, "tests/misc_testsuite/reference-types", strategy)?;
-        if spec_tests > 0 {
+            Ok(())
-            test_directory(&mut out, "tests/spec_testsuite/proposals/simd", strategy)
+        })?;
-                .expect("generating tests");
 
-            test_directory(
+        with_test_module(&mut out, "spec", |out| {
-                &mut out,
+            let spec_tests = test_directory(out, "tests/spec_testsuite", strategy)?;
-                "tests/spec_testsuite/proposals/multi-value",
+            // Skip running spec_testsuite tests if the submodule isn't checked
-                strategy,
+            // out.
-            )
+            if spec_tests > 0 {
-            .expect("generating tests");
+                test_directory_module(out, "tests/spec_testsuite/proposals/simd", strategy)?;
-
+                test_directory_module(out, "tests/spec_testsuite/proposals/multi-value", strategy)?;
-            test_directory(
+                test_directory_module(
-                &mut out,
+                    out,
-                "tests/spec_testsuite/proposals/reference-types",
+                    "tests/spec_testsuite/proposals/reference-types",
-                strategy,
+                    strategy,
-            )
+                )?;
-            .expect("generating tests");
+                test_directory_module(
-
+                    out,
-            test_directory(
+                    "tests/spec_testsuite/proposals/bulk-memory-operations",
-                &mut out,
+                    strategy,
-                "tests/spec_testsuite/proposals/bulk-memory-operations",
+                )?;
-                strategy,
+            } else {
-            )
+                println!(
-            .expect("generating tests");
+                    "cargo:warning=The spec testsuite is disabled. To enable, run `git submodule \
-        } else {
-            println!(
-                "cargo:warning=The spec testsuite is disabled. To enable, run `git submodule \
                  update --remote`."
-            );
+                );
-        }
+            }
+            Ok(())
+        })?;
 
         writeln!(out, "}}")?;
     }
@@ -72,6 +70,16 @@ fn main() -> anyhow::Result<()> {
     Ok(())
 }
 
+fn test_directory_module(
+    out: &mut String,
+    path: impl AsRef<Path>,
+    strategy: &str,
+) -> anyhow::Result<usize> {
+    let path = path.as_ref();
+    let testsuite = &extract_name(path);
+    with_test_module(out, testsuite, |out| test_directory(out, path, strategy))
+}
+
 fn test_directory(
     out: &mut String,
     path: impl AsRef<Path>,
@@ -100,11 +108,10 @@ fn test_directory(
     dir_entries.sort();
 
     let testsuite = &extract_name(path);
-    start_test_module(out, testsuite)?;
     for entry in dir_entries.iter() {
         write_testsuite_tests(out, entry, testsuite, strategy)?;
     }
-    finish_test_module(out)?;
+
     Ok(dir_entries.len())
 }
 
@@ -119,14 +126,19 @@ fn extract_name(path: impl AsRef<Path>) -> String {
         .replace("/", "_")
 }
 
-fn start_test_module(out: &mut String, testsuite: &str) -> anyhow::Result<()> {
+fn with_test_module<T>(
-    writeln!(out, "mod {} {{", testsuite)?;
+    out: &mut String,
-    Ok(())
+    testsuite: &str,
-}
+    f: impl FnOnce(&mut String) -> anyhow::Result<T>,
+) -> anyhow::Result<T> {
+    out.push_str("mod ");
+    out.push_str(testsuite);
+    out.push_str(" {\n");
+
+    let result = f(out)?;
 
-fn finish_test_module(out: &mut String) -> anyhow::Result<()> {
     out.push_str("}\n");
-    Ok(())
+    Ok(result)
 }
 
 fn write_testsuite_tests(
@@ -180,8 +192,15 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
             ("simd", "simd_load_splat") => return true, // FIXME Unsupported feature: proposed SIMD operator V8x16LoadSplat { memarg: MemoryImmediate { flags: 0, offset: 0 } }
             ("simd", "simd_splat") => return true, // FIXME Unsupported feature: proposed SIMD operator I8x16ShrS
 
+            // Still working on implementing these. See #929.
+            ("reference_types", "table_copy_on_imported_tables") => return false,
             ("reference_types", _) => return true,
-            ("bulk_memory_operations", _) => return true,
+
+            // Still working on implementing these. See #928
+            ("bulk_memory_operations", "bulk")
+            | ("bulk_memory_operations", "data")
+            | ("bulk_memory_operations", "memory_init")
+            | ("bulk_memory_operations", "imports") => return true,
 
             _ => {}
         },

crates/api/Cargo.toml

@@ -33,7 +33,7 @@ wasi-common = { path = "../wasi-common", version = "0.12.0" }
 pretty_env_logger = "0.3.0"
 rayon = "1.2.1"
 file-per-thread-logger = "0.1.1"
-wat = "1.0"
+wat = "1.0.10"
 tempfile = "3.1"
 
 [badges]

crates/api/src/externals.rs

@@ -5,8 +5,8 @@ use crate::{ExternType, GlobalType, MemoryType, TableType, ValType};
 use crate::{Func, Store};
 use anyhow::{anyhow, bail, Result};
 use std::slice;
-use wasmtime_environ::wasm;
+use wasmtime_environ::{ir, wasm};
-use wasmtime_runtime::InstanceHandle;
+use wasmtime_runtime::{self as runtime, InstanceHandle};
 
 // Externals
 
@@ -407,6 +407,41 @@ impl Table {
         }
     }
 
+    /// Copy `len` elements from `src_table[src_index..]` into
+    /// `dst_table[dst_index..]`.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the range is out of bounds of either the source or
+    /// destination tables.
+    pub fn copy(
+        dst_table: &Table,
+        dst_index: u32,
+        src_table: &Table,
+        src_index: u32,
+        len: u32,
+    ) -> Result<()> {
+        // NB: We must use the `dst_table`'s `wasmtime_handle` for the
+        // `dst_table_index` and vice versa for `src_table` since each table can
+        // come from different modules.
+
+        let dst_table_index = dst_table.wasmtime_table_index();
+        let dst_table = dst_table.wasmtime_handle.get_defined_table(dst_table_index);
+
+        let src_table_index = src_table.wasmtime_table_index();
+        let src_table = src_table.wasmtime_handle.get_defined_table(src_table_index);
+
+        runtime::Table::copy(
+            dst_table,
+            src_table,
+            dst_index,
+            src_index,
+            len,
+            ir::SourceLoc::default(),
+        )?;
+        Ok(())
+    }
+
     pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::Export {
         &self.wasmtime_export
     }

crates/api/src/instance.rs

@@ -1,6 +1,6 @@
 use crate::externals::Extern;
 use crate::module::Module;
-use crate::runtime::Store;
+use crate::runtime::{Config, Store};
 use crate::trap::Trap;
 use anyhow::{Error, Result};
 use wasmtime_jit::{CompiledModule, Resolver};
@@ -19,16 +19,22 @@ impl Resolver for SimpleResolver<'_> {
 }
 
 fn instantiate(
+    config: &Config,
     compiled_module: &CompiledModule,
     imports: &[Extern],
 ) -> Result<InstanceHandle, Error> {
     let mut resolver = SimpleResolver { imports };
     unsafe {
         let instance = compiled_module
-            .instantiate(&mut resolver)
+            .instantiate(
+                config.validating_config.operator_config.enable_bulk_memory,
+                &mut resolver,
+            )
             .map_err(|e| -> Error {
                 match e {
-                    InstantiationError::StartTrap(trap) => Trap::from_jit(trap).into(),
+                    InstantiationError::StartTrap(trap) | InstantiationError::Trap(trap) => {
+                        Trap::from_jit(trap).into()
+                    }
                     other => other.into(),
                 }
             })?;
@@ -105,7 +111,8 @@ impl Instance {
     /// [issue]: https://github.com/bytecodealliance/wasmtime/issues/727
     pub fn new(module: &Module, imports: &[Extern]) -> Result<Instance, Error> {
         let store = module.store();
-        let instance_handle = instantiate(module.compiled_module(), imports)?;
+        let config = store.engine().config();
+        let instance_handle = instantiate(config, module.compiled_module(), imports)?;
 
         let exports = {
             let mut exports = Vec::with_capacity(module.exports().len());

crates/api/src/trampoline/create_handle.rs

@@ -42,6 +42,12 @@ pub(crate) fn create_handle(
             &data_initializers,
             signatures.into_boxed_slice(),
             None,
+            store
+                .engine()
+                .config()
+                .validating_config
+                .operator_config
+                .enable_bulk_memory,
             state,
         )?)
     }

crates/environ/src/data_structures.rs

@@ -17,13 +17,13 @@ pub mod isa {
 }
 
 pub mod entity {
-    pub use cranelift_entity::{BoxedSlice, EntityRef, PrimaryMap};
+    pub use cranelift_entity::{packed_option, BoxedSlice, EntityRef, PrimaryMap};
 }
 
 pub mod wasm {
     pub use cranelift_wasm::{
         get_vmctx_value_label, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex,
         DefinedTableIndex, FuncIndex, Global, GlobalIndex, GlobalInit, Memory, MemoryIndex,
-        SignatureIndex, Table, TableElementType, TableIndex,
+        PassiveElemIndex, SignatureIndex, Table, TableElementType, TableIndex,
     };
 }

crates/environ/src/func_environ.rs

@@ -1,7 +1,7 @@
 use crate::module::{MemoryPlan, MemoryStyle, ModuleLocal, TableStyle};
 use crate::vmoffsets::VMOffsets;
 use crate::WASM_PAGE_SIZE;
-use cranelift_codegen::cursor::FuncCursor;
+use cranelift_codegen::cursor::{Cursor, FuncCursor};
 use cranelift_codegen::ir;
 use cranelift_codegen::ir::condcodes::*;
 use cranelift_codegen::ir::immediates::{Offset32, Uimm64};
@@ -23,6 +23,7 @@ pub fn get_func_name(func_index: FuncIndex) -> ir::ExternalName {
 }
 
 /// An index type for builtin functions.
+#[derive(Copy, Clone, Debug)]
 pub struct BuiltinFunctionIndex(u32);
 
 impl BuiltinFunctionIndex {
@@ -42,9 +43,38 @@ impl BuiltinFunctionIndex {
     pub const fn get_imported_memory32_size_index() -> Self {
         Self(3)
     }
+    /// Returns an index for wasm's `table.copy` when both tables are locally
+    /// defined.
+    pub const fn get_table_copy_index() -> Self {
+        Self(4)
+    }
+    /// Returns an index for wasm's `table.init`.
+    pub const fn get_table_init_index() -> Self {
+        Self(5)
+    }
+    /// Returns an index for wasm's `elem.drop`.
+    pub const fn get_elem_drop_index() -> Self {
+        Self(6)
+    }
+    /// Returns an index for wasm's `memory.copy` for locally defined memories.
+    pub const fn get_defined_memory_copy_index() -> Self {
+        Self(7)
+    }
+    /// Returns an index for wasm's `memory.copy` for imported memories.
+    pub const fn get_imported_memory_copy_index() -> Self {
+        Self(8)
+    }
+    /// Returns an index for wasm's `memory.fill` for locally defined memories.
+    pub const fn get_memory_fill_index() -> Self {
+        Self(9)
+    }
+    /// Returns an index for wasm's `memory.fill` for imported memories.
+    pub const fn get_imported_memory_fill_index() -> Self {
+        Self(10)
+    }
     /// Returns the total number of builtin functions.
     pub const fn builtin_functions_total_number() -> u32 {
-        4
+        11
     }
 
     /// Return the index as an u32 number.
@@ -72,6 +102,24 @@ pub struct FuncEnvironment<'module_environment> {
     /// for locally-defined memories.
     memory_grow_sig: Option<ir::SigRef>,
 
+    /// The external function signature for implementing wasm's `table.copy`
+    /// (it's the same for both local and imported tables).
+    table_copy_sig: Option<ir::SigRef>,
+
+    /// The external function signature for implementing wasm's `table.init`.
+    table_init_sig: Option<ir::SigRef>,
+
+    /// The external function signature for implementing wasm's `elem.drop`.
+    elem_drop_sig: Option<ir::SigRef>,
+
+    /// The external function signature for implementing wasm's `memory.copy`
+    /// (it's the same for both local and imported memories).
+    memory_copy_sig: Option<ir::SigRef>,
+
+    /// The external function signature for implementing wasm's `memory.fill`
+    /// (it's the same for both local and imported memories).
+    memory_fill_sig: Option<ir::SigRef>,
+
     /// Offsets to struct fields accessed by JIT code.
     offsets: VMOffsets,
 }
@@ -87,6 +135,11 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             vmctx: None,
             memory32_size_sig: None,
             memory_grow_sig: None,
+            table_copy_sig: None,
+            table_init_sig: None,
+            elem_drop_sig: None,
+            memory_copy_sig: None,
+            memory_fill_sig: None,
             offsets: VMOffsets::new(target_config.pointer_bytes(), module),
         }
     }
@@ -178,6 +231,198 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         }
     }
 
+    fn get_table_copy_sig(&mut self, func: &mut Function) -> ir::SigRef {
+        let sig = self.table_copy_sig.unwrap_or_else(|| {
+            func.import_signature(Signature {
+                params: vec![
+                    AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext),
+                    // Destination table index.
+                    AbiParam::new(I32),
+                    // Source table index.
+                    AbiParam::new(I32),
+                    // Index within destination table.
+                    AbiParam::new(I32),
+                    // Index within source table.
+                    AbiParam::new(I32),
+                    // Number of elements to copy.
+                    AbiParam::new(I32),
+                    // Source location.
+                    AbiParam::new(I32),
+                ],
+                returns: vec![],
+                call_conv: self.target_config.default_call_conv,
+            })
+        });
+        self.table_copy_sig = Some(sig);
+        sig
+    }
+
+    fn get_table_copy_func(
+        &mut self,
+        func: &mut Function,
+        dst_table_index: TableIndex,
+        src_table_index: TableIndex,
+    ) -> (ir::SigRef, usize, usize, BuiltinFunctionIndex) {
+        let sig = self.get_table_copy_sig(func);
+        (
+            sig,
+            dst_table_index.as_u32() as usize,
+            src_table_index.as_u32() as usize,
+            BuiltinFunctionIndex::get_table_copy_index(),
+        )
+    }
+
+    fn get_table_init_sig(&mut self, func: &mut Function) -> ir::SigRef {
+        let sig = self.table_init_sig.unwrap_or_else(|| {
+            func.import_signature(Signature {
+                params: vec![
+                    AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext),
+                    // Table index.
+                    AbiParam::new(I32),
+                    // Segment index.
+                    AbiParam::new(I32),
+                    // Destination index within table.
+                    AbiParam::new(I32),
+                    // Source index within segment.
+                    AbiParam::new(I32),
+                    // Number of elements to initialize.
+                    AbiParam::new(I32),
+                    // Source location.
+                    AbiParam::new(I32),
+                ],
+                returns: vec![],
+                call_conv: self.target_config.default_call_conv,
+            })
+        });
+        self.table_init_sig = Some(sig);
+        sig
+    }
+
+    fn get_table_init_func(
+        &mut self,
+        func: &mut Function,
+        table_index: TableIndex,
+    ) -> (ir::SigRef, usize, BuiltinFunctionIndex) {
+        let sig = self.get_table_init_sig(func);
+        let table_index = table_index.as_u32() as usize;
+        (
+            sig,
+            table_index,
+            BuiltinFunctionIndex::get_table_init_index(),
+        )
+    }
+
+    fn get_elem_drop_sig(&mut self, func: &mut Function) -> ir::SigRef {
+        let sig = self.elem_drop_sig.unwrap_or_else(|| {
+            func.import_signature(Signature {
+                params: vec![
+                    AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext),
+                    // Element index.
+                    AbiParam::new(I32),
+                ],
+                returns: vec![],
+                call_conv: self.target_config.default_call_conv,
+            })
+        });
+        self.elem_drop_sig = Some(sig);
+        sig
+    }
+
+    fn get_elem_drop_func(&mut self, func: &mut Function) -> (ir::SigRef, BuiltinFunctionIndex) {
+        let sig = self.get_elem_drop_sig(func);
+        (sig, BuiltinFunctionIndex::get_elem_drop_index())
+    }
+
+    fn get_memory_copy_sig(&mut self, func: &mut Function) -> ir::SigRef {
+        let sig = self.memory_copy_sig.unwrap_or_else(|| {
+            func.import_signature(Signature {
+                params: vec![
+                    AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext),
+                    // Memory index.
+                    AbiParam::new(I32),
+                    // Destination address.
+                    AbiParam::new(I32),
+                    // Source address.
+                    AbiParam::new(I32),
+                    // Length.
+                    AbiParam::new(I32),
+                    // Source location.
+                    AbiParam::new(I32),
+                ],
+                returns: vec![],
+                call_conv: self.target_config.default_call_conv,
+            })
+        });
+        self.memory_copy_sig = Some(sig);
+        sig
+    }
+
+    fn get_memory_copy_func(
+        &mut self,
+        func: &mut Function,
+        memory_index: MemoryIndex,
+    ) -> (ir::SigRef, usize, BuiltinFunctionIndex) {
+        let sig = self.get_memory_copy_sig(func);
+        if let Some(defined_memory_index) = self.module.defined_memory_index(memory_index) {
+            (
+                sig,
+                defined_memory_index.index(),
+                BuiltinFunctionIndex::get_defined_memory_copy_index(),
+            )
+        } else {
+            (
+                sig,
+                memory_index.index(),
+                BuiltinFunctionIndex::get_imported_memory_copy_index(),
+            )
+        }
+    }
+
+    fn get_memory_fill_sig(&mut self, func: &mut Function) -> ir::SigRef {
+        let sig = self.memory_fill_sig.unwrap_or_else(|| {
+            func.import_signature(Signature {
+                params: vec![
+                    AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext),
+                    // Memory index.
+                    AbiParam::new(I32),
+                    // Destination address.
+                    AbiParam::new(I32),
+                    // Value.
+                    AbiParam::new(I32),
+                    // Length.
+                    AbiParam::new(I32),
+                    // Source location.
+                    AbiParam::new(I32),
+                ],
+                returns: vec![],
+                call_conv: self.target_config.default_call_conv,
+            })
+        });
+        self.memory_fill_sig = Some(sig);
+        sig
+    }
+
+    fn get_memory_fill_func(
+        &mut self,
+        func: &mut Function,
+        memory_index: MemoryIndex,
+    ) -> (ir::SigRef, usize, BuiltinFunctionIndex) {
+        let sig = self.get_memory_fill_sig(func);
+        if let Some(defined_memory_index) = self.module.defined_memory_index(memory_index) {
+            (
+                sig,
+                defined_memory_index.index(),
+                BuiltinFunctionIndex::get_memory_fill_index(),
+            )
+        } else {
+            (
+                sig,
+                memory_index.index(),
+                BuiltinFunctionIndex::get_imported_memory_fill_index(),
+            )
+        }
+    }
+
     /// Translates load of builtin function and returns a pair of values `vmctx`
     /// and address of the loaded function.
     fn translate_load_builtin_function_address(
@@ -775,26 +1020,58 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
 
     fn translate_memory_copy(
         &mut self,
-        _pos: FuncCursor,
+        mut pos: FuncCursor,
-        _index: MemoryIndex,
+        memory_index: MemoryIndex,
         _heap: ir::Heap,
-        _dst: ir::Value,
+        dst: ir::Value,
-        _src: ir::Value,
+        src: ir::Value,
-        _len: ir::Value,
+        len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        let (func_sig, memory_index, func_idx) =
+            self.get_memory_copy_func(&mut pos.func, memory_index);
+
+        let memory_index_arg = pos.ins().iconst(I32, memory_index as i64);
+
+        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+
+        let src_loc = pos.srcloc();
+        let src_loc_arg = pos.ins().iconst(I32, src_loc.bits() as i64);
+
+        pos.ins().call_indirect(
+            func_sig,
+            func_addr,
+            &[vmctx, memory_index_arg, dst, src, len, src_loc_arg],
+        );
+
+        Ok(())
     }
 
     fn translate_memory_fill(
         &mut self,
-        _pos: FuncCursor,
+        mut pos: FuncCursor,
-        _index: MemoryIndex,
+        memory_index: MemoryIndex,
         _heap: ir::Heap,
-        _dst: ir::Value,
+        dst: ir::Value,
-        _val: ir::Value,
+        val: ir::Value,
-        _len: ir::Value,
+        len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        let (func_sig, memory_index, func_idx) =
+            self.get_memory_fill_func(&mut pos.func, memory_index);
+
+        let memory_index_arg = pos.ins().iconst(I32, memory_index as i64);
+
+        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+
+        let src_loc = pos.srcloc();
+        let src_loc_arg = pos.ins().iconst(I32, src_loc.bits() as i64);
+
+        pos.ins().call_indirect(
+            func_sig,
+            func_addr,
+            &[vmctx, memory_index_arg, dst, val, len, src_loc_arg],
+        );
+
+        Ok(())
     }
 
     fn translate_memory_init(
@@ -807,11 +1084,15 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         _src: ir::Value,
         _len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `memory.init`".to_string(),
+        ))
     }
 
     fn translate_data_drop(&mut self, _pos: FuncCursor, _seg_index: u32) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `data.drop`".to_string(),
+        ))
     }
 
     fn translate_table_size(
@@ -820,37 +1101,98 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         _index: TableIndex,
         _table: ir::Table,
     ) -> WasmResult<ir::Value> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `table.size`".to_string(),
+        ))
     }
 
     fn translate_table_copy(
         &mut self,
-        _pos: FuncCursor,
+        mut pos: FuncCursor,
-        _dst_table_index: TableIndex,
+        dst_table_index: TableIndex,
         _dst_table: ir::Table,
-        _src_table_index: TableIndex,
+        src_table_index: TableIndex,
         _src_table: ir::Table,
-        _dst: ir::Value,
+        dst: ir::Value,
-        _src: ir::Value,
+        src: ir::Value,
-        _len: ir::Value,
+        len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        let (func_sig, dst_table_index_arg, src_table_index_arg, func_idx) =
+            self.get_table_copy_func(&mut pos.func, dst_table_index, src_table_index);
+
+        let dst_table_index_arg = pos.ins().iconst(I32, dst_table_index_arg as i64);
+        let src_table_index_arg = pos.ins().iconst(I32, src_table_index_arg as i64);
+
+        let src_loc = pos.srcloc();
+        let src_loc_arg = pos.ins().iconst(I32, src_loc.bits() as i64);
+
+        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+
+        pos.ins().call_indirect(
+            func_sig,
+            func_addr,
+            &[
+                vmctx,
+                dst_table_index_arg,
+                src_table_index_arg,
+                dst,
+                src,
+                len,
+                src_loc_arg,
+            ],
+        );
+
+        Ok(())
     }
 
     fn translate_table_init(
         &mut self,
-        _pos: FuncCursor,
+        mut pos: FuncCursor,
-        _seg_index: u32,
+        seg_index: u32,
-        _table_index: TableIndex,
+        table_index: TableIndex,
         _table: ir::Table,
-        _dst: ir::Value,
+        dst: ir::Value,
-        _src: ir::Value,
+        src: ir::Value,
-        _len: ir::Value,
+        len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        let (func_sig, table_index_arg, func_idx) =
+            self.get_table_init_func(&mut pos.func, table_index);
+
+        let table_index_arg = pos.ins().iconst(I32, table_index_arg as i64);
+        let seg_index_arg = pos.ins().iconst(I32, seg_index as i64);
+
+        let src_loc = pos.srcloc();
+        let src_loc_arg = pos.ins().iconst(I32, src_loc.bits() as i64);
+
+        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+
+        pos.ins().call_indirect(
+            func_sig,
+            func_addr,
+            &[
+                vmctx,
+                table_index_arg,
+                seg_index_arg,
+                dst,
+                src,
+                len,
+                src_loc_arg,
+            ],
+        );
+
+        Ok(())
     }
 
-    fn translate_elem_drop(&mut self, _pos: FuncCursor, _seg_index: u32) -> WasmResult<()> {
+    fn translate_elem_drop(&mut self, mut pos: FuncCursor, elem_index: u32) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        let (func_sig, func_idx) = self.get_elem_drop_func(&mut pos.func);
+
+        let elem_index_arg = pos.ins().iconst(I32, elem_index as i64);
+
+        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+
+        pos.ins()
+            .call_indirect(func_sig, func_addr, &[vmctx, elem_index_arg]);
+
+        Ok(())
     }
 }

crates/environ/src/module.rs

@@ -6,7 +6,7 @@ use cranelift_codegen::ir;
 use cranelift_entity::{EntityRef, PrimaryMap};
 use cranelift_wasm::{
     DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, Global,
-    GlobalIndex, Memory, MemoryIndex, SignatureIndex, Table, TableIndex,
+    GlobalIndex, Memory, MemoryIndex, PassiveElemIndex, SignatureIndex, Table, TableIndex,
 };
 use indexmap::IndexMap;
 use more_asserts::assert_ge;
@@ -165,6 +165,9 @@ pub struct Module {
     /// WebAssembly table initializers.
     pub table_elements: Vec<TableElements>,
 
+    /// WebAssembly passive elements.
+    pub passive_elements: HashMap<PassiveElemIndex, Box<[FuncIndex]>>,
+
     /// WebAssembly table initializers.
     pub func_names: HashMap<FuncIndex, String>,
 }
@@ -219,6 +222,7 @@ impl Module {
             exports: IndexMap::new(),
             start_func: None,
             table_elements: Vec::new(),
+            passive_elements: HashMap::new(),
             func_names: HashMap::new(),
             local: ModuleLocal {
                 num_imported_funcs: 0,
@@ -233,6 +237,11 @@ impl Module {
             },
         }
     }
+
+    /// Get the given passive element, if it exists.
+    pub fn get_passive_element(&self, index: PassiveElemIndex) -> Option<&[FuncIndex]> {
+        self.passive_elements.get(&index).map(|es| &**es)
+    }
 }
 
 impl ModuleLocal {

crates/environ/src/module_environ.rs

@@ -336,12 +336,20 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
 
     fn declare_passive_element(
         &mut self,
-        _: PassiveElemIndex,
+        elem_index: PassiveElemIndex,
-        _: Box<[FuncIndex]>,
+        segments: Box<[FuncIndex]>,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported(
+        let old = self
-            "bulk memory: passive element segment".into(),
+            .result
-        ))
+            .module
+            .passive_elements
+            .insert(elem_index, segments);
+        debug_assert!(
+            old.is_none(),
+            "should never get duplicate element indices, that would be a bug in `cranelift_wasm`'s \
+             translation"
+        );
+        Ok(())
     }
 
     fn define_function_body(
@@ -382,9 +390,18 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
         Ok(())
     }
 
-    fn declare_passive_data(&mut self, _: PassiveDataIndex, _: &'data [u8]) -> WasmResult<()> {
+    fn reserve_passive_data(&mut self, count: u32) -> WasmResult<()> {
+        self.result.module.passive_elements.reserve(count as usize);
+        Ok(())
+    }
+
+    fn declare_passive_data(
+        &mut self,
+        _data_index: PassiveDataIndex,
+        _data: &'data [u8],
+    ) -> WasmResult<()> {
         Err(WasmError::Unsupported(
-            "bulk memory: passive data segment".into(),
+            "bulk memory: passive data".to_string(),
         ))
     }
 

crates/fuzzing/Cargo.toml

@@ -17,4 +17,4 @@ wasmprinter = "0.2.1"
 wasmtime = { path = "../api", version = "0.12.0" }
 
 [dev-dependencies]
-wat = "1.0"
+wat = "1.0.10"

crates/jit/src/instantiate.rs

@@ -205,6 +205,7 @@ impl CompiledModule {
     /// See `InstanceHandle::new`
     pub unsafe fn instantiate(
         &self,
+        is_bulk_memory: bool,
         resolver: &mut dyn Resolver,
     ) -> Result<InstanceHandle, InstantiationError> {
         let data_initializers = self
@@ -224,6 +225,7 @@ impl CompiledModule {
             &data_initializers,
             self.signatures.clone(),
             self.dbg_jit_registration.as_ref().map(|r| Rc::clone(&r)),
+            is_bulk_memory,
             Box::new(()),
         )
     }
@@ -277,9 +279,10 @@ pub unsafe fn instantiate(
     data: &[u8],
     resolver: &mut dyn Resolver,
     debug_info: bool,
+    is_bulk_memory: bool,
     profiler: Option<&Arc<Mutex<Box<dyn ProfilingAgent + Send>>>>,
 ) -> Result<InstanceHandle, SetupError> {
-    let instance =
+    let instance = CompiledModule::new(compiler, data, debug_info, profiler)?
-        CompiledModule::new(compiler, data, debug_info, profiler)?.instantiate(resolver)?;
+        .instantiate(is_bulk_memory, resolver)?;
     Ok(instance)
 }

crates/lightbeam/Cargo.toml

@@ -27,7 +27,7 @@ more-asserts = "0.2.1"
 
 [dev-dependencies]
 lazy_static = "1.2"
-wat = "1.0.2"
+wat = "1.0.9"
 quickcheck = "0.9.0"
 anyhow = "1.0"
 

crates/runtime/src/instance.rs

@@ -19,19 +19,20 @@ use memoffset::offset_of;
 use more_asserts::assert_lt;
 use std::alloc::{self, Layout};
 use std::any::Any;
-use std::cell::Cell;
+use std::cell::{Cell, RefCell};
+use std::collections::HashMap;
 use std::collections::HashSet;
 use std::convert::TryFrom;
 use std::rc::Rc;
 use std::sync::Arc;
 use std::{mem, ptr, slice};
 use thiserror::Error;
-use wasmtime_environ::entity::{BoxedSlice, EntityRef, PrimaryMap};
+use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap};
 use wasmtime_environ::wasm::{
     DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex,
-    GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, TableIndex,
+    GlobalIndex, GlobalInit, MemoryIndex, PassiveElemIndex, SignatureIndex, TableIndex,
 };
-use wasmtime_environ::{DataInitializer, Module, TableElements, VMOffsets};
+use wasmtime_environ::{ir, DataInitializer, Module, TableElements, VMOffsets};
 
 cfg_if::cfg_if! {
     if #[cfg(unix)] {
@@ -86,6 +87,11 @@ pub(crate) struct Instance {
     /// WebAssembly table data.
     tables: BoxedSlice<DefinedTableIndex, Table>,
 
+    /// Passive elements in this instantiation. As `elem.drop`s happen, these
+    /// entries get removed. A missing entry is considered equivalent to an
+    /// empty slice.
+    passive_elements: RefCell<HashMap<PassiveElemIndex, Box<[VMCallerCheckedAnyfunc]>>>,
+
     /// Pointers to functions in executable memory.
     finished_functions: BoxedSlice<DefinedFuncIndex, *mut [VMFunctionBody]>,
 
@@ -124,6 +130,14 @@ impl Instance {
         unsafe { *self.signature_ids_ptr().add(index) }
     }
 
+    pub(crate) fn module(&self) -> &Arc<Module> {
+        &self.module
+    }
+
+    pub(crate) fn module_ref(&self) -> &Module {
+        &*self.module
+    }
+
     /// Return a pointer to the `VMSharedSignatureIndex`s.
     fn signature_ids_ptr(&self) -> *mut VMSharedSignatureIndex {
         unsafe { self.vmctx_plus_offset(self.offsets.vmctx_signature_ids_begin()) }
@@ -197,6 +211,16 @@ impl Instance {
         unsafe { self.vmctx_plus_offset(self.offsets.vmctx_tables_begin()) }
     }
 
+    /// Get a locally defined or imported memory.
+    pub(crate) fn get_memory(&self, index: MemoryIndex) -> VMMemoryDefinition {
+        if let Some(defined_index) = self.module.local.defined_memory_index(index) {
+            self.memory(defined_index)
+        } else {
+            let import = self.imported_memory(index);
+            *unsafe { import.from.as_ref().unwrap() }
+        }
+    }
+
     /// Return the indexed `VMMemoryDefinition`.
     fn memory(&self, index: DefinedMemoryIndex) -> VMMemoryDefinition {
         unsafe { *self.memory_ptr(index) }
@@ -526,6 +550,226 @@ impl Instance {
         let align = mem::align_of_val(self);
         Layout::from_size_align(size, align).unwrap()
     }
+
+    /// Get a `VMCallerCheckedAnyfunc` for the given `FuncIndex`.
+    fn get_caller_checked_anyfunc(&self, index: FuncIndex) -> VMCallerCheckedAnyfunc {
+        if index == FuncIndex::reserved_value() {
+            return VMCallerCheckedAnyfunc::default();
+        }
+
+        let sig = self.module.local.functions[index];
+        let type_index = self.signature_id(sig);
+
+        let (func_ptr, vmctx) = if let Some(def_index) = self.module.local.defined_func_index(index)
+        {
+            (
+                self.finished_functions[def_index] as *const _,
+                self.vmctx_ptr(),
+            )
+        } else {
+            let import = self.imported_function(index);
+            (import.body, import.vmctx)
+        };
+        VMCallerCheckedAnyfunc {
+            func_ptr,
+            type_index,
+            vmctx,
+        }
+    }
+
+    /// The `table.init` operation: initializes a portion of a table with a
+    /// passive element.
+    ///
+    /// # Errors
+    ///
+    /// Returns a `Trap` error when the range within the table is out of bounds
+    /// or the range within the passive element is out of bounds.
+    pub(crate) fn table_init(
+        &self,
+        table_index: TableIndex,
+        elem_index: PassiveElemIndex,
+        dst: u32,
+        src: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init
+
+        let table = self.get_table(table_index);
+        let passive_elements = self.passive_elements.borrow();
+        let elem = passive_elements
+            .get(&elem_index)
+            .map(|e| &**e)
+            .unwrap_or_else(|| &[]);
+
+        if src
+            .checked_add(len)
+            .map_or(true, |n| n as usize > elem.len())
+            || dst.checked_add(len).map_or(true, |m| m > table.size())
+        {
+            return Err(Trap::wasm(source_loc, ir::TrapCode::TableOutOfBounds));
+        }
+
+        // TODO(#983): investigate replacing this get/set loop with a `memcpy`.
+        for (dst, src) in (dst..dst + len).zip(src..src + len) {
+            table
+                .set(dst, elem[src as usize].clone())
+                .expect("should never panic because we already did the bounds check above");
+        }
+
+        Ok(())
+    }
+
+    /// Drop an element.
+    pub(crate) fn elem_drop(&self, elem_index: PassiveElemIndex) {
+        // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop
+
+        let mut passive_elements = self.passive_elements.borrow_mut();
+        passive_elements.remove(&elem_index);
+        // Note that we don't check that we actually removed an element because
+        // dropping a non-passive element is a no-op (not a trap).
+    }
+
+    /// Do a `memory.copy` for a locally defined memory.
+    ///
+    /// # Errors
+    ///
+    /// Returns a `Trap` error when the source or destination ranges are out of
+    /// bounds.
+    pub(crate) fn defined_memory_copy(
+        &self,
+        memory_index: DefinedMemoryIndex,
+        dst: u32,
+        src: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy
+
+        let memory = self.memory(memory_index);
+
+        if src
+            .checked_add(len)
+            .map_or(true, |n| n as usize > memory.current_length)
+            || dst
+                .checked_add(len)
+                .map_or(true, |m| m as usize > memory.current_length)
+        {
+            return Err(Trap::wasm(source_loc, ir::TrapCode::HeapOutOfBounds));
+        }
+
+        let dst = usize::try_from(dst).unwrap();
+        let src = usize::try_from(src).unwrap();
+
+        // Bounds and casts are checked above, by this point we know that
+        // everything is safe.
+        unsafe {
+            let dst = memory.base.add(dst);
+            let src = memory.base.add(src);
+            ptr::copy(src, dst, len as usize);
+        }
+
+        Ok(())
+    }
+
+    /// Perform a `memory.copy` on an imported memory.
+    pub(crate) fn imported_memory_copy(
+        &self,
+        memory_index: MemoryIndex,
+        dst: u32,
+        src: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        let import = self.imported_memory(memory_index);
+        unsafe {
+            let foreign_instance = (&*import.vmctx).instance();
+            let foreign_memory = &*import.from;
+            let foreign_index = foreign_instance.memory_index(foreign_memory);
+            foreign_instance.defined_memory_copy(foreign_index, dst, src, len, source_loc)
+        }
+    }
+
+    /// Perform the `memory.fill` operation on a locally defined memory.
+    ///
+    /// # Errors
+    ///
+    /// Returns a `Trap` error if the memory range is out of bounds.
+    pub(crate) fn defined_memory_fill(
+        &self,
+        memory_index: DefinedMemoryIndex,
+        dst: u32,
+        val: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        let memory = self.memory(memory_index);
+
+        if dst
+            .checked_add(len)
+            .map_or(true, |m| m as usize > memory.current_length)
+        {
+            return Err(Trap::wasm(source_loc, ir::TrapCode::HeapOutOfBounds));
+        }
+
+        let dst = isize::try_from(dst).unwrap();
+        let val = val as u8;
+
+        // Bounds and casts are checked above, by this point we know that
+        // everything is safe.
+        unsafe {
+            let dst = memory.base.offset(dst);
+            ptr::write_bytes(dst, val, len as usize);
+        }
+
+        Ok(())
+    }
+
+    /// Perform the `memory.fill` operation on an imported memory.
+    ///
+    /// # Errors
+    ///
+    /// Returns a `Trap` error if the memory range is out of bounds.
+    pub(crate) fn imported_memory_fill(
+        &self,
+        memory_index: MemoryIndex,
+        dst: u32,
+        val: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        let import = self.imported_memory(memory_index);
+        unsafe {
+            let foreign_instance = (&*import.vmctx).instance();
+            let foreign_memory = &*import.from;
+            let foreign_index = foreign_instance.memory_index(foreign_memory);
+            foreign_instance.defined_memory_fill(foreign_index, dst, val, len, source_loc)
+        }
+    }
+
+    /// Get a table by index regardless of whether it is locally-defined or an
+    /// imported, foreign table.
+    pub(crate) fn get_table(&self, table_index: TableIndex) -> &Table {
+        if let Some(defined_table_index) = self.module.local.defined_table_index(table_index) {
+            self.get_defined_table(defined_table_index)
+        } else {
+            self.get_foreign_table(table_index)
+        }
+    }
+
+    /// Get a locally-defined table.
+    pub(crate) fn get_defined_table(&self, index: DefinedTableIndex) -> &Table {
+        &self.tables[index]
+    }
+
+    /// Get an imported, foreign table.
+    pub(crate) fn get_foreign_table(&self, index: TableIndex) -> &Table {
+        let import = self.imported_table(index);
+        let foreign_instance = unsafe { (&mut *(import).vmctx).instance() };
+        let foreign_table = unsafe { &mut *(import).from };
+        let foreign_index = foreign_instance.table_index(foreign_table);
+        &foreign_instance.tables[foreign_index]
+    }
 }
 
 /// A handle holding an `Instance` of a WebAssembly module.
@@ -558,6 +802,7 @@ impl InstanceHandle {
         data_initializers: &[DataInitializer<'_>],
         vmshared_signatures: BoxedSlice<SignatureIndex, VMSharedSignatureIndex>,
         dbg_jit_registration: Option<Rc<GdbJitImageRegistration>>,
+        is_bulk_memory: bool,
         host_state: Box<dyn Any>,
     ) -> Result<Self, InstantiationError> {
         let tables = create_tables(&module);
@@ -587,6 +832,7 @@ impl InstanceHandle {
                 offsets,
                 memories,
                 tables,
+                passive_elements: Default::default(),
                 finished_functions,
                 dbg_jit_registration,
                 host_state,
@@ -651,12 +897,18 @@ impl InstanceHandle {
             VMBuiltinFunctionsArray::initialized(),
         );
 
-        // Check initializer bounds before initializing anything.
+        // Check initializer bounds before initializing anything. Only do this
-        check_table_init_bounds(instance)?;
+        // when bulk memory is disabled, since the bulk memory proposal changes
-        check_memory_init_bounds(instance, data_initializers)?;
+        // instantiation such that the intermediate results of failed
+        // initializations are visible.
+        if !is_bulk_memory {
+            check_table_init_bounds(instance)?;
+            check_memory_init_bounds(instance, data_initializers)?;
+        }
 
         // Apply the initializers.
         initialize_tables(instance)?;
+        initialize_passive_elements(instance);
         initialize_memories(instance, data_initializers)?;
         initialize_globals(instance);
 
@@ -697,12 +949,12 @@ impl InstanceHandle {
 
     /// Return a reference-counting pointer to a module.
     pub fn module(&self) -> &Arc<Module> {
-        &self.instance().module
+        self.instance().module()
     }
 
     /// Return a reference to a module.
     pub fn module_ref(&self) -> &Module {
-        &self.instance().module
+        self.instance().module_ref()
     }
 
     /// Lookup an export with the given name.
@@ -778,6 +1030,11 @@ impl InstanceHandle {
         self.instance().table_set(table_index, index, val)
     }
 
+    /// Get a table defined locally within this module.
+    pub fn get_defined_table(&self, index: DefinedTableIndex) -> &Table {
+        self.instance().get_defined_table(index)
+    }
+
     /// Return a reference to the contained `Instance`.
     pub(crate) fn instance(&self) -> &Instance {
         unsafe { &*(self.instance as *const Instance) }
@@ -813,12 +1070,12 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError
     let module = Arc::clone(&instance.module);
     for init in &module.table_elements {
         let start = get_table_init_start(init, instance);
-        let table = get_table(init, instance);
+        let table = instance.get_table(init.table_index);
 
         let size = usize::try_from(table.size()).unwrap();
         if size < start + init.elements.len() {
             return Err(InstantiationError::Link(LinkError(
-                "elements segment does not fit".to_owned(),
+                "table out of bounds: elements segment does not fit".to_owned(),
             )));
         }
     }
@@ -875,7 +1132,7 @@ fn check_memory_init_bounds(
             let mem_slice = get_memory_slice(init, instance);
             if mem_slice.get_mut(start..start + init.data.len()).is_none() {
                 return Err(InstantiationError::Link(LinkError(
-                    "data segment does not fit".to_owned(),
+                    "memory out of bounds: data segment does not fit".into(),
                 )));
             }
         }
@@ -913,46 +1170,27 @@ fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize {
     start
 }
 
-/// Return a byte-slice view of a table's data.
-fn get_table<'instance>(init: &TableElements, instance: &'instance Instance) -> &'instance Table {
-    if let Some(defined_table_index) = instance.module.local.defined_table_index(init.table_index) {
-        &instance.tables[defined_table_index]
-    } else {
-        let import = instance.imported_table(init.table_index);
-        let foreign_instance = unsafe { (&mut *(import).vmctx).instance() };
-        let foreign_table = unsafe { &mut *(import).from };
-        let foreign_index = foreign_instance.table_index(foreign_table);
-        &foreign_instance.tables[foreign_index]
-    }
-}
-
 /// Initialize the table memory from the provided initializers.
 fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> {
-    let vmctx = instance.vmctx_ptr();
     let module = Arc::clone(&instance.module);
     for init in &module.table_elements {
         let start = get_table_init_start(init, instance);
-        let table = get_table(init, instance);
+        let table = instance.get_table(init.table_index);
+
+        if start
+            .checked_add(init.elements.len())
+            .map_or(true, |end| end > table.size() as usize)
+        {
+            return Err(InstantiationError::Trap(Trap::wasm(
+                ir::SourceLoc::default(),
+                ir::TrapCode::HeapOutOfBounds,
+            )));
+        }
 
         for (i, func_idx) in init.elements.iter().enumerate() {
-            let callee_sig = instance.module.local.functions[*func_idx];
+            let anyfunc = instance.get_caller_checked_anyfunc(*func_idx);
-            let (callee_ptr, callee_vmctx) =
-                if let Some(index) = instance.module.local.defined_func_index(*func_idx) {
-                    (instance.finished_functions[index] as *const _, vmctx)
-                } else {
-                    let imported_func = instance.imported_function(*func_idx);
-                    (imported_func.body, imported_func.vmctx)
-                };
-            let type_index = instance.signature_id(callee_sig);
             table
-                .set(
+                .set(u32::try_from(start + i).unwrap(), anyfunc)
-                    u32::try_from(start + i).unwrap(),
-                    VMCallerCheckedAnyfunc {
-                        func_ptr: callee_ptr,
-                        type_index,
-                        vmctx: callee_vmctx,
-                    },
-                )
                 .unwrap();
         }
     }
@@ -960,6 +1198,34 @@ fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> {
     Ok(())
 }
 
+/// Initialize the `Instance::passive_elements` map by resolving the
+/// `Module::passive_elements`'s `FuncIndex`s into `VMCallerCheckedAnyfunc`s for
+/// this instance.
+fn initialize_passive_elements(instance: &Instance) {
+    let mut passive_elements = instance.passive_elements.borrow_mut();
+    debug_assert!(
+        passive_elements.is_empty(),
+        "should only be called once, at initialization time"
+    );
+
+    passive_elements.extend(
+        instance
+            .module
+            .passive_elements
+            .iter()
+            .filter(|(_, segments)| !segments.is_empty())
+            .map(|(idx, segments)| {
+                (
+                    *idx,
+                    segments
+                        .iter()
+                        .map(|s| instance.get_caller_checked_anyfunc(*s))
+                        .collect(),
+                )
+            }),
+    );
+}
+
 /// Allocate memory for just the memories of the current module.
 fn create_memories(
     module: &Module,
@@ -979,10 +1245,23 @@ fn initialize_memories(
     data_initializers: &[DataInitializer<'_>],
 ) -> Result<(), InstantiationError> {
     for init in data_initializers {
+        let memory = instance.get_memory(init.location.memory_index);
+
         let start = get_memory_init_start(init, instance);
+        if start
+            .checked_add(init.data.len())
+            .map_or(true, |end| end > memory.current_length)
+        {
+            return Err(InstantiationError::Trap(Trap::wasm(
+                ir::SourceLoc::default(),
+                ir::TrapCode::HeapOutOfBounds,
+            )));
+        }
+
         unsafe {
             let mem_slice = get_memory_slice(init, instance);
-            let to_init = &mut mem_slice[start..start + init.data.len()];
+            let end = start + init.data.len();
+            let to_init = &mut mem_slice[start..end];
             to_init.copy_from_slice(init.data);
         }
     }
@@ -1047,6 +1326,10 @@ pub enum InstantiationError {
     #[error("Failed to link module")]
     Link(#[from] LinkError),
 
+    /// A trap ocurred during instantiation, after linking.
+    #[error("Trap occurred during instantiation")]
+    Trap(#[source] Trap),
+
     /// A compilation error occured.
     #[error("Trap occurred while invoking start function")]
     StartTrap(#[source] Trap),

crates/runtime/src/lib.rs

@@ -42,9 +42,12 @@ pub use crate::instance::{InstanceHandle, InstantiationError, LinkError};
 pub use crate::jit_int::GdbJitImageRegistration;
 pub use crate::mmap::Mmap;
 pub use crate::sig_registry::SignatureRegistry;
+pub use crate::table::Table;
 pub use crate::trap_registry::{TrapDescription, TrapRegistration, TrapRegistry};
 pub use crate::traphandlers::resume_panic;
-pub use crate::traphandlers::{catch_traps, raise_user_trap, wasmtime_call_trampoline, Trap};
+pub use crate::traphandlers::{
+    catch_traps, raise_lib_trap, raise_user_trap, wasmtime_call_trampoline, Trap,
+};
 pub use crate::vmcontext::{
     VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, VMGlobalDefinition,
     VMGlobalImport, VMInvokeArgument, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex,

crates/runtime/src/libcalls.rs

@@ -1,9 +1,42 @@
-//! Runtime library calls. Note that wasm compilers may sometimes perform these
+//! Runtime library calls.
-//! inline rather than calling them, particularly when CPUs have special
+//!
-//! instructions which compute them directly.
+//! Note that Wasm compilers may sometimes perform these inline rather than
+//! calling them, particularly when CPUs have special instructions which compute
+//! them directly.
+//!
+//! These functions are called by compiled Wasm code, and therefore must take
+//! certain care about some things:
+//!
+//! * They must always be `pub extern "C"` and should only contain basic, raw
+//!   i32/i64/f32/f64/pointer parameters that are safe to pass across the system
+//!   ABI!
+//!
+//! * If any nested function propagates an `Err(trap)` out to the library
+//!   function frame, we need to raise it. This involves some nasty and quite
+//!   unsafe code under the covers! Notable, after raising the trap, drops
+//!   **will not** be run for local variables! This can lead to things like
+//!   leaking `InstanceHandle`s which leads to never deallocating JIT code,
+//!   instances, and modules! Therefore, always use nested blocks to ensure
+//!   drops run before raising a trap:
+//!
+//!   ```ignore
+//!   pub extern "C" fn my_lib_function(...) {
+//!       let result = {
+//!           // Do everything in here so drops run at the end of the block.
+//!           ...
+//!       };
+//!       if let Err(trap) = result {
+//!           // Now we can safely raise the trap without leaking!
+//!           raise_lib_trap(trap);
+//!       }
+//!   }
+//!   ```
 
+use crate::table::Table;
+use crate::traphandlers::raise_lib_trap;
 use crate::vmcontext::VMContext;
-use wasmtime_environ::wasm::{DefinedMemoryIndex, MemoryIndex};
+use wasmtime_environ::ir;
+use wasmtime_environ::wasm::{DefinedMemoryIndex, MemoryIndex, PassiveElemIndex, TableIndex};
 
 /// Implementation of f32.ceil
 pub extern "C" fn wasmtime_f32_ceil(x: f32) -> f32 {
@@ -88,7 +121,6 @@ pub extern "C" fn wasmtime_f64_nearest(x: f64) -> f64 {
 }
 
 /// Implementation of memory.grow for locally-defined 32-bit memories.
-#[no_mangle]
 pub unsafe extern "C" fn wasmtime_memory32_grow(
     vmctx: *mut VMContext,
     delta: u32,
@@ -103,7 +135,6 @@ pub unsafe extern "C" fn wasmtime_memory32_grow(
 }
 
 /// Implementation of memory.grow for imported 32-bit memories.
-#[no_mangle]
 pub unsafe extern "C" fn wasmtime_imported_memory32_grow(
     vmctx: *mut VMContext,
     delta: u32,
@@ -118,7 +149,6 @@ pub unsafe extern "C" fn wasmtime_imported_memory32_grow(
 }
 
 /// Implementation of memory.size for locally-defined 32-bit memories.
-#[no_mangle]
 pub unsafe extern "C" fn wasmtime_memory32_size(vmctx: *mut VMContext, memory_index: u32) -> u32 {
     let instance = (&mut *vmctx).instance();
     let memory_index = DefinedMemoryIndex::from_u32(memory_index);
@@ -127,7 +157,6 @@ pub unsafe extern "C" fn wasmtime_memory32_size(vmctx: *mut VMContext, memory_in
 }
 
 /// Implementation of memory.size for imported 32-bit memories.
-#[no_mangle]
 pub unsafe extern "C" fn wasmtime_imported_memory32_size(
     vmctx: *mut VMContext,
     memory_index: u32,
@@ -137,3 +166,136 @@ pub unsafe extern "C" fn wasmtime_imported_memory32_size(
 
     instance.imported_memory_size(memory_index)
 }
+
+/// Implementation of `table.copy`.
+pub unsafe extern "C" fn wasmtime_table_copy(
+    vmctx: *mut VMContext,
+    dst_table_index: u32,
+    src_table_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let result = {
+        let dst_table_index = TableIndex::from_u32(dst_table_index);
+        let src_table_index = TableIndex::from_u32(src_table_index);
+        let source_loc = ir::SourceLoc::new(source_loc);
+        let instance = (&mut *vmctx).instance();
+        let dst_table = instance.get_table(dst_table_index);
+        let src_table = instance.get_table(src_table_index);
+        Table::copy(dst_table, src_table, dst, src, len, source_loc)
+    };
+    if let Err(trap) = result {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `table.init`.
+pub unsafe extern "C" fn wasmtime_table_init(
+    vmctx: *mut VMContext,
+    table_index: u32,
+    elem_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let result = {
+        let table_index = TableIndex::from_u32(table_index);
+        let source_loc = ir::SourceLoc::new(source_loc);
+        let elem_index = PassiveElemIndex::from_u32(elem_index);
+        let instance = (&mut *vmctx).instance();
+        instance.table_init(table_index, elem_index, dst, src, len, source_loc)
+    };
+    if let Err(trap) = result {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `elem.drop`.
+pub unsafe extern "C" fn wasmtime_elem_drop(vmctx: *mut VMContext, elem_index: u32) {
+    let elem_index = PassiveElemIndex::from_u32(elem_index);
+    let instance = (&mut *vmctx).instance();
+    instance.elem_drop(elem_index);
+}
+
+/// Implementation of `memory.copy` for locally defined memories.
+pub unsafe extern "C" fn wasmtime_defined_memory_copy(
+    vmctx: *mut VMContext,
+    memory_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let result = {
+        let memory_index = DefinedMemoryIndex::from_u32(memory_index);
+        let source_loc = ir::SourceLoc::new(source_loc);
+        let instance = (&mut *vmctx).instance();
+        instance.defined_memory_copy(memory_index, dst, src, len, source_loc)
+    };
+    if let Err(trap) = result {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `memory.copy` for imported memories.
+pub unsafe extern "C" fn wasmtime_imported_memory_copy(
+    vmctx: *mut VMContext,
+    memory_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let result = {
+        let memory_index = MemoryIndex::from_u32(memory_index);
+        let source_loc = ir::SourceLoc::new(source_loc);
+        let instance = (&mut *vmctx).instance();
+        instance.imported_memory_copy(memory_index, dst, src, len, source_loc)
+    };
+    if let Err(trap) = result {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `memory.fill` for locally defined memories.
+pub unsafe extern "C" fn wasmtime_memory_fill(
+    vmctx: *mut VMContext,
+    memory_index: u32,
+    dst: u32,
+    val: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let result = {
+        let memory_index = DefinedMemoryIndex::from_u32(memory_index);
+        let source_loc = ir::SourceLoc::new(source_loc);
+        let instance = (&mut *vmctx).instance();
+        instance.defined_memory_fill(memory_index, dst, val, len, source_loc)
+    };
+    if let Err(trap) = result {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `memory.fill` for imported memories.
+pub unsafe extern "C" fn wasmtime_imported_memory_fill(
+    vmctx: *mut VMContext,
+    memory_index: u32,
+    dst: u32,
+    val: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let result = {
+        let memory_index = MemoryIndex::from_u32(memory_index);
+        let source_loc = ir::SourceLoc::new(source_loc);
+        let instance = (&mut *vmctx).instance();
+        instance.imported_memory_fill(memory_index, dst, val, len, source_loc)
+    };
+    if let Err(trap) = result {
+        raise_lib_trap(trap);
+    }
+}

crates/runtime/src/table.rs

@@ -3,10 +3,11 @@
 //! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories.
 
 use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition};
+use crate::Trap;
 use std::cell::RefCell;
 use std::convert::{TryFrom, TryInto};
 use wasmtime_environ::wasm::TableElementType;
-use wasmtime_environ::{TablePlan, TableStyle};
+use wasmtime_environ::{ir, TablePlan, TableStyle};
 
 /// A table instance.
 #[derive(Debug)]
@@ -87,6 +88,52 @@ impl Table {
         }
     }
 
+    /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the range is out of bounds of either the source or
+    /// destination tables.
+    pub fn copy(
+        dst_table: &Self,
+        src_table: &Self,
+        dst_index: u32,
+        src_index: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-copy
+
+        if src_index
+            .checked_add(len)
+            .map_or(true, |n| n > src_table.size())
+            || dst_index
+                .checked_add(len)
+                .map_or(true, |m| m > dst_table.size())
+        {
+            return Err(Trap::wasm(source_loc, ir::TrapCode::TableOutOfBounds));
+        }
+
+        let srcs = src_index..src_index + len;
+        let dsts = dst_index..dst_index + len;
+
+        // Note on the unwraps: the bounds check above means that these will
+        // never panic.
+        //
+        // TODO(#983): investigate replacing this get/set loop with a `memcpy`.
+        if dst_index <= src_index {
+            for (s, d) in (srcs).zip(dsts) {
+                dst_table.set(d, src_table.get(s).unwrap()).unwrap();
+            }
+        } else {
+            for (s, d) in srcs.rev().zip(dsts.rev()) {
+                dst_table.set(d, src_table.get(s).unwrap()).unwrap();
+            }
+        }
+
+        Ok(())
+    }
+
     /// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
     pub fn vmtable(&self) -> VMTableDefinition {
         let mut vec = self.vec.borrow_mut();

crates/runtime/src/trap_registry.rs

@@ -84,7 +84,7 @@ fn trap_code_to_expected_string(trap_code: ir::TrapCode) -> String {
     match trap_code {
         StackOverflow => "call stack exhausted".to_string(),
         HeapOutOfBounds => "out of bounds memory access".to_string(),
-        TableOutOfBounds => "undefined element".to_string(),
+        TableOutOfBounds => "undefined element: out of bounds".to_string(),
         OutOfBounds => "out of bounds".to_string(), // Note: not covered by the test suite
         IndirectCallToNull => "uninitialized element".to_string(),
         BadSignature => "indirect call type mismatch".to_string(),

crates/runtime/src/traphandlers.rs

@@ -73,6 +73,20 @@ pub unsafe fn raise_user_trap(data: Box<dyn Error + Send + Sync>) -> ! {
     tls::with(|info| info.unwrap().unwind_with(UnwindReason::UserTrap(data)))
 }
 
+/// Raises a trap from inside library code immediately.
+///
+/// This function performs as-if a wasm trap was just executed. This trap
+/// payload is then returned from `wasmtime_call` and `wasmtime_call_trampoline`
+/// below.
+///
+/// # Safety
+///
+/// Only safe to call when wasm code is on the stack, aka `wasmtime_call` or
+/// `wasmtime_call_trampoline` must have been previously called.
+pub unsafe fn raise_lib_trap(trap: Trap) -> ! {
+    tls::with(|info| info.unwrap().unwind_with(UnwindReason::LibTrap(trap)))
+}
+
 /// Carries a Rust panic across wasm code and resumes the panic on the other
 /// side.
 ///
@@ -126,6 +140,20 @@ impl fmt::Display for Trap {
 
 impl std::error::Error for Trap {}
 
+impl Trap {
+    /// Construct a new Wasm trap with the given source location and trap code.
+    ///
+    /// Internally saves a backtrace when constructed.
+    pub fn wasm(source_loc: ir::SourceLoc, trap_code: ir::TrapCode) -> Self {
+        let desc = TrapDescription {
+            source_loc,
+            trap_code,
+        };
+        let backtrace = Backtrace::new();
+        Trap::Wasm { desc, backtrace }
+    }
+}
+
 /// Call the wasm function pointed to by `callee`.
 ///
 /// * `vmctx` - the callee vmctx argument
@@ -192,6 +220,7 @@ enum UnwindReason {
     None,
     Panic(Box<dyn Any + Send>),
     UserTrap(Box<dyn Error + Send + Sync>),
+    LibTrap(Trap),
     Trap { backtrace: Backtrace, pc: usize },
 }
 
@@ -219,6 +248,7 @@ impl CallThreadState {
                     debug_assert_eq!(ret, 0);
                     Err(Trap::User(data))
                 }
+                UnwindReason::LibTrap(trap) => Err(trap),
                 UnwindReason::Trap { backtrace, pc } => {
                     debug_assert_eq!(ret, 0);
                     let instance = unsafe { InstanceHandle::from_vmctx(self.vmctx) };

crates/runtime/src/vmcontext.rs

@@ -536,15 +536,38 @@ impl VMBuiltinFunctionsArray {
 
     pub fn initialized() -> Self {
         use crate::libcalls::*;
+
         let mut ptrs = [0; Self::len()];
+
         ptrs[BuiltinFunctionIndex::get_memory32_grow_index().index() as usize] =
             wasmtime_memory32_grow as usize;
         ptrs[BuiltinFunctionIndex::get_imported_memory32_grow_index().index() as usize] =
             wasmtime_imported_memory32_grow as usize;
+
         ptrs[BuiltinFunctionIndex::get_memory32_size_index().index() as usize] =
             wasmtime_memory32_size as usize;
         ptrs[BuiltinFunctionIndex::get_imported_memory32_size_index().index() as usize] =
             wasmtime_imported_memory32_size as usize;
+
+        ptrs[BuiltinFunctionIndex::get_table_copy_index().index() as usize] =
+            wasmtime_table_copy as usize;
+
+        ptrs[BuiltinFunctionIndex::get_table_init_index().index() as usize] =
+            wasmtime_table_init as usize;
+        ptrs[BuiltinFunctionIndex::get_elem_drop_index().index() as usize] =
+            wasmtime_elem_drop as usize;
+
+        ptrs[BuiltinFunctionIndex::get_defined_memory_copy_index().index() as usize] =
+            wasmtime_defined_memory_copy as usize;
+        ptrs[BuiltinFunctionIndex::get_imported_memory_copy_index().index() as usize] =
+            wasmtime_imported_memory_copy as usize;
+        ptrs[BuiltinFunctionIndex::get_memory_fill_index().index() as usize] =
+            wasmtime_memory_fill as usize;
+        ptrs[BuiltinFunctionIndex::get_imported_memory_fill_index().index() as usize] =
+            wasmtime_imported_memory_fill as usize;
+
+        debug_assert!(ptrs.iter().cloned().all(|p| p != 0));
+
         Self { ptrs }
     }
 }

crates/test-programs/Cargo.toml

@@ -18,7 +18,7 @@ pretty_env_logger = "0.3.0"
 tempfile = "3.1.0"
 os_pipe = "0.9"
 anyhow = "1.0.19"
-wat = "1.0.2"
+wat = "1.0.10"
 
 [features]
 test_programs = []

crates/wast/src/wast.rs

@@ -287,7 +287,7 @@ impl WastContext {
                     Err(e) => e,
                 };
                 let error_message = format!("{:?}", err);
-                if !error_message.contains(&message) {
+                if !is_matching_assert_invalid_error_message(&message, &error_message) {
                     bail!(
                         "assert_invalid: expected \"{}\", got \"{}\"",
                         message,
@@ -343,6 +343,16 @@ impl WastContext {
     }
 }
 
+fn is_matching_assert_invalid_error_message(expected: &str, actual: &str) -> bool {
+    actual.contains(expected)
+        // Waiting on https://github.com/WebAssembly/bulk-memory-operations/pull/137
+        // to propagate to WebAssembly/testsuite.
+        || (expected.contains("unknown table") && actual.contains("unknown elem"))
+        // `elem.wast` and `proposals/bulk-memory-operations/elem.wast` disagree
+        // on the expected error message for the same error.
+        || (expected.contains("out of bounds") && actual.contains("does not fit"))
+}
+
 fn extract_lane_as_i8(bytes: u128, lane: usize) -> i8 {
     (bytes >> (lane * 8)) as i8
 }

tests/instantiate.rs

@@ -24,7 +24,17 @@ fn test_environ_translate() {
     let cache_config = CacheConfig::new_cache_disabled();
     let mut compiler = Compiler::new(isa, CompilationStrategy::Auto, cache_config);
     unsafe {
-        let instance = instantiate(&mut compiler, &data, &mut resolver, false, None);
+        let instance = instantiate(
+            &mut compiler,
+            &data,
+            &mut resolver,
+            // Bulk memory.
+            false,
+            // Debug info.
+            false,
+            // Profiler.
+            None,
+        );
         assert!(instance.is_ok());
     }
 }

tests/misc_testsuite/bulk-memory-operations/elem-ref-null.wast

@@ -0,0 +1,2 @@
+(module
+  (elem funcref (ref.null)))

tests/misc_testsuite/bulk-memory-operations/elem_drop.wast

@@ -0,0 +1,7 @@
+(module
+  (table 1 1 funcref)
+  (elem (i32.const 0) funcref (ref.func 0))
+  (func (export "elem.drop non-passive element")
+    (elem.drop 0)))
+
+(invoke "elem.drop non-passive element")

tests/misc_testsuite/bulk-memory-operations/imported-memory-copy.wast

@@ -0,0 +1,129 @@
+(module $foreign
+  (memory (export "mem") 1 1)
+  (data 0 (i32.const 1000) "hello")
+  (data 0 (i32.const 2000) "olleh"))
+
+(register "foreign" $foreign)
+
+(module
+  (memory (import "foreign" "mem") 1 1)
+
+  (func $is_char (param i32 i32) (result i32)
+    local.get 0
+    i32.load8_u
+    local.get 1
+    i32.eq)
+
+  (func (export "is hello?") (param i32) (result i32)
+    local.get 0
+    i32.const 104 ;; 'h'
+    call $is_char
+
+    local.get 0
+    i32.const 1
+    i32.add
+    i32.const 101 ;; 'e'
+    call $is_char
+
+    local.get 0
+    i32.const 2
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 3
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 4
+    i32.add
+    i32.const 111 ;; 'o'
+    call $is_char
+
+    i32.and
+    i32.and
+    i32.and
+    i32.and
+  )
+
+  (func (export "is olleh?") (param i32) (result i32)
+    local.get 0
+    i32.const 111 ;; 'o'
+    call $is_char
+
+    local.get 0
+    i32.const 1
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 2
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 3
+    i32.add
+    i32.const 101 ;; 'e'
+    call $is_char
+
+    local.get 0
+    i32.const 4
+    i32.add
+    i32.const 104 ;; 'h'
+    call $is_char
+
+    i32.and
+    i32.and
+    i32.and
+    i32.and
+  )
+
+  (func (export "memory.copy") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    memory.copy))
+
+;; Our memory has our initial data in the right places.
+(assert_return
+  (invoke "is hello?" (i32.const 1000))
+  (i32.const 1))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))
+
+;; Non-overlapping memory copy with dst < src.
+(invoke "memory.copy" (i32.const 500) (i32.const 1000) (i32.const 5))
+(assert_return
+  (invoke "is hello?" (i32.const 500))
+  (i32.const 1))
+
+;; Non-overlapping memory copy with dst > src.
+(invoke "memory.copy" (i32.const 1500) (i32.const 1000) (i32.const 5))
+(assert_return
+  (invoke "is hello?" (i32.const 1500))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst < src.
+(invoke "memory.copy" (i32.const 1998) (i32.const 2000) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 1998))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst > src.
+(invoke "memory.copy" (i32.const 2000) (i32.const 1998) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst = src.
+(invoke "memory.copy" (i32.const 2000) (i32.const 2000) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))

tests/misc_testsuite/bulk-memory-operations/memory-copy.wast

@@ -0,0 +1,124 @@
+(module
+  (memory 1 1)
+  (data 0 (i32.const 1000) "hello")
+  (data 0 (i32.const 2000) "olleh")
+
+  (func $is_char (param i32 i32) (result i32)
+    local.get 0
+    i32.load8_u
+    local.get 1
+    i32.eq)
+
+  (func (export "is hello?") (param i32) (result i32)
+    local.get 0
+    i32.const 104 ;; 'h'
+    call $is_char
+
+    local.get 0
+    i32.const 1
+    i32.add
+    i32.const 101 ;; 'e'
+    call $is_char
+
+    local.get 0
+    i32.const 2
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 3
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 4
+    i32.add
+    i32.const 111 ;; 'o'
+    call $is_char
+
+    i32.and
+    i32.and
+    i32.and
+    i32.and
+  )
+
+  (func (export "is olleh?") (param i32) (result i32)
+    local.get 0
+    i32.const 111 ;; 'o'
+    call $is_char
+
+    local.get 0
+    i32.const 1
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 2
+    i32.add
+    i32.const 108 ;; 'l'
+    call $is_char
+
+    local.get 0
+    i32.const 3
+    i32.add
+    i32.const 101 ;; 'e'
+    call $is_char
+
+    local.get 0
+    i32.const 4
+    i32.add
+    i32.const 104 ;; 'h'
+    call $is_char
+
+    i32.and
+    i32.and
+    i32.and
+    i32.and
+  )
+
+  (func (export "memory.copy") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    memory.copy))
+
+;; Our memory has our initial data in the right places.
+(assert_return
+  (invoke "is hello?" (i32.const 1000))
+  (i32.const 1))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))
+
+;; Non-overlapping memory copy with dst < src.
+(invoke "memory.copy" (i32.const 500) (i32.const 1000) (i32.const 5))
+(assert_return
+  (invoke "is hello?" (i32.const 500))
+  (i32.const 1))
+
+;; Non-overlapping memory copy with dst > src.
+(invoke "memory.copy" (i32.const 1500) (i32.const 1000) (i32.const 5))
+(assert_return
+  (invoke "is hello?" (i32.const 1500))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst < src.
+(invoke "memory.copy" (i32.const 1998) (i32.const 2000) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 1998))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst > src.
+(invoke "memory.copy" (i32.const 2000) (i32.const 1998) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))
+
+;; Overlapping memory copy with dst = src.
+(invoke "memory.copy" (i32.const 2000) (i32.const 2000) (i32.const 5))
+(assert_return
+  (invoke "is olleh?" (i32.const 2000))
+  (i32.const 1))

tests/misc_testsuite/bulk-memory-operations/partial-init-memory-segment.wast

@@ -0,0 +1,35 @@
+(module $m
+  (memory (export "mem") 1)
+
+  (func (export "load") (param i32) (result i32)
+    local.get 0
+    i32.load8_u))
+
+(register "m" $m)
+
+(assert_trap
+  (module
+    (memory (import "m" "mem") 1)
+
+    ;; This is in bounds, and should get written to the memory.
+    (data (i32.const 0) "abc")
+
+    ;; Partially out of bounds. None of these bytes should get written, and
+    ;; instantiation should trap.
+    (data (i32.const 65530) "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz")
+  )
+  "out of bounds"
+)
+
+;; The first data segment got written.
+(assert_return (invoke $m "load" (i32.const 0)) (i32.const 97))
+(assert_return (invoke $m "load" (i32.const 1)) (i32.const 98))
+(assert_return (invoke $m "load" (i32.const 2)) (i32.const 99))
+
+;; The second did not get partially written.
+(assert_return (invoke $m "load" (i32.const 65530)) (i32.const 0))
+(assert_return (invoke $m "load" (i32.const 65531)) (i32.const 0))
+(assert_return (invoke $m "load" (i32.const 65532)) (i32.const 0))
+(assert_return (invoke $m "load" (i32.const 65533)) (i32.const 0))
+(assert_return (invoke $m "load" (i32.const 65534)) (i32.const 0))
+(assert_return (invoke $m "load" (i32.const 65535)) (i32.const 0))

tests/misc_testsuite/bulk-memory-operations/partial-init-table-segment.wast

@@ -0,0 +1,35 @@
+(module $m
+  (table (export "table") funcref (elem $zero $zero $zero $zero $zero $zero $zero $zero $zero $zero))
+
+  (func $zero (result i32)
+    (i32.const 0))
+
+  (func (export "indirect-call") (param i32) (result i32)
+    local.get 0
+    call_indirect (result i32)))
+
+(register "m" $m)
+
+(assert_trap
+  (module
+    (table (import "m" "table") 10 funcref)
+
+    (func $one (result i32)
+      (i32.const 1))
+
+    ;; An in-bounds segment that should get initialized in the table.
+    (elem (i32.const 7) $one)
+
+    ;; Part of this segment is out of bounds, so none of its elements should be
+    ;; initialized into the table, and it should trap.
+    (elem (i32.const 9) $one $one $one)
+  )
+  "out of bounds"
+)
+
+;; The first `$one` segment *was* initialized OK.
+(assert_return (invoke "indirect-call" (i32.const 7)) (i32.const 1))
+
+;; The second `$one` segment is partially out of bounds, and therefore none of
+;; its elements were written into the table.
+(assert_return (invoke "indirect-call" (i32.const 9)) (i32.const 0))

tests/misc_testsuite/bulk-memory-operations/table_copy.wast

@@ -0,0 +1,63 @@
+(module
+  (func $f (param i32 i32 i32) (result i32) (local.get 0))
+  (func $g (param i32 i32 i32) (result i32) (local.get 1))
+  (func $h (param i32 i32 i32) (result i32) (local.get 2))
+
+  ;; Indices:          0  1  2  3  4  5  6  7  8
+  (table funcref (elem $f $g $h $f $g $h $f $g $h))
+  ;; After table.copy: $g $h $f
+
+  (func (export "copy") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy)
+
+  (func (export "call") (param i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    call_indirect (param i32 i32 i32) (result i32))
+)
+
+;; Call $f at 0
+(assert_return
+  (invoke "call" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0))
+  (i32.const 1))
+
+;; Call $g at 1
+(assert_return
+  (invoke "call" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 1))
+  (i32.const 1))
+
+;; Call $h at 2
+(assert_return
+  (invoke "call" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 2))
+  (i32.const 1))
+
+;; Do a `table.copy` to rearrange the elements. Copy from 4..7 to 0..3.
+(invoke "copy" (i32.const 0) (i32.const 4) (i32.const 3))
+
+;; Call $g at 0
+(assert_return
+  (invoke "call" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0))
+  (i32.const 1))
+
+;; Call $h at 1
+(assert_return
+  (invoke "call" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 1))
+  (i32.const 1))
+
+;; Call $f at 2
+(assert_return
+  (invoke "call" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 2))
+  (i32.const 1))
+
+;; Copying up to the end does not trap.
+(invoke "copy" (i32.const 7) (i32.const 0) (i32.const 2))
+
+;; Copying past the end traps.
+(assert_trap
+  (invoke "copy" (i32.const 7) (i32.const 0) (i32.const 3))
+  "undefined element")

tests/misc_testsuite/reference-types/table_copy_on_imported_tables.wast

@@ -0,0 +1,165 @@
+(module $m
+  (func $f (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 0))
+  (func $g (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 1))
+  (func $h (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 2))
+
+  (table $t (export "t") funcref (elem $f $g $h $f $g $h)))
+
+(register "m" $m)
+
+(module $n
+  (table $t (import "m" "t") 6 funcref)
+
+  (func $i (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 3))
+  (func $j (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 4))
+  (func $k (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 5))
+
+  (table $u (export "u") funcref (elem $i $j $k $i $j $k))
+
+  (func (export "copy_into_t_from_u") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy $t $u)
+
+  (func (export "copy_into_u_from_t") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy $u $t)
+
+  (func (export "call_t") (param i32 i32 i32 i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    local.get 4
+    local.get 5
+    local.get 6
+    call_indirect $t (param i32 i32 i32 i32 i32 i32) (result i32))
+
+  (func (export "call_u") (param i32 i32 i32 i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    local.get 4
+    local.get 5
+    local.get 6
+    call_indirect $u (param i32 i32 i32 i32 i32 i32) (result i32)))
+
+;; Everything has what we initially expect.
+(assert_return
+  (invoke "call_t" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 0))
+  (i32.const 1))
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 1))
+  (i32.const 1))
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 2))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0)
+    (i32.const 0))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0)
+    (i32.const 1))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1)
+    (i32.const 2))
+  (i32.const 1))
+
+;; Now test copying between a local and an imported table.
+
+;; Copy $i $j $k into $t at 3..6 from $u at 0..3.
+(invoke "copy_into_t_from_u" (i32.const 3) (i32.const 0) (i32.const 3))
+
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0)
+    (i32.const 3))
+  (i32.const 1))
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0)
+    (i32.const 4))
+  (i32.const 1))
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1)
+    (i32.const 5))
+  (i32.const 1))
+
+;; Copy $f $g $h into $u at 0..3 from $t at 0..3.
+(invoke "copy_into_u_from_t" (i32.const 0) (i32.const 0) (i32.const 3))
+
+(assert_return
+  (invoke "call_u" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 0))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 1))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 2))
+  (i32.const 1))
+
+(register "n" $n)
+
+(module $o
+  (table $t (import "m" "t") 6 funcref)
+  (table $u (import "n" "u") 6 funcref)
+
+  (func (export "copy_into_t_from_u_2") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy $t $u)
+
+  (func (export "copy_into_u_from_t_2") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy $u $t)
+
+  (func (export "call_t_2") (param i32 i32 i32 i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    local.get 4
+    local.get 5
+    local.get 6
+    call_indirect $t (param i32 i32 i32 i32 i32 i32) (result i32))
+
+  (func (export "call_u_2") (param i32 i32 i32 i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    local.get 4
+    local.get 5
+    local.get 6
+    call_indirect $u (param i32 i32 i32 i32 i32 i32) (result i32)))
+
+;; Now test copying between two imported tables.
+
+;; Copy $i into $t at 0 from $u at 3.
+(invoke "copy_into_t_from_u_2" (i32.const 0) (i32.const 3) (i32.const 1))
+
+(assert_return
+  (invoke "call_t_2" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0)
+    (i32.const 0))
+  (i32.const 1))
+
+;; Copy $g into $u at 4 from $t at 1.
+(invoke "copy_into_u_from_t_2" (i32.const 4) (i32.const 1) (i32.const 1))
+
+(assert_return
+  (invoke "call_u_2" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 4))
+  (i32.const 1))

tests/wast_testsuites.rs

@@ -12,6 +12,8 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> {
 
     let simd = wast.iter().any(|s| s == "simd");
 
+    let bulk_mem = wast.iter().any(|s| s == "bulk-memory-operations");
+
     // Some simd tests assume support for multiple tables, which are introduced
     // by reference types.
     let reftypes = simd || wast.iter().any(|s| s == "reference-types");
@@ -20,6 +22,7 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> {
 
     let mut cfg = Config::new();
     cfg.wasm_simd(simd)
+        .wasm_bulk_memory(bulk_mem)
         .wasm_reference_types(reftypes)
         .wasm_multi_value(multi_val)
         .strategy(strategy)?