Add support for table.copy

This adds support for the `table.copy` instruction from the bulk memory
proposal. It also supports multiple tables, which were introduced by the
reference types proposal.

Part of #928

build.rs

@@ -181,6 +181,8 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
             ("simd", "simd_splat") => return true, // FIXME Unsupported feature: proposed SIMD operator I8x16ShrS
 
             ("reference_types", _) => return true,
+
+            ("bulk_memory_operations", "table_copy") => return false,
             ("bulk_memory_operations", _) => return true,
 
             _ => {}

crates/api/src/externals.rs

@@ -5,8 +5,8 @@ use crate::{ExternType, GlobalType, MemoryType, TableType, ValType};
 use crate::{Func, Store};
 use anyhow::{anyhow, bail, Result};
 use std::slice;
-use wasmtime_environ::wasm;
+use wasmtime_environ::{ir, wasm};
-use wasmtime_runtime::InstanceHandle;
+use wasmtime_runtime::{self as runtime, InstanceHandle};
 
 // Externals
 
@@ -407,6 +407,41 @@ impl Table {
         }
     }
 
+    /// Copy `len` elements from `src_table[src_index..]` into
+    /// `dst_table[dst_index..]`.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the range is out of bounds of either the source or
+    /// destination tables.
+    pub fn copy(
+        dst_table: &Table,
+        dst_index: u32,
+        src_table: &Table,
+        src_index: u32,
+        len: u32,
+    ) -> Result<()> {
+        // NB: We must use the `dst_table`'s `wasmtime_handle` for the
+        // `dst_table_index` and vice versa for `src_table` since each table can
+        // come from different modules.
+
+        let dst_table_index = dst_table.wasmtime_table_index();
+        let dst_table = dst_table.wasmtime_handle.get_defined_table(dst_table_index);
+
+        let src_table_index = src_table.wasmtime_table_index();
+        let src_table = src_table.wasmtime_handle.get_defined_table(src_table_index);
+
+        runtime::Table::copy(
+            dst_table,
+            src_table,
+            dst_index,
+            src_index,
+            len,
+            ir::SourceLoc::default(),
+        )?;
+        Ok(())
+    }
+
     pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::Export {
         &self.wasmtime_export
     }

crates/environ/src/func_environ.rs

@@ -23,6 +23,7 @@ pub fn get_func_name(func_index: FuncIndex) -> ir::ExternalName {
 }
 
 /// An index type for builtin functions.
+#[derive(Copy, Clone, Debug)]
 pub struct BuiltinFunctionIndex(u32);
 
 impl BuiltinFunctionIndex {
@@ -42,9 +43,28 @@ impl BuiltinFunctionIndex {
     pub const fn get_imported_memory32_size_index() -> Self {
         Self(3)
     }
+    /// Returns an index for wasm's `table.copy` when both tables are locally
+    /// defined.
+    pub const fn get_table_copy_defined_defined_index() -> Self {
+        Self(4)
+    }
+    /// Returns an index for wasm's `table.copy` when the destination table is
+    /// locally defined and the source table is imported.
+    pub const fn get_table_copy_defined_imported_index() -> Self {
+        Self(5)
+    }
+    /// Returns an index for wasm's `table.copy` when the destination table is
+    /// imported and the source table is locally defined.
+    pub const fn get_table_copy_imported_defined_index() -> Self {
+        Self(6)
+    }
+    /// Returns an index for wasm's `table.copy` when both tables are imported.
+    pub const fn get_table_copy_imported_imported_index() -> Self {
+        Self(7)
+    }
     /// Returns the total number of builtin functions.
     pub const fn builtin_functions_total_number() -> u32 {
-        4
+        8
     }
 
     /// Return the index as an u32 number.
@@ -72,6 +92,10 @@ pub struct FuncEnvironment<'module_environment> {
     /// for locally-defined memories.
     memory_grow_sig: Option<ir::SigRef>,
 
+    /// The external function signature for implementing wasm's `table.copy`
+    /// (it's the same for both local and imported tables).
+    table_copy_sig: Option<ir::SigRef>,
+
     /// Offsets to struct fields accessed by JIT code.
     offsets: VMOffsets,
 }
@@ -87,6 +111,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             vmctx: None,
             memory32_size_sig: None,
             memory_grow_sig: None,
+            table_copy_sig: None,
             offsets: VMOffsets::new(target_config.pointer_bytes(), module),
         }
     }
@@ -178,6 +203,97 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         }
     }
 
+    // NB: All `table_copy` libcall variants have the same signature.
+    fn get_table_copy_sig(&mut self, func: &mut Function) -> ir::SigRef {
+        let sig = self.table_copy_sig.unwrap_or_else(|| {
+            func.import_signature(Signature {
+                params: vec![
+                    AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext),
+                    // Destination table index.
+                    AbiParam::new(I32),
+                    // Source table index.
+                    AbiParam::new(I32),
+                    // Index within destination table.
+                    AbiParam::new(I32),
+                    // Index within source table.
+                    AbiParam::new(I32),
+                    // Number of elements to copy.
+                    AbiParam::new(I32),
+                    // Source location.
+                    AbiParam::new(I32),
+                ],
+                returns: vec![],
+                call_conv: self.target_config.default_call_conv,
+            })
+        });
+        self.table_copy_sig = Some(sig);
+        sig
+    }
+
+    fn get_table_copy_func(
+        &mut self,
+        func: &mut Function,
+        dst_table_index: TableIndex,
+        src_table_index: TableIndex,
+    ) -> (ir::SigRef, usize, usize, BuiltinFunctionIndex) {
+        let sig = self.get_table_copy_sig(func);
+        match (
+            self.module.is_imported_table(dst_table_index),
+            self.module.is_imported_table(src_table_index),
+        ) {
+            (false, false) => {
+                let dst_table_index = self
+                    .module
+                    .defined_table_index(dst_table_index)
+                    .unwrap()
+                    .index();
+                let src_table_index = self
+                    .module
+                    .defined_table_index(src_table_index)
+                    .unwrap()
+                    .index();
+                (
+                    sig,
+                    dst_table_index,
+                    src_table_index,
+                    BuiltinFunctionIndex::get_table_copy_defined_defined_index(),
+                )
+            }
+            (false, true) => {
+                let dst_table_index = self
+                    .module
+                    .defined_table_index(dst_table_index)
+                    .unwrap()
+                    .index();
+                (
+                    sig,
+                    dst_table_index,
+                    src_table_index.as_u32() as usize,
+                    BuiltinFunctionIndex::get_table_copy_defined_imported_index(),
+                )
+            }
+            (true, false) => {
+                let src_table_index = self
+                    .module
+                    .defined_table_index(src_table_index)
+                    .unwrap()
+                    .index();
+                (
+                    sig,
+                    dst_table_index.as_u32() as usize,
+                    src_table_index,
+                    BuiltinFunctionIndex::get_table_copy_imported_defined_index(),
+                )
+            }
+            (true, true) => (
+                sig,
+                dst_table_index.as_u32() as usize,
+                src_table_index.as_u32() as usize,
+                BuiltinFunctionIndex::get_table_copy_imported_imported_index(),
+            ),
+        }
+    }
+
     /// Translates load of builtin function and returns a pair of values `vmctx`
     /// and address of the loaded function.
     fn translate_load_builtin_function_address(
@@ -782,7 +898,9 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         _src: ir::Value,
         _len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `memory.copy`".to_string(),
+        ))
     }
 
     fn translate_memory_fill(
@@ -794,7 +912,9 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         _val: ir::Value,
         _len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `memory.fill`".to_string(),
+        ))
     }
 
     fn translate_memory_init(
@@ -807,11 +927,15 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         _src: ir::Value,
         _len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `memory.copy`".to_string(),
+        ))
     }
 
     fn translate_data_drop(&mut self, _pos: FuncCursor, _seg_index: u32) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `data.drop`".to_string(),
+        ))
     }
 
     fn translate_table_size(
@@ -820,21 +944,50 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         _index: TableIndex,
         _table: ir::Table,
     ) -> WasmResult<ir::Value> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `table.size`".to_string(),
+        ))
     }
 
     fn translate_table_copy(
         &mut self,
-        _pos: FuncCursor,
+        mut pos: FuncCursor,
-        _dst_table_index: TableIndex,
+        dst_table_index: TableIndex,
         _dst_table: ir::Table,
-        _src_table_index: TableIndex,
+        src_table_index: TableIndex,
         _src_table: ir::Table,
-        _dst: ir::Value,
+        dst: ir::Value,
-        _src: ir::Value,
+        src: ir::Value,
-        _len: ir::Value,
+        len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        use cranelift_codegen::cursor::Cursor;
+
+        let (func_sig, dst_table_index_arg, src_table_index_arg, func_idx) =
+            self.get_table_copy_func(&mut pos.func, dst_table_index, src_table_index);
+
+        let dst_table_index_arg = pos.ins().iconst(I32, dst_table_index_arg as i64);
+        let src_table_index_arg = pos.ins().iconst(I32, src_table_index_arg as i64);
+
+        let src_loc = pos.srcloc();
+        let src_loc_arg = pos.ins().iconst(I32, src_loc.bits() as i64);
+
+        let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx);
+
+        pos.ins().call_indirect(
+            func_sig,
+            func_addr,
+            &[
+                vmctx,
+                dst_table_index_arg,
+                src_table_index_arg,
+                dst,
+                src,
+                len,
+                src_loc_arg,
+            ],
+        );
+
+        Ok(())
     }
 
     fn translate_table_init(
@@ -847,10 +1000,14 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         _src: ir::Value,
         _len: ir::Value,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `table.init`".to_string(),
+        ))
     }
 
     fn translate_elem_drop(&mut self, _pos: FuncCursor, _seg_index: u32) -> WasmResult<()> {
-        Err(WasmError::Unsupported("bulk memory".to_string()))
+        Err(WasmError::Unsupported(
+            "bulk memory: `elem.drop`".to_string(),
+        ))
     }
 }

crates/environ/src/module.rs

@@ -6,7 +6,7 @@ use cranelift_codegen::ir;
 use cranelift_entity::{EntityRef, PrimaryMap};
 use cranelift_wasm::{
     DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, Global,
-    GlobalIndex, Memory, MemoryIndex, SignatureIndex, Table, TableIndex,
+    GlobalIndex, Memory, MemoryIndex, PassiveElemIndex, SignatureIndex, Table, TableIndex,
 };
 use indexmap::IndexMap;
 use more_asserts::assert_ge;
@@ -165,6 +165,9 @@ pub struct Module {
     /// WebAssembly table initializers.
     pub table_elements: Vec<TableElements>,
 
+    /// WebAssembly passive elements.
+    pub passive_elements: HashMap<PassiveElemIndex, Box<[FuncIndex]>>,
+
     /// WebAssembly table initializers.
     pub func_names: HashMap<FuncIndex, String>,
 }
@@ -219,6 +222,7 @@ impl Module {
             exports: IndexMap::new(),
             start_func: None,
             table_elements: Vec::new(),
+            passive_elements: HashMap::new(),
             func_names: HashMap::new(),
             local: ModuleLocal {
                 num_imported_funcs: 0,

crates/environ/src/module_environ.rs

@@ -336,12 +336,20 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
 
     fn declare_passive_element(
         &mut self,
-        _: PassiveElemIndex,
+        elem_index: PassiveElemIndex,
-        _: Box<[FuncIndex]>,
+        segments: Box<[FuncIndex]>,
     ) -> WasmResult<()> {
-        Err(WasmError::Unsupported(
+        let old = self
-            "bulk memory: passive element segment".into(),
+            .result
-        ))
+            .module
+            .passive_elements
+            .insert(elem_index, segments);
+        debug_assert!(
+            old.is_none(),
+            "should never get duplicate element indices, that would be a bug in `cranelift_wasm`'s \
+             translation"
+        );
+        Ok(())
     }
 
     fn define_function_body(
@@ -382,9 +390,18 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
         Ok(())
     }
 
-    fn declare_passive_data(&mut self, _: PassiveDataIndex, _: &'data [u8]) -> WasmResult<()> {
+    fn reserve_passive_data(&mut self, count: u32) -> WasmResult<()> {
+        self.result.module.passive_elements.reserve(count as usize);
+        Ok(())
+    }
+
+    fn declare_passive_data(
+        &mut self,
+        _data_index: PassiveDataIndex,
+        _data: &'data [u8],
+    ) -> WasmResult<()> {
         Err(WasmError::Unsupported(
-            "bulk memory: passive data segment".into(),
+            "bulk memory: passive data".to_string(),
         ))
     }
 

crates/runtime/src/instance.rs

@@ -526,6 +526,30 @@ impl Instance {
         let align = mem::align_of_val(self);
         Layout::from_size_align(size, align).unwrap()
     }
+
+    /// Get a table by index regardless of whether it is locally-defined or an
+    /// imported, foreign table.
+    pub(crate) fn get_table(&self, table_index: TableIndex) -> &Table {
+        if let Some(defined_table_index) = self.module.local.defined_table_index(table_index) {
+            &self.tables[defined_table_index]
+        } else {
+            self.get_foreign_table(table_index)
+        }
+    }
+
+    /// Get a locally-defined table.
+    pub(crate) fn get_defined_table(&self, index: DefinedTableIndex) -> &Table {
+        &self.tables[index]
+    }
+
+    /// Get an imported, foreign table.
+    pub(crate) fn get_foreign_table(&self, index: TableIndex) -> &Table {
+        let import = self.imported_table(index);
+        let foreign_instance = unsafe { (&mut *(import).vmctx).instance() };
+        let foreign_table = unsafe { &mut *(import).from };
+        let foreign_index = foreign_instance.table_index(foreign_table);
+        &foreign_instance.tables[foreign_index]
+    }
 }
 
 /// A handle holding an `Instance` of a WebAssembly module.
@@ -778,6 +802,11 @@ impl InstanceHandle {
         self.instance().table_set(table_index, index, val)
     }
 
+    /// Get a table defined locally within this module.
+    pub fn get_defined_table(&self, index: DefinedTableIndex) -> &Table {
+        self.instance().get_defined_table(index)
+    }
+
     /// Return a reference to the contained `Instance`.
     pub(crate) fn instance(&self) -> &Instance {
         unsafe { &*(self.instance as *const Instance) }
@@ -813,7 +842,7 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError
     let module = Arc::clone(&instance.module);
     for init in &module.table_elements {
         let start = get_table_init_start(init, instance);
-        let table = get_table(init, instance);
+        let table = instance.get_table(init.table_index);
 
         let size = usize::try_from(table.size()).unwrap();
         if size < start + init.elements.len() {
@@ -913,26 +942,13 @@ fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize {
     start
 }
 
-/// Return a byte-slice view of a table's data.
-fn get_table<'instance>(init: &TableElements, instance: &'instance Instance) -> &'instance Table {
-    if let Some(defined_table_index) = instance.module.local.defined_table_index(init.table_index) {
-        &instance.tables[defined_table_index]
-    } else {
-        let import = instance.imported_table(init.table_index);
-        let foreign_instance = unsafe { (&mut *(import).vmctx).instance() };
-        let foreign_table = unsafe { &mut *(import).from };
-        let foreign_index = foreign_instance.table_index(foreign_table);
-        &foreign_instance.tables[foreign_index]
-    }
-}
-
 /// Initialize the table memory from the provided initializers.
 fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> {
     let vmctx = instance.vmctx_ptr();
     let module = Arc::clone(&instance.module);
     for init in &module.table_elements {
         let start = get_table_init_start(init, instance);
-        let table = get_table(init, instance);
+        let table = instance.get_table(init.table_index);
 
         for (i, func_idx) in init.elements.iter().enumerate() {
             let callee_sig = instance.module.local.functions[*func_idx];

crates/runtime/src/lib.rs

@@ -42,9 +42,12 @@ pub use crate::instance::{InstanceHandle, InstantiationError, LinkError};
 pub use crate::jit_int::GdbJitImageRegistration;
 pub use crate::mmap::Mmap;
 pub use crate::sig_registry::SignatureRegistry;
+pub use crate::table::Table;
 pub use crate::trap_registry::{TrapDescription, TrapRegistration, TrapRegistry};
 pub use crate::traphandlers::resume_panic;
-pub use crate::traphandlers::{catch_traps, raise_user_trap, wasmtime_call_trampoline, Trap};
+pub use crate::traphandlers::{
+    catch_traps, raise_lib_trap, raise_user_trap, wasmtime_call_trampoline, Trap,
+};
 pub use crate::vmcontext::{
     VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, VMGlobalDefinition,
     VMGlobalImport, VMInvokeArgument, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex,

crates/runtime/src/libcalls.rs

@@ -2,8 +2,11 @@
 //! inline rather than calling them, particularly when CPUs have special
 //! instructions which compute them directly.
 
+use crate::table::Table;
+use crate::traphandlers::raise_lib_trap;
 use crate::vmcontext::VMContext;
-use wasmtime_environ::wasm::{DefinedMemoryIndex, MemoryIndex};
+use wasmtime_environ::ir;
+use wasmtime_environ::wasm::{DefinedMemoryIndex, DefinedTableIndex, MemoryIndex, TableIndex};
 
 /// Implementation of f32.ceil
 pub extern "C" fn wasmtime_f32_ceil(x: f32) -> f32 {
@@ -137,3 +140,93 @@ pub unsafe extern "C" fn wasmtime_imported_memory32_size(
 
     instance.imported_memory_size(memory_index)
 }
+
+/// Implementation of `table.copy` when both tables are locally defined.
+#[no_mangle]
+pub unsafe extern "C" fn wasmtime_table_copy_defined_defined(
+    vmctx: *mut VMContext,
+    dst_table_index: u32,
+    src_table_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let dst_table_index = DefinedTableIndex::from_u32(dst_table_index);
+    let src_table_index = DefinedTableIndex::from_u32(src_table_index);
+    let source_loc = ir::SourceLoc::new(source_loc);
+    let instance = (&mut *vmctx).instance();
+    let dst_table = instance.get_defined_table(dst_table_index);
+    let src_table = instance.get_defined_table(src_table_index);
+    if let Err(trap) = Table::copy(dst_table, src_table, dst, src, len, source_loc) {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `table.copy` when the destination table is locally defined
+/// and the source table is imported.
+#[no_mangle]
+pub unsafe extern "C" fn wasmtime_table_copy_defined_imported(
+    vmctx: *mut VMContext,
+    dst_table_index: u32,
+    src_table_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let dst_table_index = DefinedTableIndex::from_u32(dst_table_index);
+    let src_table_index = TableIndex::from_u32(src_table_index);
+    let source_loc = ir::SourceLoc::new(source_loc);
+    let instance = (&mut *vmctx).instance();
+    let dst_table = instance.get_defined_table(dst_table_index);
+    let src_table = instance.get_foreign_table(src_table_index);
+    if let Err(trap) = Table::copy(dst_table, src_table, dst, src, len, source_loc) {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `table.copy` when the destination table is imported
+/// and the source table is locally defined.
+#[no_mangle]
+pub unsafe extern "C" fn wasmtime_table_copy_imported_defined(
+    vmctx: *mut VMContext,
+    dst_table_index: u32,
+    src_table_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let dst_table_index = TableIndex::from_u32(dst_table_index);
+    let src_table_index = DefinedTableIndex::from_u32(src_table_index);
+    let source_loc = ir::SourceLoc::new(source_loc);
+    let instance = (&mut *vmctx).instance();
+    let dst_table = instance.get_foreign_table(dst_table_index);
+    let src_table = instance.get_defined_table(src_table_index);
+    if let Err(trap) = Table::copy(dst_table, src_table, dst, src, len, source_loc) {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `table.copy` when both tables are imported.
+#[no_mangle]
+pub unsafe extern "C" fn wasmtime_table_copy_imported_imported(
+    vmctx: *mut VMContext,
+    dst_table_index: u32,
+    src_table_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let dst_table_index = TableIndex::from_u32(dst_table_index);
+    let src_table_index = TableIndex::from_u32(src_table_index);
+    let source_loc = ir::SourceLoc::new(source_loc);
+    let instance = (&mut *vmctx).instance();
+    let dst_table = instance.get_foreign_table(dst_table_index);
+    let src_table = instance.get_foreign_table(src_table_index);
+    if let Err(trap) = Table::copy(dst_table, src_table, dst, src, len, source_loc) {
+        raise_lib_trap(trap);
+    }
+}

crates/runtime/src/table.rs

@@ -3,10 +3,12 @@
 //! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories.
 
 use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition};
+use crate::{Trap, TrapDescription};
+use backtrace::Backtrace;
 use std::cell::RefCell;
 use std::convert::{TryFrom, TryInto};
 use wasmtime_environ::wasm::TableElementType;
-use wasmtime_environ::{TablePlan, TableStyle};
+use wasmtime_environ::{ir, TablePlan, TableStyle};
 
 /// A table instance.
 #[derive(Debug)]
@@ -87,6 +89,56 @@ impl Table {
         }
     }
 
+    /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the range is out of bounds of either the source or
+    /// destination tables.
+    pub fn copy(
+        dst_table: &Self,
+        src_table: &Self,
+        dst_index: u32,
+        src_index: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-copy
+
+        if src_index
+            .checked_add(len)
+            .map_or(true, |n| n > src_table.size())
+            || dst_index
+                .checked_add(len)
+                .map_or(true, |m| m > dst_table.size())
+        {
+            return Err(Trap::Wasm {
+                desc: TrapDescription {
+                    source_loc,
+                    trap_code: ir::TrapCode::TableOutOfBounds,
+                },
+                backtrace: Backtrace::new(),
+            });
+        }
+
+        let srcs = src_index..src_index + len;
+        let dsts = dst_index..dst_index + len;
+
+        // Note on the unwraps: the bounds check above means that these will
+        // never panic.
+        if dst_index <= src_index {
+            for (s, d) in (srcs).zip(dsts) {
+                dst_table.set(d, src_table.get(s).unwrap()).unwrap();
+            }
+        } else {
+            for (s, d) in srcs.rev().zip(dsts.rev()) {
+                dst_table.set(d, src_table.get(s).unwrap()).unwrap();
+            }
+        }
+
+        Ok(())
+    }
+
     /// Return a `VMTableDefinition` for exposing the table to compiled wasm code.
     pub fn vmtable(&self) -> VMTableDefinition {
         let mut vec = self.vec.borrow_mut();

crates/runtime/src/trap_registry.rs

@@ -84,7 +84,7 @@ fn trap_code_to_expected_string(trap_code: ir::TrapCode) -> String {
     match trap_code {
         StackOverflow => "call stack exhausted".to_string(),
         HeapOutOfBounds => "out of bounds memory access".to_string(),
-        TableOutOfBounds => "undefined element".to_string(),
+        TableOutOfBounds => "undefined element: out of bounds".to_string(),
         OutOfBounds => "out of bounds".to_string(), // Note: not covered by the test suite
         IndirectCallToNull => "uninitialized element".to_string(),
         BadSignature => "indirect call type mismatch".to_string(),

crates/runtime/src/traphandlers.rs

@@ -73,6 +73,20 @@ pub unsafe fn raise_user_trap(data: Box<dyn Error + Send + Sync>) -> ! {
     tls::with(|info| info.unwrap().unwind_with(UnwindReason::UserTrap(data)))
 }
 
+/// Raises a trap from inside library code immediately.
+///
+/// This function performs as-if a wasm trap was just executed. This trap
+/// payload is then returned from `wasmtime_call` and `wasmtime_call_trampoline`
+/// below.
+///
+/// # Safety
+///
+/// Only safe to call when wasm code is on the stack, aka `wasmtime_call` or
+/// `wasmtime_call_trampoline` must have been previously called.
+pub unsafe fn raise_lib_trap(trap: Trap) -> ! {
+    tls::with(|info| info.unwrap().unwind_with(UnwindReason::LibTrap(trap)))
+}
+
 /// Carries a Rust panic across wasm code and resumes the panic on the other
 /// side.
 ///
@@ -192,6 +206,7 @@ enum UnwindReason {
     None,
     Panic(Box<dyn Any + Send>),
     UserTrap(Box<dyn Error + Send + Sync>),
+    LibTrap(Trap),
     Trap { backtrace: Backtrace, pc: usize },
 }
 
@@ -219,6 +234,7 @@ impl CallThreadState {
                     debug_assert_eq!(ret, 0);
                     Err(Trap::User(data))
                 }
+                UnwindReason::LibTrap(trap) => Err(trap),
                 UnwindReason::Trap { backtrace, pc } => {
                     debug_assert_eq!(ret, 0);
                     let instance = unsafe { InstanceHandle::from_vmctx(self.vmctx) };

crates/runtime/src/vmcontext.rs

@@ -537,6 +537,7 @@ impl VMBuiltinFunctionsArray {
     pub fn initialized() -> Self {
         use crate::libcalls::*;
         let mut ptrs = [0; Self::len()];
+
         ptrs[BuiltinFunctionIndex::get_memory32_grow_index().index() as usize] =
             wasmtime_memory32_grow as usize;
         ptrs[BuiltinFunctionIndex::get_imported_memory32_grow_index().index() as usize] =
@@ -545,6 +546,21 @@ impl VMBuiltinFunctionsArray {
             wasmtime_memory32_size as usize;
         ptrs[BuiltinFunctionIndex::get_imported_memory32_size_index().index() as usize] =
             wasmtime_imported_memory32_size as usize;
+
+        ptrs[BuiltinFunctionIndex::get_table_copy_defined_defined_index().index() as usize] =
+            wasmtime_table_copy_defined_defined as usize;
+
+        ptrs[BuiltinFunctionIndex::get_table_copy_defined_imported_index().index() as usize] =
+            wasmtime_table_copy_defined_imported as usize;
+
+        ptrs[BuiltinFunctionIndex::get_table_copy_imported_defined_index().index() as usize] =
+            wasmtime_table_copy_imported_defined as usize;
+
+        ptrs[BuiltinFunctionIndex::get_table_copy_imported_imported_index().index() as usize] =
+            wasmtime_table_copy_imported_imported as usize;
+
+        debug_assert!(ptrs.iter().cloned().all(|p| p != 0));
+
         Self { ptrs }
     }
 }

tests/misc_testsuite/table_copy.wast

@@ -0,0 +1,63 @@
+(module
+  (func $f (param i32 i32 i32) (result i32) (local.get 0))
+  (func $g (param i32 i32 i32) (result i32) (local.get 1))
+  (func $h (param i32 i32 i32) (result i32) (local.get 2))
+
+  ;; Indices:          0  1  2  3  4  5  6  7  8
+  (table funcref (elem $f $g $h $f $g $h $f $g $h))
+  ;; After table.copy: $g $h $f
+
+  (func (export "copy") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy)
+
+  (func (export "call") (param i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    call_indirect (param i32 i32 i32) (result i32))
+)
+
+;; Call $f at 0
+(assert_return
+  (invoke "call" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0))
+  (i32.const 1))
+
+;; Call $g at 1
+(assert_return
+  (invoke "call" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 1))
+  (i32.const 1))
+
+;; Call $h at 2
+(assert_return
+  (invoke "call" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 2))
+  (i32.const 1))
+
+;; Do a `table.copy` to rearrange the elements. Copy from 4..7 to 0..3.
+(invoke "copy" (i32.const 0) (i32.const 4) (i32.const 3))
+
+;; Call $g at 0
+(assert_return
+  (invoke "call" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0))
+  (i32.const 1))
+
+;; Call $h at 1
+(assert_return
+  (invoke "call" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 1))
+  (i32.const 1))
+
+;; Call $f at 2
+(assert_return
+  (invoke "call" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 2))
+  (i32.const 1))
+
+;; Copying up to the end does not trap.
+(invoke "copy" (i32.const 7) (i32.const 0) (i32.const 2))
+
+;; Copying past the end traps.
+(assert_trap
+  (invoke "copy" (i32.const 7) (i32.const 0) (i32.const 3))
+  "undefined element")

tests/misc_testsuite/table_copy_on_imported_tables.wast

@@ -0,0 +1,165 @@
+(module $m
+  (func $f (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 0))
+  (func $g (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 1))
+  (func $h (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 2))
+
+  (table $t (export "t") funcref (elem $f $g $h $f $g $h)))
+
+(register "m" $m)
+
+(module $n
+  (table $t (import "m" "t") 6 funcref)
+
+  (func $i (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 3))
+  (func $j (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 4))
+  (func $k (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 5))
+
+  (table $u (export "u") funcref (elem $i $j $k $i $j $k))
+
+  (func (export "copy_into_t_from_u") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy $t $u)
+
+  (func (export "copy_into_u_from_t") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy $u $t)
+
+  (func (export "call_t") (param i32 i32 i32 i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    local.get 4
+    local.get 5
+    local.get 6
+    call_indirect $t (param i32 i32 i32 i32 i32 i32) (result i32))
+
+  (func (export "call_u") (param i32 i32 i32 i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    local.get 4
+    local.get 5
+    local.get 6
+    call_indirect $u (param i32 i32 i32 i32 i32 i32) (result i32)))
+
+;; Everything has what we initially expect.
+(assert_return
+  (invoke "call_t" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 0))
+  (i32.const 1))
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 1))
+  (i32.const 1))
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 2))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0)
+    (i32.const 0))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0)
+    (i32.const 1))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1)
+    (i32.const 2))
+  (i32.const 1))
+
+;; Now test copying between a local and an imported table.
+
+;; Copy $i $j $k into $t at 3..6 from $u at 0..3.
+(invoke "copy_into_t_from_u" (i32.const 3) (i32.const 0) (i32.const 3))
+
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0)
+    (i32.const 3))
+  (i32.const 1))
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0)
+    (i32.const 4))
+  (i32.const 1))
+(assert_return
+  (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1)
+    (i32.const 5))
+  (i32.const 1))
+
+;; Copy $f $g $h into $u at 0..3 from $t at 0..3.
+(invoke "copy_into_u_from_t" (i32.const 0) (i32.const 0) (i32.const 3))
+
+(assert_return
+  (invoke "call_u" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 0))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 1))
+  (i32.const 1))
+(assert_return
+  (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 2))
+  (i32.const 1))
+
+(register "n" $n)
+
+(module $o
+  (table $t (import "m" "t") 6 funcref)
+  (table $u (import "n" "u") 6 funcref)
+
+  (func (export "copy_into_t_from_u_2") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy $t $u)
+
+  (func (export "copy_into_u_from_t_2") (param i32 i32 i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    table.copy $u $t)
+
+  (func (export "call_t_2") (param i32 i32 i32 i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    local.get 4
+    local.get 5
+    local.get 6
+    call_indirect $t (param i32 i32 i32 i32 i32 i32) (result i32))
+
+  (func (export "call_u_2") (param i32 i32 i32 i32 i32 i32 i32) (result i32)
+    local.get 0
+    local.get 1
+    local.get 2
+    local.get 3
+    local.get 4
+    local.get 5
+    local.get 6
+    call_indirect $u (param i32 i32 i32 i32 i32 i32) (result i32)))
+
+;; Now test copying between two imported tables.
+
+;; Copy $i into $t at 0 from $u at 3.
+(invoke "copy_into_t_from_u_2" (i32.const 0) (i32.const 3) (i32.const 1))
+
+(assert_return
+  (invoke "call_t_2" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0)
+    (i32.const 0))
+  (i32.const 1))
+
+;; Copy $g into $u at 4 from $t at 1.
+(invoke "copy_into_u_from_t_2" (i32.const 4) (i32.const 1) (i32.const 1))
+
+(assert_return
+  (invoke "call_u_2" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0)
+    (i32.const 4))
+  (i32.const 1))

tests/wast_testsuites.rs

@@ -16,12 +16,25 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> {
     // by reference types.
     let reftypes = simd || wast.iter().any(|s| s == "reference-types");
 
+    // Reference types assumes support for bulk memory.
+    let bulk_mem = reftypes
+        || wast.iter().any(|s| s == "bulk-memory-operations")
+        || wast.iter().any(|s| s == "table_copy.wast")
+        || wast
+            .iter()
+            .any(|s| s == "table_copy_on_imported_tables.wast");
+
+    // And bulk memory also assumes support for reference types (e.g. multiple
+    // tables).
+    let reftypes = reftypes || bulk_mem;
+
     let multi_val = wast.iter().any(|s| s == "multi-value");
 
     let mut cfg = Config::new();
     cfg.wasm_simd(simd)
         .wasm_reference_types(reftypes)
         .wasm_multi_value(multi_val)
+        .wasm_bulk_memory(bulk_mem)
         .strategy(strategy)?
         .cranelift_debug_verifier(true);