Implement data.drop and memory.init and get the rest of the bulk memory spec tests passing (#1264)

* Enable the already-passing `bulk-memoryoperations/imports.wast` test

* Implement support for the `memory.init` instruction and passive data

This adds support for passive data segments and the `memory.init` instruction
from the bulk memory operations proposal. Passive data segments are stored on
the Wasm module and then `memory.init` instructions copy their contents into
memory.

* Implement the `data.drop` instruction

This allows wasm modules to deallocate passive data segments that it doesn't
need anymore. We keep track of which segments have not been dropped on an
`Instance` and when dropping them, remove the entry from the instance's hash
map. The module always needs all of the segments for new instantiations.

* Enable final bulk memory operations spec test

This requires special casing an expected error message for an `assert_trap`,
since the expected error message contains the index of an uninitialized table
element, but our trap implementation doesn't save that diagnostic information
and shepherd it out.

crates/runtime/src/instance.rs

@@ -29,8 +29,8 @@ use std::{mem, ptr, slice};
 use thiserror::Error;
 use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap};
 use wasmtime_environ::wasm::{
-    DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, ElemIndex,
-    FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, TableIndex,
+    DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex,
+    ElemIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, TableIndex,
 };
 use wasmtime_environ::{ir, DataInitializer, Module, TableElements, VMOffsets};
 
@@ -92,6 +92,10 @@ pub(crate) struct Instance {
     /// empty slice.
     passive_elements: RefCell<HashMap<ElemIndex, Box<[VMCallerCheckedAnyfunc]>>>,
 
+    /// Passive data segments from our module. As `data.drop`s happen, entries
+    /// get removed. A missing entry is considered equivalent to an empty slice.
+    passive_data: RefCell<HashMap<DataIndex, Arc<[u8]>>>,
+
     /// Pointers to functions in executable memory.
     finished_functions: BoxedSlice<DefinedFuncIndex, *mut [VMFunctionBody]>,
 
@@ -747,6 +751,57 @@ impl Instance {
         }
     }
 
+    /// Performs the `memory.init` operation.
+    ///
+    /// # Errors
+    ///
+    /// Returns a `Trap` error if the destination range is out of this module's
+    /// memory's bounds or if the source range is outside the data segment's
+    /// bounds.
+    pub(crate) fn memory_init(
+        &self,
+        memory_index: MemoryIndex,
+        data_index: DataIndex,
+        dst: u32,
+        src: u32,
+        len: u32,
+        source_loc: ir::SourceLoc,
+    ) -> Result<(), Trap> {
+        // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init
+
+        let memory = self.get_memory(memory_index);
+        let passive_data = self.passive_data.borrow();
+        let data = passive_data
+            .get(&data_index)
+            .map_or(&[][..], |data| &**data);
+
+        if src
+            .checked_add(len)
+            .map_or(true, |n| n as usize > data.len())
+            || dst
+                .checked_add(len)
+                .map_or(true, |m| m as usize > memory.current_length)
+        {
+            return Err(Trap::wasm(source_loc, ir::TrapCode::HeapOutOfBounds));
+        }
+
+        let src_slice = &data[src as usize..(src + len) as usize];
+
+        unsafe {
+            let dst_start = memory.base.add(dst as usize);
+            let dst_slice = slice::from_raw_parts_mut(dst_start, len as usize);
+            dst_slice.copy_from_slice(src_slice);
+        }
+
+        Ok(())
+    }
+
+    /// Drop the given data segment, truncating its length to zero.
+    pub(crate) fn data_drop(&self, data_index: DataIndex) {
+        let mut passive_data = self.passive_data.borrow_mut();
+        passive_data.remove(&data_index);
+    }
+
     /// Get a table by index regardless of whether it is locally-defined or an
     /// imported, foreign table.
     pub(crate) fn get_table(&self, table_index: TableIndex) -> &Table {
@@ -824,6 +879,8 @@ impl InstanceHandle {
 
         let offsets = VMOffsets::new(mem::size_of::<*const u8>() as u8, &module.local);
 
+        let passive_data = RefCell::new(module.passive_data.clone());
+
         let handle = {
             let instance = Instance {
                 refcount: Cell::new(1),
@@ -833,6 +890,7 @@ impl InstanceHandle {
                 memories,
                 tables,
                 passive_elements: Default::default(),
+                passive_data,
                 finished_functions,
                 dbg_jit_registration,
                 host_state,

crates/runtime/src/libcalls.rs

@@ -36,7 +36,7 @@ use crate::table::Table;
 use crate::traphandlers::raise_lib_trap;
 use crate::vmcontext::VMContext;
 use wasmtime_environ::ir;
-use wasmtime_environ::wasm::{DefinedMemoryIndex, ElemIndex, MemoryIndex, TableIndex};
+use wasmtime_environ::wasm::{DataIndex, DefinedMemoryIndex, ElemIndex, MemoryIndex, TableIndex};
 
 /// Implementation of f32.ceil
 pub extern "C" fn wasmtime_f32_ceil(x: f32) -> f32 {
@@ -299,3 +299,32 @@ pub unsafe extern "C" fn wasmtime_imported_memory_fill(
         raise_lib_trap(trap);
     }
 }
+
+/// Implementation of `memory.init`.
+pub unsafe extern "C" fn wasmtime_memory_init(
+    vmctx: *mut VMContext,
+    memory_index: u32,
+    data_index: u32,
+    dst: u32,
+    src: u32,
+    len: u32,
+    source_loc: u32,
+) {
+    let result = {
+        let memory_index = MemoryIndex::from_u32(memory_index);
+        let data_index = DataIndex::from_u32(data_index);
+        let source_loc = ir::SourceLoc::new(source_loc);
+        let instance = (&mut *vmctx).instance();
+        instance.memory_init(memory_index, data_index, dst, src, len, source_loc)
+    };
+    if let Err(trap) = result {
+        raise_lib_trap(trap);
+    }
+}
+
+/// Implementation of `data.drop`.
+pub unsafe extern "C" fn wasmtime_data_drop(vmctx: *mut VMContext, data_index: u32) {
+    let data_index = DataIndex::from_u32(data_index);
+    let instance = (&mut *vmctx).instance();
+    instance.data_drop(data_index)
+}

crates/runtime/src/trap_registry.rs

@@ -84,7 +84,7 @@ fn trap_code_to_expected_string(trap_code: ir::TrapCode) -> String {
     match trap_code {
         StackOverflow => "call stack exhausted".to_string(),
         HeapOutOfBounds => "out of bounds memory access".to_string(),
-        TableOutOfBounds => "undefined element: out of bounds".to_string(),
+        TableOutOfBounds => "undefined element: out of bounds table access".to_string(),
         OutOfBounds => "out of bounds".to_string(), // Note: not covered by the test suite
         IndirectCallToNull => "uninitialized element".to_string(),
         BadSignature => "indirect call type mismatch".to_string(),

crates/runtime/src/vmcontext.rs

@@ -565,6 +565,10 @@ impl VMBuiltinFunctionsArray {
             wasmtime_memory_fill as usize;
         ptrs[BuiltinFunctionIndex::get_imported_memory_fill_index().index() as usize] =
             wasmtime_imported_memory_fill as usize;
+        ptrs[BuiltinFunctionIndex::get_memory_init_index().index() as usize] =
+            wasmtime_memory_init as usize;
+        ptrs[BuiltinFunctionIndex::get_data_drop_index().index() as usize] =
+            wasmtime_data_drop as usize;
 
         debug_assert!(ptrs.iter().cloned().all(|p| p != 0));