diff --git a/build.rs b/build.rs index 1d71b908fe..4d2961e9a5 100644 --- a/build.rs +++ b/build.rs @@ -184,7 +184,9 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { ("bulk_memory_operations", "table_copy") | ("bulk_memory_operations", "table_init") - | ("bulk_memory_operations", "elem") => return false, + | ("bulk_memory_operations", "elem") + | ("bulk_memory_operations", "memory_copy") + | ("bulk_memory_operations", "memory_fill") => return false, ("bulk_memory_operations", _) => return true, _ => {} diff --git a/crates/environ/src/func_environ.rs b/crates/environ/src/func_environ.rs index 39e8d5f0b4..bbddb708e6 100644 --- a/crates/environ/src/func_environ.rs +++ b/crates/environ/src/func_environ.rs @@ -78,9 +78,17 @@ impl BuiltinFunctionIndex { pub const fn get_imported_memory_copy_index() -> Self { Self(11) } + /// Returns an index for wasm's `memory.fill` for locally defined memories. + pub const fn get_memory_fill_index() -> Self { + Self(12) + } + /// Returns an index for wasm's `memory.fill` for imported memories. + pub const fn get_imported_memory_fill_index() -> Self { + Self(13) + } /// Returns the total number of builtin functions. pub const fn builtin_functions_total_number() -> u32 { - 12 + 14 } /// Return the index as an u32 number. @@ -122,6 +130,10 @@ pub struct FuncEnvironment<'module_environment> { /// (it's the same for both local and imported memories). memory_copy_sig: Option, + /// The external function signature for implementing wasm's `memory.fill` + /// (it's the same for both local and imported memories). + memory_fill_sig: Option, + /// Offsets to struct fields accessed by JIT code. offsets: VMOffsets, } @@ -141,6 +153,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { table_init_sig: None, elem_drop_sig: None, memory_copy_sig: None, + memory_fill_sig: None, offsets: VMOffsets::new(target_config.pointer_bytes(), module), } } @@ -429,6 +442,51 @@ impl<'module_environment> FuncEnvironment<'module_environment> { } } + fn get_memory_fill_sig(&mut self, func: &mut Function) -> ir::SigRef { + let sig = self.memory_fill_sig.unwrap_or_else(|| { + func.import_signature(Signature { + params: vec![ + AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), + // Memory index. + AbiParam::new(I32), + // Destination address. + AbiParam::new(I32), + // Value. + AbiParam::new(I32), + // Length. + AbiParam::new(I32), + // Source location. + AbiParam::new(I32), + ], + returns: vec![], + call_conv: self.target_config.default_call_conv, + }) + }); + self.memory_fill_sig = Some(sig); + sig + } + + fn get_memory_fill_func( + &mut self, + func: &mut Function, + memory_index: MemoryIndex, + ) -> (ir::SigRef, usize, BuiltinFunctionIndex) { + let sig = self.get_memory_fill_sig(func); + if let Some(defined_memory_index) = self.module.defined_memory_index(memory_index) { + ( + sig, + defined_memory_index.index(), + BuiltinFunctionIndex::get_memory_fill_index(), + ) + } else { + ( + sig, + memory_index.index(), + BuiltinFunctionIndex::get_imported_memory_fill_index(), + ) + } + } + /// Translates load of builtin function and returns a pair of values `vmctx` /// and address of the loaded function. fn translate_load_builtin_function_address( @@ -1054,16 +1112,30 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m fn translate_memory_fill( &mut self, - _pos: FuncCursor, - _index: MemoryIndex, + mut pos: FuncCursor, + memory_index: MemoryIndex, _heap: ir::Heap, - _dst: ir::Value, - _val: ir::Value, - _len: ir::Value, + dst: ir::Value, + val: ir::Value, + len: ir::Value, ) -> WasmResult<()> { - Err(WasmError::Unsupported( - "bulk memory: `memory.fill`".to_string(), - )) + let (func_sig, memory_index, func_idx) = + self.get_memory_fill_func(&mut pos.func, memory_index); + + let memory_index_arg = pos.ins().iconst(I32, memory_index as i64); + + let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx); + + let src_loc = pos.srcloc(); + let src_loc_arg = pos.ins().iconst(I32, src_loc.bits() as i64); + + pos.ins().call_indirect( + func_sig, + func_addr, + &[vmctx, memory_index_arg, dst, val, len, src_loc_arg], + ); + + Ok(()) } fn translate_memory_init( diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 395969388a..2d13b65d7a 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -643,6 +643,7 @@ impl Instance { // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy let memory = self.memory(memory_index); + if src .checked_add(len) .map_or(true, |n| n as usize > memory.current_length) @@ -691,6 +692,69 @@ impl Instance { } } + /// Perform the `memory.fill` operation on a locally defined memory. + /// + /// # Errors + /// + /// Returns a `Trap` error if the memory range is out of bounds. + pub(crate) fn defined_memory_fill( + &self, + memory_index: DefinedMemoryIndex, + dst: u32, + val: u32, + len: u32, + source_loc: ir::SourceLoc, + ) -> Result<(), Trap> { + let memory = self.memory(memory_index); + + if dst + .checked_add(len) + .map_or(true, |m| m as usize > memory.current_length) + { + return Err(Trap::Wasm { + desc: TrapDescription { + source_loc, + trap_code: ir::TrapCode::HeapOutOfBounds, + }, + backtrace: Backtrace::new(), + }); + } + + let dst = isize::try_from(dst).unwrap(); + let val = val as u8; + + // Bounds and casts are checked above, by this point we know that + // everything is safe. + unsafe { + let dst = memory.base.offset(dst); + ptr::write_bytes(dst, val, len as usize); + } + + Ok(()) + } + + /// Perform the `memory.fill` operation on an imported memory. + /// + /// # Errors + /// + /// Returns a `Trap` error if the memory range is out of bounds. + pub(crate) fn imported_memory_fill( + &self, + memory_index: MemoryIndex, + dst: u32, + val: u32, + len: u32, + source_loc: ir::SourceLoc, + ) -> Result<(), Trap> { + let import = self.imported_memory(memory_index); + unsafe { + let foreign_instance = (&*import.vmctx).instance(); + let foreign_memory = &*import.from; + let foreign_index = foreign_instance.memory_index(foreign_memory); + foreign_instance.defined_memory_fill(foreign_index, dst, val, len, source_loc) + } + } + /// Get a table by index regardless of whether it is locally-defined or an /// imported, foreign table. pub(crate) fn get_table(&self, table_index: TableIndex) -> &Table { diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index b83627f72b..65d6331aa4 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -298,3 +298,39 @@ pub unsafe extern "C" fn wasmtime_imported_memory_copy( raise_lib_trap(trap); } } + +/// Implementation of `memory.fill` for locally defined memories. +#[no_mangle] +pub unsafe extern "C" fn wasmtime_memory_fill( + vmctx: *mut VMContext, + memory_index: u32, + dst: u32, + val: u32, + len: u32, + source_loc: u32, +) { + let memory_index = DefinedMemoryIndex::from_u32(memory_index); + let source_loc = ir::SourceLoc::new(source_loc); + let instance = (&mut *vmctx).instance(); + if let Err(trap) = instance.defined_memory_fill(memory_index, dst, val, len, source_loc) { + raise_lib_trap(trap); + } +} + +/// Implementation of `memory.fill` for imported memories. +#[no_mangle] +pub unsafe extern "C" fn wasmtime_imported_memory_fill( + vmctx: *mut VMContext, + memory_index: u32, + dst: u32, + val: u32, + len: u32, + source_loc: u32, +) { + let memory_index = MemoryIndex::from_u32(memory_index); + let source_loc = ir::SourceLoc::new(source_loc); + let instance = (&mut *vmctx).instance(); + if let Err(trap) = instance.imported_memory_fill(memory_index, dst, val, len, source_loc) { + raise_lib_trap(trap); + } +} diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index 50b02f7b48..6cb640f0e2 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -567,6 +567,10 @@ impl VMBuiltinFunctionsArray { wasmtime_memory_copy as usize; ptrs[BuiltinFunctionIndex::get_imported_memory_copy_index().index() as usize] = wasmtime_imported_memory_copy as usize; + ptrs[BuiltinFunctionIndex::get_memory_fill_index().index() as usize] = + wasmtime_memory_fill as usize; + ptrs[BuiltinFunctionIndex::get_imported_memory_fill_index().index() as usize] = + wasmtime_imported_memory_fill as usize; debug_assert!(ptrs.iter().cloned().all(|p| p != 0));