From 1639f2c844e5c06de9d102d6298af192af99d11c Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 20 Mar 2021 18:46:16 +0100 Subject: [PATCH 01/43] Allow passing arbitrary MemFlags to emit_small_mem{cpy,move} --- cranelift/frontend/src/frontend.rs | 34 ++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/cranelift/frontend/src/frontend.rs b/cranelift/frontend/src/frontend.rs index 365e5aeffb..3ebfc8efa0 100644 --- a/cranelift/frontend/src/frontend.rs +++ b/cranelift/frontend/src/frontend.rs @@ -640,6 +640,7 @@ impl<'a> FunctionBuilder<'a> { dest_align: u8, src_align: u8, non_overlapping: bool, + mut flags: MemFlags, ) { // Currently the result of guess work, not actual profiling. const THRESHOLD: u64 = 4; @@ -676,7 +677,6 @@ impl<'a> FunctionBuilder<'a> { return; } - let mut flags = MemFlags::new(); flags.set_aligned(); // Load all of the memory first. This is necessary in case `dest` overlaps. @@ -732,6 +732,7 @@ impl<'a> FunctionBuilder<'a> { ch: u8, size: u64, buffer_align: u8, + mut flags: MemFlags, ) { // Currently the result of guess work, not actual profiling. const THRESHOLD: u64 = 4; @@ -763,7 +764,6 @@ impl<'a> FunctionBuilder<'a> { let size = self.ins().iconst(config.pointer_type(), size as i64); self.call_memset(config, buffer, ch, size); } else { - let mut flags = MemFlags::new(); flags.set_aligned(); let ch = u64::from(ch); @@ -851,7 +851,9 @@ mod tests { use alloc::string::ToString; use cranelift_codegen::entity::EntityRef; use cranelift_codegen::ir::types::*; - use cranelift_codegen::ir::{AbiParam, ExternalName, Function, InstBuilder, Signature}; + use cranelift_codegen::ir::{ + AbiParam, ExternalName, Function, InstBuilder, MemFlags, Signature, + }; use cranelift_codegen::isa::CallConv; use cranelift_codegen::settings; use cranelift_codegen::verifier::verify_function; @@ -1063,7 +1065,16 @@ block0: let src = builder.use_var(x); let dest = builder.use_var(y); let size = 8; - builder.emit_small_memory_copy(target.frontend_config(), dest, src, size, 8, 8, true); + builder.emit_small_memory_copy( + target.frontend_config(), + dest, + src, + size, + 8, + 8, + true, + MemFlags::new(), + ); builder.ins().return_(&[dest]); builder.seal_all_blocks(); @@ -1121,7 +1132,16 @@ block0: let src = builder.use_var(x); let dest = builder.use_var(y); let size = 8192; - builder.emit_small_memory_copy(target.frontend_config(), dest, src, size, 8, 8, true); + builder.emit_small_memory_copy( + target.frontend_config(), + dest, + src, + size, + 8, + 8, + true, + MemFlags::new(), + ); builder.ins().return_(&[dest]); builder.seal_all_blocks(); @@ -1179,7 +1199,7 @@ block0: let dest = builder.use_var(y); let size = 8; - builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8); + builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8, MemFlags::new()); builder.ins().return_(&[dest]); builder.seal_all_blocks(); @@ -1232,7 +1252,7 @@ block0: let dest = builder.use_var(y); let size = 8192; - builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8); + builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8, MemFlags::new()); builder.ins().return_(&[dest]); builder.seal_all_blocks(); From e3bb36ba77f1f0bf1b1acc40885572ddcbfd1807 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 14 Apr 2021 13:46:08 +0200 Subject: [PATCH 02/43] Fix frame size after unwind rework After the unwind rework (commit 2d5db92a) the space used to save clobbered registers now lies between the nominal SP and the FP. Therefore, the size of that space should now be included in the frame size as reported by frame_size(), since this value is used to compute the nominal_sp_to_fp offset. --- cranelift/codegen/src/machinst/abi_impl.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 85a6d6831b..73d320a433 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -1285,7 +1285,7 @@ impl ABICallee for ABICalleeImpl { } // Save clobbered registers. - let (_, clobber_insts) = M::gen_clobber_save( + let (clobber_size, clobber_insts) = M::gen_clobber_save( self.call_conv, &self.flags, &self.clobbered, @@ -1304,7 +1304,7 @@ impl ABICallee for ABICalleeImpl { // [crate::machinst::abi_impl](this module) for more details // on stackframe layout and nominal SP maintenance. - self.total_frame_size = Some(total_stacksize); + self.total_frame_size = Some(total_stacksize + clobber_size as u32); insts } From 336c6369b40272ba432d0b03b28fad51fd571ee7 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 14 Apr 2021 13:51:16 +0200 Subject: [PATCH 03/43] Add back support for accumulating outgoing arguments The unwind rework (commit 2d5db92a) removed support for the feature to allow a target to allocate the space for outgoing function arguments right in the prologue (originally added via commit 80c2d70d). This patch adds it back. --- cranelift/codegen/src/isa/aarch64/abi.rs | 2 ++ cranelift/codegen/src/isa/arm32/abi.rs | 2 ++ cranelift/codegen/src/isa/x64/abi.rs | 2 ++ cranelift/codegen/src/machinst/abi.rs | 13 +++++++++++++ cranelift/codegen/src/machinst/abi_impl.rs | 18 ++++++++++++++++++ 5 files changed, 37 insertions(+) diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 380d2dfe0b..72ff0e6d64 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -587,6 +587,7 @@ impl ABIMachineSpec for AArch64MachineDeps { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> (u64, SmallVec<[Inst; 16]>) { let mut insts = SmallVec::new(); let (clobbered_int, clobbered_vec) = get_regs_saved_in_prologue(call_conv, clobbers); @@ -691,6 +692,7 @@ impl ABIMachineSpec for AArch64MachineDeps { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> SmallVec<[Inst; 16]> { let mut insts = SmallVec::new(); let (clobbered_int, clobbered_vec) = get_regs_saved_in_prologue(call_conv, clobbers); diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs index 94627a227c..5a4145d8b7 100644 --- a/cranelift/codegen/src/isa/arm32/abi.rs +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -319,6 +319,7 @@ impl ABIMachineSpec for Arm32MachineDeps { _flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> (u64, SmallVec<[Inst; 16]>) { let mut insts = SmallVec::new(); if fixed_frame_storage_size > 0 { @@ -348,6 +349,7 @@ impl ABIMachineSpec for Arm32MachineDeps { _flags: &settings::Flags, clobbers: &Set>, _fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> SmallVec<[Inst; 16]> { let mut insts = SmallVec::new(); let clobbered_vec = get_callee_saves(clobbers); diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 29a0dbd982..63a8af7c9b 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -500,6 +500,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> (u64, SmallVec<[Self::I; 16]>) { let mut insts = SmallVec::new(); // Find all clobbered registers that are callee-save. @@ -574,6 +575,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> SmallVec<[Self::I; 16]> { let mut insts = SmallVec::new(); diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index ac4cb7fad2..7af9b087c8 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -30,6 +30,12 @@ pub trait ABICallee { /// Access the (possibly legalized) signature. fn signature(&self) -> &Signature; + /// Accumulate outgoing arguments. This ensures that at least SIZE bytes + /// are allocated in the prologue to be available for use in function calls + /// to hold arguments and/or return values. If this function is called + /// multiple times, the maximum of all SIZE values will be available. + fn accumulate_outgoing_args_size(&mut self, size: u32); + /// Get the settings controlling this function's compilation. fn flags(&self) -> &settings::Flags; @@ -242,6 +248,13 @@ pub trait ABICaller { /// Emit code to post-adjust the satck, after call return and return-value copies. fn emit_stack_post_adjust>(&self, ctx: &mut C); + /// Accumulate outgoing arguments. This ensures that the caller (as + /// identified via the CTX argument) allocates enough space in the + /// prologue to hold all arguments and return values for this call. + /// There is no code emitted at the call site, everything is done + /// in the caller's function prologue. + fn accumulate_outgoing_args_size>(&self, ctx: &mut C); + /// Emit the call itself. /// /// The returned instruction should have proper use- and def-sets according diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 85a6d6831b..2b1b0cae05 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -444,6 +444,7 @@ pub trait ABIMachineSpec { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + outgoing_args_size: u32, ) -> (u64, SmallVec<[Self::I; 16]>); /// Generate a clobber-restore sequence. This sequence should perform the @@ -455,6 +456,7 @@ pub trait ABIMachineSpec { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + outgoing_args_size: u32, ) -> SmallVec<[Self::I; 16]>; /// Generate a call instruction/sequence. This method is provided one @@ -576,6 +578,8 @@ pub struct ABICalleeImpl { stackslots: PrimaryMap, /// Total stack size of all stackslots. stackslots_size: u32, + /// Stack size to be reserved for outgoing arguments. + outgoing_args_size: u32, /// Clobbered registers, from regalloc. clobbered: Set>, /// Total number of spillslots, from regalloc. @@ -691,6 +695,7 @@ impl ABICalleeImpl { sig, stackslots, stackslots_size: stack_offset, + outgoing_args_size: 0, clobbered: Set::empty(), spillslots: None, fixed_frame_storage_size: 0, @@ -917,6 +922,12 @@ impl ABICallee for ABICalleeImpl { } } + fn accumulate_outgoing_args_size(&mut self, size: u32) { + if size > self.outgoing_args_size { + self.outgoing_args_size = size; + } + } + fn flags(&self) -> &settings::Flags { &self.flags } @@ -1290,6 +1301,7 @@ impl ABICallee for ABICalleeImpl { &self.flags, &self.clobbered, self.fixed_frame_storage_size, + self.outgoing_args_size, ); insts.extend(clobber_insts); @@ -1317,6 +1329,7 @@ impl ABICallee for ABICalleeImpl { &self.flags, &self.clobbered, self.fixed_frame_storage_size, + self.outgoing_args_size, )); // N.B.: we do *not* emit a nominal SP adjustment here, because (i) there will be no @@ -1519,6 +1532,11 @@ impl ABICaller for ABICallerImpl { } } + fn accumulate_outgoing_args_size>(&self, ctx: &mut C) { + let off = self.sig.stack_arg_space + self.sig.stack_ret_space; + ctx.abi().accumulate_outgoing_args_size(off as u32); + } + fn emit_stack_pre_adjust>(&self, ctx: &mut C) { let off = self.sig.stack_arg_space + self.sig.stack_ret_space; adjust_stack_and_nominal_sp::(ctx, off as i32, /* is_sub = */ true) From 5904c09682f8f3340756300b693eab5f69ee8a5d Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 14 Apr 2021 13:54:02 +0200 Subject: [PATCH 04/43] Allow unwind support to work without a frame pointer The patch extends the unwinder to support targets that do not need to use a dedicated frame pointer register. Specifically, the changes include: - Change the "fp" routine in the RegisterMapper to return an *optional* frame pointer regsiter via Option. - On targets that choose to not define a FP register via the above routine, the UnwindInst::DefineNewFrame operation no longer switches the CFA to be defined in terms of the FP. (The operation still can be used to define the location of the clobber area.) - In addition, on targets that choose not to define a FP register, the UnwindInst::PushFrameRegs operation is not supported. - There is a new operation UnwindInst::StackAlloc that needs to be called on targets without FP whenever the stack pointer is updated. This caused the CFA offset to be adjusted accordingly. (On targets with FP this operation is a no-op.) --- .../src/isa/aarch64/inst/unwind/systemv.rs | 4 +- cranelift/codegen/src/isa/unwind.rs | 5 +++ cranelift/codegen/src/isa/unwind/systemv.rs | 39 ++++++++++++++----- cranelift/codegen/src/isa/unwind/winx64.rs | 6 +++ .../src/isa/x64/inst/unwind/systemv.rs | 4 +- .../codegen/src/isa/x86/unwind/systemv.rs | 4 +- 6 files changed, 47 insertions(+), 15 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs index 9f2eb741a0..b514dc20b8 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs @@ -56,8 +56,8 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { fn sp(&self) -> u16 { regs::stack_reg().get_hw_encoding().into() } - fn fp(&self) -> u16 { - regs::fp_reg().get_hw_encoding().into() + fn fp(&self) -> Option { + Some(regs::fp_reg().get_hw_encoding().into()) } fn lr(&self) -> Option { Some(regs::link_reg().get_hw_encoding().into()) diff --git a/cranelift/codegen/src/isa/unwind.rs b/cranelift/codegen/src/isa/unwind.rs index 7c9718a570..13397c3266 100644 --- a/cranelift/codegen/src/isa/unwind.rs +++ b/cranelift/codegen/src/isa/unwind.rs @@ -225,6 +225,11 @@ pub enum UnwindInst { /// the clobber area. offset_downward_to_clobbers: u32, }, + /// The stack pointer was adjusted to allocate the stack. + StackAlloc { + /// Size to allocate. + size: u32, + }, /// The stack slot at the given offset from the clobber-area base has been /// used to save the given register. /// diff --git a/cranelift/codegen/src/isa/unwind/systemv.rs b/cranelift/codegen/src/isa/unwind/systemv.rs index 965603d4e1..5ceadef93f 100644 --- a/cranelift/codegen/src/isa/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/unwind/systemv.rs @@ -122,8 +122,10 @@ pub(crate) trait RegisterMapper { fn map(&self, reg: Reg) -> Result; /// Gets stack pointer register. fn sp(&self) -> Register; - /// Gets the frame pointer register. - fn fp(&self) -> Register; + /// Gets the frame pointer register, if any. + fn fp(&self) -> Option { + None + } /// Gets the link register, if any. fn lr(&self) -> Option { None @@ -151,6 +153,7 @@ pub(crate) fn create_unwind_info_from_insts>( ) -> CodegenResult { let mut instructions = vec![]; + let mut cfa_offset = 0; let mut clobber_offset_to_cfa = 0; for &(instruction_offset, ref inst) in insts { match inst { @@ -163,10 +166,14 @@ pub(crate) fn create_unwind_info_from_insts>( instruction_offset, CallFrameInstruction::CfaOffset(offset_upward_to_caller_sp as i32), )); - // Note that we saved the old FP value on the stack. + // Note that we saved the old FP value on the stack. Use of this + // operation implies that the target defines a FP register. instructions.push(( instruction_offset, - CallFrameInstruction::Offset(mr.fp(), -(offset_upward_to_caller_sp as i32)), + CallFrameInstruction::Offset( + mr.fp().unwrap(), + -(offset_upward_to_caller_sp as i32), + ), )); // If there is a link register on this architecture, note that // we saved it as well. @@ -188,15 +195,29 @@ pub(crate) fn create_unwind_info_from_insts>( // Define CFA in terms of FP. Note that we assume it was already // defined correctly in terms of the current SP, and FP has just // been set to the current SP, so we do not need to change the - // offset, only the register. - instructions.push(( - instruction_offset, - CallFrameInstruction::CfaRegister(mr.fp()), - )); + // offset, only the register. (This is done only if the target + // defines a frame pointer register.) + if let Some(fp) = mr.fp() { + instructions.push((instruction_offset, CallFrameInstruction::CfaRegister(fp))); + } + // Record initial CFA offset. This will be used with later + // StackAlloc calls if we do not have a frame pointer. + cfa_offset = offset_upward_to_caller_sp; // Record distance from CFA downward to clobber area so we can // express clobber offsets later in terms of CFA. clobber_offset_to_cfa = offset_upward_to_caller_sp + offset_downward_to_clobbers; } + &UnwindInst::StackAlloc { size } => { + // If we do not use a frame pointer, we need to update the + // CFA offset whenever the stack pointer changes. + if mr.fp().is_none() { + cfa_offset += size; + instructions.push(( + instruction_offset, + CallFrameInstruction::CfaOffset(cfa_offset as i32), + )); + } + } &UnwindInst::SaveReg { clobber_offset, reg, diff --git a/cranelift/codegen/src/isa/unwind/winx64.rs b/cranelift/codegen/src/isa/unwind/winx64.rs index f447d341ae..adac7caefa 100644 --- a/cranelift/codegen/src/isa/unwind/winx64.rs +++ b/cranelift/codegen/src/isa/unwind/winx64.rs @@ -356,6 +356,12 @@ pub(crate) fn create_unwind_info_from_insts>( frame_register_offset = ensure_unwind_offset(offset_downward_to_clobbers)?; unwind_codes.push(UnwindCode::SetFPReg { instruction_offset }); } + &UnwindInst::StackAlloc { size } => { + unwind_codes.push(UnwindCode::StackAlloc { + instruction_offset, + size, + }); + } &UnwindInst::SaveReg { clobber_offset, reg, diff --git a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs index c2a04a5c8e..9115db0671 100644 --- a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs @@ -89,8 +89,8 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { fn sp(&self) -> u16 { X86_64::RSP.0 } - fn fp(&self) -> u16 { - X86_64::RBP.0 + fn fp(&self) -> Option { + Some(X86_64::RBP.0) } } diff --git a/cranelift/codegen/src/isa/x86/unwind/systemv.rs b/cranelift/codegen/src/isa/x86/unwind/systemv.rs index 0c3e749762..31fc64c9fb 100644 --- a/cranelift/codegen/src/isa/x86/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x86/unwind/systemv.rs @@ -121,8 +121,8 @@ pub(crate) fn create_unwind_info( fn sp(&self) -> u16 { X86_64::RSP.0 } - fn fp(&self) -> u16 { - X86_64::RBP.0 + fn fp(&self) -> Option { + Some(X86_64::RBP.0) } } let map = RegisterMapper(isa); From e7bced95125055b41e449d414e55ccf2f05c09cf Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Wed, 14 Apr 2021 18:08:52 +0200 Subject: [PATCH 05/43] cranelift: always spill i32 with i64 stores; Fixes #2839. See also the issue description and comments in this commits for details of what the fix is about here. --- cranelift/codegen/src/machinst/abi_impl.rs | 13 ++ .../isa/x64/store-stack-full-width-i32.clif | 127 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 85a6d6831b..5636b83f82 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -1213,6 +1213,19 @@ impl ABICallee for ABICalleeImpl { let spill_off = islot * M::word_bytes() as i64; let sp_off = self.stackslots_size as i64 + spill_off; trace!("store_spillslot: slot {:?} -> sp_off {}", slot, sp_off); + + // When reloading from a spill slot, we might have lost information about real integer + // types. For instance, on the x64 backend, a zero-extension can become spurious and + // optimized into a move, causing vregs of types I32 and I64 to share the same coalescing + // equivalency class. As a matter of fact, such a value can be spilled as an I32 and later + // reloaded as an I64; to make sure the high bits are always defined, do a word-sized store + // all the time, in this case. + let ty = if ty.is_int() && ty.bytes() < M::word_bytes() { + M::word_type() + } else { + ty + }; + gen_store_stack_multi::(StackAMode::NominalSPOffset(sp_off, ty), from_regs, ty) } diff --git a/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif b/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif new file mode 100644 index 0000000000..10b3102c17 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif @@ -0,0 +1,127 @@ +test compile +target x86_64 machinst + +;; The goal of this test is to ensure that stack spills of an integer value, +;; which width is less than the machine word's size, cause the full word to be +;; stored, and not only the lower bits. + +;; Because of unsigned extensions which can be transformed into simple moves, +;; the source vreg of the extension operation can be coalesced with its +;; destination vreg, and if it happens to be spill, then the reload may use a +;; reload of a different, larger size. + +function %f0(i32, i32, i32) -> i64 { + fn0 = %g(i32) -> i64 + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: subq $$64, %rsp + +;; Stash all the callee-saved registers. + +; nextln: movq %r12, 16(%rsp) +; nextln: movq %r13, 24(%rsp) +; nextln: movq %r14, 32(%rsp) +; nextln: movq %rbx, 40(%rsp) +; nextln: movq %r15, 48(%rsp) + +block0(v0: i32, v1: i32, v2: i32): + ;; First, create enough virtual registers so that the call instructions + ;; causes at least one of them to be spilled onto the stack. + + v3 = iadd.i32 v0, v1 + v4 = iadd.i32 v1, v2 + v5 = iadd.i32 v0, v2 + v6 = iadd.i32 v3, v0 + v7 = iadd.i32 v4, v0 + v8 = iadd.i32 v5, v0 + +; nextln: movq %rdi, %r12 +; nextln: addl %esi, %r12d +; nextln: movq %rsi, %r13 +; nextln: addl %edx, %r13d +; nextln: movq %rdi, %r14 +; nextln: addl %edx, %r14d +; nextln: movq %r12, %rbx +; nextln: addl %edi, %ebx +; nextln: movq %r13, %r15 +; nextln: addl %edi, %r15d +; nextln: movq %r14, %rsi + +;; This should be movq below, not movl. +; nextln: movq %rsi, rsp(0 + virtual offset) + +; nextln: movslq rsp(0 + virtual offset), %rsi +; nextln: addl %edi, %esi + + ;; Put an effectful instruction so that the live-ranges of the adds and + ;; uextends are split here, and to prevent the uextend to be emitted + ;; before the call. This will effectively causing the above i32 to be + ;; spilled as an i32, and not a full i64. + + v300 = call fn0(v0) + +;; This should be movq below, not movl. +; nextln: movq %rsi, rsp(0 + virtual offset) + +; nextln: load_ext_name %g+0, %rsi +; nextln: call *%rsi + + v31 = uextend.i64 v3 + v41 = uextend.i64 v4 + v51 = uextend.i64 v5 + v61 = uextend.i64 v6 + v71 = uextend.i64 v7 + v81 = uextend.i64 v8 + + ;; None of the uextends are generated here yet. + + ;; At this point, I'd expect that this second call below would be not + ;; necessary, but if it is removed, the uextend is applied before the call, + ;; and the i64 is spilled (then reloaded), causing the bug to not appear. So + ;; an additional call it is! + + v100 = call fn0(v3) + +; nextln: movq %r12, %rsi +; nextln: movq %rsi, rsp(8 + virtual offset) +; nextln: nop len=0 +; nextln: movq %r12, %rdi +; nextln: load_ext_name %g+0, %rsi +; nextln: call *%rsi + + ;; Cause reloads of all the values. Most are in registers, but one of them + ;; is on the stack. Make sure they're all used in the final computation. + + v101 = iadd.i64 v100, v31 + v102 = iadd.i64 v101, v41 + v103 = iadd.i64 v102, v51 + v104 = iadd.i64 v103, v61 + v105 = iadd.i64 v104, v71 + v200 = iadd.i64 v105, v81 + +; nextln: movq %rax, %rsi +; nextln: movq rsp(8 + virtual offset), %rdi +; nextln: addq %rdi, %rsi +; nextln: addq %r13, %rsi +; nextln: addq %r14, %rsi +; nextln: addq %rbx, %rsi +; nextln: addq %r15, %rsi + +;; The reload operates on a full word, so uses movq. +; nextln: movq rsp(0 + virtual offset), %rdi + +; nextln: addq %rdi, %rsi +; nextln: movq %rsi, %rax +; nextln: movq 16(%rsp), %r12 +; nextln: movq 24(%rsp), %r13 +; nextln: movq 32(%rsp), %r14 +; nextln: movq 40(%rsp), %rbx +; nextln: movq 48(%rsp), %r15 +; nextln: addq $$64, %rsp + + return v200 +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} From 1243cea4554c6792a27c1e090817a4c89d335df6 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Tue, 13 Apr 2021 13:22:03 +0200 Subject: [PATCH 06/43] Update cap-std dependency to 0.13.9 This fixes a build failure on s390x. --- Cargo.lock | 20 ++++++++++---------- crates/wasi-common/cap-std-sync/Cargo.toml | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0eb0b74a07..f61d3abc14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,9 +287,9 @@ checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" [[package]] name = "cap-fs-ext" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee87a3a916d6f051fc6809c39c4627f0c3a73b2a803bcfbb5fdf2bdfa1da0cb" +checksum = "c4a7f9bdd49f3979c659811723041929803c986b3ed5381af726a4082e806c27" dependencies = [ "cap-primitives", "cap-std", @@ -300,9 +300,9 @@ dependencies = [ [[package]] name = "cap-primitives" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3e3ea29994a34f3bc67b5396a43c87597d302d9e2e5e3b3d5ba952d86c7b41" +checksum = "0e206e688244a8f8158f91d77d3d8ed5891eb27aef2542bb26ceb98b970d5597" dependencies = [ "errno", "fs-set-times", @@ -320,18 +320,18 @@ dependencies = [ [[package]] name = "cap-rand" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0418058b38db7efc6021c5ce012e3a39c57e1a4d7bf2ddcd3789771de505d2f" +checksum = "0842ea6c9a2e078ab901b3b805608af65e11fcbfb87024d37fb129b2b74cb1ac" dependencies = [ "rand 0.8.3", ] [[package]] name = "cap-std" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5f20cbb3055e9c72b16ba45913fe9f92836d2aa7a880e1ffacb8d244f454319" +checksum = "36d8cb4b761a2fb1ccf0e92d795c717bba9dcb1e54636ee0579fca628abfa305" dependencies = [ "cap-primitives", "posish", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "cap-time-ext" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b684f9db089b0558520076b4eeda2b719a5c4c06f329be96c9497f2b48c3944" +checksum = "bc5c279d213dfe8f71c02c4cd740bd130d1dce39f45fdcdb745ae3de8a777fbd" dependencies = [ "cap-primitives", "once_cell", diff --git a/crates/wasi-common/cap-std-sync/Cargo.toml b/crates/wasi-common/cap-std-sync/Cargo.toml index 821fa8a5bb..d1e22339fc 100644 --- a/crates/wasi-common/cap-std-sync/Cargo.toml +++ b/crates/wasi-common/cap-std-sync/Cargo.toml @@ -14,10 +14,10 @@ include = ["src/**/*", "LICENSE" ] [dependencies] wasi-common = { path = "../", version = "0.26.0" } anyhow = "1.0" -cap-std = "0.13.7" -cap-fs-ext = "0.13.7" -cap-time-ext = "0.13.7" -cap-rand = "0.13.2" +cap-std = "0.13.9" +cap-fs-ext = "0.13.9" +cap-time-ext = "0.13.9" +cap-rand = "0.13.9" fs-set-times = "0.3.1" unsafe-io = "0.6.2" system-interface = { version = "0.6.3", features = ["cap_std_impls"] } From 0acc1451eaecfcc73021fc619ea88b0e866bccb2 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Thu, 15 Apr 2021 11:53:58 -0700 Subject: [PATCH 07/43] x64: lower iabs.i64x2 using a single AVX512 instruction when possible (#2819) * x64: add EVEX encoding mechanism Also, includes an empty stub module for the VEX encoding. * x64: lower abs.i64x2 to VPABSQ when available * x64: refactor EVEX encodings to use `EvexInstruction` This change replaces the `encode_evex` function with a builder-style struct, `EvexInstruction`. This approach clarifies the code, adds documentation, and results in slight speedups when benchmarked. * x64: rename encoding CodeSink to ByteSink --- cranelift/codegen/src/isa/x64/inst/args.rs | 4 - cranelift/codegen/src/isa/x64/inst/emit.rs | 22 +- .../codegen/src/isa/x64/inst/emit_tests.rs | 7 + .../codegen/src/isa/x64/inst/encoding/evex.rs | 396 ++++++++++++++++++ .../codegen/src/isa/x64/inst/encoding/mod.rs | 56 +++ .../codegen/src/isa/x64/inst/encoding/rex.rs | 65 ++- .../codegen/src/isa/x64/inst/encoding/vex.rs | 2 + cranelift/codegen/src/isa/x64/inst/mod.rs | 30 +- cranelift/codegen/src/isa/x64/lower.rs | 40 +- 9 files changed, 590 insertions(+), 32 deletions(-) create mode 100644 cranelift/codegen/src/isa/x64/inst/encoding/evex.rs create mode 100644 cranelift/codegen/src/isa/x64/inst/encoding/vex.rs diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 2eebf4140e..b54f1b6126 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -460,9 +460,7 @@ pub(crate) enum InstructionSet { BMI1, #[allow(dead_code)] // never constructed (yet). BMI2, - #[allow(dead_code)] AVX512F, - #[allow(dead_code)] AVX512VL, } @@ -995,13 +993,11 @@ impl fmt::Display for SseOpcode { #[derive(Clone)] pub enum Avx512Opcode { - #[allow(dead_code)] Vpabsq, } impl Avx512Opcode { /// Which `InstructionSet`s support the opcode? - #[allow(dead_code)] pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> { match self { Avx512Opcode::Vpabsq => smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL], diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 3bd6a58e7c..a35bbe0a99 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -6,9 +6,11 @@ use crate::isa::x64::inst::args::*; use crate::isa::x64::inst::*; use crate::machinst::{inst_common, MachBuffer, MachInstEmit, MachLabel}; use core::convert::TryInto; +use encoding::evex::{EvexInstruction, EvexVectorLength}; use encoding::rex::{ emit_simm, emit_std_enc_enc, emit_std_enc_mem, emit_std_reg_mem, emit_std_reg_reg, int_reg_enc, - low8_will_sign_extend_to_32, low8_will_sign_extend_to_64, reg_enc, LegacyPrefixes, RexFlags, + low8_will_sign_extend_to_32, low8_will_sign_extend_to_64, reg_enc, LegacyPrefixes, OpcodeMap, + RexFlags, }; use log::debug; use regalloc::{Reg, Writable}; @@ -1404,6 +1406,24 @@ pub(crate) fn emit( }; } + Inst::XmmUnaryRmREvex { op, src, dst } => { + let opcode = match op { + Avx512Opcode::Vpabsq => 0x1f, + }; + match src { + RegMem::Reg { reg: src } => EvexInstruction::new() + .length(EvexVectorLength::V128) + .prefix(LegacyPrefixes::_66) + .map(OpcodeMap::_0F38) + .w(true) + .opcode(opcode) + .reg(dst.to_reg().get_hw_encoding()) + .rm(src.get_hw_encoding()) + .encode(sink), + _ => todo!(), + }; + } + Inst::XmmRmR { op, src: src_e, diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index f730d25e93..f03762b97b 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3865,6 +3865,12 @@ fn test_x64_emit() { "cvtdq2pd %xmm2, %xmm8", )); + insns.push(( + Inst::xmm_unary_rm_r_evex(Avx512Opcode::Vpabsq, RegMem::reg(xmm2), w_xmm8), + "6272FD081FC2", + "vpabsq %xmm2, %xmm8", + )); + // Xmm to int conversions, and conversely. insns.push(( @@ -4276,6 +4282,7 @@ fn test_x64_emit() { let mut isa_flag_builder = x64::settings::builder(); isa_flag_builder.enable("has_ssse3").unwrap(); isa_flag_builder.enable("has_sse41").unwrap(); + isa_flag_builder.enable("has_avx512f").unwrap(); let isa_flags = x64::settings::Flags::new(&flags, isa_flag_builder); let rru = regs::create_reg_universe_systemv(&flags); diff --git a/cranelift/codegen/src/isa/x64/inst/encoding/evex.rs b/cranelift/codegen/src/isa/x64/inst/encoding/evex.rs new file mode 100644 index 0000000000..9029b9f43e --- /dev/null +++ b/cranelift/codegen/src/isa/x64/inst/encoding/evex.rs @@ -0,0 +1,396 @@ +//! Encodes EVEX instructions. These instructions are those added by the AVX-512 extensions. The +//! EVEX encoding requires a 4-byte prefix: +//! +//! Byte 0: 0x62 +//! ┌───┬───┬───┬───┬───┬───┬───┬───┐ +//! Byte 1: │ R │ X │ B │ R'│ 0 │ 0 │ m │ m │ +//! ├───┼───┼───┼───┼───┼───┼───┼───┤ +//! Byte 2: │ W │ v │ v │ v │ v │ 1 │ p │ p │ +//! ├───┼───┼───┼───┼───┼───┼───┼───┤ +//! Byte 3: │ z │ L'│ L │ b │ V'│ a │ a │ a │ +//! └───┴───┴───┴───┴───┴───┴───┴───┘ +//! +//! The prefix is then followeded by the opcode byte, the ModR/M byte, and other optional suffixes +//! (e.g. SIB byte, displacements, immediates) based on the instruction (see section 2.6, Intel +//! Software Development Manual, volume 2A). +use super::rex::{encode_modrm, LegacyPrefixes, OpcodeMap}; +use super::ByteSink; +use core::ops::RangeInclusive; + +/// Constructs an EVEX-encoded instruction using a builder pattern. This approach makes it visually +/// easier to transform something the manual's syntax, `EVEX.256.66.0F38.W1 1F /r` to code: +/// `EvexInstruction::new().length(...).prefix(...).map(...).w(true).opcode(0x1F).reg(...).rm(...)`. +pub struct EvexInstruction { + bits: u32, + opcode: u8, + reg: Register, + rm: Register, +} + +/// Because some of the bit flags in the EVEX prefix are reversed and users of `EvexInstruction` may +/// choose to skip setting fields, here we set some sane defaults. Note that: +/// - the first byte is always `0x62` but you will notice it at the end of the default `bits` value +/// implemented--remember the little-endian order +/// - some bits are always set to certain values: bits 10-11 to 0, bit 18 to 1 +/// - the other bits set correspond to reversed bits: R, X, B, R' (byte 1), vvvv (byte 2), V' (byte +/// 3). +/// +/// See the `default_emission` test for what these defaults are equivalent to (e.g. using RAX, +/// unsetting the W bit, etc.) +impl Default for EvexInstruction { + fn default() -> Self { + Self { + bits: 0x08_7C_F0_62, + opcode: 0, + reg: Register::default(), + rm: Register::default(), + } + } +} + +#[allow(non_upper_case_globals)] // This makes it easier to match the bit range names to the manual's names. +impl EvexInstruction { + /// Construct a default EVEX instruction. + pub fn new() -> Self { + Self::default() + } + + /// Set the length of the instruction . Note that there are sets of instructions (i.e. rounding, + /// memory broadcast) that modify the same underlying bits--at some point (TODO) we can add a + /// way to set those context bits and verify that both are not used (e.g. rounding AND length). + /// For now, this method is very convenient. + #[inline(always)] + pub fn length(mut self, length: EvexVectorLength) -> Self { + self.write(Self::LL, EvexContext::Other { length }.bits() as u32); + self + } + + /// Set the legacy prefix byte of the instruction: None | 66 | F0 | F2 | F3. EVEX instructions + /// pack these into the prefix, not as separate bytes. + #[inline(always)] + pub fn prefix(mut self, prefix: LegacyPrefixes) -> Self { + self.write(Self::pp, prefix.bits() as u32); + self + } + + /// Set the opcode map byte of the instruction: None | 0F | 0F38 | 0F3A. EVEX instructions pack + /// these into the prefix, not as separate bytes. + #[inline(always)] + pub fn map(mut self, map: OpcodeMap) -> Self { + self.write(Self::mm, map.bits() as u32); + self + } + + /// Set the W bit, typically used to indicate an instruction using 64 bits of an operand (e.g. + /// 64 bit lanes). EVEX packs this bit in the EVEX prefix; previous encodings used the REX + /// prefix. + #[inline(always)] + pub fn w(mut self, w: bool) -> Self { + self.write(Self::W, w as u32); + self + } + + /// Set the instruction opcode byte. + #[inline(always)] + pub fn opcode(mut self, opcode: u8) -> Self { + self.opcode = opcode; + self + } + + /// Set the register to use for the `reg` bits; many instructions use this as the write operand. + /// Setting this affects both the ModRM byte (`reg` section) and the EVEX prefix (the extension + /// bits for register encodings > 8). + #[inline(always)] + pub fn reg(mut self, reg: impl Into) -> Self { + self.reg = reg.into(); + let r = !(self.reg.0 >> 3) & 1; + let r_ = !(self.reg.0 >> 4) & 1; + self.write(Self::R, r as u32); + self.write(Self::R_, r_ as u32); + self + } + + /// Set the mask to use. See section 2.6 in the Intel Software Developer's Manual, volume 2A for + /// more details. + #[allow(dead_code)] + #[inline(always)] + pub fn mask(mut self, mask: EvexMasking) -> Self { + self.write(Self::aaa, mask.aaa_bits() as u32); + self.write(Self::z, mask.z_bit() as u32); + self + } + + /// Set the `vvvvv` register; some instructions allow using this as a second, non-destructive + /// source register in 3-operand instructions (e.g. 2 read, 1 write). + #[allow(dead_code)] + #[inline(always)] + pub fn vvvvv(mut self, reg: impl Into) -> Self { + let reg = reg.into(); + self.write(Self::vvvv, !(reg.0 as u32) & 0b1111); + self.write(Self::V_, !(reg.0 as u32 >> 4) & 0b1); + self + } + + /// Set the register to use for the `rm` bits; many instructions use this as the "read from + /// register/memory" operand. Currently this does not support memory addressing (TODO).Setting + /// this affects both the ModRM byte (`rm` section) and the EVEX prefix (the extension bits for + /// register encodings > 8). + #[inline(always)] + pub fn rm(mut self, reg: impl Into) -> Self { + self.rm = reg.into(); + let b = !(self.rm.0 >> 3) & 1; + let x = !(self.rm.0 >> 4) & 1; + self.write(Self::X, x as u32); + self.write(Self::B, b as u32); + self + } + + /// Emit the EVEX-encoded instruction to the code sink: + /// - first, the 4-byte EVEX prefix; + /// - then, the opcode byte; + /// - finally, the ModR/M byte. + /// + /// Eventually this method should support encodings of more than just the reg-reg addressing mode (TODO). + pub fn encode(&self, sink: &mut CS) { + sink.put4(self.bits); + sink.put1(self.opcode); + sink.put1(encode_modrm(3, self.reg.0 & 7, self.rm.0 & 7)); + } + + // In order to simplify the encoding of the various bit ranges in the prefix, we specify those + // ranges according to the table below (extracted from the Intel Software Development Manual, + // volume 2A). Remember that, because we pack the 4-byte prefix into a little-endian `u32`, this + // chart should be read from right-to-left, top-to-bottom. Note also that we start ranges at bit + // 8, leaving bits 0-7 for the mandatory `0x62`. + // ┌───┬───┬───┬───┬───┬───┬───┬───┐ + // Byte 1: │ R │ X │ B │ R'│ 0 │ 0 │ m │ m │ + // ├───┼───┼───┼───┼───┼───┼───┼───┤ + // Byte 2: │ W │ v │ v │ v │ v │ 1 │ p │ p │ + // ├───┼───┼───┼───┼───┼───┼───┼───┤ + // Byte 3: │ z │ L'│ L │ b │ V'│ a │ a │ a │ + // └───┴───┴───┴───┴───┴───┴───┴───┘ + + // Byte 1: + const mm: RangeInclusive = 8..=9; + const R_: RangeInclusive = 12..=12; + const B: RangeInclusive = 13..=13; + const X: RangeInclusive = 14..=14; + const R: RangeInclusive = 15..=15; + + // Byte 2: + const pp: RangeInclusive = 16..=17; + const vvvv: RangeInclusive = 19..=22; + const W: RangeInclusive = 23..=23; + + // Byte 3: + const aaa: RangeInclusive = 24..=26; + const V_: RangeInclusive = 27..=27; + #[allow(dead_code)] // Will be used once broadcast and rounding controls are exposed. + const b: RangeInclusive = 28..=28; + const LL: RangeInclusive = 29..=30; + const z: RangeInclusive = 31..=31; + + // A convenience method for writing the `value` bits to the given range in `self.bits`. + #[inline] + fn write(&mut self, range: RangeInclusive, value: u32) { + assert!(ExactSizeIterator::len(&range) > 0); + let size = range.end() - range.start() + 1; // Calculate the number of bits in the range. + let mask: u32 = (1 << size) - 1; // Generate a bit mask. + debug_assert!( + value <= mask, + "The written value should have fewer than {} bits.", + size + ); + let mask_complement = !(mask << *range.start()); // Create the bitwise complement for the clear mask. + self.bits &= mask_complement; // Clear the bits in `range`; otherwise the OR below may allow previously-set bits to slip through. + let value = value << *range.start(); // Place the value in the correct location (assumes `value <= mask`). + self.bits |= value; // Modify the bits in `range`. + } +} + +#[derive(Copy, Clone, Default)] +pub struct Register(u8); +impl From for Register { + fn from(reg: u8) -> Self { + debug_assert!(reg < 16); + Self(reg) + } +} + +/// Defines the EVEX context for the `L'`, `L`, and `b` bits (bits 6:4 of EVEX P2 byte). Table 2-36 in +/// section 2.6.10 (Intel Software Development Manual, volume 2A) describes how these bits can be +/// used together for certain classes of instructions; i.e., special care should be taken to ensure +/// that instructions use an applicable correct `EvexContext`. Table 2-39 contains cases where +/// opcodes can result in an #UD. +#[allow(dead_code)] // Rounding and broadcast modes are not yet used. +pub enum EvexContext { + RoundingRegToRegFP { + rc: EvexRoundingControl, + }, + NoRoundingFP { + sae: bool, + length: EvexVectorLength, + }, + MemoryOp { + broadcast: bool, + length: EvexVectorLength, + }, + Other { + length: EvexVectorLength, + }, +} + +impl Default for EvexContext { + fn default() -> Self { + Self::Other { + length: EvexVectorLength::default(), + } + } +} + +impl EvexContext { + /// Encode the `L'`, `L`, and `b` bits (bits 6:4 of EVEX P2 byte) for merging with the P2 byte. + fn bits(&self) -> u8 { + match self { + Self::RoundingRegToRegFP { rc } => 0b001 | rc.bits() << 1, + Self::NoRoundingFP { sae, length } => (*sae as u8) | length.bits() << 1, + Self::MemoryOp { broadcast, length } => (*broadcast as u8) | length.bits() << 1, + Self::Other { length } => length.bits() << 1, + } + } +} + +/// The EVEX format allows choosing a vector length in the `L'` and `L` bits; see `EvexContext`. +#[allow(dead_code)] // Wider-length vectors are not yet used. +pub enum EvexVectorLength { + V128, + V256, + V512, +} + +impl EvexVectorLength { + /// Encode the `L'` and `L` bits for merging with the P2 byte. + fn bits(&self) -> u8 { + match self { + Self::V128 => 0b00, + Self::V256 => 0b01, + Self::V512 => 0b10, + // 0b11 is reserved (#UD). + } + } +} + +impl Default for EvexVectorLength { + fn default() -> Self { + Self::V128 + } +} + +/// The EVEX format allows defining rounding control in the `L'` and `L` bits; see `EvexContext`. +#[allow(dead_code)] // Rounding controls are not yet used. +pub enum EvexRoundingControl { + RNE, + RD, + RU, + RZ, +} + +impl EvexRoundingControl { + /// Encode the `L'` and `L` bits for merging with the P2 byte. + fn bits(&self) -> u8 { + match self { + Self::RNE => 0b00, + Self::RD => 0b01, + Self::RU => 0b10, + Self::RZ => 0b11, + } + } +} + +/// Defines the EVEX masking behavior; masking support is described in section 2.6.4 of the Intel +/// Software Development Manual, volume 2A. +#[allow(dead_code)] // Masking is not yet used. +pub enum EvexMasking { + None, + Merging { k: u8 }, + Zeroing { k: u8 }, +} + +impl Default for EvexMasking { + fn default() -> Self { + EvexMasking::None + } +} + +impl EvexMasking { + /// Encode the `z` bit for merging with the P2 byte. + fn z_bit(&self) -> u8 { + match self { + Self::None | Self::Merging { .. } => 0, + Self::Zeroing { .. } => 1, + } + } + + /// Encode the `aaa` bits for merging with the P2 byte. + fn aaa_bits(&self) -> u8 { + match self { + Self::None => 0b000, + Self::Merging { k } | Self::Zeroing { k } => { + debug_assert!(*k <= 7); + *k + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::isa::x64::inst::regs; + use std::vec::Vec; + + // As a sanity test, we verify that the output of `xed-asmparse-main 'vpabsq xmm0{k0}, + // xmm1'` matches this EVEX encoding machinery. + #[test] + fn vpabsq() { + let dst = regs::xmm0(); + let src = regs::xmm1(); + let mut sink0 = Vec::new(); + + EvexInstruction::new() + .prefix(LegacyPrefixes::_66) + .map(OpcodeMap::_0F38) + .w(true) + .opcode(0x1F) + .reg(dst.get_hw_encoding()) + .rm(src.get_hw_encoding()) + .length(EvexVectorLength::V128) + .encode(&mut sink0); + + assert_eq!(sink0, vec![0x62, 0xf2, 0xfd, 0x08, 0x1f, 0xc1]); + } + + /// Verify that the defaults are equivalent to an instruction with a `0x00` opcode using the + /// "0" register (i.e. `rax`), with sane defaults for the various configurable parameters. This + /// test is more interesting than it may appear because some of the parameters have flipped-bit + /// representations (e.g. `vvvvv`) so emitting 0s as a default will not work. + #[test] + fn default_emission() { + let mut sink0 = Vec::new(); + EvexInstruction::new().encode(&mut sink0); + + let mut sink1 = Vec::new(); + EvexInstruction::new() + .length(EvexVectorLength::V128) + .prefix(LegacyPrefixes::None) + .map(OpcodeMap::None) + .w(false) + .opcode(0x00) + .reg(regs::rax().get_hw_encoding()) + .rm(regs::rax().get_hw_encoding()) + .mask(EvexMasking::None) + .encode(&mut sink1); + + assert_eq!(sink0, sink1); + } +} diff --git a/cranelift/codegen/src/isa/x64/inst/encoding/mod.rs b/cranelift/codegen/src/isa/x64/inst/encoding/mod.rs index 7fd3aeeae7..a269e58609 100644 --- a/cranelift/codegen/src/isa/x64/inst/encoding/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/encoding/mod.rs @@ -1 +1,57 @@ +use crate::{isa::x64, machinst::MachBuffer}; +use std::vec::Vec; + +pub mod evex; pub mod rex; +pub mod vex; + +pub trait ByteSink { + /// Add 1 byte to the code section. + fn put1(&mut self, _: u8); + + /// Add 2 bytes to the code section. + fn put2(&mut self, _: u16); + + /// Add 4 bytes to the code section. + fn put4(&mut self, _: u32); + + /// Add 8 bytes to the code section. + fn put8(&mut self, _: u64); +} + +impl ByteSink for MachBuffer { + fn put1(&mut self, value: u8) { + self.put1(value) + } + + fn put2(&mut self, value: u16) { + self.put2(value) + } + + fn put4(&mut self, value: u32) { + self.put4(value) + } + + fn put8(&mut self, value: u64) { + self.put8(value) + } +} + +/// Provide a convenient implementation for testing. +impl ByteSink for Vec { + fn put1(&mut self, v: u8) { + self.extend_from_slice(&[v]) + } + + fn put2(&mut self, v: u16) { + self.extend_from_slice(&v.to_le_bytes()) + } + + fn put4(&mut self, v: u32) { + self.extend_from_slice(&v.to_le_bytes()) + } + + fn put8(&mut self, v: u64) { + self.extend_from_slice(&v.to_le_bytes()) + } +} diff --git a/cranelift/codegen/src/isa/x64/inst/encoding/rex.rs b/cranelift/codegen/src/isa/x64/inst/encoding/rex.rs index 923f2e3615..648941790e 100644 --- a/cranelift/codegen/src/isa/x64/inst/encoding/rex.rs +++ b/cranelift/codegen/src/isa/x64/inst/encoding/rex.rs @@ -153,9 +153,37 @@ impl From<(OperandSize, Reg)> for RexFlags { } } +/// Allows using the same opcode byte in different "opcode maps" to allow for more instruction +/// encodings. See appendix A in the Intel Software Developer's Manual, volume 2A, for more details. +pub enum OpcodeMap { + None, + _0F, + _0F38, + _0F3A, +} + +impl OpcodeMap { + /// Normally the opcode map is specified as bytes in the instruction, but some x64 encoding + /// formats pack this information as bits in a prefix (e.g. EVEX). + pub(crate) fn bits(&self) -> u8 { + match self { + OpcodeMap::None => 0b00, + OpcodeMap::_0F => 0b01, + OpcodeMap::_0F38 => 0b10, + OpcodeMap::_0F3A => 0b11, + } + } +} + +impl Default for OpcodeMap { + fn default() -> Self { + Self::None + } +} + /// We may need to include one or more legacy prefix bytes before the REX prefix. This enum /// covers only the small set of possibilities that we actually need. -pub(crate) enum LegacyPrefixes { +pub enum LegacyPrefixes { /// No prefix bytes. None, /// Operand Size Override -- here, denoting "16-bit operation". @@ -173,26 +201,47 @@ pub(crate) enum LegacyPrefixes { } impl LegacyPrefixes { + /// Emit the legacy prefix as bytes (e.g. in REX instructions). #[inline(always)] pub(crate) fn emit(&self, sink: &mut MachBuffer) { match self { - LegacyPrefixes::_66 => sink.put1(0x66), - LegacyPrefixes::_F0 => sink.put1(0xF0), - LegacyPrefixes::_66F0 => { + Self::_66 => sink.put1(0x66), + Self::_F0 => sink.put1(0xF0), + Self::_66F0 => { // I don't think the order matters, but in any case, this is the same order that // the GNU assembler uses. sink.put1(0x66); sink.put1(0xF0); } - LegacyPrefixes::_F2 => sink.put1(0xF2), - LegacyPrefixes::_F3 => sink.put1(0xF3), - LegacyPrefixes::_66F3 => { + Self::_F2 => sink.put1(0xF2), + Self::_F3 => sink.put1(0xF3), + Self::_66F3 => { sink.put1(0x66); sink.put1(0xF3); } - LegacyPrefixes::None => (), + Self::None => (), } } + + /// Emit the legacy prefix as bits (e.g. for EVEX instructions). + #[inline(always)] + pub(crate) fn bits(&self) -> u8 { + match self { + Self::None => 0b00, + Self::_66 => 0b01, + Self::_F3 => 0b10, + Self::_F2 => 0b11, + _ => panic!( + "VEX and EVEX bits can only be extracted from single prefixes: None, 66, F3, F2" + ), + } + } +} + +impl Default for LegacyPrefixes { + fn default() -> Self { + Self::None + } } /// This is the core 'emit' function for instructions that reference memory. diff --git a/cranelift/codegen/src/isa/x64/inst/encoding/vex.rs b/cranelift/codegen/src/isa/x64/inst/encoding/vex.rs new file mode 100644 index 0000000000..f2f3feebba --- /dev/null +++ b/cranelift/codegen/src/isa/x64/inst/encoding/vex.rs @@ -0,0 +1,2 @@ +//! Encodes VEX instructions. These instructions are those added by the Advanced Vector Extensions +//! (AVX). diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 0e8b8d9f17..cfb0351bf3 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -225,6 +225,12 @@ pub enum Inst { dst: Writable, }, + XmmUnaryRmREvex { + op: Avx512Opcode, + src: RegMem, + dst: Writable, + }, + /// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq XmmMovRM { op: SseOpcode, @@ -571,6 +577,8 @@ impl Inst { | Inst::XmmRmRImm { op, .. } | Inst::XmmToGpr { op, .. } | Inst::XmmUnaryRmR { op, .. } => smallvec![op.available_from()], + + Inst::XmmUnaryRmREvex { op, .. } => op.available_from(), } } } @@ -705,6 +713,12 @@ impl Inst { Inst::XmmUnaryRmR { op, src, dst } } + pub(crate) fn xmm_unary_rm_r_evex(op: Avx512Opcode, src: RegMem, dst: Writable) -> Inst { + src.assert_regclass_is(RegClass::V128); + debug_assert!(dst.to_reg().get_class() == RegClass::V128); + Inst::XmmUnaryRmREvex { op, src, dst } + } + pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); @@ -1391,6 +1405,13 @@ impl PrettyPrint for Inst { show_ireg_sized(dst.to_reg(), mb_rru, 8), ), + Inst::XmmUnaryRmREvex { op, src, dst, .. } => format!( + "{} {}, {}", + ljustify(op.to_string()), + src.show_rru_sized(mb_rru, 8), + show_ireg_sized(dst.to_reg(), mb_rru, 8), + ), + Inst::XmmMovRM { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), @@ -1863,7 +1884,9 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(Writable::from_reg(regs::rdx())); } }, - Inst::UnaryRmR { src, dst, .. } | Inst::XmmUnaryRmR { src, dst, .. } => { + Inst::UnaryRmR { src, dst, .. } + | Inst::XmmUnaryRmR { src, dst, .. } + | Inst::XmmUnaryRmREvex { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); } @@ -2210,6 +2233,11 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut dst, .. } + | Inst::XmmUnaryRmREvex { + ref mut src, + ref mut dst, + .. + } | Inst::UnaryRmR { ref mut src, ref mut dst, diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 26c0b89740..6f675b9232 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1855,25 +1855,29 @@ fn lower_insn_to_regs>( let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); let ty = ty.unwrap(); if ty == types::I64X2 { - // This lowering could be a single instruction with AVX512F/VL's VPABSQ instruction. - // Instead, we use a separate register, `tmp`, to contain the results of `0 - src` - // and then blend in those results with `BLENDVPD` if the MSB of `tmp` was set to 1 - // (i.e. if `tmp` was negative or, conversely, if `src` was originally positive). + if isa_flags.use_avx512f_simd() || isa_flags.use_avx512vl_simd() { + ctx.emit(Inst::xmm_unary_rm_r_evex(Avx512Opcode::Vpabsq, src, dst)); + } else { + // If `VPABSQ` from AVX512 is unavailable, we use a separate register, `tmp`, to + // contain the results of `0 - src` and then blend in those results with + // `BLENDVPD` if the MSB of `tmp` was set to 1 (i.e. if `tmp` was negative or, + // conversely, if `src` was originally positive). - // Emit all 0s into the `tmp` register. - let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp), tmp)); - // Subtract the lanes from 0 and set up `dst`. - ctx.emit(Inst::xmm_rm_r(SseOpcode::Psubq, src.clone(), tmp)); - ctx.emit(Inst::gen_move(dst, tmp.to_reg(), ty)); - // Choose the subtracted lanes when `tmp` has an MSB of 1. BLENDVPD's semantics - // require the "choice" mask to be in XMM0. - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::xmm0()), - tmp.to_reg(), - ty, - )); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Blendvpd, src, dst)); + // Emit all 0s into the `tmp` register. + let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp), tmp)); + // Subtract the lanes from 0 and set up `dst`. + ctx.emit(Inst::xmm_rm_r(SseOpcode::Psubq, src.clone(), tmp)); + ctx.emit(Inst::gen_move(dst, tmp.to_reg(), ty)); + // Choose the subtracted lanes when `tmp` has an MSB of 1. BLENDVPD's semantics + // require the "choice" mask to be in XMM0. + ctx.emit(Inst::gen_move( + Writable::from_reg(regs::xmm0()), + tmp.to_reg(), + ty, + )); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Blendvpd, src, dst)); + } } else if ty.is_vector() { let opcode = match ty { types::I8X16 => SseOpcode::Pabsb, From 10efe8e780c9276e79a0bb8f8afb293e67b8f846 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Thu, 15 Apr 2021 20:35:42 +0200 Subject: [PATCH 08/43] cranelift: Fix spillslot regression on big-endian platforms PR 2840 changed the store_spillslot routine to always store integer registers in full word size to a spill slot. However, the load_spillslot routine was not updated, which may causes the contents to be reloaded in a different type. On big-endian systems this will fetch wrong data. Fixed by using the same type override in load_spillslot. --- cranelift/codegen/src/machinst/abi_impl.rs | 9 +++++++++ .../filetests/isa/x64/store-stack-full-width-i32.clif | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 925ee8bf6d..56ebec48f0 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -1209,6 +1209,15 @@ impl ABICallee for ABICalleeImpl { let spill_off = islot * M::word_bytes() as i64; let sp_off = self.stackslots_size as i64 + spill_off; trace!("load_spillslot: slot {:?} -> sp_off {}", slot, sp_off); + + // Integer types smaller than word size have been spilled as words below, + // and therefore must be reloaded in the same type. + let ty = if ty.is_int() && ty.bytes() < M::word_bytes() { + M::word_type() + } else { + ty + }; + gen_load_stack_multi::(StackAMode::NominalSPOffset(sp_off, ty), into_regs, ty) } diff --git a/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif b/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif index 10b3102c17..31edd7bdca 100644 --- a/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif +++ b/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif @@ -51,7 +51,7 @@ block0(v0: i32, v1: i32, v2: i32): ;; This should be movq below, not movl. ; nextln: movq %rsi, rsp(0 + virtual offset) -; nextln: movslq rsp(0 + virtual offset), %rsi +; nextln: movq rsp(0 + virtual offset), %rsi ; nextln: addl %edi, %esi ;; Put an effectful instruction so that the live-ranges of the adds and From 50aa6457692fcb319e578ddceaef26536548824e Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 23 Mar 2021 20:09:06 +0100 Subject: [PATCH 09/43] cranelift: use a deferred display wrapper for logging the vcode's IR --- cranelift/codegen/src/lib.rs | 1 + cranelift/codegen/src/log.rs | 39 +++++++++++++++++++++++ cranelift/codegen/src/machinst/compile.rs | 28 +++++++--------- 3 files changed, 52 insertions(+), 16 deletions(-) create mode 100644 cranelift/codegen/src/log.rs diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 5b80073b7f..331c8f81b7 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -97,6 +97,7 @@ mod inst_predicates; mod iterators; mod legalizer; mod licm; +mod log; mod nan_canonicalization; mod partition_slice; mod postopt; diff --git a/cranelift/codegen/src/log.rs b/cranelift/codegen/src/log.rs new file mode 100644 index 0000000000..c5bd59aa58 --- /dev/null +++ b/cranelift/codegen/src/log.rs @@ -0,0 +1,39 @@ +//! This module implements deferred display helpers. +//! +//! These are particularly useful in logging contexts, where the maximum logging level filter might +//! be enabled, but we don't want the arguments to be evaluated early: +//! +//! ``` +//! log::set_max_level(log::LevelFilter::max()); +//! fn expensive_calculation() -> String { +//! "a string that is very slow to generate".into() +//! } +//! log::debug!("{}", expensive_calculation()); +//! ``` +//! +//! If the associated log implementation filters out log debug entries, the expensive calculation +//! would have been spurious. In this case, we can wrap the expensive computation within an +//! `DeferredDisplay`, so that the computation only happens when the actual `fmt` function is +//! called. + +use core::fmt; + +pub(crate) struct DeferredDisplay(F); + +impl T, T: fmt::Display> DeferredDisplay { + pub(crate) fn new(f: F) -> Self { + Self(f) + } +} + +impl T, T: fmt::Display> fmt::Display for DeferredDisplay { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0().fmt(f) + } +} + +impl T, T: fmt::Debug> fmt::Debug for DeferredDisplay { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0().fmt(f) + } +} diff --git a/cranelift/codegen/src/machinst/compile.rs b/cranelift/codegen/src/machinst/compile.rs index 0aaee5e98f..2dfbb85785 100644 --- a/cranelift/codegen/src/machinst/compile.rs +++ b/cranelift/codegen/src/machinst/compile.rs @@ -1,11 +1,12 @@ //! Compilation backend pipeline: optimized IR to VCode / binemit. use crate::ir::Function; +use crate::log::DeferredDisplay; use crate::machinst::*; use crate::settings; use crate::timing; -use log::{debug, log_enabled, Level}; +use log::debug; use regalloc::{allocate_registers_with_opts, Algorithm, Options, PrettyPrint}; /// Compile the given function down to VCode with allocated registers, ready @@ -29,15 +30,12 @@ where lower.lower(b)? }; - // Creating the vcode string representation may be costly for large functions, so don't do it - // if the Debug level hasn't been statically (through features) or dynamically (through - // RUST_LOG) enabled. - if log_enabled!(Level::Debug) { - debug!( - "vcode from lowering: \n{}", - vcode.show_rru(Some(b.reg_universe())) - ); - } + // Creating the vcode string representation may be costly for large functions, so defer its + // rendering. + debug!( + "vcode from lowering: \n{}", + DeferredDisplay::new(|| vcode.show_rru(Some(b.reg_universe()))) + ); // Perform register allocation. let (run_checker, algorithm) = match vcode.flags().regalloc() { @@ -106,12 +104,10 @@ where vcode.replace_insns_from_regalloc(result); } - if log_enabled!(Level::Debug) { - debug!( - "vcode after regalloc: final version:\n{}", - vcode.show_rru(Some(b.reg_universe())) - ); - } + debug!( + "vcode after regalloc: final version:\n{}", + DeferredDisplay::new(|| vcode.show_rru(Some(b.reg_universe()))) + ); Ok(vcode) } From 8ab3511b3b0d9a7e0706a617c6b79d71f1a93775 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Fri, 16 Apr 2021 18:05:49 +0200 Subject: [PATCH 10/43] Generate unwind information on Win64 with the old backend Following the new ABI introduced for efficient support of multiple return values, the old-backend test for generating unwind information was incomplete, resulting in no unwind information being generated and traps not being correctly caught by the runtime. --- cranelift/codegen/src/isa/x86/unwind/winx64.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cranelift/codegen/src/isa/x86/unwind/winx64.rs b/cranelift/codegen/src/isa/x86/unwind/winx64.rs index b2da0bc8b9..33e5463bb8 100644 --- a/cranelift/codegen/src/isa/x86/unwind/winx64.rs +++ b/cranelift/codegen/src/isa/x86/unwind/winx64.rs @@ -2,7 +2,7 @@ use crate::ir::Function; use crate::isa::x86::registers::{FPR, GPR}; -use crate::isa::{unwind::winx64::UnwindInfo, CallConv, RegUnit, TargetIsa}; +use crate::isa::{unwind::winx64::UnwindInfo, RegUnit, TargetIsa}; use crate::result::CodegenResult; pub(crate) fn create_unwind_info( @@ -10,7 +10,7 @@ pub(crate) fn create_unwind_info( isa: &dyn TargetIsa, ) -> CodegenResult> { // Only Windows fastcall is supported for unwind information - if func.signature.call_conv != CallConv::WindowsFastcall || func.prologue_end.is_none() { + if !func.signature.call_conv.extends_windows_fastcall() || func.prologue_end.is_none() { return Ok(None); } From ba73b458b87ec15f7844d4d6a4bea3612038277c Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Fri, 16 Apr 2021 18:15:35 +0200 Subject: [PATCH 11/43] Introduce a new API that allows notifying that a Store has moved to a new thread (#2822) * Introduce a new API that allows notifying that a Store has moved to a new thread * Add backlink to documentation, and mention the new API in the multithreading doc; --- crates/wasmtime/src/store.rs | 19 ++++++++++++++ docs/examples-rust-multithreading.md | 15 +++++++---- tests/all/traps.rs | 39 ++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 18dbde303c..4c78e82c8f 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -498,6 +498,25 @@ impl Store { &self.inner.frame_info } + /// Notifies that the current Store (and all referenced entities) has been moved over to a + /// different thread. + /// + /// See also the multithreading documentation for more details: + /// . + /// + /// # Safety + /// + /// In general, it's not possible to move a `Store` to a different thread, because it isn't `Send`. + /// That being said, it is possible to create an unsafe `Send` wrapper over a `Store`, assuming + /// the safety guidelines exposed in the multithreading documentation have been applied. So it + /// is in general unnecessary to do this, if you're not doing unsafe things. + /// + /// It is fine to call this several times: only the first call will have an effect. + pub unsafe fn notify_switched_thread(&self) { + wasmtime_runtime::init_traps(frame_info::GlobalFrameInfo::is_wasm_pc) + .expect("failed to initialize per-threads traps"); + } + /// Perform garbage collection of `ExternRef`s. pub fn gc(&self) { // For this crate's API, we ensure that `set_stack_canary` invariants diff --git a/docs/examples-rust-multithreading.md b/docs/examples-rust-multithreading.md index 5d3fdcac2c..d99977759f 100644 --- a/docs/examples-rust-multithreading.md +++ b/docs/examples-rust-multithreading.md @@ -129,11 +129,16 @@ some possibilities include: `Store::set` or `Func::wrap`) implement the `Send` trait. If these requirements are met it is technically safe to move a store and its - objects between threads. The reason that this strategy isn't recommended, - however, is that you will receive no assistance from the Rust compiler in - verifying that the transfer across threads is indeed actually safe. This will - require auditing your embedding of Wasmtime itself to ensure it meets these - requirements. + objects between threads. When you move a store to another thread, it is + required that you run the `Store::notify_switched_thread()` method after the + store has landed on the new thread, so that per-thread initialization is + correctly re-run. Failure to do so may cause wasm traps to crash the whole + application. + + The reason that this strategy isn't recommended, however, is that you will + receive no assistance from the Rust compiler in verifying that the transfer + across threads is indeed actually safe. This will require auditing your + embedding of Wasmtime itself to ensure it meets these requirements. It's important to note that the requirements here also apply to the futures returned from `Func::call_async`. These futures are not `Send` due to them diff --git a/tests/all/traps.rs b/tests/all/traps.rs index 4fcc6f3882..e409b98189 100644 --- a/tests/all/traps.rs +++ b/tests/all/traps.rs @@ -591,3 +591,42 @@ note: run with `WASMTIME_BACKTRACE_DETAILS=1` environment variable to display mo ); Ok(()) } + +#[test] +fn multithreaded_traps() -> Result<()> { + // Compile and run unreachable on a thread, then moves over the whole store to another thread, + // and make sure traps are still correctly caught after notifying the store of the move. + let instance = { + let store = Store::default(); + let module = Module::new( + store.engine(), + r#"(module (func (export "run") unreachable))"#, + )?; + Instance::new(&store, &module, &[])? + }; + + assert!(instance.get_typed_func::<(), ()>("run")?.call(()).is_err()); + + struct SendInstance { + inner: Instance, + } + unsafe impl Send for SendInstance {} + + let instance = SendInstance { inner: instance }; + + let handle = std::thread::spawn(move || { + let instance = instance.inner; + unsafe { + instance.store().notify_switched_thread(); + } + assert!(instance + .get_typed_func::<(), ()>("run") + .unwrap() + .call(()) + .is_err()); + }); + + handle.join().expect("couldn't join thread"); + + Ok(()) +} From a2466b3c232f82a0e13a18e1ed9ddd9ad677bd86 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Sun, 11 Apr 2021 00:37:27 -0700 Subject: [PATCH 12/43] Move the signature registry into `Engine`. This commit moves the shared signature registry out of `Store` and into `Engine`. This helps eliminate work that was performed whenever a `Module` was instantiated into a `Store`. Now a `Module` is registered with the shared signature registry upon creation, storing the mapping from the module's signature index space to the shared index space. This also refactors the "frame info" registry into a general purpose "module registry" that is used to look up trap information, signature information, and (soon) stack map information. --- crates/runtime/src/externref.rs | 2 +- crates/runtime/src/traphandlers/macos.rs | 2 +- crates/wasmtime/src/config.rs | 8 + crates/wasmtime/src/engine.rs | 100 ++++++- crates/wasmtime/src/func.rs | 111 ++++---- crates/wasmtime/src/instance.rs | 5 +- crates/wasmtime/src/lib.rs | 6 +- crates/wasmtime/src/module.rs | 60 ++++- .../src/{frame_info.rs => module/registry.rs} | 225 ++++++++-------- crates/wasmtime/src/module/serialization.rs | 243 +++++++++++------- crates/wasmtime/src/sig_registry.rs | 155 ----------- crates/wasmtime/src/signatures.rs | 185 +++++++++++++ crates/wasmtime/src/store.rs | 152 +++++------ crates/wasmtime/src/trampoline/func.rs | 10 +- crates/wasmtime/src/trap.rs | 4 +- crates/wasmtime/src/types.rs | 7 +- crates/wasmtime/src/types/matching.rs | 42 +-- 17 files changed, 768 insertions(+), 549 deletions(-) rename crates/wasmtime/src/{frame_info.rs => module/registry.rs} (78%) delete mode 100644 crates/wasmtime/src/sig_registry.rs create mode 100644 crates/wasmtime/src/signatures.rs diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index cefce0d507..02943c1c01 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -753,7 +753,7 @@ pub struct StackMapRegistry { struct StackMapRegistryInner { /// A map from the highest pc in a module, to its stack maps. /// - /// For details, see the comment above `GlobalFrameInfo::ranges`. + /// For details, see the comment above `GlobalModuleRegistry`. ranges: BTreeMap, } diff --git a/crates/runtime/src/traphandlers/macos.rs b/crates/runtime/src/traphandlers/macos.rs index 0f5e593012..2f92167526 100644 --- a/crates/runtime/src/traphandlers/macos.rs +++ b/crates/runtime/src/traphandlers/macos.rs @@ -25,7 +25,7 @@ //! use a thread-local to store information about how to unwind. Additionally //! this requires that the check of whether a pc is a wasm trap or not is a //! global check rather than a per-thread check. This necessitates the existence -//! of `GlobalFrameInfo` in the `wasmtime` crate. +//! of `GlobalModuleRegistry` in the `wasmtime` crate. //! //! Otherwise this file heavily uses the `mach` Rust crate for type and //! function declarations. Many bits and pieces are copied or translated from diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 43bd809c84..8351fea426 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -319,6 +319,10 @@ impl HostFuncMap { fn async_required(&self) -> bool { self.funcs.values().any(|f| f.1) } + + fn iter(&self) -> impl Iterator { + self.funcs.values().map(|v| &*v.0) + } } macro_rules! generate_wrap_async_host_func { @@ -1318,6 +1322,10 @@ impl Config { for_each_function_signature!(generate_wrap_async_host_func); + pub(crate) fn host_funcs(&self) -> impl Iterator { + self.host_funcs.iter() + } + pub(crate) fn get_host_func(&self, module: &str, name: &str) -> Option<&HostFunc> { self.host_funcs.get(module, name) } diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index f7d9c4eaa4..54d8769d19 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -1,10 +1,31 @@ +use crate::signatures::{SharedSignatures, SignatureRegistry, TrampolineMap}; use crate::Config; use anyhow::Result; -use std::sync::Arc; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; #[cfg(feature = "cache")] use wasmtime_cache::CacheConfig; +use wasmtime_environ::{ + entity::PrimaryMap, + wasm::{SignatureIndex, WasmFuncType}, +}; use wasmtime_jit::Compiler; -use wasmtime_runtime::{debug_builtins, InstanceAllocator}; +use wasmtime_runtime::{ + debug_builtins, InstanceAllocator, InstanceHandle, VMCallerCheckedAnyfunc, + VMSharedSignatureIndex, VMTrampoline, +}; + +#[derive(Default)] +struct EngineHostFuncs { + anyfuncs: HashMap>, + trampolines: TrampolineMap, +} + +// This is safe for send and sync as it is read-only once the +// engine is constructed and the host functions live with the config, +// which the engine keeps a strong reference to. +unsafe impl Send for EngineHostFuncs {} +unsafe impl Sync for EngineHostFuncs {} /// An `Engine` which is a global context for compilation and management of wasm /// modules. @@ -37,6 +58,16 @@ struct EngineInner { config: Config, compiler: Compiler, allocator: Box, + signatures: RwLock, + host_funcs: EngineHostFuncs, +} + +impl Drop for EngineInner { + fn drop(&mut self) { + let mut signatures = self.signatures.write().unwrap(); + signatures.unregister(self.host_funcs.trampolines.indexes()); + assert!(signatures.is_empty()); + } } impl Engine { @@ -46,11 +77,29 @@ impl Engine { debug_builtins::ensure_exported(); config.validate()?; let allocator = config.build_allocator()?; + let mut signatures = SignatureRegistry::default(); + let mut host_funcs = EngineHostFuncs::default(); + + // Register all the host function signatures + for func in config.host_funcs() { + let sig = signatures.register(func.ty.as_wasm_func_type()); + + // Cloning the instance handle is safe as host functions outlive the engine + host_funcs.anyfuncs.insert( + unsafe { func.instance.clone() }, + Box::new(func.anyfunc(sig)), + ); + + host_funcs.trampolines.insert(sig, func.trampoline); + } + Ok(Engine { inner: Arc::new(EngineInner { config: config.clone(), compiler: config.build_compiler(allocator.as_ref()), allocator, + signatures: RwLock::new(signatures), + host_funcs, }), }) } @@ -79,6 +128,53 @@ impl Engine { Arc::ptr_eq(&a.inner, &b.inner) } + pub(crate) fn register_module_signatures( + &self, + signatures: &PrimaryMap, + trampolines: impl Iterator, + ) -> (SharedSignatures, TrampolineMap) { + self.inner + .signatures + .write() + .unwrap() + .register_module(signatures, trampolines) + } + + pub(crate) fn register_signature(&self, ty: &WasmFuncType) -> VMSharedSignatureIndex { + self.inner.signatures.write().unwrap().register(ty) + } + + pub(crate) fn unregister_signatures( + &self, + indexes: impl Iterator, + ) { + self.inner.signatures.write().unwrap().unregister(indexes); + } + + pub(crate) fn lookup_func_type(&self, index: VMSharedSignatureIndex) -> Option { + self.inner + .signatures + .read() + .unwrap() + .lookup_type(index) + .cloned() + } + + pub(crate) fn host_func_trampolines(&self) -> &TrampolineMap { + &self.inner.host_funcs.trampolines + } + + pub(crate) fn host_func_anyfunc( + &self, + instance: &InstanceHandle, + ) -> Option<&VMCallerCheckedAnyfunc> { + self.inner + .host_funcs + .anyfuncs + .get(instance) + .map(AsRef::as_ref) + } + /// Ahead-of-time (AOT) compiles a WebAssembly module. /// /// The `bytes` provided must be in one of two formats: diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 82805e2c11..6bb24597e1 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -1,4 +1,4 @@ -use crate::{sig_registry::SignatureRegistry, trampoline::StoreInstanceHandle}; +use crate::trampoline::StoreInstanceHandle; use crate::{Config, Extern, FuncType, Store, Trap, Val, ValType}; use anyhow::{bail, Context as _, Result}; use smallvec::{smallvec, SmallVec}; @@ -22,9 +22,9 @@ use wasmtime_runtime::{ /// This differs from `Func` in that it is not associated with a `Store`. /// Host functions are associated with a `Config`. pub(crate) struct HostFunc { - ty: FuncType, - instance: InstanceHandle, - trampoline: VMTrampoline, + pub ty: FuncType, + pub instance: InstanceHandle, + pub trampoline: VMTrampoline, } impl HostFunc { @@ -73,6 +73,23 @@ impl HostFunc { } } + /// Gets a caller-checked anyfunc for this host function given a shared signature index. + /// + /// The shared signature index must have been registered for the signature of + /// this host function. + pub fn anyfunc(&self, sig: VMSharedSignatureIndex) -> VMCallerCheckedAnyfunc { + let mut anyfunc = match self + .instance + .lookup_by_declaration(&EntityIndex::Function(FuncIndex::from_u32(0))) + { + wasmtime_runtime::Export::Function(f) => unsafe { f.anyfunc.as_ref() }.clone(), + _ => unreachable!(), + }; + + anyfunc.type_index = sig; + anyfunc + } + /// Converts a `HostFunc` to a `Func`. /// /// # Safety @@ -88,11 +105,11 @@ impl HostFunc { }; let export = ExportFunction { - anyfunc: std::ptr::NonNull::new_unchecked(store.get_host_anyfunc( - &self.instance, - &self.ty, - self.trampoline, - )), + anyfunc: store + .engine() + .host_func_anyfunc(&self.instance) + .unwrap() + .into(), }; Func { @@ -408,13 +425,9 @@ impl Func { Func::invoke(&store, &ty_clone, caller_vmctx, values_vec, &func) }); - let (instance, trampoline) = crate::trampoline::create_function( - &ty, - func, - store.engine().config(), - Some(&mut store.signatures().borrow_mut()), - ) - .expect("failed to create function"); + let (instance, trampoline) = + crate::trampoline::create_function(&ty, func, store.engine().config(), Some(store)) + .expect("failed to create function"); let idx = EntityIndex::Function(FuncIndex::from_u32(0)); let (instance, export) = match instance.lookup_by_declaration(&idx) { @@ -734,7 +747,7 @@ impl Func { /// # } /// ``` pub fn wrap(store: &Store, func: impl IntoFunc) -> Func { - let (_, instance, trampoline) = func.into_func(Some(&mut store.signatures().borrow_mut())); + let (_, instance, trampoline) = func.into_func(Some(store)); let (instance, export) = unsafe { let idx = EntityIndex::Function(FuncIndex::from_u32(0)); @@ -759,33 +772,35 @@ impl Func { /// Returns the underlying wasm type that this `Func` has. pub fn ty(&self) -> FuncType { - // Signatures should always be registered in the store's registry of + // Signatures should always be registered in the engine's registry of // shared signatures, so we should be able to unwrap safely here. - let signatures = self.instance.store.signatures().borrow(); - let (wft, _) = signatures - .lookup_shared(self.sig_index()) - .expect("signature should be registered"); - - // This is only called with `Export::Function`, and since it's coming - // from wasmtime_runtime itself we should support all the types coming - // out of it, so assert such here. - FuncType::from_wasm_func_type(&wft) + FuncType::from_wasm_func_type( + self.instance + .store + .engine() + .lookup_func_type(self.sig_index()) + .expect("signature should be registered"), + ) } /// Returns the number of parameters that this function takes. pub fn param_arity(&self) -> usize { - let signatures = self.instance.store.signatures().borrow(); - let (sig, _) = signatures - .lookup_shared(self.sig_index()) + let sig = self + .instance + .store + .engine() + .lookup_func_type(self.sig_index()) .expect("signature should be registered"); sig.params.len() } /// Returns the number of results this function produces. pub fn result_arity(&self) -> usize { - let signatures = self.instance.store.signatures().borrow(); - let (sig, _) = signatures - .lookup_shared(self.sig_index()) + let sig = self + .instance + .store + .engine() + .lookup_func_type(self.sig_index()) .expect("signature should be registered"); sig.returns.len() } @@ -907,21 +922,12 @@ impl Func { } pub(crate) unsafe fn from_wasmtime_function(export: &ExportFunction, store: &Store) -> Self { - // Each function signature in a module should have a trampoline stored - // on that module as well, so unwrap the result here since otherwise - // it's a bug in wasmtime. let anyfunc = export.anyfunc.as_ref(); - let trampoline = store - .signatures() - .borrow() - .lookup_shared(anyfunc.type_index) - .expect("failed to retrieve trampoline from module") - .1; Func { instance: store.existing_vmctx(anyfunc.vmctx), export: export.clone(), - trampoline, + trampoline: store.lookup_trampoline(anyfunc.type_index), } } @@ -1542,10 +1548,7 @@ for_each_function_signature!(impl_host_abi); /// as an implementation detail of this crate. pub trait IntoFunc { #[doc(hidden)] - fn into_func( - self, - registry: Option<&mut SignatureRegistry>, - ) -> (FuncType, InstanceHandle, VMTrampoline); + fn into_func(self, store: Option<&Store>) -> (FuncType, InstanceHandle, VMTrampoline); } /// A structure representing the *caller's* context when creating a function @@ -1658,12 +1661,12 @@ macro_rules! impl_into_func { $($args: WasmTy,)* R: WasmRet, { - fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { + fn into_func(self, store: Option<&Store>) -> (FuncType, InstanceHandle, VMTrampoline) { let f = move |_: Caller<'_>, $($args:$args),*| { self($($args),*) }; - f.into_func(registry) + f.into_func(store) } } @@ -1674,7 +1677,7 @@ macro_rules! impl_into_func { $($args: WasmTy,)* R: WasmRet, { - fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { + fn into_func(self, store: Option<&Store>) -> (FuncType, InstanceHandle, VMTrampoline) { /// This shim is called by Wasm code, constructs a `Caller`, /// calls the wrapped host function, and returns the translated /// result back to Wasm. @@ -1807,10 +1810,10 @@ macro_rules! impl_into_func { let trampoline = host_trampoline::<$($args,)* R>; - // If not given a registry, use a default signature index that is guaranteed to trap - // if the function is called indirectly without first being associated with a store (a bug condition). - let shared_signature_id = registry - .map(|r| r.register(ty.as_wasm_func_type(), trampoline)) + // If not given a store, use a default signature index that is guaranteed to trap. + // If the function is called indirectly without first being associated with a store (a bug condition). + let shared_signature_id = store + .map(|s| s.register_signature(ty.as_wasm_func_type(), trampoline)) .unwrap_or(VMSharedSignatureIndex::default()); let instance = unsafe { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 01765d02bd..00cc90fd82 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -362,6 +362,7 @@ impl<'a> Instantiator<'a> { let expected_ty = self.cur.module.compiled_module().module().type_of(*index); matching::MatchCx { + signatures: self.cur.module.signatures(), types: self.cur.module.types(), store: self.store, } @@ -513,14 +514,12 @@ impl<'a> Instantiator<'a> { unsafe { let engine = self.store.engine(); let allocator = engine.allocator(); - let signatures = self.store.signatures().borrow(); - let signatures = signatures.lookup_table(&self.cur.module); let instance = allocator.allocate(InstanceAllocationRequest { module: compiled_module.module().clone(), finished_functions: compiled_module.finished_functions(), imports: self.cur.build(), - shared_signatures: (&signatures).into(), + shared_signatures: self.cur.module.signatures().into(), host_state: Box::new(()), interrupts: self.store.interrupts(), externref_activations_table: self.store.externref_activations_table() diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index d843b9db43..98a86a279e 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -282,13 +282,12 @@ mod func; mod config; mod engine; mod externals; -mod frame_info; mod instance; mod linker; mod memory; mod module; mod r#ref; -mod sig_registry; +mod signatures; mod store; mod trampoline; mod trap; @@ -298,12 +297,11 @@ mod values; pub use crate::config::*; pub use crate::engine::*; pub use crate::externals::*; -pub use crate::frame_info::{FrameInfo, FrameSymbol}; pub use crate::func::*; pub use crate::instance::Instance; pub use crate::linker::*; pub use crate::memory::*; -pub use crate::module::Module; +pub use crate::module::{FrameInfo, FrameSymbol, Module}; pub use crate::r#ref::ExternRef; pub use crate::store::*; pub use crate::trap::*; diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 8b460207cf..7e1db3c876 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -1,4 +1,7 @@ -use crate::types::{ExportType, ExternType, ImportType}; +use crate::{ + signatures::{SharedSignatures, TrampolineMap}, + types::{ExportType, ExternType, ImportType}, +}; use crate::{Engine, ModuleType}; use anyhow::{bail, Context, Result}; use std::fs; @@ -8,13 +11,36 @@ use wasmparser::Validator; #[cfg(feature = "cache")] use wasmtime_cache::ModuleCacheEntry; use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::wasm::ModuleIndex; +use wasmtime_environ::wasm::{ModuleIndex, SignatureIndex}; use wasmtime_jit::{CompilationArtifacts, CompiledModule, TypeTables}; +use wasmtime_runtime::VMSharedSignatureIndex; +mod registry; mod serialization; +pub use registry::{FrameInfo, FrameSymbol, GlobalModuleRegistry, ModuleRegistry}; pub use serialization::SerializedModule; +// A wrapper around registered signatures and trampolines that will automatically +/// unregister the signatures when dropped. +pub(crate) struct ModuleSharedSignatures { + engine: Engine, + signatures: SharedSignatures, + trampolines: TrampolineMap, +} + +impl Drop for ModuleSharedSignatures { + fn drop(&mut self) { + if !self.signatures.is_empty() { + // Use the shared signatures map to unregister as not every registered + // signature will have a trampoline, but every index in the trampoline map + // will be present in the shared signatures map. + self.engine + .unregister_signatures(self.signatures.values().cloned()); + } + } +} + /// A compiled WebAssembly module, ready to be instantiated. /// /// A `Module` is a compiled in-memory representation of an input WebAssembly @@ -102,6 +128,8 @@ struct ModuleInner { /// Type information of this module and all `artifact_upvars` compiled /// modules. types: Arc, + /// Registered shared signature for the module. + signatures: Arc, } impl Module { @@ -318,10 +346,16 @@ impl Module { engine.compiler().isa(), &*engine.config().profiler, )?; - let module = modules.remove(main_module); // Validate the module can be used with the current allocator - engine.allocator().validate(module.module())?; + engine.allocator().validate(modules[main_module].module())?; + + let (signatures, trampolines) = engine.register_module_signatures( + &types.wasm_signatures, + modules.iter().flat_map(|m| m.trampolines().iter().cloned()), + ); + + let module = modules.remove(main_module); Ok(Module { inner: Arc::new(ModuleInner { @@ -330,6 +364,11 @@ impl Module { types: Arc::new(types), artifact_upvars: modules, module_upvars: Vec::new(), + signatures: Arc::new(ModuleSharedSignatures { + engine: engine.clone(), + signatures, + trampolines, + }), }), }) } @@ -416,8 +455,8 @@ impl Module { ) -> Module { Module { inner: Arc::new(ModuleInner { - types: self.types().clone(), - engine: self.engine().clone(), + types: self.inner.types.clone(), + engine: self.inner.engine.clone(), module: self.inner.artifact_upvars[artifact_index].clone(), artifact_upvars: artifact_upvars .iter() @@ -432,6 +471,7 @@ impl Module { wasmtime_environ::ModuleUpvar::Local(i) => modules[i].clone(), }) .collect(), + signatures: self.inner.signatures.clone(), }), } } @@ -448,6 +488,14 @@ impl Module { &self.inner.types } + pub(crate) fn signatures(&self) -> &PrimaryMap { + &self.inner.signatures.signatures + } + + pub(crate) fn shared_signatures(&self) -> &Arc { + &self.inner.signatures + } + /// Looks up the module upvar value at the `index` specified. /// /// Note that this panics if `index` is out of bounds since this should diff --git a/crates/wasmtime/src/frame_info.rs b/crates/wasmtime/src/module/registry.rs similarity index 78% rename from crates/wasmtime/src/frame_info.rs rename to crates/wasmtime/src/module/registry.rs index 8bd326ebdb..161ef408d6 100644 --- a/crates/wasmtime/src/frame_info.rs +++ b/crates/wasmtime/src/module/registry.rs @@ -1,35 +1,30 @@ -use std::collections::BTreeMap; -use std::sync::Arc; -use std::sync::Mutex; -use wasmtime_environ::entity::EntityRef; -use wasmtime_environ::ir; -use wasmtime_environ::wasm::DefinedFuncIndex; -use wasmtime_environ::{FunctionAddressMap, TrapInformation}; -use wasmtime_jit::CompiledModule; +//! Implements a registry of modules for a store. -/// This is a structure that lives within a `Store` and retains information -/// about all modules registered with the `Store` via instantiation. -/// -/// "frame information" here refers to things like determining whether a -/// program counter is a wasm program counter, and additionally mapping program -/// counters to wasm filenames, modules, line numbers, etc. This store of -/// information lives as long as a `Store` lives since modules are never -/// unloaded today. -#[derive(Default)] -pub struct StoreFrameInfo { - /// An internal map that keeps track of backtrace frame information for - /// each module. - /// - /// This map is morally a map of ranges to a map of information for that - /// module. Each module is expected to reside in a disjoint section of - /// contiguous memory. No modules can overlap. - /// - /// The key of this map is the highest address in the module and the value - /// is the module's information, which also contains the start address. - ranges: BTreeMap, +use crate::{module::ModuleSharedSignatures, Module}; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; +use wasmtime_environ::{ + entity::EntityRef, ir, wasm::DefinedFuncIndex, FunctionAddressMap, TrapInformation, +}; +use wasmtime_jit::CompiledModule; +use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline}; + +lazy_static::lazy_static! { + static ref GLOBAL_MODULES: Mutex = Default::default(); } -impl StoreFrameInfo { +/// Used for registering modules with a store. +/// +/// The map is from the ending (exclusive) address for the module code to +/// the registered module. +/// +/// The `BTreeMap` is used to quickly locate a module based on a program counter value. +#[derive(Default)] +pub struct ModuleRegistry(BTreeMap); + +impl ModuleRegistry { /// Fetches frame information about a program counter in a backtrace. /// /// Returns an object if this `pc` is known to some previously registered @@ -48,8 +43,8 @@ impl StoreFrameInfo { self.module(pc)?.lookup_trap_info(pc) } - fn module(&self, pc: usize) -> Option<&ModuleFrameInfo> { - let (end, info) = self.ranges.range(pc..).next()?; + fn module(&self, pc: usize) -> Option<&RegisteredModule> { + let (end, info) = self.0.range(pc..).next()?; if pc < info.start || *end < pc { return None; } @@ -57,57 +52,78 @@ impl StoreFrameInfo { Some(info) } - /// Registers a new compiled module's frame information. - pub fn register(&mut self, module: &Arc) { - let (start, end) = module.code().range(); + /// Registers a new module with the registry. + pub fn register(&mut self, module: &Module) -> bool { + let compiled_module = module.compiled_module(); + let (start, end) = compiled_module.code().range(); - // Ignore modules with no code or finished functions - if start == end || module.finished_functions().is_empty() { - return; + // Ignore modules with no code, finished functions, or if the module is already registered + if start == end || compiled_module.finished_functions().is_empty() { + return false; } // The module code range is exclusive for end, so make it inclusive as it // may be a valid PC value let end = end - 1; + if self.0.get(&end).is_some() { + return false; + } + // Assert that this module's code doesn't collide with any other registered modules - if let Some((_, prev)) = self.ranges.range(end..).next() { + if let Some((_, prev)) = self.0.range(end..).next() { assert!(prev.start > end); } - if let Some((prev_end, _)) = self.ranges.range(..=start).next_back() { + + if let Some((prev_end, _)) = self.0.range(..=start).next_back() { assert!(*prev_end < start); } - let prev = self.ranges.insert( + let prev = self.0.insert( end, - ModuleFrameInfo { + RegisteredModule { start, - module: module.clone(), + module: compiled_module.clone(), + signatures: module.shared_signatures().clone(), }, ); assert!(prev.is_none()); - GLOBAL_INFO.lock().unwrap().register(start, end, module); + GLOBAL_MODULES.lock().unwrap().register(start, end, module); + true + } + + /// Looks up a trampoline from a shared signature index. + /// + /// This will search all modules associated with the store for a suitable trampoline + /// given the shared signature index. + pub fn lookup_trampoline(&self, index: VMSharedSignatureIndex) -> Option { + for (_, m) in &self.0 { + if let Some(trampoline) = m.signatures.trampolines.get(index) { + return Some(trampoline); + } + } + + None } } -impl Drop for StoreFrameInfo { +impl Drop for ModuleRegistry { fn drop(&mut self) { - let mut info = GLOBAL_INFO.lock().unwrap(); - for end in self.ranges.keys() { + let mut info = GLOBAL_MODULES.lock().unwrap(); + for end in self.0.keys() { info.unregister(*end); } } } -/// Represents a module's frame information. -#[derive(Clone)] -pub struct ModuleFrameInfo { +struct RegisteredModule { start: usize, module: Arc, + signatures: Arc, } -impl ModuleFrameInfo { +impl RegisteredModule { /// Determines if the related module has unparsed debug information. pub fn has_unparsed_debuginfo(&self) -> bool { self.module.has_unparsed_debuginfo() @@ -214,47 +230,29 @@ impl ModuleFrameInfo { } } -/// This is the dual of `StoreFrameInfo` and is stored globally (as the name -/// implies) rather than simply in one `Store`. +/// This is the global module registry that stores information for all modules +/// that are currently in use by any `Store`. /// /// The purpose of this map is to be called from signal handlers to determine /// whether a program counter is a wasm trap or not. Specifically macOS has /// no contextual information about the thread available, hence the necessity /// for global state rather than using thread local state. /// -/// This is similar to `StoreFrameInfo` except that it has less information and -/// supports removal. Any time anything is registered with a `StoreFrameInfo` -/// it is also automatically registered with the singleton global frame -/// information. When a `StoreFrameInfo` is destroyed then all of its entries -/// are removed from the global frame information. +/// This is similar to `ModuleRegistry` except that it has less information and +/// supports removal. Any time anything is registered with a `ModuleRegistry` +/// it is also automatically registered with the singleton global module +/// registry. When a `ModuleRegistry` is destroyed then all of its entries +/// are removed from the global module registry. #[derive(Default)] -pub struct GlobalFrameInfo { - // The map here behaves the same way as `StoreFrameInfo`. - ranges: BTreeMap, -} +pub struct GlobalModuleRegistry(BTreeMap); -/// This is the equivalent of `ModuleFrameInfo` except it keeps a reference count. -struct GlobalModuleFrameInfo { - module: ModuleFrameInfo, - - /// Note that modules can be instantiated in many stores, so the purpose of - /// this field is to keep track of how many stores have registered a - /// module. Information is only removed from the global store when this - /// reference count reaches 0. - references: usize, -} - -lazy_static::lazy_static! { - static ref GLOBAL_INFO: Mutex = Default::default(); -} - -impl GlobalFrameInfo { +impl GlobalModuleRegistry { /// Returns whether the `pc`, according to globally registered information, /// is a wasm trap or not. pub(crate) fn is_wasm_pc(pc: usize) -> bool { - let info = GLOBAL_INFO.lock().unwrap(); + let info = GLOBAL_MODULES.lock().unwrap(); - match info.ranges.range(pc..).next() { + match info.0.range(pc..).next() { Some((end, info)) => { if pc < info.module.start || *end < pc { return false; @@ -263,7 +261,7 @@ impl GlobalFrameInfo { match info.module.func(pc) { Some((index, offset)) => { let (addr_map, _) = info.module.module.func_info(index); - ModuleFrameInfo::instr_pos(offset, addr_map).is_some() + RegisteredModule::instr_pos(offset, addr_map).is_some() } None => false, } @@ -274,17 +272,15 @@ impl GlobalFrameInfo { /// Registers a new region of code, described by `(start, end)` and with /// the given function information, with the global information. - fn register(&mut self, start: usize, end: usize, module: &Arc) { - let info = self - .ranges - .entry(end) - .or_insert_with(|| GlobalModuleFrameInfo { - module: ModuleFrameInfo { - start, - module: module.clone(), - }, - references: 0, - }); + fn register(&mut self, start: usize, end: usize, module: &Module) { + let info = self.0.entry(end).or_insert_with(|| GlobalRegisteredModule { + module: RegisteredModule { + start, + module: module.compiled_module().clone(), + signatures: module.shared_signatures().clone(), + }, + references: 0, + }); // Note that ideally we'd debug_assert that the information previously // stored, if any, matches the `functions` we were given, but for now we @@ -293,17 +289,28 @@ impl GlobalFrameInfo { info.references += 1; } - /// Unregisters a region of code (keyed by the `end` address) from this + /// Unregisters a region of code (keyed by the `end` address) from the /// global information. fn unregister(&mut self, end: usize) { - let info = self.ranges.get_mut(&end).unwrap(); + let info = self.0.get_mut(&end).unwrap(); info.references -= 1; if info.references == 0 { - self.ranges.remove(&end); + self.0.remove(&end); } } } +/// This is the equivalent of `RegisteredModule` except it keeps a reference count. +struct GlobalRegisteredModule { + module: RegisteredModule, + + /// Note that modules can be instantiated in many stores, so the purpose of + /// this field is to keep track of how many stores have registered a + /// module. Information is only removed from the global registry when this + /// reference count reaches 0. + references: usize, +} + /// Description of a frame in a backtrace for a [`Trap`]. /// /// Whenever a WebAssembly trap occurs an instance of [`Trap`] is created. Each @@ -321,19 +328,6 @@ pub struct FrameInfo { symbols: Vec, } -/// Debug information for a symbol that is attached to a [`FrameInfo`]. -/// -/// When DWARF debug information is present in a wasm file then this structure -/// can be found on a [`FrameInfo`] and can be used to learn about filenames, -/// line numbers, etc, which are the origin of a function in a stack trace. -#[derive(Debug)] -pub struct FrameSymbol { - name: Option, - file: Option, - line: Option, - column: Option, -} - impl FrameInfo { /// Returns the WebAssembly function index for this frame. /// @@ -405,6 +399,19 @@ impl FrameInfo { } } +/// Debug information for a symbol that is attached to a [`FrameInfo`]. +/// +/// When DWARF debug information is present in a wasm file then this structure +/// can be found on a [`FrameInfo`] and can be used to learn about filenames, +/// line numbers, etc, which are the origin of a function in a stack trace. +#[derive(Debug)] +pub struct FrameSymbol { + name: Option, + file: Option, + line: Option, + column: Option, +} + impl FrameSymbol { /// Returns the function name associated with this symbol. /// @@ -463,7 +470,7 @@ fn test_frame_info() -> Result<(), anyhow::Error> { )?; // Create an instance to ensure the frame information is registered. Instance::new(&store, &module, &[])?; - let info = store.frame_info().borrow(); + let modules = store.modules().borrow(); for (i, alloc) in module.compiled_module().finished_functions() { let (start, end) = unsafe { let ptr = (**alloc).as_ptr(); @@ -471,7 +478,7 @@ fn test_frame_info() -> Result<(), anyhow::Error> { (ptr as usize, ptr as usize + len) }; for pc in start..end { - let (frame, _) = info.lookup_frame_info(pc).unwrap(); + let (frame, _) = modules.lookup_frame_info(pc).unwrap(); assert!(frame.func_index() == i.as_u32()); } } diff --git a/crates/wasmtime/src/module/serialization.rs b/crates/wasmtime/src/module/serialization.rs index cc28cdeaf7..1d74aaaf43 100644 --- a/crates/wasmtime/src/module/serialization.rs +++ b/crates/wasmtime/src/module/serialization.rs @@ -1,6 +1,6 @@ //! Implements module serialization. -use super::ModuleInner; +use super::{ModuleInner, ModuleSharedSignatures}; use crate::{Engine, Module, OptLevel}; use anyhow::{anyhow, bail, Context, Result}; use bincode::Options; @@ -10,8 +10,7 @@ use std::fmt; use std::str::FromStr; use std::sync::Arc; use std::{collections::HashMap, fmt::Display}; -use wasmtime_environ::Tunables; -use wasmtime_environ::{isa::TargetIsa, settings}; +use wasmtime_environ::{isa::TargetIsa, settings, Tunables}; use wasmtime_jit::{ CompilationArtifacts, CompilationStrategy, CompiledModule, Compiler, TypeTables, }; @@ -123,55 +122,44 @@ impl From for OptLevel { } } -/// A small helper struct which defines modules are serialized. +/// A small helper struct for serialized module upvars. #[derive(Serialize, Deserialize)] -struct SerializedModuleData<'a> { - /// All compiled artifacts needed by this module, where the last entry in - /// this list is the artifacts for the module itself. - artifacts: Vec>, +struct SerializedModuleUpvar { + /// The module's index into the compilation artifact. + index: usize, + /// Indexes into the list of all compilation artifacts for this module. + artifact_upvars: Vec, /// Closed-over module values that are also needed for this module. - modules: Vec>, - /// The index into the list of type tables that are used for this module's - /// type tables. - type_tables: usize, + module_upvars: Vec, } -impl<'a> SerializedModuleData<'a> { - pub fn new(module: &'a Module) -> (Self, Vec>) { - let mut pushed = HashMap::new(); - let mut tables = Vec::new(); - return (module_data(module, &mut pushed, &mut tables), tables); +impl SerializedModuleUpvar { + pub fn new(module: &Module, artifacts: &[Arc]) -> Self { + // TODO: improve upon the linear searches in the artifact list + let index = artifacts + .iter() + .position(|a| Arc::as_ptr(a) == Arc::as_ptr(&module.inner.module)) + .expect("module should be in artifacts list"); - fn module_data<'a>( - module: &'a Module, - type_tables_pushed: &mut HashMap, - type_tables: &mut Vec>, - ) -> SerializedModuleData<'a> { - // Deduplicate `Arc` using our two parameters to ensure we - // serialize type tables as little as possible. - let ptr = Arc::as_ptr(module.types()); - let type_tables_idx = *type_tables_pushed.entry(ptr as usize).or_insert_with(|| { - type_tables.push(MyCow::Borrowed(module.types())); - type_tables.len() - 1 - }); - SerializedModuleData { - artifacts: module - .inner - .artifact_upvars - .iter() - .map(|i| MyCow::Borrowed(i.compilation_artifacts())) - .chain(Some(MyCow::Borrowed( - module.compiled_module().compilation_artifacts(), - ))) - .collect(), - modules: module - .inner - .module_upvars - .iter() - .map(|i| module_data(i, type_tables_pushed, type_tables)) - .collect(), - type_tables: type_tables_idx, - } + SerializedModuleUpvar { + index, + artifact_upvars: module + .inner + .artifact_upvars + .iter() + .map(|m| { + artifacts + .iter() + .position(|a| Arc::as_ptr(a) == Arc::as_ptr(m)) + .expect("artifact should be in artifacts list") + }) + .collect(), + module_upvars: module + .inner + .module_upvars + .iter() + .map(|m| SerializedModuleUpvar::new(m, artifacts)) + .collect(), } } } @@ -212,14 +200,36 @@ pub struct SerializedModule<'a> { strategy: CompilationStrategy, tunables: Tunables, features: WasmFeatures, - data: SerializedModuleData<'a>, - tables: Vec>, + artifacts: Vec>, + module_upvars: Vec, + types: MyCow<'a, TypeTables>, } impl<'a> SerializedModule<'a> { pub fn new(module: &'a Module) -> Self { - let (data, tables) = SerializedModuleData::new(module); - Self::with_data(module.engine().compiler(), data, tables) + let compiler = module.engine().compiler(); + let artifacts = module + .inner + .artifact_upvars + .iter() + .map(|m| MyCow::Borrowed(m.compilation_artifacts())) + .chain(Some(MyCow::Borrowed( + module.inner.module.compilation_artifacts(), + ))) + .collect::>(); + let module_upvars = module + .inner + .module_upvars + .iter() + .map(|m| SerializedModuleUpvar::new(m, &module.inner.artifact_upvars)) + .collect::>(); + + Self::with_data( + compiler, + artifacts, + module_upvars, + MyCow::Borrowed(module.types()), + ) } pub fn from_artifacts( @@ -229,19 +239,17 @@ impl<'a> SerializedModule<'a> { ) -> Self { Self::with_data( compiler, - SerializedModuleData { - artifacts: artifacts.iter().map(MyCow::Borrowed).collect(), - modules: Vec::new(), - type_tables: 0, - }, - vec![MyCow::Borrowed(types)], + artifacts.iter().map(MyCow::Borrowed).collect(), + Vec::new(), + MyCow::Borrowed(types), ) } fn with_data( compiler: &Compiler, - data: SerializedModuleData<'a>, - tables: Vec>, + artifacts: Vec>, + module_upvars: Vec, + types: MyCow<'a, TypeTables>, ) -> Self { let isa = compiler.isa(); @@ -260,8 +268,9 @@ impl<'a> SerializedModule<'a> { strategy: compiler.strategy(), tunables: compiler.tunables().clone(), features: compiler.features().into(), - data, - tables, + artifacts, + module_upvars, + types, } } @@ -276,45 +285,95 @@ impl<'a> SerializedModule<'a> { self.check_tunables(compiler)?; self.check_features(compiler)?; - let types = self - .tables - .into_iter() - .map(|t| Arc::new(t.unwrap_owned())) - .collect::>(); - let module = mk(engine, &types, self.data)?; + let types = Arc::new(self.types.unwrap_owned()); + let mut modules = CompiledModule::from_artifacts_list( + self.artifacts + .into_iter() + .map(|i| i.unwrap_owned()) + .collect(), + engine.compiler().isa(), + &*engine.config().profiler, + )?; // Validate the module can be used with the current allocator - engine.allocator().validate(module.inner.module.module())?; + engine + .allocator() + .validate(modules.last().unwrap().module())?; - return Ok(module); + let (signatures, trampolines) = engine.register_module_signatures( + &types.wasm_signatures, + modules.iter().flat_map(|m| m.trampolines().iter().cloned()), + ); + + let signatures = Arc::new(ModuleSharedSignatures { + engine: engine.clone(), + signatures, + trampolines, + }); + + let module = modules.pop().unwrap(); + + let module_upvars = self + .module_upvars + .iter() + .map(|m| { + mk( + engine, + &modules, + &types, + m.index, + &m.artifact_upvars, + &m.module_upvars, + &signatures, + ) + }) + .collect::>>()?; + + return Ok(Module { + inner: Arc::new(ModuleInner { + engine: engine.clone(), + types, + module, + artifact_upvars: modules, + module_upvars, + signatures, + }), + }); fn mk( engine: &Engine, - types: &Vec>, - data: SerializedModuleData<'_>, + artifacts: &[Arc], + types: &Arc, + module_index: usize, + artifact_upvars: &[usize], + module_upvars: &[SerializedModuleUpvar], + signatures: &Arc, ) -> Result { - let mut artifacts = CompiledModule::from_artifacts_list( - data.artifacts - .into_iter() - .map(|i| i.unwrap_owned()) - .collect(), - engine.compiler().isa(), - &*engine.config().profiler, - )?; - let inner = ModuleInner { - engine: engine.clone(), - types: types[data.type_tables].clone(), - module: artifacts.pop().unwrap(), - artifact_upvars: artifacts, - module_upvars: data - .modules - .into_iter() - .map(|m| mk(engine, types, m)) - .collect::>>()?, - }; - Ok(Module { - inner: Arc::new(inner), + inner: Arc::new(ModuleInner { + engine: engine.clone(), + types: types.clone(), + module: artifacts[module_index].clone(), + artifact_upvars: artifact_upvars + .iter() + .map(|i| artifacts[*i].clone()) + .collect(), + module_upvars: module_upvars + .into_iter() + .map(|m| { + mk( + engine, + artifacts, + types, + m.index, + &m.artifact_upvars, + &m.module_upvars, + signatures, + ) + }) + .collect::>>()?, + signatures: signatures.clone(), + }), }) } } diff --git a/crates/wasmtime/src/sig_registry.rs b/crates/wasmtime/src/sig_registry.rs deleted file mode 100644 index e5811a8424..0000000000 --- a/crates/wasmtime/src/sig_registry.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! Implement a registry of function signatures, for fast indirect call -//! signature checking. - -use crate::Module; -use std::collections::{hash_map, HashMap}; -use std::convert::TryFrom; -use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::wasm::{SignatureIndex, WasmFuncType}; -use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline}; - -/// WebAssembly requires that the caller and callee signatures in an indirect -/// call must match. To implement this efficiently, keep a registry of all -/// signatures, shared by all instances, so that call sites can just do an -/// index comparison. -#[derive(Debug, Default)] -pub struct SignatureRegistry { - // Map from a wasm actual function type to the index that it is assigned, - // shared amongst all wasm modules. - wasm2index: HashMap, - - // Map of all known wasm function signatures in this registry. This is - // keyed by `VMSharedSignatureIndex` above. - index_map: Vec, -} - -#[derive(Debug)] -struct Entry { - // The WebAssembly type signature, using wasm types. - wasm: WasmFuncType, - // The native trampoline used to invoke this type signature from `Func`. - // Note that the code memory for this trampoline is not owned by this - // type, but instead it's expected to be owned by the store that this - // registry lives within. - trampoline: Option, -} - -impl SignatureRegistry { - /// Registers all signatures within a module into this registry all at once. - /// - /// This will also internally register trampolines compiled in the module. - pub fn register_module(&mut self, module: &Module) { - // Register a unique index for all types in this module, even if they - // don't have a trampoline. - let signatures = &module.types().wasm_signatures; - for ty in module.compiled_module().module().types.values() { - if let wasmtime_environ::ModuleType::Function(index) = ty { - self.register_one(&signatures[*index], None); - } - } - - // Once we've got a shared index for all types used then also fill in - // any trampolines that the module has compiled as well. - for (index, trampoline) in module.compiled_module().trampolines() { - let shared = self.wasm2index[&signatures[*index]]; - let entry = &mut self.index_map[shared.bits() as usize]; - if entry.trampoline.is_none() { - entry.trampoline = Some(*trampoline); - } - } - } - - /// Register a signature and return its unique index. - pub fn register( - &mut self, - wasm: &WasmFuncType, - trampoline: VMTrampoline, - ) -> VMSharedSignatureIndex { - self.register_one(wasm, Some(trampoline)) - } - - fn register_one( - &mut self, - wasm: &WasmFuncType, - trampoline: Option, - ) -> VMSharedSignatureIndex { - let len = self.wasm2index.len(); - - match self.wasm2index.entry(wasm.clone()) { - hash_map::Entry::Occupied(entry) => { - let ret = *entry.get(); - let entry = &mut self.index_map[ret.bits() as usize]; - // If the entry does not previously have a trampoline, then - // overwrite it with whatever was specified by this function. - if entry.trampoline.is_none() { - entry.trampoline = trampoline; - } - ret - } - hash_map::Entry::Vacant(entry) => { - // Keep `signature_hash` len under 2**32 -- VMSharedSignatureIndex::new(std::u32::MAX) - // is reserved for VMSharedSignatureIndex::default(). - assert!( - len < std::u32::MAX as usize, - "Invariant check: signature_hash.len() < std::u32::MAX" - ); - debug_assert_eq!(len, self.index_map.len()); - let index = VMSharedSignatureIndex::new(u32::try_from(len).unwrap()); - self.index_map.push(Entry { - wasm: wasm.clone(), - trampoline, - }); - entry.insert(index); - index - } - } - } - - /// Looks up a shared index from the wasm signature itself. - pub fn lookup(&self, wasm: &WasmFuncType) -> Option { - self.wasm2index.get(wasm).cloned() - } - - /// Builds a lookup table for a module from the possible module's signature - /// indices to the shared signature index within this registry. - pub fn lookup_table( - &self, - module: &Module, - ) -> PrimaryMap { - // For module-linking using modules this builds up a map that is - // too large. This builds up a map for everything in `TypeTables` but - // that's all the types for all modules in a whole module linking graph, - // which our `module` may not be using. - // - // For all non-module-linking-using modules, though, this is not an - // issue. This is optimizing for the non-module-linking case right now - // and it seems like module linking will likely change to the point that - // this will no longer be an issue in the future. - let signatures = &module.types().wasm_signatures; - let mut map = PrimaryMap::with_capacity(signatures.len()); - for wasm in signatures.values() { - map.push( - self.wasm2index - .get(wasm) - .cloned() - .unwrap_or(VMSharedSignatureIndex::new(u32::MAX)), - ); - } - map - } - - /// Looks up information known about a shared signature index. - /// - /// Note that for this operation to be semantically correct the `idx` must - /// have previously come from a call to `register` of this same object. - pub fn lookup_shared( - &self, - idx: VMSharedSignatureIndex, - ) -> Option<(&WasmFuncType, VMTrampoline)> { - let (wasm, trampoline) = self - .index_map - .get(idx.bits() as usize) - .map(|e| (&e.wasm, e.trampoline))?; - Some((wasm, trampoline?)) - } -} diff --git a/crates/wasmtime/src/signatures.rs b/crates/wasmtime/src/signatures.rs new file mode 100644 index 0000000000..bd68f0d8ce --- /dev/null +++ b/crates/wasmtime/src/signatures.rs @@ -0,0 +1,185 @@ +//! Implement a registry of function signatures, for fast indirect call +//! signature checking. + +use std::collections::{hash_map::Entry, HashMap}; +use std::convert::TryFrom; +use wasmtime_environ::entity::PrimaryMap; +use wasmtime_environ::wasm::{SignatureIndex, WasmFuncType}; +use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline}; + +/// Represents a mapping of shared signature index to trampolines. +/// +/// This is used in various places to store trampolines associated with shared +/// signature indexes. +/// +/// As multiple trampolines may exist for a single signature, the map entries +/// are internally reference counted. +#[derive(Default)] +pub struct TrampolineMap(HashMap); + +impl TrampolineMap { + /// Inserts a trampoline into the map. + pub fn insert(&mut self, index: VMSharedSignatureIndex, trampoline: VMTrampoline) { + let entry = match self.0.entry(index) { + Entry::Occupied(e) => e.into_mut(), + Entry::Vacant(e) => e.insert((0, trampoline)), + }; + + // Increment the ref count + entry.0 += 1; + } + + /// Gets a trampoline from the map. + pub fn get(&self, index: VMSharedSignatureIndex) -> Option { + self.0.get(&index).map(|(_, trampoline)| *trampoline) + } + + /// Iterates the shared signature indexes stored in the map. + /// + /// A shared signature index will be returned by the iterator for every + /// trampoline registered for that index, so duplicates may be present. + /// + /// This iterator can be used for deregistering signatures with the + /// signature registry. + pub fn indexes<'a>(&'a self) -> impl Iterator + 'a { + self.0 + .iter() + .flat_map(|(index, (count, _))| std::iter::repeat(*index).take(*count)) + } + + /// Determines if the trampoline map is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +/// Represents a map between module signature indexes and +/// shared signature indexes. +pub type SharedSignatures = PrimaryMap; + +#[derive(Debug)] +struct RegistryEntry { + references: usize, + ty: WasmFuncType, +} + +/// WebAssembly requires that the caller and callee signatures in an indirect +/// call must match. To implement this efficiently, keep a registry of all +/// signatures, shared by all instances, so that call sites can just do an +/// index comparison. +#[derive(Debug, Default)] +pub struct SignatureRegistry { + map: HashMap, + entries: Vec>, + free: Vec, +} + +impl SignatureRegistry { + /// Registers a module with the signature registry from the collection of + /// all signatures and trampolines in the module. + pub fn register_module( + &mut self, + signatures: &PrimaryMap, + trampolines: impl Iterator, + ) -> (SharedSignatures, TrampolineMap) { + let mut sigs = SharedSignatures::default(); + let mut map = TrampolineMap::default(); + + for (_, ty) in signatures.iter() { + sigs.push(self.register(ty)); + } + + for (index, trampoline) in trampolines { + let index = self.map[&signatures[index]]; + map.insert(index, trampoline); + } + + (sigs, map) + } + + /// Registers a single signature with the registry. + /// + /// This is used for registering host functions created with the Wasmtime API. + pub fn register(&mut self, ty: &WasmFuncType) -> VMSharedSignatureIndex { + let len = self.map.len(); + + let index = match self.map.entry(ty.clone()) { + Entry::Occupied(e) => *e.get(), + Entry::Vacant(e) => { + let (index, entry) = match self.free.pop() { + Some(index) => (index, &mut self.entries[index.bits() as usize]), + None => { + // Keep `index_map` len under 2**32 -- VMSharedSignatureIndex::new(std::u32::MAX) + // is reserved for VMSharedSignatureIndex::default(). + assert!( + len < std::u32::MAX as usize, + "Invariant check: index_map.len() < std::u32::MAX" + ); + debug_assert_eq!(len, self.entries.len()); + + let index = VMSharedSignatureIndex::new(u32::try_from(len).unwrap()); + self.entries.push(None); + + (index, self.entries.last_mut().unwrap()) + } + }; + + *entry = Some(RegistryEntry { + references: 0, + ty: ty.clone(), + }); + + *e.insert(index) + } + }; + + self.entries[index.bits() as usize] + .as_mut() + .unwrap() + .references += 1; + + index + } + + /// Unregisters a collection of shared indexes from the registry. + pub fn unregister(&mut self, indexes: impl Iterator) { + for index in indexes { + let removed = { + let entry = self.entries[index.bits() as usize].as_mut().unwrap(); + + debug_assert!(entry.references > 0); + entry.references -= 1; + + if entry.references == 0 { + self.map.remove(&entry.ty); + self.free.push(index); + true + } else { + false + } + }; + + if removed { + self.entries[index.bits() as usize] = None; + } + } + } + + /// Looks up a function type from a shared signature index. + pub fn lookup_type(&self, index: VMSharedSignatureIndex) -> Option<&WasmFuncType> { + self.entries + .get(index.bits() as usize) + .and_then(|e| e.as_ref().map(|e| &e.ty)) + } + + /// Determines if the registry is semantically empty. + pub fn is_empty(&self) -> bool { + // If the map is empty, assert that all remaining entries are "free" + if self.map.is_empty() { + assert!(self.free.len() == self.entries.len()); + true + } else { + false + } + } +} diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 4c78e82c8f..5cbe0119b5 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -1,12 +1,11 @@ -use crate::frame_info; -use crate::frame_info::StoreFrameInfo; -use crate::sig_registry::SignatureRegistry; -use crate::trampoline::StoreInstanceHandle; -use crate::{Engine, Func, FuncType, Module, Trap}; +use crate::{ + module::ModuleRegistry, signatures::TrampolineMap, trampoline::StoreInstanceHandle, Engine, + Func, Module, Trap, +}; use anyhow::{bail, Result}; use std::any::{Any, TypeId}; use std::cell::{Cell, RefCell}; -use std::collections::{hash_map::Entry, HashMap, HashSet}; +use std::collections::{hash_map::Entry, HashMap}; use std::convert::TryFrom; use std::fmt; use std::future::Future; @@ -16,12 +15,12 @@ use std::ptr; use std::rc::Rc; use std::sync::Arc; use std::task::{Context, Poll}; -use wasmtime_environ::wasm; +use wasmtime_environ::wasm::WasmFuncType; use wasmtime_jit::{CompiledModule, ModuleCode}; use wasmtime_runtime::{ - Export, InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler, - StackMapRegistry, TrapInfo, VMCallerCheckedAnyfunc, VMContext, VMExternRef, - VMExternRefActivationsTable, VMInterrupts, VMTrampoline, + InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler, StackMapRegistry, + TrapInfo, VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, + VMSharedSignatureIndex, VMTrampoline, }; /// Used to associate instances with the store. @@ -72,20 +71,13 @@ pub struct Store { pub(crate) struct StoreInner { engine: Engine, - /// The map of all host functions registered with this store's signature registry - host_funcs: RefCell>>, interrupts: Arc, - signatures: RefCell, instances: RefCell>, signal_handler: RefCell>>>, externref_activations_table: VMExternRefActivationsTable, stack_map_registry: StackMapRegistry, - /// Information about JIT code which allows us to test if a program counter - /// is in JIT code, lookup trap information, etc. - frame_info: RefCell, - /// Set of all compiled modules that we're holding a strong reference to - /// the module's code for. This includes JIT functions, trampolines, etc. - modules: RefCell>, + modules: RefCell, + trampolines: RefCell, // Numbers of resources instantiated in this store. instance_count: Cell, memory_count: Cell, @@ -137,21 +129,19 @@ impl Store { // once-per-thread. Platforms like Unix, however, only require this // once-per-program. In any case this is safe to call many times and // each one that's not relevant just won't do anything. - wasmtime_runtime::init_traps(frame_info::GlobalFrameInfo::is_wasm_pc) + wasmtime_runtime::init_traps(crate::module::GlobalModuleRegistry::is_wasm_pc) .expect("failed to initialize trap handling"); Store { inner: Rc::new(StoreInner { engine: engine.clone(), - host_funcs: RefCell::new(HashMap::new()), interrupts: Arc::new(Default::default()), - signatures: RefCell::new(Default::default()), instances: RefCell::new(Vec::new()), signal_handler: RefCell::new(None), externref_activations_table: VMExternRefActivationsTable::new(), stack_map_registry: StackMapRegistry::default(), - frame_info: Default::default(), - modules: Default::default(), + modules: RefCell::new(ModuleRegistry::default()), + trampolines: RefCell::new(TrampolineMap::default()), instance_count: Default::default(), memory_count: Default::default(), table_count: Default::default(), @@ -181,35 +171,6 @@ impl Store { }) } - pub(crate) fn get_host_anyfunc( - &self, - instance: &InstanceHandle, - ty: &FuncType, - trampoline: VMTrampoline, - ) -> *mut VMCallerCheckedAnyfunc { - let mut funcs = self.inner.host_funcs.borrow_mut(); - - let anyfunc = funcs.entry(unsafe { instance.clone() }).or_insert_with(|| { - let mut anyfunc = match instance - .lookup_by_declaration(&wasm::EntityIndex::Function(wasm::FuncIndex::from_u32(0))) - { - Export::Function(f) => unsafe { f.anyfunc.as_ref() }.clone(), - _ => unreachable!(), - }; - - // Register the function with this store's signature registry - anyfunc.type_index = self - .inner - .signatures - .borrow_mut() - .register(ty.as_wasm_func_type(), trampoline); - - Box::new(anyfunc) - }); - - &mut **anyfunc - } - /// Returns the [`Engine`] that this store is associated with. #[inline] pub fn engine(&self) -> &Engine { @@ -244,52 +205,16 @@ impl Store { } } - pub(crate) fn signatures(&self) -> &RefCell { - &self.inner.signatures - } - pub(crate) fn register_module(&self, module: &Module) { - // With a module being instantiated into this `Store` we need to - // preserve its jit-code. References to this module's code and - // trampolines are not owning-references so it's our responsibility to - // keep it all alive within the `Store`. - // - // If this module is already present in the store then we skip all - // further registration steps. - let first = self - .inner - .modules - .borrow_mut() - .insert(ArcModuleCode(module.compiled_module().code().clone())); - if !first { + // Register the module with the registry + if !self.inner.modules.borrow_mut().register(module) { return; } - // All modules register their JIT code in a store for two reasons - // currently: - // - // * First we only catch signals/traps if the program counter falls - // within the jit code of an instantiated wasm module. This ensures - // we don't catch accidental Rust/host segfaults. - // - // * Second when generating a backtrace we'll use this mapping to - // only generate wasm frames for instruction pointers that fall - // within jit code. - self.inner - .frame_info - .borrow_mut() - .register(module.compiled_module()); - // We need to know about all the stack maps of all instantiated modules // so when performing a GC we know about all wasm frames that we find // on the stack. self.register_stack_maps(module.compiled_module()); - - // Signatures are loaded into our `SignatureRegistry` here - // once-per-module (and once-per-signature). This allows us to create - // a `Func` wrapper for any function in the module, which requires that - // we know about the signature and trampoline for all instances. - self.signatures().borrow_mut().register_module(module); } fn register_stack_maps(&self, module: &CompiledModule) { @@ -304,6 +229,39 @@ impl Store { })); } + // This is used to register a `Func` with the store + pub(crate) fn register_signature( + &self, + ty: &WasmFuncType, + trampoline: VMTrampoline, + ) -> VMSharedSignatureIndex { + let index = self.inner.engine.register_signature(ty); + self.inner + .trampolines + .borrow_mut() + .insert(index, trampoline); + index + } + + pub(crate) fn lookup_trampoline(&self, index: VMSharedSignatureIndex) -> VMTrampoline { + // Look up the trampoline with the store's trampolines (from `Func`). + if let Some(trampoline) = self.inner.trampolines.borrow().get(index) { + return trampoline; + } + + // Look up the trampoline with the registered modules + if let Some(trampoline) = self.inner.modules.borrow().lookup_trampoline(index) { + return trampoline; + } + + // Lastly, check with the engine (for `HostFunc`) + self.inner + .engine + .host_func_trampolines() + .get(index) + .expect("trampoline missing") + } + pub(crate) fn bump_resource_counts(&self, module: &Module) -> Result<()> { let config = self.engine().config(); @@ -363,7 +321,7 @@ impl Store { .borrow() .iter() .any(|i| i.handle.vmctx_ptr() == handle.vmctx_ptr()) - || self.inner.host_funcs.borrow().get(&handle).is_some() + || self.inner.engine.host_func_anyfunc(&handle).is_some() ); StoreInstanceHandle { store: self.clone(), @@ -494,8 +452,8 @@ impl Store { &self.inner.stack_map_registry } - pub(crate) fn frame_info(&self) -> &RefCell { - &self.inner.frame_info + pub(crate) fn modules(&self) -> &RefCell { + &self.inner.modules } /// Notifies that the current Store (and all referenced entities) has been moved over to a @@ -984,6 +942,12 @@ impl Drop for StoreInner { } } } + + let trampolines = self.trampolines.borrow(); + + if !trampolines.is_empty() { + self.engine.unregister_signatures(trampolines.indexes()); + } } } diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 9879880900..f2402efa6c 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -1,6 +1,6 @@ //! Support for a calling of an imported function. -use crate::{sig_registry::SignatureRegistry, Config, FuncType, Trap}; +use crate::{Config, FuncType, Store, Trap}; use anyhow::Result; use std::any::Any; use std::cmp; @@ -262,15 +262,15 @@ pub fn create_function( ft: &FuncType, func: Box Result<(), Trap>>, config: &Config, - registry: Option<&mut SignatureRegistry>, + store: Option<&Store>, ) -> Result<(InstanceHandle, VMTrampoline)> { let (module, finished_functions, trampoline, trampoline_state) = create_function_trampoline(config, ft, func)?; - // If there is no signature registry, use the default signature index which is + // If there is no store, use the default signature index which is // guaranteed to trap if there is ever an indirect call on the function (should not happen) - let shared_signature_id = registry - .map(|r| r.register(ft.as_wasm_func_type(), trampoline)) + let shared_signature_id = store + .map(|s| s.register_signature(ft.as_wasm_func_type(), trampoline)) .unwrap_or(VMSharedSignatureIndex::default()); unsafe { diff --git a/crates/wasmtime/src/trap.rs b/crates/wasmtime/src/trap.rs index b50f5353ee..1d59aa564c 100644 --- a/crates/wasmtime/src/trap.rs +++ b/crates/wasmtime/src/trap.rs @@ -161,7 +161,7 @@ impl Trap { maybe_interrupted, } => { let mut code = store - .frame_info() + .modules() .borrow() .lookup_trap_info(pc) .map(|info| info.trap_code) @@ -239,7 +239,7 @@ impl Trap { // (the call instruction) so we subtract one as the lookup. let pc_to_lookup = if Some(pc) == trap_pc { pc } else { pc - 1 }; if let Some((info, has_unparsed_debuginfo)) = - store.frame_info().borrow().lookup_frame_info(pc_to_lookup) + store.modules().borrow().lookup_frame_info(pc_to_lookup) { wasm_trace.push(info); diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs index 2a91af473c..0241aee282 100644 --- a/crates/wasmtime/src/types.rs +++ b/crates/wasmtime/src/types.rs @@ -204,8 +204,7 @@ impl ExternType { ) -> ExternType { match ty { EntityType::Function(idx) => { - let sig = &types.wasm_signatures[*idx]; - FuncType::from_wasm_func_type(sig).into() + FuncType::from_wasm_func_type(types.wasm_signatures[*idx].clone()).into() } EntityType::Global(ty) => GlobalType::from_wasmtime_global(ty).into(), EntityType::Memory(ty) => MemoryType::from_wasmtime_memory(ty).into(), @@ -298,8 +297,8 @@ impl FuncType { &self.sig } - pub(crate) fn from_wasm_func_type(sig: &wasm::WasmFuncType) -> FuncType { - FuncType { sig: sig.clone() } + pub(crate) fn from_wasm_func_type(sig: wasm::WasmFuncType) -> FuncType { + Self { sig } } } diff --git a/crates/wasmtime/src/types/matching.rs b/crates/wasmtime/src/types/matching.rs index a767bbb7e1..55a55c1d62 100644 --- a/crates/wasmtime/src/types/matching.rs +++ b/crates/wasmtime/src/types/matching.rs @@ -1,4 +1,4 @@ -use crate::{Extern, Store}; +use crate::{signatures::SharedSignatures, Extern, Store}; use anyhow::{bail, Context, Result}; use wasmtime_environ::wasm::{ EntityType, Global, InstanceTypeIndex, Memory, ModuleTypeIndex, SignatureIndex, Table, @@ -6,6 +6,7 @@ use wasmtime_environ::wasm::{ use wasmtime_jit::TypeTables; pub struct MatchCx<'a> { + pub signatures: &'a SharedSignatures, pub types: &'a TypeTables, pub store: &'a Store, } @@ -70,13 +71,8 @@ impl MatchCx<'_> { } pub fn func(&self, expected: SignatureIndex, actual: &crate::Func) -> Result<()> { - let matches = match self - .store - .signatures() - .borrow() - .lookup(&self.types.wasm_signatures[expected]) - { - Some(idx) => actual.sig_index() == idx, + let matches = match self.signatures.get(expected) { + Some(idx) => actual.sig_index() == *idx, // If our expected signature isn't registered, then there's no way // that `actual` can match it. None => false, @@ -114,15 +110,19 @@ impl MatchCx<'_> { let module = actual.compiled_module().module(); self.imports_match( expected, + actual.signatures(), actual.types(), module.imports().map(|(name, field, ty)| { assert!(field.is_none()); // should be true if module linking is enabled (name, ty) }), )?; - self.exports_match(expected_sig.exports, actual.types(), |name| { - module.exports.get(name).map(|idx| module.type_of(*idx)) - })?; + self.exports_match( + expected_sig.exports, + actual.signatures(), + actual.types(), + |name| module.exports.get(name).map(|idx| module.type_of(*idx)), + )?; Ok(()) } @@ -133,6 +133,7 @@ impl MatchCx<'_> { fn imports_match<'a>( &self, expected: ModuleTypeIndex, + actual_signatures: &SharedSignatures, actual_types: &TypeTables, actual_imports: impl Iterator, ) -> Result<()> { @@ -146,10 +147,11 @@ impl MatchCx<'_> { None => bail!("expected type doesn't import {:?}", name), }; MatchCx { + signatures: actual_signatures, types: actual_types, store: self.store, } - .extern_ty_matches(&actual_ty, expected_ty, self.types) + .extern_ty_matches(&actual_ty, expected_ty, self.signatures, self.types) .with_context(|| format!("module import {:?} incompatible", name))?; } Ok(()) @@ -160,6 +162,7 @@ impl MatchCx<'_> { fn exports_match( &self, expected: InstanceTypeIndex, + actual_signatures: &SharedSignatures, actual_types: &TypeTables, lookup: impl Fn(&str) -> Option, ) -> Result<()> { @@ -169,7 +172,7 @@ impl MatchCx<'_> { for (name, expected) in self.types.instance_signatures[expected].exports.iter() { match lookup(name) { Some(ty) => self - .extern_ty_matches(expected, &ty, actual_types) + .extern_ty_matches(expected, &ty, actual_signatures, actual_types) .with_context(|| format!("export {:?} incompatible", name))?, None => bail!("failed to find export {:?}", name), } @@ -183,6 +186,7 @@ impl MatchCx<'_> { &self, expected: &EntityType, actual_ty: &EntityType, + actual_signatures: &SharedSignatures, actual_types: &TypeTables, ) -> Result<()> { let actual_desc = match actual_ty { @@ -221,7 +225,7 @@ impl MatchCx<'_> { EntityType::Instance(expected) => match actual_ty { EntityType::Instance(actual) => { let sig = &actual_types.instance_signatures[*actual]; - self.exports_match(*expected, actual_types, |name| { + self.exports_match(*expected, actual_signatures, actual_types, |name| { sig.exports.get(name).cloned() })?; Ok(()) @@ -237,15 +241,19 @@ impl MatchCx<'_> { self.imports_match( *expected, + actual_signatures, actual_types, actual_module_sig .imports .iter() .map(|(module, ty)| (module.as_str(), ty.clone())), )?; - self.exports_match(expected_module_sig.exports, actual_types, |name| { - actual_instance_sig.exports.get(name).cloned() - })?; + self.exports_match( + expected_module_sig.exports, + actual_signatures, + actual_types, + |name| actual_instance_sig.exports.get(name).cloned(), + )?; Ok(()) } _ => bail!("expected module, but found {}", actual_desc), From ea72c621f07644609c87252d5fb4454d8d8f0e49 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Wed, 14 Apr 2021 13:45:56 -0700 Subject: [PATCH 13/43] Remove the stack map registry. This commit removes the stack map registry and instead uses the existing information from the store's module registry to lookup stack maps. A trait is now used to pass the lookup context to the runtime, implemented by `Store` to do the lookup. With this change, module registration in `Store` is now entirely limited to inserting the module into the module registry. --- crates/environ/src/vmoffsets.rs | 18 +- crates/jit/src/instantiate.rs | 11 +- crates/runtime/src/externref.rs | 201 ++---------------- crates/runtime/src/instance.rs | 8 +- crates/runtime/src/instance/allocator.rs | 8 +- .../runtime/src/instance/allocator/pooling.rs | 4 +- .../src/instance/allocator/pooling/uffd.rs | 2 +- crates/runtime/src/libcalls.rs | 8 +- crates/wasmtime/src/func/typed.rs | 2 +- crates/wasmtime/src/instance.rs | 5 +- crates/wasmtime/src/module/registry.rs | 82 ++++++- crates/wasmtime/src/store.rs | 57 ++--- crates/wasmtime/src/trampoline.rs | 6 +- .../wasmtime/src/trampoline/create_handle.rs | 4 +- crates/wasmtime/src/trampoline/func.rs | 4 +- crates/wasmtime/src/values.rs | 2 +- 16 files changed, 158 insertions(+), 264 deletions(-) diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index e5d6604610..20f3b8a298 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -6,7 +6,7 @@ // struct VMContext { // interrupts: *const VMInterrupts, // externref_activations_table: *mut VMExternRefActivationsTable, -// stack_map_registry: *mut StackMapRegistry, +// stack_map_lookup: *const dyn StackMapLookup, // signature_ids: [VMSharedSignatureIndex; module.num_signature_ids], // imported_functions: [VMFunctionImport; module.num_imported_functions], // imported_tables: [VMTableImport; module.num_imported_tables], @@ -77,7 +77,7 @@ pub struct VMOffsets { // precalculated offsets of various member fields interrupts: u32, externref_activations_table: u32, - stack_map_registry: u32, + stack_map_lookup: u32, signature_ids: u32, imported_functions: u32, imported_tables: u32, @@ -149,7 +149,7 @@ impl From for VMOffsets { num_defined_globals: fields.num_defined_globals, interrupts: 0, externref_activations_table: 0, - stack_map_registry: 0, + stack_map_lookup: 0, signature_ids: 0, imported_functions: 0, imported_tables: 0, @@ -168,13 +168,13 @@ impl From for VMOffsets { .interrupts .checked_add(u32::from(fields.pointer_size)) .unwrap(); - ret.stack_map_registry = ret + ret.stack_map_lookup = ret .externref_activations_table .checked_add(u32::from(fields.pointer_size)) .unwrap(); ret.signature_ids = ret - .stack_map_registry - .checked_add(u32::from(fields.pointer_size)) + .stack_map_lookup + .checked_add(u32::from(fields.pointer_size * 2)) .unwrap(); ret.imported_functions = ret .signature_ids @@ -507,10 +507,10 @@ impl VMOffsets { self.externref_activations_table } - /// The offset of the `*mut StackMapRegistry` member. + /// The offset of the `*const dyn StackMapLookup` member. #[inline] - pub fn vmctx_stack_map_registry(&self) -> u32 { - self.stack_map_registry + pub fn vmctx_stack_map_lookup(&self) -> u32 { + self.stack_map_lookup } /// The offset of the `signature_ids` array. diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 2d155e3d7b..5527051ecd 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -362,11 +362,18 @@ impl CompiledModule { } /// Gets the function information for a given function index. - pub fn func_info(&self, index: DefinedFuncIndex) -> (&FunctionAddressMap, &[TrapInformation]) { + pub fn func_info( + &self, + index: DefinedFuncIndex, + ) -> ( + &FunctionAddressMap, + &[TrapInformation], + &[StackMapInformation], + ) { self.artifacts .funcs .get(index) - .map(|f| (&f.address_map, f.traps.as_ref())) + .map(|f| (&f.address_map, f.traps.as_ref(), f.stack_maps.as_ref())) .expect("defined function should be present") } diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index 02943c1c01..0606a941e6 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -103,14 +103,12 @@ use std::alloc::Layout; use std::any::Any; use std::cell::{Cell, RefCell, UnsafeCell}; use std::cmp::Ordering; -use std::collections::BTreeMap; use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::mem; use std::ops::Deref; use std::ptr::{self, NonNull}; -use std::rc::Rc; -use wasmtime_environ::{ir::StackMap, StackMapInformation}; +use wasmtime_environ::ir::StackMap; /// An external reference to some opaque data. /// @@ -596,10 +594,10 @@ impl VMExternRefActivationsTable { pub unsafe fn insert_with_gc( &self, externref: VMExternRef, - stack_maps_registry: &StackMapRegistry, + stack_map_lookup: &dyn StackMapLookup, ) { if let Err(externref) = self.try_insert(externref) { - self.gc_and_insert_slow(externref, stack_maps_registry); + self.gc_and_insert_slow(externref, stack_map_lookup); } } @@ -607,9 +605,9 @@ impl VMExternRefActivationsTable { unsafe fn gc_and_insert_slow( &self, externref: VMExternRef, - stack_maps_registry: &StackMapRegistry, + stack_map_lookup: &dyn StackMapLookup, ) { - gc(stack_maps_registry, self); + gc(stack_map_lookup, self); // Might as well insert right into the hash set, rather than the bump // chunk, since we are already on a slow path and we get de-duplication @@ -743,182 +741,21 @@ impl VMExternRefActivationsTable { } } -/// A registry of stack maps for currently active Wasm modules. -#[derive(Default)] -pub struct StackMapRegistry { - inner: RefCell, +/// Used by the runtime to lookup a stack map from a PC value. +pub trait StackMapLookup { + /// Lookup the stack map at a program counter (PC) value. + fn lookup(&self, pc: usize) -> Option<*const StackMap>; } -#[derive(Default)] -struct StackMapRegistryInner { - /// A map from the highest pc in a module, to its stack maps. - /// - /// For details, see the comment above `GlobalModuleRegistry`. - ranges: BTreeMap, -} +pub(crate) struct EmptyStackMapLookup; -#[derive(Debug)] -struct ModuleStackMaps { - /// The range of PCs that this module covers. Different modules must always - /// have distinct ranges. - range: std::ops::Range, - - /// A map from a PC in this module (that is a GC safepoint) to its - /// associated stack map. If `None` then it means that the PC is the start - /// of a range which has no stack map. - pc_to_stack_map: Vec<(usize, Option>)>, -} - -impl StackMapRegistry { - /// Register the stack maps for a given module. - /// - /// The stack maps should be given as an iterator over a function's PC range - /// in memory (that is, where the JIT actually allocated and emitted the - /// function's code at), and the stack maps and code offsets within that - /// range for each of its GC safepoints. - pub fn register_stack_maps<'a>( - &self, - stack_maps: impl IntoIterator, &'a [StackMapInformation])>, - ) { - let mut min = usize::max_value(); - let mut max = 0; - let mut pc_to_stack_map = vec![]; - let mut last_is_none_marker = true; - - for (range, infos) in stack_maps { - let len = range.end - range.start; - - min = std::cmp::min(min, range.start); - max = std::cmp::max(max, range.end); - - // Add a marker between functions indicating that this function's pc - // starts with no stack map so when our binary search later on finds - // a pc between the start of the function and the function's first - // stack map it doesn't think the previous stack map is our stack - // map. - // - // We skip this if the previous entry pushed was also a `None` - // marker, in which case the starting pc already has no stack map. - // This is also skipped if the first `code_offset` is zero since - // what we'll push applies for the first pc anyway. - if !last_is_none_marker && (infos.is_empty() || infos[0].code_offset > 0) { - pc_to_stack_map.push((range.start, None)); - last_is_none_marker = true; - } - - for info in infos { - assert!((info.code_offset as usize) < len); - pc_to_stack_map.push(( - range.start + (info.code_offset as usize), - Some(Rc::new(info.stack_map.clone())), - )); - last_is_none_marker = false; - } - } - - if pc_to_stack_map.is_empty() { - // Nothing to register. - return; - } - - let module_stack_maps = ModuleStackMaps { - range: min..max, - pc_to_stack_map, - }; - - let mut inner = self.inner.borrow_mut(); - - // Assert that this chunk of ranges doesn't collide with any other known - // chunks. - if let Some((_, prev)) = inner.ranges.range(max..).next() { - assert!(prev.range.start > max); - } - if let Some((prev_end, _)) = inner.ranges.range(..=min).next_back() { - assert!(*prev_end < min); - } - - let old = inner.ranges.insert(max, module_stack_maps); - assert!(old.is_none()); - } - - /// Lookup the stack map for the given PC, if any. - pub fn lookup_stack_map(&self, pc: usize) -> Option> { - let inner = self.inner.borrow(); - let stack_maps = inner.module_stack_maps(pc)?; - - // Do a binary search to find the stack map for the given PC. - // - // Because GC safepoints are technically only associated with a single - // PC, we should ideally only care about `Ok(index)` values returned - // from the binary search. However, safepoints are inserted right before - // calls, and there are two things that can disturb the PC/offset - // associated with the safepoint versus the PC we actually use to query - // for the stack map: - // - // 1. The `backtrace` crate gives us the PC in a frame that will be - // *returned to*, and where execution will continue from, rather than - // the PC of the call we are currently at. So we would need to - // disassemble one instruction backwards to query the actual PC for - // the stack map. - // - // TODO: One thing we *could* do to make this a little less error - // prone, would be to assert/check that the nearest GC safepoint - // found is within `max_encoded_size(any kind of call instruction)` - // our queried PC for the target architecture. - // - // 2. Cranelift's stack maps only handle the stack, not - // registers. However, some references that are arguments to a call - // may need to be in registers. In these cases, what Cranelift will - // do is: - // - // a. spill all the live references, - // b. insert a GC safepoint for those references, - // c. reload the references into registers, and finally - // d. make the call. - // - // Step (c) adds drift between the GC safepoint and the location of - // the call, which is where we actually walk the stack frame and - // collect its live references. - // - // Luckily, the spill stack slots for the live references are still - // up to date, so we can still find all the on-stack roots. - // Furthermore, we do not have a moving GC, so we don't need to worry - // whether the following code will reuse the references in registers - // (which would not have been updated to point to the moved objects) - // or reload from the stack slots (which would have been updated to - // point to the moved objects). - let index = match stack_maps - .pc_to_stack_map - .binary_search_by_key(&pc, |(pc, _stack_map)| *pc) - { - // Exact hit. - Ok(i) => i, - - // `Err(0)` means that the associated stack map would have been the - // first element in the array if this pc had an associated stack - // map, but this pc does not have an associated stack map. This can - // only happen inside a Wasm frame if there are no live refs at this - // pc. - Err(0) => return None, - - Err(n) => n - 1, - }; - - let stack_map = stack_maps.pc_to_stack_map[index].1.as_ref()?.clone(); - Some(stack_map) +impl StackMapLookup for EmptyStackMapLookup { + fn lookup(&self, _pc: usize) -> Option<*const StackMap> { + None } } -impl StackMapRegistryInner { - fn module_stack_maps(&self, pc: usize) -> Option<&ModuleStackMaps> { - let (end, stack_maps) = self.ranges.range(pc..).next()?; - if pc < stack_maps.range.start || *end < pc { - None - } else { - Some(stack_maps) - } - } -} +pub(crate) const EMPTY_STACK_MAP_LOOKUP: EmptyStackMapLookup = EmptyStackMapLookup; #[derive(Debug, Default)] struct DebugOnly { @@ -965,7 +802,7 @@ impl std::ops::DerefMut for DebugOnly { /// Additionally, you must have registered the stack maps for every Wasm module /// that has frames on the stack with the given `stack_maps_registry`. pub unsafe fn gc( - stack_maps_registry: &StackMapRegistry, + stack_map_lookup: &dyn StackMapLookup, externref_activations_table: &VMExternRefActivationsTable, ) { // We borrow the precise stack roots `RefCell` for the whole duration of @@ -1003,7 +840,7 @@ pub unsafe fn gc( if cfg!(debug_assertions) { // Assert that there aren't any Wasm frames on the stack. backtrace::trace(|frame| { - let stack_map = stack_maps_registry.lookup_stack_map(frame.ip() as usize); + let stack_map = stack_map_lookup.lookup(frame.ip() as usize); assert!(stack_map.is_none()); true }); @@ -1048,11 +885,11 @@ pub unsafe fn gc( let pc = frame.ip() as usize; let sp = frame.sp() as usize; - if let Some(stack_map) = stack_maps_registry.lookup_stack_map(pc) { + if let Some(stack_map) = stack_map_lookup.lookup(pc) { debug_assert!(sp != 0, "we should always get a valid SP for Wasm frames"); - for i in 0..(stack_map.mapped_words() as usize) { - if stack_map.get_bit(i) { + for i in 0..((*stack_map).mapped_words() as usize) { + if (*stack_map).get_bit(i) { // Stack maps have one bit per word in the frame, and the // zero^th bit is the *lowest* addressed word in the frame, // i.e. the closest to the SP. So to get the `i`^th word in diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index bf4234f6b8..33a2583cff 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -3,7 +3,7 @@ //! `InstanceHandle` is a reference-counting handle for an `Instance`. use crate::export::Export; -use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; +use crate::externref::{StackMapLookup, VMExternRefActivationsTable}; use crate::memory::{Memory, RuntimeMemoryCreator}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; @@ -249,9 +249,9 @@ impl Instance { unsafe { self.vmctx_plus_offset(self.offsets.vmctx_externref_activations_table()) } } - /// Return a pointer to the `StackMapRegistry`. - pub fn stack_map_registry(&self) -> *mut *mut StackMapRegistry { - unsafe { self.vmctx_plus_offset(self.offsets.vmctx_stack_map_registry()) } + /// Return a pointer to the `StackMapLookup`. + pub fn stack_map_lookup(&self) -> *mut *const dyn StackMapLookup { + unsafe { self.vmctx_plus_offset(self.offsets.vmctx_stack_map_lookup()) } } /// Return a reference to the vmctx used by compiled wasm code. diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index d236b4723c..14a9d2eff3 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -1,4 +1,4 @@ -use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; +use crate::externref::{StackMapLookup, VMExternRefActivationsTable, EMPTY_STACK_MAP_LOOKUP}; use crate::imports::Imports; use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator}; use crate::memory::{DefaultMemoryCreator, Memory}; @@ -57,8 +57,8 @@ pub struct InstanceAllocationRequest<'a> { /// The pointer to the reference activations table to use for the instance. pub externref_activations_table: *mut VMExternRefActivationsTable, - /// The pointer to the stack map registry to use for the instance. - pub stack_map_registry: *mut StackMapRegistry, + /// The pointer to the stack map lookup to use for the instance. + pub stack_map_lookup: Option<*const dyn StackMapLookup>, } /// An link error while instantiating a module. @@ -447,7 +447,7 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque *instance.interrupts() = req.interrupts; *instance.externref_activations_table() = req.externref_activations_table; - *instance.stack_map_registry() = req.stack_map_registry; + *instance.stack_map_lookup() = req.stack_map_lookup.unwrap_or(&EMPTY_STACK_MAP_LOOKUP); // Initialize shared signatures let mut ptr = instance.signature_ids_ptr(); diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index d9604c348f..6b89ce603e 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -1370,7 +1370,7 @@ mod test { host_state: Box::new(()), interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_registry: std::ptr::null_mut(), + stack_map_lookup: None, }, ) .expect("allocation should succeed"), @@ -1394,7 +1394,7 @@ mod test { host_state: Box::new(()), interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_registry: std::ptr::null_mut(), + stack_map_lookup: None, }, ) { Err(InstantiationError::Limit(3)) => {} diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index 5578ca1746..fb811680bb 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -523,7 +523,7 @@ mod test { host_state: Box::new(()), interrupts: ptr::null(), externref_activations_table: ptr::null_mut(), - stack_map_registry: ptr::null_mut(), + stack_map_lookup: None, }, ) .expect("instance should allocate"), diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index 026738a8a1..58eb585312 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -449,8 +449,8 @@ pub unsafe extern "C" fn wasmtime_activations_table_insert_with_gc( let externref = VMExternRef::clone_from_raw(externref); let instance = (&mut *vmctx).instance(); let activations_table = &**instance.externref_activations_table(); - let registry = &**instance.stack_map_registry(); - activations_table.insert_with_gc(externref, registry); + let stack_map_lookup = &**instance.stack_map_lookup(); + activations_table.insert_with_gc(externref, stack_map_lookup); } /// Perform a Wasm `global.get` for `externref` globals. @@ -466,8 +466,8 @@ pub unsafe extern "C" fn wasmtime_externref_global_get( Some(externref) => { let raw = externref.as_raw(); let activations_table = &**instance.externref_activations_table(); - let registry = &**instance.stack_map_registry(); - activations_table.insert_with_gc(externref, registry); + let stack_map_lookup = &**instance.stack_map_lookup(); + activations_table.insert_with_gc(externref, stack_map_lookup); raw } } diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index c0a1d834da..b0fe8b574f 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -207,7 +207,7 @@ unsafe impl WasmTy for Option { unsafe { store .externref_activations_table() - .insert_with_gc(x.inner, store.stack_map_registry()); + .insert_with_gc(x.inner, &*store.stack_map_lookup()); } abi } else { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 00cc90fd82..a30ef98414 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -13,7 +13,7 @@ use wasmtime_environ::wasm::{ }; use wasmtime_environ::Initializer; use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstantiationError, RuntimeInstance, StackMapRegistry, + Imports, InstanceAllocationRequest, InstantiationError, RuntimeInstance, StackMapLookup, VMContext, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport, }; @@ -525,8 +525,7 @@ impl<'a> Instantiator<'a> { externref_activations_table: self.store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_registry: self.store.stack_map_registry() as *const StackMapRegistry - as *mut _, + stack_map_lookup: Some(self.store.stack_map_lookup() as *const dyn StackMapLookup), })?; // After we've created the `InstanceHandle` we still need to run diff --git a/crates/wasmtime/src/module/registry.rs b/crates/wasmtime/src/module/registry.rs index 161ef408d6..bec0434a91 100644 --- a/crates/wasmtime/src/module/registry.rs +++ b/crates/wasmtime/src/module/registry.rs @@ -43,6 +43,11 @@ impl ModuleRegistry { self.module(pc)?.lookup_trap_info(pc) } + /// Looks up a stack map from a program counter. + pub fn lookup_stack_map<'a>(&'a self, pc: usize) -> Option<&'a ir::StackMap> { + self.module(pc)?.lookup_stack_map(pc) + } + fn module(&self, pc: usize) -> Option<&RegisteredModule> { let (end, info) = self.0.range(pc..).next()?; if pc < info.start || *end < pc { @@ -53,13 +58,13 @@ impl ModuleRegistry { } /// Registers a new module with the registry. - pub fn register(&mut self, module: &Module) -> bool { + pub fn register(&mut self, module: &Module) { let compiled_module = module.compiled_module(); let (start, end) = compiled_module.code().range(); // Ignore modules with no code, finished functions, or if the module is already registered if start == end || compiled_module.finished_functions().is_empty() { - return false; + return; } // The module code range is exclusive for end, so make it inclusive as it @@ -67,7 +72,7 @@ impl ModuleRegistry { let end = end - 1; if self.0.get(&end).is_some() { - return false; + return; } // Assert that this module's code doesn't collide with any other registered modules @@ -90,7 +95,6 @@ impl ModuleRegistry { assert!(prev.is_none()); GLOBAL_MODULES.lock().unwrap().register(start, end, module); - true } /// Looks up a trampoline from a shared signature index. @@ -135,7 +139,7 @@ impl RegisteredModule { /// if no information can be found. pub fn lookup_frame_info(&self, pc: usize) -> Option { let (index, offset) = self.func(pc)?; - let (addr_map, _) = self.module.func_info(index); + let (addr_map, _, _) = self.module.func_info(index); let pos = Self::instr_pos(offset, addr_map); // In debug mode for now assert that we found a mapping for `pc` within @@ -197,13 +201,77 @@ impl RegisteredModule { /// Fetches trap information about a program counter in a backtrace. pub fn lookup_trap_info(&self, pc: usize) -> Option<&TrapInformation> { let (index, offset) = self.func(pc)?; - let (_, traps) = self.module.func_info(index); + let (_, traps, _) = self.module.func_info(index); let idx = traps .binary_search_by_key(&offset, |info| info.code_offset) .ok()?; Some(&traps[idx]) } + /// Looks up a stack map from a program counter + pub fn lookup_stack_map(&self, pc: usize) -> Option<&ir::StackMap> { + let (index, offset) = self.func(pc)?; + let (_, _, stack_maps) = self.module.func_info(index); + + // Do a binary search to find the stack map for the given offset. + // + // Because GC safepoints are technically only associated with a single + // PC, we should ideally only care about `Ok(index)` values returned + // from the binary search. However, safepoints are inserted right before + // calls, and there are two things that can disturb the PC/offset + // associated with the safepoint versus the PC we actually use to query + // for the stack map: + // + // 1. The `backtrace` crate gives us the PC in a frame that will be + // *returned to*, and where execution will continue from, rather than + // the PC of the call we are currently at. So we would need to + // disassemble one instruction backwards to query the actual PC for + // the stack map. + // + // TODO: One thing we *could* do to make this a little less error + // prone, would be to assert/check that the nearest GC safepoint + // found is within `max_encoded_size(any kind of call instruction)` + // our queried PC for the target architecture. + // + // 2. Cranelift's stack maps only handle the stack, not + // registers. However, some references that are arguments to a call + // may need to be in registers. In these cases, what Cranelift will + // do is: + // + // a. spill all the live references, + // b. insert a GC safepoint for those references, + // c. reload the references into registers, and finally + // d. make the call. + // + // Step (c) adds drift between the GC safepoint and the location of + // the call, which is where we actually walk the stack frame and + // collect its live references. + // + // Luckily, the spill stack slots for the live references are still + // up to date, so we can still find all the on-stack roots. + // Furthermore, we do not have a moving GC, so we don't need to worry + // whether the following code will reuse the references in registers + // (which would not have been updated to point to the moved objects) + // or reload from the stack slots (which would have been updated to + // point to the moved objects). + + let index = match stack_maps.binary_search_by_key(&offset, |i| i.code_offset) { + // Exact hit. + Ok(i) => i, + + // `Err(0)` means that the associated stack map would have been the + // first element in the array if this pc had an associated stack + // map, but this pc does not have an associated stack map. This can + // only happen inside a Wasm frame if there are no live refs at this + // pc. + Err(0) => return None, + + Err(i) => i - 1, + }; + + Some(&stack_maps[index].stack_map) + } + fn func(&self, pc: usize) -> Option<(DefinedFuncIndex, u32)> { let (index, start, _) = self.module.func_by_pc(pc)?; Some((index, (pc - start) as u32)) @@ -260,7 +328,7 @@ impl GlobalModuleRegistry { match info.module.func(pc) { Some((index, offset)) => { - let (addr_map, _) = info.module.module.func_info(index); + let (addr_map, _, _) = info.module.module.func_info(index); RegisteredModule::instr_pos(offset, addr_map).is_some() } None => false, diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 5cbe0119b5..f337b39975 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -16,11 +16,11 @@ use std::rc::Rc; use std::sync::Arc; use std::task::{Context, Poll}; use wasmtime_environ::wasm::WasmFuncType; -use wasmtime_jit::{CompiledModule, ModuleCode}; +use wasmtime_jit::ModuleCode; use wasmtime_runtime::{ - InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler, StackMapRegistry, - TrapInfo, VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, - VMSharedSignatureIndex, VMTrampoline, + InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler, TrapInfo, + VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, + VMTrampoline, }; /// Used to associate instances with the store. @@ -75,7 +75,6 @@ pub(crate) struct StoreInner { instances: RefCell>, signal_handler: RefCell>>>, externref_activations_table: VMExternRefActivationsTable, - stack_map_registry: StackMapRegistry, modules: RefCell, trampolines: RefCell, // Numbers of resources instantiated in this store. @@ -139,7 +138,6 @@ impl Store { instances: RefCell::new(Vec::new()), signal_handler: RefCell::new(None), externref_activations_table: VMExternRefActivationsTable::new(), - stack_map_registry: StackMapRegistry::default(), modules: RefCell::new(ModuleRegistry::default()), trampolines: RefCell::new(TrampolineMap::default()), instance_count: Default::default(), @@ -207,26 +205,7 @@ impl Store { pub(crate) fn register_module(&self, module: &Module) { // Register the module with the registry - if !self.inner.modules.borrow_mut().register(module) { - return; - } - - // We need to know about all the stack maps of all instantiated modules - // so when performing a GC we know about all wasm frames that we find - // on the stack. - self.register_stack_maps(module.compiled_module()); - } - - fn register_stack_maps(&self, module: &CompiledModule) { - self.stack_map_registry() - .register_stack_maps(module.stack_maps().map(|(func, stack_maps)| unsafe { - let ptr = (*func).as_ptr(); - let len = (*func).len(); - let start = ptr as usize; - let end = ptr as usize + len; - let range = start..end; - (range, stack_maps) - })); + self.inner.modules.borrow_mut().register(module); } // This is used to register a `Func` with the store @@ -448,10 +427,6 @@ impl Store { } #[inline] - pub(crate) fn stack_map_registry(&self) -> &StackMapRegistry { - &self.inner.stack_map_registry - } - pub(crate) fn modules(&self) -> &RefCell { &self.inner.modules } @@ -471,20 +446,21 @@ impl Store { /// /// It is fine to call this several times: only the first call will have an effect. pub unsafe fn notify_switched_thread(&self) { - wasmtime_runtime::init_traps(frame_info::GlobalFrameInfo::is_wasm_pc) + wasmtime_runtime::init_traps(crate::module::GlobalModuleRegistry::is_wasm_pc) .expect("failed to initialize per-threads traps"); } + #[inline] + pub(crate) fn stack_map_lookup(&self) -> *const dyn wasmtime_runtime::StackMapLookup { + self.inner.as_ref() + } + /// Perform garbage collection of `ExternRef`s. pub fn gc(&self) { // For this crate's API, we ensure that `set_stack_canary` invariants - // are upheld for all host-->Wasm calls, and we register every module - // used with this store in `self.inner.stack_map_registry`. + // are upheld for all host-->Wasm calls. unsafe { - wasmtime_runtime::gc( - &self.inner.stack_map_registry, - &self.inner.externref_activations_table, - ); + wasmtime_runtime::gc(self.inner.as_ref(), &self.inner.externref_activations_table); } } @@ -951,6 +927,13 @@ impl Drop for StoreInner { } } +impl wasmtime_runtime::StackMapLookup for StoreInner { + fn lookup(&self, pc: usize) -> Option<*const wasmtime_environ::ir::StackMap> { + // The address of the stack map is stable for the lifetime of the store + self.modules.borrow().lookup_stack_map(pc).map(|m| m as _) + } +} + /// A threadsafe handle used to interrupt instances executing within a /// particular `Store`. /// diff --git a/crates/wasmtime/src/trampoline.rs b/crates/wasmtime/src/trampoline.rs index 7cb588f7d4..21eac2a134 100644 --- a/crates/wasmtime/src/trampoline.rs +++ b/crates/wasmtime/src/trampoline.rs @@ -19,8 +19,8 @@ use std::sync::Arc; use wasmtime_environ::{entity::PrimaryMap, wasm, Module}; use wasmtime_runtime::{ Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, - OnDemandInstanceAllocator, StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, - VMFunctionImport, VMSharedSignatureIndex, + OnDemandInstanceAllocator, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, + VMSharedSignatureIndex, }; /// A wrapper around `wasmtime_runtime::InstanceHandle` which pairs it with the @@ -77,7 +77,7 @@ fn create_handle( externref_activations_table: store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, + stack_map_lookup: Some(store.stack_map_lookup()), }, )?; diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs index dde937f81c..182e421499 100644 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ b/crates/wasmtime/src/trampoline/create_handle.rs @@ -9,7 +9,7 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::Module; use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstanceAllocator, StackMapRegistry, + Imports, InstanceAllocationRequest, InstanceAllocator, StackMapLookup, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMSharedSignatureIndex, }; @@ -43,7 +43,7 @@ pub(crate) fn create_handle( externref_activations_table: store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, + stack_map_lookup: &store, })?; Ok(store.add_instance(handle, true)) diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index f2402efa6c..0265c0b3c1 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -283,7 +283,7 @@ pub fn create_function( host_state: Box::new(trampoline_state), interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_registry: std::ptr::null_mut(), + stack_map_lookup: None, })?, trampoline, )) @@ -315,7 +315,7 @@ pub unsafe fn create_raw_function( host_state, interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_registry: std::ptr::null_mut(), + stack_map_lookup: None, })?, ) } diff --git a/crates/wasmtime/src/values.rs b/crates/wasmtime/src/values.rs index 4dc6474513..9e10f0aa83 100644 --- a/crates/wasmtime/src/values.rs +++ b/crates/wasmtime/src/values.rs @@ -98,7 +98,7 @@ impl Val { let externref_ptr = x.inner.as_raw(); store .externref_activations_table() - .insert_with_gc(x.inner, store.stack_map_registry()); + .insert_with_gc(x.inner, &*store.stack_map_lookup()); ptr::write(p as *mut *mut u8, externref_ptr) } Val::FuncRef(f) => ptr::write( From 510fc717281327971f35b40879567f1dc9918e9c Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 15 Apr 2021 20:15:24 -0700 Subject: [PATCH 14/43] Code review feedback. * Make `FunctionInfo` public and `CompiledModule::func_info` return it. * Make the `StackMapLookup` trait unsafe. * Add comments for the purpose of `EngineHostFuncs`. * Rework ownership model of shared signatures: `SignatureCollection` in conjunction with `SignatureRegistry` is now used so that the `Engine`, `Store`, and `Module` don't need to worry about unregistering shared signatures. * Implement `Func::param_arity` and `Func::result_arity` in terms of `Func::ty`. * Make looking up a trampoline with the module registry more efficient by doing a binary search on the function's starting PC value for the owning module and then looking up the trampoline with only that module. * Remove reference to the shared signatures from `GlobalRegisteredModule`. --- crates/jit/src/instantiate.rs | 20 +- crates/runtime/src/externref.rs | 12 +- crates/wasmtime/src/engine.rs | 94 +++---- crates/wasmtime/src/func.rs | 23 +- crates/wasmtime/src/instance.rs | 17 +- crates/wasmtime/src/module.rs | 44 +--- crates/wasmtime/src/module/registry.rs | 118 +++++---- crates/wasmtime/src/module/serialization.rs | 17 +- crates/wasmtime/src/signatures.rs | 257 +++++++++++++------- crates/wasmtime/src/store.rs | 56 ++--- crates/wasmtime/src/trampoline.rs | 2 +- crates/wasmtime/src/trampoline/func.rs | 6 +- crates/wasmtime/src/types/matching.rs | 14 +- 13 files changed, 336 insertions(+), 344 deletions(-) diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 5527051ecd..3ff67d9c7a 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -176,11 +176,13 @@ struct FinishedFunctions(PrimaryMap); unsafe impl Send for FinishedFunctions {} unsafe impl Sync for FinishedFunctions {} +/// Information about a function, such as trap information, address map, +/// and stack maps. #[derive(Serialize, Deserialize, Clone)] -struct FunctionInfo { - traps: Vec, - address_map: FunctionAddressMap, - stack_maps: Vec, +pub struct FunctionInfo { + pub traps: Vec, + pub address_map: FunctionAddressMap, + pub stack_maps: Vec, } /// This is intended to mirror the type tables in `wasmtime_environ`, except that @@ -362,18 +364,10 @@ impl CompiledModule { } /// Gets the function information for a given function index. - pub fn func_info( - &self, - index: DefinedFuncIndex, - ) -> ( - &FunctionAddressMap, - &[TrapInformation], - &[StackMapInformation], - ) { + pub fn func_info(&self, index: DefinedFuncIndex) -> &FunctionInfo { self.artifacts .funcs .get(index) - .map(|f| (&f.address_map, f.traps.as_ref(), f.stack_maps.as_ref())) .expect("defined function should be present") } diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index 0606a941e6..44a60ce797 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -742,14 +742,22 @@ impl VMExternRefActivationsTable { } /// Used by the runtime to lookup a stack map from a PC value. -pub trait StackMapLookup { +/// +/// # Safety +/// +/// This trait is unsafe as it returns pointers to a stack map without +/// any clear ownership. +/// +/// It is the responsibility of the caller to not have the pointer outlive +/// the stack map lookup trait object. +pub unsafe trait StackMapLookup { /// Lookup the stack map at a program counter (PC) value. fn lookup(&self, pc: usize) -> Option<*const StackMap>; } pub(crate) struct EmptyStackMapLookup; -impl StackMapLookup for EmptyStackMapLookup { +unsafe impl StackMapLookup for EmptyStackMapLookup { fn lookup(&self, _pc: usize) -> Option<*const StackMap> { None } diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index 54d8769d19..acafaa5797 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -1,24 +1,34 @@ -use crate::signatures::{SharedSignatures, SignatureRegistry, TrampolineMap}; +use crate::signatures::{SignatureCollection, SignatureRegistry}; use crate::Config; use anyhow::Result; use std::collections::HashMap; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; #[cfg(feature = "cache")] use wasmtime_cache::CacheConfig; -use wasmtime_environ::{ - entity::PrimaryMap, - wasm::{SignatureIndex, WasmFuncType}, -}; use wasmtime_jit::Compiler; -use wasmtime_runtime::{ - debug_builtins, InstanceAllocator, InstanceHandle, VMCallerCheckedAnyfunc, - VMSharedSignatureIndex, VMTrampoline, -}; +use wasmtime_runtime::{debug_builtins, InstanceAllocator, InstanceHandle, VMCallerCheckedAnyfunc}; -#[derive(Default)] +/// This is used as a Send+Sync wrapper around two data structures relating to +/// host functions defined on `Config`: +/// +/// * `anyfuncs` - this stores a mapping between the host function instance and +/// a `VMCallerCheckedAnyfunc` that can be used as the function's value in Wasmtime's ABI. +/// The address of the anyfunc needs to be stable, thus the boxed value. +/// +/// * `signatures` - this stores the collection of shared signatures registered for every +/// usable host functions with this engine. struct EngineHostFuncs { anyfuncs: HashMap>, - trampolines: TrampolineMap, + signatures: SignatureCollection, +} + +impl EngineHostFuncs { + fn new(registry: &SignatureRegistry) -> Self { + Self { + anyfuncs: HashMap::new(), + signatures: SignatureCollection::new(registry), + } + } } // This is safe for send and sync as it is read-only once the @@ -58,18 +68,10 @@ struct EngineInner { config: Config, compiler: Compiler, allocator: Box, - signatures: RwLock, + signatures: SignatureRegistry, host_funcs: EngineHostFuncs, } -impl Drop for EngineInner { - fn drop(&mut self) { - let mut signatures = self.signatures.write().unwrap(); - signatures.unregister(self.host_funcs.trampolines.indexes()); - assert!(signatures.is_empty()); - } -} - impl Engine { /// Creates a new [`Engine`] with the specified compilation and /// configuration settings. @@ -77,20 +79,20 @@ impl Engine { debug_builtins::ensure_exported(); config.validate()?; let allocator = config.build_allocator()?; - let mut signatures = SignatureRegistry::default(); - let mut host_funcs = EngineHostFuncs::default(); + let registry = SignatureRegistry::new(); + let mut host_funcs = EngineHostFuncs::new(®istry); - // Register all the host function signatures + // Register all the host function signatures with the collection for func in config.host_funcs() { - let sig = signatures.register(func.ty.as_wasm_func_type()); + let sig = host_funcs + .signatures + .register(func.ty.as_wasm_func_type(), func.trampoline); // Cloning the instance handle is safe as host functions outlive the engine host_funcs.anyfuncs.insert( unsafe { func.instance.clone() }, Box::new(func.anyfunc(sig)), ); - - host_funcs.trampolines.insert(sig, func.trampoline); } Ok(Engine { @@ -98,7 +100,7 @@ impl Engine { config: config.clone(), compiler: config.build_compiler(allocator.as_ref()), allocator, - signatures: RwLock::new(signatures), + signatures: registry, host_funcs, }), }) @@ -128,40 +130,12 @@ impl Engine { Arc::ptr_eq(&a.inner, &b.inner) } - pub(crate) fn register_module_signatures( - &self, - signatures: &PrimaryMap, - trampolines: impl Iterator, - ) -> (SharedSignatures, TrampolineMap) { - self.inner - .signatures - .write() - .unwrap() - .register_module(signatures, trampolines) + pub(crate) fn signatures(&self) -> &SignatureRegistry { + &self.inner.signatures } - pub(crate) fn register_signature(&self, ty: &WasmFuncType) -> VMSharedSignatureIndex { - self.inner.signatures.write().unwrap().register(ty) - } - - pub(crate) fn unregister_signatures( - &self, - indexes: impl Iterator, - ) { - self.inner.signatures.write().unwrap().unregister(indexes); - } - - pub(crate) fn lookup_func_type(&self, index: VMSharedSignatureIndex) -> Option { - self.inner - .signatures - .read() - .unwrap() - .lookup_type(index) - .cloned() - } - - pub(crate) fn host_func_trampolines(&self) -> &TrampolineMap { - &self.inner.host_funcs.trampolines + pub(crate) fn host_func_signatures(&self) -> &SignatureCollection { + &self.inner.host_funcs.signatures } pub(crate) fn host_func_anyfunc( diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 6bb24597e1..0319ee0002 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -778,31 +778,20 @@ impl Func { self.instance .store .engine() - .lookup_func_type(self.sig_index()) + .signatures() + .lookup_type(self.sig_index()) .expect("signature should be registered"), ) } /// Returns the number of parameters that this function takes. pub fn param_arity(&self) -> usize { - let sig = self - .instance - .store - .engine() - .lookup_func_type(self.sig_index()) - .expect("signature should be registered"); - sig.params.len() + self.ty().params().len() } /// Returns the number of results this function produces. pub fn result_arity(&self) -> usize { - let sig = self - .instance - .store - .engine() - .lookup_func_type(self.sig_index()) - .expect("signature should be registered"); - sig.returns.len() + self.ty().results().len() } /// Invokes this function with the `params` given, returning the results and @@ -927,7 +916,7 @@ impl Func { Func { instance: store.existing_vmctx(anyfunc.vmctx), export: export.clone(), - trampoline: store.lookup_trampoline(anyfunc.type_index), + trampoline: store.lookup_trampoline(&*anyfunc), } } @@ -1813,7 +1802,7 @@ macro_rules! impl_into_func { // If not given a store, use a default signature index that is guaranteed to trap. // If the function is called indirectly without first being associated with a store (a bug condition). let shared_signature_id = store - .map(|s| s.register_signature(ty.as_wasm_func_type(), trampoline)) + .map(|s| s.signatures().borrow_mut().register(ty.as_wasm_func_type(), trampoline)) .unwrap_or(VMSharedSignatureIndex::default()); let instance = unsafe { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index a30ef98414..cf807aee6a 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -13,9 +13,9 @@ use wasmtime_environ::wasm::{ }; use wasmtime_environ::Initializer; use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstantiationError, RuntimeInstance, StackMapLookup, - VMContext, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, - VMMemoryImport, VMTableImport, + Imports, InstanceAllocationRequest, InstantiationError, RuntimeInstance, VMContext, + VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, + VMTableImport, }; /// An instantiated WebAssembly module. @@ -506,10 +506,9 @@ impl<'a> Instantiator<'a> { fn instantiate_raw(&self) -> Result { let compiled_module = self.cur.module.compiled_module(); - // Register the module just before instantiation to ensure we have a - // trampoline registered for every signature and to preserve the module's - // compiled JIT code within the `Store`. - self.store.register_module(&self.cur.module); + // Register the module just before instantiation to ensure we keep the module + // properly referenced while in use by the store. + self.store.modules().borrow_mut().register(&self.cur.module); unsafe { let engine = self.store.engine(); @@ -519,13 +518,13 @@ impl<'a> Instantiator<'a> { module: compiled_module.module().clone(), finished_functions: compiled_module.finished_functions(), imports: self.cur.build(), - shared_signatures: self.cur.module.signatures().into(), + shared_signatures: self.cur.module.signatures().as_module_map().into(), host_state: Box::new(()), interrupts: self.store.interrupts(), externref_activations_table: self.store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_lookup: Some(self.store.stack_map_lookup() as *const dyn StackMapLookup), + stack_map_lookup: Some(std::mem::transmute(self.store.stack_map_lookup())), })?; // After we've created the `InstanceHandle` we still need to run diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 7e1db3c876..af108d2bac 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -1,5 +1,5 @@ use crate::{ - signatures::{SharedSignatures, TrampolineMap}, + signatures::SignatureCollection, types::{ExportType, ExternType, ImportType}, }; use crate::{Engine, ModuleType}; @@ -11,9 +11,8 @@ use wasmparser::Validator; #[cfg(feature = "cache")] use wasmtime_cache::ModuleCacheEntry; use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::wasm::{ModuleIndex, SignatureIndex}; +use wasmtime_environ::wasm::ModuleIndex; use wasmtime_jit::{CompilationArtifacts, CompiledModule, TypeTables}; -use wasmtime_runtime::VMSharedSignatureIndex; mod registry; mod serialization; @@ -21,26 +20,6 @@ mod serialization; pub use registry::{FrameInfo, FrameSymbol, GlobalModuleRegistry, ModuleRegistry}; pub use serialization::SerializedModule; -// A wrapper around registered signatures and trampolines that will automatically -/// unregister the signatures when dropped. -pub(crate) struct ModuleSharedSignatures { - engine: Engine, - signatures: SharedSignatures, - trampolines: TrampolineMap, -} - -impl Drop for ModuleSharedSignatures { - fn drop(&mut self) { - if !self.signatures.is_empty() { - // Use the shared signatures map to unregister as not every registered - // signature will have a trampoline, but every index in the trampoline map - // will be present in the shared signatures map. - self.engine - .unregister_signatures(self.signatures.values().cloned()); - } - } -} - /// A compiled WebAssembly module, ready to be instantiated. /// /// A `Module` is a compiled in-memory representation of an input WebAssembly @@ -129,7 +108,7 @@ struct ModuleInner { /// modules. types: Arc, /// Registered shared signature for the module. - signatures: Arc, + signatures: Arc, } impl Module { @@ -350,10 +329,11 @@ impl Module { // Validate the module can be used with the current allocator engine.allocator().validate(modules[main_module].module())?; - let (signatures, trampolines) = engine.register_module_signatures( + let signatures = Arc::new(SignatureCollection::new_for_module( + engine.signatures(), &types.wasm_signatures, modules.iter().flat_map(|m| m.trampolines().iter().cloned()), - ); + )); let module = modules.remove(main_module); @@ -364,11 +344,7 @@ impl Module { types: Arc::new(types), artifact_upvars: modules, module_upvars: Vec::new(), - signatures: Arc::new(ModuleSharedSignatures { - engine: engine.clone(), - signatures, - trampolines, - }), + signatures, }), }) } @@ -488,11 +464,7 @@ impl Module { &self.inner.types } - pub(crate) fn signatures(&self) -> &PrimaryMap { - &self.inner.signatures.signatures - } - - pub(crate) fn shared_signatures(&self) -> &Arc { + pub(crate) fn signatures(&self) -> &Arc { &self.inner.signatures } diff --git a/crates/wasmtime/src/module/registry.rs b/crates/wasmtime/src/module/registry.rs index bec0434a91..3691c38834 100644 --- a/crates/wasmtime/src/module/registry.rs +++ b/crates/wasmtime/src/module/registry.rs @@ -1,6 +1,6 @@ //! Implements a registry of modules for a store. -use crate::{module::ModuleSharedSignatures, Module}; +use crate::{signatures::SignatureCollection, Module}; use std::{ collections::BTreeMap, sync::{Arc, Mutex}, @@ -9,12 +9,17 @@ use wasmtime_environ::{ entity::EntityRef, ir, wasm::DefinedFuncIndex, FunctionAddressMap, TrapInformation, }; use wasmtime_jit::CompiledModule; -use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline}; +use wasmtime_runtime::{VMCallerCheckedAnyfunc, VMTrampoline}; lazy_static::lazy_static! { static ref GLOBAL_MODULES: Mutex = Default::default(); } +fn func_by_pc(module: &CompiledModule, pc: usize) -> Option<(DefinedFuncIndex, u32)> { + let (index, start, _) = module.func_by_pc(pc)?; + Some((index, (pc - start) as u32)) +} + /// Used for registering modules with a store. /// /// The map is from the ending (exclusive) address for the module code to @@ -62,7 +67,7 @@ impl ModuleRegistry { let compiled_module = module.compiled_module(); let (start, end) = compiled_module.code().range(); - // Ignore modules with no code, finished functions, or if the module is already registered + // Ignore modules with no code or finished functions if start == end || compiled_module.finished_functions().is_empty() { return; } @@ -71,7 +76,10 @@ impl ModuleRegistry { // may be a valid PC value let end = end - 1; - if self.0.get(&end).is_some() { + // Ensure the module isn't already present in the registry + // This is expected when a module is instantiated multiple times in the same store + if let Some(m) = self.0.get(&end) { + assert_eq!(m.start, start); return; } @@ -89,7 +97,7 @@ impl ModuleRegistry { RegisteredModule { start, module: compiled_module.clone(), - signatures: module.shared_signatures().clone(), + signatures: module.signatures().clone(), }, ); assert!(prev.is_none()); @@ -97,18 +105,10 @@ impl ModuleRegistry { GLOBAL_MODULES.lock().unwrap().register(start, end, module); } - /// Looks up a trampoline from a shared signature index. - /// - /// This will search all modules associated with the store for a suitable trampoline - /// given the shared signature index. - pub fn lookup_trampoline(&self, index: VMSharedSignatureIndex) -> Option { - for (_, m) in &self.0 { - if let Some(trampoline) = m.signatures.trampolines.get(index) { - return Some(trampoline); - } - } - - None + /// Looks up a trampoline from an anyfunc. + pub fn lookup_trampoline(&self, anyfunc: &VMCallerCheckedAnyfunc) -> Option { + let module = self.module(anyfunc.func_ptr.as_ptr() as usize)?; + module.signatures.trampoline(anyfunc.type_index) } } @@ -124,7 +124,7 @@ impl Drop for ModuleRegistry { struct RegisteredModule { start: usize, module: Arc, - signatures: Arc, + signatures: Arc, } impl RegisteredModule { @@ -138,9 +138,9 @@ impl RegisteredModule { /// Returns an object if this `pc` is known to this module, or returns `None` /// if no information can be found. pub fn lookup_frame_info(&self, pc: usize) -> Option { - let (index, offset) = self.func(pc)?; - let (addr_map, _, _) = self.module.func_info(index); - let pos = Self::instr_pos(offset, addr_map); + let (index, offset) = func_by_pc(&self.module, pc)?; + let info = self.module.func_info(index); + let pos = Self::instr_pos(offset, &info.address_map); // In debug mode for now assert that we found a mapping for `pc` within // the function, because otherwise something is buggy along the way and @@ -149,8 +149,8 @@ impl RegisteredModule { debug_assert!(pos.is_some(), "failed to find instruction for {:x}", pc); let instr = match pos { - Some(pos) => addr_map.instructions[pos].srcloc, - None => addr_map.start_srcloc, + Some(pos) => info.address_map.instructions[pos].srcloc, + None => info.address_map.start_srcloc, }; // Use our wasm-relative pc to symbolize this frame. If there's a @@ -193,25 +193,26 @@ impl RegisteredModule { func_index: index.index() as u32, func_name: module.func_names.get(&index).cloned(), instr, - func_start: addr_map.start_srcloc, + func_start: info.address_map.start_srcloc, symbols, }) } /// Fetches trap information about a program counter in a backtrace. pub fn lookup_trap_info(&self, pc: usize) -> Option<&TrapInformation> { - let (index, offset) = self.func(pc)?; - let (_, traps, _) = self.module.func_info(index); - let idx = traps + let (index, offset) = func_by_pc(&self.module, pc)?; + let info = self.module.func_info(index); + let idx = info + .traps .binary_search_by_key(&offset, |info| info.code_offset) .ok()?; - Some(&traps[idx]) + Some(&info.traps[idx]) } /// Looks up a stack map from a program counter pub fn lookup_stack_map(&self, pc: usize) -> Option<&ir::StackMap> { - let (index, offset) = self.func(pc)?; - let (_, _, stack_maps) = self.module.func_info(index); + let (index, offset) = func_by_pc(&self.module, pc)?; + let info = self.module.func_info(index); // Do a binary search to find the stack map for the given offset. // @@ -255,7 +256,10 @@ impl RegisteredModule { // or reload from the stack slots (which would have been updated to // point to the moved objects). - let index = match stack_maps.binary_search_by_key(&offset, |i| i.code_offset) { + let index = match info + .stack_maps + .binary_search_by_key(&offset, |i| i.code_offset) + { // Exact hit. Ok(i) => i, @@ -269,12 +273,7 @@ impl RegisteredModule { Err(i) => i - 1, }; - Some(&stack_maps[index].stack_map) - } - - fn func(&self, pc: usize) -> Option<(DefinedFuncIndex, u32)> { - let (index, start, _) = self.module.func_by_pc(pc)?; - Some((index, (pc - start) as u32)) + Some(&info.stack_maps[index].stack_map) } fn instr_pos(offset: u32, addr_map: &FunctionAddressMap) -> Option { @@ -298,6 +297,17 @@ impl RegisteredModule { } } +// Counterpart to `RegisteredModule`, but stored in the global registry. +struct GlobalRegisteredModule { + start: usize, + module: Arc, + /// Note that modules can be instantiated in many stores, so the purpose of + /// this field is to keep track of how many stores have registered a + /// module. Information is only removed from the global registry when this + /// reference count reaches 0. + references: usize, +} + /// This is the global module registry that stores information for all modules /// that are currently in use by any `Store`. /// @@ -318,18 +328,18 @@ impl GlobalModuleRegistry { /// Returns whether the `pc`, according to globally registered information, /// is a wasm trap or not. pub(crate) fn is_wasm_pc(pc: usize) -> bool { - let info = GLOBAL_MODULES.lock().unwrap(); + let modules = GLOBAL_MODULES.lock().unwrap(); - match info.0.range(pc..).next() { - Some((end, info)) => { - if pc < info.module.start || *end < pc { + match modules.0.range(pc..).next() { + Some((end, entry)) => { + if pc < entry.start || *end < pc { return false; } - match info.module.func(pc) { + match func_by_pc(&entry.module, pc) { Some((index, offset)) => { - let (addr_map, _, _) = info.module.module.func_info(index); - RegisteredModule::instr_pos(offset, addr_map).is_some() + let info = entry.module.func_info(index); + RegisteredModule::instr_pos(offset, &info.address_map).is_some() } None => false, } @@ -342,18 +352,15 @@ impl GlobalModuleRegistry { /// the given function information, with the global information. fn register(&mut self, start: usize, end: usize, module: &Module) { let info = self.0.entry(end).or_insert_with(|| GlobalRegisteredModule { - module: RegisteredModule { - start, - module: module.compiled_module().clone(), - signatures: module.shared_signatures().clone(), - }, + start, + module: module.compiled_module().clone(), references: 0, }); // Note that ideally we'd debug_assert that the information previously // stored, if any, matches the `functions` we were given, but for now we // just do some simple checks to hope it's the same. - assert_eq!(info.module.start, start); + assert_eq!(info.start, start); info.references += 1; } @@ -368,17 +375,6 @@ impl GlobalModuleRegistry { } } -/// This is the equivalent of `RegisteredModule` except it keeps a reference count. -struct GlobalRegisteredModule { - module: RegisteredModule, - - /// Note that modules can be instantiated in many stores, so the purpose of - /// this field is to keep track of how many stores have registered a - /// module. Information is only removed from the global registry when this - /// reference count reaches 0. - references: usize, -} - /// Description of a frame in a backtrace for a [`Trap`]. /// /// Whenever a WebAssembly trap occurs an instance of [`Trap`] is created. Each diff --git a/crates/wasmtime/src/module/serialization.rs b/crates/wasmtime/src/module/serialization.rs index 1d74aaaf43..0d2836889e 100644 --- a/crates/wasmtime/src/module/serialization.rs +++ b/crates/wasmtime/src/module/serialization.rs @@ -1,7 +1,7 @@ //! Implements module serialization. -use super::{ModuleInner, ModuleSharedSignatures}; -use crate::{Engine, Module, OptLevel}; +use super::ModuleInner; +use crate::{signatures::SignatureCollection, Engine, Module, OptLevel}; use anyhow::{anyhow, bail, Context, Result}; use bincode::Options; use serde::{Deserialize, Serialize}; @@ -300,16 +300,11 @@ impl<'a> SerializedModule<'a> { .allocator() .validate(modules.last().unwrap().module())?; - let (signatures, trampolines) = engine.register_module_signatures( + let signatures = Arc::new(SignatureCollection::new_for_module( + engine.signatures(), &types.wasm_signatures, modules.iter().flat_map(|m| m.trampolines().iter().cloned()), - ); - - let signatures = Arc::new(ModuleSharedSignatures { - engine: engine.clone(), - signatures, - trampolines, - }); + )); let module = modules.pop().unwrap(); @@ -347,7 +342,7 @@ impl<'a> SerializedModule<'a> { module_index: usize, artifact_upvars: &[usize], module_upvars: &[SerializedModuleUpvar], - signatures: &Arc, + signatures: &Arc, ) -> Result { Ok(Module { inner: Arc::new(ModuleInner { diff --git a/crates/wasmtime/src/signatures.rs b/crates/wasmtime/src/signatures.rs index bd68f0d8ce..2faf3fa059 100644 --- a/crates/wasmtime/src/signatures.rs +++ b/crates/wasmtime/src/signatures.rs @@ -1,61 +1,108 @@ //! Implement a registry of function signatures, for fast indirect call //! signature checking. -use std::collections::{hash_map::Entry, HashMap}; -use std::convert::TryFrom; +use std::{ + collections::{hash_map::Entry, HashMap}, + sync::RwLock, +}; +use std::{convert::TryFrom, sync::Arc}; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::{SignatureIndex, WasmFuncType}; use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline}; -/// Represents a mapping of shared signature index to trampolines. +/// Represents a collection of shared signatures. /// -/// This is used in various places to store trampolines associated with shared -/// signature indexes. +/// This is used to register shared signatures with a shared signature registry. /// -/// As multiple trampolines may exist for a single signature, the map entries -/// are internally reference counted. -#[derive(Default)] -pub struct TrampolineMap(HashMap); +/// The collection will unregister any contained signatures with the registry +/// when dropped. +#[derive(Debug)] +pub struct SignatureCollection { + registry: Arc>, + signatures: PrimaryMap, + trampolines: HashMap, +} -impl TrampolineMap { - /// Inserts a trampoline into the map. - pub fn insert(&mut self, index: VMSharedSignatureIndex, trampoline: VMTrampoline) { - let entry = match self.0.entry(index) { +impl SignatureCollection { + /// Creates a new, empty signature collection given a signature registry. + pub fn new(registry: &SignatureRegistry) -> Self { + Self { + registry: registry.0.clone(), + signatures: PrimaryMap::new(), + trampolines: HashMap::new(), + } + } + + /// Creates a signature collection for a module given the module's signatures + /// and trampolines. + pub fn new_for_module( + registry: &SignatureRegistry, + signatures: &PrimaryMap, + trampolines: impl Iterator, + ) -> Self { + let (signatures, trampolines) = registry + .0 + .write() + .unwrap() + .register_for_module(signatures, trampolines); + + Self { + registry: registry.0.clone(), + signatures, + trampolines, + } + } + + /// Treats the signature collection as a map from a module signature index to + /// registered shared signature indexes. + /// + /// This is used for looking up module shared signature indexes during module + /// instantiation. + pub fn as_module_map(&self) -> &PrimaryMap { + &self.signatures + } + + /// Gets the shared signature index given a module signature index. + pub fn shared_signature(&self, index: SignatureIndex) -> Option { + self.signatures.get(index).copied() + } + + /// Gets a trampoline for a registered signature. + pub fn trampoline(&self, index: VMSharedSignatureIndex) -> Option { + self.trampolines + .get(&index) + .map(|(_, trampoline)| *trampoline) + } + + /// Registers a single function with the collection. + /// + /// Returns the shared signature index for the function. + pub fn register( + &mut self, + ty: &WasmFuncType, + trampoline: VMTrampoline, + ) -> VMSharedSignatureIndex { + let index = self.registry.write().unwrap().register(ty); + + let entry = match self.trampolines.entry(index) { Entry::Occupied(e) => e.into_mut(), Entry::Vacant(e) => e.insert((0, trampoline)), }; // Increment the ref count entry.0 += 1; - } - /// Gets a trampoline from the map. - pub fn get(&self, index: VMSharedSignatureIndex) -> Option { - self.0.get(&index).map(|(_, trampoline)| *trampoline) - } - - /// Iterates the shared signature indexes stored in the map. - /// - /// A shared signature index will be returned by the iterator for every - /// trampoline registered for that index, so duplicates may be present. - /// - /// This iterator can be used for deregistering signatures with the - /// signature registry. - pub fn indexes<'a>(&'a self) -> impl Iterator + 'a { - self.0 - .iter() - .flat_map(|(index, (count, _))| std::iter::repeat(*index).take(*count)) - } - - /// Determines if the trampoline map is empty. - pub fn is_empty(&self) -> bool { - self.0.is_empty() + index } } -/// Represents a map between module signature indexes and -/// shared signature indexes. -pub type SharedSignatures = PrimaryMap; +impl Drop for SignatureCollection { + fn drop(&mut self) { + if !self.signatures.is_empty() || !self.trampolines.is_empty() { + self.registry.write().unwrap().unregister_signatures(self); + } + } +} #[derive(Debug)] struct RegistryEntry { @@ -63,44 +110,37 @@ struct RegistryEntry { ty: WasmFuncType, } -/// WebAssembly requires that the caller and callee signatures in an indirect -/// call must match. To implement this efficiently, keep a registry of all -/// signatures, shared by all instances, so that call sites can just do an -/// index comparison. #[derive(Debug, Default)] -pub struct SignatureRegistry { +struct SignatureRegistryInner { map: HashMap, entries: Vec>, free: Vec, } -impl SignatureRegistry { - /// Registers a module with the signature registry from the collection of - /// all signatures and trampolines in the module. - pub fn register_module( +impl SignatureRegistryInner { + fn register_for_module( &mut self, signatures: &PrimaryMap, trampolines: impl Iterator, - ) -> (SharedSignatures, TrampolineMap) { - let mut sigs = SharedSignatures::default(); - let mut map = TrampolineMap::default(); + ) -> ( + PrimaryMap, + HashMap, + ) { + let mut sigs = PrimaryMap::default(); + let mut map = HashMap::default(); for (_, ty) in signatures.iter() { sigs.push(self.register(ty)); } for (index, trampoline) in trampolines { - let index = self.map[&signatures[index]]; - map.insert(index, trampoline); + map.insert(sigs[index], (1, trampoline)); } (sigs, map) } - /// Registers a single signature with the registry. - /// - /// This is used for registering host functions created with the Wasmtime API. - pub fn register(&mut self, ty: &WasmFuncType) -> VMSharedSignatureIndex { + fn register(&mut self, ty: &WasmFuncType) -> VMSharedSignatureIndex { let len = self.map.len(); let index = match self.map.entry(ty.clone()) { @@ -124,6 +164,10 @@ impl SignatureRegistry { } }; + // The entry should be missing for one just allocated or + // taken from the free list + assert!(entry.is_none()); + *entry = Some(RegistryEntry { references: 0, ty: ty.clone(), @@ -141,45 +185,78 @@ impl SignatureRegistry { index } - /// Unregisters a collection of shared indexes from the registry. - pub fn unregister(&mut self, indexes: impl Iterator) { - for index in indexes { - let removed = { - let entry = self.entries[index.bits() as usize].as_mut().unwrap(); - - debug_assert!(entry.references > 0); - entry.references -= 1; - - if entry.references == 0 { - self.map.remove(&entry.ty); - self.free.push(index); - true - } else { - false - } - }; - - if removed { - self.entries[index.bits() as usize] = None; + fn unregister_signatures(&mut self, collection: &SignatureCollection) { + // If the collection has a populated signatures map, use it to deregister + // This is always 1:1 from entry to registration + if !collection.signatures.is_empty() { + for (_, index) in collection.signatures.iter() { + self.unregister_entry(*index, 1); + } + } else { + // Otherwise, use the trampolines map, which has reference counts related + // to the stored index + for (index, (count, _)) in collection.trampolines.iter() { + self.unregister_entry(*index, *count); } } } - /// Looks up a function type from a shared signature index. - pub fn lookup_type(&self, index: VMSharedSignatureIndex) -> Option<&WasmFuncType> { - self.entries - .get(index.bits() as usize) - .and_then(|e| e.as_ref().map(|e| &e.ty)) - } + fn unregister_entry(&mut self, index: VMSharedSignatureIndex, count: usize) { + let removed = { + let entry = self.entries[index.bits() as usize].as_mut().unwrap(); - /// Determines if the registry is semantically empty. - pub fn is_empty(&self) -> bool { - // If the map is empty, assert that all remaining entries are "free" - if self.map.is_empty() { - assert!(self.free.len() == self.entries.len()); - true - } else { - false + debug_assert!(entry.references >= count); + entry.references -= count; + + if entry.references == 0 { + self.map.remove(&entry.ty); + self.free.push(index); + true + } else { + false + } + }; + + if removed { + self.entries[index.bits() as usize] = None; } } } + +// `SignatureRegistryInner` implements `Drop` in debug builds to assert that +// all signatures have been unregistered for the registry. +#[cfg(debug_assertions)] +impl Drop for SignatureRegistryInner { + fn drop(&mut self) { + assert!( + self.map.is_empty() && self.free.len() == self.entries.len(), + "signature registry not empty" + ); + } +} + +/// Implements a shared signature registry. +/// +/// WebAssembly requires that the caller and callee signatures in an indirect +/// call must match. To implement this efficiently, keep a registry of all +/// signatures, shared by all instances, so that call sites can just do an +/// index comparison. +#[derive(Debug)] +pub struct SignatureRegistry(Arc>); + +impl SignatureRegistry { + /// Creates a new shared signature registry. + pub fn new() -> Self { + Self(Arc::new(RwLock::new(SignatureRegistryInner::default()))) + } + + /// Looks up a function type from a shared signature index. + pub fn lookup_type(&self, index: VMSharedSignatureIndex) -> Option { + self.0 + .read() + .unwrap() + .entries + .get(index.bits() as usize) + .and_then(|e| e.as_ref().map(|e| &e.ty).cloned()) + } +} diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index f337b39975..6b4c65a1fc 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -1,6 +1,6 @@ use crate::{ - module::ModuleRegistry, signatures::TrampolineMap, trampoline::StoreInstanceHandle, Engine, - Func, Module, Trap, + module::ModuleRegistry, signatures::SignatureCollection, trampoline::StoreInstanceHandle, + Engine, Func, Module, Trap, }; use anyhow::{bail, Result}; use std::any::{Any, TypeId}; @@ -15,11 +15,10 @@ use std::ptr; use std::rc::Rc; use std::sync::Arc; use std::task::{Context, Poll}; -use wasmtime_environ::wasm::WasmFuncType; use wasmtime_jit::ModuleCode; use wasmtime_runtime::{ InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler, TrapInfo, - VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, + VMCallerCheckedAnyfunc, VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMTrampoline, }; @@ -76,7 +75,8 @@ pub(crate) struct StoreInner { signal_handler: RefCell>>>, externref_activations_table: VMExternRefActivationsTable, modules: RefCell, - trampolines: RefCell, + // The signatures and trampolines for `Func` objects + signatures: RefCell, // Numbers of resources instantiated in this store. instance_count: Cell, memory_count: Cell, @@ -139,7 +139,7 @@ impl Store { signal_handler: RefCell::new(None), externref_activations_table: VMExternRefActivationsTable::new(), modules: RefCell::new(ModuleRegistry::default()), - trampolines: RefCell::new(TrampolineMap::default()), + signatures: RefCell::new(SignatureCollection::new(engine.signatures())), instance_count: Default::default(), memory_count: Default::default(), table_count: Default::default(), @@ -203,41 +203,31 @@ impl Store { } } - pub(crate) fn register_module(&self, module: &Module) { - // Register the module with the registry - self.inner.modules.borrow_mut().register(module); + pub(crate) fn signatures(&self) -> &RefCell { + &self.inner.signatures } - // This is used to register a `Func` with the store - pub(crate) fn register_signature( - &self, - ty: &WasmFuncType, - trampoline: VMTrampoline, - ) -> VMSharedSignatureIndex { - let index = self.inner.engine.register_signature(ty); - self.inner - .trampolines - .borrow_mut() - .insert(index, trampoline); - index - } - - pub(crate) fn lookup_trampoline(&self, index: VMSharedSignatureIndex) -> VMTrampoline { + pub(crate) fn lookup_trampoline(&self, anyfunc: &VMCallerCheckedAnyfunc) -> VMTrampoline { // Look up the trampoline with the store's trampolines (from `Func`). - if let Some(trampoline) = self.inner.trampolines.borrow().get(index) { + if let Some(trampoline) = self + .inner + .signatures + .borrow() + .trampoline(anyfunc.type_index) + { return trampoline; } // Look up the trampoline with the registered modules - if let Some(trampoline) = self.inner.modules.borrow().lookup_trampoline(index) { + if let Some(trampoline) = self.inner.modules.borrow().lookup_trampoline(anyfunc) { return trampoline; } // Lastly, check with the engine (for `HostFunc`) self.inner .engine - .host_func_trampolines() - .get(index) + .host_func_signatures() + .trampoline(anyfunc.type_index) .expect("trampoline missing") } @@ -451,7 +441,7 @@ impl Store { } #[inline] - pub(crate) fn stack_map_lookup(&self) -> *const dyn wasmtime_runtime::StackMapLookup { + pub(crate) fn stack_map_lookup(&self) -> &dyn wasmtime_runtime::StackMapLookup { self.inner.as_ref() } @@ -918,16 +908,10 @@ impl Drop for StoreInner { } } } - - let trampolines = self.trampolines.borrow(); - - if !trampolines.is_empty() { - self.engine.unregister_signatures(trampolines.indexes()); - } } } -impl wasmtime_runtime::StackMapLookup for StoreInner { +unsafe impl wasmtime_runtime::StackMapLookup for StoreInner { fn lookup(&self, pc: usize) -> Option<*const wasmtime_environ::ir::StackMap> { // The address of the stack map is stable for the lifetime of the store self.modules.borrow().lookup_stack_map(pc).map(|m| m as _) diff --git a/crates/wasmtime/src/trampoline.rs b/crates/wasmtime/src/trampoline.rs index 21eac2a134..1590bf444a 100644 --- a/crates/wasmtime/src/trampoline.rs +++ b/crates/wasmtime/src/trampoline.rs @@ -77,7 +77,7 @@ fn create_handle( externref_activations_table: store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_lookup: Some(store.stack_map_lookup()), + stack_map_lookup: Some(std::mem::transmute(store.stack_map_lookup())), }, )?; diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 0265c0b3c1..0f95703268 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -270,7 +270,11 @@ pub fn create_function( // If there is no store, use the default signature index which is // guaranteed to trap if there is ever an indirect call on the function (should not happen) let shared_signature_id = store - .map(|s| s.register_signature(ft.as_wasm_func_type(), trampoline)) + .map(|s| { + s.signatures() + .borrow_mut() + .register(ft.as_wasm_func_type(), trampoline) + }) .unwrap_or(VMSharedSignatureIndex::default()); unsafe { diff --git a/crates/wasmtime/src/types/matching.rs b/crates/wasmtime/src/types/matching.rs index 55a55c1d62..7405194c12 100644 --- a/crates/wasmtime/src/types/matching.rs +++ b/crates/wasmtime/src/types/matching.rs @@ -1,4 +1,4 @@ -use crate::{signatures::SharedSignatures, Extern, Store}; +use crate::{signatures::SignatureCollection, Extern, Store}; use anyhow::{bail, Context, Result}; use wasmtime_environ::wasm::{ EntityType, Global, InstanceTypeIndex, Memory, ModuleTypeIndex, SignatureIndex, Table, @@ -6,7 +6,7 @@ use wasmtime_environ::wasm::{ use wasmtime_jit::TypeTables; pub struct MatchCx<'a> { - pub signatures: &'a SharedSignatures, + pub signatures: &'a SignatureCollection, pub types: &'a TypeTables, pub store: &'a Store, } @@ -71,8 +71,8 @@ impl MatchCx<'_> { } pub fn func(&self, expected: SignatureIndex, actual: &crate::Func) -> Result<()> { - let matches = match self.signatures.get(expected) { - Some(idx) => actual.sig_index() == *idx, + let matches = match self.signatures.shared_signature(expected) { + Some(idx) => actual.sig_index() == idx, // If our expected signature isn't registered, then there's no way // that `actual` can match it. None => false, @@ -133,7 +133,7 @@ impl MatchCx<'_> { fn imports_match<'a>( &self, expected: ModuleTypeIndex, - actual_signatures: &SharedSignatures, + actual_signatures: &SignatureCollection, actual_types: &TypeTables, actual_imports: impl Iterator, ) -> Result<()> { @@ -162,7 +162,7 @@ impl MatchCx<'_> { fn exports_match( &self, expected: InstanceTypeIndex, - actual_signatures: &SharedSignatures, + actual_signatures: &SignatureCollection, actual_types: &TypeTables, lookup: impl Fn(&str) -> Option, ) -> Result<()> { @@ -186,7 +186,7 @@ impl MatchCx<'_> { &self, expected: &EntityType, actual_ty: &EntityType, - actual_signatures: &SharedSignatures, + actual_signatures: &SignatureCollection, actual_types: &TypeTables, ) -> Result<()> { let actual_desc = match actual_ty { From 726a936474b9dcf2e4e8406fdb4a877bf6809812 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 16 Apr 2021 10:41:55 -0700 Subject: [PATCH 15/43] Remove ArcModuleCode as it is no longer used. --- crates/wasmtime/src/store.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 6b4c65a1fc..8103278b1f 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -15,7 +15,6 @@ use std::ptr; use std::rc::Rc; use std::sync::Arc; use std::task::{Context, Poll}; -use wasmtime_jit::ModuleCode; use wasmtime_runtime::{ InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler, TrapInfo, VMCallerCheckedAnyfunc, VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, @@ -946,24 +945,6 @@ impl InterruptHandle { } } -// Wrapper struct to implement hash/equality based on the pointer value of the -// `Arc` in question. -struct ArcModuleCode(Arc); - -impl PartialEq for ArcModuleCode { - fn eq(&self, other: &ArcModuleCode) -> bool { - Arc::ptr_eq(&self.0, &other.0) - } -} - -impl Eq for ArcModuleCode {} - -impl Hash for ArcModuleCode { - fn hash(&self, hasher: &mut H) { - Arc::as_ptr(&self.0).hash(hasher) - } -} - struct Reset<'a, T: Copy>(&'a Cell, T); impl Drop for Reset<'_, T> { From 6ac1321162b7a7035082804118d4d1eccbc3259a Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 16 Apr 2021 11:06:24 -0700 Subject: [PATCH 16/43] Minor corrections with latest changes. --- crates/runtime/src/externref.rs | 2 +- crates/wasmtime/src/func/typed.rs | 2 +- crates/wasmtime/src/instance.rs | 2 +- crates/wasmtime/src/trampoline.rs | 2 +- crates/wasmtime/src/values.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index 44a60ce797..0eed6562b5 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -750,7 +750,7 @@ impl VMExternRefActivationsTable { /// /// It is the responsibility of the caller to not have the pointer outlive /// the stack map lookup trait object. -pub unsafe trait StackMapLookup { +pub unsafe trait StackMapLookup: 'static { /// Lookup the stack map at a program counter (PC) value. fn lookup(&self, pc: usize) -> Option<*const StackMap>; } diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index b0fe8b574f..627c2f9f3f 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -207,7 +207,7 @@ unsafe impl WasmTy for Option { unsafe { store .externref_activations_table() - .insert_with_gc(x.inner, &*store.stack_map_lookup()); + .insert_with_gc(x.inner, store.stack_map_lookup()); } abi } else { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index cf807aee6a..56c2ec3f30 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -524,7 +524,7 @@ impl<'a> Instantiator<'a> { externref_activations_table: self.store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_lookup: Some(std::mem::transmute(self.store.stack_map_lookup())), + stack_map_lookup: Some(self.store.stack_map_lookup()), })?; // After we've created the `InstanceHandle` we still need to run diff --git a/crates/wasmtime/src/trampoline.rs b/crates/wasmtime/src/trampoline.rs index 1590bf444a..21eac2a134 100644 --- a/crates/wasmtime/src/trampoline.rs +++ b/crates/wasmtime/src/trampoline.rs @@ -77,7 +77,7 @@ fn create_handle( externref_activations_table: store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_lookup: Some(std::mem::transmute(store.stack_map_lookup())), + stack_map_lookup: Some(store.stack_map_lookup()), }, )?; diff --git a/crates/wasmtime/src/values.rs b/crates/wasmtime/src/values.rs index 9e10f0aa83..477f0e36b8 100644 --- a/crates/wasmtime/src/values.rs +++ b/crates/wasmtime/src/values.rs @@ -98,7 +98,7 @@ impl Val { let externref_ptr = x.inner.as_raw(); store .externref_activations_table() - .insert_with_gc(x.inner, &*store.stack_map_lookup()); + .insert_with_gc(x.inner, store.stack_map_lookup()); ptr::write(p as *mut *mut u8, externref_ptr) } Val::FuncRef(f) => ptr::write( From b775b68cfb623f8dbf0448986cc43e8e5458aa36 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 16 Apr 2021 12:05:38 -0700 Subject: [PATCH 17/43] Make module information lookup from runtime safe. This commit uses a two-phase lookup of stack map information from modules rather than giving back raw pointers to stack maps. First the runtime looks up information about a module from a pc value, which returns an `Arc` it keeps a reference on while completing the stack map lookup. Second it then queries the module information for the stack map from a pc value, getting a reference to the stack map (which is now safe because of the `Arc` held by the runtime). --- crates/environ/src/vmoffsets.rs | 16 ++-- crates/runtime/src/externref.rs | 86 +++++++++---------- crates/runtime/src/instance.rs | 8 +- crates/runtime/src/instance/allocator.rs | 8 +- .../runtime/src/instance/allocator/pooling.rs | 4 +- .../src/instance/allocator/pooling/uffd.rs | 2 +- crates/runtime/src/libcalls.rs | 8 +- crates/wasmtime/src/func/typed.rs | 2 +- crates/wasmtime/src/instance.rs | 2 +- crates/wasmtime/src/module/registry.rs | 67 ++++++++------- crates/wasmtime/src/store.rs | 15 ++-- crates/wasmtime/src/trampoline.rs | 2 +- .../wasmtime/src/trampoline/create_handle.rs | 2 +- crates/wasmtime/src/trampoline/func.rs | 4 +- crates/wasmtime/src/values.rs | 2 +- 15 files changed, 116 insertions(+), 112 deletions(-) diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 20f3b8a298..6fc68f5e28 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -6,7 +6,7 @@ // struct VMContext { // interrupts: *const VMInterrupts, // externref_activations_table: *mut VMExternRefActivationsTable, -// stack_map_lookup: *const dyn StackMapLookup, +// module_info_lookup: *const dyn ModuleInfoLookup, // signature_ids: [VMSharedSignatureIndex; module.num_signature_ids], // imported_functions: [VMFunctionImport; module.num_imported_functions], // imported_tables: [VMTableImport; module.num_imported_tables], @@ -77,7 +77,7 @@ pub struct VMOffsets { // precalculated offsets of various member fields interrupts: u32, externref_activations_table: u32, - stack_map_lookup: u32, + module_info_lookup: u32, signature_ids: u32, imported_functions: u32, imported_tables: u32, @@ -149,7 +149,7 @@ impl From for VMOffsets { num_defined_globals: fields.num_defined_globals, interrupts: 0, externref_activations_table: 0, - stack_map_lookup: 0, + module_info_lookup: 0, signature_ids: 0, imported_functions: 0, imported_tables: 0, @@ -168,12 +168,12 @@ impl From for VMOffsets { .interrupts .checked_add(u32::from(fields.pointer_size)) .unwrap(); - ret.stack_map_lookup = ret + ret.module_info_lookup = ret .externref_activations_table .checked_add(u32::from(fields.pointer_size)) .unwrap(); ret.signature_ids = ret - .stack_map_lookup + .module_info_lookup .checked_add(u32::from(fields.pointer_size * 2)) .unwrap(); ret.imported_functions = ret @@ -507,10 +507,10 @@ impl VMOffsets { self.externref_activations_table } - /// The offset of the `*const dyn StackMapLookup` member. + /// The offset of the `*const dyn ModuleInfoLookup` member. #[inline] - pub fn vmctx_stack_map_lookup(&self) -> u32 { - self.stack_map_lookup + pub fn vmctx_module_info_lookup(&self) -> u32 { + self.module_info_lookup } /// The offset of the `signature_ids` array. diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index 0eed6562b5..4d7b8f9c47 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -99,7 +99,6 @@ //! Examination of Deferred Reference Counting and Cycle Detection* by Quinane: //! -use std::alloc::Layout; use std::any::Any; use std::cell::{Cell, RefCell, UnsafeCell}; use std::cmp::Ordering; @@ -108,6 +107,7 @@ use std::hash::{Hash, Hasher}; use std::mem; use std::ops::Deref; use std::ptr::{self, NonNull}; +use std::{alloc::Layout, sync::Arc}; use wasmtime_environ::ir::StackMap; /// An external reference to some opaque data. @@ -594,10 +594,10 @@ impl VMExternRefActivationsTable { pub unsafe fn insert_with_gc( &self, externref: VMExternRef, - stack_map_lookup: &dyn StackMapLookup, + module_info_lookup: &dyn ModuleInfoLookup, ) { if let Err(externref) = self.try_insert(externref) { - self.gc_and_insert_slow(externref, stack_map_lookup); + self.gc_and_insert_slow(externref, module_info_lookup); } } @@ -605,9 +605,9 @@ impl VMExternRefActivationsTable { unsafe fn gc_and_insert_slow( &self, externref: VMExternRef, - stack_map_lookup: &dyn StackMapLookup, + module_info_lookup: &dyn ModuleInfoLookup, ) { - gc(stack_map_lookup, self); + gc(module_info_lookup, self); // Might as well insert right into the hash set, rather than the bump // chunk, since we are already on a slow path and we get de-duplication @@ -741,29 +741,28 @@ impl VMExternRefActivationsTable { } } -/// Used by the runtime to lookup a stack map from a PC value. -/// -/// # Safety -/// -/// This trait is unsafe as it returns pointers to a stack map without -/// any clear ownership. -/// -/// It is the responsibility of the caller to not have the pointer outlive -/// the stack map lookup trait object. -pub unsafe trait StackMapLookup: 'static { - /// Lookup the stack map at a program counter (PC) value. - fn lookup(&self, pc: usize) -> Option<*const StackMap>; +/// Used by the runtime to lookup information about a module given a +/// program counter value. +pub trait ModuleInfoLookup: 'static { + /// Lookup the module information from a program counter value. + fn lookup(&self, pc: usize) -> Option>; } -pub(crate) struct EmptyStackMapLookup; +/// Used by the runtime to query module information. +pub trait ModuleInfo { + /// Lookup the stack map at a program counter value. + fn lookup_stack_map(&self, pc: usize) -> Option<&StackMap>; +} -unsafe impl StackMapLookup for EmptyStackMapLookup { - fn lookup(&self, _pc: usize) -> Option<*const StackMap> { +pub(crate) struct EmptyModuleInfoLookup; + +impl ModuleInfoLookup for EmptyModuleInfoLookup { + fn lookup(&self, _pc: usize) -> Option> { None } } -pub(crate) const EMPTY_STACK_MAP_LOOKUP: EmptyStackMapLookup = EmptyStackMapLookup; +pub(crate) const EMPTY_MODULE_LOOKUP: EmptyModuleInfoLookup = EmptyModuleInfoLookup; #[derive(Debug, Default)] struct DebugOnly { @@ -810,7 +809,7 @@ impl std::ops::DerefMut for DebugOnly { /// Additionally, you must have registered the stack maps for every Wasm module /// that has frames on the stack with the given `stack_maps_registry`. pub unsafe fn gc( - stack_map_lookup: &dyn StackMapLookup, + module_info_lookup: &dyn ModuleInfoLookup, externref_activations_table: &VMExternRefActivationsTable, ) { // We borrow the precise stack roots `RefCell` for the whole duration of @@ -848,8 +847,7 @@ pub unsafe fn gc( if cfg!(debug_assertions) { // Assert that there aren't any Wasm frames on the stack. backtrace::trace(|frame| { - let stack_map = stack_map_lookup.lookup(frame.ip() as usize); - assert!(stack_map.is_none()); + assert!(module_info_lookup.lookup(frame.ip() as usize).is_none()); true }); } @@ -893,28 +891,30 @@ pub unsafe fn gc( let pc = frame.ip() as usize; let sp = frame.sp() as usize; - if let Some(stack_map) = stack_map_lookup.lookup(pc) { - debug_assert!(sp != 0, "we should always get a valid SP for Wasm frames"); + if let Some(module_info) = module_info_lookup.lookup(pc) { + if let Some(stack_map) = module_info.lookup_stack_map(pc) { + debug_assert!(sp != 0, "we should always get a valid SP for Wasm frames"); - for i in 0..((*stack_map).mapped_words() as usize) { - if (*stack_map).get_bit(i) { - // Stack maps have one bit per word in the frame, and the - // zero^th bit is the *lowest* addressed word in the frame, - // i.e. the closest to the SP. So to get the `i`^th word in - // this frame, we add `i * sizeof(word)` to the SP. - let ptr_to_ref = sp + i * mem::size_of::(); + for i in 0..(stack_map.mapped_words() as usize) { + if stack_map.get_bit(i) { + // Stack maps have one bit per word in the frame, and the + // zero^th bit is the *lowest* addressed word in the frame, + // i.e. the closest to the SP. So to get the `i`^th word in + // this frame, we add `i * sizeof(word)` to the SP. + let ptr_to_ref = sp + i * mem::size_of::(); - let r = std::ptr::read(ptr_to_ref as *const *mut VMExternData); - debug_assert!( - r.is_null() || activations_table_set.contains(&r), - "every on-stack externref inside a Wasm frame should \ - have an entry in the VMExternRefActivationsTable" - ); - if let Some(r) = NonNull::new(r) { - VMExternRefActivationsTable::insert_precise_stack_root( - &mut precise_stack_roots, - r, + let r = std::ptr::read(ptr_to_ref as *const *mut VMExternData); + debug_assert!( + r.is_null() || activations_table_set.contains(&r), + "every on-stack externref inside a Wasm frame should \ + have an entry in the VMExternRefActivationsTable" ); + if let Some(r) = NonNull::new(r) { + VMExternRefActivationsTable::insert_precise_stack_root( + &mut precise_stack_roots, + r, + ); + } } } } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 33a2583cff..7da8d11ee5 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -3,7 +3,7 @@ //! `InstanceHandle` is a reference-counting handle for an `Instance`. use crate::export::Export; -use crate::externref::{StackMapLookup, VMExternRefActivationsTable}; +use crate::externref::{ModuleInfoLookup, VMExternRefActivationsTable}; use crate::memory::{Memory, RuntimeMemoryCreator}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; @@ -249,9 +249,9 @@ impl Instance { unsafe { self.vmctx_plus_offset(self.offsets.vmctx_externref_activations_table()) } } - /// Return a pointer to the `StackMapLookup`. - pub fn stack_map_lookup(&self) -> *mut *const dyn StackMapLookup { - unsafe { self.vmctx_plus_offset(self.offsets.vmctx_stack_map_lookup()) } + /// Return a pointer to the `ModuleInfoLookup`. + pub fn module_info_lookup(&self) -> *mut *const dyn ModuleInfoLookup { + unsafe { self.vmctx_plus_offset(self.offsets.vmctx_module_info_lookup()) } } /// Return a reference to the vmctx used by compiled wasm code. diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 14a9d2eff3..03618a69d1 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -1,4 +1,4 @@ -use crate::externref::{StackMapLookup, VMExternRefActivationsTable, EMPTY_STACK_MAP_LOOKUP}; +use crate::externref::{ModuleInfoLookup, VMExternRefActivationsTable, EMPTY_MODULE_LOOKUP}; use crate::imports::Imports; use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator}; use crate::memory::{DefaultMemoryCreator, Memory}; @@ -57,8 +57,8 @@ pub struct InstanceAllocationRequest<'a> { /// The pointer to the reference activations table to use for the instance. pub externref_activations_table: *mut VMExternRefActivationsTable, - /// The pointer to the stack map lookup to use for the instance. - pub stack_map_lookup: Option<*const dyn StackMapLookup>, + /// The pointer to the module info lookup to use for the instance. + pub module_info_lookup: Option<*const dyn ModuleInfoLookup>, } /// An link error while instantiating a module. @@ -447,7 +447,7 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque *instance.interrupts() = req.interrupts; *instance.externref_activations_table() = req.externref_activations_table; - *instance.stack_map_lookup() = req.stack_map_lookup.unwrap_or(&EMPTY_STACK_MAP_LOOKUP); + *instance.module_info_lookup() = req.module_info_lookup.unwrap_or(&EMPTY_MODULE_LOOKUP); // Initialize shared signatures let mut ptr = instance.signature_ids_ptr(); diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 6b89ce603e..5bc342dafc 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -1370,7 +1370,7 @@ mod test { host_state: Box::new(()), interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_lookup: None, + module_info_lookup: None, }, ) .expect("allocation should succeed"), @@ -1394,7 +1394,7 @@ mod test { host_state: Box::new(()), interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_lookup: None, + module_info_lookup: None, }, ) { Err(InstantiationError::Limit(3)) => {} diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index fb811680bb..4c4b22caba 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -523,7 +523,7 @@ mod test { host_state: Box::new(()), interrupts: ptr::null(), externref_activations_table: ptr::null_mut(), - stack_map_lookup: None, + module_info_lookup: None, }, ) .expect("instance should allocate"), diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index 58eb585312..9c5800fd8e 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -449,8 +449,8 @@ pub unsafe extern "C" fn wasmtime_activations_table_insert_with_gc( let externref = VMExternRef::clone_from_raw(externref); let instance = (&mut *vmctx).instance(); let activations_table = &**instance.externref_activations_table(); - let stack_map_lookup = &**instance.stack_map_lookup(); - activations_table.insert_with_gc(externref, stack_map_lookup); + let module_info_lookup = &**instance.module_info_lookup(); + activations_table.insert_with_gc(externref, module_info_lookup); } /// Perform a Wasm `global.get` for `externref` globals. @@ -466,8 +466,8 @@ pub unsafe extern "C" fn wasmtime_externref_global_get( Some(externref) => { let raw = externref.as_raw(); let activations_table = &**instance.externref_activations_table(); - let stack_map_lookup = &**instance.stack_map_lookup(); - activations_table.insert_with_gc(externref, stack_map_lookup); + let module_info_lookup = &**instance.module_info_lookup(); + activations_table.insert_with_gc(externref, module_info_lookup); raw } } diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index 627c2f9f3f..107810f84b 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -207,7 +207,7 @@ unsafe impl WasmTy for Option { unsafe { store .externref_activations_table() - .insert_with_gc(x.inner, store.stack_map_lookup()); + .insert_with_gc(x.inner, store.module_info_lookup()); } abi } else { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 56c2ec3f30..04a47337ca 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -524,7 +524,7 @@ impl<'a> Instantiator<'a> { externref_activations_table: self.store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_lookup: Some(self.store.stack_map_lookup()), + module_info_lookup: Some(self.store.module_info_lookup()), })?; // After we've created the `InstanceHandle` we still need to run diff --git a/crates/wasmtime/src/module/registry.rs b/crates/wasmtime/src/module/registry.rs index 3691c38834..6687119106 100644 --- a/crates/wasmtime/src/module/registry.rs +++ b/crates/wasmtime/src/module/registry.rs @@ -6,10 +6,13 @@ use std::{ sync::{Arc, Mutex}, }; use wasmtime_environ::{ - entity::EntityRef, ir, wasm::DefinedFuncIndex, FunctionAddressMap, TrapInformation, + entity::EntityRef, + ir::{self, StackMap}, + wasm::DefinedFuncIndex, + FunctionAddressMap, TrapInformation, }; use wasmtime_jit::CompiledModule; -use wasmtime_runtime::{VMCallerCheckedAnyfunc, VMTrampoline}; +use wasmtime_runtime::{ModuleInfo, VMCallerCheckedAnyfunc, VMTrampoline}; lazy_static::lazy_static! { static ref GLOBAL_MODULES: Mutex = Default::default(); @@ -27,7 +30,7 @@ fn func_by_pc(module: &CompiledModule, pc: usize) -> Option<(DefinedFuncIndex, u /// /// The `BTreeMap` is used to quickly locate a module based on a program counter value. #[derive(Default)] -pub struct ModuleRegistry(BTreeMap); +pub struct ModuleRegistry(BTreeMap>); impl ModuleRegistry { /// Fetches frame information about a program counter in a backtrace. @@ -48,12 +51,13 @@ impl ModuleRegistry { self.module(pc)?.lookup_trap_info(pc) } - /// Looks up a stack map from a program counter. - pub fn lookup_stack_map<'a>(&'a self, pc: usize) -> Option<&'a ir::StackMap> { - self.module(pc)?.lookup_stack_map(pc) + /// Fetches information about a registered module given a program counter value. + pub fn lookup_module(&self, pc: usize) -> Option> { + self.module(pc) + .map(|m| -> Arc { m.clone() }) } - fn module(&self, pc: usize) -> Option<&RegisteredModule> { + fn module(&self, pc: usize) -> Option<&Arc> { let (end, info) = self.0.range(pc..).next()?; if pc < info.start || *end < pc { return None; @@ -94,11 +98,11 @@ impl ModuleRegistry { let prev = self.0.insert( end, - RegisteredModule { + Arc::new(RegisteredModule { start, module: compiled_module.clone(), signatures: module.signatures().clone(), - }, + }), ); assert!(prev.is_none()); @@ -209,8 +213,29 @@ impl RegisteredModule { Some(&info.traps[idx]) } - /// Looks up a stack map from a program counter - pub fn lookup_stack_map(&self, pc: usize) -> Option<&ir::StackMap> { + fn instr_pos(offset: u32, addr_map: &FunctionAddressMap) -> Option { + // Use our relative position from the start of the function to find the + // machine instruction that corresponds to `pc`, which then allows us to + // map that to a wasm original source location. + match addr_map + .instructions + .binary_search_by_key(&offset, |map| map.code_offset) + { + // Exact hit! + Ok(pos) => Some(pos), + + // This *would* be at the first slot in the array, so no + // instructions cover `pc`. + Err(0) => None, + + // This would be at the `nth` slot, so we're at the `n-1`th slot. + Err(n) => Some(n - 1), + } + } +} + +impl ModuleInfo for RegisteredModule { + fn lookup_stack_map(&self, pc: usize) -> Option<&StackMap> { let (index, offset) = func_by_pc(&self.module, pc)?; let info = self.module.func_info(index); @@ -275,26 +300,6 @@ impl RegisteredModule { Some(&info.stack_maps[index].stack_map) } - - fn instr_pos(offset: u32, addr_map: &FunctionAddressMap) -> Option { - // Use our relative position from the start of the function to find the - // machine instruction that corresponds to `pc`, which then allows us to - // map that to a wasm original source location. - match addr_map - .instructions - .binary_search_by_key(&offset, |map| map.code_offset) - { - // Exact hit! - Ok(pos) => Some(pos), - - // This *would* be at the first slot in the array, so no - // instructions cover `pc`. - Err(0) => None, - - // This would be at the `nth` slot, so we're at the `n-1`th slot. - Err(n) => Some(n - 1), - } - } } // Counterpart to `RegisteredModule`, but stored in the global registry. diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 8103278b1f..301b1192c1 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -16,9 +16,9 @@ use std::rc::Rc; use std::sync::Arc; use std::task::{Context, Poll}; use wasmtime_runtime::{ - InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler, TrapInfo, - VMCallerCheckedAnyfunc, VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, - VMTrampoline, + InstanceAllocator, InstanceHandle, ModuleInfo, OnDemandInstanceAllocator, SignalHandler, + TrapInfo, VMCallerCheckedAnyfunc, VMContext, VMExternRef, VMExternRefActivationsTable, + VMInterrupts, VMTrampoline, }; /// Used to associate instances with the store. @@ -440,7 +440,7 @@ impl Store { } #[inline] - pub(crate) fn stack_map_lookup(&self) -> &dyn wasmtime_runtime::StackMapLookup { + pub(crate) fn module_info_lookup(&self) -> &dyn wasmtime_runtime::ModuleInfoLookup { self.inner.as_ref() } @@ -910,10 +910,9 @@ impl Drop for StoreInner { } } -unsafe impl wasmtime_runtime::StackMapLookup for StoreInner { - fn lookup(&self, pc: usize) -> Option<*const wasmtime_environ::ir::StackMap> { - // The address of the stack map is stable for the lifetime of the store - self.modules.borrow().lookup_stack_map(pc).map(|m| m as _) +impl wasmtime_runtime::ModuleInfoLookup for StoreInner { + fn lookup(&self, pc: usize) -> Option> { + self.modules.borrow().lookup_module(pc) } } diff --git a/crates/wasmtime/src/trampoline.rs b/crates/wasmtime/src/trampoline.rs index 21eac2a134..ee0bf25c9e 100644 --- a/crates/wasmtime/src/trampoline.rs +++ b/crates/wasmtime/src/trampoline.rs @@ -77,7 +77,7 @@ fn create_handle( externref_activations_table: store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_lookup: Some(store.stack_map_lookup()), + module_info_lookup: Some(store.module_info_lookup()), }, )?; diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs index 182e421499..d04d2899f9 100644 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ b/crates/wasmtime/src/trampoline/create_handle.rs @@ -43,7 +43,7 @@ pub(crate) fn create_handle( externref_activations_table: store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_lookup: &store, + module_info_lookup: &store, })?; Ok(store.add_instance(handle, true)) diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 0f95703268..0110e7b2a9 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -287,7 +287,7 @@ pub fn create_function( host_state: Box::new(trampoline_state), interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_lookup: None, + module_info_lookup: None, })?, trampoline, )) @@ -319,7 +319,7 @@ pub unsafe fn create_raw_function( host_state, interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_lookup: None, + module_info_lookup: None, })?, ) } diff --git a/crates/wasmtime/src/values.rs b/crates/wasmtime/src/values.rs index 477f0e36b8..7d6adbbedc 100644 --- a/crates/wasmtime/src/values.rs +++ b/crates/wasmtime/src/values.rs @@ -98,7 +98,7 @@ impl Val { let externref_ptr = x.inner.as_raw(); store .externref_activations_table() - .insert_with_gc(x.inner, store.stack_map_lookup()); + .insert_with_gc(x.inner, store.module_info_lookup()); ptr::write(p as *mut *mut u8, externref_ptr) } Val::FuncRef(f) => ptr::write( From dfab471ce5bf8b5cbf29d9b542d143930ea5cf05 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 16 Apr 2021 12:26:42 -0700 Subject: [PATCH 18/43] Remove unused file. This file hasn't been used for a while and was mistakenly not deleted. --- .../wasmtime/src/trampoline/create_handle.rs | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 crates/wasmtime/src/trampoline/create_handle.rs diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs deleted file mode 100644 index d04d2899f9..0000000000 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Support for a calling of an imported function. - -use crate::trampoline::StoreInstanceHandle; -use crate::Store; -use anyhow::Result; -use std::any::Any; -use std::sync::Arc; -use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::wasm::DefinedFuncIndex; -use wasmtime_environ::Module; -use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstanceAllocator, StackMapLookup, - VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMSharedSignatureIndex, -}; - -pub(crate) fn create_handle( - module: Module, - store: &Store, - finished_functions: PrimaryMap, - host_state: Box, - func_imports: &[VMFunctionImport], - shared_signature_id: Option, -) -> Result { - let mut imports = Imports::default(); - imports.functions = func_imports; - let module = Arc::new(module); - - unsafe { - // Use the default allocator when creating handles associated with host objects - // The configured instance allocator should only be used when creating module instances - // as we don't want host objects to count towards instance limits. - let handle = store - .engine() - .config() - .default_instance_allocator - .allocate(InstanceAllocationRequest { - module: module.clone(), - finished_functions: &finished_functions, - imports, - shared_signatures: shared_signature_id.into(), - host_state, - interrupts: store.interrupts(), - externref_activations_table: store.externref_activations_table() - as *const VMExternRefActivationsTable - as *mut _, - module_info_lookup: &store, - })?; - - Ok(store.add_instance(handle, true)) - } -} From ef2ad6375d0dd22d1122088c47aea01a86305d1d Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 16 Apr 2021 12:28:27 -0700 Subject: [PATCH 19/43] Consolidate module construction. This commit adds `Module::from_parts` as an internal constructor that shared the implementation between `Module::from_binary` and module deserialization. --- crates/wasmtime/src/module.rs | 72 +++++++++++++-- crates/wasmtime/src/module/serialization.rs | 97 ++++----------------- 2 files changed, 82 insertions(+), 87 deletions(-) diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index af108d2bac..e9ae9579e3 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -320,12 +320,22 @@ impl Module { } }; - let mut modules = CompiledModule::from_artifacts_list( + let modules = CompiledModule::from_artifacts_list( artifacts, engine.compiler().isa(), &*engine.config().profiler, )?; + Self::from_parts(engine, modules, main_module, Arc::new(types), &[]) + } + + fn from_parts( + engine: &Engine, + mut modules: Vec>, + main_module: usize, + types: Arc, + module_upvars: &[serialization::SerializedModuleUpvar], + ) -> Result { // Validate the module can be used with the current allocator engine.allocator().validate(modules[main_module].module())?; @@ -337,16 +347,68 @@ impl Module { let module = modules.remove(main_module); - Ok(Module { + let module_upvars = module_upvars + .iter() + .map(|m| { + mk( + engine, + &modules, + &types, + m.index, + &m.artifact_upvars, + &m.module_upvars, + &signatures, + ) + }) + .collect::>>()?; + + return Ok(Self { inner: Arc::new(ModuleInner { engine: engine.clone(), + types, module, - types: Arc::new(types), artifact_upvars: modules, - module_upvars: Vec::new(), + module_upvars, signatures, }), - }) + }); + + fn mk( + engine: &Engine, + artifacts: &[Arc], + types: &Arc, + module_index: usize, + artifact_upvars: &[usize], + module_upvars: &[serialization::SerializedModuleUpvar], + signatures: &Arc, + ) -> Result { + Ok(Module { + inner: Arc::new(ModuleInner { + engine: engine.clone(), + types: types.clone(), + module: artifacts[module_index].clone(), + artifact_upvars: artifact_upvars + .iter() + .map(|i| artifacts[*i].clone()) + .collect(), + module_upvars: module_upvars + .into_iter() + .map(|m| { + mk( + engine, + artifacts, + types, + m.index, + &m.artifact_upvars, + &m.module_upvars, + signatures, + ) + }) + .collect::>>()?, + signatures: signatures.clone(), + }), + }) + } } /// Validates `binary` input data as a WebAssembly binary given the diff --git a/crates/wasmtime/src/module/serialization.rs b/crates/wasmtime/src/module/serialization.rs index 0d2836889e..8b3620a75d 100644 --- a/crates/wasmtime/src/module/serialization.rs +++ b/crates/wasmtime/src/module/serialization.rs @@ -1,7 +1,6 @@ //! Implements module serialization. -use super::ModuleInner; -use crate::{signatures::SignatureCollection, Engine, Module, OptLevel}; +use crate::{Engine, Module, OptLevel}; use anyhow::{anyhow, bail, Context, Result}; use bincode::Options; use serde::{Deserialize, Serialize}; @@ -124,13 +123,13 @@ impl From for OptLevel { /// A small helper struct for serialized module upvars. #[derive(Serialize, Deserialize)] -struct SerializedModuleUpvar { +pub struct SerializedModuleUpvar { /// The module's index into the compilation artifact. - index: usize, + pub index: usize, /// Indexes into the list of all compilation artifacts for this module. - artifact_upvars: Vec, + pub artifact_upvars: Vec, /// Closed-over module values that are also needed for this module. - module_upvars: Vec, + pub module_upvars: Vec, } impl SerializedModuleUpvar { @@ -285,8 +284,7 @@ impl<'a> SerializedModule<'a> { self.check_tunables(compiler)?; self.check_features(compiler)?; - let types = Arc::new(self.types.unwrap_owned()); - let mut modules = CompiledModule::from_artifacts_list( + let modules = CompiledModule::from_artifacts_list( self.artifacts .into_iter() .map(|i| i.unwrap_owned()) @@ -295,82 +293,17 @@ impl<'a> SerializedModule<'a> { &*engine.config().profiler, )?; - // Validate the module can be used with the current allocator - engine - .allocator() - .validate(modules.last().unwrap().module())?; + assert!(!modules.is_empty()); - let signatures = Arc::new(SignatureCollection::new_for_module( - engine.signatures(), - &types.wasm_signatures, - modules.iter().flat_map(|m| m.trampolines().iter().cloned()), - )); + let main_module = modules.len() - 1; - let module = modules.pop().unwrap(); - - let module_upvars = self - .module_upvars - .iter() - .map(|m| { - mk( - engine, - &modules, - &types, - m.index, - &m.artifact_upvars, - &m.module_upvars, - &signatures, - ) - }) - .collect::>>()?; - - return Ok(Module { - inner: Arc::new(ModuleInner { - engine: engine.clone(), - types, - module, - artifact_upvars: modules, - module_upvars, - signatures, - }), - }); - - fn mk( - engine: &Engine, - artifacts: &[Arc], - types: &Arc, - module_index: usize, - artifact_upvars: &[usize], - module_upvars: &[SerializedModuleUpvar], - signatures: &Arc, - ) -> Result { - Ok(Module { - inner: Arc::new(ModuleInner { - engine: engine.clone(), - types: types.clone(), - module: artifacts[module_index].clone(), - artifact_upvars: artifact_upvars - .iter() - .map(|i| artifacts[*i].clone()) - .collect(), - module_upvars: module_upvars - .into_iter() - .map(|m| { - mk( - engine, - artifacts, - types, - m.index, - &m.artifact_upvars, - &m.module_upvars, - signatures, - ) - }) - .collect::>>()?, - signatures: signatures.clone(), - }), - }) - } + Module::from_parts( + engine, + modules, + main_module, + Arc::new(self.types.unwrap_owned()), + &self.module_upvars, + ) } pub fn to_bytes(&self) -> Result> { From 52b1166918d6690dae69b4545b984c76a89774d2 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Mon, 19 Apr 2021 23:08:15 +0900 Subject: [PATCH 20/43] Update iter-enum to 1 (#2846) --- Cargo.lock | 4 ++-- crates/lightbeam/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f61d3abc14..76c99d6e9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1368,9 +1368,9 @@ checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" [[package]] name = "iter-enum" -version = "0.2.7" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad34f24d3b48ceffdff38af2df5ce1b7d1d9cc113e503d8e86fe8cdb889c871" +checksum = "4f947f0d9df7e69c4df60a950c0a83741455bb9ebd8fd9b5a87994dda4dbb005" dependencies = [ "derive_utils", "quote", diff --git a/crates/lightbeam/Cargo.toml b/crates/lightbeam/Cargo.toml index 7e6e5f6509..6f30be3d49 100644 --- a/crates/lightbeam/Cargo.toml +++ b/crates/lightbeam/Cargo.toml @@ -17,7 +17,7 @@ cranelift-codegen = { path = "../../cranelift/codegen", version = "0.73.0" } derive_more = "0.99" dynasm = "1.0.0" dynasmrt = "1.0.0" -iter-enum = "0.2" +iter-enum = "1" itertools = "0.10.0" memoffset = "0.6.0" more-asserts = "0.2.1" From f12b4c467c3ac5101791614532f31444575bcdf5 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Mon, 19 Apr 2021 07:19:20 -0700 Subject: [PATCH 21/43] Add resource limiting to the Wasmtime API. (#2736) * Add resource limiting to the Wasmtime API. This commit adds a `ResourceLimiter` trait to the Wasmtime API. When used in conjunction with `Store::new_with_limiter`, this can be used to monitor and prevent WebAssembly code from growing linear memories and tables. This is particularly useful when hosts need to take into account host resource usage to determine if WebAssembly code can consume more resources. A simple `StaticResourceLimiter` is also included with these changes that will simply limit the size of linear memories or tables for all instances created in the store based on static values. * Code review feedback. * Implemented `StoreLimits` and `StoreLimitsBuilder`. * Moved `max_instances`, `max_memories`, `max_tables` out of `Config` and into `StoreLimits`. * Moved storage of the limiter in the runtime into `Memory` and `Table`. * Made `InstanceAllocationRequest` use a reference to the limiter. * Updated docs. * Made `ResourceLimiterProxy` generic to remove a level of indirection. * Fixed the limiter not being used for `wasmtime::Memory` and `wasmtime::Table`. * Code review feedback and bug fix. * `Memory::new` now returns `Result` so that an error can be returned if the initial requested memory exceeds any limits placed on the store. * Changed an `Arc` to `Rc` as the `Arc` wasn't necessary. * Removed `Store` from the `ResourceLimiter` callbacks. Custom resource limiter implementations are free to capture any context they want, so no need to unnecessarily store a weak reference to `Store` from the proxy type. * Fixed a bug in the pooling instance allocator where an instance would be leaked from the pool. Previously, this would only have happened if the OS was unable to make the necessary linear memory available for the instance. With these changes, however, the instance might not be created due to limits placed on the store. We now properly deallocate the instance on error. * Added more tests, including one that covers the fix mentioned above. * Code review feedback. * Add another memory to `test_pooling_allocator_initial_limits_exceeded` to ensure a partially created instance is successfully deallocated. * Update some doc comments for better documentation of `Store` and `ResourceLimiter`. --- RELEASES.md | 13 + crates/c-api/src/config.rs | 5 - crates/c-api/src/memory.rs | 8 +- crates/fuzzing/src/lib.rs | 7 - crates/fuzzing/src/oracles.rs | 27 +- crates/fuzzing/src/oracles/dummy.rs | 2 +- crates/runtime/src/instance.rs | 78 +++- crates/runtime/src/instance/allocator.rs | 30 +- .../runtime/src/instance/allocator/pooling.rs | 81 ++-- .../src/instance/allocator/pooling/uffd.rs | 1 + crates/runtime/src/lib.rs | 2 +- crates/runtime/src/memory.rs | 153 +++++-- crates/runtime/src/table.rs | 103 +++-- crates/wasmtime/src/config.rs | 39 -- crates/wasmtime/src/instance.rs | 1 + crates/wasmtime/src/lib.rs | 2 + crates/wasmtime/src/limits.rs | 208 ++++++++++ crates/wasmtime/src/memory.rs | 19 +- crates/wasmtime/src/store.rs | 81 +++- crates/wasmtime/src/trampoline.rs | 1 + crates/wasmtime/src/trampoline/func.rs | 2 + crates/wasmtime/src/trampoline/memory.rs | 4 + crates/wast/src/spectest.rs | 2 +- examples/memory.rs | 2 +- tests/all/externals.rs | 4 +- tests/all/limits.rs | 381 ++++++++++++++++++ tests/all/linker.rs | 4 +- tests/all/main.rs | 1 + tests/all/memory_creator.rs | 11 +- tests/all/module_linking.rs | 9 +- 30 files changed, 1063 insertions(+), 218 deletions(-) create mode 100644 crates/wasmtime/src/limits.rs create mode 100644 tests/all/limits.rs diff --git a/RELEASES.md b/RELEASES.md index 73ebf304c4..acf115f1c5 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -4,6 +4,19 @@ ## Unreleased +### Added + +* Added `Store::with_limits`, `StoreLimits`, and `ResourceLimiter` to the + Wasmtime API to help with enforcing resource limits at runtime. The + `ResourceLimiter` trait can be implemented by custom resource limiters to + decide if linear memories or tables can be grown. + +### Changed + +* Breaking: `Memory::new` has been changed to return `Result` as creating a + host memory object is now a fallible operation when the initial size of + the memory exceeds the store limits. + ## 0.26.0 Released 2021-04-05. diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index f84bc320cd..3e6e313ba9 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -176,8 +176,3 @@ pub extern "C" fn wasmtime_config_static_memory_guard_size_set(c: &mut wasm_conf pub extern "C" fn wasmtime_config_dynamic_memory_guard_size_set(c: &mut wasm_config_t, size: u64) { c.config.dynamic_memory_guard_size(size); } - -#[no_mangle] -pub extern "C" fn wasmtime_config_max_instances_set(c: &mut wasm_config_t, limit: usize) { - c.config.max_instances(limit); -} diff --git a/crates/c-api/src/memory.rs b/crates/c-api/src/memory.rs index 55a001deed..54d3936849 100644 --- a/crates/c-api/src/memory.rs +++ b/crates/c-api/src/memory.rs @@ -31,13 +31,13 @@ impl wasm_memory_t { pub extern "C" fn wasm_memory_new( store: &wasm_store_t, mt: &wasm_memorytype_t, -) -> Box { - let memory = Memory::new(&store.store, mt.ty().ty.clone()); - Box::new(wasm_memory_t { +) -> Option> { + let memory = Memory::new(&store.store, mt.ty().ty.clone()).ok()?; + Some(Box::new(wasm_memory_t { ext: wasm_extern_t { which: memory.into(), }, - }) + })) } #[no_mangle] diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs index 6e4e991c4c..7ef3382411 100644 --- a/crates/fuzzing/src/lib.rs +++ b/crates/fuzzing/src/lib.rs @@ -39,13 +39,6 @@ pub fn fuzz_default_config(strategy: wasmtime::Strategy) -> anyhow::Result Store { + Store::new_with_limits( + &engine, + StoreLimitsBuilder::new() + // The limits here are chosen based on the default "maximum type size" + // configured in wasm-smith, which is 1000. This means that instances + // are allowed to, for example, export up to 1000 memories. We bump that + // a little bit here to give us some slop. + .instances(1100) + .tables(1100) + .memories(1100) + .build(), + ) +} + /// Methods of timing out execution of a WebAssembly module #[derive(Debug)] pub enum Timeout { @@ -95,7 +110,7 @@ pub fn instantiate_with_config( _ => false, }); let engine = Engine::new(&config).unwrap(); - let store = Store::new(&engine); + let store = create_store(&engine); let mut timeout_state = SignalOnDrop::default(); match timeout { @@ -203,7 +218,7 @@ pub fn differential_execution( config.wasm_module_linking(false); let engine = Engine::new(&config).unwrap(); - let store = Store::new(&engine); + let store = create_store(&engine); let module = Module::new(&engine, &wasm).unwrap(); @@ -348,7 +363,7 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) { ApiCall::StoreNew => { log::trace!("creating store"); assert!(store.is_none()); - store = Some(Store::new(engine.as_ref().unwrap())); + store = Some(create_store(engine.as_ref().unwrap())); } ApiCall::ModuleNew { id, wasm } => { @@ -439,7 +454,7 @@ pub fn spectest(fuzz_config: crate::generators::Config, test: crate::generators: config.wasm_reference_types(false); config.wasm_bulk_memory(false); config.wasm_module_linking(false); - let store = Store::new(&Engine::new(&config).unwrap()); + let store = create_store(&Engine::new(&config).unwrap()); if fuzz_config.consume_fuel { store.add_fuel(u64::max_value()).unwrap(); } @@ -463,7 +478,7 @@ pub fn table_ops( let mut config = fuzz_config.to_wasmtime(); config.wasm_reference_types(true); let engine = Engine::new(&config).unwrap(); - let store = Store::new(&engine); + let store = create_store(&engine); if fuzz_config.consume_fuel { store.add_fuel(u64::max_value()).unwrap(); } @@ -578,7 +593,7 @@ pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Con let mut wasmtime_config = config.to_wasmtime(); wasmtime_config.cranelift_nan_canonicalization(true); let wasmtime_engine = Engine::new(&wasmtime_config).unwrap(); - let wasmtime_store = Store::new(&wasmtime_engine); + let wasmtime_store = create_store(&wasmtime_engine); if config.consume_fuel { wasmtime_store.add_fuel(u64::max_value()).unwrap(); } diff --git a/crates/fuzzing/src/oracles/dummy.rs b/crates/fuzzing/src/oracles/dummy.rs index f03bbfa067..99b5be736b 100644 --- a/crates/fuzzing/src/oracles/dummy.rs +++ b/crates/fuzzing/src/oracles/dummy.rs @@ -87,7 +87,7 @@ pub fn dummy_table(store: &Store, ty: TableType) -> Table { /// Construct a dummy memory for the given memory type. pub fn dummy_memory(store: &Store, ty: MemoryType) -> Memory { - Memory::new(store, ty) + Memory::new(store, ty).unwrap() } /// Construct a dummy instance for the given instance type. diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 7da8d11ee5..916aa90f26 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -37,6 +37,52 @@ mod allocator; pub use allocator::*; +/// Used by hosts to limit resource consumption of instances. +/// +/// An instance can be created with a resource limiter so that hosts can take into account +/// non-WebAssembly resource usage to determine if a linear memory or table should grow. +pub trait ResourceLimiter { + /// Notifies the resource limiter that an instance's linear memory has been requested to grow. + /// + /// * `current` is the current size of the linear memory in WebAssembly page units. + /// * `desired` is the desired size of the linear memory in WebAssembly page units. + /// * `maximum` is either the linear memory's maximum or a maximum from an instance allocator, + /// also in WebAssembly page units. A value of `None` indicates that the linear memory is + /// unbounded. + /// + /// This function should return `true` to indicate that the growing operation is permitted or + /// `false` if not permitted. Returning `true` when a maximum has been exceeded will have no + /// effect as the linear memory will not grow. + fn memory_growing(&self, current: u32, desired: u32, maximum: Option) -> bool; + + /// Notifies the resource limiter that an instance's table has been requested to grow. + /// + /// * `current` is the current number of elements in the table. + /// * `desired` is the desired number of elements in the table. + /// * `maximum` is either the table's maximum or a maximum from an instance allocator. + /// A value of `None` indicates that the table is unbounded. + /// + /// This function should return `true` to indicate that the growing operation is permitted or + /// `false` if not permitted. Returning `true` when a maximum has been exceeded will have no + /// effect as the table will not grow. + fn table_growing(&self, current: u32, desired: u32, maximum: Option) -> bool; + + /// The maximum number of instances that can be created for a `Store`. + /// + /// Module instantiation will fail if this limit is exceeded. + fn instances(&self) -> usize; + + /// The maximum number of tables that can be created for a `Store`. + /// + /// Module instantiation will fail if this limit is exceeded. + fn tables(&self) -> usize; + + /// The maximum number of tables that can be created for a `Store`. + /// + /// Module instantiation will fail if this limit is exceeded. + fn memories(&self) -> usize; +} + /// Runtime representation of an instance value, which erases all `Instance` /// information since instances are just a collection of values. pub type RuntimeInstance = Rc>; @@ -378,11 +424,12 @@ impl Instance { /// Returns `None` if memory can't be grown by the specified amount /// of pages. pub(crate) fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option { - let result = self + let memory = self .memories .get(memory_index) - .unwrap_or_else(|| panic!("no memory for index {}", memory_index.index())) - .grow(delta); + .unwrap_or_else(|| panic!("no memory for index {}", memory_index.index())); + + let result = unsafe { memory.grow(delta) }; // Keep current the VMContext pointers used by compiled wasm code. self.set_memory(memory_index, self.memories[memory_index].vmmemory()); @@ -460,19 +507,18 @@ impl Instance { delta: u32, init_value: TableElement, ) -> Option { - unsafe { - let orig_size = self - .tables - .get(table_index) - .unwrap_or_else(|| panic!("no table for index {}", table_index.index())) - .grow(delta, init_value)?; + let table = self + .tables + .get(table_index) + .unwrap_or_else(|| panic!("no table for index {}", table_index.index())); - // Keep the `VMContext` pointers used by compiled Wasm code up to - // date. - self.set_table(table_index, self.tables[table_index].vmtable()); + let result = unsafe { table.grow(delta, init_value) }; - Some(orig_size) - } + // Keep the `VMContext` pointers used by compiled Wasm code up to + // date. + self.set_table(table_index, self.tables[table_index].vmtable()); + + result } pub(crate) fn defined_table_fill( @@ -818,10 +864,6 @@ pub struct InstanceHandle { } impl InstanceHandle { - pub(crate) unsafe fn new(instance: *mut Instance) -> Self { - Self { instance } - } - /// Create a new `InstanceHandle` pointing at the instance /// pointed to by the given `VMContext` pointer. /// diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 03618a69d1..1e4d800877 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -1,6 +1,6 @@ use crate::externref::{ModuleInfoLookup, VMExternRefActivationsTable, EMPTY_MODULE_LOOKUP}; use crate::imports::Imports; -use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator}; +use crate::instance::{Instance, InstanceHandle, ResourceLimiter, RuntimeMemoryCreator}; use crate::memory::{DefaultMemoryCreator, Memory}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; @@ -15,6 +15,7 @@ use std::any::Any; use std::cell::RefCell; use std::convert::TryFrom; use std::ptr::{self, NonNull}; +use std::rc::Rc; use std::slice; use std::sync::Arc; use thiserror::Error; @@ -59,6 +60,9 @@ pub struct InstanceAllocationRequest<'a> { /// The pointer to the module info lookup to use for the instance. pub module_info_lookup: Option<*const dyn ModuleInfoLookup>, + + /// The resource limiter to use for the instance. + pub limiter: Option<&'a Rc>, } /// An link error while instantiating a module. @@ -590,19 +594,23 @@ impl OnDemandInstanceAllocator { } } - fn create_tables(module: &Module) -> PrimaryMap { + fn create_tables( + module: &Module, + limiter: Option<&Rc>, + ) -> Result, InstantiationError> { let num_imports = module.num_imported_tables; let mut tables: PrimaryMap = PrimaryMap::with_capacity(module.table_plans.len() - num_imports); for table in &module.table_plans.values().as_slice()[num_imports..] { - tables.push(Table::new_dynamic(table)); + tables.push(Table::new_dynamic(table, limiter).map_err(InstantiationError::Resource)?); } - tables + Ok(tables) } fn create_memories( &self, module: &Module, + limiter: Option<&Rc>, ) -> Result, InstantiationError> { let creator = self .mem_creator @@ -612,8 +620,10 @@ impl OnDemandInstanceAllocator { let mut memories: PrimaryMap = PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); for plan in &module.memory_plans.values().as_slice()[num_imports..] { - memories - .push(Memory::new_dynamic(plan, creator).map_err(InstantiationError::Resource)?); + memories.push( + Memory::new_dynamic(plan, creator, limiter) + .map_err(InstantiationError::Resource)?, + ); } Ok(memories) } @@ -633,8 +643,8 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { &self, mut req: InstanceAllocationRequest, ) -> Result { - let memories = self.create_memories(&req.module)?; - let tables = Self::create_tables(&req.module); + let memories = self.create_memories(&req.module, req.limiter)?; + let tables = Self::create_tables(&req.module, req.limiter)?; let host_state = std::mem::replace(&mut req.host_state, Box::new(())); @@ -657,7 +667,9 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { alloc::handle_alloc_error(layout); } ptr::write(instance_ptr, instance); - InstanceHandle::new(instance_ptr) + InstanceHandle { + instance: instance_ptr, + } }; initialize_vmcontext(handle.instance(), req); diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 5bc342dafc..7a17b2b143 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -9,7 +9,7 @@ use super::{ initialize_instance, initialize_vmcontext, InstanceAllocationRequest, InstanceAllocator, - InstanceHandle, InstantiationError, + InstanceHandle, InstantiationError, ResourceLimiter, }; use crate::{instance::Instance, Memory, Mmap, Table, VMContext}; use anyhow::{anyhow, bail, Context, Result}; @@ -18,6 +18,7 @@ use std::cell::RefCell; use std::cmp::min; use std::convert::TryFrom; use std::mem; +use std::rc::Rc; use std::sync::{Arc, Mutex}; use wasmtime_environ::{ entity::{EntitySet, PrimaryMap}, @@ -376,10 +377,45 @@ impl InstancePool { } } + unsafe fn setup_instance( + &self, + index: usize, + mut req: InstanceAllocationRequest, + ) -> Result { + let instance = self.instance(index); + + instance.module = req.module.clone(); + instance.offsets = VMOffsets::new( + std::mem::size_of::<*const u8>() as u8, + instance.module.as_ref(), + ); + instance.host_state = std::mem::replace(&mut req.host_state, Box::new(())); + + Self::set_instance_memories( + instance, + self.memories.get(index), + self.memories.max_wasm_pages, + req.limiter, + )?; + + Self::set_instance_tables( + instance, + self.tables.get(index), + self.tables.max_elements, + req.limiter, + )?; + + initialize_vmcontext(instance, req); + + Ok(InstanceHandle { + instance: instance as _, + }) + } + fn allocate( &self, strategy: PoolingAllocationStrategy, - mut req: InstanceAllocationRequest, + req: InstanceAllocationRequest, ) -> Result { let index = { let mut free_list = self.free_list.lock().unwrap(); @@ -390,28 +426,15 @@ impl InstancePool { free_list.swap_remove(free_index) }; - let host_state = std::mem::replace(&mut req.host_state, Box::new(())); - unsafe { - let instance = self.instance(index); - - instance.module = req.module.clone(); - instance.offsets = VMOffsets::new( - std::mem::size_of::<*const u8>() as u8, - instance.module.as_ref(), - ); - instance.host_state = host_state; - - Self::set_instance_memories( - instance, - self.memories.get(index), - self.memories.max_wasm_pages, - )?; - Self::set_instance_tables(instance, self.tables.get(index), self.tables.max_elements)?; - - initialize_vmcontext(instance, req); - - Ok(InstanceHandle::new(instance as _)) + self.setup_instance(index, req).or_else(|e| { + // Deallocate the allocated instance on error + let instance = self.instance(index); + self.deallocate(&InstanceHandle { + instance: instance as _, + }); + Err(e) + }) } } @@ -473,6 +496,7 @@ impl InstancePool { instance: &mut Instance, mut memories: impl Iterator, max_pages: u32, + limiter: Option<&Rc>, ) -> Result<(), InstantiationError> { let module = instance.module.as_ref(); @@ -487,6 +511,7 @@ impl InstancePool { memories.next().unwrap(), max_pages, commit_memory_pages, + limiter, ) .map_err(InstantiationError::Resource)?, ); @@ -503,6 +528,7 @@ impl InstancePool { instance: &mut Instance, mut tables: impl Iterator, max_elements: u32, + limiter: Option<&Rc>, ) -> Result<(), InstantiationError> { let module = instance.module.as_ref(); @@ -514,9 +540,10 @@ impl InstancePool { commit_table_pages(base, max_elements as usize * mem::size_of::<*mut u8>()) .map_err(InstantiationError::Resource)?; - instance - .tables - .push(Table::new_static(plan, base as _, max_elements)); + instance.tables.push( + Table::new_static(plan, base as _, max_elements, limiter) + .map_err(InstantiationError::Resource)?, + ); } let mut dropped_elements = instance.dropped_elements.borrow_mut(); @@ -1371,6 +1398,7 @@ mod test { interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), module_info_lookup: None, + limiter: None, }, ) .expect("allocation should succeed"), @@ -1395,6 +1423,7 @@ mod test { interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), module_info_lookup: None, + limiter: None, }, ) { Err(InstantiationError::Limit(3)) => {} diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index 4c4b22caba..43ba9a654a 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -524,6 +524,7 @@ mod test { interrupts: ptr::null(), externref_activations_table: ptr::null_mut(), module_info_lookup: None, + limiter: None, }, ) .expect("instance should allocate"), diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 20f28ffd97..1c3ce53a3d 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -40,7 +40,7 @@ pub use crate::imports::Imports; pub use crate::instance::{ InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstanceLimits, InstantiationError, LinkError, ModuleLimits, OnDemandInstanceAllocator, - PoolingAllocationStrategy, PoolingInstanceAllocator, RuntimeInstance, + PoolingAllocationStrategy, PoolingInstanceAllocator, ResourceLimiter, RuntimeInstance, }; pub use crate::jit_int::GdbJitImageRegistration; pub use crate::memory::{Memory, RuntimeLinearMemory, RuntimeMemoryCreator}; diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index 024d901246..973016224d 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -4,12 +4,14 @@ use crate::mmap::Mmap; use crate::vmcontext::VMMemoryDefinition; -use anyhow::Result; +use crate::ResourceLimiter; +use anyhow::{bail, Result}; use more_asserts::{assert_ge, assert_le}; use std::cell::{Cell, RefCell}; use std::cmp::min; use std::convert::TryFrom; use std::ptr; +use std::rc::Rc; use wasmtime_environ::{MemoryPlan, MemoryStyle, WASM_MAX_PAGES, WASM_PAGE_SIZE}; /// A memory allocator @@ -33,6 +35,10 @@ pub trait RuntimeLinearMemory { /// Returns the number of allocated wasm pages. fn size(&self) -> u32; + /// Returns the maximum number of pages the memory can grow to. + /// Returns `None` if the memory is unbounded. + fn maximum(&self) -> Option; + /// Grow memory by the specified amount of wasm pages. /// /// Returns `None` if memory can't be grown by the specified amount @@ -105,6 +111,12 @@ impl RuntimeLinearMemory for MmapMemory { self.mmap.borrow().size } + /// Returns the maximum number of pages the memory can grow to. + /// Returns `None` if the memory is unbounded. + fn maximum(&self) -> Option { + self.maximum + } + /// Grow memory by the specified amount of wasm pages. /// /// Returns `None` if memory can't be grown by the specified amount @@ -189,12 +201,23 @@ enum MemoryStorage { } /// Represents an instantiation of a WebAssembly memory. -pub struct Memory(MemoryStorage); +pub struct Memory { + storage: MemoryStorage, + limiter: Option>, +} impl Memory { /// Create a new dynamic (movable) memory instance for the specified plan. - pub fn new_dynamic(plan: &MemoryPlan, creator: &dyn RuntimeMemoryCreator) -> Result { - Ok(Self(MemoryStorage::Dynamic(creator.new_memory(plan)?))) + pub fn new_dynamic( + plan: &MemoryPlan, + creator: &dyn RuntimeMemoryCreator, + limiter: Option<&Rc>, + ) -> Result { + Self::new( + plan, + MemoryStorage::Dynamic(creator.new_memory(plan)?), + limiter, + ) } /// Create a new static (immovable) memory instance for the specified plan. @@ -203,32 +226,78 @@ impl Memory { base: *mut u8, maximum: u32, make_accessible: fn(*mut u8, usize) -> Result<()>, + limiter: Option<&Rc>, ) -> Result { - if plan.memory.minimum > 0 { - make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize)?; - } - - Ok(Self(MemoryStorage::Static { + let storage = MemoryStorage::Static { base, size: Cell::new(plan.memory.minimum), maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum), make_accessible, #[cfg(all(feature = "uffd", target_os = "linux"))] guard_page_faults: RefCell::new(Vec::new()), - })) + }; + + Self::new(plan, storage, limiter) + } + + fn new( + plan: &MemoryPlan, + storage: MemoryStorage, + limiter: Option<&Rc>, + ) -> Result { + if let Some(limiter) = limiter { + if !limiter.memory_growing(0, plan.memory.minimum, plan.memory.maximum) { + bail!( + "memory minimum size of {} pages exceeds memory limits", + plan.memory.minimum + ); + } + } + + if let MemoryStorage::Static { + base, + make_accessible, + .. + } = &storage + { + if plan.memory.minimum > 0 { + make_accessible( + *base, + plan.memory.minimum as usize * WASM_PAGE_SIZE as usize, + )?; + } + } + + Ok(Self { + storage, + limiter: limiter.cloned(), + }) } /// Returns the number of allocated wasm pages. pub fn size(&self) -> u32 { - match &self.0 { + match &self.storage { MemoryStorage::Static { size, .. } => size.get(), MemoryStorage::Dynamic(mem) => mem.size(), } } + /// Returns the maximum number of pages the memory can grow to at runtime. + /// + /// Returns `None` if the memory is unbounded. + /// + /// The runtime maximum may not be equal to the maximum from the linear memory's + /// Wasm type when it is being constrained by an instance allocator. + pub fn maximum(&self) -> Option { + match &self.storage { + MemoryStorage::Static { maximum, .. } => Some(*maximum), + MemoryStorage::Dynamic(mem) => mem.maximum(), + } + } + /// Returns whether or not the underlying storage of the memory is "static". pub(crate) fn is_static(&self) -> bool { - if let MemoryStorage::Static { .. } = &self.0 { + if let MemoryStorage::Static { .. } = &self.storage { true } else { false @@ -239,8 +308,30 @@ impl Memory { /// /// Returns `None` if memory can't be grown by the specified amount /// of wasm pages. - pub fn grow(&self, delta: u32) -> Option { - match &self.0 { + /// + /// # Safety + /// + /// Resizing the memory can reallocate the memory buffer for dynamic memories. + /// An instance's `VMContext` may have pointers to the memory's base and will + /// need to be fixed up after growing the memory. + /// + /// Generally, prefer using `InstanceHandle::memory_grow`, which encapsulates + /// this unsafety. + pub unsafe fn grow(&self, delta: u32) -> Option { + let old_size = self.size(); + if delta == 0 { + return Some(old_size); + } + + let new_size = old_size.checked_add(delta)?; + + if let Some(limiter) = &self.limiter { + if !limiter.memory_growing(old_size, new_size, self.maximum()) { + return None; + } + } + + match &self.storage { MemoryStorage::Static { base, size, @@ -252,13 +343,6 @@ impl Memory { #[cfg(all(feature = "uffd", target_os = "linux"))] self.reset_guard_pages().ok()?; - let old_size = size.get(); - if delta == 0 { - return Some(old_size); - } - - let new_size = old_size.checked_add(delta)?; - if new_size > *maximum || new_size >= WASM_MAX_PAGES { return None; } @@ -266,7 +350,7 @@ impl Memory { let start = usize::try_from(old_size).unwrap() * WASM_PAGE_SIZE as usize; let len = usize::try_from(delta).unwrap() * WASM_PAGE_SIZE as usize; - make_accessible(unsafe { base.add(start) }, len).ok()?; + make_accessible(base.add(start), len).ok()?; size.set(new_size); @@ -278,7 +362,7 @@ impl Memory { /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. pub fn vmmemory(&self) -> VMMemoryDefinition { - match &self.0 { + match &self.storage { MemoryStorage::Static { base, size, .. } => VMMemoryDefinition { base: *base, current_length: size.get() as usize * WASM_PAGE_SIZE as usize, @@ -299,7 +383,7 @@ impl Memory { size: usize, reset: fn(*mut u8, usize) -> Result<()>, ) { - match &self.0 { + match &self.storage { MemoryStorage::Static { guard_page_faults, .. } => { @@ -320,7 +404,7 @@ impl Memory { /// This function will panic if called on a dynamic memory. #[cfg(all(feature = "uffd", target_os = "linux"))] pub(crate) fn reset_guard_pages(&self) -> Result<()> { - match &self.0 { + match &self.storage { MemoryStorage::Static { guard_page_faults, .. } => { @@ -345,13 +429,16 @@ impl Default for Memory { unreachable!() } - Self(MemoryStorage::Static { - base: ptr::null_mut(), - size: Cell::new(0), - maximum: 0, - make_accessible, - #[cfg(all(feature = "uffd", target_os = "linux"))] - guard_page_faults: RefCell::new(Vec::new()), - }) + Self { + storage: MemoryStorage::Static { + base: ptr::null_mut(), + size: Cell::new(0), + maximum: 0, + make_accessible, + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), + }, + limiter: None, + } } } diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index 8c857add45..34cd654da2 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -3,19 +3,21 @@ //! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories. use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition}; -use crate::{Trap, VMExternRef}; +use crate::{ResourceLimiter, Trap, VMExternRef}; +use anyhow::{bail, Result}; use std::cell::{Cell, RefCell}; use std::cmp::min; use std::convert::TryInto; use std::ops::Range; use std::ptr; +use std::rc::Rc; use wasmtime_environ::wasm::TableElementType; use wasmtime_environ::{ir, TablePlan}; /// An element going into or coming out of a table. /// /// Table elements are stored as pointers and are default-initialized with `ptr::null_mut`. -#[derive(Clone, Debug)] +#[derive(Clone)] pub enum TableElement { /// A `funcref`. FuncRef(*mut VMCallerCheckedAnyfunc), @@ -92,7 +94,6 @@ impl From for TableElement { } } -#[derive(Debug)] enum TableStorage { Static { data: *mut *mut u8, @@ -108,38 +109,74 @@ enum TableStorage { } /// Represents an instance's table. -#[derive(Debug)] -pub struct Table(TableStorage); +pub struct Table { + storage: TableStorage, + limiter: Option>, +} impl Table { /// Create a new dynamic (movable) table instance for the specified table plan. - pub fn new_dynamic(plan: &TablePlan) -> Self { + pub fn new_dynamic( + plan: &TablePlan, + limiter: Option<&Rc>, + ) -> Result { let elements = RefCell::new(vec![ptr::null_mut(); plan.table.minimum as usize]); let ty = plan.table.ty.clone(); let maximum = plan.table.maximum; - Self(TableStorage::Dynamic { + + let storage = TableStorage::Dynamic { elements, ty, maximum, - }) + }; + + Self::new(plan, storage, limiter) } /// Create a new static (immovable) table instance for the specified table plan. - pub fn new_static(plan: &TablePlan, data: *mut *mut u8, maximum: u32) -> Self { + pub fn new_static( + plan: &TablePlan, + data: *mut *mut u8, + maximum: u32, + limiter: Option<&Rc>, + ) -> Result { let size = Cell::new(plan.table.minimum); let ty = plan.table.ty.clone(); let maximum = min(plan.table.maximum.unwrap_or(maximum), maximum); - Self(TableStorage::Static { + + let storage = TableStorage::Static { data, size, ty, maximum, + }; + + Self::new(plan, storage, limiter) + } + + fn new( + plan: &TablePlan, + storage: TableStorage, + limiter: Option<&Rc>, + ) -> Result { + if let Some(limiter) = limiter { + if !limiter.table_growing(0, plan.table.minimum, plan.table.maximum) { + bail!( + "table minimum size of {} elements exceeds table limits", + plan.table.minimum + ); + } + } + + Ok(Self { + storage, + limiter: limiter.cloned(), }) } /// Returns the type of the elements in this table. pub fn element_type(&self) -> TableElementType { - match &self.0 { + match &self.storage { TableStorage::Static { ty, .. } => *ty, TableStorage::Dynamic { ty, .. } => *ty, } @@ -147,7 +184,7 @@ impl Table { /// Returns whether or not the underlying storage of the table is "static". pub(crate) fn is_static(&self) -> bool { - if let TableStorage::Static { .. } = &self.0 { + if let TableStorage::Static { .. } = &self.storage { true } else { false @@ -156,15 +193,20 @@ impl Table { /// Returns the number of allocated elements. pub fn size(&self) -> u32 { - match &self.0 { + match &self.storage { TableStorage::Static { size, .. } => size.get(), TableStorage::Dynamic { elements, .. } => elements.borrow().len().try_into().unwrap(), } } - /// Returns the maximum number of elements. + /// Returns the maximum number of elements at runtime. + /// + /// Returns `None` if the table is unbounded. + /// + /// The runtime maximum may not be equal to the maximum from the table's Wasm type + /// when it is being constrained by an instance allocator. pub fn maximum(&self) -> Option { - match &self.0 { + match &self.storage { TableStorage::Static { maximum, .. } => Some(*maximum), TableStorage::Dynamic { maximum, .. } => maximum.clone(), } @@ -218,8 +260,14 @@ impl Table { /// this unsafety. pub unsafe fn grow(&self, delta: u32, init_value: TableElement) -> Option { let old_size = self.size(); - let new_size = old_size.checked_add(delta)?; + + if let Some(limiter) = &self.limiter { + if !limiter.table_growing(old_size, new_size, self.maximum()) { + return None; + } + } + if let Some(max) = self.maximum() { if new_size > max { return None; @@ -229,7 +277,7 @@ impl Table { debug_assert!(self.type_matches(&init_value)); // First resize the storage and then fill with the init value - match &self.0 { + match &self.storage { TableStorage::Static { size, .. } => { size.set(new_size); } @@ -319,7 +367,7 @@ impl Table { /// Return a `VMTableDefinition` for exposing the table to compiled wasm code. pub fn vmtable(&self) -> VMTableDefinition { - match &self.0 { + match &self.storage { TableStorage::Static { data, size, .. } => VMTableDefinition { base: *data as _, current_elements: size.get(), @@ -346,7 +394,7 @@ impl Table { where F: FnOnce(&[*mut u8]) -> R, { - match &self.0 { + match &self.storage { TableStorage::Static { data, size, .. } => unsafe { f(std::slice::from_raw_parts(*data, size.get() as usize)) }, @@ -361,7 +409,7 @@ impl Table { where F: FnOnce(&mut [*mut u8]) -> R, { - match &self.0 { + match &self.storage { TableStorage::Static { data, size, .. } => unsafe { f(std::slice::from_raw_parts_mut(*data, size.get() as usize)) }, @@ -463,11 +511,14 @@ impl Drop for Table { // The default table representation is an empty funcref table that cannot grow. impl Default for Table { fn default() -> Self { - Self(TableStorage::Static { - data: std::ptr::null_mut(), - size: Cell::new(0), - ty: TableElementType::Func, - maximum: 0, - }) + Self { + storage: TableStorage::Static { + data: std::ptr::null_mut(), + size: Cell::new(0), + ty: TableElementType::Func, + maximum: 0, + }, + limiter: None, + } } } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 8351fea426..f133fa787b 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -383,9 +383,6 @@ pub struct Config { pub(crate) max_wasm_stack: usize, pub(crate) features: WasmFeatures, pub(crate) wasm_backtrace_details_env_used: bool, - pub(crate) max_instances: usize, - pub(crate) max_tables: usize, - pub(crate) max_memories: usize, #[cfg(feature = "async")] pub(crate) async_stack_size: usize, host_funcs: HostFuncMap, @@ -422,9 +419,6 @@ impl Config { max_wasm_stack: 1 << 20, wasm_backtrace_details_env_used: false, features: WasmFeatures::default(), - max_instances: 10_000, - max_tables: 10_000, - max_memories: 10_000, #[cfg(feature = "async")] async_stack_size: 2 << 20, host_funcs: HostFuncMap::new(), @@ -1196,39 +1190,6 @@ impl Config { self } - /// Configures the maximum number of instances which can be created within - /// this `Store`. - /// - /// Instantiation will fail with an error if this limit is exceeded. - /// - /// This value defaults to 10,000. - pub fn max_instances(&mut self, instances: usize) -> &mut Self { - self.max_instances = instances; - self - } - - /// Configures the maximum number of tables which can be created within - /// this `Store`. - /// - /// Instantiation will fail with an error if this limit is exceeded. - /// - /// This value defaults to 10,000. - pub fn max_tables(&mut self, tables: usize) -> &mut Self { - self.max_tables = tables; - self - } - - /// Configures the maximum number of memories which can be created within - /// this `Store`. - /// - /// Instantiation will fail with an error if this limit is exceeded. - /// - /// This value defaults to 10,000. - pub fn max_memories(&mut self, memories: usize) -> &mut Self { - self.max_memories = memories; - self - } - /// Defines a host function for the [`Config`] for the given callback. /// /// Use [`Store::get_host_func`](crate::Store::get_host_func) to get a [`Func`](crate::Func) representing the function. diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 04a47337ca..7a30f850aa 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -525,6 +525,7 @@ impl<'a> Instantiator<'a> { as *const VMExternRefActivationsTable as *mut _, module_info_lookup: Some(self.store.module_info_lookup()), + limiter: self.store.limiter().as_ref(), })?; // After we've created the `InstanceHandle` we still need to run diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index 98a86a279e..332f63b4d2 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -283,6 +283,7 @@ mod config; mod engine; mod externals; mod instance; +mod limits; mod linker; mod memory; mod module; @@ -299,6 +300,7 @@ pub use crate::engine::*; pub use crate::externals::*; pub use crate::func::*; pub use crate::instance::Instance; +pub use crate::limits::*; pub use crate::linker::*; pub use crate::memory::*; pub use crate::module::{FrameInfo, FrameSymbol, Module}; diff --git a/crates/wasmtime/src/limits.rs b/crates/wasmtime/src/limits.rs new file mode 100644 index 0000000000..fc65aa2e90 --- /dev/null +++ b/crates/wasmtime/src/limits.rs @@ -0,0 +1,208 @@ +pub(crate) const DEFAULT_INSTANCE_LIMIT: usize = 10000; +pub(crate) const DEFAULT_TABLE_LIMIT: usize = 10000; +pub(crate) const DEFAULT_MEMORY_LIMIT: usize = 10000; + +/// Used by hosts to limit resource consumption of instances at runtime. +/// +/// [`Store::new_with_limits`](crate::Store::new_with_limits) can be used +/// with a resource limiter to take into account non-WebAssembly resource +/// usage to determine if a linear memory or table should be grown. +pub trait ResourceLimiter { + /// Notifies the resource limiter that an instance's linear memory has been requested to grow. + /// + /// * `current` is the current size of the linear memory in WebAssembly page units. + /// * `desired` is the desired size of the linear memory in WebAssembly page units. + /// * `maximum` is either the linear memory's maximum or a maximum from an instance allocator, + /// also in WebAssembly page units. A value of `None` indicates that the linear memory is + /// unbounded. + /// + /// This function should return `true` to indicate that the growing operation is permitted or + /// `false` if not permitted. + /// + /// Note that this function will be called even when the desired count exceeds the given maximum. + /// + /// Returning `true` when a maximum has been exceeded will have no effect as the linear memory + /// will not be grown. + fn memory_growing(&self, current: u32, desired: u32, maximum: Option) -> bool; + + /// Notifies the resource limiter that an instance's table has been requested to grow. + /// + /// * `current` is the current number of elements in the table. + /// * `desired` is the desired number of elements in the table. + /// * `maximum` is either the table's maximum or a maximum from an instance allocator, + /// A value of `None` indicates that the table is unbounded. + /// + /// This function should return `true` to indicate that the growing operation is permitted or + /// `false` if not permitted. + /// + /// Note that this function will be called even when the desired count exceeds the given maximum. + /// + /// Returning `true` when a maximum has been exceeded will have no effect as the table will + /// not be grown. + fn table_growing(&self, current: u32, desired: u32, maximum: Option) -> bool; + + /// The maximum number of instances that can be created for a [`Store`](crate::Store). + /// + /// Module instantiation will fail if this limit is exceeded. + /// + /// This value defaults to 10,000. + fn instances(&self) -> usize { + DEFAULT_INSTANCE_LIMIT + } + + /// The maximum number of tables that can be created for a [`Store`](crate::Store). + /// + /// Module instantiation will fail if this limit is exceeded. + /// + /// This value defaults to 10,000. + fn tables(&self) -> usize { + DEFAULT_TABLE_LIMIT + } + + /// The maximum number of linear memories that can be created for a [`Store`](crate::Store). + /// + /// Instantiation will fail with an error if this limit is exceeded. + /// + /// This value defaults to 10,000. + fn memories(&self) -> usize { + DEFAULT_MEMORY_LIMIT + } +} + +pub(crate) struct ResourceLimiterProxy(pub T); + +impl wasmtime_runtime::ResourceLimiter for ResourceLimiterProxy { + fn memory_growing(&self, current: u32, desired: u32, maximum: Option) -> bool { + self.0.memory_growing(current, desired, maximum) + } + + fn table_growing(&self, current: u32, desired: u32, maximum: Option) -> bool { + self.0.table_growing(current, desired, maximum) + } + + fn instances(&self) -> usize { + self.0.instances() + } + + fn tables(&self) -> usize { + self.0.tables() + } + + fn memories(&self) -> usize { + self.0.memories() + } +} + +/// Used to build [`StoreLimits`]. +pub struct StoreLimitsBuilder(StoreLimits); + +impl StoreLimitsBuilder { + /// Creates a new [`StoreLimitsBuilder`]. + pub fn new() -> Self { + Self(StoreLimits::default()) + } + + /// The maximum number of WebAssembly pages a linear memory can grow to. + /// + /// Growing a linear memory beyond this limit will fail. + /// + /// By default, linear memory pages will not be limited. + pub fn memory_pages(mut self, limit: u32) -> Self { + self.0.memory_pages = Some(limit); + self + } + + /// The maximum number of elements in a table. + /// + /// Growing a table beyond this limit will fail. + /// + /// By default, table elements will not be limited. + pub fn table_elements(mut self, limit: u32) -> Self { + self.0.table_elements = Some(limit); + self + } + + /// The maximum number of instances that can be created for a [`Store`](crate::Store). + /// + /// Module instantiation will fail if this limit is exceeded. + /// + /// This value defaults to 10,000. + pub fn instances(mut self, limit: usize) -> Self { + self.0.instances = limit; + self + } + + /// The maximum number of tables that can be created for a [`Store`](crate::Store). + /// + /// Module instantiation will fail if this limit is exceeded. + /// + /// This value defaults to 10,000. + pub fn tables(mut self, tables: usize) -> Self { + self.0.tables = tables; + self + } + + /// The maximum number of linear memories that can be created for a [`Store`](crate::Store). + /// + /// Instantiation will fail with an error if this limit is exceeded. + /// + /// This value defaults to 10,000. + pub fn memories(mut self, memories: usize) -> Self { + self.0.memories = memories; + self + } + + /// Consumes this builder and returns the [`StoreLimits`]. + pub fn build(self) -> StoreLimits { + self.0 + } +} + +/// Provides limits for a [`Store`](crate::Store). +pub struct StoreLimits { + memory_pages: Option, + table_elements: Option, + instances: usize, + tables: usize, + memories: usize, +} + +impl Default for StoreLimits { + fn default() -> Self { + Self { + memory_pages: None, + table_elements: None, + instances: DEFAULT_INSTANCE_LIMIT, + tables: DEFAULT_TABLE_LIMIT, + memories: DEFAULT_MEMORY_LIMIT, + } + } +} + +impl ResourceLimiter for StoreLimits { + fn memory_growing(&self, _current: u32, desired: u32, _maximum: Option) -> bool { + match self.memory_pages { + Some(limit) if desired > limit => false, + _ => true, + } + } + + fn table_growing(&self, _current: u32, desired: u32, _maximum: Option) -> bool { + match self.table_elements { + Some(limit) if desired > limit => false, + _ => true, + } + } + + fn instances(&self) -> usize { + self.instances + } + + fn tables(&self) -> usize { + self.tables + } + + fn memories(&self) -> usize { + self.memories + } +} diff --git a/crates/wasmtime/src/memory.rs b/crates/wasmtime/src/memory.rs index 39b6349274..57aee031ff 100644 --- a/crates/wasmtime/src/memory.rs +++ b/crates/wasmtime/src/memory.rs @@ -262,7 +262,7 @@ impl Memory { /// let store = Store::new(&engine); /// /// let memory_ty = MemoryType::new(Limits::new(1, None)); - /// let memory = Memory::new(&store, memory_ty); + /// let memory = Memory::new(&store, memory_ty)?; /// /// let module = Module::new(&engine, "(module (memory (import \"\" \"\") 1))")?; /// let instance = Instance::new(&store, &module, &[memory.into()])?; @@ -270,13 +270,12 @@ impl Memory { /// # Ok(()) /// # } /// ``` - pub fn new(store: &Store, ty: MemoryType) -> Memory { - let (instance, wasmtime_export) = - generate_memory_export(store, &ty).expect("generated memory"); - Memory { + pub fn new(store: &Store, ty: MemoryType) -> Result { + let (instance, wasmtime_export) = generate_memory_export(store, &ty)?; + Ok(Memory { instance, wasmtime_export, - } + }) } /// Returns the underlying type of this memory. @@ -454,7 +453,7 @@ impl Memory { .memory_index(unsafe { &*self.wasmtime_export.definition }); self.instance .memory_grow(index, delta) - .ok_or_else(|| anyhow!("failed to grow memory")) + .ok_or_else(|| anyhow!("failed to grow memory by `{}`", delta)) } pub(crate) unsafe fn from_wasmtime_memory( @@ -500,6 +499,10 @@ pub unsafe trait LinearMemory { /// Returns the number of allocated wasm pages. fn size(&self) -> u32; + /// Returns the maximum number of pages the memory can grow to. + /// Returns `None` if the memory is unbounded. + fn maximum(&self) -> Option; + /// Grow memory by the specified amount of wasm pages. /// /// Returns `None` if memory can't be grown by the specified amount @@ -568,7 +571,7 @@ mod tests { .dynamic_memory_guard_size(0); let store = Store::new(&Engine::new(&cfg).unwrap()); let ty = MemoryType::new(Limits::new(1, None)); - let mem = Memory::new(&store, ty); + let mem = Memory::new(&store, ty).unwrap(); assert_eq!(mem.wasmtime_export.memory.offset_guard_size, 0); match mem.wasmtime_export.memory.style { wasmtime_environ::MemoryStyle::Dynamic => {} diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 301b1192c1..fb60ded7d0 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -1,6 +1,7 @@ use crate::{ module::ModuleRegistry, signatures::SignatureCollection, trampoline::StoreInstanceHandle, - Engine, Func, Module, Trap, + Engine, Func, Module, ResourceLimiter, ResourceLimiterProxy, Trap, DEFAULT_INSTANCE_LIMIT, + DEFAULT_MEMORY_LIMIT, DEFAULT_TABLE_LIMIT, }; use anyhow::{bail, Result}; use std::any::{Any, TypeId}; @@ -89,6 +90,7 @@ pub(crate) struct StoreInner { current_poll_cx: Cell<*mut Context<'static>>, out_of_gas_behavior: Cell, context_values: RefCell>>, + limiter: Option>, } #[derive(Copy, Clone)] @@ -120,8 +122,43 @@ impl Hash for HostInfoKey { } impl Store { - /// Creates a new store to be associated with the given [`Engine`]. - pub fn new(engine: &Engine) -> Store { + /// Creates a new [`Store`] to be associated with the given [`Engine`]. + /// + /// The created [`Store`] will place no additional limits on the size of linear + /// memories or tables at runtime. Linear memories and tables will be allowed to + /// grow to any upper limit specified in their definitions. + /// + /// The store will limit the number of instances, linear memories, and tables created to 10,000. + /// + /// Use [`Store::new_with_limits`] with a [`StoreLimitsBuilder`](crate::StoreLimitsBuilder) to + /// specify different limits for the store. + pub fn new(engine: &Engine) -> Self { + Self::new_(engine, None) + } + + /// Creates a new [`Store`] to be associated with the given [`Engine`] and using the supplied + /// resource limiter. + /// + /// A [`ResourceLimiter`] can be implemented by hosts to control the size of WebAssembly + /// linear memories and tables when a request is made to grow them. + /// + /// [`StoreLimitsBuilder`](crate::StoreLimitsBuilder) can be used to create a + /// [`StoreLimits`](crate::StoreLimits) that implements [`ResourceLimiter`] using + /// static limit values. + /// + /// # Example + /// + /// ```rust + /// # use wasmtime::{Engine, Store, StoreLimitsBuilder}; + /// // Place a limit on linear memories so they cannot grow beyond 1 MiB + /// let engine = Engine::default(); + /// let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memory_pages(16).build()); + /// ``` + pub fn new_with_limits(engine: &Engine, limiter: impl ResourceLimiter + 'static) -> Self { + Self::new_(engine, Some(Rc::new(ResourceLimiterProxy(limiter)))) + } + + fn new_(engine: &Engine, limiter: Option>) -> Self { // Ensure that wasmtime_runtime's signal handlers are configured. Note // that at the `Store` level it means we should perform this // once-per-thread. Platforms like Unix, however, only require this @@ -130,7 +167,7 @@ impl Store { wasmtime_runtime::init_traps(crate::module::GlobalModuleRegistry::is_wasm_pc) .expect("failed to initialize trap handling"); - Store { + Self { inner: Rc::new(StoreInner { engine: engine.clone(), interrupts: Arc::new(Default::default()), @@ -149,6 +186,7 @@ impl Store { current_poll_cx: Cell::new(ptr::null_mut()), out_of_gas_behavior: Cell::new(OutOfGas::Trap), context_values: RefCell::new(HashMap::new()), + limiter, }), } } @@ -202,6 +240,10 @@ impl Store { } } + pub(crate) fn limiter(&self) -> &Option> { + &self.inner.limiter + } + pub(crate) fn signatures(&self) -> &RefCell { &self.inner.signatures } @@ -231,8 +273,6 @@ impl Store { } pub(crate) fn bump_resource_counts(&self, module: &Module) -> Result<()> { - let config = self.engine().config(); - fn bump(slot: &Cell, max: usize, amt: usize, desc: &str) -> Result<()> { let new = slot.get().saturating_add(amt); if new > max { @@ -249,20 +289,11 @@ impl Store { let module = module.env_module(); let memories = module.memory_plans.len() - module.num_imported_memories; let tables = module.table_plans.len() - module.num_imported_tables; + let (max_instances, max_memories, max_tables) = self.limits(); - bump( - &self.inner.instance_count, - config.max_instances, - 1, - "instance", - )?; - bump( - &self.inner.memory_count, - config.max_memories, - memories, - "memory", - )?; - bump(&self.inner.table_count, config.max_tables, tables, "table")?; + bump(&self.inner.instance_count, max_instances, 1, "instance")?; + bump(&self.inner.memory_count, max_memories, memories, "memory")?; + bump(&self.inner.table_count, max_tables, tables, "table")?; Ok(()) } @@ -838,6 +869,18 @@ impl Store { Err(trap) => unsafe { wasmtime_runtime::raise_user_trap(trap.into()) }, } } + + fn limits(&self) -> (usize, usize, usize) { + self.inner + .limiter + .as_ref() + .map(|l| (l.instances(), l.memories(), l.tables())) + .unwrap_or(( + DEFAULT_INSTANCE_LIMIT, + DEFAULT_MEMORY_LIMIT, + DEFAULT_TABLE_LIMIT, + )) + } } unsafe impl TrapInfo for Store { diff --git a/crates/wasmtime/src/trampoline.rs b/crates/wasmtime/src/trampoline.rs index ee0bf25c9e..5ebc436eba 100644 --- a/crates/wasmtime/src/trampoline.rs +++ b/crates/wasmtime/src/trampoline.rs @@ -78,6 +78,7 @@ fn create_handle( as *const VMExternRefActivationsTable as *mut _, module_info_lookup: Some(store.module_info_lookup()), + limiter: store.limiter().as_ref(), }, )?; diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 0110e7b2a9..b381c561ea 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -288,6 +288,7 @@ pub fn create_function( interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), module_info_lookup: None, + limiter: None, })?, trampoline, )) @@ -320,6 +321,7 @@ pub unsafe fn create_raw_function( interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), module_info_lookup: None, + limiter: None, })?, ) } diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs index 0876ead225..9f3ea772b0 100644 --- a/crates/wasmtime/src/trampoline/memory.rs +++ b/crates/wasmtime/src/trampoline/memory.rs @@ -37,6 +37,10 @@ impl RuntimeLinearMemory for LinearMemoryProxy { self.mem.size() } + fn maximum(&self) -> Option { + self.mem.maximum() + } + fn grow(&self, delta: u32) -> Option { self.mem.grow(delta) } diff --git a/crates/wast/src/spectest.rs b/crates/wast/src/spectest.rs index ef744fa42d..cd8b7bc9d1 100644 --- a/crates/wast/src/spectest.rs +++ b/crates/wast/src/spectest.rs @@ -39,7 +39,7 @@ pub fn link_spectest(linker: &mut Linker) -> Result<()> { linker.define("spectest", "table", table)?; let ty = MemoryType::new(Limits::new(1, Some(2))); - let memory = Memory::new(linker.store(), ty); + let memory = Memory::new(linker.store(), ty)?; linker.define("spectest", "memory", memory)?; Ok(()) diff --git a/examples/memory.rs b/examples/memory.rs index 70e1b724a9..e47c249e5f 100644 --- a/examples/memory.rs +++ b/examples/memory.rs @@ -75,7 +75,7 @@ fn main() -> Result<()> { println!("Creating stand-alone memory..."); let memorytype = MemoryType::new(Limits::new(5, Some(5))); - let memory2 = Memory::new(&wasmtime_store, memorytype); + let memory2 = Memory::new(&wasmtime_store, memorytype)?; assert_eq!(memory2.size(), 5); assert!(memory2.grow(1).is_err()); assert!(memory2.grow(0).is_ok()); diff --git a/tests/all/externals.rs b/tests/all/externals.rs index 9bffa2c08d..0e26166f07 100644 --- a/tests/all/externals.rs +++ b/tests/all/externals.rs @@ -67,7 +67,7 @@ fn cross_store() -> anyhow::Result<()> { let ty = GlobalType::new(ValType::I32, Mutability::Const); let global = Global::new(&store2, ty, Val::I32(0))?; let ty = MemoryType::new(Limits::new(1, None)); - let memory = Memory::new(&store2, ty); + let memory = Memory::new(&store2, ty)?; let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); let table = Table::new(&store2, ty, Val::FuncRef(None))?; @@ -356,7 +356,7 @@ fn read_write_memory_via_api() { let cfg = Config::new(); let store = Store::new(&Engine::new(&cfg).unwrap()); let ty = MemoryType::new(Limits::new(1, None)); - let mem = Memory::new(&store, ty); + let mem = Memory::new(&store, ty).unwrap(); mem.grow(1).unwrap(); let value = b"hello wasm"; diff --git a/tests/all/limits.rs b/tests/all/limits.rs new file mode 100644 index 0000000000..f4c74be612 --- /dev/null +++ b/tests/all/limits.rs @@ -0,0 +1,381 @@ +use anyhow::Result; +use std::cell::RefCell; +use std::rc::Rc; +use wasmtime::*; + +#[test] +fn test_limits() -> Result<()> { + let engine = Engine::default(); + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (table (export "t") 0 anyfunc))"#, + )?; + + let store = Store::new_with_limits( + &engine, + StoreLimitsBuilder::new() + .memory_pages(10) + .table_elements(5) + .build(), + ); + + let instance = Instance::new(&store, &module, &[])?; + + // Test instance exports and host objects hitting the limit + for memory in std::array::IntoIter::new([ + instance.get_memory("m").unwrap(), + Memory::new(&store, MemoryType::new(Limits::new(0, None)))?, + ]) { + memory.grow(3)?; + memory.grow(5)?; + memory.grow(2)?; + + assert_eq!( + memory.grow(1).map_err(|e| e.to_string()).unwrap_err(), + "failed to grow memory by `1`" + ); + } + + // Test instance exports and host objects hitting the limit + for table in std::array::IntoIter::new([ + instance.get_table("t").unwrap(), + Table::new( + &store, + TableType::new(ValType::FuncRef, Limits::new(0, None)), + Val::FuncRef(None), + )?, + ]) { + table.grow(2, Val::FuncRef(None))?; + table.grow(1, Val::FuncRef(None))?; + table.grow(2, Val::FuncRef(None))?; + + assert_eq!( + table + .grow(1, Val::FuncRef(None)) + .map_err(|e| e.to_string()) + .unwrap_err(), + "failed to grow table by `1`" + ); + } + + Ok(()) +} + +#[test] +fn test_limits_memory_only() -> Result<()> { + let engine = Engine::default(); + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (table (export "t") 0 anyfunc))"#, + )?; + + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memory_pages(10).build()); + + let instance = Instance::new(&store, &module, &[])?; + + // Test instance exports and host objects hitting the limit + for memory in std::array::IntoIter::new([ + instance.get_memory("m").unwrap(), + Memory::new(&store, MemoryType::new(Limits::new(0, None)))?, + ]) { + memory.grow(3)?; + memory.grow(5)?; + memory.grow(2)?; + + assert_eq!( + memory.grow(1).map_err(|e| e.to_string()).unwrap_err(), + "failed to grow memory by `1`" + ); + } + + // Test instance exports and host objects *not* hitting the limit + for table in std::array::IntoIter::new([ + instance.get_table("t").unwrap(), + Table::new( + &store, + TableType::new(ValType::FuncRef, Limits::new(0, None)), + Val::FuncRef(None), + )?, + ]) { + table.grow(2, Val::FuncRef(None))?; + table.grow(1, Val::FuncRef(None))?; + table.grow(2, Val::FuncRef(None))?; + table.grow(1, Val::FuncRef(None))?; + } + + Ok(()) +} + +#[test] +fn test_initial_memory_limits_exceeded() -> Result<()> { + let engine = Engine::default(); + let module = Module::new(&engine, r#"(module (memory (export "m") 11))"#)?; + + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memory_pages(10).build()); + + match Instance::new(&store, &module, &[]) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: memory minimum size of 11 pages exceeds memory limits" + ), + } + + match Memory::new(&store, MemoryType::new(Limits::new(25, None))) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: memory minimum size of 25 pages exceeds memory limits" + ), + } + + Ok(()) +} + +#[test] +fn test_limits_table_only() -> Result<()> { + let engine = Engine::default(); + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (table (export "t") 0 anyfunc))"#, + )?; + + let store = + Store::new_with_limits(&engine, StoreLimitsBuilder::new().table_elements(5).build()); + + let instance = Instance::new(&store, &module, &[])?; + + // Test instance exports and host objects *not* hitting the limit + for memory in std::array::IntoIter::new([ + instance.get_memory("m").unwrap(), + Memory::new(&store, MemoryType::new(Limits::new(0, None)))?, + ]) { + memory.grow(3)?; + memory.grow(5)?; + memory.grow(2)?; + memory.grow(1)?; + } + + // Test instance exports and host objects hitting the limit + for table in std::array::IntoIter::new([ + instance.get_table("t").unwrap(), + Table::new( + &store, + TableType::new(ValType::FuncRef, Limits::new(0, None)), + Val::FuncRef(None), + )?, + ]) { + table.grow(2, Val::FuncRef(None))?; + table.grow(1, Val::FuncRef(None))?; + table.grow(2, Val::FuncRef(None))?; + + assert_eq!( + table + .grow(1, Val::FuncRef(None)) + .map_err(|e| e.to_string()) + .unwrap_err(), + "failed to grow table by `1`" + ); + } + + Ok(()) +} + +#[test] +fn test_initial_table_limits_exceeded() -> Result<()> { + let engine = Engine::default(); + let module = Module::new(&engine, r#"(module (table (export "t") 23 anyfunc))"#)?; + + let store = + Store::new_with_limits(&engine, StoreLimitsBuilder::new().table_elements(4).build()); + + match Instance::new(&store, &module, &[]) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: table minimum size of 23 elements exceeds table limits" + ), + } + + match Table::new( + &store, + TableType::new(ValType::FuncRef, Limits::new(99, None)), + Val::FuncRef(None), + ) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: table minimum size of 99 elements exceeds table limits" + ), + } + + Ok(()) +} + +#[test] +fn test_pooling_allocator_initial_limits_exceeded() -> Result<()> { + let mut config = Config::new(); + config.wasm_multi_memory(true); + config.allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memories: 2, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + ..Default::default() + }, + }); + + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#"(module (memory (export "m1") 2) (memory (export "m2") 5))"#, + )?; + + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memory_pages(3).build()); + + match Instance::new(&store, &module, &[]) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: memory minimum size of 5 pages exceeds memory limits" + ), + } + + // An instance should still be able to be created after the failure above + let module = Module::new(&engine, r#"(module (memory (export "m") 2))"#)?; + + Instance::new(&store, &module, &[])?; + + Ok(()) +} + +struct MemoryContext { + host_memory_used: usize, + wasm_memory_used: usize, + memory_limit: usize, + limit_exceeded: bool, + limiter_dropped: bool, +} + +struct HostMemoryLimiter(Rc>); + +impl ResourceLimiter for HostMemoryLimiter { + fn memory_growing(&self, current: u32, desired: u32, maximum: Option) -> bool { + let mut ctx = self.0.borrow_mut(); + + // Check if the desired exceeds a maximum (either from Wasm or from the host) + if desired > maximum.unwrap_or(u32::MAX) { + ctx.limit_exceeded = true; + return false; + } + + assert_eq!(current as usize * 0x10000, ctx.wasm_memory_used); + let desired = desired as usize * 0x10000; + + if desired + ctx.host_memory_used > ctx.memory_limit { + ctx.limit_exceeded = true; + return false; + } + + ctx.wasm_memory_used = desired; + true + } + + fn table_growing(&self, _current: u32, _desired: u32, _maximum: Option) -> bool { + true + } +} + +impl Drop for HostMemoryLimiter { + fn drop(&mut self) { + self.0.borrow_mut().limiter_dropped = true; + } +} + +#[test] +fn test_custom_limiter() -> Result<()> { + let mut config = Config::default(); + + // This approximates a function that would "allocate" resources that the host tracks. + // Here this is a simple function that increments the current host memory "used". + config.wrap_host_func("", "alloc", |caller: Caller, size: u32| -> u32 { + if let Some(ctx) = caller.store().get::>>() { + let mut ctx = ctx.borrow_mut(); + let size = size as usize; + + if size + ctx.host_memory_used + ctx.wasm_memory_used <= ctx.memory_limit { + ctx.host_memory_used += size; + return 1; + } + + ctx.limit_exceeded = true; + } + + 0 + }); + + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#"(module (import "" "alloc" (func $alloc (param i32) (result i32))) (memory (export "m") 0) (func (export "f") (param i32) (result i32) local.get 0 call $alloc))"#, + )?; + + let context = Rc::new(RefCell::new(MemoryContext { + host_memory_used: 0, + wasm_memory_used: 0, + memory_limit: 1 << 20, // 16 wasm pages is the limit for both wasm + host memory + limit_exceeded: false, + limiter_dropped: false, + })); + + let store = Store::new_with_limits(&engine, HostMemoryLimiter(context.clone())); + + assert!(store.set(context.clone()).is_ok()); + + let linker = Linker::new(&store); + let instance = linker.instantiate(&module)?; + let memory = instance.get_memory("m").unwrap(); + + // Grow the memory by 640 KiB + memory.grow(3)?; + memory.grow(5)?; + memory.grow(2)?; + + assert!(!context.borrow().limit_exceeded); + + // Grow the host "memory" by 384 KiB + let f = instance.get_typed_func::("f")?; + + assert_eq!(f.call(1 * 0x10000).unwrap(), 1); + assert_eq!(f.call(3 * 0x10000).unwrap(), 1); + assert_eq!(f.call(2 * 0x10000).unwrap(), 1); + + // Memory is at the maximum, but the limit hasn't been exceeded + assert!(!context.borrow().limit_exceeded); + + // Try to grow the memory again + assert_eq!( + memory.grow(1).map_err(|e| e.to_string()).unwrap_err(), + "failed to grow memory by `1`" + ); + + assert!(context.borrow().limit_exceeded); + + // Try to grow the host "memory" again + assert_eq!(f.call(1).unwrap(), 0); + + assert!(context.borrow().limit_exceeded); + + drop(f); + drop(memory); + drop(instance); + drop(linker); + drop(store); + + assert!(context.borrow().limiter_dropped); + + Ok(()) +} diff --git a/tests/all/linker.rs b/tests/all/linker.rs index e3d21e96bb..2f1707dd71 100644 --- a/tests/all/linker.rs +++ b/tests/all/linker.rs @@ -51,11 +51,11 @@ fn link_twice_bad() -> Result<()> { // memories let ty = MemoryType::new(Limits::new(1, None)); - let memory = Memory::new(&store, ty); + let memory = Memory::new(&store, ty)?; linker.define("m", "", memory.clone())?; assert!(linker.define("m", "", memory.clone()).is_err()); let ty = MemoryType::new(Limits::new(2, None)); - let memory = Memory::new(&store, ty); + let memory = Memory::new(&store, ty)?; assert!(linker.define("m", "", memory.clone()).is_err()); // tables diff --git a/tests/all/main.rs b/tests/all/main.rs index 4c921e60a3..89c686119e 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -13,6 +13,7 @@ mod import_calling_export; mod import_indexes; mod instance; mod invoke_func_via_table; +mod limits; mod linker; mod memory_creator; mod module; diff --git a/tests/all/memory_creator.rs b/tests/all/memory_creator.rs index e877b5e22d..5c49414706 100644 --- a/tests/all/memory_creator.rs +++ b/tests/all/memory_creator.rs @@ -17,6 +17,7 @@ mod not_for_windows { struct CustomMemory { mem: *mut c_void, size: usize, + guard_size: usize, used_wasm_pages: RefCell, glob_page_counter: Arc>, } @@ -43,6 +44,7 @@ mod not_for_windows { Self { mem, size, + guard_size, used_wasm_pages: RefCell::new(num_wasm_pages), glob_page_counter: glob_counter, } @@ -63,6 +65,10 @@ mod not_for_windows { *self.used_wasm_pages.borrow() } + fn maximum(&self) -> Option { + Some((self.size as u32 - self.guard_size as u32) / WASM_PAGE_SIZE) + } + fn grow(&self, delta: u32) -> Option { let delta_size = (delta as usize).checked_mul(WASM_PAGE_SIZE as usize)?; @@ -70,11 +76,8 @@ mod not_for_windows { let prev_size = (prev_pages as usize).checked_mul(WASM_PAGE_SIZE as usize)?; let new_pages = prev_pages.checked_add(delta)?; - let new_size = (new_pages as usize).checked_mul(WASM_PAGE_SIZE as usize)?; - let guard_size = unsafe { sysconf(_SC_PAGESIZE) as usize }; - - if new_size > self.size - guard_size { + if new_pages > self.maximum().unwrap() { return None; } unsafe { diff --git a/tests/all/module_linking.rs b/tests/all/module_linking.rs index 101655eff3..357caa982f 100644 --- a/tests/all/module_linking.rs +++ b/tests/all/module_linking.rs @@ -190,7 +190,6 @@ fn imports_exports() -> Result<()> { fn limit_instances() -> Result<()> { let mut config = Config::new(); config.wasm_module_linking(true); - config.max_instances(10); let engine = Engine::new(&config)?; let module = Module::new( &engine, @@ -216,7 +215,7 @@ fn limit_instances() -> Result<()> { ) "#, )?; - let store = Store::new(&engine); + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().instances(10).build()); let err = Instance::new(&store, &module, &[]).err().unwrap(); assert!( err.to_string().contains("resource limit exceeded"), @@ -231,7 +230,6 @@ fn limit_memories() -> Result<()> { let mut config = Config::new(); config.wasm_module_linking(true); config.wasm_multi_memory(true); - config.max_memories(10); let engine = Engine::new(&config)?; let module = Module::new( &engine, @@ -252,7 +250,7 @@ fn limit_memories() -> Result<()> { ) "#, )?; - let store = Store::new(&engine); + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memories(10).build()); let err = Instance::new(&store, &module, &[]).err().unwrap(); assert!( err.to_string().contains("resource limit exceeded"), @@ -266,7 +264,6 @@ fn limit_memories() -> Result<()> { fn limit_tables() -> Result<()> { let mut config = Config::new(); config.wasm_module_linking(true); - config.max_tables(10); let engine = Engine::new(&config)?; let module = Module::new( &engine, @@ -287,7 +284,7 @@ fn limit_tables() -> Result<()> { ) "#, )?; - let store = Store::new(&engine); + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().tables(10).build()); let err = Instance::new(&store, &module, &[]).err().unwrap(); assert!( err.to_string().contains("resource limit exceeded"), From 8507eb77087a0860d073784123432c55f906e082 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Mon, 19 Apr 2021 14:18:55 -0700 Subject: [PATCH 22/43] Use `map_or` instead of `map` and `unwrap_or` in `TableElement::into_raw` --- crates/runtime/src/table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index 8c857add45..368fdcd115 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -69,7 +69,7 @@ impl TableElement { unsafe fn into_raw(self) -> *mut u8 { match self { Self::FuncRef(e) => e as _, - Self::ExternRef(e) => e.map(|e| e.into_raw()).unwrap_or(ptr::null_mut()), + Self::ExternRef(e) => e.map_or(ptr::null_mut(), |e| e.into_raw()), } } } From 193551a8d6d15559de7f768b901d5452d20524da Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 19 Apr 2021 18:44:48 -0500 Subject: [PATCH 23/43] Optimize `table.init` instruction and instantiation (#2847) * Optimize `table.init` instruction and instantiation This commit optimizes table initialization as part of instance instantiation and also applies the same optimization to the `table.init` instruction. One part of this commit is to remove some preexisting duplication between instance instantiation and the `table.init` instruction itself, after this the actual implementation of `table.init` is optimized to effectively have fewer bounds checks in fewer places and have a much tighter loop for instantiation. A big fallout from this change is that memory/table initializer offsets are now stored as `u32` instead of `usize` to remove a few casts in a few places. This ended up requiring moving some overflow checks that happened in parsing to later in code itself because otherwise the wrong spec test errors are emitted during testing. I've tried to trace where these can possibly overflow but I think that I managed to get everything. In a local synthetic test where an empty module with a single 80,000 element initializer this improves total instantiation time by 4x (562us => 141us) * Review comments --- cranelift/wasm/src/environ/dummy.rs | 4 +- cranelift/wasm/src/environ/spec.rs | 4 +- cranelift/wasm/src/sections_translator.rs | 16 +--- crates/environ/src/module.rs | 19 ++-- crates/environ/src/module_environ.rs | 4 +- crates/runtime/src/instance.rs | 88 ++++++++++++------- crates/runtime/src/instance/allocator.rs | 100 +++++++--------------- crates/runtime/src/table.rs | 28 +++++- 8 files changed, 137 insertions(+), 126 deletions(-) diff --git a/cranelift/wasm/src/environ/dummy.rs b/cranelift/wasm/src/environ/dummy.rs index ea9157e552..6ac82b73f0 100644 --- a/cranelift/wasm/src/environ/dummy.rs +++ b/cranelift/wasm/src/environ/dummy.rs @@ -747,7 +747,7 @@ impl<'data> ModuleEnvironment<'data> for DummyEnvironment { &mut self, _table_index: TableIndex, _base: Option, - _offset: usize, + _offset: u32, _elements: Box<[FuncIndex]>, ) -> WasmResult<()> { // We do nothing @@ -792,7 +792,7 @@ impl<'data> ModuleEnvironment<'data> for DummyEnvironment { &mut self, _memory_index: MemoryIndex, _base: Option, - _offset: usize, + _offset: u32, _data: &'data [u8], ) -> WasmResult<()> { // We do nothing diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index 3a2389c44d..ab7974f576 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -937,7 +937,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { &mut self, table_index: TableIndex, base: Option, - offset: usize, + offset: u32, elements: Box<[FuncIndex]>, ) -> WasmResult<()>; @@ -984,7 +984,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { &mut self, memory_index: MemoryIndex, base: Option, - offset: usize, + offset: u32, data: &'data [u8], ) -> WasmResult<()>; diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index c617c9943c..3ce00e61de 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -377,7 +377,7 @@ pub fn parse_element_section<'data>( } => { let mut init_expr_reader = init_expr.get_binary_reader(); let (base, offset) = match init_expr_reader.read_operator()? { - Operator::I32Const { value } => (None, value as u32 as usize), + Operator::I32Const { value } => (None, value as u32), Operator::GlobalGet { global_index } => { (Some(GlobalIndex::from_u32(global_index)), 0) } @@ -388,12 +388,6 @@ pub fn parse_element_section<'data>( )); } }; - // Check for offset + len overflow - if offset.checked_add(segments.len()).is_none() { - return Err(wasm_unsupported!( - "element segment offset and length overflows" - )); - } environ.declare_table_elements( TableIndex::from_u32(table_index), base, @@ -429,7 +423,7 @@ pub fn parse_data_section<'data>( } => { let mut init_expr_reader = init_expr.get_binary_reader(); let (base, offset) = match init_expr_reader.read_operator()? { - Operator::I32Const { value } => (None, value as u32 as usize), + Operator::I32Const { value } => (None, value as u32), Operator::GlobalGet { global_index } => { (Some(GlobalIndex::from_u32(global_index)), 0) } @@ -440,12 +434,6 @@ pub fn parse_data_section<'data>( )) } }; - // Check for offset + len overflow - if offset.checked_add(data.len()).is_none() { - return Err(wasm_unsupported!( - "data segment offset and length overflows" - )); - } environ.declare_data_initialization( MemoryIndex::from_u32(memory_index), base, diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index dd98484118..38a4293825 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -7,6 +7,7 @@ use cranelift_wasm::*; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; +use std::convert::TryFrom; use std::sync::Arc; /// Implemenation styles for WebAssembly linear memory. @@ -86,7 +87,7 @@ pub struct MemoryInitializer { /// Optionally, a global variable giving a base index. pub base: Option, /// The offset to add to the base. - pub offset: usize, + pub offset: u32, /// The data to write into the linear memory. pub data: Box<[u8]>, } @@ -168,7 +169,15 @@ impl MemoryInitialization { // Perform a bounds check on the segment // As this segment is referencing a defined memory without a global base, the last byte // written to by the segment cannot exceed the memory's initial minimum size - if (initializer.offset + initializer.data.len()) + let offset = usize::try_from(initializer.offset).unwrap(); + let end = match offset.checked_add(initializer.data.len()) { + Some(end) => end, + None => { + out_of_bounds = true; + continue; + } + }; + if end > ((module.memory_plans[initializer.memory_index].memory.minimum as usize) * WASM_PAGE_SIZE) @@ -178,8 +187,8 @@ impl MemoryInitialization { } let pages = &mut map[index]; - let mut page_index = initializer.offset / WASM_PAGE_SIZE; - let mut page_offset = initializer.offset % WASM_PAGE_SIZE; + let mut page_index = offset / WASM_PAGE_SIZE; + let mut page_offset = offset % WASM_PAGE_SIZE; let mut data_offset = 0; let mut data_remaining = initializer.data.len(); @@ -268,7 +277,7 @@ pub struct TableInitializer { /// Optionally, a global variable giving a base index. pub base: Option, /// The offset to add to the base. - pub offset: usize, + pub offset: u32, /// The values to write into the table elements. pub elements: Box<[FuncIndex]>, } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 8106c1a3fb..393a83975e 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -705,7 +705,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data &mut self, table_index: TableIndex, base: Option, - offset: usize, + offset: u32, elements: Box<[FuncIndex]>, ) -> WasmResult<()> { for element in elements.iter() { @@ -794,7 +794,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data &mut self, memory_index: MemoryIndex, base: Option, - offset: usize, + offset: u32, data: &'data [u8], ) -> WasmResult<()> { match &mut self.result.module.memory_initialization { diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 916aa90f26..f2143681fa 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -573,11 +573,11 @@ impl Instance { return None; } - Some(unsafe { &*self.anyfunc_ptr(index) }) + unsafe { Some(&*self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index))) } } - unsafe fn anyfunc_ptr(&self, index: FuncIndex) -> *mut VMCallerCheckedAnyfunc { - self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index)) + unsafe fn anyfunc_base(&self) -> *mut VMCallerCheckedAnyfunc { + self.vmctx_plus_offset(self.offsets.vmctx_anyfuncs_begin()) } fn find_passive_segment<'a, I, D, T>( @@ -611,38 +611,56 @@ impl Instance { src: u32, len: u32, ) -> Result<(), Trap> { - // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init - - let table = self.get_table(table_index); - let elements = Self::find_passive_segment( elem_index, &self.module.passive_elements_map, &self.module.passive_elements, &self.dropped_elements, ); + self.table_init_segment(table_index, elements, dst, src, len) + } - if src - .checked_add(len) - .map_or(true, |n| n as usize > elements.len()) - || dst.checked_add(len).map_or(true, |m| m > table.size()) + pub(crate) fn table_init_segment( + &self, + table_index: TableIndex, + elements: &[FuncIndex], + dst: u32, + src: u32, + len: u32, + ) -> Result<(), Trap> { + // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init + + let table = self.get_table(table_index); + + let elements = match elements + .get(usize::try_from(src).unwrap()..) + .and_then(|s| s.get(..usize::try_from(len).unwrap())) { - return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); + Some(elements) => elements, + None => return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)), + }; + + match table.element_type() { + TableElementType::Func => unsafe { + let base = self.anyfunc_base(); + table.init_funcs( + dst, + elements.iter().map(|idx| { + if *idx == FuncIndex::reserved_value() { + ptr::null_mut() + } else { + debug_assert!(idx.as_u32() < self.offsets.num_defined_functions); + base.add(usize::try_from(idx.as_u32()).unwrap()) + } + }), + )?; + }, + + TableElementType::Val(_) => { + debug_assert!(elements.iter().all(|e| *e == FuncIndex::reserved_value())); + table.fill(dst, TableElement::ExternRef(None), len)?; + } } - - // TODO(#983): investigate replacing this get/set loop with a `memcpy`. - for (dst, src) in (dst..dst + len).zip(src..src + len) { - let elem = self - .get_caller_checked_anyfunc(elements[src as usize]) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut _ - }); - - table - .set(dst, TableElement::FuncRef(elem)) - .expect("should never panic because we already did the bounds check above"); - } - Ok(()) } @@ -773,16 +791,26 @@ impl Instance { src: u32, len: u32, ) -> Result<(), Trap> { - // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init - - let memory = self.get_memory(memory_index); - let data = Self::find_passive_segment( data_index, &self.module.passive_data_map, &self.module.passive_data, &self.dropped_data, ); + self.memory_init_segment(memory_index, &data, dst, src, len) + } + + pub(crate) fn memory_init_segment( + &self, + memory_index: MemoryIndex, + data: &[u8], + dst: u32, + src: u32, + len: u32, + ) -> Result<(), Trap> { + // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init + + let memory = self.get_memory(memory_index); if src .checked_add(len) diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 1e4d800877..c91f89a74f 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -2,7 +2,7 @@ use crate::externref::{ModuleInfoLookup, VMExternRefActivationsTable, EMPTY_MODU use crate::imports::Imports; use crate::instance::{Instance, InstanceHandle, ResourceLimiter, RuntimeMemoryCreator}; use crate::memory::{DefaultMemoryCreator, Memory}; -use crate::table::{Table, TableElement}; +use crate::table::Table; use crate::traphandlers::Trap; use crate::vmcontext::{ VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, @@ -19,10 +19,9 @@ use std::rc::Rc; use std::slice; use std::sync::Arc; use thiserror::Error; -use wasmtime_environ::entity::{packed_option::ReservedValue, EntityRef, EntitySet, PrimaryMap}; +use wasmtime_environ::entity::{EntityRef, EntitySet, PrimaryMap}; use wasmtime_environ::wasm::{ - DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalInit, SignatureIndex, - TableElementType, WasmType, + DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, GlobalInit, SignatureIndex, WasmType, }; use wasmtime_environ::{ ir, MemoryInitialization, MemoryInitializer, Module, ModuleType, TableInitializer, VMOffsets, @@ -212,7 +211,7 @@ impl<'a> From<&'a PrimaryMap> for Shared fn get_table_init_start( init: &TableInitializer, instance: &Instance, -) -> Result { +) -> Result { match init.base { Some(base) => { let val = unsafe { @@ -223,7 +222,7 @@ fn get_table_init_start( } }; - init.offset.checked_add(val as usize).ok_or_else(|| { + init.offset.checked_add(val).ok_or_else(|| { InstantiationError::Link(LinkError( "element segment global base overflows".to_owned(), )) @@ -237,6 +236,7 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError for init in &instance.module.table_initializers { let table = instance.get_table(init.table_index); let start = get_table_init_start(init, instance)?; + let start = usize::try_from(start).unwrap(); let end = start.checked_add(init.elements.len()); match end { @@ -256,34 +256,15 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { for init in &instance.module.table_initializers { - let table = instance.get_table(init.table_index); - let start = get_table_init_start(init, instance)?; - let end = start.checked_add(init.elements.len()); - - match end { - Some(end) if end <= table.size() as usize => { - for (i, func_idx) in init.elements.iter().enumerate() { - let item = match table.element_type() { - TableElementType::Func => instance - .get_caller_checked_anyfunc(*func_idx) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc - }) - .into(), - TableElementType::Val(_) => { - assert!(*func_idx == FuncIndex::reserved_value()); - TableElement::ExternRef(None) - } - }; - table.set(u32::try_from(start + i).unwrap(), item).unwrap(); - } - } - _ => { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::TableOutOfBounds, - ))) - } - } + instance + .table_init_segment( + init.table_index, + &init.elements, + get_table_init_start(init, instance)?, + 0, + init.elements.len() as u32, + ) + .map_err(InstantiationError::Trap)?; } Ok(()) @@ -292,7 +273,7 @@ fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { fn get_memory_init_start( init: &MemoryInitializer, instance: &Instance, -) -> Result { +) -> Result { match init.base { Some(base) => { let val = unsafe { @@ -303,7 +284,7 @@ fn get_memory_init_start( } }; - init.offset.checked_add(val as usize).ok_or_else(|| { + init.offset.checked_add(val).ok_or_else(|| { InstantiationError::Link(LinkError("data segment global base overflows".to_owned())) }) } @@ -311,24 +292,6 @@ fn get_memory_init_start( } } -unsafe fn get_memory_slice<'instance>( - init: &MemoryInitializer, - instance: &'instance Instance, -) -> &'instance mut [u8] { - let memory = if let Some(defined_memory_index) = - instance.module.defined_memory_index(init.memory_index) - { - instance.memory(defined_memory_index) - } else { - let import = instance.imported_memory(init.memory_index); - let foreign_instance = (&mut *(import).vmctx).instance(); - let foreign_memory = &mut *(import).from; - let foreign_index = foreign_instance.memory_index(foreign_memory); - foreign_instance.memory(foreign_index) - }; - &mut *ptr::slice_from_raw_parts_mut(memory.base, memory.current_length) -} - fn check_memory_init_bounds( instance: &Instance, initializers: &[MemoryInitializer], @@ -336,6 +299,7 @@ fn check_memory_init_bounds( for init in initializers { let memory = instance.get_memory(init.memory_index); let start = get_memory_init_start(init, instance)?; + let start = usize::try_from(start).unwrap(); let end = start.checked_add(init.data.len()); match end { @@ -358,21 +322,15 @@ fn initialize_memories( initializers: &[MemoryInitializer], ) -> Result<(), InstantiationError> { for init in initializers { - let memory = instance.get_memory(init.memory_index); - let start = get_memory_init_start(init, instance)?; - let end = start.checked_add(init.data.len()); - - match end { - Some(end) if end <= memory.current_length => { - let mem_slice = unsafe { get_memory_slice(init, instance) }; - mem_slice[start..end].copy_from_slice(&init.data); - } - _ => { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::HeapOutOfBounds, - ))) - } - } + instance + .memory_init_segment( + init.memory_index, + &init.data, + get_memory_init_start(init, instance)?, + 0, + init.data.len() as u32, + ) + .map_err(InstantiationError::Trap)?; } Ok(()) @@ -496,6 +454,7 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque ); // Initialize the functions + let mut base = instance.anyfunc_base(); for (index, sig) in instance.module.functions.iter() { let type_index = req.shared_signatures.lookup(*sig); @@ -510,13 +469,14 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque }; ptr::write( - instance.anyfunc_ptr(index), + base, VMCallerCheckedAnyfunc { func_ptr, type_index, vmctx, }, ); + base = base.add(1); } // Initialize the defined tables diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index a06bdfc5ee..9e26721fb4 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -7,7 +7,7 @@ use crate::{ResourceLimiter, Trap, VMExternRef}; use anyhow::{bail, Result}; use std::cell::{Cell, RefCell}; use std::cmp::min; -use std::convert::TryInto; +use std::convert::{TryFrom, TryInto}; use std::ops::Range; use std::ptr; use std::rc::Rc; @@ -212,6 +212,32 @@ impl Table { } } + /// Fill `table[dst..]` with values from `items` + /// + /// Returns a trap error on out-of-bounds accesses. + pub fn init_funcs( + &self, + dst: u32, + items: impl ExactSizeIterator, + ) -> Result<(), Trap> { + assert!(self.element_type() == TableElementType::Func); + + self.with_elements_mut(|elements| { + let elements = match elements + .get_mut(usize::try_from(dst).unwrap()..) + .and_then(|s| s.get_mut(..items.len())) + { + Some(elements) => elements, + None => return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)), + }; + + for (item, slot) in items.zip(elements) { + *slot = item as *mut u8; + } + Ok(()) + }) + } + /// Fill `table[dst..dst + len]` with `val`. /// /// Returns a trap error on out-of-bounds accesses. From 200d7f1df668db31baf14f1453ab8cee54724292 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 19 Apr 2021 20:49:33 -0500 Subject: [PATCH 24/43] Delete signature for no-longer-present function (#2849) Accidental omission from #2736 --- crates/c-api/include/wasmtime.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h index d6b0349e64..8babeabdff 100644 --- a/crates/c-api/include/wasmtime.h +++ b/crates/c-api/include/wasmtime.h @@ -283,14 +283,6 @@ WASMTIME_CONFIG_PROP(void, static_memory_guard_size, uint64_t) */ WASMTIME_CONFIG_PROP(void, dynamic_memory_guard_size, uint64_t) -/** - * \brief Configures the maximum number of instances that can be created. - * - * For more information see the Rust documentation at - * https://bytecodealliance.github.io/wasmtime/api/wasmtime/struct.Config.html#method.max_instances. - */ -WASMTIME_CONFIG_PROP(void, max_instances, size_t) - /** * \brief Enables Wasmtime's cache and loads configuration from the specified * path. From 196bcec6cfd495a5c75ea169f75a878b5d78ffaf Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 20 Apr 2021 16:52:51 -0500 Subject: [PATCH 25/43] Process declared element segments for "possibly exported funcs" (#2851) Now that we're using "possibly exported" as an impactful decision for codegen (which trampolines to generate and which ABI a function has) it's important that we calculate this property of a wasm function correctly! Previously Wasmtime forgot to processed "declared" elements in apart from active/passive element segments, but this updates Wasmtime to ensure that these entries are processed and all the functions contained within are flagged as "possibly exported". Closes #2850 --- cranelift/wasm/src/environ/spec.rs | 7 ++++++ cranelift/wasm/src/sections_translator.rs | 2 +- crates/environ/src/module_environ.rs | 7 ++++++ tests/all/func.rs | 27 +++++++++++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index ab7974f576..31c8d86a4e 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -948,6 +948,13 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { elements: Box<[FuncIndex]>, ) -> WasmResult<()>; + /// Indicates that a declarative element segment was seen in the wasm + /// module. + fn declare_elements(&mut self, elements: Box<[FuncIndex]>) -> WasmResult<()> { + drop(elements); + Ok(()) + } + /// Provides the number of passive data segments up front. /// /// By default this does nothing, but implementations may use this to diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index 3ce00e61de..bd4dcd1136 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -400,7 +400,7 @@ pub fn parse_element_section<'data>( environ.declare_passive_element(index, segments)?; } ElementKind::Declared => { - // Nothing to do here. + environ.declare_elements(segments)?; } } } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 393a83975e..324c4b9296 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -746,6 +746,13 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data Ok(()) } + fn declare_elements(&mut self, segments: Box<[FuncIndex]>) -> WasmResult<()> { + for element in segments.iter() { + self.flag_func_possibly_exported(*element); + } + Ok(()) + } + fn reserve_function_bodies(&mut self, _count: u32, offset: u64) { self.result.debuginfo.wasm_file.code_section_offset = offset; } diff --git a/tests/all/func.rs b/tests/all/func.rs index e3bee038c2..67e4f9ddc2 100644 --- a/tests/all/func.rs +++ b/tests/all/func.rs @@ -774,3 +774,30 @@ fn wrap_multiple_results() -> anyhow::Result<()> { f64 "f64" F64 f64::from_bits(a), } } + +#[test] +fn trampoline_for_declared_elem() -> anyhow::Result<()> { + let engine = Engine::default(); + + let module = Module::new( + &engine, + r#" + (module + (elem declare func $f) + (func $f) + (func (export "g") (result funcref) + (ref.func $f) + ) + ) + "#, + )?; + + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + + let g = instance.get_typed_func::<(), Option>("g")?; + + let func = g.call(())?; + func.unwrap().call(&[])?; + Ok(()) +} From 801358333dabfb9348aafede7efb4e6dd56eba2f Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 21 Apr 2021 14:14:59 +0200 Subject: [PATCH 26/43] debug: Support big-endian architectures This fixes some hard-coded assumptions in the debug crate that the native ELF files being accessed are little-endian; specifically in create_gdbjit_image as well as in emit_dwarf. In addition, data in WebAssembly memory always uses little-endian byte order. Therefore, if the native architecture is big-endian, all references to base types need to be marked as little-endian using the DW_AT_endianity attribute, so that the debugger will be able to correctly access them. --- crates/debug/src/lib.rs | 77 ++++++++++++++++++--------- crates/debug/src/transform/unit.rs | 14 +++++ crates/debug/src/write_debuginfo.rs | 23 ++++---- crates/environ/src/data_structures.rs | 4 +- 4 files changed, 79 insertions(+), 39 deletions(-) diff --git a/crates/debug/src/lib.rs b/crates/debug/src/lib.rs index 597ab2e25c..b438051b45 100644 --- a/crates/debug/src/lib.rs +++ b/crates/debug/src/lib.rs @@ -3,6 +3,7 @@ #![allow(clippy::cast_ptr_alignment)] use anyhow::{bail, ensure, Error}; +use object::endian::{BigEndian, Endian, Endianness, LittleEndian}; use object::{RelocationEncoding, RelocationKind}; use std::collections::HashMap; @@ -18,13 +19,20 @@ pub fn create_gdbjit_image( defined_funcs_offset: usize, funcs: &[*const u8], ) -> Result, Error> { - ensure_supported_elf_format(&mut bytes)?; + let e = ensure_supported_elf_format(&mut bytes)?; // patch relocs relocate_dwarf_sections(&mut bytes, defined_funcs_offset, funcs)?; // elf is still missing details... - convert_object_elf_to_loadable_file(&mut bytes, code_region); + match e { + Endianness::Little => { + convert_object_elf_to_loadable_file::(&mut bytes, code_region) + } + Endianness::Big => { + convert_object_elf_to_loadable_file::(&mut bytes, code_region) + } + } // let mut file = ::std::fs::File::create(::std::path::Path::new("test.o")).expect("file"); // ::std::io::Write::write_all(&mut file, &bytes).expect("write"); @@ -83,18 +91,33 @@ fn relocate_dwarf_sections( Ok(()) } -fn ensure_supported_elf_format(bytes: &mut Vec) -> Result<(), Error> { +fn ensure_supported_elf_format(bytes: &mut Vec) -> Result { use object::elf::*; - use object::endian::LittleEndian; + use object::read::elf::*; + use object::Bytes; use std::mem::size_of; - let e = LittleEndian; - let header: &FileHeader64 = - unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) }; - ensure!( - header.e_ident.class == ELFCLASS64 && header.e_ident.data == ELFDATA2LSB, - "bits and endianess in .ELF", - ); + let kind = match object::FileKind::parse(bytes) { + Ok(file) => file, + Err(err) => { + bail!("Failed to parse file: {}", err); + } + }; + let header = match kind { + object::FileKind::Elf64 => { + match object::elf::FileHeader64::::parse(Bytes(bytes)) { + Ok(header) => header, + Err(err) => { + bail!("Unsupported ELF file: {}", err); + } + } + } + _ => { + bail!("only 64-bit ELF files currently supported") + } + }; + let e = header.endian().unwrap(); + match header.e_machine.get(e) { EM_X86_64 => (), machine => { @@ -106,23 +129,25 @@ fn ensure_supported_elf_format(bytes: &mut Vec) -> Result<(), Error> { "program header table is empty" ); let e_shentsize = header.e_shentsize.get(e); - ensure!( - e_shentsize as usize == size_of::>(), - "size of sh" - ); - Ok(()) + let req_shentsize = match e { + Endianness::Little => size_of::>(), + Endianness::Big => size_of::>(), + }; + ensure!(e_shentsize as usize == req_shentsize, "size of sh"); + Ok(e) } -fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const u8, usize)) { +fn convert_object_elf_to_loadable_file( + bytes: &mut Vec, + code_region: (*const u8, usize), +) { use object::elf::*; - use object::endian::LittleEndian; use std::ffi::CStr; use std::mem::size_of; use std::os::raw::c_char; - let e = LittleEndian; - let header: &FileHeader64 = - unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) }; + let e = E::default(); + let header: &FileHeader64 = unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) }; let e_shentsize = header.e_shentsize.get(e); let e_shoff = header.e_shoff.get(e); @@ -130,7 +155,7 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const let mut shstrtab_off = 0; for i in 0..e_shnum { let off = e_shoff as isize + i as isize * e_shentsize as isize; - let section: &SectionHeader64 = + let section: &SectionHeader64 = unsafe { &*(bytes.as_ptr().offset(off) as *const SectionHeader64<_>) }; if section.sh_type.get(e) != SHT_STRTAB { continue; @@ -140,7 +165,7 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const let mut segment: Option<_> = None; for i in 0..e_shnum { let off = e_shoff as isize + i as isize * e_shentsize as isize; - let section: &mut SectionHeader64 = + let section: &mut SectionHeader64 = unsafe { &mut *(bytes.as_mut_ptr().offset(off) as *mut SectionHeader64<_>) }; if section.sh_type.get(e) != SHT_PROGBITS { continue; @@ -171,12 +196,12 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const // LLDB wants segment with virtual address set, placing them at the end of ELF. let ph_off = bytes.len(); - let e_phentsize = size_of::>(); + let e_phentsize = size_of::>(); let e_phnum = 1; bytes.resize(ph_off + e_phentsize * e_phnum, 0); if let Some((sh_offset, sh_size)) = segment { let (v_offset, size) = code_region; - let program: &mut ProgramHeader64 = + let program: &mut ProgramHeader64 = unsafe { &mut *(bytes.as_ptr().add(ph_off) as *mut ProgramHeader64<_>) }; program.p_type.set(e, PT_LOAD); program.p_offset.set(e, sh_offset); @@ -189,7 +214,7 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const } // It is somewhat loadable ELF file at this moment. - let header: &mut FileHeader64 = + let header: &mut FileHeader64 = unsafe { &mut *(bytes.as_mut_ptr() as *mut FileHeader64<_>) }; header.e_type.set(e, ET_DYN); header.e_phoff.set(e, ph_off as u64); diff --git a/crates/debug/src/transform/unit.rs b/crates/debug/src/transform/unit.rs index 15f165418a..655d7738ee 100644 --- a/crates/debug/src/transform/unit.rs +++ b/crates/debug/src/transform/unit.rs @@ -10,6 +10,7 @@ use anyhow::{Context, Error}; use gimli::write; use gimli::{AttributeValue, DebuggingInformationEntry, Unit}; use std::collections::HashSet; +use wasmtime_environ::ir::Endianness; use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::{CompiledFunctions, ModuleMemoryOffset}; @@ -463,6 +464,19 @@ where isa, )?; + // Data in WebAssembly memory always uses little-endian byte order. + // If the native architecture is big-endian, we need to mark all + // base types used to refer to WebAssembly memory as little-endian + // using the DW_AT_endianity attribute, so that the debugger will + // be able to correctly access them. + if entry.tag() == gimli::DW_TAG_base_type && isa.endianness() == Endianness::Big { + let current_scope = comp_unit.get_mut(die_id); + current_scope.set( + gimli::DW_AT_endianity, + write::AttributeValue::Endianity(gimli::DW_END_little), + ); + } + if entry.tag() == gimli::DW_TAG_subprogram && !current_scope_ranges.is_empty() { append_vmctx_info( comp_unit, diff --git a/crates/debug/src/write_debuginfo.rs b/crates/debug/src/write_debuginfo.rs index 56c7231db0..491267b495 100644 --- a/crates/debug/src/write_debuginfo.rs +++ b/crates/debug/src/write_debuginfo.rs @@ -2,6 +2,7 @@ pub use crate::transform::transform_dwarf; use gimli::write::{Address, Dwarf, EndianVec, FrameTable, Result, Sections, Writer}; use gimli::{RunTimeEndian, SectionId}; use wasmtime_environ::entity::EntityRef; +use wasmtime_environ::ir::Endianness; use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; use wasmtime_environ::{CompiledFunctions, DebugInfoData, ModuleMemoryOffset}; @@ -26,10 +27,19 @@ pub struct DwarfSection { } fn emit_dwarf_sections( + isa: &dyn TargetIsa, mut dwarf: Dwarf, frames: Option, ) -> anyhow::Result> { - let mut sections = Sections::new(WriterRelocate::default()); + let endian = match isa.endianness() { + Endianness::Little => RunTimeEndian::Little, + Endianness::Big => RunTimeEndian::Big, + }; + let writer = WriterRelocate { + relocs: Vec::new(), + writer: EndianVec::new(endian), + }; + let mut sections = Sections::new(writer); dwarf.write(&mut sections)?; if let Some(frames) = frames { frames.write_debug_frame(&mut sections.debug_frame)?; @@ -54,15 +64,6 @@ pub struct WriterRelocate { writer: EndianVec, } -impl Default for WriterRelocate { - fn default() -> Self { - WriterRelocate { - relocs: Vec::new(), - writer: EndianVec::new(RunTimeEndian::Little), - } - } -} - impl Writer for WriterRelocate { type Endian = RunTimeEndian; @@ -156,6 +157,6 @@ pub fn emit_dwarf<'a>( ) -> anyhow::Result> { let dwarf = transform_dwarf(isa, debuginfo_data, funcs, memory_offset)?; let frame_table = create_frame_table(isa, funcs); - let sections = emit_dwarf_sections(dwarf, frame_table)?; + let sections = emit_dwarf_sections(isa, dwarf, frame_table)?; Ok(sections) } diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 36eec310ec..12b321d779 100644 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -3,8 +3,8 @@ pub mod ir { pub use cranelift_codegen::binemit::{Reloc, StackMap}; pub use cranelift_codegen::ir::{ - types, AbiParam, ArgumentPurpose, JumpTableOffsets, LabelValueLoc, LibCall, Signature, - SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc, + types, AbiParam, ArgumentPurpose, Endianness, JumpTableOffsets, LabelValueLoc, LibCall, + Signature, SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc, }; pub use cranelift_codegen::{ValueLabelsRanges, ValueLocRange}; } From 9637bc5a09ac00cbad2b890f455988613c542fb6 Mon Sep 17 00:00:00 2001 From: StackDoubleFlow Date: Wed, 21 Apr 2021 09:29:02 -0400 Subject: [PATCH 27/43] Fix cranelift `Module` and `ObjectModule` docs links (#2852) --- cranelift/codegen/src/ir/entities.rs | 6 +++--- cranelift/object/src/backend.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cranelift/codegen/src/ir/entities.rs b/cranelift/codegen/src/ir/entities.rs index 09eaed3bec..d8ca7cef36 100644 --- a/cranelift/codegen/src/ir/entities.rs +++ b/cranelift/codegen/src/ir/entities.rs @@ -146,7 +146,7 @@ impl StackSlot { /// [`VmContext`](super::GlobalValueData::VMContext) using /// [`FuncEnvironment::make_global`](https://docs.rs/cranelift-wasm/*/cranelift_wasm/trait.FuncEnvironment.html#tymethod.make_global). /// - When compiling to native code, you can use it for objects in static memory with -/// [`Module::declare_data_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/struct.Module.html#method.declare_data_in_func). +/// [`Module::declare_data_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/trait.Module.html#method.declare_data_in_func). /// - For any compilation target, it can be registered with /// [`FunctionBuilder::create_global_value`](https://docs.rs/cranelift-frontend/*/cranelift_frontend/struct.FunctionBuilder.html#method.create_global_value). /// @@ -264,9 +264,9 @@ impl JumpTable { /// /// - [`FunctionBuilder::import_function`](https://docs.rs/cranelift-frontend/*/cranelift_frontend/struct.FunctionBuilder.html#method.import_function) /// for external functions -/// - [`Module::declare_func_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/struct.Module.html#method.declare_func_in_func) +/// - [`Module::declare_func_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/trait.Module.html#method.declare_func_in_func) /// for functions declared elsewhere in the same native -/// [`Module`](https://docs.rs/cranelift-module/*/cranelift_module/struct.Module.html) +/// [`Module`](https://docs.rs/cranelift-module/*/cranelift_module/trait.Module.html) /// - [`FuncEnvironment::make_direct_func`](https://docs.rs/cranelift-wasm/*/cranelift_wasm/trait.FuncEnvironment.html#tymethod.make_direct_func) /// for functions declared in the same WebAssembly /// [`FuncEnvironment`](https://docs.rs/cranelift-wasm/*/cranelift_wasm/trait.FuncEnvironment.html#tymethod.make_direct_func) diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index abfdcf10c9..6cf54fc30a 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -639,8 +639,8 @@ fn translate_linkage(linkage: Linkage) -> (SymbolScope, bool) { (scope, weak) } -/// This is the output of `Module`'s -/// [`finish`](../cranelift_module/struct.Module.html#method.finish) function. +/// This is the output of `ObjectModule`'s +/// [`finish`](../struct.ObjectModule.html#method.finish) function. /// It contains the generated `Object` and other information produced during /// compilation. pub struct ObjectProduct { From a8c956ede1ee1366035b37bc773392fa590d0f7a Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 21 Apr 2021 20:15:41 -0700 Subject: [PATCH 28/43] Factor out byteorder in cranelift This removes an existing dependency on the byteorder crate in favor of using std equivalents directly. While not an issue for wasmtime per se, cranelift is now part of the critical path of building and testing Rust, and minimizing dependencies, even small ones, can help reduce the time and bandwidth required. --- Cargo.lock | 2 -- cranelift/codegen/Cargo.toml | 1 - cranelift/codegen/src/isa/unwind/winx64.rs | 21 +++++++------- cranelift/filetests/Cargo.toml | 1 - cranelift/filetests/src/test_unwind.rs | 32 +++++++++++----------- 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76c99d6e9d..37c27174dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,7 +544,6 @@ name = "cranelift-codegen" version = "0.73.0" dependencies = [ "bincode", - "byteorder", "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", @@ -591,7 +590,6 @@ name = "cranelift-filetests" version = "0.73.0" dependencies = [ "anyhow", - "byteorder", "cranelift-codegen", "cranelift-frontend", "cranelift-interpreter", diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 90ca998ae8..9eb990f896 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -24,7 +24,6 @@ bincode = { version = "1.2.1", optional = true } gimli = { version = "0.23.0", default-features = false, features = ["write"], optional = true } smallvec = { version = "1.6.1" } thiserror = "1.0.4" -byteorder = { version = "1.3.2", default-features = false } peepmatic = { path = "../peepmatic", optional = true, version = "0.73.0" } peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.73.0" } peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.73.0" } diff --git a/cranelift/codegen/src/isa/unwind/winx64.rs b/cranelift/codegen/src/isa/unwind/winx64.rs index adac7caefa..1c232f6855 100644 --- a/cranelift/codegen/src/isa/unwind/winx64.rs +++ b/cranelift/codegen/src/isa/unwind/winx64.rs @@ -3,7 +3,6 @@ use crate::isa::unwind::input; use crate::result::{CodegenError, CodegenResult}; use alloc::vec::Vec; -use byteorder::{ByteOrder, LittleEndian}; use log::warn; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -31,13 +30,13 @@ impl<'a> Writer<'a> { self.offset += 1; } - fn write_u16(&mut self, v: u16) { - T::write_u16(&mut self.buf[self.offset..(self.offset + 2)], v); + fn write_u16_le(&mut self, v: u16) { + self.buf[self.offset..(self.offset + 2)].copy_from_slice(&v.to_le_bytes()); self.offset += 2; } - fn write_u32(&mut self, v: u32) { - T::write_u32(&mut self.buf[self.offset..(self.offset + 4)], v); + fn write_u32_le(&mut self, v: u32) { + self.buf[self.offset..(self.offset + 4)].copy_from_slice(&v.to_le_bytes()); self.offset += 4; } } @@ -121,11 +120,11 @@ impl UnwindCode { let scaled_stack_offset = stack_offset / 16; if scaled_stack_offset <= core::u16::MAX as u32 { writer.write_u8((*reg << 4) | (op_small as u8)); - writer.write_u16::(scaled_stack_offset as u16); + writer.write_u16_le(scaled_stack_offset as u16); } else { writer.write_u8((*reg << 4) | (op_large as u8)); - writer.write_u16::(*stack_offset as u16); - writer.write_u16::((stack_offset >> 16) as u16); + writer.write_u16_le(*stack_offset as u16); + writer.write_u16_le((stack_offset >> 16) as u16); } } Self::StackAlloc { @@ -143,10 +142,10 @@ impl UnwindCode { ); } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { writer.write_u8(UnwindOperation::LargeStackAlloc as u8); - writer.write_u16::((*size / 8) as u16); + writer.write_u16_le((*size / 8) as u16); } else { writer.write_u8((1 << 4) | (UnwindOperation::LargeStackAlloc as u8)); - writer.write_u32::(*size); + writer.write_u32_le(*size); } } Self::SetFPReg { instruction_offset } => { @@ -248,7 +247,7 @@ impl UnwindInfo { // To keep a 32-bit alignment, emit 2 bytes of padding if there's an odd number of 16-bit nodes if (node_count & 1) == 1 { - writer.write_u16::(0); + writer.write_u16_le(0); } // Ensure the correct number of bytes was emitted diff --git a/cranelift/filetests/Cargo.toml b/cranelift/filetests/Cargo.toml index ffeecd7ac6..f25cbcf5ac 100644 --- a/cranelift/filetests/Cargo.toml +++ b/cranelift/filetests/Cargo.toml @@ -16,7 +16,6 @@ cranelift-interpreter = { path = "../interpreter", version = "0.73.0" } cranelift-native = { path = "../native", version = "0.73.0" } cranelift-reader = { path = "../reader", version = "0.73.0" } cranelift-preopt = { path = "../preopt", version = "0.73.0" } -byteorder = { version = "1.3.2", default-features = false } file-per-thread-logger = "0.1.2" filecheck = "0.5.0" gimli = { version = "0.23.0", default-features = false, features = ["read"] } diff --git a/cranelift/filetests/src/test_unwind.rs b/cranelift/filetests/src/test_unwind.rs index 0c7124e8b2..3d22c4d3d9 100644 --- a/cranelift/filetests/src/test_unwind.rs +++ b/cranelift/filetests/src/test_unwind.rs @@ -72,7 +72,6 @@ impl SubTest for TestUnwind { } mod windowsx64 { - use byteorder::{ByteOrder, LittleEndian}; use std::fmt::Write; pub fn dump(text: &mut W, mem: &[u8]) { @@ -165,23 +164,24 @@ mod windowsx64 { let op_and_info = mem[1]; let op = UnwindOperation::from(op_and_info & 0xF); let info = (op_and_info & 0xF0) >> 4; + let unwind_le_bytes = |bytes| match (bytes, &mem[2..]) { + (2, &[b0, b1, ..]) => UnwindValue::U16(u16::from_le_bytes([b0, b1])), + (4, &[b0, b1, b2, b3, ..]) => { + UnwindValue::U32(u32::from_le_bytes([b0, b1, b2, b3])) + } + (_, _) => panic!("not enough bytes to unwind value"), + }; - let value = match op { - UnwindOperation::LargeStackAlloc => match info { - 0 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), - 1 => UnwindValue::U32(LittleEndian::read_u32(&mem[2..])), - _ => panic!("unexpected stack alloc info value"), - }, - UnwindOperation::SaveNonvolatileRegister => { - UnwindValue::U16(LittleEndian::read_u16(&mem[2..])) - } - UnwindOperation::SaveNonvolatileRegisterFar => { - UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) - } - UnwindOperation::SaveXmm128 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), - UnwindOperation::SaveXmm128Far => { - UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) + let value = match (&op, info) { + (UnwindOperation::LargeStackAlloc, 0) => unwind_le_bytes(2), + (UnwindOperation::LargeStackAlloc, 1) => unwind_le_bytes(4), + (UnwindOperation::LargeStackAlloc, _) => { + panic!("unexpected stack alloc info value") } + (UnwindOperation::SaveNonvolatileRegister, _) => unwind_le_bytes(2), + (UnwindOperation::SaveNonvolatileRegisterFar, _) => unwind_le_bytes(4), + (UnwindOperation::SaveXmm128, _) => unwind_le_bytes(2), + (UnwindOperation::SaveXmm128Far, _) => unwind_le_bytes(4), _ => UnwindValue::None, }; From 8384f3a3479e071175194710ea71873e9cabd3d3 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 27 Apr 2021 10:55:12 -0500 Subject: [PATCH 29/43] Bring back `Module::deserialize` (#2858) * Bring back `Module::deserialize` I thought I was being clever suggesting that `Module::deserialize` was removed from #2791 by funneling all module constructors into `Module::new`. As our studious fuzzers have found, though, this means that `Module::new` is not safe currently to pass arbitrary user-defined input into. Now one might pretty reasonable expect to be able to do that, however, being a WebAssembly engine and all. This PR as a result separates the `deserialize` part of `Module::new` back into `Module::deserialize`. This means that binary blobs created with `Module::serialize` and `Engine::precompile_module` will need to be passed to `Module::deserialize` to "rehydrate" them back into a `Module`. This restores the property that it should be safe to pass arbitrary input to `Module::new` since it's always expected to be a wasm module. This also means that fuzzing will no longer attempt to fuzz `Module::deserialize` which isn't something we want to do anyway. * Fix an example * Mark `Module::deserialize` as `unsafe` --- crates/c-api/include/wasmtime.h | 6 +- crates/c-api/src/module.rs | 11 ++-- crates/wasmtime/src/engine.rs | 5 +- crates/wasmtime/src/module.rs | 63 +++++++++++++++------ crates/wasmtime/src/module/serialization.rs | 12 ++-- examples/serialize.rs | 7 ++- src/commands/compile.rs | 3 +- tests/all/module.rs | 54 +++++++++--------- tests/all/module_linking.rs | 4 +- tests/all/module_serialize.rs | 10 ++-- 10 files changed, 110 insertions(+), 65 deletions(-) diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h index 8babeabdff..f163f923bb 100644 --- a/crates/c-api/include/wasmtime.h +++ b/crates/c-api/include/wasmtime.h @@ -992,9 +992,13 @@ WASM_API_EXTERN own wasmtime_error_t* wasmtime_module_serialize( /** * \brief Build a module from serialized data. - * * + * * This function does not take ownership of any of its arguments, but the * returned error and module are owned by the caller. + * + * This function is not safe to receive arbitrary user input. See the Rust + * documentation for more information on what inputs are safe to pass in here + * (e.g. only that of #wasmtime_module_serialize) */ WASM_API_EXTERN own wasmtime_error_t *wasmtime_module_deserialize( wasm_engine_t *engine, diff --git a/crates/c-api/src/module.rs b/crates/c-api/src/module.rs index 1757191026..4b60c67e73 100644 --- a/crates/c-api/src/module.rs +++ b/crates/c-api/src/module.rs @@ -185,10 +185,13 @@ pub extern "C" fn wasmtime_module_deserialize( binary: &wasm_byte_vec_t, ret: &mut *mut wasm_module_t, ) -> Option> { - handle_result(Module::new(&engine.engine, binary.as_slice()), |module| { - let module = Box::new(wasm_module_t::new(module)); - *ret = Box::into_raw(module); - }) + handle_result( + unsafe { Module::deserialize(&engine.engine, binary.as_slice()) }, + |module| { + let module = Box::new(wasm_module_t::new(module)); + *ret = Box::into_raw(module); + }, + ) } #[no_mangle] diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index acafaa5797..8876eb042d 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -160,8 +160,9 @@ impl Engine { /// Note that the `wat` feature is enabled by default. /// /// This method may be used to compile a module for use with a different target - /// host. The output of this method may be used with [`Module::new`](crate::Module::new) - /// on hosts compatible with the [`Config`] associated with this [`Engine`]. + /// host. The output of this method may be used with + /// [`Module::deserialize`](crate::Module::deserialize) on hosts compatible + /// with the [`Config`] associated with this [`Engine`]. /// /// The output of this method is safe to send to another host machine for later /// execution. As the output is already a compiled module, translation and code diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index e9ae9579e3..4964189d7b 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -121,9 +121,6 @@ impl Module { /// This is only supported when the `wat` feature of this crate is enabled. /// If this is supplied then the text format will be parsed before validation. /// Note that the `wat` feature is enabled by default. - /// * A module serialized with [`Module::serialize`]. - /// * A module compiled with [`Engine::precompile_module`] or the - /// `wasmtime compile` command. /// /// The data for the wasm module must be loaded in-memory if it's present /// elsewhere, for example on disk. This requires that the entire binary is @@ -182,11 +179,6 @@ impl Module { /// ``` pub fn new(engine: &Engine, bytes: impl AsRef<[u8]>) -> Result { let bytes = bytes.as_ref(); - - if let Some(module) = SerializedModule::from_bytes(bytes)? { - return module.into_module(engine); - } - #[cfg(feature = "wat")] let bytes = wat::parse_bytes(bytes)?; Self::from_binary(engine, &bytes) @@ -258,10 +250,10 @@ impl Module { /// data. /// /// This is similar to [`Module::new`] except that it requires that the - /// `binary` input is a WebAssembly binary or a compiled module, the - /// text format is not supported by this function. It's generally - /// recommended to use [`Module::new`], but if it's required to not - /// support the text format this function can be used instead. + /// `binary` input is a WebAssembly binary, the text format is not supported + /// by this function. It's generally recommended to use [`Module::new`], but + /// if it's required to not support the text format this function can be + /// used instead. /// /// # Examples /// @@ -286,10 +278,6 @@ impl Module { /// # } /// ``` pub fn from_binary(engine: &Engine, binary: &[u8]) -> Result { - if let Some(module) = SerializedModule::from_bytes(binary)? { - return module.into_module(engine); - } - // Check to see that the config's target matches the host let target = engine.config().isa_flags.triple(); if *target != target_lexicon::Triple::host() { @@ -329,6 +317,49 @@ impl Module { Self::from_parts(engine, modules, main_module, Arc::new(types), &[]) } + /// Deserializes an in-memory compiled module previously created with + /// [`Module::serialize`] or [`Engine::precompile_module`]. + /// + /// This function will deserialize the binary blobs emitted by + /// [`Module::serialize`] and [`Engine::precompile_module`] back into an + /// in-memory [`Module`] that's ready to be instantiated. + /// + /// # Unsafety + /// + /// This function is marked as `unsafe` because if fed invalid input or used + /// improperly this could lead to memory safety vulnerabilities. This method + /// should not, for example, be exposed to arbitrary user input. + /// + /// The structure of the binary blob read here is only lightly validated + /// internally in `wasmtime`. This is intended to be an efficient + /// "rehydration" for a [`Module`] which has very few runtime checks beyond + /// deserialization. Arbitrary input could, for example, replace valid + /// compiled code with any other valid compiled code, meaning that this can + /// trivially be used to execute arbitrary code otherwise. + /// + /// For these reasons this function is `unsafe`. This function is only + /// designed to receive the previous input from [`Module::serialize`] and + /// [`Engine::precompile_module`]. If the exact output of those functions + /// (unmodified) is passed to this function then calls to this function can + /// be considered safe. It is the caller's responsibility to provide the + /// guarantee that only previously-serialized bytes are being passed in + /// here. + /// + /// Note that this function is designed to be safe receiving output from + /// *any* compiled version of `wasmtime` itself. This means that it is safe + /// to feed output from older versions of Wasmtime into this function, in + /// addition to newer versions of wasmtime (from the future!). These inputs + /// will deterministically and safely produce an `Err`. This function only + /// successfully accepts inputs from the same version of `wasmtime`, but the + /// safety guarantee only applies to externally-defined blobs of bytes, not + /// those defined by any version of wasmtime. (this means that if you cache + /// blobs across versions of wasmtime you can be safely guaranteed that + /// future versions of wasmtime will reject old cache entries). + pub unsafe fn deserialize(engine: &Engine, bytes: impl AsRef<[u8]>) -> Result { + let module = SerializedModule::from_bytes(bytes.as_ref())?; + module.into_module(engine) + } + fn from_parts( engine: &Engine, mut modules: Vec>, diff --git a/crates/wasmtime/src/module/serialization.rs b/crates/wasmtime/src/module/serialization.rs index 8b3620a75d..e566d01ed5 100644 --- a/crates/wasmtime/src/module/serialization.rs +++ b/crates/wasmtime/src/module/serialization.rs @@ -329,9 +329,9 @@ impl<'a> SerializedModule<'a> { Ok(bytes) } - pub fn from_bytes(bytes: &[u8]) -> Result> { + pub fn from_bytes(bytes: &[u8]) -> Result { if !bytes.starts_with(HEADER) { - return Ok(None); + bail!("bytes are not a compatible serialized wasmtime module"); } let bytes = &bytes[HEADER.len()..]; @@ -353,11 +353,9 @@ impl<'a> SerializedModule<'a> { ); } - Ok(Some( - bincode_options() - .deserialize::>(&bytes[1 + version_len..]) - .context("deserialize compilation artifacts")?, - )) + Ok(bincode_options() + .deserialize::>(&bytes[1 + version_len..]) + .context("deserialize compilation artifacts")?) } fn check_triple(&self, isa: &dyn TargetIsa) -> Result<()> { diff --git a/examples/serialize.rs b/examples/serialize.rs index 3cd709e748..70a875c3bf 100644 --- a/examples/serialize.rs +++ b/examples/serialize.rs @@ -29,9 +29,12 @@ fn deserialize(buffer: &[u8]) -> Result<()> { println!("Initializing..."); let store = Store::default(); - // Compile the wasm binary into an in-memory instance of a `Module`. + // Compile the wasm binary into an in-memory instance of a `Module`. Note + // that this is `unsafe` because it is our responsibility for guaranteeing + // that these bytes are valid precompiled module bytes. We know that from + // the structure of this example program. println!("Deserialize module..."); - let module = Module::new(store.engine(), buffer)?; + let module = unsafe { Module::deserialize(store.engine(), buffer)? }; // Here we handle the imports of the module, which in this case is our // `HelloCallback` type and its associated implementation of `Callback. diff --git a/src/commands/compile.rs b/src/commands/compile.rs index 7b9a7ebcc2..1ad611623c 100644 --- a/src/commands/compile.rs +++ b/src/commands/compile.rs @@ -126,7 +126,8 @@ mod test { command.execute()?; let engine = Engine::default(); - let module = Module::from_file(&engine, output_path)?; + let contents = std::fs::read(output_path)?; + let module = unsafe { Module::deserialize(&engine, contents)? }; let store = Store::new(&engine); let instance = Instance::new(&store, &module, &[])?; let f = instance.get_typed_func::("f")?; diff --git a/tests/all/module.rs b/tests/all/module.rs index 279c91e829..f6b3b5b0d0 100644 --- a/tests/all/module.rs +++ b/tests/all/module.rs @@ -27,35 +27,37 @@ fn caches_across_engines() { .serialize() .unwrap(); - let res = Module::new(&Engine::new(&Config::new()).unwrap(), &bytes); - assert!(res.is_ok()); + unsafe { + let res = Module::deserialize(&Engine::new(&Config::new()).unwrap(), &bytes); + assert!(res.is_ok()); - // differ in shared cranelift flags - let res = Module::new( - &Engine::new(Config::new().cranelift_nan_canonicalization(true)).unwrap(), - &bytes, - ); - assert!(res.is_err()); - - // differ in cranelift settings - let res = Module::new( - &Engine::new(Config::new().cranelift_opt_level(OptLevel::None)).unwrap(), - &bytes, - ); - assert!(res.is_err()); - - // Missing required cpu flags - if cfg!(target_arch = "x86_64") { - let res = Module::new( - &Engine::new( - Config::new() - .target(&target_lexicon::Triple::host().to_string()) - .unwrap(), - ) - .unwrap(), + // differ in shared cranelift flags + let res = Module::deserialize( + &Engine::new(Config::new().cranelift_nan_canonicalization(true)).unwrap(), &bytes, ); assert!(res.is_err()); + + // differ in cranelift settings + let res = Module::deserialize( + &Engine::new(Config::new().cranelift_opt_level(OptLevel::None)).unwrap(), + &bytes, + ); + assert!(res.is_err()); + + // Missing required cpu flags + if cfg!(target_arch = "x86_64") { + let res = Module::deserialize( + &Engine::new( + Config::new() + .target(&target_lexicon::Triple::host().to_string()) + .unwrap(), + ) + .unwrap(), + &bytes, + ); + assert!(res.is_err()); + } } } @@ -66,7 +68,7 @@ fn aot_compiles() -> Result<()> { "(module (func (export \"f\") (param i32) (result i32) local.get 0))".as_bytes(), )?; - let module = Module::from_binary(&engine, &bytes)?; + let module = unsafe { Module::deserialize(&engine, &bytes)? }; let store = Store::new(&engine); let instance = Instance::new(&store, &module, &[])?; diff --git a/tests/all/module_linking.rs b/tests/all/module_linking.rs index 357caa982f..28fb802fae 100644 --- a/tests/all/module_linking.rs +++ b/tests/all/module_linking.rs @@ -39,7 +39,9 @@ fn compile() -> Result<()> { assert_eq!(m.imports().len(), 0); assert_eq!(m.exports().len(), 0); let bytes = m.serialize()?; - Module::new(&engine, &bytes)?; + unsafe { + Module::deserialize(&engine, &bytes)?; + } assert_eq!(m.imports().len(), 0); assert_eq!(m.exports().len(), 0); Ok(()) diff --git a/tests/all/module_serialize.rs b/tests/all/module_serialize.rs index 4409659fcd..7f68b48a58 100644 --- a/tests/all/module_serialize.rs +++ b/tests/all/module_serialize.rs @@ -6,8 +6,8 @@ fn serialize(engine: &Engine, wat: &'static str) -> Result> { Ok(module.serialize()?) } -fn deserialize_and_instantiate(store: &Store, buffer: &[u8]) -> Result { - let module = Module::new(store.engine(), buffer)?; +unsafe fn deserialize_and_instantiate(store: &Store, buffer: &[u8]) -> Result { + let module = Module::deserialize(store.engine(), buffer)?; Ok(Instance::new(&store, &module, &[])?) } @@ -17,7 +17,7 @@ fn test_version_mismatch() -> Result<()> { let mut buffer = serialize(&engine, "(module)")?; buffer[13 /* header length */ + 1 /* version length */] = 'x' as u8; - match Module::new(&engine, &buffer) { + match unsafe { Module::deserialize(&engine, &buffer) } { Ok(_) => bail!("expected deserialization to fail"), Err(e) => assert!(e .to_string() @@ -35,7 +35,7 @@ fn test_module_serialize_simple() -> Result<()> { )?; let store = Store::default(); - let instance = deserialize_and_instantiate(&store, &buffer)?; + let instance = unsafe { deserialize_and_instantiate(&store, &buffer)? }; let run = instance.get_typed_func::<(), i32>("run")?; let result = run.call(())?; @@ -53,7 +53,7 @@ fn test_module_serialize_fail() -> Result<()> { let mut config = Config::new(); config.cranelift_opt_level(OptLevel::None); let store = Store::new(&Engine::new(&config)?); - match deserialize_and_instantiate(&store, &buffer) { + match unsafe { deserialize_and_instantiate(&store, &buffer) } { Ok(_) => bail!("expected failure at deserialization"), Err(_) => (), } From 480670e17f3a614cb30efecaeff6a5cb83379b61 Mon Sep 17 00:00:00 2001 From: Anton Kirilov Date: Fri, 16 Apr 2021 13:10:30 +0100 Subject: [PATCH 30/43] Enable the simd_boolean test for AArch64 Also, enable the simd_i64x2_arith2 test because it doesn't need any code changes. Copyright (c) 2021, Arm Limited. --- build.rs | 4 - .../codegen/src/isa/aarch64/inst/emit.rs | 17 +++ .../src/isa/aarch64/inst/emit_tests.rs | 21 ++++ cranelift/codegen/src/isa/aarch64/inst/mod.rs | 100 ++++++++++++------ .../codegen/src/isa/aarch64/lower_inst.rs | 75 +++++++++++++ 5 files changed, 182 insertions(+), 35 deletions(-) diff --git a/build.rs b/build.rs index 411593c505..42d786799f 100644 --- a/build.rs +++ b/build.rs @@ -220,10 +220,6 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { }, "Cranelift" => match (testsuite, testname) { ("simd", _) if cfg!(feature = "old-x86-backend") => return true, // skip all SIMD tests on old backend. - // These are only implemented on x64. - ("simd", "simd_i64x2_arith2") | ("simd", "simd_boolean") => { - return !platform_is_x64() || cfg!(feature = "old-x86-backend") - } // These are new instructions that are not really implemented in any backend. ("simd", "simd_i8x16_arith2") | ("simd", "simd_conversions") diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 1e8ca78317..6621e3f409 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -427,6 +427,15 @@ fn enc_vec_rr_misc(qu: u32, size: u32, bits_12_16: u32, rd: Writable, rn: R | machreg_to_vec(rd.to_reg()) } +fn enc_vec_rr_pair(bits_12_16: u32, rd: Writable, rn: Reg) -> u32 { + debug_assert_eq!(bits_12_16 & 0b11111, bits_12_16); + + 0b010_11110_11_11000_11011_10_00000_00000 + | bits_12_16 << 12 + | machreg_to_vec(rn) << 5 + | machreg_to_vec(rd.to_reg()) +} + fn enc_vec_lanes(q: u32, u: u32, size: u32, opcode: u32, rd: Writable, rn: Reg) -> u32 { debug_assert_eq!(q & 0b1, q); debug_assert_eq!(u & 0b1, u); @@ -1628,6 +1637,7 @@ impl MachInstEmit for Inst { debug_assert!(size == VectorSize::Size8x8 || size == VectorSize::Size8x16); (0b0, 0b00101, enc_size) } + VecMisc2::Cmeq0 => (0b0, 0b01001, enc_size), }; sink.put4(enc_vec_rr_misc((q << 1) | u, size, bits_12_16, rd, rn)); } @@ -2054,6 +2064,13 @@ impl MachInstEmit for Inst { | machreg_to_vec(rd.to_reg()), ); } + &Inst::VecRRPair { op, rd, rn } => { + let bits_12_16 = match op { + VecPairOp::Addp => 0b11011, + }; + + sink.put4(enc_vec_rr_pair(bits_12_16, rd, rn)); + } &Inst::VecRRR { rd, rn, diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index 505fd2c86b..9f628fced6 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -2311,6 +2311,16 @@ fn test_aarch64_binemit() { "sqxtun v16.8b, v23.8h", )); + insns.push(( + Inst::VecRRPair { + op: VecPairOp::Addp, + rd: writable_vreg(0), + rn: vreg(30), + }, + "C0BBF15E", + "addp d0, v30.2d", + )); + insns.push(( Inst::VecRRR { alu_op: VecALUOp::Sqadd, @@ -3803,6 +3813,17 @@ fn test_aarch64_binemit() { "cnt v23.8b, v5.8b", )); + insns.push(( + Inst::VecMisc { + op: VecMisc2::Cmeq0, + rd: writable_vreg(12), + rn: vreg(27), + size: VectorSize::Size16x8, + }, + "6C9B604E", + "cmeq v12.8h, v27.8h, #0", + )); + insns.push(( Inst::VecLanes { op: VecLanesOp::Uminv, diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index f6a6aa59d0..35903c18d0 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -334,6 +334,8 @@ pub enum VecMisc2 { Frintp, /// Population count per byte Cnt, + /// Compare bitwise equal to 0 + Cmeq0, } /// A Vector narrowing operation with two registers. @@ -347,6 +349,13 @@ pub enum VecMiscNarrowOp { Sqxtun, } +/// A vector operation on a pair of elements with one register. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum VecPairOp { + /// Add pair of elements + Addp, +} + /// An operation across the lanes of vectors. #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum VecLanesOp { @@ -1011,6 +1020,13 @@ pub enum Inst { high_half: bool, }, + /// 1-operand vector instruction that operates on a pair of elements. + VecRRPair { + op: VecPairOp, + rd: Writable, + rn: Reg, + }, + /// A vector ALU op. VecRRR { alu_op: VecALUOp, @@ -2028,6 +2044,10 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(rd); } } + &Inst::VecRRPair { rd, rn, .. } => { + collector.add_def(rd); + collector.add_use(rn); + } &Inst::VecRRR { alu_op, rd, rn, rm, .. } => { @@ -2816,6 +2836,14 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { map_def(mapper, rd); } } + &mut Inst::VecRRPair { + ref mut rd, + ref mut rn, + .. + } => { + map_def(mapper, rd); + map_use(mapper, rn); + } &mut Inst::VecRRR { alu_op, ref mut rd, @@ -3856,6 +3884,15 @@ impl Inst { }; format!("{} {}, {}", op, rd, rn) } + &Inst::VecRRPair { op, rd, rn } => { + let op = match op { + VecPairOp::Addp => "addp", + }; + let rd = show_vreg_scalar(rd.to_reg(), mb_rru, ScalarSize::Size64); + let rn = show_vreg_vector(rn, mb_rru, VectorSize::Size64x2); + + format!("{} {}, {}", op, rd, rn) + } &Inst::VecRRR { rd, rn, @@ -3919,43 +3956,44 @@ impl Inst { format!("{} {}, {}, {}", op, rd, rn, rm) } &Inst::VecMisc { op, rd, rn, size } => { - let is_shll = op == VecMisc2::Shll; - let suffix = match (is_shll, size) { - (true, VectorSize::Size8x8) => ", #8", - (true, VectorSize::Size16x4) => ", #16", - (true, VectorSize::Size32x2) => ", #32", - _ => "", - }; - - let (op, size) = match op { - VecMisc2::Not => ( - "mvn", - if size.is_128bits() { + let (op, rd_size, size, suffix) = match op { + VecMisc2::Not => { + let size = if size.is_128bits() { VectorSize::Size8x16 } else { VectorSize::Size8x8 + }; + + ("mvn", size, size, "") + } + VecMisc2::Neg => ("neg", size, size, ""), + VecMisc2::Abs => ("abs", size, size, ""), + VecMisc2::Fabs => ("fabs", size, size, ""), + VecMisc2::Fneg => ("fneg", size, size, ""), + VecMisc2::Fsqrt => ("fsqrt", size, size, ""), + VecMisc2::Rev64 => ("rev64", size, size, ""), + VecMisc2::Shll => ( + "shll", + size.widen(), + size, + match size { + VectorSize::Size8x8 => ", #8", + VectorSize::Size16x4 => ", #16", + VectorSize::Size32x2 => ", #32", + _ => panic!("Unexpected vector size: {:?}", size), }, ), - VecMisc2::Neg => ("neg", size), - VecMisc2::Abs => ("abs", size), - VecMisc2::Fabs => ("fabs", size), - VecMisc2::Fneg => ("fneg", size), - VecMisc2::Fsqrt => ("fsqrt", size), - VecMisc2::Rev64 => ("rev64", size), - VecMisc2::Shll => ("shll", size), - VecMisc2::Fcvtzs => ("fcvtzs", size), - VecMisc2::Fcvtzu => ("fcvtzu", size), - VecMisc2::Scvtf => ("scvtf", size), - VecMisc2::Ucvtf => ("ucvtf", size), - VecMisc2::Frintn => ("frintn", size), - VecMisc2::Frintz => ("frintz", size), - VecMisc2::Frintm => ("frintm", size), - VecMisc2::Frintp => ("frintp", size), - VecMisc2::Cnt => ("cnt", size), + VecMisc2::Fcvtzs => ("fcvtzs", size, size, ""), + VecMisc2::Fcvtzu => ("fcvtzu", size, size, ""), + VecMisc2::Scvtf => ("scvtf", size, size, ""), + VecMisc2::Ucvtf => ("ucvtf", size, size, ""), + VecMisc2::Frintn => ("frintn", size, size, ""), + VecMisc2::Frintz => ("frintz", size, size, ""), + VecMisc2::Frintm => ("frintm", size, size, ""), + VecMisc2::Frintp => ("frintp", size, size, ""), + VecMisc2::Cnt => ("cnt", size, size, ""), + VecMisc2::Cmeq0 => ("cmeq", size, size, ", #0"), }; - - let rd_size = if is_shll { size.widen() } else { size }; - let rd = show_vreg_vector(rd.to_reg(), mb_rru, rd_size); let rn = show_vreg_vector(rn, mb_rru, size); format!("{} {}, {}{}", op, rd, rn, suffix) diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index be3edd953b..ede66295e9 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -1950,6 +1950,40 @@ pub(crate) fn lower_insn_to_regs>( } } + Opcode::VallTrue if ctx.input_ty(insn, 0) == I64X2 => { + let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let tmp = ctx.alloc_tmp(I64X2).only_reg().unwrap(); + + // cmeq vtmp.2d, vm.2d, #0 + // addp dtmp, vtmp.2d + // fcmp dtmp, dtmp + // cset xd, eq + // + // Note that after the ADDP the value of the temporary register will + // be either 0 when all input elements are true, i.e. non-zero, or a + // NaN otherwise (either -1 or -2 when represented as an integer); + // NaNs are the only floating-point numbers that compare unequal to + // themselves. + + ctx.emit(Inst::VecMisc { + op: VecMisc2::Cmeq0, + rd: tmp, + rn: rm, + size: VectorSize::Size64x2, + }); + ctx.emit(Inst::VecRRPair { + op: VecPairOp::Addp, + rd: tmp, + rn: tmp.to_reg(), + }); + ctx.emit(Inst::FpuCmp64 { + rn: tmp.to_reg(), + rm: tmp.to_reg(), + }); + materialize_bool_result(ctx, insn, rd, Cond::Eq); + } + Opcode::VanyTrue | Opcode::VallTrue => { let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); @@ -2180,6 +2214,47 @@ pub(crate) fn lower_insn_to_regs>( size: VectorSize::Size32x4, }); } + I64X2 => { + // mov dst_r, src_v.d[0] + // mov tmp_r0, src_v.d[1] + // lsr dst_r, dst_r, #63 + // lsr tmp_r0, tmp_r0, #63 + // add dst_r, dst_r, tmp_r0, lsl #1 + ctx.emit(Inst::MovFromVec { + rd: dst_r, + rn: src_v, + idx: 0, + size: VectorSize::Size64x2, + }); + ctx.emit(Inst::MovFromVec { + rd: tmp_r0, + rn: src_v, + idx: 1, + size: VectorSize::Size64x2, + }); + ctx.emit(Inst::AluRRImmShift { + alu_op: ALUOp::Lsr64, + rd: dst_r, + rn: dst_r.to_reg(), + immshift: ImmShift::maybe_from_u64(63).unwrap(), + }); + ctx.emit(Inst::AluRRImmShift { + alu_op: ALUOp::Lsr64, + rd: tmp_r0, + rn: tmp_r0.to_reg(), + immshift: ImmShift::maybe_from_u64(63).unwrap(), + }); + ctx.emit(Inst::AluRRRShift { + alu_op: ALUOp::Add32, + rd: dst_r, + rn: dst_r.to_reg(), + rm: tmp_r0.to_reg(), + shiftop: ShiftOpAndAmt::new( + ShiftOp::LSL, + ShiftOpShiftImm::maybe_from_shift(1).unwrap(), + ), + }); + } _ => panic!("arm64 isel: VhighBits unhandled, ty = {:?}", ty), } } From 7ec073cef118fa444908644da7f6b9f76ff303d7 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 28 Apr 2021 12:08:27 -0500 Subject: [PATCH 31/43] Bring back per-thread lazy initialization (#2863) * Bring back per-thread lazy initialization Platforms Wasmtime supports may have per-thread initialization that needs to run before WebAssembly. For example Unix needs to setup a sigaltstack and macOS needs to set up mach ports. In #2757 this per-thread setup was moved out of the invocation of a wasm function, relying on the lack of Send for Store to initialize the thread at Store creation time and never worry about it later. This conflicted with [wasmtime's desired multithreading story](https://github.com/bytecodealliance/wasmtime/pull/2812) so a new [`Store::notify_switched_thread` was added](https://github.com/bytecodealliance/wasmtime/pull/2822) to explicitly indicate a Store has moved to another thread (if it unsafely did so). It turns out though that it's not always easy to determine when a `Store` moves to a new thread. For example the Go bindings for Wasmtime are generally unaware when a goroutine switches OS threads. This led to https://github.com/bytecodealliance/wasmtime-go/issues/74 where a SIGILL was left uncaught, making it appear that traps aren't working properly. This commit revisits the decision in #2757 and moves per-thread initialization back into the path of calling into WebAssembly. This is differently from before, though, where there's still only one TLS access on the path of calling into WebAssembly, unlike before where it was a separate access. This allows us to get the speed benefits of #2757 as well as the flexibility benefits of not having to explicitly move a store between threads. With this new ability this commit deletes the recently added `Store::notify_switched_thread` method since it's no longer necessary. * Fix a test compiling --- crates/runtime/src/traphandlers.rs | 53 +++++++++++++++--------- crates/runtime/src/traphandlers/macos.rs | 31 +++++--------- crates/runtime/src/traphandlers/unix.rs | 50 ++++++++-------------- crates/wasmtime/src/store.rs | 33 +++------------ docs/examples-rust-multithreading.md | 15 +++---- tests/all/traps.rs | 3 -- 6 files changed, 72 insertions(+), 113 deletions(-) diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 36d7ceba86..75f7a7d6d4 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -58,13 +58,12 @@ static mut IS_WASM_PC: fn(usize) -> bool = |_| false; /// program counter is the pc of an actual wasm trap or not. This is then used /// to disambiguate faults that happen due to wasm and faults that happen due to /// bugs in Rust or elsewhere. -pub fn init_traps(is_wasm_pc: fn(usize) -> bool) -> Result<(), Trap> { +pub fn init_traps(is_wasm_pc: fn(usize) -> bool) { static INIT: Once = Once::new(); INIT.call_once(|| unsafe { IS_WASM_PC = is_wasm_pc; sys::platform_init(); }); - sys::lazy_per_thread_init() } /// Raises a user-defined trap immediately. @@ -256,7 +255,7 @@ impl<'a> CallThreadState<'a> { } fn with(self, closure: impl FnOnce(&CallThreadState) -> i32) -> Result<(), Trap> { - let ret = tls::set(&self, || closure(&self)); + let ret = tls::set(&self, || closure(&self))?; if ret != 0 { return Ok(()); } @@ -366,6 +365,7 @@ impl Drop for ResetCell<'_, T> { // the caller to the trap site. mod tls { use super::CallThreadState; + use crate::Trap; use std::mem; use std::ptr; @@ -384,21 +384,38 @@ mod tls { // these TLS values when the runtime may have crossed threads. mod raw { use super::CallThreadState; + use crate::Trap; use std::cell::Cell; use std::ptr; pub type Ptr = *const CallThreadState<'static>; - thread_local!(static PTR: Cell = Cell::new(ptr::null())); + // The first entry here is the `Ptr` which is what's used as part of the + // public interface of this module. The second entry is a boolean which + // allows the runtime to perform per-thread initialization if necessary + // for handling traps (e.g. setting up ports on macOS and sigaltstack on + // Unix). + thread_local!(static PTR: Cell<(Ptr, bool)> = Cell::new((ptr::null(), false))); #[inline(never)] // see module docs for why this is here - pub fn replace(val: Ptr) -> Ptr { - PTR.with(|p| p.replace(val)) + pub fn replace(val: Ptr) -> Result { + PTR.with(|p| { + // When a new value is configured that means that we may be + // entering WebAssembly so check to see if this thread has + // performed per-thread initialization for traps. + let (prev, mut initialized) = p.get(); + if !initialized { + super::super::sys::lazy_per_thread_init()?; + initialized = true; + } + p.set((val, initialized)); + Ok(prev) + }) } #[inline(never)] // see module docs for why this is here pub fn get() -> Ptr { - PTR.with(|p| p.get()) + PTR.with(|p| p.get().0) } } @@ -412,7 +429,7 @@ mod tls { /// /// This is not a safe operation since it's intended to only be used /// with stack switching found with fibers and async wasmtime. - pub unsafe fn take() -> TlsRestore { + pub unsafe fn take() -> Result { // Our tls pointer must be set at this time, and it must not be // null. We need to restore the previous pointer since we're // removing ourselves from the call-stack, and in the process we @@ -421,8 +438,8 @@ mod tls { let raw = raw::get(); assert!(!raw.is_null()); let prev = (*raw).prev.replace(ptr::null()); - raw::replace(prev); - TlsRestore(raw) + raw::replace(prev)?; + Ok(TlsRestore(raw)) } /// Restores a previous tls state back into this thread's TLS. @@ -430,17 +447,12 @@ mod tls { /// This is unsafe because it's intended to only be used within the /// context of stack switching within wasmtime. pub unsafe fn replace(self) -> Result<(), super::Trap> { - // When replacing to the previous value of TLS, we might have - // crossed a thread: make sure the trap-handling lazy initializer - // runs. - super::sys::lazy_per_thread_init()?; - // We need to configure our previous TLS pointer to whatever is in // TLS at this time, and then we set the current state to ourselves. let prev = raw::get(); assert!((*self.0).prev.get().is_null()); (*self.0).prev.set(prev); - raw::replace(self.0); + raw::replace(self.0)?; Ok(()) } } @@ -448,13 +460,14 @@ mod tls { /// Configures thread local state such that for the duration of the /// execution of `closure` any call to `with` will yield `ptr`, unless this /// is recursively called again. - pub fn set(state: &CallThreadState<'_>, closure: impl FnOnce() -> R) -> R { + pub fn set(state: &CallThreadState<'_>, closure: impl FnOnce() -> R) -> Result { struct Reset<'a, 'b>(&'a CallThreadState<'b>); impl Drop for Reset<'_, '_> { #[inline] fn drop(&mut self) { - raw::replace(self.0.prev.replace(ptr::null())); + raw::replace(self.0.prev.replace(ptr::null())) + .expect("tls should be previously initialized"); } } @@ -464,10 +477,10 @@ mod tls { let ptr = unsafe { mem::transmute::<*const CallThreadState<'_>, *const CallThreadState<'static>>(state) }; - let prev = raw::replace(ptr); + let prev = raw::replace(ptr)?; state.prev.set(prev); let _reset = Reset(state); - closure() + Ok(closure()) } /// Returns the last pointer configured with `set` above. Panics if `set` diff --git a/crates/runtime/src/traphandlers/macos.rs b/crates/runtime/src/traphandlers/macos.rs index 2f92167526..f48ae034a1 100644 --- a/crates/runtime/src/traphandlers/macos.rs +++ b/crates/runtime/src/traphandlers/macos.rs @@ -42,7 +42,6 @@ use mach::message::*; use mach::port::*; use mach::thread_act::*; use mach::traps::*; -use std::cell::Cell; use std::mem; use std::thread; @@ -425,26 +424,16 @@ impl Drop for ClosePort { /// task-level port which is where we'd expected things like breakpad/crashpad /// exception handlers to get registered. pub fn lazy_per_thread_init() -> Result<(), Trap> { - thread_local! { - static PORTS_SET: Cell = Cell::new(false); + unsafe { + assert!(WASMTIME_PORT != MACH_PORT_NULL); + let kret = thread_set_exception_ports( + MY_PORT.with(|p| p.0), + EXC_MASK_BAD_ACCESS | EXC_MASK_BAD_INSTRUCTION, + WASMTIME_PORT, + EXCEPTION_DEFAULT | MACH_EXCEPTION_CODES, + mach_addons::THREAD_STATE_NONE, + ); + assert_eq!(kret, KERN_SUCCESS, "failed to set thread exception port"); } - - PORTS_SET.with(|ports| { - if ports.replace(true) { - return; - } - - unsafe { - assert!(WASMTIME_PORT != MACH_PORT_NULL); - let kret = thread_set_exception_ports( - MY_PORT.with(|p| p.0), - EXC_MASK_BAD_ACCESS | EXC_MASK_BAD_INSTRUCTION, - WASMTIME_PORT, - EXCEPTION_DEFAULT | MACH_EXCEPTION_CODES, - mach_addons::THREAD_STATE_NONE, - ); - assert_eq!(kret, KERN_SUCCESS, "failed to set thread exception port"); - } - }); Ok(()) } diff --git a/crates/runtime/src/traphandlers/unix.rs b/crates/runtime/src/traphandlers/unix.rs index 579acf59c3..955d00a62b 100644 --- a/crates/runtime/src/traphandlers/unix.rs +++ b/crates/runtime/src/traphandlers/unix.rs @@ -154,41 +154,35 @@ unsafe fn get_pc(cx: *mut libc::c_void) -> *const u8 { /// and registering our own alternate stack that is large enough and has a guard /// page. pub fn lazy_per_thread_init() -> Result<(), Trap> { + // This thread local is purely used to register a `Stack` to get deallocated + // when the thread exists. Otherwise this function is only ever called at + // most once per-thread. thread_local! { - /// Thread-local state is lazy-initialized on the first time it's used, - /// and dropped when the thread exits. - static TLS: RefCell = RefCell::new(Tls::None); + static STACK: RefCell> = RefCell::new(None); } /// The size of the sigaltstack (not including the guard, which will be /// added). Make this large enough to run our signal handlers. const MIN_STACK_SIZE: usize = 16 * 4096; - enum Tls { - None, - Allocated { - mmap_ptr: *mut libc::c_void, - mmap_size: usize, - }, - BigEnough, + struct Stack { + mmap_ptr: *mut libc::c_void, + mmap_size: usize, } - return TLS.with(|slot| unsafe { - let mut slot = slot.borrow_mut(); - match *slot { - Tls::None => {} - // already checked - _ => return Ok(()), - } + return STACK.with(|s| { + *s.borrow_mut() = unsafe { allocate_sigaltstack()? }; + Ok(()) + }); + unsafe fn allocate_sigaltstack() -> Result, Trap> { // Check to see if the existing sigaltstack, if it exists, is big // enough. If so we don't need to allocate our own. let mut old_stack = mem::zeroed(); let r = libc::sigaltstack(ptr::null(), &mut old_stack); assert_eq!(r, 0, "learning about sigaltstack failed"); if old_stack.ss_flags & libc::SS_DISABLE == 0 && old_stack.ss_size >= MIN_STACK_SIZE { - *slot = Tls::BigEnough; - return Ok(()); + return Ok(None); } // ... but failing that we need to allocate our own, so do all that @@ -226,25 +220,17 @@ pub fn lazy_per_thread_init() -> Result<(), Trap> { let r = libc::sigaltstack(&new_stack, ptr::null_mut()); assert_eq!(r, 0, "registering new sigaltstack failed"); - *slot = Tls::Allocated { + Ok(Some(Stack { mmap_ptr: ptr, mmap_size: alloc_size, - }; - Ok(()) - }); + })) + } - impl Drop for Tls { + impl Drop for Stack { fn drop(&mut self) { - let (ptr, size) = match self { - Tls::Allocated { - mmap_ptr, - mmap_size, - } => (*mmap_ptr, *mmap_size), - _ => return, - }; unsafe { // Deallocate the stack memory. - let r = libc::munmap(ptr, size); + let r = libc::munmap(self.mmap_ptr, self.mmap_size); debug_assert_eq!(r, 0, "munmap failed during thread shutdown"); } } diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index fb60ded7d0..c3737b3bff 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -159,13 +159,10 @@ impl Store { } fn new_(engine: &Engine, limiter: Option>) -> Self { - // Ensure that wasmtime_runtime's signal handlers are configured. Note - // that at the `Store` level it means we should perform this - // once-per-thread. Platforms like Unix, however, only require this - // once-per-program. In any case this is safe to call many times and - // each one that's not relevant just won't do anything. - wasmtime_runtime::init_traps(crate::module::GlobalModuleRegistry::is_wasm_pc) - .expect("failed to initialize trap handling"); + // Ensure that wasmtime_runtime's signal handlers are configured. This + // is the per-program initialization required for handling traps, such + // as configuring signals, vectored exception handlers, etc. + wasmtime_runtime::init_traps(crate::module::GlobalModuleRegistry::is_wasm_pc); Self { inner: Rc::new(StoreInner { @@ -451,25 +448,6 @@ impl Store { &self.inner.modules } - /// Notifies that the current Store (and all referenced entities) has been moved over to a - /// different thread. - /// - /// See also the multithreading documentation for more details: - /// . - /// - /// # Safety - /// - /// In general, it's not possible to move a `Store` to a different thread, because it isn't `Send`. - /// That being said, it is possible to create an unsafe `Send` wrapper over a `Store`, assuming - /// the safety guidelines exposed in the multithreading documentation have been applied. So it - /// is in general unnecessary to do this, if you're not doing unsafe things. - /// - /// It is fine to call this several times: only the first call will have an effect. - pub unsafe fn notify_switched_thread(&self) { - wasmtime_runtime::init_traps(crate::module::GlobalModuleRegistry::is_wasm_pc) - .expect("failed to initialize per-threads traps"); - } - #[inline] pub(crate) fn module_info_lookup(&self) -> &dyn wasmtime_runtime::ModuleInfoLookup { self.inner.as_ref() @@ -673,7 +651,8 @@ impl Store { } unsafe { - let before = wasmtime_runtime::TlsRestore::take(); + let before = wasmtime_runtime::TlsRestore::take() + .map_err(|e| Trap::from_runtime(self, e))?; let res = (*suspend).suspend(()); before.replace().map_err(|e| Trap::from_runtime(self, e))?; res?; diff --git a/docs/examples-rust-multithreading.md b/docs/examples-rust-multithreading.md index d99977759f..5d3fdcac2c 100644 --- a/docs/examples-rust-multithreading.md +++ b/docs/examples-rust-multithreading.md @@ -129,16 +129,11 @@ some possibilities include: `Store::set` or `Func::wrap`) implement the `Send` trait. If these requirements are met it is technically safe to move a store and its - objects between threads. When you move a store to another thread, it is - required that you run the `Store::notify_switched_thread()` method after the - store has landed on the new thread, so that per-thread initialization is - correctly re-run. Failure to do so may cause wasm traps to crash the whole - application. - - The reason that this strategy isn't recommended, however, is that you will - receive no assistance from the Rust compiler in verifying that the transfer - across threads is indeed actually safe. This will require auditing your - embedding of Wasmtime itself to ensure it meets these requirements. + objects between threads. The reason that this strategy isn't recommended, + however, is that you will receive no assistance from the Rust compiler in + verifying that the transfer across threads is indeed actually safe. This will + require auditing your embedding of Wasmtime itself to ensure it meets these + requirements. It's important to note that the requirements here also apply to the futures returned from `Func::call_async`. These futures are not `Send` due to them diff --git a/tests/all/traps.rs b/tests/all/traps.rs index e409b98189..af702c247b 100644 --- a/tests/all/traps.rs +++ b/tests/all/traps.rs @@ -616,9 +616,6 @@ fn multithreaded_traps() -> Result<()> { let handle = std::thread::spawn(move || { let instance = instance.inner; - unsafe { - instance.store().notify_switched_thread(); - } assert!(instance .get_typed_func::<(), ()>("run") .unwrap() From e4f9eebe3a95275f934fb009008e7205beabd1e9 Mon Sep 17 00:00:00 2001 From: Alan Egerton Date: Thu, 29 Apr 2021 09:28:29 +0100 Subject: [PATCH 32/43] Expose new `JITModule::read_got_entry` function --- cranelift/jit/src/backend.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index 26768ffc74..4389455d8d 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -285,6 +285,13 @@ impl JITModule { } } + /// Returns the given function's entry in the Global Offset Table. + /// + /// Panics if there's no entry in the table for the given function. + pub fn read_got_entry(&self, func_id: FuncId) -> *const u8 { + unsafe { *self.function_got_entries[func_id].unwrap().as_ptr() } + } + fn get_got_address(&self, name: &ir::ExternalName) -> *const u8 { match *name { ir::ExternalName::User { .. } => { From 92e0b6b9e8231bedfac0687da84b9d5683b29339 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Thu, 29 Apr 2021 13:03:28 -0700 Subject: [PATCH 33/43] wasi-nn: turn it on by default (#2859) * wasi-nn: turn it on by default This change makes the wasi-nn Cargo feature a default feature. Previously, a wasi-nn user would have to build a separate Wasmtime binary (e.g. `cargo build --features wasi-nn ...`) to use wasi-nn and the resulting binary would require OpenVINO shared libraries to be present in the environment in order to run (otherwise it would fail immediately with linking errors). With recent changes to the `openvino` crate, the wasi-nn implementation can defer the loading of the OpenVINO shared libraries until runtime (i.e., when the user Wasm program calls `wasi_ephemeral_nn::load`) and display a user-level error if anything goes wrong (e.g., the OpenVINO libraries are not present on the system). This runtime-linking addition allows the wasi-nn feature to be turned on by default and shipped with upcoming releases of Wasmtime. This change should be transparent for users who do not use wasi-nn: the `openvino` crate is small and the newly-available wasi-nn imports only affect programs in which they are used. For those interested in reviewing the runtime linking approach added to the `openvino` crate, see https://github.com/intel/openvino-rs/pull/19. * wasi-nn spec path: don't use canonicalize * Allow dependencies using the ISC license The ISC license should be [just as permissive](https://choosealicense.com/licenses/isc) as MIT, e.g., with no additional limitations. * Add a `--wasi-modules` flag This flag controls which WASI modules are made available to the Wasm program. This initial commit enables `wasi-common` by default (equivalent to `--wasi-modules=all`) and allows `wasi-nn` and `wasi-crypto` to be added in either individually (e.g., `--wasi-modules=wasi-nn`) or as a group (e.g., `--wasi-modules=all-experimental`). * wasi-crypto: fix unused dependency Co-authored-by: Pat Hickey --- Cargo.lock | 55 ++---- Cargo.toml | 2 +- ci/run-wasi-crypto-example.sh | 2 +- ci/run-wasi-nn-example.sh | 2 +- .../src/wiggle_interfaces/error.rs | 2 +- crates/wasi-nn/Cargo.toml | 2 +- crates/wasi-nn/build.rs | 6 +- crates/wasi-nn/src/ctx.rs | 12 +- crates/wasi-nn/src/impl.rs | 20 ++- crates/wasi-nn/src/witx.rs | 3 +- deny.toml | 1 + src/commands/compile.rs | 2 +- src/commands/run.rs | 62 ++++--- src/commands/wasm2obj.rs | 2 +- src/commands/wast.rs | 2 +- src/lib.rs | 164 +++++++++++++++++- 16 files changed, 257 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37c27174dd..05b1a92bd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,30 +189,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.55.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b13ce559e6433d360c26305643803cb52cfbabbc2b9c47ce04a58493dfb443" -dependencies = [ - "bitflags", - "cexpr", - "cfg-if 0.1.10", - "clang-sys", - "clap", - "env_logger 0.7.1", - "lazy_static", - "lazycell", - "log", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "which", -] - [[package]] name = "bindgen" version = "0.57.0" @@ -1730,22 +1706,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openvino" -version = "0.1.8" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43eeb44285b7ce8e2012b92bec32968622e1dad452e812e6edea9e001e5e9410" +checksum = "0cb74b3d8c653f7a9928bda494d329e6363ea0b428d3a3e5805b45ebb74ace76" dependencies = [ "openvino-sys", "thiserror", ] [[package]] -name = "openvino-sys" -version = "0.1.8" +name = "openvino-finder" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb64bef270a1ff665b0b2e28ebfa213e6205a007ce88223d020730225d6008f" +checksum = "426587a131841eb1e1111b0fea96cbd4fd0fd5d7b6526fb9c41400587d1c525c" + +[[package]] +name = "openvino-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d5e5d5e913f4e9aa42b2a7ae9c8719aedb4bc0eb443bf92f07d9ee9a05e7b1" dependencies = [ - "bindgen 0.55.1", "cmake", + "lazy_static", + "libloading", + "openvino-finder", ] [[package]] @@ -2984,7 +2968,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ada4f4ae167325015f52cc65f9fb6c251b868d8fb3b6dd0ce2d60e497c4870a" dependencies = [ - "bindgen 0.57.0", + "bindgen", "cc", "cfg-if 0.1.10", ] @@ -3604,15 +3588,6 @@ dependencies = [ "wast 35.0.2", ] -[[package]] -name = "which" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" -dependencies = [ - "libc", -] - [[package]] name = "wiggle" version = "0.26.0" diff --git a/Cargo.toml b/Cargo.toml index ffff6d865e..e23bddb5e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,7 +84,7 @@ members = [ ] [features] -default = ["jitdump", "wasmtime/wat", "wasmtime/parallel-compilation"] +default = ["jitdump", "wasmtime/wat", "wasmtime/parallel-compilation", "wasi-nn"] lightbeam = ["wasmtime/lightbeam"] jitdump = ["wasmtime/jitdump"] vtune = ["wasmtime/vtune"] diff --git a/ci/run-wasi-crypto-example.sh b/ci/run-wasi-crypto-example.sh index d2582c71b1..b027e48340 100755 --- a/ci/run-wasi-crypto-example.sh +++ b/ci/run-wasi-crypto-example.sh @@ -7,4 +7,4 @@ pushd "$RUST_BINDINGS" cargo build --release --target=wasm32-wasi popd -cargo run --features wasi-crypto -- run "$RUST_BINDINGS/target/wasm32-wasi/release/wasi-crypto-guest.wasm" +cargo run --features wasi-crypto -- run "$RUST_BINDINGS/target/wasm32-wasi/release/wasi-crypto-guest.wasm" --wasi-modules=experimental-wasi-crypto diff --git a/ci/run-wasi-nn-example.sh b/ci/run-wasi-nn-example.sh index 49c472c95e..97b55362c6 100755 --- a/ci/run-wasi-nn-example.sh +++ b/ci/run-wasi-nn-example.sh @@ -37,7 +37,7 @@ cp target/wasm32-wasi/release/wasi-nn-example.wasm $TMP_DIR popd # Run the example in Wasmtime (note that the example uses `fixture` as the expected location of the model/tensor files). -OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo run --features wasi-nn -- run --mapdir fixture::$TMP_DIR $TMP_DIR/wasi-nn-example.wasm +cargo run -- run --mapdir fixture::$TMP_DIR $TMP_DIR/wasi-nn-example.wasm --wasi-modules=experimental-wasi-nn # Clean up the temporary directory only if it was not specified (users may want to keep the directory around). if [[ $REMOVE_TMP_DIR -eq 1 ]]; then diff --git a/crates/wasi-crypto/src/wiggle_interfaces/error.rs b/crates/wasi-crypto/src/wiggle_interfaces/error.rs index fd13a87331..e54a104b03 100644 --- a/crates/wasi-crypto/src/wiggle_interfaces/error.rs +++ b/crates/wasi-crypto/src/wiggle_interfaces/error.rs @@ -1,4 +1,4 @@ -use super::{guest_types, WasiCryptoCtx}; +use super::guest_types; use std::num::TryFromIntError; use wasi_crypto::CryptoError; diff --git a/crates/wasi-nn/Cargo.toml b/crates/wasi-nn/Cargo.toml index 1ce73aeed7..a5de00ed3a 100644 --- a/crates/wasi-nn/Cargo.toml +++ b/crates/wasi-nn/Cargo.toml @@ -22,7 +22,7 @@ wasmtime-wasi = { path = "../wasi", version = "0.26.0" } wiggle = { path = "../wiggle", version = "0.26.0" } # These dependencies are necessary for the wasi-nn implementation: -openvino = "0.1.5" +openvino = { version = "0.3.1", features = ["runtime-linking"] } thiserror = "1.0" [build-dependencies] diff --git a/crates/wasi-nn/build.rs b/crates/wasi-nn/build.rs index 189b9513a6..535d0e0f80 100644 --- a/crates/wasi-nn/build.rs +++ b/crates/wasi-nn/build.rs @@ -1,11 +1,9 @@ //! This build script: //! - has the configuration necessary for the wiggle and witx macros. - -use std::path::PathBuf; - fn main() { // This is necessary for Wiggle/Witx macros. - let wasi_root = PathBuf::from("./spec").canonicalize().unwrap(); + let cwd = std::env::current_dir().unwrap(); + let wasi_root = cwd.join("spec"); println!("cargo:rustc-env=WASI_ROOT={}", wasi_root.display()); // Also automatically rebuild if the Witx files change diff --git a/crates/wasi-nn/src/ctx.rs b/crates/wasi-nn/src/ctx.rs index f180f113fb..dbe93c7224 100644 --- a/crates/wasi-nn/src/ctx.rs +++ b/crates/wasi-nn/src/ctx.rs @@ -2,7 +2,7 @@ //! wasi-nn API. use crate::r#impl::UsageError; use crate::witx::types::{Graph, GraphExecutionContext}; -use openvino::InferenceError; +use openvino::{InferenceError, SetupError}; use std::cell::RefCell; use std::collections::HashMap; use std::hash::Hash; @@ -14,8 +14,10 @@ use wiggle::GuestError; pub enum WasiNnError { #[error("guest error")] GuestError(#[from] GuestError), - #[error("openvino error")] - OpenvinoError(#[from] InferenceError), + #[error("openvino inference error")] + OpenvinoInferenceError(#[from] InferenceError), + #[error("openvino setup error")] + OpenvinoSetupError(#[from] SetupError), #[error("usage error")] UsageError(#[from] UsageError), } @@ -74,7 +76,7 @@ impl ExecutionContext { /// Capture the state necessary for calling into `openvino`. pub struct Ctx { - pub(crate) core: openvino::Core, + pub(crate) core: Option, pub(crate) graphs: Table, pub(crate) executions: Table, } @@ -83,7 +85,7 @@ impl Ctx { /// Make a new `WasiNnCtx` with the default settings. pub fn new() -> WasiNnResult { Ok(Self { - core: openvino::Core::new(None)?, + core: Option::default(), graphs: Table::default(), executions: Table::default(), }) diff --git a/crates/wasi-nn/src/impl.rs b/crates/wasi-nn/src/impl.rs index e18caafdc2..47fd323fdb 100644 --- a/crates/wasi-nn/src/impl.rs +++ b/crates/wasi-nn/src/impl.rs @@ -12,6 +12,8 @@ use wiggle::GuestPtr; #[derive(Debug, Error)] pub enum UsageError { + #[error("Invalid context; has the load function been called?")] + InvalidContext, #[error("Only OpenVINO's IR is currently supported, passed encoding: {0:?}")] InvalidEncoding(GraphEncoding), #[error("OpenVINO expects only two buffers (i.e. [ir, weights]), passed: {0}")] @@ -34,9 +36,21 @@ impl<'a> WasiEphemeralNn for WasiNnCtx { if encoding != GraphEncoding::Openvino { return Err(UsageError::InvalidEncoding(encoding).into()); } + if builders.len() != 2 { return Err(UsageError::InvalidNumberOfBuilders(builders.len()).into()); } + + // Construct the context if none is present; this is done lazily (i.e. upon actually loading + // a model) because it may fail to find and load the OpenVINO libraries. The laziness limits + // the extent of the error only to wasi-nn users, not all WASI users. + if self.ctx.borrow().core.is_none() { + self.ctx + .borrow_mut() + .core + .replace(openvino::Core::new(None)?); + } + let builders = builders.as_ptr(); let xml = builders.read()?.as_slice()?; let weights = builders.add(1)?.read()?.as_slice()?; @@ -44,11 +58,15 @@ impl<'a> WasiEphemeralNn for WasiNnCtx { .ctx .borrow_mut() .core + .as_mut() + .ok_or(UsageError::InvalidContext)? .read_network_from_buffer(&xml, &weights)?; let executable_graph = self .ctx .borrow_mut() .core + .as_mut() + .ok_or(UsageError::InvalidContext)? .load_network(&graph, map_execution_target_to_string(target))?; let id = self .ctx @@ -94,7 +112,7 @@ impl<'a> WasiEphemeralNn for WasiNnCtx { .dimensions .as_slice()? .iter() - .map(|d| *d as u64) + .map(|d| *d as usize) .collect::>(); let precision = match tensor.type_ { TensorType::F16 => Precision::FP16, diff --git a/crates/wasi-nn/src/witx.rs b/crates/wasi-nn/src/witx.rs index 32cc0167d8..8244f99644 100644 --- a/crates/wasi-nn/src/witx.rs +++ b/crates/wasi-nn/src/witx.rs @@ -14,7 +14,8 @@ impl<'a> types::UserErrorConversion for WasiNnCtx { fn nn_errno_from_wasi_nn_error(&self, e: WasiNnError) -> Result { eprintln!("Host error: {:?}", e); match e { - WasiNnError::OpenvinoError(_) => unimplemented!(), + WasiNnError::OpenvinoSetupError(_) => unimplemented!(), + WasiNnError::OpenvinoInferenceError(_) => unimplemented!(), WasiNnError::GuestError(_) => unimplemented!(), WasiNnError::UsageError(_) => unimplemented!(), } diff --git a/deny.toml b/deny.toml index d9baefa136..a63b6864e4 100644 --- a/deny.toml +++ b/deny.toml @@ -23,6 +23,7 @@ allow = [ "Apache-2.0", "BSD-2-Clause", "CC0-1.0", + "ISC", "MIT", "MPL-2.0", "Zlib", diff --git a/src/commands/compile.rs b/src/commands/compile.rs index 1ad611623c..9d601ab7a0 100644 --- a/src/commands/compile.rs +++ b/src/commands/compile.rs @@ -28,7 +28,7 @@ lazy_static::lazy_static! { Compiling for a specific platform (Linux) and CPU preset (Skylake):\n\ \n \ wasmtime compile --target x86_64-unknown-linux --cranelift-enable skylake foo.wasm\n", - crate::WASM_FEATURES.as_str() + crate::FLAG_EXPLANATIONS.as_str() ) }; } diff --git a/src/commands/run.rs b/src/commands/run.rs index 9c57add91f..4d6bce5b0b 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -1,6 +1,6 @@ //! The module that implements the `wasmtime run` command. -use crate::CommonOptions; +use crate::{CommonOptions, WasiModules}; use anyhow::{bail, Context as _, Result}; use std::thread; use std::time::Duration; @@ -70,7 +70,7 @@ fn parse_preloads(s: &str) -> Result<(String, PathBuf)> { lazy_static::lazy_static! { static ref AFTER_HELP: String = { - crate::WASM_FEATURES.to_string() + crate::FLAG_EXPLANATIONS.to_string() }; } @@ -146,7 +146,13 @@ impl RunCommand { let argv = self.compute_argv(); let mut linker = Linker::new(&store); - populate_with_wasi(&mut linker, preopen_dirs, &argv, &self.vars)?; + populate_with_wasi( + &mut linker, + preopen_dirs, + &argv, + &self.vars, + &self.common.wasi_modules.unwrap_or(WasiModules::default()), + )?; // Load the preload wasm modules. for (name, path) in self.preloads.iter() { @@ -351,6 +357,7 @@ fn populate_with_wasi( preopen_dirs: Vec<(String, Dir)>, argv: &[String], vars: &[(String, String)], + wasi_modules: &WasiModules, ) -> Result<()> { // Add the current snapshot to the linker. let mut builder = WasiCtxBuilder::new(); @@ -360,25 +367,40 @@ fn populate_with_wasi( builder = builder.preopened_dir(dir, name)?; } - Wasi::new(linker.store(), builder.build()?).add_to_linker(linker)?; - - #[cfg(feature = "wasi-nn")] - { - use std::cell::RefCell; - use std::rc::Rc; - let wasi_nn = WasiNn::new(linker.store(), Rc::new(RefCell::new(WasiNnCtx::new()?))); - wasi_nn.add_to_linker(linker)?; + if wasi_modules.wasi_common { + Wasi::new(linker.store(), builder.build()?).add_to_linker(linker)?; } - #[cfg(feature = "wasi-crypto")] - { - use std::cell::RefCell; - use std::rc::Rc; - let cx_crypto = Rc::new(RefCell::new(WasiCryptoCtx::new())); - WasiCryptoCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; - WasiCryptoAsymmetricCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; - WasiCryptoSignatures::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; - WasiCryptoSymmetric::new(linker.store(), cx_crypto).add_to_linker(linker)?; + if wasi_modules.wasi_nn { + #[cfg(not(feature = "wasi-nn"))] + { + bail!("Cannot enable wasi-nn when the binary is not compiled with this feature."); + } + #[cfg(feature = "wasi-nn")] + { + use std::cell::RefCell; + use std::rc::Rc; + let wasi_nn = WasiNn::new(linker.store(), Rc::new(RefCell::new(WasiNnCtx::new()?))); + wasi_nn.add_to_linker(linker)?; + } + } + + if wasi_modules.wasi_crypto { + #[cfg(not(feature = "wasi-crypto"))] + { + bail!("Cannot enable wasi-crypto when the binary is not compiled with this feature."); + } + #[cfg(feature = "wasi-crypto")] + { + use std::cell::RefCell; + use std::rc::Rc; + let cx_crypto = Rc::new(RefCell::new(WasiCryptoCtx::new())); + WasiCryptoCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; + WasiCryptoAsymmetricCommon::new(linker.store(), cx_crypto.clone()) + .add_to_linker(linker)?; + WasiCryptoSignatures::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; + WasiCryptoSymmetric::new(linker.store(), cx_crypto).add_to_linker(linker)?; + } } Ok(()) diff --git a/src/commands/wasm2obj.rs b/src/commands/wasm2obj.rs index ab914d8edc..b6426ed13a 100644 --- a/src/commands/wasm2obj.rs +++ b/src/commands/wasm2obj.rs @@ -18,7 +18,7 @@ lazy_static::lazy_static! { The default is a dummy environment that produces placeholder values.\n\ \n\ {}", - crate::WASM_FEATURES.as_str() + crate::FLAG_EXPLANATIONS.as_str() ) }; } diff --git a/src/commands/wast.rs b/src/commands/wast.rs index cd749e61a6..08ad8e0a2a 100644 --- a/src/commands/wast.rs +++ b/src/commands/wast.rs @@ -9,7 +9,7 @@ use wasmtime_wast::WastContext; lazy_static::lazy_static! { static ref AFTER_HELP: String = { - crate::WASM_FEATURES.to_string() + crate::FLAG_EXPLANATIONS.to_string() }; } diff --git a/src/lib.rs b/src/lib.rs index 3485b62889..facb0383c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,19 +43,47 @@ const SUPPORTED_WASM_FEATURES: &[(&str, &str)] = &[ ("threads", "enables support for WebAssembly threads"), ]; +const SUPPORTED_WASI_MODULES: &[(&str, &str)] = &[ + ( + "default", + "enables all stable WASI modules (no experimental modules)", + ), + ( + "wasi-common", + "enables support for the WASI common APIs, see https://github.com/WebAssembly/WASI", + ), + ( + "experimental-wasi-nn", + "enables support for the WASI neural network API (experimental), see https://github.com/WebAssembly/wasi-nn", + ), + ( + "experimental-wasi-crypto", + "enables support for the WASI cryptography APIs (experimental), see https://github.com/WebAssembly/wasi-crypto", + ), +]; + lazy_static::lazy_static! { - static ref WASM_FEATURES: String = { + static ref FLAG_EXPLANATIONS: String = { use std::fmt::Write; let mut s = String::new(); + + // Explain --wasm-features. writeln!(&mut s, "Supported values for `--wasm-features`:").unwrap(); writeln!(&mut s).unwrap(); - let max = SUPPORTED_WASM_FEATURES.iter().max_by_key(|(name, _)| name.len()).unwrap(); - for (name, desc) in SUPPORTED_WASM_FEATURES.iter() { writeln!(&mut s, "{:width$} {}", name, desc, width = max.0.len() + 2).unwrap(); } + writeln!(&mut s).unwrap(); + + // Explain --wasi-modules. + writeln!(&mut s, "Supported values for `--wasi-modules`:").unwrap(); + writeln!(&mut s).unwrap(); + let max = SUPPORTED_WASI_MODULES.iter().max_by_key(|(name, _)| name.len()).unwrap(); + for (name, desc) in SUPPORTED_WASI_MODULES.iter() { + writeln!(&mut s, "{:width$} {}", name, desc, width = max.0.len() + 2).unwrap(); + } writeln!(&mut s).unwrap(); writeln!(&mut s, "Features prefixed with '-' will be disabled.").unwrap(); @@ -186,6 +214,10 @@ struct CommonOptions { #[structopt(long, value_name = "FEATURE,FEATURE,...", parse(try_from_str = parse_wasm_features))] wasm_features: Option, + /// Enables or disables WASI modules + #[structopt(long, value_name = "MODULE,MODULE,...", parse(try_from_str = parse_wasi_modules))] + wasi_modules: Option, + /// Use Lightbeam for all compilation #[structopt(long, conflicts_with = "cranelift")] lightbeam: bool, @@ -408,6 +440,75 @@ fn parse_wasm_features(features: &str) -> Result { memory64: false, }) } + +fn parse_wasi_modules(modules: &str) -> Result { + let modules = modules.trim(); + match modules { + "default" => Ok(WasiModules::default()), + "-default" => Ok(WasiModules::none()), + _ => { + // Starting from the default set of WASI modules, enable or disable a list of + // comma-separated modules. + let mut wasi_modules = WasiModules::default(); + let mut set = |module: &str, enable: bool| match module { + "" => Ok(()), + "wasi-common" => Ok(wasi_modules.wasi_common = enable), + "experimental-wasi-nn" => Ok(wasi_modules.wasi_nn = enable), + "experimental-wasi-crypto" => Ok(wasi_modules.wasi_crypto = enable), + "default" => bail!("'default' cannot be specified with other WASI modules"), + _ => bail!("unsupported WASI module '{}'", module), + }; + + for module in modules.split(',') { + let module = module.trim(); + let (module, value) = if module.starts_with('-') { + (&module[1..], false) + } else { + (module, true) + }; + set(module, value)?; + } + + Ok(wasi_modules) + } + } +} + +/// Select which WASI modules are available at runtime for use by Wasm programs. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct WasiModules { + /// Enable the wasi-common implementation; eventually this should be split into its separate + /// parts once the implementation allows for it (e.g. wasi-fs, wasi-clocks, etc.). + pub wasi_common: bool, + + /// Enable the experimental wasi-nn implementation. + pub wasi_nn: bool, + + /// Enable the experimental wasi-crypto implementation. + pub wasi_crypto: bool, +} + +impl Default for WasiModules { + fn default() -> Self { + Self { + wasi_common: true, + wasi_nn: false, + wasi_crypto: false, + } + } +} + +impl WasiModules { + /// Enable no modules. + pub fn none() -> Self { + Self { + wasi_common: false, + wasi_nn: false, + wasi_crypto: false, + } + } +} + fn parse_cranelift_flag(name_and_value: &str) -> Result<(String, String)> { let mut split = name_and_value.splitn(2, '='); let name = if let Some(name) = split.next() { @@ -574,4 +675,61 @@ mod test { feature_test!(test_simd_feature, simd, "simd"); feature_test!(test_threads_feature, threads, "threads"); feature_test!(test_multi_memory_feature, multi_memory, "multi-memory"); + + #[test] + fn test_default_modules() { + let options = CommonOptions::from_iter_safe(vec!["foo", "--wasi-modules=default"]).unwrap(); + assert_eq!( + options.wasi_modules.unwrap(), + WasiModules { + wasi_common: true, + wasi_nn: false, + wasi_crypto: false + } + ); + } + + #[test] + fn test_empty_modules() { + let options = CommonOptions::from_iter_safe(vec!["foo", "--wasi-modules="]).unwrap(); + assert_eq!( + options.wasi_modules.unwrap(), + WasiModules { + wasi_common: true, + wasi_nn: false, + wasi_crypto: false + } + ); + } + + #[test] + fn test_some_modules() { + let options = CommonOptions::from_iter_safe(vec![ + "foo", + "--wasi-modules=experimental-wasi-nn,-wasi-common", + ]) + .unwrap(); + assert_eq!( + options.wasi_modules.unwrap(), + WasiModules { + wasi_common: false, + wasi_nn: true, + wasi_crypto: false + } + ); + } + + #[test] + fn test_no_modules() { + let options = + CommonOptions::from_iter_safe(vec!["foo", "--wasi-modules=-default"]).unwrap(); + assert_eq!( + options.wasi_modules.unwrap(), + WasiModules { + wasi_common: false, + wasi_nn: false, + wasi_crypto: false + } + ); + } } From 7a3791f9e9905387c747476334253732fa3e9199 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Mon, 3 May 2021 18:01:10 +0200 Subject: [PATCH 34/43] Fiber support for s390x (#2870) Add fiber support for the s390x architecture. --- crates/fiber/src/arch/s390x.S | 112 ++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 crates/fiber/src/arch/s390x.S diff --git a/crates/fiber/src/arch/s390x.S b/crates/fiber/src/arch/s390x.S new file mode 100644 index 0000000000..8d9548bbb0 --- /dev/null +++ b/crates/fiber/src/arch/s390x.S @@ -0,0 +1,112 @@ +// A WORD OF CAUTION +// +// This entire file basically needs to be kept in sync with itself. It's not +// really possible to modify just one bit of this file without understanding +// all the other bits. Documentation tries to reference various bits here and +// there but try to make sure to read over everything before tweaking things! +// +// Also at this time this file is heavily based off the x86_64 file, so you'll +// probably want to read that one as well. + +#include "header.h" + +// fn(top_of_stack(%x0): *mut u8) +HIDDEN(wasmtime_fiber_switch) +GLOBL(wasmtime_fiber_switch) +.p2align 2 +TYPE(wasmtime_fiber_switch) +FUNCTION(wasmtime_fiber_switch): + // Save all callee-saved registers on the stack since we're assuming + // they're clobbered as a result of the stack switch. + stmg %r6, %r15, 48(%r15) + aghi %r15, -64 + std %f8, 0(%r15) + std %f9, 8(%r15) + std %f10, 16(%r15) + std %f11, 24(%r15) + std %f12, 32(%r15) + std %f13, 40(%r15) + std %f14, 48(%r15) + std %f15, 56(%r15) + + // Load our previously saved stack pointer to resume to, and save off our + // current stack pointer on where to come back to eventually. + lg %r1, -16(%r2) + stg %r15, -16(%r2) + + // Switch to the new stack and restore all our callee-saved registers after + // the switch and return to our new stack. + ld %f8, 0(%r1) + ld %f9, 8(%r1) + ld %f10, 16(%r1) + ld %f11, 24(%r1) + ld %f12, 32(%r1) + ld %f13, 40(%r1) + ld %f14, 48(%r1) + ld %f15, 56(%r1) + lmg %r6, %r15, 112(%r1) + br %r14 +SIZE(wasmtime_fiber_switch) + +// fn( +// top_of_stack(%x0): *mut u8, +// entry_point(%x1): extern fn(*mut u8, *mut u8), +// entry_arg0(%x2): *mut u8, +// ) +HIDDEN(wasmtime_fiber_init) +GLOBL(wasmtime_fiber_init) +.p2align 2 +TYPE(wasmtime_fiber_init) +FUNCTION(wasmtime_fiber_init): + larl %r1, FUNCTION(wasmtime_fiber_start) + stg %r1, -48(%r2) // wasmtime_fiber_start - restored into %r14 + stg %r2, -112(%r2) // top_of_stack - restored into %r6 + stg %r3, -104(%r2) // entry_point - restored into %r7 + stg %r4, -96(%r2) // entry_arg0 - restored into %r8 + aghi %r2, -160 // 160 bytes register save area + stg %r2, 120(%r2) // bottom of register save area - restored into %r15 + + // `wasmtime_fiber_switch` has a 64 byte stack. + aghi %r2, -64 + stg %r2, 208(%r2) + br %r14 +SIZE(wasmtime_fiber_init) + +.p2align 2 +TYPE(wasmtime_fiber_start) +FUNCTION(wasmtime_fiber_start): +.cfi_startproc simple + + // See the x86_64 file for more commentary on what these CFI directives are + // doing. Like over there note that the relative offsets to registers here + // match the frame layout in `wasmtime_fiber_switch`. + .cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \ + 7, /* the byte length of this expression */ \ + 0x7f, 0x90, 0x1, /* DW_OP_breg15 0x90 */ \ + 0x06, /* DW_OP_deref */ \ + 0x23, 0xe0, 0x1 /* DW_OP_plus_uconst 0xe0 */ + + .cfi_rel_offset 6, -112 + .cfi_rel_offset 7, -104 + .cfi_rel_offset 8, -96 + .cfi_rel_offset 9, -88 + .cfi_rel_offset 10, -80 + .cfi_rel_offset 11, -72 + .cfi_rel_offset 12, -64 + .cfi_rel_offset 13, -56 + .cfi_rel_offset 14, -48 + .cfi_rel_offset 15, -40 + + // Load our two arguments prepared by `wasmtime_fiber_init`. + lgr %r2, %r8 // entry_arg0 + lgr %r3, %r6 // top_of_stack + + // ... and then we call the function! Note that this is a function call so + // our frame stays on the stack to backtrace through. + basr %r14, %r7 // entry_point + // .. technically we shouldn't get here, so just trap. + .word 0x0000 + .cfi_endproc +SIZE(wasmtime_fiber_start) + +FOOTER From e1cc1a67d592f64d7ac5ab0e4fb0517c848760fd Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Mon, 3 May 2021 18:50:00 +0200 Subject: [PATCH 35/43] Object file support for s390x (#2872) Add support for s390x binary format object files. In particular, add support for s390x ELF relocation types (currently only S390xPCRel32Dbl). --- cranelift/codegen/src/binemit/mod.rs | 3 +++ cranelift/jit/src/compiled_blob.rs | 9 +++++++++ cranelift/src/disasm.rs | 5 +++++ crates/debug/src/lib.rs | 1 + crates/jit/src/link.rs | 13 +++++++++++++ crates/obj/src/builder.rs | 2 ++ crates/profiling/src/jitdump_linux.rs | 1 + 7 files changed, 34 insertions(+) diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index b534ec9765..aa3102797e 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -60,6 +60,8 @@ pub enum Reloc { Arm64Call, /// RISC-V call target RiscvCall, + /// s390x PC-relative 4-byte offset + S390xPCRel32Dbl, /// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol. ElfX86_64TlsGd, @@ -75,6 +77,7 @@ impl fmt::Display for Reloc { match *self { Self::Abs4 => write!(f, "Abs4"), Self::Abs8 => write!(f, "Abs8"), + Self::S390xPCRel32Dbl => write!(f, "PCRel32Dbl"), Self::X86PCRel4 => write!(f, "PCRel4"), Self::X86PCRelRodata4 => write!(f, "PCRelRodata4"), Self::X86CallPCRel4 => write!(f, "CallPCRel4"), diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs index d44497ae9e..f00165dbab 100644 --- a/cranelift/jit/src/compiled_blob.rs +++ b/cranelift/jit/src/compiled_blob.rs @@ -72,6 +72,15 @@ impl CompiledBlob { write_unaligned(at as *mut i32, pcrel) }; } + Reloc::S390xPCRel32Dbl => { + let base = get_address(name); + let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; + let pcrel = i32::try_from(((what as isize) - (at as isize)) >> 1).unwrap(); + #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] + unsafe { + write_unaligned(at as *mut i32, pcrel) + }; + } _ => unimplemented!(), } } diff --git a/cranelift/src/disasm.rs b/cranelift/src/disasm.rs index a98e867380..35a581d344 100644 --- a/cranelift/src/disasm.rs +++ b/cranelift/src/disasm.rs @@ -153,6 +153,11 @@ cfg_if! { cs.set_skipdata(true).map_err(map_caperr)?; cs } + Architecture::S390x {..} => Capstone::new() + .sysz() + .mode(arch::sysz::ArchMode::Default) + .build() + .map_err(map_caperr)?, _ => anyhow::bail!("Unknown ISA"), }; diff --git a/crates/debug/src/lib.rs b/crates/debug/src/lib.rs index b438051b45..b6d6830d51 100644 --- a/crates/debug/src/lib.rs +++ b/crates/debug/src/lib.rs @@ -120,6 +120,7 @@ fn ensure_supported_elf_format(bytes: &mut Vec) -> Result match header.e_machine.get(e) { EM_X86_64 => (), + EM_S390 => (), machine => { bail!("Unsupported ELF target machine: {:x}", machine); } diff --git a/crates/jit/src/link.rs b/crates/jit/src/link.rs index 9c58a138a5..fffd9ecca8 100644 --- a/crates/jit/src/link.rs +++ b/crates/jit/src/link.rs @@ -111,6 +111,19 @@ fn apply_reloc( ); write_unaligned(reloc_address as *mut u32, reloc_delta_u64 as u32); }, + #[cfg(target_pointer_width = "64")] + (RelocationKind::Relative, RelocationEncoding::S390xDbl, 32) => unsafe { + let reloc_address = body.add(offset as usize) as usize; + let reloc_addend = r.addend() as isize; + let reloc_delta_u64 = (target_func_address as u64) + .wrapping_sub(reloc_address as u64) + .wrapping_add(reloc_addend as u64); + assert!( + (reloc_delta_u64 as isize) >> 1 <= i32::max_value() as isize, + "relocation too large to fit in i32" + ); + write_unaligned(reloc_address as *mut u32, (reloc_delta_u64 >> 1) as u32); + }, (RelocationKind::Elf(elf::R_AARCH64_CALL26), RelocationEncoding::Generic, 32) => unsafe { let reloc_address = body.add(offset as usize) as usize; let reloc_addend = r.addend() as isize; diff --git a/crates/obj/src/builder.rs b/crates/obj/src/builder.rs index 805adbb806..f606d72078 100644 --- a/crates/obj/src/builder.rs +++ b/crates/obj/src/builder.rs @@ -80,6 +80,7 @@ fn to_object_relocations<'a>( RelocationEncoding::Generic, 32, ), + Reloc::S390xPCRel32Dbl => (RelocationKind::Relative, RelocationEncoding::S390xDbl, 32), other => unimplemented!("Unimplemented relocation {:?}", other), }; Some(ObjectRelocation { @@ -102,6 +103,7 @@ fn to_object_architecture( X86_64 => Architecture::X86_64, Arm(_) => Architecture::Arm, Aarch64(_) => Architecture::Aarch64, + S390x => Architecture::S390x, architecture => { anyhow::bail!("target architecture {:?} is unsupported", architecture,); } diff --git a/crates/profiling/src/jitdump_linux.rs b/crates/profiling/src/jitdump_linux.rs index 0f3a4bb00d..b43e8155af 100644 --- a/crates/profiling/src/jitdump_linux.rs +++ b/crates/profiling/src/jitdump_linux.rs @@ -241,6 +241,7 @@ impl State { Architecture::X86_32(_) => elf::EM_386 as u32, Architecture::Arm(_) => elf::EM_ARM as u32, Architecture::Aarch64(_) => elf::EM_AARCH64 as u32, + Architecture::S390x => elf::EM_S390 as u32, _ => unimplemented!("unrecognized architecture"), } } From dfb1bc4d02a653ac9bdfd7ba69e0b20ff3796149 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Mon, 3 May 2021 18:56:18 +0200 Subject: [PATCH 36/43] Trap handler changes to support s390x (#2871) On s390x, SIGILL and SIGFPE are delivered with the PSW address pointing *after* the faulting instruction, while SIGSEGV and SIGBUS are delivered with the PSW address pointing *to* the faulting instruction. In order to support this, the common code trap handler has to distinguish between those cases. Also, enable SIGFPE on s390x (just like on x86). --- crates/runtime/src/traphandlers/unix.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/runtime/src/traphandlers/unix.rs b/crates/runtime/src/traphandlers/unix.rs index 955d00a62b..b4046f0944 100644 --- a/crates/runtime/src/traphandlers/unix.rs +++ b/crates/runtime/src/traphandlers/unix.rs @@ -47,8 +47,8 @@ pub unsafe fn platform_init() { // Handle `unreachable` instructions which execute `ud2` right now register(&mut PREV_SIGILL, libc::SIGILL); - // x86 uses SIGFPE to report division by zero - if cfg!(target_arch = "x86") || cfg!(target_arch = "x86_64") { + // x86 and s390x use SIGFPE to report division by zero + if cfg!(target_arch = "x86") || cfg!(target_arch = "x86_64") || cfg!(target_arch = "s390x") { register(&mut PREV_SIGFPE, libc::SIGFPE); } @@ -85,7 +85,7 @@ unsafe extern "C" fn trap_handler( // Otherwise flag ourselves as handling a trap, do the trap // handling, and reset our trap handling flag. Then we figure // out what to do based on the result of the trap handling. - let pc = get_pc(context); + let pc = get_pc(context, signum); let jmp_buf = info.jmp_buf_if_trap(pc, |handler| handler(signum, siginfo, context)); // Figure out what to do based on the result of this handling of @@ -127,7 +127,7 @@ unsafe extern "C" fn trap_handler( } } -unsafe fn get_pc(cx: *mut libc::c_void) -> *const u8 { +unsafe fn get_pc(cx: *mut libc::c_void, _signum: libc::c_int) -> *const u8 { cfg_if::cfg_if! { if #[cfg(all(target_os = "linux", target_arch = "x86_64"))] { let cx = &*(cx as *const libc::ucontext_t); @@ -138,6 +138,23 @@ unsafe fn get_pc(cx: *mut libc::c_void) -> *const u8 { } else if #[cfg(all(any(target_os = "linux", target_os = "android"), target_arch = "aarch64"))] { let cx = &*(cx as *const libc::ucontext_t); cx.uc_mcontext.pc as *const u8 + } else if #[cfg(all(target_os = "linux", target_arch = "s390x"))] { + // On s390x, SIGILL and SIGFPE are delivered with the PSW address + // pointing *after* the faulting instruction, while SIGSEGV and + // SIGBUS are delivered with the PSW address pointing *to* the + // faulting instruction. To handle this, the code generator registers + // any trap that results in one of "late" signals on the last byte + // of the instruction, and any trap that results in one of the "early" + // signals on the first byte of the instruction (as usual). This + // means we simply need to decrement the reported PSW address by + // one in the case of a "late" signal here to ensure we always + // correctly find the associated trap handler. + let trap_offset = match _signum { + libc::SIGILL | libc::SIGFPE => 1, + _ => 0, + }; + let cx = &*(cx as *const libc::ucontext_t); + (cx.uc_mcontext.psw.addr - trap_offset) as *const u8 } else if #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] { let cx = &*(cx as *const libc::ucontext_t); cx.uc_mcontext.mc_rip as *const u8 From 8811246a9fe9fc8f4277e9876aec693baaa2f8d3 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Mon, 3 May 2021 18:58:40 +0200 Subject: [PATCH 37/43] debug: Avoid underflow when scanning for landing pad bytes (#2866) --- crates/debug/src/transform/expression.rs | 28 ++++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/crates/debug/src/transform/expression.rs b/crates/debug/src/transform/expression.rs index 57f3cc7a21..556a460905 100644 --- a/crates/debug/src/transform/expression.rs +++ b/crates/debug/src/transform/expression.rs @@ -512,24 +512,28 @@ where } }; } + // Find all landing pads by scanning bytes, do not care about // false location at this moment. // Looks hacky but it is fast; does not need to be really exact. - for i in 0..buf.len() - 2 { - let op = buf[i]; - if op == gimli::constants::DW_OP_bra.0 || op == gimli::constants::DW_OP_skip.0 { - // TODO fix for big-endian - let offset = i16::from_le_bytes([buf[i + 1], buf[i + 2]]); - let origin = i + 3; - // Discarding out-of-bounds jumps (also some of falsely detected ops) - if (offset >= 0 && offset as usize + origin <= buf.len()) - || (offset < 0 && -offset as usize <= origin) - { - let target = buf.len() as isize - origin as isize - offset as isize; - jump_targets.insert(target as u64, JumpTargetMarker::new()); + if buf.len() > 2 { + for i in 0..buf.len() - 2 { + let op = buf[i]; + if op == gimli::constants::DW_OP_bra.0 || op == gimli::constants::DW_OP_skip.0 { + // TODO fix for big-endian + let offset = i16::from_le_bytes([buf[i + 1], buf[i + 2]]); + let origin = i + 3; + // Discarding out-of-bounds jumps (also some of falsely detected ops) + if (offset >= 0 && offset as usize + origin <= buf.len()) + || (offset < 0 && -offset as usize <= origin) + { + let target = buf.len() as isize - origin as isize - offset as isize; + jump_targets.insert(target as u64, JumpTargetMarker::new()); + } } } } + while !pc.is_empty() { let unread_bytes = pc.len().into_u64(); if let Some(marker) = jump_targets.get(&unread_bytes) { From 147cda3b99a4febcd428ee51c89c57bd735aaaa7 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 16 Mar 2021 11:09:06 +0100 Subject: [PATCH 38/43] Remove thiserror dependency from cranelift_module --- Cargo.lock | 1 - cranelift/module/Cargo.toml | 1 - cranelift/module/src/module.rs | 74 +++++++++++++++++++++++++++++----- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05b1a92bd2..e1f4a79617 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -635,7 +635,6 @@ dependencies = [ "cranelift-entity", "hashbrown", "log", - "thiserror", ] [[package]] diff --git a/cranelift/module/Cargo.toml b/cranelift/module/Cargo.toml index 6650714cf6..a3fe4a0901 100644 --- a/cranelift/module/Cargo.toml +++ b/cranelift/module/Cargo.toml @@ -15,7 +15,6 @@ cranelift-codegen = { path = "../codegen", version = "0.73.0", default-features cranelift-entity = { path = "../entity", version = "0.73.0" } hashbrown = { version = "0.9.1", optional = true } log = { version = "0.4.6", default-features = false } -thiserror = "1.0.4" anyhow = "1.0" [features] diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index 6047dda103..97db899342 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -12,7 +12,6 @@ use cranelift_codegen::entity::{entity_impl, PrimaryMap}; use cranelift_codegen::{ir, isa, CodegenError, Context}; use std::borrow::ToOwned; use std::string::String; -use thiserror::Error; /// A function identifier for use in the `Module` interface. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -168,30 +167,83 @@ impl FunctionDeclaration { } /// Error messages for all `Module` methods -#[derive(Error, Debug)] +#[derive(Debug)] pub enum ModuleError { /// Indicates an identifier was used before it was declared - #[error("Undeclared identifier: {0}")] Undeclared(String), + /// Indicates an identifier was used as data/function first, but then used as the other - #[error("Incompatible declaration of identifier: {0}")] IncompatibleDeclaration(String), + /// Indicates a function identifier was declared with a /// different signature than declared previously - #[error("Function {0} signature {2:?} is incompatible with previous declaration {1:?}")] IncompatibleSignature(String, ir::Signature, ir::Signature), + /// Indicates an identifier was defined more than once - #[error("Duplicate definition of identifier: {0}")] DuplicateDefinition(String), + /// Indicates an identifier was defined, but was declared as an import - #[error("Invalid to define identifier declared as an import: {0}")] InvalidImportDefinition(String), + /// Wraps a `cranelift-codegen` error - #[error("Compilation error: {0}")] - Compilation(#[from] CodegenError), + Compilation(CodegenError), + /// Wraps a generic error from a backend - #[error("Backend error: {0}")] - Backend(#[source] anyhow::Error), + Backend(anyhow::Error), +} + +impl std::error::Error for ModuleError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Undeclared { .. } + | Self::IncompatibleDeclaration { .. } + | Self::IncompatibleSignature { .. } + | Self::DuplicateDefinition { .. } + | Self::InvalidImportDefinition { .. } => None, + Self::Compilation(source) => Some(source), + Self::Backend(source) => Some(&**source), + } + } +} + +impl std::fmt::Display for ModuleError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Undeclared(name) => { + write!(f, "Undeclared identifier: {}", name) + } + Self::IncompatibleDeclaration(name) => { + write!(f, "Incompatible declaration of identifier: {}", name,) + } + Self::IncompatibleSignature(name, prev_sig, new_sig) => { + write!( + f, + "Function {} signature {:?} is incompatible with previous declaration {:?}", + name, new_sig, prev_sig, + ) + } + Self::DuplicateDefinition(name) => { + write!(f, "Duplicate definition of identifier: {}", name) + } + Self::InvalidImportDefinition(name) => { + write!( + f, + "Invalid to define identifier declared as an import: {}", + name, + ) + } + Self::Compilation(err) => { + write!(f, "Compilation error: {}", err) + } + Self::Backend(err) => write!(f, "Backend error: {}", err), + } + } +} + +impl std::convert::From for ModuleError { + fn from(source: CodegenError) -> Self { + Self::Compilation { 0: source } + } } /// A convenient alias for a `Result` that uses `ModuleError` as the error type. From 03fdbadfb4e2f6e86ac983fecd10d8be069da7e8 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 16 Mar 2021 11:51:05 +0100 Subject: [PATCH 39/43] Remove thiserror dependency from cranelift_codegen --- Cargo.lock | 1 - cranelift/codegen/Cargo.toml | 1 - cranelift/codegen/src/data_value.rs | 28 ++++++++++++-- cranelift/codegen/src/isa/mod.rs | 18 +++++++-- cranelift/codegen/src/isa/unwind/systemv.rs | 23 ++++++++--- cranelift/codegen/src/result.rs | 42 +++++++++++++++++---- cranelift/codegen/src/settings.rs | 22 ++++++++--- cranelift/codegen/src/verifier/mod.rs | 21 ++++++----- 8 files changed, 119 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1f4a79617..3fbc57af9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -535,7 +535,6 @@ dependencies = [ "smallvec", "souper-ir", "target-lexicon", - "thiserror", "wast 35.0.2", ] diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 9eb990f896..5ef650f1b3 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -23,7 +23,6 @@ serde = { version = "1.0.94", features = ["derive"], optional = true } bincode = { version = "1.2.1", optional = true } gimli = { version = "0.23.0", default-features = false, features = ["write"], optional = true } smallvec = { version = "1.6.1" } -thiserror = "1.0.4" peepmatic = { path = "../peepmatic", optional = true, version = "0.73.0" } peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.73.0" } peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.73.0" } diff --git a/cranelift/codegen/src/data_value.rs b/cranelift/codegen/src/data_value.rs index 193607f392..8819859f09 100644 --- a/cranelift/codegen/src/data_value.rs +++ b/cranelift/codegen/src/data_value.rs @@ -5,7 +5,6 @@ use crate::ir::{types, ConstantData, Type}; use core::convert::TryInto; use core::fmt::{self, Display, Formatter}; use core::ptr; -use thiserror::Error; /// Represent a data value. Where [Value] is an SSA reference, [DataValue] is the type + value /// that would be referred to by a [Value]. @@ -97,15 +96,36 @@ impl DataValue { } /// Record failures to cast [DataValue]. -#[derive(Error, Debug, PartialEq)] +#[derive(Debug, PartialEq)] #[allow(missing_docs)] pub enum DataValueCastFailure { - #[error("unable to cast data value of type {0} to type {1}")] TryInto(Type, Type), - #[error("unable to cast i64({0}) to a data value of type {1}")] FromInteger(i64, Type), } +impl std::error::Error for DataValueCastFailure {} + +impl Display for DataValueCastFailure { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + DataValueCastFailure::TryInto(from, to) => { + write!( + f, + "unable to cast data value of type {} to type {}", + from, to + ) + } + DataValueCastFailure::FromInteger(val, to) => { + write!( + f, + "unable to cast i64({}) to a data value of type {}", + val, to + ) + } + } + } +} + /// Helper for creating conversion implementations for [DataValue]. macro_rules! build_conversion_impl { ( $rust_ty:ty, $data_value_ty:ident, $cranelift_ty:ident ) => { diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index a24f64a256..811ad8c9d7 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -69,7 +69,6 @@ use core::fmt; use core::fmt::{Debug, Formatter}; use core::hash::Hasher; use target_lexicon::{triple, Architecture, OperatingSystem, PointerWidth, Triple}; -use thiserror::Error; #[cfg(feature = "riscv")] mod riscv; @@ -178,17 +177,28 @@ pub fn lookup_by_name(name: &str) -> Result { } /// Describes reason for target lookup failure -#[derive(Error, PartialEq, Eq, Copy, Clone, Debug)] +#[derive(PartialEq, Eq, Copy, Clone, Debug)] pub enum LookupError { /// Support for this target was disabled in the current build. - #[error("Support for this target is disabled")] SupportDisabled, /// Support for this target has not yet been implemented. - #[error("Support for this target has not been implemented yet")] Unsupported, } +impl std::error::Error for LookupError {} + +impl fmt::Display for LookupError { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + LookupError::SupportDisabled => write!(f, "Support for this target is disabled"), + LookupError::Unsupported => { + write!(f, "Support for this target has not been implemented yet") + } + } + } +} + /// Builder for a `TargetIsa`. /// Modify the ISA-specific settings before creating the `TargetIsa` trait object with `finish`. #[derive(Clone)] diff --git a/cranelift/codegen/src/isa/unwind/systemv.rs b/cranelift/codegen/src/isa/unwind/systemv.rs index 5ceadef93f..9e537907a8 100644 --- a/cranelift/codegen/src/isa/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/unwind/systemv.rs @@ -6,7 +6,6 @@ use crate::isa::unwind::UnwindInst; use crate::result::{CodegenError, CodegenResult}; use alloc::vec::Vec; use gimli::write::{Address, FrameDescriptionEntry}; -use thiserror::Error; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -15,16 +14,30 @@ type Register = u16; /// Enumerate the errors possible in mapping Cranelift registers to their DWARF equivalent. #[allow(missing_docs)] -#[derive(Error, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] pub enum RegisterMappingError { - #[error("unable to find bank for register info")] MissingBank, - #[error("register mapping is currently only implemented for x86_64")] UnsupportedArchitecture, - #[error("unsupported register bank: {0}")] UnsupportedRegisterBank(&'static str), } +impl std::error::Error for RegisterMappingError {} + +impl std::fmt::Display for RegisterMappingError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + RegisterMappingError::MissingBank => write!(f, "unable to find bank for register info"), + RegisterMappingError::UnsupportedArchitecture => write!( + f, + "register mapping is currently only implemented for x86_64" + ), + RegisterMappingError::UnsupportedRegisterBank(bank) => { + write!(f, "unsupported register bank: {}", bank) + } + } + } +} + // This mirrors gimli's CallFrameInstruction, but is serializable // This excludes CfaExpression, Expression, ValExpression due to // https://github.com/gimli-rs/gimli/issues/513. diff --git a/cranelift/codegen/src/result.rs b/cranelift/codegen/src/result.rs index 493545c151..aa264c7601 100644 --- a/cranelift/codegen/src/result.rs +++ b/cranelift/codegen/src/result.rs @@ -2,19 +2,17 @@ use crate::verifier::VerifierErrors; use std::string::String; -use thiserror::Error; /// A compilation error. /// /// When Cranelift fails to compile a function, it will return one of these error codes. -#[derive(Error, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] pub enum CodegenError { /// A list of IR verifier errors. /// /// This always represents a bug, either in the code that generated IR for Cranelift, or a bug /// in Cranelift itself. - #[error("Verifier errors")] - Verifier(#[from] VerifierErrors), + Verifier(VerifierErrors), /// An implementation limit was exceeded. /// @@ -22,27 +20,55 @@ pub enum CodegenError { /// limits][limits] that cause compilation to fail when they are exceeded. /// /// [limits]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/docs/ir.md#implementation-limits - #[error("Implementation limit exceeded")] ImplLimitExceeded, /// The code size for the function is too large. /// /// Different target ISAs may impose a limit on the size of a compiled function. If that limit /// is exceeded, compilation fails. - #[error("Code for function is too large")] CodeTooLarge, /// Something is not supported by the code generator. This might be an indication that a /// feature is used without explicitly enabling it, or that something is temporarily /// unsupported by a given target backend. - #[error("Unsupported feature: {0}")] Unsupported(String), /// A failure to map Cranelift register representation to a DWARF register representation. #[cfg(feature = "unwind")] - #[error("Register mapping error")] RegisterMappingError(crate::isa::unwind::systemv::RegisterMappingError), } /// A convenient alias for a `Result` that uses `CodegenError` as the error type. pub type CodegenResult = Result; + +impl std::error::Error for CodegenError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + CodegenError::Verifier(source) => Some(source), + CodegenError::ImplLimitExceeded { .. } + | CodegenError::CodeTooLarge { .. } + | CodegenError::Unsupported { .. } => None, + #[cfg(feature = "unwind")] + CodegenError::RegisterMappingError { .. } => None, + } + } +} + +impl std::fmt::Display for CodegenError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + CodegenError::Verifier(_) => write!(f, "Verifier errors"), + CodegenError::ImplLimitExceeded => write!(f, "Implementation limit exceeded"), + CodegenError::CodeTooLarge => write!(f, "Code for function is too large"), + CodegenError::Unsupported(feature) => write!(f, "Unsupported feature: {}", feature), + #[cfg(feature = "unwind")] + CodegenError::RegisterMappingError(_0) => write!(f, "Register mapping error"), + } + } +} + +impl From for CodegenError { + fn from(source: VerifierErrors) -> Self { + CodegenError::Verifier { 0: source } + } +} diff --git a/cranelift/codegen/src/settings.rs b/cranelift/codegen/src/settings.rs index 88a3c62157..0f36db82a9 100644 --- a/cranelift/codegen/src/settings.rs +++ b/cranelift/codegen/src/settings.rs @@ -26,7 +26,6 @@ use alloc::boxed::Box; use alloc::string::{String, ToString}; use core::fmt; use core::str; -use thiserror::Error; /// A string-based configurator for settings groups. /// @@ -261,21 +260,34 @@ impl Configurable for Builder { } /// An error produced when changing a setting. -#[derive(Error, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] pub enum SetError { /// No setting by this name exists. - #[error("No existing setting named '{0}'")] BadName(String), /// Type mismatch for setting (e.g., setting an enum setting as a bool). - #[error("Trying to set a setting with the wrong type")] BadType, /// This is not a valid value for this setting. - #[error("Unexpected value for a setting, expected {0}")] BadValue(String), } +impl std::error::Error for SetError {} + +impl fmt::Display for SetError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + SetError::BadName(name) => write!(f, "No existing setting named '{}'", name), + SetError::BadType => { + write!(f, "Trying to set a setting with the wrong type") + } + SetError::BadValue(value) => { + write!(f, "Unexpected value for a setting, expected {}", value) + } + } + } +} + /// A result returned when changing a setting. pub type SetResult = Result; diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs index e20570c951..7c64c0464c 100644 --- a/cranelift/codegen/src/verifier/mod.rs +++ b/cranelift/codegen/src/verifier/mod.rs @@ -80,7 +80,6 @@ use alloc::vec::Vec; use core::cmp::Ordering; use core::fmt::{self, Display, Formatter, Write}; use log::debug; -use thiserror::Error; pub use self::cssa::verify_cssa; pub use self::liveness::verify_liveness; @@ -92,8 +91,7 @@ mod liveness; mod locations; /// A verifier error. -#[derive(Error, Debug, PartialEq, Eq, Clone)] -#[error("{}{}: {}", .location, format_context(.context), .message)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct VerifierError { /// The entity causing the verifier error. pub location: AnyEntity, @@ -104,11 +102,14 @@ pub struct VerifierError { pub message: String, } -/// Helper for formatting Verifier::Error context. -fn format_context(context: &Option) -> String { - match context { - None => "".to_string(), - Some(c) => format!(" ({})", c), +impl std::error::Error for VerifierError {} + +impl Display for VerifierError { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match &self.context { + None => write!(f, "{}: {}", self.location, self.message), + Some(context) => write!(f, "{} ({}): {}", self.location, context, self.message), + } } } @@ -175,9 +176,11 @@ pub type VerifierStepResult = Result; pub type VerifierResult = Result; /// List of verifier errors. -#[derive(Error, Debug, Default, PartialEq, Eq, Clone)] +#[derive(Debug, Default, PartialEq, Eq, Clone)] pub struct VerifierErrors(pub Vec); +impl std::error::Error for VerifierErrors {} + impl VerifierErrors { /// Return a new `VerifierErrors` struct. #[inline] From 84c79982e7f89969c1e034e511457b1bbbf4fe83 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 16 Mar 2021 12:16:11 +0100 Subject: [PATCH 40/43] Remove unnecessary dependencies Found using cargo-udeps --- Cargo.lock | 3 --- cranelift/interpreter/Cargo.toml | 2 +- cranelift/reader/Cargo.toml | 1 - crates/environ/Cargo.toml | 2 -- 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3fbc57af9a..102a126904 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -673,7 +673,6 @@ dependencies = [ "cranelift-codegen", "smallvec", "target-lexicon", - "thiserror", ] [[package]] @@ -3313,7 +3312,6 @@ dependencies = [ name = "wasmtime-environ" version = "0.26.0" dependencies = [ - "anyhow", "cfg-if 1.0.0", "cranelift-codegen", "cranelift-entity", @@ -3322,7 +3320,6 @@ dependencies = [ "indexmap", "log", "more-asserts", - "region", "serde", "thiserror", "wasmparser", diff --git a/cranelift/interpreter/Cargo.toml b/cranelift/interpreter/Cargo.toml index 0439c964d8..c539f70fe1 100644 --- a/cranelift/interpreter/Cargo.toml +++ b/cranelift/interpreter/Cargo.toml @@ -13,13 +13,13 @@ edition = "2018" [dependencies] cranelift-codegen = { path = "../codegen", version = "0.73.0", features = ["all-arch"] } cranelift-entity = { path = "../entity", version = "0.73.0" } -cranelift-reader = { path = "../reader", version = "0.73.0" } log = { version = "0.4.8", default-features = false } smallvec = "1.6.1" thiserror = "1.0.15" [dev-dependencies] cranelift-frontend = { path = "../frontend", version = "0.73.0" } +cranelift-reader = { path = "../reader", version = "0.73.0" } [badges] maintenance = { status = "experimental" } diff --git a/cranelift/reader/Cargo.toml b/cranelift/reader/Cargo.toml index d1d8a29ae3..2d7e93fee0 100644 --- a/cranelift/reader/Cargo.toml +++ b/cranelift/reader/Cargo.toml @@ -13,7 +13,6 @@ edition = "2018" cranelift-codegen = { path = "../codegen", version = "0.73.0" } smallvec = "1.6.1" target-lexicon = "0.12" -thiserror = "1.0.15" [badges] maintenance = { status = "experimental" } diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 751be91cb6..383d030b43 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -12,8 +12,6 @@ readme = "README.md" edition = "2018" [dependencies] -anyhow = "1.0" -region = "2.2.0" cranelift-codegen = { path = "../../cranelift/codegen", version = "0.73.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.73.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.73.0", features = ["enable-serde"] } From 82f3ad4f1a412ae94958bb5eb82788bc95082b2a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 19 Mar 2021 16:41:39 +0100 Subject: [PATCH 41/43] Add comment why thiserror is not used --- cranelift/codegen/src/data_value.rs | 2 ++ cranelift/codegen/src/isa/mod.rs | 2 ++ cranelift/codegen/src/isa/unwind/systemv.rs | 2 ++ cranelift/codegen/src/result.rs | 2 ++ cranelift/codegen/src/verifier/mod.rs | 4 ++++ cranelift/module/src/module.rs | 2 ++ 6 files changed, 14 insertions(+) diff --git a/cranelift/codegen/src/data_value.rs b/cranelift/codegen/src/data_value.rs index 8819859f09..a317c7f394 100644 --- a/cranelift/codegen/src/data_value.rs +++ b/cranelift/codegen/src/data_value.rs @@ -103,6 +103,8 @@ pub enum DataValueCastFailure { FromInteger(i64, Type), } +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. impl std::error::Error for DataValueCastFailure {} impl Display for DataValueCastFailure { diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 811ad8c9d7..fccb46f7a6 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -186,6 +186,8 @@ pub enum LookupError { Unsupported, } +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. impl std::error::Error for LookupError {} impl fmt::Display for LookupError { diff --git a/cranelift/codegen/src/isa/unwind/systemv.rs b/cranelift/codegen/src/isa/unwind/systemv.rs index 9e537907a8..da3bfea869 100644 --- a/cranelift/codegen/src/isa/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/unwind/systemv.rs @@ -21,6 +21,8 @@ pub enum RegisterMappingError { UnsupportedRegisterBank(&'static str), } +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. impl std::error::Error for RegisterMappingError {} impl std::fmt::Display for RegisterMappingError { diff --git a/cranelift/codegen/src/result.rs b/cranelift/codegen/src/result.rs index aa264c7601..3178cd5ba9 100644 --- a/cranelift/codegen/src/result.rs +++ b/cranelift/codegen/src/result.rs @@ -41,6 +41,8 @@ pub enum CodegenError { /// A convenient alias for a `Result` that uses `CodegenError` as the error type. pub type CodegenResult = Result; +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. impl std::error::Error for CodegenError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs index 7c64c0464c..1d1801016b 100644 --- a/cranelift/codegen/src/verifier/mod.rs +++ b/cranelift/codegen/src/verifier/mod.rs @@ -102,6 +102,8 @@ pub struct VerifierError { pub message: String, } +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. impl std::error::Error for VerifierError {} impl Display for VerifierError { @@ -179,6 +181,8 @@ pub type VerifierResult = Result; #[derive(Debug, Default, PartialEq, Eq, Clone)] pub struct VerifierErrors(pub Vec); +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. impl std::error::Error for VerifierErrors {} impl VerifierErrors { diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index 97db899342..191d468ed7 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -192,6 +192,8 @@ pub enum ModuleError { Backend(anyhow::Error), } +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. impl std::error::Error for ModuleError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { From 2145855954dfd3c5370c41c477a8cafe6be1a25d Mon Sep 17 00:00:00 2001 From: Ryan Brewster Date: Tue, 4 May 2021 08:39:01 -0700 Subject: [PATCH 42/43] Add example of execution limits using fuel consumption (#2869) * Add example of execution limits using fuel consumption * run rustfmt * Use a more naive WAT implementation * Catch error and return cleanly * Add a C example to demonstrate fuel consumption * Add error handling for add_fuel --- examples/fuel.c | 132 ++++++++++++++++++++++++++++++++++++++++++++++ examples/fuel.rs | 33 ++++++++++++ examples/fuel.wat | 13 +++++ 3 files changed, 178 insertions(+) create mode 100644 examples/fuel.c create mode 100644 examples/fuel.rs create mode 100644 examples/fuel.wat diff --git a/examples/fuel.c b/examples/fuel.c new file mode 100644 index 0000000000..af3b51782d --- /dev/null +++ b/examples/fuel.c @@ -0,0 +1,132 @@ +/* +Example of instantiating of the WebAssembly module and invoking its exported +function. + +You can compile and run this example on Linux with: + + cargo build --release -p wasmtime-c-api + cc examples/fuel.c \ + -I crates/c-api/include \ + -I crates/c-api/wasm-c-api/include \ + target/release/libwasmtime.a \ + -lpthread -ldl -lm \ + -o fuel + ./fuel + +Note that on Windows and macOS the command will be similar, but you'll need +to tweak the `-lpthread` and such annotations. +*/ + +#include +#include +#include +#include +#include + +static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap); + +int main() { + wasmtime_error_t *error = NULL; + + wasm_config_t *config = wasm_config_new(); + assert(config != NULL); + wasmtime_config_consume_fuel_set(config, true); + + // Create an *engine*, which is a compilation context, with our configured options. + wasm_engine_t *engine = wasm_engine_new_with_config(config); + assert(engine != NULL); + wasm_store_t *store = wasm_store_new(engine); + assert(store != NULL); + error = wasmtime_store_add_fuel(store, 10000); + if (error != NULL) + exit_with_error("failed to add fuel", error, NULL); + + // Load our input file to parse it next + FILE* file = fopen("examples/fuel.wat", "r"); + if (!file) { + printf("> Error loading file!\n"); + return 1; + } + fseek(file, 0L, SEEK_END); + size_t file_size = ftell(file); + fseek(file, 0L, SEEK_SET); + wasm_byte_vec_t wat; + wasm_byte_vec_new_uninitialized(&wat, file_size); + if (fread(wat.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + return 1; + } + fclose(file); + + // Parse the wat into the binary wasm format + wasm_byte_vec_t wasm; + error = wasmtime_wat2wasm(&wat, &wasm); + if (error != NULL) + exit_with_error("failed to parse wat", error, NULL); + wasm_byte_vec_delete(&wat); + + // Compile and instantiate our module + wasm_module_t *module = NULL; + error = wasmtime_module_new(engine, &wasm, &module); + if (module == NULL) + exit_with_error("failed to compile module", error, NULL); + wasm_byte_vec_delete(&wasm); + wasm_trap_t *trap = NULL; + wasm_instance_t *instance = NULL; + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + error = wasmtime_instance_new(store, module, &imports, &instance, &trap); + if (instance == NULL) + exit_with_error("failed to instantiate", error, trap); + + // Lookup our `fibonacci` export function + wasm_extern_vec_t externs; + wasm_instance_exports(instance, &externs); + assert(externs.size == 1); + wasm_func_t *fibonacci = wasm_extern_as_func(externs.data[0]); + assert(fibonacci != NULL); + + // Call it repeatedly until it fails + for (int n = 1; ; n++) { + uint64_t fuel_before; + wasmtime_store_fuel_consumed(store, &fuel_before); + wasm_val_t params[1] = { WASM_I32_VAL(n) }; + wasm_val_t results[1]; + wasm_val_vec_t params_vec = WASM_ARRAY_VEC(params); + wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results); + error = wasmtime_func_call(fibonacci, ¶ms_vec, &results_vec, &trap); + if (error != NULL || trap != NULL) { + printf("Exhausted fuel computing fib(%d)\n", n); + break; + } + + uint64_t fuel_after; + wasmtime_store_fuel_consumed(store, &fuel_after); + assert(results[0].kind == WASM_I32); + printf("fib(%d) = %d [consumed %lld fuel]\n", n, results[0].of.i32, fuel_after - fuel_before); + + error = wasmtime_store_add_fuel(store, fuel_after - fuel_before); + if (error != NULL) + exit_with_error("failed to add fuel", error, NULL); + } + + // Clean up after ourselves at this point + wasm_extern_vec_delete(&externs); + wasm_instance_delete(instance); + wasm_module_delete(module); + wasm_store_delete(store); + wasm_engine_delete(engine); + return 0; +} + +static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap) { + fprintf(stderr, "error: %s\n", message); + wasm_byte_vec_t error_message; + if (error != NULL) { + wasmtime_error_message(error, &error_message); + } else { + wasm_trap_message(trap, &error_message); + } + fprintf(stderr, "%.*s\n", (int) error_message.size, error_message.data); + wasm_byte_vec_delete(&error_message); + exit(1); +} diff --git a/examples/fuel.rs b/examples/fuel.rs new file mode 100644 index 0000000000..00181a67e1 --- /dev/null +++ b/examples/fuel.rs @@ -0,0 +1,33 @@ +//! Example of limiting a WebAssembly function's runtime using "fuel consumption". + +// You can execute this example with `cargo run --example fuel` + +use anyhow::Result; +use wasmtime::*; + +fn main() -> Result<()> { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config)?; + let store = Store::new(&engine); + store.add_fuel(10_000)?; + let module = Module::from_file(store.engine(), "examples/fuel.wat")?; + let instance = Instance::new(&store, &module, &[])?; + + // Invoke `fibonacci` export with higher and higher numbers until we exhaust our fuel. + let fibonacci = instance.get_typed_func::("fibonacci")?; + for n in 1.. { + let fuel_before = store.fuel_consumed().unwrap(); + let output = match fibonacci.call(n) { + Ok(v) => v, + Err(_) => { + println!("Exhausted fuel computing fib({})", n); + break; + } + }; + let fuel_consumed = store.fuel_consumed().unwrap() - fuel_before; + println!("fib({}) = {} [consumed {} fuel]", n, output, fuel_consumed); + store.add_fuel(fuel_consumed)?; + } + Ok(()) +} diff --git a/examples/fuel.wat b/examples/fuel.wat new file mode 100644 index 0000000000..48622c2e21 --- /dev/null +++ b/examples/fuel.wat @@ -0,0 +1,13 @@ +(module + (func $fibonacci (param $n i32) (result i32) + (if + (i32.lt_s (local.get $n) (i32.const 2)) + (return (local.get $n)) + ) + (i32.add + (call $fibonacci (i32.sub (local.get $n) (i32.const 1))) + (call $fibonacci (i32.sub (local.get $n) (i32.const 2))) + ) + ) + (export "fibonacci" (func $fibonacci)) +) From 9d6f64b33d7a1c5472d63881f267e09ded1302b6 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Thu, 6 May 2021 17:08:05 -0700 Subject: [PATCH 43/43] cargo deny: ignore RUSTSEC-2021-0064 Transient dependencies depend on two different versions of `cpuid-bool`. This advisory does not appear to be urgent. We should review this ignore after a few weeks to see if our deps have switched over. text of the advisory: Issued May 6, 2021 Package cpuid-bool (crates.io) Type Unmaintained Details https://github.com/RustCrypto/utils/pull/381 Patched no patched versions Description Please use the `cpufeatures`` crate going forward: https://github.com/RustCrypto/utils/tree/master/cpufeatures There will be no further releases of cpuid-bool. --- deny.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/deny.toml b/deny.toml index a63b6864e4..7f909d470e 100644 --- a/deny.toml +++ b/deny.toml @@ -14,6 +14,7 @@ vulnerability = "deny" unmaintained = "deny" yanked = "deny" ignore = [ + "RUSTSEC-2021-0064" ] # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html