diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index d1368a4553..1989fb8dce 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -83,7 +83,9 @@ fn in_int_reg(ty: types::Type) -> bool { | types::B8 | types::B16 | types::B32 - | types::B64 => true, + | types::B64 + | types::R64 => true, + types::R32 => panic!("unexpected 32-bits refs on x64!"), _ => false, } } @@ -497,8 +499,26 @@ impl ABIBody for X64ABIBody { ) } - fn spillslots_to_stackmap(&self, _slots: &[SpillSlot], _state: &EmitState) -> Stackmap { - unimplemented!("spillslots_to_stackmap") + fn spillslots_to_stackmap(&self, slots: &[SpillSlot], state: &EmitState) -> Stackmap { + assert!(state.virtual_sp_offset >= 0); + trace!( + "spillslots_to_stackmap: slots = {:?}, state = {:?}", + slots, + state + ); + let map_size = (state.virtual_sp_offset + state.nominal_sp_to_fp) as u32; + let map_words = (map_size + 7) / 8; + let mut bits = std::iter::repeat(false) + .take(map_words as usize) + .collect::>(); + + let first_spillslot_word = (self.stack_slots_size + state.virtual_sp_offset as usize) / 8; + for &slot in slots { + let slot = slot.get() as usize; + bits[first_spillslot_word + slot] = true; + } + + Stackmap::from_slice(&bits[..]) } fn gen_prologue(&mut self) -> Vec { @@ -971,7 +991,7 @@ fn load_stack(mem: impl Into, into_reg: Writable, ty: Type) types::B1 | types::B8 | types::I8 => (true, Some(ExtMode::BQ)), types::B16 | types::I16 => (true, Some(ExtMode::WQ)), types::B32 | types::I32 => (true, Some(ExtMode::LQ)), - types::B64 | types::I64 => (true, None), + types::B64 | types::I64 | types::R64 => (true, None), types::F32 | types::F64 => (false, None), _ => panic!("load_stack({})", ty), }; @@ -1008,7 +1028,7 @@ fn store_stack(mem: impl Into, from_reg: Reg, ty: Type) -> Inst types::B1 | types::B8 | types::I8 => (true, 1), types::B16 | types::I16 => (true, 2), types::B32 | types::I32 => (true, 4), - types::B64 | types::I64 => (true, 8), + types::B64 | types::I64 | types::R64 => (true, 8), types::F32 => (false, 4), types::F64 => (false, 8), _ => unimplemented!("store_stack({})", ty), @@ -1166,13 +1186,9 @@ impl ABICall for X64ABICall { } match &self.dest { - &CallDest::ExtName(ref name, RelocDistance::Near) => ctx.emit(Inst::call_known( - name.clone(), - uses, - defs, - self.loc, - self.opcode, - )), + &CallDest::ExtName(ref name, RelocDistance::Near) => ctx.emit_safepoint( + Inst::call_known(name.clone(), uses, defs, self.loc, self.opcode), + ), &CallDest::ExtName(ref name, RelocDistance::Far) => { let tmp = ctx.alloc_tmp(RegClass::I64, I64); ctx.emit(Inst::LoadExtName { @@ -1181,7 +1197,7 @@ impl ABICall for X64ABICall { offset: 0, srcloc: self.loc, }); - ctx.emit(Inst::call_unknown( + ctx.emit_safepoint(Inst::call_unknown( RegMem::reg(tmp.to_reg()), uses, defs, @@ -1189,7 +1205,7 @@ impl ABICall for X64ABICall { self.opcode, )); } - &CallDest::Reg(reg) => ctx.emit(Inst::call_unknown( + &CallDest::Reg(reg) => ctx.emit_safepoint(Inst::call_unknown( RegMem::reg(reg), uses, defs, diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index e58a03a1e0..2d4ae85d03 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1282,6 +1282,9 @@ pub(crate) fn emit( Inst::CallKnown { dest, loc, opcode, .. } => { + if let Some(s) = state.take_stackmap() { + sink.add_stackmap(5, s); + } sink.put1(0xE8); // The addend adjusts for the difference between the end of the instruction and the // beginning of the immediate field. @@ -1295,6 +1298,7 @@ pub(crate) fn emit( Inst::CallUnknown { dest, opcode, loc, .. } => { + let start_offset = sink.cur_offset(); match dest { RegMem::Reg { reg } => { let reg_enc = int_reg_enc(*reg); @@ -1322,6 +1326,9 @@ pub(crate) fn emit( ); } } + if let Some(s) = state.take_stackmap() { + sink.add_stackmap_ending_at(start_offset, s); + } if opcode.is_call() { sink.add_call_site(*loc, *opcode); } @@ -2298,6 +2305,9 @@ pub(crate) fn emit( Inst::Ud2 { trap_info } => { sink.add_trap(trap_info.0, trap_info.1); + if let Some(s) = state.take_stackmap() { + sink.add_stackmap(2, s); + } sink.put1(0x0f); sink.put1(0x0b); } @@ -2315,4 +2325,6 @@ pub(crate) fn emit( // Generate no code. } } + + state.clear_post_insn(); } diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index ffae1d1443..572f8c4865 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -13,7 +13,7 @@ use regalloc::RegUsageCollector; use regalloc::{RealRegUniverse, Reg, RegClass, RegUsageMapper, SpillSlot, VirtualReg, Writable}; use smallvec::SmallVec; -use crate::binemit::CodeOffset; +use crate::binemit::{CodeOffset, Stackmap}; use crate::ir::types::*; use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type}; use crate::machinst::*; @@ -1913,7 +1913,7 @@ impl MachInst for Inst { fn rc_for_type(ty: Type) -> CodegenResult { match ty { - I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 => Ok(RegClass::I64), + I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 | R32 | R64 => Ok(RegClass::I64), F32 | F64 | I128 | B128 => Ok(RegClass::V128), IFLAGS | FFLAGS => Ok(RegClass::I64), _ => Err(CodegenError::Unsupported(format!( @@ -1987,7 +1987,13 @@ impl MachInst for Inst { /// State carried between emissions of a sequence of instructions. #[derive(Default, Clone, Debug)] pub struct EmitState { - virtual_sp_offset: i64, + /// Addend to convert nominal-SP offsets to real-SP offsets at the current + /// program point. + pub(crate) virtual_sp_offset: i64, + /// Offset of FP from nominal-SP. + pub(crate) nominal_sp_to_fp: i64, + /// Safepoint stackmap for upcoming instruction, as provided to `pre_safepoint()`. + stackmap: Option, } impl MachInstEmit for Inst { @@ -2003,11 +2009,27 @@ impl MachInstEmit for Inst { } impl MachInstEmitState for EmitState { - fn new(_: &dyn ABIBody) -> Self { + fn new(abi: &dyn ABIBody) -> Self { EmitState { virtual_sp_offset: 0, + nominal_sp_to_fp: abi.frame_size() as i64, + stackmap: None, } } + + fn pre_safepoint(&mut self, stackmap: Stackmap) { + self.stackmap = Some(stackmap); + } +} + +impl EmitState { + fn take_stackmap(&mut self) -> Option { + self.stackmap.take() + } + + fn clear_post_insn(&mut self) { + self.stackmap = None; + } } /// A label-use (internal relocation) in generated code. diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 0ffb4cef93..36b659da23 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -37,7 +37,8 @@ type Ctx<'a> = &'a mut dyn LowerCtx; fn is_int_ty(ty: Type) -> bool { match ty { - types::I8 | types::I16 | types::I32 | types::I64 => true, + types::I8 | types::I16 | types::I32 | types::I64 | types::R64 => true, + types::R32 => panic!("shouldn't have 32-bits refs on x64"), _ => false, } } @@ -45,6 +46,7 @@ fn is_int_ty(ty: Type) -> bool { fn is_bool_ty(ty: Type) -> bool { match ty { types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true, + types::R32 => panic!("shouldn't have 32-bits refs on x64"), _ => false, } } @@ -52,6 +54,7 @@ fn is_bool_ty(ty: Type) -> bool { fn is_float_ty(ty: Type) -> bool { match ty { types::F32 | types::F64 => true, + types::R32 => panic!("shouldn't have 32-bits refs on x64"), _ => false, } } @@ -329,7 +332,7 @@ fn lower_insn_to_regs>( }; match op { - Opcode::Iconst => { + Opcode::Iconst | Opcode::Null => { if let Some(w64) = iri_to_u64_imm(ctx, insn) { let dst_is_64 = w64 > 0x7fffffff; let dst = output_to_reg(ctx, outputs[0]); @@ -771,6 +774,23 @@ fn lower_insn_to_regs>( } } + Opcode::IsNull | Opcode::IsInvalid => { + // Null references are represented by the constant value 0; invalid references are + // represented by the constant value -1. See `define_reftypes()` in + // `meta/src/isa/x86/encodings.rs` to confirm. + let src = input_to_reg(ctx, inputs[0]); + let dst = output_to_reg(ctx, outputs[0]); + let ty = ctx.input_ty(insn, 0); + let imm = match op { + // TODO could use tst src, src for IsNull + Opcode::IsNull => 0, + Opcode::IsInvalid => 0xffffffff, + _ => unreachable!(), + }; + ctx.emit(Inst::cmp_rmi_r(ty.bytes() as u8, RegMemImm::imm(imm), src)); + ctx.emit(Inst::setcc(CC::Z, dst)); + } + Opcode::Uextend | Opcode::Sextend | Opcode::Bint @@ -979,7 +999,7 @@ fn lower_insn_to_regs>( Opcode::Trap | Opcode::ResumableTrap => { let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); - ctx.emit(Inst::Ud2 { trap_info }) + ctx.emit_safepoint(Inst::Ud2 { trap_info }); } Opcode::Trapif | Opcode::Trapff => { @@ -1033,7 +1053,7 @@ fn lower_insn_to_regs>( cc }; - ctx.emit(Inst::TrapIf { + ctx.emit_safepoint(Inst::TrapIf { trap_code, srcloc, cc, @@ -1658,7 +1678,7 @@ fn lower_insn_to_regs>( let ty = ctx.output_ty(insn, 0); - if ty.is_int() { + if is_int_ty(ty) { let size = ty.bytes() as u8; if size == 1 { // Sign-extend operands to 32, then do a cmove of size 4. diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index bcdc9449a7..d78d0e325b 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -1174,6 +1174,20 @@ impl MachBuffer { stackmap, }); } + + /// Add stackmap metadata for this program point: a set of stack offsets + /// (from SP upward) that contain live references. + /// + /// Method variant of `add_stackmap` that is easier to manipulate for non-fixed-sizes + /// instructions. + pub fn add_stackmap_ending_at(&mut self, start_offset: CodeOffset, stackmap: Stackmap) { + let cur_offset = self.cur_offset(); + self.stackmaps.push(MachStackMap { + offset: start_offset, + offset_end: cur_offset, + stackmap, + }); + } } impl MachBufferFinalized {