machinst x64: implement support for reference types;

cranelift/codegen/src/isa/x64/abi.rs

@@ -83,7 +83,9 @@ fn in_int_reg(ty: types::Type) -> bool {
         | types::B8
         | types::B16
         | types::B32
-        | types::B64 => true,
+        | types::B64
+        | types::R64 => true,
+        types::R32 => panic!("unexpected 32-bits refs on x64!"),
         _ => false,
     }
 }
@@ -497,8 +499,26 @@ impl ABIBody for X64ABIBody {
         )
     }
 
-    fn spillslots_to_stackmap(&self, _slots: &[SpillSlot], _state: &EmitState) -> Stackmap {
+    fn spillslots_to_stackmap(&self, slots: &[SpillSlot], state: &EmitState) -> Stackmap {
-        unimplemented!("spillslots_to_stackmap")
+        assert!(state.virtual_sp_offset >= 0);
+        trace!(
+            "spillslots_to_stackmap: slots = {:?}, state = {:?}",
+            slots,
+            state
+        );
+        let map_size = (state.virtual_sp_offset + state.nominal_sp_to_fp) as u32;
+        let map_words = (map_size + 7) / 8;
+        let mut bits = std::iter::repeat(false)
+            .take(map_words as usize)
+            .collect::<Vec<bool>>();
+
+        let first_spillslot_word = (self.stack_slots_size + state.virtual_sp_offset as usize) / 8;
+        for &slot in slots {
+            let slot = slot.get() as usize;
+            bits[first_spillslot_word + slot] = true;
+        }
+
+        Stackmap::from_slice(&bits[..])
     }
 
     fn gen_prologue(&mut self) -> Vec<Inst> {
@@ -971,7 +991,7 @@ fn load_stack(mem: impl Into<SyntheticAmode>, into_reg: Writable<Reg>, ty: Type)
         types::B1 | types::B8 | types::I8 => (true, Some(ExtMode::BQ)),
         types::B16 | types::I16 => (true, Some(ExtMode::WQ)),
         types::B32 | types::I32 => (true, Some(ExtMode::LQ)),
-        types::B64 | types::I64 => (true, None),
+        types::B64 | types::I64 | types::R64 => (true, None),
         types::F32 | types::F64 => (false, None),
         _ => panic!("load_stack({})", ty),
     };
@@ -1008,7 +1028,7 @@ fn store_stack(mem: impl Into<SyntheticAmode>, from_reg: Reg, ty: Type) -> Inst
         types::B1 | types::B8 | types::I8 => (true, 1),
         types::B16 | types::I16 => (true, 2),
         types::B32 | types::I32 => (true, 4),
-        types::B64 | types::I64 => (true, 8),
+        types::B64 | types::I64 | types::R64 => (true, 8),
         types::F32 => (false, 4),
         types::F64 => (false, 8),
         _ => unimplemented!("store_stack({})", ty),
@@ -1166,13 +1186,9 @@ impl ABICall for X64ABICall {
         }
 
         match &self.dest {
-            &CallDest::ExtName(ref name, RelocDistance::Near) => ctx.emit(Inst::call_known(
+            &CallDest::ExtName(ref name, RelocDistance::Near) => ctx.emit_safepoint(
-                name.clone(),
+                Inst::call_known(name.clone(), uses, defs, self.loc, self.opcode),
-                uses,
+            ),
-                defs,
-                self.loc,
-                self.opcode,
-            )),
             &CallDest::ExtName(ref name, RelocDistance::Far) => {
                 let tmp = ctx.alloc_tmp(RegClass::I64, I64);
                 ctx.emit(Inst::LoadExtName {
@@ -1181,7 +1197,7 @@ impl ABICall for X64ABICall {
                     offset: 0,
                     srcloc: self.loc,
                 });
-                ctx.emit(Inst::call_unknown(
+                ctx.emit_safepoint(Inst::call_unknown(
                     RegMem::reg(tmp.to_reg()),
                     uses,
                     defs,
@@ -1189,7 +1205,7 @@ impl ABICall for X64ABICall {
                     self.opcode,
                 ));
             }
-            &CallDest::Reg(reg) => ctx.emit(Inst::call_unknown(
+            &CallDest::Reg(reg) => ctx.emit_safepoint(Inst::call_unknown(
                 RegMem::reg(reg),
                 uses,
                 defs,

cranelift/codegen/src/isa/x64/inst/emit.rs

@@ -1282,6 +1282,9 @@ pub(crate) fn emit(
         Inst::CallKnown {
             dest, loc, opcode, ..
         } => {
+            if let Some(s) = state.take_stackmap() {
+                sink.add_stackmap(5, s);
+            }
             sink.put1(0xE8);
             // The addend adjusts for the difference between the end of the instruction and the
             // beginning of the immediate field.
@@ -1295,6 +1298,7 @@ pub(crate) fn emit(
         Inst::CallUnknown {
             dest, opcode, loc, ..
         } => {
+            let start_offset = sink.cur_offset();
             match dest {
                 RegMem::Reg { reg } => {
                     let reg_enc = int_reg_enc(*reg);
@@ -1322,6 +1326,9 @@ pub(crate) fn emit(
                     );
                 }
             }
+            if let Some(s) = state.take_stackmap() {
+                sink.add_stackmap_ending_at(start_offset, s);
+            }
             if opcode.is_call() {
                 sink.add_call_site(*loc, *opcode);
             }
@@ -2298,6 +2305,9 @@ pub(crate) fn emit(
 
         Inst::Ud2 { trap_info } => {
             sink.add_trap(trap_info.0, trap_info.1);
+            if let Some(s) = state.take_stackmap() {
+                sink.add_stackmap(2, s);
+            }
             sink.put1(0x0f);
             sink.put1(0x0b);
         }
@@ -2315,4 +2325,6 @@ pub(crate) fn emit(
             // Generate no code.
         }
     }
+
+    state.clear_post_insn();
 }

cranelift/codegen/src/isa/x64/inst/mod.rs

@@ -13,7 +13,7 @@ use regalloc::RegUsageCollector;
 use regalloc::{RealRegUniverse, Reg, RegClass, RegUsageMapper, SpillSlot, VirtualReg, Writable};
 use smallvec::SmallVec;
 
-use crate::binemit::CodeOffset;
+use crate::binemit::{CodeOffset, Stackmap};
 use crate::ir::types::*;
 use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type};
 use crate::machinst::*;
@@ -1913,7 +1913,7 @@ impl MachInst for Inst {
 
     fn rc_for_type(ty: Type) -> CodegenResult<RegClass> {
         match ty {
-            I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 => Ok(RegClass::I64),
+            I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 | R32 | R64 => Ok(RegClass::I64),
             F32 | F64 | I128 | B128 => Ok(RegClass::V128),
             IFLAGS | FFLAGS => Ok(RegClass::I64),
             _ => Err(CodegenError::Unsupported(format!(
@@ -1987,7 +1987,13 @@ impl MachInst for Inst {
 /// State carried between emissions of a sequence of instructions.
 #[derive(Default, Clone, Debug)]
 pub struct EmitState {
-    virtual_sp_offset: i64,
+    /// Addend to convert nominal-SP offsets to real-SP offsets at the current
+    /// program point.
+    pub(crate) virtual_sp_offset: i64,
+    /// Offset of FP from nominal-SP.
+    pub(crate) nominal_sp_to_fp: i64,
+    /// Safepoint stackmap for upcoming instruction, as provided to `pre_safepoint()`.
+    stackmap: Option<Stackmap>,
 }
 
 impl MachInstEmit for Inst {
@@ -2003,11 +2009,27 @@ impl MachInstEmit for Inst {
 }
 
 impl MachInstEmitState<Inst> for EmitState {
-    fn new(_: &dyn ABIBody<I = Inst>) -> Self {
+    fn new(abi: &dyn ABIBody<I = Inst>) -> Self {
         EmitState {
             virtual_sp_offset: 0,
+            nominal_sp_to_fp: abi.frame_size() as i64,
+            stackmap: None,
         }
     }
+
+    fn pre_safepoint(&mut self, stackmap: Stackmap) {
+        self.stackmap = Some(stackmap);
+    }
+}
+
+impl EmitState {
+    fn take_stackmap(&mut self) -> Option<Stackmap> {
+        self.stackmap.take()
+    }
+
+    fn clear_post_insn(&mut self) {
+        self.stackmap = None;
+    }
 }
 
 /// A label-use (internal relocation) in generated code.

cranelift/codegen/src/isa/x64/lower.rs

@@ -37,7 +37,8 @@ type Ctx<'a> = &'a mut dyn LowerCtx<I = Inst>;
 
 fn is_int_ty(ty: Type) -> bool {
     match ty {
-        types::I8 | types::I16 | types::I32 | types::I64 => true,
+        types::I8 | types::I16 | types::I32 | types::I64 | types::R64 => true,
+        types::R32 => panic!("shouldn't have 32-bits refs on x64"),
         _ => false,
     }
 }
@@ -45,6 +46,7 @@ fn is_int_ty(ty: Type) -> bool {
 fn is_bool_ty(ty: Type) -> bool {
     match ty {
         types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true,
+        types::R32 => panic!("shouldn't have 32-bits refs on x64"),
         _ => false,
     }
 }
@@ -52,6 +54,7 @@ fn is_bool_ty(ty: Type) -> bool {
 fn is_float_ty(ty: Type) -> bool {
     match ty {
         types::F32 | types::F64 => true,
+        types::R32 => panic!("shouldn't have 32-bits refs on x64"),
         _ => false,
     }
 }
@@ -329,7 +332,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
     };
 
     match op {
-        Opcode::Iconst => {
+        Opcode::Iconst | Opcode::Null => {
             if let Some(w64) = iri_to_u64_imm(ctx, insn) {
                 let dst_is_64 = w64 > 0x7fffffff;
                 let dst = output_to_reg(ctx, outputs[0]);
@@ -771,6 +774,23 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             }
         }
 
+        Opcode::IsNull | Opcode::IsInvalid => {
+            // Null references are represented by the constant value 0; invalid references are
+            // represented by the constant value -1. See `define_reftypes()` in
+            // `meta/src/isa/x86/encodings.rs` to confirm.
+            let src = input_to_reg(ctx, inputs[0]);
+            let dst = output_to_reg(ctx, outputs[0]);
+            let ty = ctx.input_ty(insn, 0);
+            let imm = match op {
+                // TODO could use tst src, src for IsNull
+                Opcode::IsNull => 0,
+                Opcode::IsInvalid => 0xffffffff,
+                _ => unreachable!(),
+            };
+            ctx.emit(Inst::cmp_rmi_r(ty.bytes() as u8, RegMemImm::imm(imm), src));
+            ctx.emit(Inst::setcc(CC::Z, dst));
+        }
+
         Opcode::Uextend
         | Opcode::Sextend
         | Opcode::Bint
@@ -979,7 +999,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::Trap | Opcode::ResumableTrap => {
             let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap());
-            ctx.emit(Inst::Ud2 { trap_info })
+            ctx.emit_safepoint(Inst::Ud2 { trap_info });
         }
 
         Opcode::Trapif | Opcode::Trapff => {
@@ -1033,7 +1053,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 cc
             };
 
-            ctx.emit(Inst::TrapIf {
+            ctx.emit_safepoint(Inst::TrapIf {
                 trap_code,
                 srcloc,
                 cc,
@@ -1658,7 +1678,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
             let ty = ctx.output_ty(insn, 0);
 
-            if ty.is_int() {
+            if is_int_ty(ty) {
                 let size = ty.bytes() as u8;
                 if size == 1 {
                     // Sign-extend operands to 32, then do a cmove of size 4.

cranelift/codegen/src/machinst/buffer.rs

@@ -1174,6 +1174,20 @@ impl<I: VCodeInst> MachBuffer<I> {
             stackmap,
         });
     }
+
+    /// Add stackmap metadata for this program point: a set of stack offsets
+    /// (from SP upward) that contain live references.
+    ///
+    /// Method variant of `add_stackmap` that is easier to manipulate for non-fixed-sizes
+    /// instructions.
+    pub fn add_stackmap_ending_at(&mut self, start_offset: CodeOffset, stackmap: Stackmap) {
+        let cur_offset = self.cur_offset();
+        self.stackmaps.push(MachStackMap {
+            offset: start_offset,
+            offset_end: cur_offset,
+            stackmap,
+        });
+    }
 }
 
 impl MachBufferFinalized {