Initial reftype support in aarch64, modulo safepoints.

This commit adds the inital support to allow reftypes to flow through
the program when targetting aarch64. It also adds a fix to the
`ModuleTranslationState` needed to send R32/R64 types over from the
SpiderMonkey embedding.

This commit does not include any support for safepoints in aarch64
or the `MachInst` infrastructure; that is in the next commit.

This commit also makes a drive-by improvement to `Bint`, avoiding an
unneeded zero-extension op when the extended value comes directly from a
conditional-set (which produces a full-width 0 or 1).

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -400,6 +400,7 @@ fn in_int_reg(ty: ir::Type) -> bool {
     match ty {
         types::I8 | types::I16 | types::I32 | types::I64 => true,
         types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true,
+        types::R32 | types::R64 => true,
         _ => false,
     }
 }
@@ -653,12 +654,12 @@ fn load_stack(mem: MemArg, into_reg: Writable<Reg>, ty: Type) -> Inst {
             mem,
             srcloc: None,
         },
-        types::B32 | types::I32 => Inst::ULoad32 {
+        types::B32 | types::I32 | types::R32 => Inst::ULoad32 {
             rd: into_reg,
             mem,
             srcloc: None,
         },
-        types::B64 | types::I64 => Inst::ULoad64 {
+        types::B64 | types::I64 | types::R64 => Inst::ULoad64 {
             rd: into_reg,
             mem,
             srcloc: None,
@@ -689,12 +690,12 @@ fn store_stack(mem: MemArg, from_reg: Reg, ty: Type) -> Inst {
             mem,
             srcloc: None,
         },
-        types::B32 | types::I32 => Inst::Store32 {
+        types::B32 | types::I32 | types::R32 => Inst::Store32 {
             rd: from_reg,
             mem,
             srcloc: None,
         },
-        types::B64 | types::I64 => Inst::Store64 {
+        types::B64 | types::I64 | types::R64 => Inst::Store64 {
             rd: from_reg,
             mem,
             srcloc: None,

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -6,7 +6,7 @@
 use crate::binemit::CodeOffset;
 use crate::ir::types::{
     B1, B16, B16X8, B32, B32X4, B64, B64X2, B8, B8X16, F32, F32X2, F32X4, F64, F64X2, FFLAGS, I16,
-    I16X4, I16X8, I32, I32X2, I32X4, I64, I64X2, I8, I8X16, I8X8, IFLAGS,
+    I16X4, I16X8, I32, I32X2, I32X4, I64, I64X2, I8, I8X16, I8X8, IFLAGS, R32, R64,
 };
 use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type};
 use crate::machinst::*;
@@ -2081,6 +2081,8 @@ impl MachInst for Inst {
                     || ty == B32
                     || ty == I64
                     || ty == B64
+                    || ty == R32
+                    || ty == R64
             );
             Inst::load_constant(to_reg, value)
         }
@@ -2102,7 +2104,7 @@ impl MachInst for Inst {
 
     fn rc_for_type(ty: Type) -> CodegenResult<RegClass> {
         match ty {
-            I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 => Ok(RegClass::I64),
+            I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 | R32 | R64 => Ok(RegClass::I64),
             F32 | F64 => Ok(RegClass::V128),
             IFLAGS | FFLAGS => Ok(RegClass::I64),
             B8X16 | I8X16 | B16X8 | I16X8 | B32X4 | I32X4 | B64X2 | I64X2 | F32X4 | F64X2 => {

cranelift/codegen/src/isa/aarch64/lower.rs

@@ -829,8 +829,8 @@ pub fn ty_bits(ty: Type) -> usize {
         B1 => 1,
         B8 | I8 => 8,
         B16 | I16 => 16,
-        B32 | I32 | F32 => 32,
+        B32 | I32 | F32 | R32 => 32,
-        B64 | I64 | F64 => 64,
+        B64 | I64 | F64 | R64 => 64,
         B128 | I128 => 128,
         IFLAGS | FFLAGS => 32,
         B8X8 | I8X8 | B16X4 | I16X4 | B32X2 | I32X2 => 64,
@@ -842,7 +842,7 @@ pub fn ty_bits(ty: Type) -> usize {
 
 pub(crate) fn ty_is_int(ty: Type) -> bool {
     match ty {
-        B1 | B8 | I8 | B16 | I16 | B32 | I32 | B64 | I64 => true,
+        B1 | B8 | I8 | B16 | I16 | B32 | I32 | B64 | I64 | R32 | R64 => true,
         F32 | F64 | B128 | I128 | I8X8 | I8X16 | I16X4 | I16X8 | I32X2 | I32X4 | I64X2 => false,
         IFLAGS | FFLAGS => panic!("Unexpected flags type"),
         _ => panic!("ty_is_int() on unknown type: {:?}", ty),

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -1204,7 +1204,26 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::IsNull | Opcode::IsInvalid => {
-            panic!("Reference types not supported");
+            // Null references are represented by the constant value 0; invalid references are
+            // represented by the constant value -1. See `define_reftypes()` in
+            // `meta/src/isa/x86/encodings.rs` to confirm.
+            let rd = get_output_reg(ctx, outputs[0]);
+            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
+            let ty = ctx.input_ty(insn, 0);
+            let (alu_op, const_value) = match op {
+                Opcode::IsNull => {
+                    // cmp rn, #0
+                    (choose_32_64(ty, ALUOp::SubS32, ALUOp::SubS64), 0)
+                }
+                Opcode::IsInvalid => {
+                    // cmn rn, #1
+                    (choose_32_64(ty, ALUOp::AddS32, ALUOp::AddS64), 1)
+                }
+                _ => unreachable!(),
+            };
+            let const_value = ResultRSEImm12::Imm12(Imm12::maybe_from_u64(const_value).unwrap());
+            ctx.emit(alu_inst_imm12(alu_op, writable_zero_reg(), rn, const_value));
+            ctx.emit(Inst::CSet { rd, cond: Cond::Eq });
         }
 
         Opcode::Copy => {
@@ -1215,6 +1234,21 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Bint | Opcode::Breduce | Opcode::Bextend | Opcode::Ireduce => {
+            // If this is a Bint from a Trueif/Trueff/IsNull/IsInvalid, then the result is already
+            // 64-bit-zero-extended, even if the CLIF type doesn't say so, because it was produced
+            // by a CSet. In this case, we do not need to do any zero-extension.
+            let input_info = ctx.get_input(insn, 0);
+            let src_op = input_info
+                .inst
+                .map(|(src_inst, _)| ctx.data(src_inst).opcode());
+            let narrow_mode = match (src_op, op) {
+                (Some(Opcode::Trueif), Opcode::Bint)
+                | (Some(Opcode::Trueff), Opcode::Bint)
+                | (Some(Opcode::IsNull), Opcode::Bint)
+                | (Some(Opcode::IsInvalid), Opcode::Bint) => NarrowValueMode::None,
+                _ => NarrowValueMode::ZeroExtend64,
+            };
+
             // All of these ops are simply a move from a zero-extended source.
             // Here is why this works, in each case:
             //
@@ -1227,7 +1261,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // - Ireduce: changing width of an integer. Smaller ints are stored
             //   with undefined high-order bits, so we can simply do a copy.
 
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64);
+            let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
             let rd = get_output_reg(ctx, outputs[0]);
             let ty = ctx.input_ty(insn, 0);
             ctx.emit(Inst::gen_move(rd, rn, ty));

cranelift/filetests/filetests/vcode/aarch64/reftypes.clif

@@ -0,0 +1,65 @@
+test compile
+target aarch64
+
+function %f0(r64) -> r64 {
+block0(v0: r64):
+  return v0
+}
+
+; check: stp fp, lr, [sp, #-16]!
+; nextln: mov fp, sp
+; nextln: mov sp, fp
+; nextln: ldp fp, lr, [sp], #16
+; nextln: ret
+
+function %f1(r32) -> r32 {
+block0(v0: r32):
+  return v0
+}
+
+; check: stp fp, lr, [sp, #-16]!
+; nextln: mov fp, sp
+; nextln: mov sp, fp
+; nextln: ldp fp, lr, [sp], #16
+; nextln: ret
+
+function %f2(r64) -> b1 {
+block0(v0: r64):
+  v1 = is_null v0
+  return v1
+}
+
+; check: stp fp, lr, [sp, #-16]!
+; nextln: mov fp, sp
+; nextln: subs xzr, x0, #0
+; nextln: cset x0, eq
+; nextln: mov sp, fp
+; nextln: ldp fp, lr, [sp], #16
+; nextln: ret
+
+function %f3(r64) -> b1 {
+block0(v0: r64):
+  v1 = is_invalid v0
+  return v1
+}
+
+; check: stp fp, lr, [sp, #-16]!
+; nextln: mov fp, sp
+; nextln: adds xzr, x0, #1
+; nextln: cset x0, eq
+; nextln: mov sp, fp
+; nextln: ldp fp, lr, [sp], #16
+; nextln: ret
+
+function %f4() -> r64 {
+block0:
+  v0 = null.r64
+  return v0
+}
+
+; check: stp fp, lr, [sp, #-16]!
+; nextln: mov fp, sp
+; nextln: movz x0, #0
+; nextln: mov sp, fp
+; nextln: ldp fp, lr, [sp], #16
+; nextln: ret

cranelift/wasm/src/state/module_state.rs

@@ -30,6 +30,7 @@ fn cranelift_to_wasmparser_type(ty: Type) -> WasmResult<wasmparser::Type> {
         types::I64 => wasmparser::Type::I64,
         types::F32 => wasmparser::Type::F32,
         types::F64 => wasmparser::Type::F64,
+        types::R32 | types::R64 => wasmparser::Type::ExternRef,
         _ => {
             return Err(WasmError::Unsupported(format!(
                 "Cannot convert Cranelift type to Wasm signature: {:?}",