Multi-register value support: framework for Values wider than machine regs.

This will allow for support for `I128` values everywhere, and `I64`
values on 32-bit targets (e.g., ARM32 and x86-32). It does not alter the
machine backends to build such support; it just adds the framework for
the MachInst backends to *reason* about a `Value` residing in more than
one register.

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -77,7 +77,7 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
             &ir::ArgumentPurpose::VMContext => {
                 // This is SpiderMonkey's `WasmTlsReg`.
                 Some(ABIArg::Reg(
-                    xreg(BALDRDASH_TLS_REG).to_real_reg(),
+                    ValueRegs::one(xreg(BALDRDASH_TLS_REG).to_real_reg()),
                     ir::types::I64,
                     param.extension,
                     param.purpose,
@@ -86,7 +86,7 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
             &ir::ArgumentPurpose::SignatureId => {
                 // This is SpiderMonkey's `WasmTableCallSigReg`.
                 Some(ABIArg::Reg(
-                    xreg(BALDRDASH_SIG_REG).to_real_reg(),
+                    ValueRegs::one(xreg(BALDRDASH_SIG_REG).to_real_reg()),
                     ir::types::I64,
                     param.extension,
                     param.purpose,
@@ -220,7 +220,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
                 "Invalid type for AArch64: {:?}",
                 param.value_type
             );
-            let rc = Inst::rc_for_type(param.value_type).unwrap();
+            let (rcs, _) = Inst::rc_for_type(param.value_type).unwrap();
+            assert!(rcs.len() == 1, "Multi-reg values not supported yet");
+            let rc = rcs[0];
 
             let next_reg = match rc {
                 RegClass::I64 => &mut next_xreg,
@@ -238,7 +240,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                     _ => unreachable!(),
                 };
                 ret.push(ABIArg::Reg(
-                    reg.to_real_reg(),
+                    ValueRegs::one(reg.to_real_reg()),
                     param.value_type,
                     param.extension,
                     param.purpose,
@@ -271,7 +273,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
             if next_xreg < max_per_class_reg_vals && remaining_reg_vals > 0 {
                 ret.push(ABIArg::Reg(
-                    xreg(next_xreg).to_real_reg(),
+                    ValueRegs::one(xreg(next_xreg).to_real_reg()),
                     I64,
                     ir::ArgumentExtension::None,
                     ir::ArgumentPurpose::Normal,
@@ -345,7 +347,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         Inst::Ret
     }
 
-    fn gen_add_imm(into_reg: Writable<Reg>, from_reg: Reg, imm: u32) -> SmallVec<[Inst; 4]> {
+    fn gen_add_imm(into_reg: Writable<Reg>, from_reg: Reg, imm: u32) -> SmallInstVec<Inst> {
         let imm = imm as u64;
         let mut insts = SmallVec::new();
         if let Some(imm12) = Imm12::maybe_from_u64(imm) {
@@ -370,7 +372,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         insts
     }
 
-    fn gen_stack_lower_bound_trap(limit_reg: Reg) -> SmallVec<[Inst; 2]> {
+    fn gen_stack_lower_bound_trap(limit_reg: Reg) -> SmallInstVec<Inst> {
         let mut insts = SmallVec::new();
         insts.push(Inst::AluRRRExtend {
             alu_op: ALUOp::SubS64,
@@ -411,7 +413,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         Inst::gen_store(mem, from_reg, ty, MemFlags::trusted())
     }
 
-    fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Inst; 2]> {
+    fn gen_sp_reg_adjust(amount: i32) -> SmallInstVec<Inst> {
         if amount == 0 {
             return SmallVec::new();
         }
@@ -455,7 +457,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         }
     }
 
-    fn gen_prologue_frame_setup() -> SmallVec<[Inst; 2]> {
+    fn gen_prologue_frame_setup() -> SmallInstVec<Inst> {
         let mut insts = SmallVec::new();
         // stp fp (x29), lr (x30), [sp, #-16]!
         insts.push(Inst::StoreP64 {
@@ -481,7 +483,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         insts
     }
 
-    fn gen_epilogue_frame_restore() -> SmallVec<[Inst; 2]> {
+    fn gen_epilogue_frame_restore() -> SmallInstVec<Inst> {
         let mut insts = SmallVec::new();
 
         // MOV (alias of ORR) interprets x31 as XZR, so use an ADD here.
@@ -508,7 +510,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         insts
     }
 
-    fn gen_probestack(_: u32) -> SmallVec<[Self::I; 2]> {
+    fn gen_probestack(_: u32) -> SmallInstVec<Self::I> {
         // TODO: implement if we ever require stack probes on an AArch64 host
         // (unlikely unless Lucet is ported)
         smallvec![]

cranelift/codegen/src/isa/aarch64/inst/args.rs

@@ -3,7 +3,7 @@
 // Some variants are never constructed, but we still want them as options in the future.
 #![allow(dead_code)]
 
-use crate::ir::types::{F32X2, F32X4, F64X2, I16X4, I16X8, I32X2, I32X4, I64X2, I8X16, I8X8};
+use crate::ir::types::*;
 use crate::ir::Type;
 use crate::isa::aarch64::inst::*;
 use crate::machinst::{ty_bits, MachLabel};

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -5,9 +5,7 @@
 
 use crate::binemit::CodeOffset;
 use crate::ir::types::{
-    B1, B16, B16X4, B16X8, B32, B32X2, B32X4, B64, B64X2, B8, B8X16, B8X8, F32, F32X2, F32X4, F64,
+    B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64,
-    F64X2, FFLAGS, I16, I16X4, I16X8, I32, I32X2, I32X4, I64, I64X2, I8, I8X16, I8X8, IFLAGS, R32,
-    R64,
 };
 use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type};
 use crate::isa::CallConv;
@@ -1304,7 +1302,7 @@ impl Inst {
     }
 
     /// Create instructions that load a 32-bit floating-point constant.
-    pub fn load_fp_constant32<F: FnMut(RegClass, Type) -> Writable<Reg>>(
+    pub fn load_fp_constant32<F: FnMut(Type) -> Writable<Reg>>(
         rd: Writable<Reg>,
         value: u32,
         mut alloc_tmp: F,
@@ -1322,7 +1320,7 @@ impl Inst {
         } else {
             // TODO: use FMOV immediate form when `value` has sufficiently few mantissa/exponent
             // bits.
-            let tmp = alloc_tmp(RegClass::I64, I32);
+            let tmp = alloc_tmp(I32);
             let mut insts = Inst::load_constant(tmp, value as u64);
 
             insts.push(Inst::MovToFpu {
@@ -1336,7 +1334,7 @@ impl Inst {
     }
 
     /// Create instructions that load a 64-bit floating-point constant.
-    pub fn load_fp_constant64<F: FnMut(RegClass, Type) -> Writable<Reg>>(
+    pub fn load_fp_constant64<F: FnMut(Type) -> Writable<Reg>>(
         rd: Writable<Reg>,
         const_data: u64,
         mut alloc_tmp: F,
@@ -1350,7 +1348,7 @@ impl Inst {
         // bits.  Also, treat it as half of a 128-bit vector and consider replicated
         // patterns. Scalar MOVI might also be an option.
         } else if const_data & (u32::MAX as u64) == 0 {
-            let tmp = alloc_tmp(RegClass::I64, I64);
+            let tmp = alloc_tmp(I64);
             let mut insts = Inst::load_constant(tmp, const_data);
 
             insts.push(Inst::MovToFpu {
@@ -1366,7 +1364,7 @@ impl Inst {
     }
 
     /// Create instructions that load a 128-bit vector constant.
-    pub fn load_fp_constant128<F: FnMut(RegClass, Type) -> Writable<Reg>>(
+    pub fn load_fp_constant128<F: FnMut(Type) -> Writable<Reg>>(
         rd: Writable<Reg>,
         const_data: u128,
         alloc_tmp: F,
@@ -1416,7 +1414,7 @@ impl Inst {
 
     /// Create instructions that load a vector constant consisting of elements with
     /// the same value.
-    pub fn load_replicated_vector_pattern<F: FnMut(RegClass, Type) -> Writable<Reg>>(
+    pub fn load_replicated_vector_pattern<F: FnMut(Type) -> Writable<Reg>>(
         rd: Writable<Reg>,
         pattern: u64,
         size: VectorSize,
@@ -1472,7 +1470,7 @@ impl Inst {
         } else if let Some(imm) = ASIMDFPModImm::maybe_from_u64(pattern, lane_size) {
             smallvec![Inst::VecDupFPImm { rd, imm, size }]
         } else {
-            let tmp = alloc_tmp(RegClass::I64, I64);
+            let tmp = alloc_tmp(I64);
             let mut insts = SmallVec::from(&Inst::load_constant(tmp, pattern)[..]);
 
             insts.push(Inst::VecDup {
@@ -2862,12 +2860,16 @@ impl MachInst for Inst {
         }
     }
 
-    fn gen_constant<F: FnMut(RegClass, Type) -> Writable<Reg>>(
+    fn gen_constant<F: FnMut(Type) -> Writable<Reg>>(
-        to_reg: Writable<Reg>,
+        to_regs: ValueRegs<Writable<Reg>>,
-        value: u64,
+        value: u128,
         ty: Type,
         alloc_tmp: F,
     ) -> SmallVec<[Inst; 4]> {
+        let to_reg = to_regs
+            .only_reg()
+            .expect("multi-reg values not supported yet");
+        let value = value as u64;
         if ty == F64 {
             Inst::load_fp_constant64(to_reg, value, alloc_tmp)
         } else if ty == F32 {
@@ -2905,14 +2907,28 @@ impl MachInst for Inst {
         None
     }
 
-    fn rc_for_type(ty: Type) -> CodegenResult<RegClass> {
+    fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
         match ty {
-            I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 | R32 | R64 => Ok(RegClass::I64),
+            I8 => Ok((&[RegClass::I64], &[I8])),
-            F32 | F64 => Ok(RegClass::V128),
+            I16 => Ok((&[RegClass::I64], &[I16])),
-            IFLAGS | FFLAGS => Ok(RegClass::I64),
+            I32 => Ok((&[RegClass::I64], &[I32])),
-            B8X8 | B8X16 | B16X4 | B16X8 | B32X2 | B32X4 | B64X2 => Ok(RegClass::V128),
+            I64 => Ok((&[RegClass::I64], &[I64])),
-            F32X2 | I8X8 | I16X4 | I32X2 => Ok(RegClass::V128),
+            B1 => Ok((&[RegClass::I64], &[B1])),
-            F32X4 | F64X2 | I8X16 | I16X8 | I32X4 | I64X2 => Ok(RegClass::V128),
+            B8 => Ok((&[RegClass::I64], &[B8])),
+            B16 => Ok((&[RegClass::I64], &[B16])),
+            B32 => Ok((&[RegClass::I64], &[B32])),
+            B64 => Ok((&[RegClass::I64], &[B64])),
+            R32 => panic!("32-bit reftype pointer should never be seen on AArch64"),
+            R64 => Ok((&[RegClass::I64], &[R64])),
+            F32 => Ok((&[RegClass::V128], &[F32])),
+            F64 => Ok((&[RegClass::V128], &[F64])),
+            I128 => Ok((&[RegClass::I64, RegClass::I64], &[I64, I64])),
+            B128 => Ok((&[RegClass::I64, RegClass::I64], &[B64, B64])),
+            _ if ty.is_vector() => {
+                assert!(ty.bits() <= 128);
+                Ok((&[RegClass::V128], &[I8X16]))
+            }
+            IFLAGS | FFLAGS => Ok((&[RegClass::I64], &[I64])),
             _ => Err(CodegenError::Unsupported(format!(
                 "Unexpected SSA-value type: {}",
                 ty

cranelift/codegen/src/isa/aarch64/lower.rs

@@ -22,7 +22,7 @@ use super::lower_inst;
 
 use crate::data_value::DataValue;
 use log::{debug, trace};
-use regalloc::{Reg, RegClass, Writable};
+use regalloc::{Reg, Writable};
 use smallvec::SmallVec;
 
 //============================================================================
@@ -179,9 +179,9 @@ pub(crate) fn put_input_in_reg<C: LowerCtx<I = Inst>>(
         } else {
             c
         };
-        let to_reg = ctx.alloc_tmp(Inst::rc_for_type(ty).unwrap(), ty);
+        let to_reg = ctx.alloc_tmp(ty).only_reg().unwrap();
-        for inst in Inst::gen_constant(to_reg, masked, ty, |reg_class, ty| {
+        for inst in Inst::gen_constant(ValueRegs::one(to_reg), masked as u128, ty, |ty| {
-            ctx.alloc_tmp(reg_class, ty)
+            ctx.alloc_tmp(ty).only_reg().unwrap()
         })
         .into_iter()
         {
@@ -189,13 +189,15 @@ pub(crate) fn put_input_in_reg<C: LowerCtx<I = Inst>>(
         }
         to_reg.to_reg()
     } else {
-        ctx.put_input_in_reg(input.insn, input.input)
+        ctx.put_input_in_regs(input.insn, input.input)
+            .only_reg()
+            .unwrap()
     };
 
     match (narrow_mode, from_bits) {
         (NarrowValueMode::None, _) => in_reg,
         (NarrowValueMode::ZeroExtend32, n) if n < 32 => {
-            let tmp = ctx.alloc_tmp(RegClass::I64, I32);
+            let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
             ctx.emit(Inst::Extend {
                 rd: tmp,
                 rn: in_reg,
@@ -206,7 +208,7 @@ pub(crate) fn put_input_in_reg<C: LowerCtx<I = Inst>>(
             tmp.to_reg()
         }
         (NarrowValueMode::SignExtend32, n) if n < 32 => {
-            let tmp = ctx.alloc_tmp(RegClass::I64, I32);
+            let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
             ctx.emit(Inst::Extend {
                 rd: tmp,
                 rn: in_reg,
@@ -223,7 +225,7 @@ pub(crate) fn put_input_in_reg<C: LowerCtx<I = Inst>>(
                 // Constants are zero-extended to full 64-bit width on load already.
                 in_reg
             } else {
-                let tmp = ctx.alloc_tmp(RegClass::I64, I32);
+                let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
                 ctx.emit(Inst::Extend {
                     rd: tmp,
                     rn: in_reg,
@@ -235,7 +237,7 @@ pub(crate) fn put_input_in_reg<C: LowerCtx<I = Inst>>(
             }
         }
         (NarrowValueMode::SignExtend64, n) if n < 64 => {
-            let tmp = ctx.alloc_tmp(RegClass::I64, I32);
+            let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
             ctx.emit(Inst::Extend {
                 rd: tmp,
                 rn: in_reg,
@@ -696,7 +698,7 @@ pub(crate) fn lower_address<C: LowerCtx<I = Inst>>(
     /* addends64.len() == 0 */
     {
         if addends32.len() > 0 {
-            let tmp = ctx.alloc_tmp(RegClass::I64, I64);
+            let tmp = ctx.alloc_tmp(I64).only_reg().unwrap();
             let (reg1, extendop) = addends32.pop().unwrap();
             let signed = match extendop {
                 ExtendOp::SXTW => true,
@@ -718,7 +720,7 @@ pub(crate) fn lower_address<C: LowerCtx<I = Inst>>(
         } else
         /* addends32.len() == 0 */
         {
-            let off_reg = ctx.alloc_tmp(RegClass::I64, I64);
+            let off_reg = ctx.alloc_tmp(I64).only_reg().unwrap();
             lower_constant_u64(ctx, off_reg, offset as u64);
             offset = 0;
             AMode::reg(off_reg.to_reg())
@@ -734,7 +736,7 @@ pub(crate) fn lower_address<C: LowerCtx<I = Inst>>(
     }
 
     // Allocate the temp and shoehorn it into the AMode.
-    let addr = ctx.alloc_tmp(RegClass::I64, I64);
+    let addr = ctx.alloc_tmp(I64).only_reg().unwrap();
     let (reg, memarg) = match memarg {
         AMode::RegExtended(r1, r2, extendop) => {
             (r1, AMode::RegExtended(addr.to_reg(), r2, extendop))
@@ -782,7 +784,7 @@ pub(crate) fn lower_address<C: LowerCtx<I = Inst>>(
         // If the register is the stack reg, we must move it to another reg
         // before adding it.
         let reg = if reg == stack_reg() {
-            let tmp = ctx.alloc_tmp(RegClass::I64, I64);
+            let tmp = ctx.alloc_tmp(I64).only_reg().unwrap();
             ctx.emit(Inst::gen_move(tmp, stack_reg(), I64));
             tmp.to_reg()
         } else {
@@ -824,7 +826,7 @@ pub(crate) fn lower_constant_f32<C: LowerCtx<I = Inst>>(
     rd: Writable<Reg>,
     value: f32,
 ) {
-    let alloc_tmp = |class, ty| ctx.alloc_tmp(class, ty);
+    let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap();
 
     for inst in Inst::load_fp_constant32(rd, value.to_bits(), alloc_tmp) {
         ctx.emit(inst);
@@ -836,7 +838,7 @@ pub(crate) fn lower_constant_f64<C: LowerCtx<I = Inst>>(
     rd: Writable<Reg>,
     value: f64,
 ) {
-    let alloc_tmp = |class, ty| ctx.alloc_tmp(class, ty);
+    let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap();
 
     for inst in Inst::load_fp_constant64(rd, value.to_bits(), alloc_tmp) {
         ctx.emit(inst);
@@ -858,7 +860,7 @@ pub(crate) fn lower_constant_f128<C: LowerCtx<I = Inst>>(
             size: VectorSize::Size8x16,
         });
     } else {
-        let alloc_tmp = |class, ty| ctx.alloc_tmp(class, ty);
+        let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap();
         for inst in Inst::load_fp_constant128(rd, value, alloc_tmp) {
             ctx.emit(inst);
         }
@@ -885,7 +887,7 @@ pub(crate) fn lower_splat_const<C: LowerCtx<I = Inst>>(
         ),
         None => (value, size),
     };
-    let alloc_tmp = |class, ty| ctx.alloc_tmp(class, ty);
+    let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap();
 
     for inst in Inst::load_replicated_vector_pattern(rd, value, size, alloc_tmp) {
         ctx.emit(inst);
@@ -1217,7 +1219,7 @@ pub(crate) fn lower_load<C: LowerCtx<I = Inst>, F: FnMut(&mut C, Writable<Reg>,
 
     let off = ctx.data(ir_inst).load_store_offset().unwrap();
     let mem = lower_address(ctx, elem_ty, &inputs[..], off);
-    let rd = get_output_reg(ctx, output);
+    let rd = get_output_reg(ctx, output).only_reg().unwrap();
 
     f(ctx, rd, elem_ty, mem);
 }

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -12,7 +12,7 @@ use crate::{CodegenError, CodegenResult};
 use crate::isa::aarch64::abi::*;
 use crate::isa::aarch64::inst::*;
 
-use regalloc::{RegClass, Writable};
+use regalloc::Writable;
 
 use alloc::boxed::Box;
 use alloc::vec::Vec;
@@ -46,21 +46,21 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 ty if ty.is_bool() => value,
                 ty => unreachable!("Unknown type for const: {}", ty),
             };
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             lower_constant_u64(ctx, rd, value);
         }
         Opcode::F32const => {
             let value = f32::from_bits(ctx.get_constant(insn).unwrap() as u32);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             lower_constant_f32(ctx, rd, value);
         }
         Opcode::F64const => {
             let value = f64::from_bits(ctx.get_constant(insn).unwrap());
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             lower_constant_f64(ctx, rd, value);
         }
         Opcode::Iadd => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ty.unwrap();
             if !ty.is_vector() {
                 let mul_insn =
@@ -116,7 +116,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             }
         }
         Opcode::Isub => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let ty = ty.unwrap();
             if !ty.is_vector() {
@@ -148,7 +148,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // (SQADD / UQADD / SQSUB / UQSUB), which require scalar FP registers.
             let is_signed = op == Opcode::SaddSat || op == Opcode::SsubSat;
             let ty = ty.unwrap();
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             if !ty.is_vector() {
                 let narrow_mode = if is_signed {
                     NarrowValueMode::SignExtend64
@@ -162,8 +162,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     Opcode::SsubSat => FPUOp2::Sqsub64,
                     _ => unreachable!(),
                 };
-                let va = ctx.alloc_tmp(RegClass::V128, I128);
+                let va = ctx.alloc_tmp(I8X16).only_reg().unwrap();
-                let vb = ctx.alloc_tmp(RegClass::V128, I128);
+                let vb = ctx.alloc_tmp(I8X16).only_reg().unwrap();
                 let ra = put_input_in_reg(ctx, inputs[0], narrow_mode);
                 let rb = put_input_in_reg(ctx, inputs[1], narrow_mode);
                 ctx.emit(Inst::MovToFpu {
@@ -211,7 +211,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Ineg => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ty.unwrap();
             if !ty.is_vector() {
                 let rn = zero_reg();
@@ -230,7 +230,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Imul => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let ty = ty.unwrap();
@@ -245,8 +245,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 });
             } else {
                 if ty == I64X2 {
-                    let tmp1 = ctx.alloc_tmp(RegClass::V128, I64X2);
+                    let tmp1 = ctx.alloc_tmp(I64X2).only_reg().unwrap();
-                    let tmp2 = ctx.alloc_tmp(RegClass::V128, I64X2);
+                    let tmp2 = ctx.alloc_tmp(I64X2).only_reg().unwrap();
 
                     // This I64X2 multiplication is performed with several 32-bit
                     // operations.
@@ -362,7 +362,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Umulhi | Opcode::Smulhi => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let is_signed = op == Opcode::Smulhi;
             let input_ty = ctx.input_ty(insn, 0);
             assert!(ctx.input_ty(insn, 1) == input_ty);
@@ -443,7 +443,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 ALUOp::UDiv64
             };
 
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
             let rm = put_input_in_reg(ctx, inputs[1], narrow_mode);
             // The div instruction does not trap on divide by zero or signed overflow
@@ -550,7 +550,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             assert!(from_bits <= to_bits);
             if from_bits < to_bits {
                 let signed = op == Opcode::Sextend;
-                let rd = get_output_reg(ctx, outputs[0]);
+                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
 
                 if let Some(extract_insn) = maybe_input_insn(ctx, inputs[0], Opcode::Extractlane) {
                     let idx =
@@ -596,7 +596,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Bnot => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ty.unwrap();
             if !ty.is_vector() {
                 let rm = put_input_in_rs_immlogic(ctx, inputs[0], NarrowValueMode::None);
@@ -620,7 +620,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         | Opcode::BandNot
         | Opcode::BorNot
         | Opcode::BxorNot => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ty.unwrap();
             if !ty.is_vector() {
                 let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
@@ -646,7 +646,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
                 let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
                 let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                let rd = get_output_reg(ctx, outputs[0]);
+                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
 
                 ctx.emit(Inst::VecRRR {
                     alu_op,
@@ -660,7 +660,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => {
             let ty = ty.unwrap();
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             if !ty.is_vector() {
                 let size = OperandSize::from_bits(ty_bits(ty));
                 let narrow_mode = match (op, size) {
@@ -692,7 +692,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
                 let rm = if is_right_shift {
                     // Right shifts are implemented with a negative left shift.
-                    let tmp = ctx.alloc_tmp(RegClass::I64, I32);
+                    let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
                     let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
                     let rn = zero_reg();
                     ctx.emit(Inst::AluRRR {
@@ -751,7 +751,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let ty = ty.unwrap();
             let ty_bits_size = ty_bits(ty) as u8;
 
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(
                 ctx,
                 inputs[0],
@@ -785,7 +785,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                             // ignored (because of the implicit masking done by the instruction),
                             // so this is equivalent to negating the input.
                             let alu_op = choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64);
-                            let tmp = ctx.alloc_tmp(RegClass::I64, ty);
+                            let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
                             ctx.emit(Inst::AluRRR {
                                 alu_op,
                                 rd: tmp,
@@ -808,7 +808,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                             // Really ty_bits_size - rn, but the upper bits of the result are
                             // ignored (because of the implicit masking done by the instruction),
                             // so this is equivalent to negating the input.
-                            let tmp = ctx.alloc_tmp(RegClass::I64, I32);
+                            let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
                             ctx.emit(Inst::AluRRR {
                                 alu_op: ALUOp::Sub32,
                                 rd: tmp,
@@ -821,7 +821,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                         };
 
                         // Explicitly mask the rotation count.
-                        let tmp_masked_rm = ctx.alloc_tmp(RegClass::I64, I32);
+                        let tmp_masked_rm = ctx.alloc_tmp(I32).only_reg().unwrap();
                         ctx.emit(Inst::AluRRImmLogic {
                             alu_op: ALUOp::And32,
                             rd: tmp_masked_rm,
@@ -830,8 +830,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                         });
                         let tmp_masked_rm = tmp_masked_rm.to_reg();
 
-                        let tmp1 = ctx.alloc_tmp(RegClass::I64, I32);
+                        let tmp1 = ctx.alloc_tmp(I32).only_reg().unwrap();
-                        let tmp2 = ctx.alloc_tmp(RegClass::I64, I32);
+                        let tmp2 = ctx.alloc_tmp(I32).only_reg().unwrap();
                         ctx.emit(Inst::AluRRImm12 {
                             alu_op: ALUOp::Sub32,
                             rd: tmp1,
@@ -870,7 +870,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                         }
                         immshift.imm &= ty_bits_size - 1;
 
-                        let tmp1 = ctx.alloc_tmp(RegClass::I64, I32);
+                        let tmp1 = ctx.alloc_tmp(I32).only_reg().unwrap();
                         ctx.emit(Inst::AluRRImmShift {
                             alu_op: ALUOp::Lsr32,
                             rd: tmp1,
@@ -900,7 +900,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Bitrev | Opcode::Clz | Opcode::Cls | Opcode::Ctz => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let needs_zext = match op {
                 Opcode::Bitrev | Opcode::Ctz => false,
                 Opcode::Clz | Opcode::Cls => true,
@@ -970,12 +970,12 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             //   x += x << 32
             //   x >> 56
             let ty = ty.unwrap();
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             // FIXME(#1537): zero-extend 8/16/32-bit operands only to 32 bits,
             // and fix the sequence below to work properly for this.
             let narrow_mode = NarrowValueMode::ZeroExtend64;
             let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-            let tmp = ctx.alloc_tmp(RegClass::I64, I64);
+            let tmp = ctx.alloc_tmp(I64).only_reg().unwrap();
 
             // If this is a 32-bit Popcnt, use Lsr32 to clear the top 32 bits of the register, then
             // the rest of the code is identical to the 64-bit version.
@@ -1236,7 +1236,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 } => (stack_slot, offset),
                 _ => unreachable!(),
             };
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let offset: i32 = offset.into();
             let inst = ctx
                 .abi()
@@ -1245,7 +1245,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::AtomicRmw => {
-            let r_dst = get_output_reg(ctx, outputs[0]);
+            let r_dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let mut r_addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let mut r_arg2 = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let ty_access = ty.unwrap();
@@ -1270,7 +1270,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // This is very similar to, but not identical to, the AtomicRmw case.  Note
             // that the AtomicCAS sequence does its own masking, so we don't need to worry
             // about zero-extending narrow (I8/I16/I32) values here.
-            let r_dst = get_output_reg(ctx, outputs[0]);
+            let r_dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let mut r_addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let mut r_expected = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let mut r_replacement = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
@@ -1301,7 +1301,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::AtomicLoad => {
-            let r_data = get_output_reg(ctx, outputs[0]);
+            let r_data = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let r_addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let ty_access = ty.unwrap();
             assert!(is_valid_atomic_transaction_ty(ty_access));
@@ -1382,7 +1382,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             };
 
             // csel.cond rd, rn, rm
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
             let ty = ctx.output_ty(insn, 0);
@@ -1409,7 +1409,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             lower_icmp_or_ifcmp_to_flags(ctx, ifcmp_insn, is_signed);
 
             // csel.COND rd, rn, rm
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
             let ty = ctx.output_ty(insn, 0);
@@ -1428,8 +1428,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let ty = ty.unwrap();
             if !ty.is_vector() {
                 debug_assert_ne!(Opcode::Vselect, op);
-                let tmp = ctx.alloc_tmp(RegClass::I64, I64);
+                let tmp = ctx.alloc_tmp(I64).only_reg().unwrap();
-                let rd = get_output_reg(ctx, outputs[0]);
+                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
                 let rcond = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
                 let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
                 let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
@@ -1458,7 +1458,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 let rcond = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
                 let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
                 let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
-                let rd = get_output_reg(ctx, outputs[0]);
+                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
                 ctx.emit(Inst::gen_move(rd, rcond, ty));
 
                 ctx.emit(Inst::VecRRR {
@@ -1479,7 +1479,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // single-def ifcmp.
             let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
             lower_icmp_or_ifcmp_to_flags(ctx, ifcmp_insn, is_signed);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             materialize_bool_result(ctx, insn, rd, cond);
         }
 
@@ -1488,7 +1488,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let cond = lower_fp_condcode(condcode);
             let ffcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ffcmp).unwrap();
             lower_fcmp_or_ffcmp_to_flags(ctx, ffcmp_insn);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             materialize_bool_result(ctx, insn, rd, cond);
         }
 
@@ -1496,7 +1496,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // Null references are represented by the constant value 0; invalid references are
             // represented by the constant value -1. See `define_reftypes()` in
             // `meta/src/isa/x86/encodings.rs` to confirm.
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let ty = ctx.input_ty(insn, 0);
             let (alu_op, const_value) = match op {
@@ -1516,7 +1516,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Copy => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let ty = ctx.input_ty(insn, 0);
             ctx.emit(Inst::gen_move(rd, rn, ty));
@@ -1526,7 +1526,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // Smaller integers/booleans are stored with high-order bits
             // undefined, so we can simply do a copy.
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ctx.input_ty(insn, 0);
             ctx.emit(Inst::gen_move(rd, rn, ty));
         }
@@ -1553,7 +1553,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 // Nothing.
             } else {
                 let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                let rd = get_output_reg(ctx, outputs[0]);
+                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
                 let to_bits = if to_bits == 64 {
                     64
                 } else {
@@ -1575,7 +1575,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // Booleans are stored as all-zeroes (0) or all-ones (-1). We AND
             // out the LSB to give a 0 / 1-valued integer result.
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let output_bits = ty_bits(ctx.output_ty(insn, 0));
 
             let (imm_ty, alu_op) = if output_bits > 32 {
@@ -1592,7 +1592,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Bitcast => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ity = ctx.input_ty(insn, 0);
             let oty = ctx.output_ty(insn, 0);
             let ity_bits = ty_bits(ity);
@@ -1644,7 +1644,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 // (above the bits for the value's type) are undefined, so we
                 // need not extend the return values.
                 let reg = put_input_in_reg(ctx, *input, NarrowValueMode::None);
-                let retval_reg = ctx.retval(i);
+                let retval_reg = ctx.retval(i).only_reg().unwrap();
                 let ty = ctx.input_ty(insn, i);
                 ctx.emit(Inst::gen_move(retval_reg, reg, ty));
             }
@@ -1663,7 +1663,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let condcode = ctx.data(insn).cond_code().unwrap();
             let cond = lower_condcode(condcode);
             let is_signed = condcode_is_signed(condcode);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ctx.input_ty(insn, 0);
             let bits = ty_bits(ty);
             let narrow_mode = match (bits <= 32, is_signed) {
@@ -1691,7 +1691,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let ty = ctx.input_ty(insn, 0);
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
 
             if !ty.is_vector() {
                 match ty_bits(ty) {
@@ -1768,7 +1768,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::FuncAddr => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let (extname, _) = ctx.call_target(insn).unwrap();
             let extname = extname.clone();
             ctx.emit(Inst::LoadExtName {
@@ -1783,7 +1783,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::SymbolValue => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let (extname, _, offset) = ctx.symbol_value(insn).unwrap();
             let extname = extname.clone();
             ctx.emit(Inst::LoadExtName {
@@ -1824,18 +1824,18 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             assert!(inputs.len() == abi.num_args());
             for (i, input) in inputs.iter().enumerate() {
                 let arg_reg = put_input_in_reg(ctx, *input, NarrowValueMode::None);
-                abi.emit_copy_reg_to_arg(ctx, i, arg_reg);
+                abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg));
             }
             abi.emit_call(ctx);
             for (i, output) in outputs.iter().enumerate() {
-                let retval_reg = get_output_reg(ctx, *output);
+                let retval_reg = get_output_reg(ctx, *output).only_reg().unwrap();
-                abi.emit_copy_retval_to_reg(ctx, i, retval_reg);
+                abi.emit_copy_retval_to_regs(ctx, i, ValueRegs::one(retval_reg));
             }
             abi.emit_stack_post_adjust(ctx);
         }
 
         Opcode::GetPinnedReg => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             ctx.emit(Inst::gen_move(rd, xreg(PINNED_REG), I64));
         }
 
@@ -1874,13 +1874,13 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::Vconst => {
             let value = const_param_to_u128(ctx, insn).expect("Invalid immediate bytes");
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             lower_constant_f128(ctx, rd, value);
         }
 
         Opcode::RawBitcast => {
             let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ctx.input_ty(insn, 0);
             ctx.emit(Inst::gen_move(rd, rm, ty));
         }
@@ -1888,7 +1888,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         Opcode::Extractlane => {
             if let InstructionData::BinaryImm8 { imm, .. } = ctx.data(insn) {
                 let idx = *imm;
-                let rd = get_output_reg(ctx, outputs[0]);
+                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
                 let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
                 let size = VectorSize::from_ty(ctx.input_ty(insn, 0));
                 let ty = ty.unwrap();
@@ -1913,7 +1913,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 unreachable!();
             };
             let input_ty = ctx.input_ty(insn, 1);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let ty = ty.unwrap();
@@ -1935,7 +1935,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Splat => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let size = VectorSize::from_ty(ty.unwrap());
 
             if let Some((_, insn)) = maybe_input_insn_multi(
@@ -1979,7 +1979,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     &load_inputs[..],
                     load_outputs[0],
                     |ctx, _rd, _elem_ty, mem| {
-                        let tmp = ctx.alloc_tmp(RegClass::I64, I64);
+                        let tmp = ctx.alloc_tmp(I64).only_reg().unwrap();
                         let (addr, addr_inst) = Inst::gen_load_addr(tmp, mem);
                         if let Some(addr_inst) = addr_inst {
                             ctx.emit(addr_inst);
@@ -2002,7 +2002,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::ScalarToVector => {
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let input_ty = ctx.input_ty(insn, 0);
             if (input_ty == I32 && ty.unwrap() == I32X4)
                 || (input_ty == I64 && ty.unwrap() == I64X2)
@@ -2021,9 +2021,10 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::VanyTrue | Opcode::VallTrue => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let tmp = ctx.alloc_tmp(RegClass::V128, ty.unwrap());
+            let src_ty = ctx.input_ty(insn, 0);
+            let tmp = ctx.alloc_tmp(src_ty).only_reg().unwrap();
 
             // This operation is implemented by using umaxp or uminv to
             // create a scalar value, which is then compared against zero.
@@ -2070,7 +2071,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::VhighBits => {
-            let dst_r = get_output_reg(ctx, outputs[0]);
+            let dst_r = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let src_v = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let ty = ctx.input_ty(insn, 0);
             // All three sequences use one integer temporary and two vector temporaries.  The
@@ -2080,9 +2081,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // derivation of these sequences.  Alternative sequences are discussed in
             // https://github.com/bytecodealliance/wasmtime/issues/2296, although they are not
             // used here.
-            let tmp_r0 = ctx.alloc_tmp(RegClass::I64, I64);
+            let tmp_r0 = ctx.alloc_tmp(I64).only_reg().unwrap();
-            let tmp_v0 = ctx.alloc_tmp(RegClass::V128, I8X16);
+            let tmp_v0 = ctx.alloc_tmp(I8X16).only_reg().unwrap();
-            let tmp_v1 = ctx.alloc_tmp(RegClass::V128, I8X16);
+            let tmp_v1 = ctx.alloc_tmp(I8X16).only_reg().unwrap();
             match ty {
                 I8X16 => {
                     // sshr  tmp_v1.16b, src_v.16b, #7
@@ -2255,7 +2256,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::Shuffle => {
             let mask = const_param_to_u128(ctx, insn).expect("Invalid immediate mask bytes");
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rn2 = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             // 2 register table vector lookups require consecutive table registers;
@@ -2283,7 +2284,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Swizzle => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
 
@@ -2310,7 +2311,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 Opcode::Imax => VecALUOp::Smax,
                 _ => unreachable!(),
             };
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let ty = ty.unwrap();
@@ -2324,12 +2325,12 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::WideningPairwiseDotProductS => {
-            let r_y = get_output_reg(ctx, outputs[0]);
+            let r_y = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let r_a = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let r_b = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let ty = ty.unwrap();
             if ty == I32X4 {
-                let tmp = ctx.alloc_tmp(RegClass::V128, I8X16);
+                let tmp = ctx.alloc_tmp(I8X16).only_reg().unwrap();
                 // The args have type I16X8.
                 // "y = i32x4.dot_i16x8_s(a, b)"
                 // => smull  tmp, a, b
@@ -2369,7 +2370,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let bits = ty_bits(ty);
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             if !ty.is_vector() {
                 let fpu_op = match (op, bits) {
                     (Opcode::Fadd, 32) => FPUOp2::Add32,
@@ -2413,7 +2414,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             if ty == F32X4 || ty == F64X2 {
                 // pmin(a,b) => bitsel(b, a, cmpgt(a, b))
                 // pmax(a,b) => bitsel(b, a, cmpgt(b, a))
-                let r_dst = get_output_reg(ctx, outputs[0]);
+                let r_dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
                 let r_a = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
                 let r_b = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
                 // Since we're going to write the output register `r_dst` anyway, we might as
@@ -2449,7 +2450,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let ty = ty.unwrap();
             let bits = ty_bits(ty);
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             if !ty.is_vector() {
                 let fpu_op = match (op, bits) {
                     (Opcode::Sqrt, 32) => FPUOp1::Sqrt32,
@@ -2498,7 +2499,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     _ => panic!("Unknown op/bits combination (scalar)"),
                 };
                 let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                let rd = get_output_reg(ctx, outputs[0]);
+                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
                 ctx.emit(Inst::FpuRound { op, rd, rn });
             } else {
                 let (op, size) = match (op, ty) {
@@ -2513,7 +2514,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     _ => panic!("Unknown op/ty combination (vector){:?}", ty),
                 };
                 let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                let rd = get_output_reg(ctx, outputs[0]);
+                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
                 ctx.emit(Inst::VecMisc { op, rd, rn, size });
             }
         }
@@ -2528,7 +2529,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let ra = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             ctx.emit(Inst::FpuRRRR {
                 fpu_op,
                 rn,
@@ -2554,8 +2555,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             assert!(bits == 32 || bits == 64);
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let tmp = ctx.alloc_tmp(RegClass::V128, F64);
+            let tmp = ctx.alloc_tmp(F64).only_reg().unwrap();
 
             // Copy LHS to rd.
             ctx.emit(Inst::gen_move(rd, rn, ty));
@@ -2594,7 +2595,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             };
 
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
 
             // First, check the output: it's important to carry the NaN conversion before the
             // in-bounds conversion, per wasm semantics.
@@ -2611,7 +2612,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 kind: CondBrKind::Cond(lower_fp_condcode(FloatCC::Unordered)),
             });
 
-            let tmp = ctx.alloc_tmp(RegClass::V128, I128);
+            let tmp = ctx.alloc_tmp(I8X16).only_reg().unwrap();
 
             // Check that the input is in range, with "truncate towards zero" semantics. This means
             // we allow values that are slightly out of range:
@@ -2736,7 +2737,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         Opcode::FcvtFromUint | Opcode::FcvtFromSint => {
             let ty = ty.unwrap();
             let signed = op == Opcode::FcvtFromSint;
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
 
             if ty.is_vector() {
                 let op = if signed {
@@ -2782,7 +2783,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let ty = ty.unwrap();
             let out_signed = op == Opcode::FcvtToSintSat;
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
 
             if ty.is_vector() {
                 let op = if out_signed {
@@ -2829,8 +2830,8 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     _ => unreachable!(),
                 };
 
-                let rtmp1 = ctx.alloc_tmp(RegClass::V128, in_ty);
+                let rtmp1 = ctx.alloc_tmp(in_ty).only_reg().unwrap();
-                let rtmp2 = ctx.alloc_tmp(RegClass::V128, in_ty);
+                let rtmp2 = ctx.alloc_tmp(in_ty).only_reg().unwrap();
 
                 if in_bits == 32 {
                     lower_constant_f32(ctx, rtmp1, max as f32);
@@ -2920,7 +2921,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
             // Now handle the iadd as above, except use an AddS opcode that sets
             // flags.
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rm = put_input_in_rse_imm12(ctx, inputs[1], NarrowValueMode::None);
             let ty = ty.unwrap();
@@ -3001,7 +3002,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         Opcode::DummySargT => unreachable!(),
 
         Opcode::Iabs => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let ty = ty.unwrap();
             ctx.emit(Inst::VecMisc {
@@ -3012,7 +3013,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             });
         }
         Opcode::AvgRound => {
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let ty = ty.unwrap();
@@ -3031,7 +3032,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             } else {
                 VecMiscNarrowOp::Sqxtun
             };
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let rn2 = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
             let ty = ty.unwrap();
@@ -3054,7 +3055,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::SwidenLow | Opcode::SwidenHigh | Opcode::UwidenLow | Opcode::UwidenHigh => {
             let lane_type = ty.unwrap().lane_type();
-            let rd = get_output_reg(ctx, outputs[0]);
+            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
             let (t, high_half) = match (lane_type, op) {
                 (I16, Opcode::SwidenLow) => (VecExtendOp::Sxtl8, false),
@@ -3313,8 +3314,8 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
                     NarrowValueMode::ZeroExtend32,
                 );
 
-                let rtmp1 = ctx.alloc_tmp(RegClass::I64, I32);
+                let rtmp1 = ctx.alloc_tmp(I32).only_reg().unwrap();
-                let rtmp2 = ctx.alloc_tmp(RegClass::I64, I32);
+                let rtmp2 = ctx.alloc_tmp(I32).only_reg().unwrap();
 
                 // Bounds-check, leaving condition codes for JTSequence's
                 // branch to default target below.

cranelift/codegen/src/isa/arm32/abi.rs

@@ -82,7 +82,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
                 let reg = rreg(next_rreg);
 
                 ret.push(ABIArg::Reg(
-                    reg.to_real_reg(),
+                    ValueRegs::one(reg.to_real_reg()),
                     param.value_type,
                     param.extension,
                     param.purpose,
@@ -102,7 +102,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
             if next_rreg < max_reg_val {
                 ret.push(ABIArg::Reg(
-                    rreg(next_rreg).to_real_reg(),
+                    ValueRegs::one(rreg(next_rreg).to_real_reg()),
                     I32,
                     ir::ArgumentExtension::None,
                     ir::ArgumentPurpose::Normal,
@@ -185,7 +185,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
         Inst::EpiloguePlaceholder
     }
 
-    fn gen_add_imm(into_reg: Writable<Reg>, from_reg: Reg, imm: u32) -> SmallVec<[Inst; 4]> {
+    fn gen_add_imm(into_reg: Writable<Reg>, from_reg: Reg, imm: u32) -> SmallInstVec<Inst> {
         let mut insts = SmallVec::new();
 
         if let Some(imm12) = UImm12::maybe_from_i64(imm as i64) {
@@ -209,7 +209,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
         insts
     }
 
-    fn gen_stack_lower_bound_trap(limit_reg: Reg) -> SmallVec<[Inst; 2]> {
+    fn gen_stack_lower_bound_trap(limit_reg: Reg) -> SmallInstVec<Inst> {
         let mut insts = SmallVec::new();
         insts.push(Inst::Cmp {
             rn: sp_reg(),
@@ -243,7 +243,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
         Inst::gen_store(from_reg, mem, ty)
     }
 
-    fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Inst; 2]> {
+    fn gen_sp_reg_adjust(amount: i32) -> SmallInstVec<Inst> {
         let mut ret = SmallVec::new();
 
         if amount == 0 {
@@ -283,7 +283,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
         Inst::VirtualSPOffsetAdj { offset }
     }
 
-    fn gen_prologue_frame_setup() -> SmallVec<[Inst; 2]> {
+    fn gen_prologue_frame_setup() -> SmallInstVec<Inst> {
         let mut ret = SmallVec::new();
         let reg_list = vec![fp_reg(), lr_reg()];
         ret.push(Inst::Push { reg_list });
@@ -294,7 +294,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
         ret
     }
 
-    fn gen_epilogue_frame_restore() -> SmallVec<[Inst; 2]> {
+    fn gen_epilogue_frame_restore() -> SmallInstVec<Inst> {
         let mut ret = SmallVec::new();
         ret.push(Inst::Mov {
             rd: writable_sp_reg(),
@@ -305,7 +305,7 @@ impl ABIMachineSpec for Arm32MachineDeps {
         ret
     }
 
-    fn gen_probestack(_: u32) -> SmallVec<[Self::I; 2]> {
+    fn gen_probestack(_: u32) -> SmallInstVec<Self::I> {
         // TODO: implement if we ever require stack probes on ARM32 (unlikely
         // unless Lucet is ported)
         smallvec![]

cranelift/codegen/src/isa/arm32/inst/mod.rs

@@ -807,12 +807,17 @@ impl MachInst for Inst {
         Inst::mov(to_reg, from_reg)
     }
 
-    fn gen_constant<F: FnMut(RegClass, Type) -> Writable<Reg>>(
+    fn gen_constant<F: FnMut(Type) -> Writable<Reg>>(
-        to_reg: Writable<Reg>,
+        to_regs: ValueRegs<Writable<Reg>>,
-        value: u64,
+        value: u128,
         ty: Type,
         _alloc_tmp: F,
     ) -> SmallVec<[Inst; 4]> {
+        let to_reg = to_regs
+            .only_reg()
+            .expect("multi-reg values not supported yet");
+        let value = value as u64;
+
         match ty {
             B1 | I8 | B8 | I16 | B16 | I32 | B32 => {
                 let v: i64 = value as i64;
@@ -839,10 +844,10 @@ impl MachInst for Inst {
         None
     }
 
-    fn rc_for_type(ty: Type) -> CodegenResult<RegClass> {
+    fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
         match ty {
-            I8 | I16 | I32 | B1 | B8 | B16 | B32 => Ok(RegClass::I32),
+            I8 | I16 | I32 | B1 | B8 | B16 | B32 => Ok((&[RegClass::I32], &[I32])),
-            IFLAGS => Ok(RegClass::I32),
+            IFLAGS => Ok((&[RegClass::I32], &[I32])),
             _ => Err(CodegenError::Unsupported(format!(
                 "Unexpected SSA-value type: {}",
                 ty

cranelift/codegen/src/isa/arm32/lower.rs

@@ -13,7 +13,7 @@ use crate::isa::arm32::Arm32Backend;
 
 use super::lower_inst;
 
-use regalloc::{Reg, RegClass, Writable};
+use regalloc::{Reg, Writable};
 
 //============================================================================
 // Lowering: convert instruction outputs to result types.
@@ -55,7 +55,7 @@ pub(crate) enum NarrowValueMode {
 
 /// Lower an instruction output to a reg.
 pub(crate) fn output_to_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, out: InsnOutput) -> Writable<Reg> {
-    ctx.get_output(out.insn, out.output)
+    ctx.get_output(out.insn, out.output).only_reg().unwrap()
 }
 
 /// Lower an instruction input to a reg.
@@ -70,21 +70,25 @@ pub(crate) fn input_to_reg<C: LowerCtx<I = Inst>>(
     let from_bits = ty.bits() as u8;
     let inputs = ctx.get_input_as_source_or_const(input.insn, input.input);
     let in_reg = if let Some(c) = inputs.constant {
-        let to_reg = ctx.alloc_tmp(Inst::rc_for_type(ty).unwrap(), ty);
+        let to_reg = ctx.alloc_tmp(ty).only_reg().unwrap();
-        for inst in Inst::gen_constant(to_reg, c, ty, |reg_class, ty| ctx.alloc_tmp(reg_class, ty))
+        for inst in Inst::gen_constant(ValueRegs::one(to_reg), c as u128, ty, |ty| {
+            ctx.alloc_tmp(ty).only_reg().unwrap()
+        })
         .into_iter()
         {
             ctx.emit(inst);
         }
         to_reg.to_reg()
     } else {
-        ctx.put_input_in_reg(input.insn, input.input)
+        ctx.put_input_in_regs(input.insn, input.input)
+            .only_reg()
+            .unwrap()
     };
 
     match (narrow_mode, from_bits) {
         (NarrowValueMode::None, _) => in_reg,
         (NarrowValueMode::ZeroExtend, 1) => {
-            let tmp = ctx.alloc_tmp(RegClass::I32, I32);
+            let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
             ctx.emit(Inst::AluRRImm8 {
                 alu_op: ALUOp::And,
                 rd: tmp,
@@ -94,7 +98,7 @@ pub(crate) fn input_to_reg<C: LowerCtx<I = Inst>>(
             tmp.to_reg()
         }
         (NarrowValueMode::ZeroExtend, n) if n < 32 => {
-            let tmp = ctx.alloc_tmp(RegClass::I32, I32);
+            let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
             ctx.emit(Inst::Extend {
                 rd: tmp,
                 rm: in_reg,
@@ -104,7 +108,7 @@ pub(crate) fn input_to_reg<C: LowerCtx<I = Inst>>(
             tmp.to_reg()
         }
         (NarrowValueMode::SignExtend, n) if n < 32 => {
-            let tmp = ctx.alloc_tmp(RegClass::I32, I32);
+            let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
             ctx.emit(Inst::Extend {
                 rd: tmp,
                 rm: in_reg,

cranelift/codegen/src/isa/arm32/lower_inst.rs

@@ -10,7 +10,6 @@ use crate::CodegenResult;
 use crate::isa::arm32::abi::*;
 use crate::isa::arm32::inst::*;
 
-use regalloc::RegClass;
 use smallvec::SmallVec;
 
 use super::lower::*;
@@ -143,7 +142,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let rd = output_to_reg(ctx, outputs[0]);
             let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None);
             let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None);
-            let tmp = ctx.alloc_tmp(RegClass::I32, I32);
+            let tmp = ctx.alloc_tmp(I32).only_reg().unwrap();
 
             // ror rd, rn, 32 - (rm & 31)
             ctx.emit(Inst::AluRRImm8 {
@@ -171,7 +170,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             match ty {
                 I32 => {
                     let rd_hi = output_to_reg(ctx, outputs[0]);
-                    let rd_lo = ctx.alloc_tmp(RegClass::I32, ty);
+                    let rd_lo = ctx.alloc_tmp(ty).only_reg().unwrap();
                     let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None);
                     let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None);
 
@@ -487,7 +486,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         Opcode::FallthroughReturn | Opcode::Return => {
             for (i, input) in inputs.iter().enumerate() {
                 let reg = input_to_reg(ctx, *input, NarrowValueMode::None);
-                let retval_reg = ctx.retval(i);
+                let retval_reg = ctx.retval(i).only_reg().unwrap();
                 let ty = ctx.input_ty(insn, i);
 
                 ctx.emit(Inst::gen_move(retval_reg, reg, ty));
@@ -522,12 +521,12 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             assert_eq!(inputs.len(), abi.num_args());
             for (i, input) in inputs.iter().enumerate().filter(|(i, _)| *i <= 3) {
                 let arg_reg = input_to_reg(ctx, *input, NarrowValueMode::None);
-                abi.emit_copy_reg_to_arg(ctx, i, arg_reg);
+                abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg));
             }
             abi.emit_call(ctx);
             for (i, output) in outputs.iter().enumerate() {
                 let retval_reg = output_to_reg(ctx, *output);
-                abi.emit_copy_retval_to_reg(ctx, i, retval_reg);
+                abi.emit_copy_retval_to_regs(ctx, i, ValueRegs::one(retval_reg));
             }
         }
         _ => panic!("lowering {} unimplemented!", op),

cranelift/codegen/src/isa/x64/abi.rs

@@ -32,7 +32,7 @@ fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option<A
             &ir::ArgumentPurpose::VMContext => {
                 // This is SpiderMonkey's `WasmTlsReg`.
                 Some(ABIArg::Reg(
-                    regs::r14().to_real_reg(),
+                    ValueRegs::one(regs::r14().to_real_reg()),
                     types::I64,
                     param.extension,
                     param.purpose,
@@ -41,7 +41,7 @@ fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option<A
             &ir::ArgumentPurpose::SignatureId => {
                 // This is SpiderMonkey's `WasmTableCallSigReg`.
                 Some(ABIArg::Reg(
-                    regs::r10().to_real_reg(),
+                    ValueRegs::one(regs::r10().to_real_reg()),
                     types::I64,
                     param.extension,
                     param.purpose,
@@ -168,7 +168,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                 ret.push(param);
             } else if let Some(reg) = candidate {
                 ret.push(ABIArg::Reg(
-                    reg.to_real_reg(),
+                    ValueRegs::one(reg.to_real_reg()),
                     param.value_type,
                     param.extension,
                     param.purpose,
@@ -200,7 +200,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
             if let Some(reg) = get_intreg_for_arg_systemv(&call_conv, next_gpr) {
                 ret.push(ABIArg::Reg(
-                    reg.to_real_reg(),
+                    ValueRegs::one(reg.to_real_reg()),
                     types::I64,
                     ir::ArgumentExtension::None,
                     ir::ArgumentPurpose::Normal,
@@ -288,7 +288,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         Inst::epilogue_placeholder()
     }
 
-    fn gen_add_imm(into_reg: Writable<Reg>, from_reg: Reg, imm: u32) -> SmallVec<[Self::I; 4]> {
+    fn gen_add_imm(into_reg: Writable<Reg>, from_reg: Reg, imm: u32) -> SmallInstVec<Self::I> {
         let mut ret = SmallVec::new();
         if from_reg != into_reg.to_reg() {
             ret.push(Inst::gen_move(into_reg, from_reg, I64));
@@ -302,7 +302,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         ret
     }
 
-    fn gen_stack_lower_bound_trap(limit_reg: Reg) -> SmallVec<[Self::I; 2]> {
+    fn gen_stack_lower_bound_trap(limit_reg: Reg) -> SmallInstVec<Self::I> {
         smallvec![
             Inst::cmp_rmi_r(/* bytes = */ 8, RegMemImm::reg(regs::rsp()), limit_reg),
             Inst::TrapIf {
@@ -343,7 +343,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         Inst::store(ty, from_reg, mem)
     }
 
-    fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Self::I; 2]> {
+    fn gen_sp_reg_adjust(amount: i32) -> SmallInstVec<Self::I> {
         let (alu_op, amount) = if amount >= 0 {
             (AluRmiROpcode::Add, amount)
         } else {
@@ -366,7 +366,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         }
     }
 
-    fn gen_prologue_frame_setup() -> SmallVec<[Self::I; 2]> {
+    fn gen_prologue_frame_setup() -> SmallInstVec<Self::I> {
         let r_rsp = regs::rsp();
         let r_rbp = regs::rbp();
         let w_rbp = Writable::from_reg(r_rbp);
@@ -378,7 +378,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         insts
     }
 
-    fn gen_epilogue_frame_restore() -> SmallVec<[Self::I; 2]> {
+    fn gen_epilogue_frame_restore() -> SmallInstVec<Self::I> {
         let mut insts = SmallVec::new();
         insts.push(Inst::mov_r_r(
             true,
@@ -389,7 +389,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         insts
     }
 
-    fn gen_probestack(frame_size: u32) -> SmallVec<[Self::I; 2]> {
+    fn gen_probestack(frame_size: u32) -> SmallInstVec<Self::I> {
         let mut insts = SmallVec::new();
         insts.push(Inst::imm(
             OperandSize::Size32,

cranelift/codegen/src/isa/x64/inst/mod.rs

@@ -2506,22 +2506,28 @@ impl MachInst for Inst {
         None
     }
 
-    fn rc_for_type(ty: Type) -> CodegenResult<RegClass> {
+    fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
         match ty {
-            types::I8
+            types::I8 => Ok((&[RegClass::I64], &[types::I8])),
-            | types::I16
+            types::I16 => Ok((&[RegClass::I64], &[types::I16])),
-            | types::I32
+            types::I32 => Ok((&[RegClass::I64], &[types::I32])),
-            | types::I64
+            types::I64 => Ok((&[RegClass::I64], &[types::I64])),
-            | types::B1
+            types::B1 => Ok((&[RegClass::I64], &[types::B1])),
-            | types::B8
+            types::B8 => Ok((&[RegClass::I64], &[types::B8])),
-            | types::B16
+            types::B16 => Ok((&[RegClass::I64], &[types::B16])),
-            | types::B32
+            types::B32 => Ok((&[RegClass::I64], &[types::B32])),
-            | types::B64
+            types::B64 => Ok((&[RegClass::I64], &[types::B64])),
-            | types::R32
+            types::R32 => panic!("32-bit reftype pointer should never be seen on x86-64"),
-            | types::R64 => Ok(RegClass::I64),
+            types::R64 => Ok((&[RegClass::I64], &[types::R64])),
-            types::F32 | types::F64 => Ok(RegClass::V128),
+            types::F32 => Ok((&[RegClass::V128], &[types::F32])),
-            _ if ty.bits() == 128 => Ok(RegClass::V128),
+            types::F64 => Ok((&[RegClass::V128], &[types::F64])),
-            types::IFLAGS | types::FFLAGS => Ok(RegClass::I64),
+            types::I128 => Ok((&[RegClass::I64, RegClass::I64], &[types::I64, types::I64])),
+            types::B128 => Ok((&[RegClass::I64, RegClass::I64], &[types::B64, types::B64])),
+            _ if ty.is_vector() => {
+                assert!(ty.bits() <= 128);
+                Ok((&[RegClass::V128], &[types::I8X16]))
+            }
+            types::IFLAGS | types::FFLAGS => Ok((&[RegClass::I64], &[types::I64])),
             _ => Err(CodegenError::Unsupported(format!(
                 "Unexpected SSA-value type: {}",
                 ty
@@ -2533,13 +2539,18 @@ impl MachInst for Inst {
         Inst::jmp_known(label)
     }
 
-    fn gen_constant<F: FnMut(RegClass, Type) -> Writable<Reg>>(
+    fn gen_constant<F: FnMut(Type) -> Writable<Reg>>(
-        to_reg: Writable<Reg>,
+        to_regs: ValueRegs<Writable<Reg>>,
-        value: u64,
+        value: u128,
         ty: Type,
         mut alloc_tmp: F,
     ) -> SmallVec<[Self; 4]> {
+        // We don't support 128-bit constants.
+        assert!(value <= u64::MAX as u128);
         let mut ret = SmallVec::new();
+        let to_reg = to_regs
+            .only_reg()
+            .expect("multi-reg values not supported on x64");
         if ty == types::F32 {
             if value == 0 {
                 ret.push(Inst::xmm_rm_r(
@@ -2548,8 +2559,8 @@ impl MachInst for Inst {
                     to_reg,
                 ));
             } else {
-                let tmp = alloc_tmp(RegClass::I64, types::I32);
+                let tmp = alloc_tmp(types::I32);
-                ret.push(Inst::imm(OperandSize::Size32, value, tmp));
+                ret.push(Inst::imm(OperandSize::Size32, value as u64, tmp));
 
                 ret.push(Inst::gpr_to_xmm(
                     SseOpcode::Movd,
@@ -2566,8 +2577,8 @@ impl MachInst for Inst {
                     to_reg,
                 ));
             } else {
-                let tmp = alloc_tmp(RegClass::I64, types::I64);
+                let tmp = alloc_tmp(types::I64);
-                ret.push(Inst::imm(OperandSize::Size64, value, tmp));
+                ret.push(Inst::imm(OperandSize::Size64, value as u64, tmp));
 
                 ret.push(Inst::gpr_to_xmm(
                     SseOpcode::Movq,
@@ -2599,6 +2610,7 @@ impl MachInst for Inst {
                     to_reg,
                 ));
             } else {
+                let value = value as u64;
                 ret.push(Inst::imm(
                     OperandSize::from_bytes(ty.bytes()),
                     value.into(),

cranelift/codegen/src/machinst/abi.rs

@@ -5,8 +5,12 @@ use crate::ir::StackSlot;
 use crate::isa::CallConv;
 use crate::machinst::*;
 use crate::settings;
-
 use regalloc::{Reg, Set, SpillSlot, Writable};
+use smallvec::SmallVec;
+
+/// A small vector of instructions (with some reasonable size); appropriate for
+/// a small fixed sequence implementing one operation.
+pub type SmallInstVec<I> = SmallVec<[I; 4]>;
 
 /// Trait implemented by an object that tracks ABI-related state (e.g., stack
 /// layout) and can generate code while emitting the *body* of a function.
@@ -14,9 +18,9 @@ pub trait ABICallee {
     /// The instruction type for the ISA associated with this ABI.
     type I: VCodeInst;
 
-    /// Does the ABI-body code need a temp reg? One will be provided to `init()`
+    /// Does the ABI-body code need a temp reg (and if so, of what type)? One
-    /// as the `maybe_tmp` arg if so.
+    /// will be provided to `init()` as the `maybe_tmp` arg if so.
-    fn temp_needed(&self) -> bool;
+    fn temp_needed(&self) -> Option<Type>;
 
     /// Initialize. This is called after the ABICallee is constructed because it
     /// may be provided with a temp vreg, which can only be allocated once the
@@ -52,7 +56,11 @@ pub trait ABICallee {
 
     /// Generate an instruction which copies an argument to a destination
     /// register.
-    fn gen_copy_arg_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Self::I;
+    fn gen_copy_arg_to_regs(
+        &self,
+        idx: usize,
+        into_reg: ValueRegs<Writable<Reg>>,
+    ) -> SmallInstVec<Self::I>;
 
     /// Is the given argument needed in the body (as opposed to, e.g., serving
     /// only as a special ABI-specific placeholder)? This controls whether
@@ -67,7 +75,11 @@ pub trait ABICallee {
     fn gen_retval_area_setup(&self) -> Option<Self::I>;
 
     /// Generate an instruction which copies a source register to a return value slot.
-    fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Writable<Reg>) -> Vec<Self::I>;
+    fn gen_copy_regs_to_retval(
+        &self,
+        idx: usize,
+        from_reg: ValueRegs<Writable<Reg>>,
+    ) -> SmallInstVec<Self::I>;
 
     /// Generate a return instruction.
     fn gen_ret(&self) -> Self::I;
@@ -99,17 +111,33 @@ pub trait ABICallee {
         slot: StackSlot,
         offset: u32,
         ty: Type,
-        into_reg: Writable<Reg>,
+        into_reg: ValueRegs<Writable<Reg>>,
-    ) -> Self::I;
+    ) -> SmallInstVec<Self::I>;
 
     /// Store to a stackslot.
-    fn store_stackslot(&self, slot: StackSlot, offset: u32, ty: Type, from_reg: Reg) -> Self::I;
+    fn store_stackslot(
+        &self,
+        slot: StackSlot,
+        offset: u32,
+        ty: Type,
+        from_reg: ValueRegs<Reg>,
+    ) -> SmallInstVec<Self::I>;
 
     /// Load from a spillslot.
-    fn load_spillslot(&self, slot: SpillSlot, ty: Type, into_reg: Writable<Reg>) -> Self::I;
+    fn load_spillslot(
+        &self,
+        slot: SpillSlot,
+        ty: Type,
+        into_reg: ValueRegs<Writable<Reg>>,
+    ) -> SmallInstVec<Self::I>;
 
     /// Store to a spillslot.
-    fn store_spillslot(&self, slot: SpillSlot, ty: Type, from_reg: Reg) -> Self::I;
+    fn store_spillslot(
+        &self,
+        slot: SpillSlot,
+        ty: Type,
+        from_reg: ValueRegs<Reg>,
+    ) -> SmallInstVec<Self::I>;
 
     /// Generate a stack map, given a list of spillslots and the emission state
     /// at a given program point (prior to emission fo the safepointing
@@ -125,13 +153,13 @@ pub trait ABICallee {
     /// `store_retval`, and spillslot accesses.)  `self` is mutable so that we
     /// can store information in it which will be useful when creating the
     /// epilogue.
-    fn gen_prologue(&mut self) -> Vec<Self::I>;
+    fn gen_prologue(&mut self) -> SmallInstVec<Self::I>;
 
     /// Generate an epilogue, post-regalloc. Note that this must generate the
     /// actual return instruction (rather than emitting this in the lowering
     /// logic), because the epilogue code comes before the return and the two are
     /// likely closely related.
-    fn gen_epilogue(&self) -> Vec<Self::I>;
+    fn gen_epilogue(&self) -> SmallInstVec<Self::I>;
 
     /// Returns the full frame size for the given function, after prologue
     /// emission has run. This comprises the spill slots and stack-storage slots
@@ -188,19 +216,19 @@ pub trait ABICaller {
     fn num_args(&self) -> usize;
 
     /// Emit a copy of an argument value from a source register, prior to the call.
-    fn emit_copy_reg_to_arg<C: LowerCtx<I = Self::I>>(
+    fn emit_copy_regs_to_arg<C: LowerCtx<I = Self::I>>(
         &self,
         ctx: &mut C,
         idx: usize,
-        from_reg: Reg,
+        from_reg: ValueRegs<Reg>,
     );
 
     /// Emit a copy a return value into a destination register, after the call returns.
-    fn emit_copy_retval_to_reg<C: LowerCtx<I = Self::I>>(
+    fn emit_copy_retval_to_regs<C: LowerCtx<I = Self::I>>(
         &self,
         ctx: &mut C,
         idx: usize,
-        into_reg: Writable<Reg>,
+        into_reg: ValueRegs<Writable<Reg>>,
     );
 
     /// Emit code to pre-adjust the stack, prior to argument copies and call.

cranelift/codegen/src/machinst/abi_impl.rs

@@ -119,6 +119,7 @@ use crate::{ir, isa};
 use alloc::vec::Vec;
 use log::{debug, trace};
 use regalloc::{RealReg, Reg, RegClass, Set, SpillSlot, Writable};
+use smallvec::{smallvec, SmallVec};
 use std::convert::TryFrom;
 use std::marker::PhantomData;
 use std::mem;
@@ -126,9 +127,9 @@ use std::mem;
 /// A location for an argument or return value.
 #[derive(Clone, Copy, Debug)]
 pub enum ABIArg {
-    /// In a real register.
+    /// In a real register (or set of registers).
     Reg(
-        RealReg,
+        ValueRegs<RealReg>,
         ir::Type,
         ir::ArgumentExtension,
         ir::ArgumentPurpose,
@@ -183,6 +184,17 @@ pub enum StackAMode {
     SPOffset(i64, ir::Type),
 }
 
+impl StackAMode {
+    /// Offset by an addend.
+    pub fn offset(self, addend: i64) -> Self {
+        match self {
+            StackAMode::FPOffset(off, ty) => StackAMode::FPOffset(off + addend, ty),
+            StackAMode::NominalSPOffset(off, ty) => StackAMode::NominalSPOffset(off + addend, ty),
+            StackAMode::SPOffset(off, ty) => StackAMode::SPOffset(off + addend, ty),
+        }
+    }
+}
+
 /// Trait implemented by machine-specific backend to provide information about
 /// register assignments and to allow generating the specific instructions for
 /// stack loads/saves, prologues/epilogues, etc.
@@ -270,12 +282,12 @@ pub trait ABIMachineSpec {
     ///
     /// - The add-imm sequence must work correctly when `from_reg` and/or
     ///   `into_reg` are the register returned by `get_stacklimit_reg()`.
-    fn gen_add_imm(into_reg: Writable<Reg>, from_reg: Reg, imm: u32) -> SmallVec<[Self::I; 4]>;
+    fn gen_add_imm(into_reg: Writable<Reg>, from_reg: Reg, imm: u32) -> SmallInstVec<Self::I>;
 
     /// Generate a sequence that traps with a `TrapCode::StackOverflow` code if
     /// the stack pointer is less than the given limit register (assuming the
     /// stack grows downward).
-    fn gen_stack_lower_bound_trap(limit_reg: Reg) -> SmallVec<[Self::I; 2]>;
+    fn gen_stack_lower_bound_trap(limit_reg: Reg) -> SmallInstVec<Self::I>;
 
     /// Generate an instruction to compute an address of a stack slot (FP- or
     /// SP-based offset).
@@ -301,7 +313,7 @@ pub trait ABIMachineSpec {
     fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Self::I;
 
     /// Adjust the stack pointer up or down.
-    fn gen_sp_reg_adjust(amount: i32) -> SmallVec<[Self::I; 2]>;
+    fn gen_sp_reg_adjust(amount: i32) -> SmallInstVec<Self::I>;
 
     /// Generate a meta-instruction that adjusts the nominal SP offset.
     fn gen_nominal_sp_adj(amount: i32) -> Self::I;
@@ -309,13 +321,13 @@ pub trait ABIMachineSpec {
     /// Generate the usual frame-setup sequence for this architecture: e.g.,
     /// `push rbp / mov rbp, rsp` on x86-64, or `stp fp, lr, [sp, #-16]!` on
     /// AArch64.
-    fn gen_prologue_frame_setup() -> SmallVec<[Self::I; 2]>;
+    fn gen_prologue_frame_setup() -> SmallInstVec<Self::I>;
 
     /// Generate the usual frame-restore sequence for this architecture.
-    fn gen_epilogue_frame_restore() -> SmallVec<[Self::I; 2]>;
+    fn gen_epilogue_frame_restore() -> SmallInstVec<Self::I>;
 
     /// Generate a probestack call.
-    fn gen_probestack(_frame_size: u32) -> SmallVec<[Self::I; 2]>;
+    fn gen_probestack(_frame_size: u32) -> SmallInstVec<Self::I>;
 
     /// Generate a clobber-save sequence. This takes the list of *all* registers
     /// written/modified by the function body. The implementation here is
@@ -483,7 +495,7 @@ pub struct ABICalleeImpl<M: ABIMachineSpec> {
     /// need to be extremely careful with each instruction. The instructions are
     /// manually register-allocated and carefully only use caller-saved
     /// registers and keep nothing live after this sequence of instructions.
-    stack_limit: Option<(Reg, Vec<M::I>)>,
+    stack_limit: Option<(Reg, SmallInstVec<M::I>)>,
     /// Are we to invoke the probestack function in the prologue? If so,
     /// what is the minimum size at which we must invoke it?
     probestack_min_frame: Option<u32>,
@@ -498,7 +510,7 @@ fn get_special_purpose_param_register(
 ) -> Option<Reg> {
     let idx = f.signature.special_param_index(purpose)?;
     match abi.args[idx] {
-        ABIArg::Reg(reg, ..) => Some(reg.to_reg()),
+        ABIArg::Reg(regs, ..) => Some(regs.only_reg().unwrap().to_reg()),
         ABIArg::Stack(..) => None,
     }
 }
@@ -539,7 +551,7 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
         // from the arguments.
         let stack_limit =
             get_special_purpose_param_register(f, &sig, ir::ArgumentPurpose::StackLimit)
-                .map(|reg| (reg, Vec::new()))
+                .map(|reg| (reg, smallvec![]))
                 .or_else(|| f.stack_limit.map(|gv| gen_stack_limit::<M>(f, &sig, gv)));
 
         // Determine whether a probestack call is required for large enough
@@ -596,7 +608,12 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
     /// No values can be live after the prologue, but in this case that's ok
     /// because we just need to perform a stack check before progressing with
     /// the rest of the function.
-    fn insert_stack_check(&self, stack_limit: Reg, stack_size: u32, insts: &mut Vec<M::I>) {
+    fn insert_stack_check(
+        &self,
+        stack_limit: Reg,
+        stack_size: u32,
+        insts: &mut SmallInstVec<M::I>,
+    ) {
         // With no explicit stack allocated we can just emit the simple check of
         // the stack registers against the stack limit register, and trap if
         // it's out of bounds.
@@ -649,8 +666,8 @@ fn gen_stack_limit<M: ABIMachineSpec>(
     f: &ir::Function,
     abi: &ABISig,
     gv: ir::GlobalValue,
-) -> (Reg, Vec<M::I>) {
+) -> (Reg, SmallInstVec<M::I>) {
-    let mut insts = Vec::new();
+    let mut insts = smallvec![];
     let reg = generate_gv::<M>(f, abi, gv, &mut insts);
     return (reg, insts);
 }
@@ -659,7 +676,7 @@ fn generate_gv<M: ABIMachineSpec>(
     f: &ir::Function,
     abi: &ABISig,
     gv: ir::GlobalValue,
-    insts: &mut Vec<M::I>,
+    insts: &mut SmallInstVec<M::I>,
 ) -> Reg {
     match f.global_values[gv] {
         // Return the direct register the vmcontext is in
@@ -709,11 +726,76 @@ fn ty_from_ty_hint_or_reg_class<M: ABIMachineSpec>(r: Reg, ty: Option<Type>) ->
     }
 }
 
+fn gen_move_multi<M: ABIMachineSpec>(
+    dst: ValueRegs<Writable<Reg>>,
+    src: ValueRegs<Reg>,
+    ty: Type,
+) -> SmallInstVec<M::I> {
+    let mut ret = smallvec![];
+    let (_, tys) = M::I::rc_for_type(ty).unwrap();
+    for ((&dst, &src), &ty) in dst.regs().iter().zip(src.regs().iter()).zip(tys.iter()) {
+        ret.push(M::gen_move(dst, src, ty));
+    }
+    ret
+}
+
+fn gen_load_stack_multi<M: ABIMachineSpec>(
+    from: StackAMode,
+    dst: ValueRegs<Writable<Reg>>,
+    ty: Type,
+) -> SmallInstVec<M::I> {
+    let mut ret = smallvec![];
+    let (_, tys) = M::I::rc_for_type(ty).unwrap();
+    let mut offset = 0;
+    // N.B.: registers are given in the `ValueRegs` in target endian order.
+    for (&dst, &ty) in dst.regs().iter().zip(tys.iter()) {
+        ret.push(M::gen_load_stack(from.offset(offset), dst, ty));
+        offset += ty.bytes() as i64;
+    }
+    ret
+}
+
+fn gen_store_stack_multi<M: ABIMachineSpec>(
+    from: StackAMode,
+    src: ValueRegs<Reg>,
+    ty: Type,
+) -> SmallInstVec<M::I> {
+    let mut ret = smallvec![];
+    let (_, tys) = M::I::rc_for_type(ty).unwrap();
+    let mut offset = 0;
+    // N.B.: registers are given in the `ValueRegs` in target endian order.
+    for (&src, &ty) in src.regs().iter().zip(tys.iter()) {
+        ret.push(M::gen_store_stack(from.offset(offset), src, ty));
+        offset += ty.bytes() as i64;
+    }
+    ret
+}
+
+fn gen_store_base_offset_multi<M: ABIMachineSpec>(
+    base: Reg,
+    mut offset: i32,
+    src: ValueRegs<Reg>,
+    ty: Type,
+) -> SmallInstVec<M::I> {
+    let mut ret = smallvec![];
+    let (_, tys) = M::I::rc_for_type(ty).unwrap();
+    // N.B.: registers are given in the `ValueRegs` in target endian order.
+    for (&src, &ty) in src.regs().iter().zip(tys.iter()) {
+        ret.push(M::gen_store_base_offset(base, offset, src, ty));
+        offset += ty.bytes() as i32;
+    }
+    ret
+}
+
 impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
     type I = M::I;
 
-    fn temp_needed(&self) -> bool {
+    fn temp_needed(&self) -> Option<Type> {
-        self.sig.stack_ret_arg.is_some()
+        if self.sig.stack_ret_arg.is_some() {
+            Some(M::word_type())
+        } else {
+            None
+        }
     }
 
     fn init(&mut self, maybe_tmp: Option<Writable<Reg>>) {
@@ -740,20 +822,24 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
     fn liveins(&self) -> Set<RealReg> {
         let mut set: Set<RealReg> = Set::empty();
         for &arg in &self.sig.args {
-            if let ABIArg::Reg(r, ..) = arg {
+            if let ABIArg::Reg(regs, ..) = arg {
+                for &r in regs.regs() {
                     set.insert(r);
                 }
             }
+        }
         set
     }
 
     fn liveouts(&self) -> Set<RealReg> {
         let mut set: Set<RealReg> = Set::empty();
         for &ret in &self.sig.rets {
-            if let ABIArg::Reg(r, ..) = ret {
+            if let ABIArg::Reg(regs, ..) = ret {
+                for &r in regs.regs() {
                     set.insert(r);
                 }
             }
+        }
         set
     }
 
@@ -769,14 +855,20 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         self.stackslots.len()
     }
 
-    fn gen_copy_arg_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Self::I {
+    fn gen_copy_arg_to_regs(
+        &self,
+        idx: usize,
+        into_regs: ValueRegs<Writable<Reg>>,
+    ) -> SmallInstVec<Self::I> {
         match &self.sig.args[idx] {
             // Extension mode doesn't matter (we're copying out, not in; we
             // ignore high bits by convention).
-            &ABIArg::Reg(r, ty, ..) => M::gen_move(into_reg, r.to_reg(), ty),
+            &ABIArg::Reg(regs, ty, ..) => {
-            &ABIArg::Stack(off, ty, ..) => M::gen_load_stack(
+                gen_move_multi::<M>(into_regs, regs.map(|r| r.to_reg()), ty)
+            }
+            &ABIArg::Stack(off, ty, ..) => gen_load_stack_multi::<M>(
                 StackAMode::FPOffset(M::fp_to_arg_offset(self.call_conv, &self.flags) + off, ty),
-                into_reg,
+                into_regs,
                 ty,
             ),
         }
@@ -792,19 +884,29 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         }
     }
 
-    fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Writable<Reg>) -> Vec<Self::I> {
+    fn gen_copy_regs_to_retval(
-        let mut ret = Vec::new();
+        &self,
+        idx: usize,
+        from_regs: ValueRegs<Writable<Reg>>,
+    ) -> SmallInstVec<Self::I> {
+        let mut ret = smallvec![];
         let word_bits = M::word_bits() as u8;
         match &self.sig.rets[idx] {
-            &ABIArg::Reg(r, ty, ext, ..) => {
+            &ABIArg::Reg(regs, ty, ext, ..) => {
                 let from_bits = ty_bits(ty) as u8;
-                let dest_reg = Writable::from_reg(r.to_reg());
+                let dest_regs = writable_value_regs(regs.map(|r| r.to_reg()));
                 let ext = M::get_ext_mode(self.sig.call_conv, ext);
                 match (ext, from_bits) {
                     (ArgumentExtension::Uext, n) | (ArgumentExtension::Sext, n)
                         if n < word_bits =>
                     {
                         let signed = ext == ArgumentExtension::Sext;
+                        let dest_reg = dest_regs
+                            .only_reg()
+                            .expect("extension only possible from one-reg value");
+                        let from_reg = from_regs
+                            .only_reg()
+                            .expect("extension only possible from one-reg value");
                         ret.push(M::gen_extend(
                             dest_reg,
                             from_reg.to_reg(),
@@ -813,7 +915,10 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
                             /* to_bits = */ word_bits,
                         ));
                     }
-                    _ => ret.push(M::gen_move(dest_reg, from_reg.to_reg(), ty)),
+                    _ => ret.extend(
+                        gen_move_multi::<M>(dest_regs, non_writable_value_regs(from_regs), ty)
+                            .into_iter(),
+                    ),
                 };
             }
             &ABIArg::Stack(off, mut ty, ext, ..) => {
@@ -829,6 +934,9 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
                     (ArgumentExtension::Uext, n) | (ArgumentExtension::Sext, n)
                         if n < word_bits =>
                     {
+                        let from_reg = from_regs
+                            .only_reg()
+                            .expect("extension only possible from one-reg value");
                         assert_eq!(M::word_reg_class(), from_reg.to_reg().get_class());
                         let signed = ext == ArgumentExtension::Sext;
                         ret.push(M::gen_extend(
@@ -843,12 +951,15 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
                     }
                     _ => {}
                 };
-                ret.push(M::gen_store_base_offset(
+                ret.extend(
+                    gen_store_base_offset_multi::<M>(
                         self.ret_area_ptr.unwrap().to_reg(),
                         off,
-                    from_reg.to_reg(),
+                        non_writable_value_regs(from_regs),
                         ty,
-                ));
+                    )
+                    .into_iter(),
+                );
             }
         }
         ret
@@ -856,7 +967,8 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
 
     fn gen_retval_area_setup(&self) -> Option<Self::I> {
         if let Some(i) = self.sig.stack_ret_arg {
-            let inst = self.gen_copy_arg_to_reg(i, self.ret_area_ptr.unwrap());
+            let insts = self.gen_copy_arg_to_regs(i, ValueRegs::one(self.ret_area_ptr.unwrap()));
+            let inst = insts.into_iter().next().unwrap();
             trace!(
                 "gen_retval_area_setup: inst {:?}; ptr reg is {:?}",
                 inst,
@@ -891,24 +1003,30 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         slot: StackSlot,
         offset: u32,
         ty: Type,
-        into_reg: Writable<Reg>,
+        into_regs: ValueRegs<Writable<Reg>>,
-    ) -> Self::I {
+    ) -> SmallInstVec<Self::I> {
         // Offset from beginning of stackslot area, which is at nominal SP (see
         // [MemArg::NominalSPOffset] for more details on nominal SP tracking).
         let stack_off = self.stackslots[slot.as_u32() as usize] as i64;
         let sp_off: i64 = stack_off + (offset as i64);
         trace!("load_stackslot: slot {} -> sp_off {}", slot, sp_off);
-        M::gen_load_stack(StackAMode::NominalSPOffset(sp_off, ty), into_reg, ty)
+        gen_load_stack_multi::<M>(StackAMode::NominalSPOffset(sp_off, ty), into_regs, ty)
     }
 
     /// Store to a stackslot.
-    fn store_stackslot(&self, slot: StackSlot, offset: u32, ty: Type, from_reg: Reg) -> Self::I {
+    fn store_stackslot(
+        &self,
+        slot: StackSlot,
+        offset: u32,
+        ty: Type,
+        from_regs: ValueRegs<Reg>,
+    ) -> SmallInstVec<Self::I> {
         // Offset from beginning of stackslot area, which is at nominal SP (see
         // [MemArg::NominalSPOffset] for more details on nominal SP tracking).
         let stack_off = self.stackslots[slot.as_u32() as usize] as i64;
         let sp_off: i64 = stack_off + (offset as i64);
         trace!("store_stackslot: slot {} -> sp_off {}", slot, sp_off);
-        M::gen_store_stack(StackAMode::NominalSPOffset(sp_off, ty), from_reg, ty)
+        gen_store_stack_multi::<M>(StackAMode::NominalSPOffset(sp_off, ty), from_regs, ty)
     }
 
     /// Produce an instruction that computes a stackslot address.
@@ -921,23 +1039,33 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
     }
 
     /// Load from a spillslot.
-    fn load_spillslot(&self, slot: SpillSlot, ty: Type, into_reg: Writable<Reg>) -> Self::I {
+    fn load_spillslot(
+        &self,
+        slot: SpillSlot,
+        ty: Type,
+        into_regs: ValueRegs<Writable<Reg>>,
+    ) -> SmallInstVec<Self::I> {
         // Offset from beginning of spillslot area, which is at nominal SP + stackslots_size.
         let islot = slot.get() as i64;
         let spill_off = islot * M::word_bytes() as i64;
         let sp_off = self.stackslots_size as i64 + spill_off;
         trace!("load_spillslot: slot {:?} -> sp_off {}", slot, sp_off);
-        M::gen_load_stack(StackAMode::NominalSPOffset(sp_off, ty), into_reg, ty)
+        gen_load_stack_multi::<M>(StackAMode::NominalSPOffset(sp_off, ty), into_regs, ty)
     }
 
     /// Store to a spillslot.
-    fn store_spillslot(&self, slot: SpillSlot, ty: Type, from_reg: Reg) -> Self::I {
+    fn store_spillslot(
+        &self,
+        slot: SpillSlot,
+        ty: Type,
+        from_regs: ValueRegs<Reg>,
+    ) -> SmallInstVec<Self::I> {
         // Offset from beginning of spillslot area, which is at nominal SP + stackslots_size.
         let islot = slot.get() as i64;
         let spill_off = islot * M::word_bytes() as i64;
         let sp_off = self.stackslots_size as i64 + spill_off;
         trace!("store_spillslot: slot {:?} -> sp_off {}", slot, sp_off);
-        M::gen_store_stack(StackAMode::NominalSPOffset(sp_off, ty), from_reg, ty)
+        gen_store_stack_multi::<M>(StackAMode::NominalSPOffset(sp_off, ty), from_regs, ty)
     }
 
     fn spillslots_to_stack_map(
@@ -970,8 +1098,8 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         StackMap::from_slice(&bits[..])
     }
 
-    fn gen_prologue(&mut self) -> Vec<Self::I> {
+    fn gen_prologue(&mut self) -> SmallInstVec<Self::I> {
-        let mut insts = vec![];
+        let mut insts = smallvec![];
         if !self.call_conv.extends_baldrdash() {
             // set up frame
             insts.extend(M::gen_prologue_frame_setup().into_iter());
@@ -994,7 +1122,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
             // specified, otherwise always insert the stack check.
             if total_stacksize > 0 || !self.is_leaf {
                 if let Some((reg, stack_limit_load)) = &self.stack_limit {
-                    insts.extend_from_slice(stack_limit_load);
+                    insts.extend(stack_limit_load.clone());
                     self.insert_stack_check(*reg, total_stacksize, &mut insts);
                 }
                 if let Some(min_frame) = &self.probestack_min_frame {
@@ -1037,8 +1165,8 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         insts
     }
 
-    fn gen_epilogue(&self) -> Vec<M::I> {
+    fn gen_epilogue(&self) -> SmallInstVec<M::I> {
-        let mut insts = vec![];
+        let mut insts = smallvec![];
 
         // Restore clobbered registers.
         insts.extend(M::gen_clobber_restore(
@@ -1079,7 +1207,10 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
 
     fn gen_spill(&self, to_slot: SpillSlot, from_reg: RealReg, ty: Option<Type>) -> Self::I {
         let ty = ty_from_ty_hint_or_reg_class::<M>(from_reg.to_reg(), ty);
-        self.store_spillslot(to_slot, ty, from_reg.to_reg())
+        self.store_spillslot(to_slot, ty, ValueRegs::one(from_reg.to_reg()))
+            .into_iter()
+            .next()
+            .unwrap()
     }
 
     fn gen_reload(
@@ -1089,7 +1220,14 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         ty: Option<Type>,
     ) -> Self::I {
         let ty = ty_from_ty_hint_or_reg_class::<M>(to_reg.to_reg().to_reg(), ty);
-        self.load_spillslot(from_slot, ty, to_reg.map(|r| r.to_reg()))
+        self.load_spillslot(
+            from_slot,
+            ty,
+            writable_value_regs(ValueRegs::one(to_reg.to_reg().to_reg())),
+        )
+        .into_iter()
+        .next()
+        .unwrap()
     }
 
     fn unwind_info_kind(&self) -> UnwindInfoKind {
@@ -1110,7 +1248,7 @@ fn abisig_to_uses_and_defs<M: ABIMachineSpec>(sig: &ABISig) -> (Vec<Reg>, Vec<Wr
     let mut uses = Vec::new();
     for arg in &sig.args {
         match arg {
-            &ABIArg::Reg(reg, ..) => uses.push(reg.to_reg()),
+            &ABIArg::Reg(regs, ..) => uses.extend(regs.regs().iter().map(|r| r.to_reg())),
             _ => {}
         }
     }
@@ -1119,7 +1257,9 @@ fn abisig_to_uses_and_defs<M: ABIMachineSpec>(sig: &ABISig) -> (Vec<Reg>, Vec<Wr
     let mut defs = M::get_regs_clobbered_by_call(sig.call_conv);
     for ret in &sig.rets {
         match ret {
-            &ABIArg::Reg(reg, ..) => defs.push(Writable::from_reg(reg.to_reg())),
+            &ABIArg::Reg(regs, ..) => {
+                defs.extend(regs.regs().iter().map(|r| Writable::from_reg(r.to_reg())))
+            }
             _ => {}
         }
     }
@@ -1238,18 +1378,19 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
         adjust_stack_and_nominal_sp::<M, C>(ctx, off as i32, /* is_sub = */ false)
     }
 
-    fn emit_copy_reg_to_arg<C: LowerCtx<I = Self::I>>(
+    fn emit_copy_regs_to_arg<C: LowerCtx<I = Self::I>>(
         &self,
         ctx: &mut C,
         idx: usize,
-        from_reg: Reg,
+        from_regs: ValueRegs<Reg>,
     ) {
         let word_rc = M::word_reg_class();
         let word_bits = M::word_bits() as usize;
         match &self.sig.args[idx] {
-            &ABIArg::Reg(reg, ty, ext, _) => {
+            &ABIArg::Reg(regs, ty, ext, _) => {
                 let ext = M::get_ext_mode(self.sig.call_conv, ext);
                 if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits {
+                    let reg = regs.only_reg().unwrap();
                     assert_eq!(word_rc, reg.get_class());
                     let signed = match ext {
                         ir::ArgumentExtension::Uext => false,
@@ -1258,18 +1399,27 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
                     };
                     ctx.emit(M::gen_extend(
                         Writable::from_reg(reg.to_reg()),
-                        from_reg,
+                        from_regs.only_reg().unwrap(),
                         signed,
                         ty_bits(ty) as u8,
                         word_bits as u8,
                     ));
                 } else {
-                    ctx.emit(M::gen_move(Writable::from_reg(reg.to_reg()), from_reg, ty));
+                    for insn in gen_move_multi::<M>(
+                        writable_value_regs(regs.map(|r| r.to_reg())),
+                        from_regs,
+                        ty,
+                    ) {
+                        ctx.emit(insn);
+                    }
                 }
             }
             &ABIArg::Stack(off, mut ty, ext, _) => {
                 let ext = M::get_ext_mode(self.sig.call_conv, ext);
                 if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits {
+                    let from_reg = from_regs
+                        .only_reg()
+                        .expect("only one reg for sub-word value width");
                     assert_eq!(word_rc, from_reg.get_class());
                     let signed = match ext {
                         ir::ArgumentExtension::Uext => false,
@@ -1289,32 +1439,37 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
                     // Store the extended version.
                     ty = M::word_type();
                 }
-                ctx.emit(M::gen_store_stack(
+                for insn in gen_store_stack_multi::<M>(StackAMode::SPOffset(off, ty), from_regs, ty)
-                    StackAMode::SPOffset(off, ty),
+                {
-                    from_reg,
+                    ctx.emit(insn);
-                    ty,
+                }
-                ));
             }
         }
     }
 
-    fn emit_copy_retval_to_reg<C: LowerCtx<I = Self::I>>(
+    fn emit_copy_retval_to_regs<C: LowerCtx<I = Self::I>>(
         &self,
         ctx: &mut C,
         idx: usize,
-        into_reg: Writable<Reg>,
+        into_regs: ValueRegs<Writable<Reg>>,
     ) {
         match &self.sig.rets[idx] {
             // Extension mode doesn't matter because we're copying out, not in,
             // and we ignore high bits in our own registers by convention.
-            &ABIArg::Reg(reg, ty, _, _) => ctx.emit(M::gen_move(into_reg, reg.to_reg(), ty)),
+            &ABIArg::Reg(regs, ty, _, _) => {
+                for insn in gen_move_multi::<M>(into_regs, regs.map(|r| r.to_reg()), ty) {
+                    ctx.emit(insn);
+                }
+            }
             &ABIArg::Stack(off, ty, _, _) => {
                 let ret_area_base = self.sig.stack_arg_space;
-                ctx.emit(M::gen_load_stack(
+                for insn in gen_load_stack_multi::<M>(
                     StackAMode::SPOffset(off + ret_area_base, ty),
-                    into_reg,
+                    into_regs,
                     ty,
-                ));
+                ) {
+                    ctx.emit(insn);
+                }
             }
         }
     }
@@ -1324,19 +1479,18 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
             mem::replace(&mut self.uses, Default::default()),
             mem::replace(&mut self.defs, Default::default()),
         );
-        let word_rc = M::word_reg_class();
         let word_type = M::word_type();
         if let Some(i) = self.sig.stack_ret_arg {
-            let rd = ctx.alloc_tmp(word_rc, word_type);
+            let rd = ctx.alloc_tmp(word_type).only_reg().unwrap();
             let ret_area_base = self.sig.stack_arg_space;
             ctx.emit(M::gen_get_stack_addr(
                 StackAMode::SPOffset(ret_area_base, I8),
                 rd,
                 I8,
             ));
-            self.emit_copy_reg_to_arg(ctx, i, rd.to_reg());
+            self.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(rd.to_reg()));
         }
-        let tmp = ctx.alloc_tmp(word_rc, word_type);
+        let tmp = ctx.alloc_tmp(word_type).only_reg().unwrap();
         for (is_safepoint, inst) in M::gen_call(
             &self.dest,
             uses,

cranelift/codegen/src/machinst/helpers.rs

@@ -1,6 +1,6 @@
 //! Miscellaneous helpers for machine backends.
 
-use super::{InsnOutput, LowerCtx, VCodeInst};
+use super::{InsnOutput, LowerCtx, VCodeInst, ValueRegs};
 use crate::ir::Type;
 use regalloc::{Reg, Writable};
 
@@ -23,6 +23,6 @@ pub(crate) fn ty_has_float_or_vec_representation(ty: Type) -> bool {
 pub(crate) fn get_output_reg<I: VCodeInst, C: LowerCtx<I = I>>(
     ctx: &mut C,
     spec: InsnOutput,
-) -> Writable<Reg> {
+) -> ValueRegs<Writable<Reg>> {
     ctx.get_output(spec.insn, spec.output)
 }

cranelift/codegen/src/machinst/lower.rs

@@ -5,29 +5,27 @@
 // TODO: separate the IR-query core of `LowerCtx` from the lowering logic built
 // on top of it, e.g. the side-effect/coloring analysis and the scan support.
 
+use crate::data_value::DataValue;
 use crate::entity::SecondaryMap;
 use crate::fx::{FxHashMap, FxHashSet};
 use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit};
 use crate::ir::instructions::BranchInfo;
-use crate::ir::types::I64;
 use crate::ir::{
     ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst,
     InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef,
 };
 use crate::machinst::{
-    ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode, VCodeBuilder,
+    writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode,
-    VCodeConstant, VCodeConstantData, VCodeConstants, VCodeInst,
+    VCodeBuilder, VCodeConstant, VCodeConstantData, VCodeConstants, VCodeInst, ValueRegs,
 };
 use crate::CodegenResult;
-
-use regalloc::{Reg, RegClass, StackmapRequestInfo, VirtualReg, Writable};
-
-use crate::data_value::DataValue;
 use alloc::boxed::Box;
 use alloc::vec::Vec;
 use core::convert::TryInto;
 use log::debug;
+use regalloc::{Reg, StackmapRequestInfo, Writable};
 use smallvec::SmallVec;
+use std::fmt::Debug;
 
 /// An "instruction color" partitions CLIF instructions by side-effecting ops.
 /// All instructions with the same "color" are guaranteed not to be separated by
@@ -71,7 +69,7 @@ pub trait LowerCtx {
     /// instruction should lower into a sequence that fills this register. (Why
     /// not allow the backend to specify its own result register for the return?
     /// Because there may be multiple return points.)
-    fn retval(&self, idx: usize) -> Writable<Reg>;
+    fn retval(&self, idx: usize) -> ValueRegs<Writable<Reg>>;
     /// Returns the vreg containing the VmContext parameter, if there's one.
     fn get_vm_context(&self) -> Option<Reg>;
 
@@ -118,7 +116,7 @@ pub trait LowerCtx {
     ///
     /// The instruction input may be available in either of these forms.  It may
     /// be available in neither form, if the conditions are not met; if so, use
-    /// `put_input_in_reg()` instead to get it in a register.
+    /// `put_input_in_regs()` instead to get it in a register.
     ///
     /// If the backend merges the effect of a side-effecting instruction, it
     /// must call `sink_inst()`. When this is called, it indicates that the
@@ -126,29 +124,29 @@ pub trait LowerCtx {
     /// instruction's result(s) must have *no* uses remaining, because it will
     /// not be codegen'd (it has been integrated into the current instruction).
     fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput;
-    /// Put the `idx`th input into a register and return the assigned register.
+    /// Put the `idx`th input into register(s) and return the assigned register.
-    fn put_input_in_reg(&mut self, ir_inst: Inst, idx: usize) -> Reg;
+    fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg>;
-    /// Get the `idx`th output register of the given IR instruction. When
+    /// Get the `idx`th output register(s) of the given IR instruction. When
     /// `backend.lower_inst_to_regs(ctx, inst)` is called, it is expected that
     /// the backend will write results to these output register(s).  This
     /// register will always be "fresh"; it is guaranteed not to overlap with
     /// any of the inputs, and can be freely used as a scratch register within
     /// the lowered instruction sequence, as long as its final value is the
     /// result of the computation.
-    fn get_output(&self, ir_inst: Inst, idx: usize) -> Writable<Reg>;
+    fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>>;
 
     // Codegen primitives: allocate temps, emit instructions, set result registers,
     // ask for an input to be gen'd into a register.
 
     /// Get a new temp.
-    fn alloc_tmp(&mut self, rc: RegClass, ty: Type) -> Writable<Reg>;
+    fn alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>>;
     /// Emit a machine instruction.
     fn emit(&mut self, mach_inst: Self::I);
     /// Emit a machine instruction that is a safepoint.
     fn emit_safepoint(&mut self, mach_inst: Self::I);
     /// Indicate that the side-effect of an instruction has been sunk to the
     /// current scan location. This should only be done with the instruction's
-    /// original results are not used (i.e., `put_input_in_reg` is not invoked
+    /// original results are not used (i.e., `put_input_in_regs` is not invoked
     /// for the input produced by the sunk instruction), otherwise the
     /// side-effect will occur twice.
     fn sink_inst(&mut self, ir_inst: Inst);
@@ -234,10 +232,10 @@ pub struct Lower<'func, I: VCodeInst> {
     vcode: VCodeBuilder<I>,
 
     /// Mapping from `Value` (SSA value in IR) to virtual register.
-    value_regs: SecondaryMap<Value, Reg>,
+    value_regs: SecondaryMap<Value, ValueRegs<Reg>>,
 
     /// Return-value vregs.
-    retval_regs: Vec<Reg>,
+    retval_regs: Vec<ValueRegs<Reg>>,
 
     /// Instruction colors at block exits. From this map, we can recover all
     /// instruction colors by scanning backward from the block end and
@@ -306,20 +304,30 @@ pub enum RelocDistance {
     Far,
 }
 
-fn alloc_vreg(
+fn alloc_vregs<I: VCodeInst>(
-    value_regs: &mut SecondaryMap<Value, Reg>,
+    ty: Type,
-    regclass: RegClass,
-    value: Value,
     next_vreg: &mut u32,
-) -> VirtualReg {
+    vcode: &mut VCodeBuilder<I>,
-    if value_regs[value].is_invalid() {
+) -> CodegenResult<ValueRegs<Reg>> {
-        // default value in map.
     let v = *next_vreg;
-        *next_vreg += 1;
+    let (regclasses, tys) = I::rc_for_type(ty)?;
-        value_regs[value] = Reg::new_virtual(regclass, v);
+    *next_vreg += regclasses.len() as u32;
-        debug!("value {} gets vreg {:?}", value, v);
+    let regs = match regclasses {
+        &[rc0] => ValueRegs::one(Reg::new_virtual(rc0, v)),
+        &[rc0, rc1] => ValueRegs::two(Reg::new_virtual(rc0, v), Reg::new_virtual(rc1, v + 1)),
+        #[cfg(feature = "arm32")]
+        &[rc0, rc1, rc2, rc3] => ValueRegs::four(
+            Reg::new_virtual(rc0, v),
+            Reg::new_virtual(rc1, v + 1),
+            Reg::new_virtual(rc2, v + 2),
+            Reg::new_virtual(rc3, v + 3),
+        ),
+        _ => panic!("Value must reside in 1, 2 or 4 registers"),
+    };
+    for (&reg_ty, &reg) in tys.iter().zip(regs.regs().iter()) {
+        vcode.set_vreg_type(reg.to_virtual_reg(), reg_ty);
     }
-    value_regs[value].as_virtual_reg().unwrap()
+    Ok(regs)
 }
 
 enum GenerateReturn {
@@ -340,29 +348,32 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
 
         let mut next_vreg: u32 = 0;
 
-        let mut value_regs = SecondaryMap::with_default(Reg::invalid());
+        let mut value_regs = SecondaryMap::with_default(ValueRegs::invalid());
 
         // Assign a vreg to each block param and each inst result.
         for bb in f.layout.blocks() {
             for &param in f.dfg.block_params(bb) {
                 let ty = f.dfg.value_type(param);
-                let vreg = alloc_vreg(&mut value_regs, I::rc_for_type(ty)?, param, &mut next_vreg);
+                if value_regs[param].is_invalid() {
-                vcode.set_vreg_type(vreg, ty);
+                    let regs = alloc_vregs(ty, &mut next_vreg, &mut vcode)?;
-                debug!("bb {} param {}: vreg {:?}", bb, param, vreg);
+                    value_regs[param] = regs;
+                    debug!("bb {} param {}: regs {:?}", bb, param, regs);
+                }
             }
             for inst in f.layout.block_insts(bb) {
                 for &result in f.dfg.inst_results(inst) {
                     let ty = f.dfg.value_type(result);
-                    let vreg =
+                    if value_regs[result].is_invalid() {
-                        alloc_vreg(&mut value_regs, I::rc_for_type(ty)?, result, &mut next_vreg);
+                        let regs = alloc_vregs(ty, &mut next_vreg, &mut vcode)?;
-                    vcode.set_vreg_type(vreg, ty);
+                        value_regs[result] = regs;
                         debug!(
-                        "bb {} inst {} ({:?}): result vreg {:?}",
+                            "bb {} inst {} ({:?}): result regs {:?}",
-                        bb, inst, f.dfg[inst], vreg
+                            bb, inst, f.dfg[inst], regs,
                         );
                     }
                 }
             }
+        }
 
         let vm_context = f
             .signature
@@ -370,18 +381,15 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             .map(|vm_context_index| {
                 let entry_block = f.layout.entry_block().unwrap();
                 let param = f.dfg.block_params(entry_block)[vm_context_index];
-                value_regs[param]
+                value_regs[param].only_reg().unwrap()
             });
 
-        // Assign a vreg to each return value.
+        // Assign vreg(s) to each return value.
         let mut retval_regs = vec![];
         for ret in &f.signature.returns {
-            let v = next_vreg;
+            let regs = alloc_vregs(ret.value_type, &mut next_vreg, &mut vcode)?;
-            next_vreg += 1;
+            retval_regs.push(regs);
-            let regclass = I::rc_for_type(ret.value_type)?;
+            debug!("retval gets regs {:?}", regs);
-            let vreg = Reg::new_virtual(regclass, v);
-            retval_regs.push(vreg);
-            vcode.set_vreg_type(vreg.as_virtual_reg().unwrap(), ret.value_type);
         }
 
         // Compute instruction colors, find constant instructions, and find instructions with
@@ -453,10 +461,11 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
                 if !self.vcode.abi().arg_is_needed_in_body(i) {
                     continue;
                 }
-                let reg = Writable::from_reg(self.value_regs[*param]);
+                let regs = writable_value_regs(self.value_regs[*param]);
-                let insn = self.vcode.abi().gen_copy_arg_to_reg(i, reg);
+                for insn in self.vcode.abi().gen_copy_arg_to_regs(i, regs).into_iter() {
                     self.emit(insn);
                 }
+            }
             if let Some(insn) = self.vcode.abi().gen_retval_area_setup() {
                 self.emit(insn);
             }
@@ -465,10 +474,14 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
 
     fn gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn) {
         let retval_regs = self.retval_regs.clone();
-        for (i, reg) in retval_regs.into_iter().enumerate() {
+        for (i, regs) in retval_regs.into_iter().enumerate() {
-            let reg = Writable::from_reg(reg);
+            let regs = writable_value_regs(regs);
-            let insns = self.vcode.abi().gen_copy_reg_to_retval(i, reg);
+            for insn in self
-            for insn in insns {
+                .vcode
+                .abi()
+                .gen_copy_regs_to_retval(i, regs)
+                .into_iter()
+            {
                 self.emit(insn);
             }
         }
@@ -499,8 +512,8 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         //
         // * one for dsts whose sources are non-constants.
 
-        let mut const_bundles = SmallVec::<[(Type, Writable<Reg>, u64); 16]>::new();
+        let mut const_bundles: SmallVec<[_; 16]> = SmallVec::new();
-        let mut var_bundles = SmallVec::<[(Type, Writable<Reg>, Reg); 16]>::new();
+        let mut var_bundles: SmallVec<[_; 16]> = SmallVec::new();
 
         let mut i = 0;
         for (dst_val, src_val) in self
@@ -514,7 +527,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             let ty = self.f.dfg.value_type(src_val);
 
             debug_assert!(ty == self.f.dfg.value_type(*dst_val));
-            let dst_reg = self.value_regs[*dst_val];
+            let dst_regs = self.value_regs[*dst_val];
 
             let input = self.get_value_as_source_or_const(src_val);
             debug!("jump arg {} is {}", i, src_val);
@@ -522,15 +535,15 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
 
             if let Some(c) = input.constant {
                 debug!(" -> constant {}", c);
-                const_bundles.push((ty, Writable::from_reg(dst_reg), c));
+                const_bundles.push((ty, writable_value_regs(dst_regs), c));
             } else {
-                let src_reg = self.put_value_in_reg(src_val);
+                let src_regs = self.put_value_in_regs(src_val);
-                debug!(" -> reg {:?}", src_reg);
+                debug!(" -> reg {:?}", src_regs);
                 // Skip self-assignments.  Not only are they pointless, they falsely trigger the
                 // overlap-check below and hence can cause a lot of unnecessary copying through
                 // temporaries.
-                if dst_reg != src_reg {
+                if dst_regs != src_regs {
-                    var_bundles.push((ty, Writable::from_reg(dst_reg), src_reg));
+                    var_bundles.push((ty, writable_value_regs(dst_regs), src_regs));
                 }
             }
         }
@@ -541,41 +554,69 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         // for cases of up to circa 16 args.  Currently not possible because regalloc.rs
         // does not export it.
         let mut src_reg_set = FxHashSet::<Reg>::default();
-        for (_, _, src_reg) in &var_bundles {
+        for (_, _, src_regs) in &var_bundles {
-            src_reg_set.insert(*src_reg);
+            for &reg in src_regs.regs() {
+                src_reg_set.insert(reg);
+            }
         }
         let mut overlaps = false;
-        for (_, dst_reg, _) in &var_bundles {
+        'outer: for (_, dst_regs, _) in &var_bundles {
-            if src_reg_set.contains(&dst_reg.to_reg()) {
+            for &reg in dst_regs.regs() {
+                if src_reg_set.contains(&reg.to_reg()) {
                     overlaps = true;
-                break;
+                    break 'outer;
+                }
             }
         }
 
         // If, as is mostly the case, the source and destination register sets are non
         // overlapping, then we can copy directly, so as to save the register allocator work.
         if !overlaps {
-            for (ty, dst_reg, src_reg) in &var_bundles {
+            for (ty, dst_regs, src_regs) in &var_bundles {
-                self.emit(I::gen_move(*dst_reg, *src_reg, *ty));
+                let (_, reg_tys) = I::rc_for_type(*ty)?;
+                for ((dst, src), reg_ty) in dst_regs
+                    .regs()
+                    .iter()
+                    .zip(src_regs.regs().iter())
+                    .zip(reg_tys.iter())
+                {
+                    self.emit(I::gen_move(*dst, *src, *reg_ty));
+                }
             }
         } else {
             // There's some overlap, so play safe and copy via temps.
-            let mut tmp_regs = SmallVec::<[Writable<Reg>; 16]>::new();
+            let mut tmp_regs = SmallVec::<[ValueRegs<Writable<Reg>>; 16]>::new();
             for (ty, _, _) in &var_bundles {
-                tmp_regs.push(self.alloc_tmp(I::rc_for_type(*ty)?, *ty));
+                tmp_regs.push(self.alloc_tmp(*ty));
             }
             for ((ty, _, src_reg), tmp_reg) in var_bundles.iter().zip(tmp_regs.iter()) {
-                self.emit(I::gen_move(*tmp_reg, *src_reg, *ty));
+                let (_, reg_tys) = I::rc_for_type(*ty)?;
+                for ((tmp, src), reg_ty) in tmp_reg
+                    .regs()
+                    .iter()
+                    .zip(src_reg.regs().iter())
+                    .zip(reg_tys.iter())
+                {
+                    self.emit(I::gen_move(*tmp, *src, *reg_ty));
+                }
             }
             for ((ty, dst_reg, _), tmp_reg) in var_bundles.iter().zip(tmp_regs.iter()) {
-                self.emit(I::gen_move(*dst_reg, (*tmp_reg).to_reg(), *ty));
+                let (_, reg_tys) = I::rc_for_type(*ty)?;
+                for ((dst, tmp), reg_ty) in dst_reg
+                    .regs()
+                    .iter()
+                    .zip(tmp_reg.regs().iter())
+                    .zip(reg_tys.iter())
+                {
+                    self.emit(I::gen_move(*dst, tmp.to_reg(), *reg_ty));
+                }
             }
         }
 
         // Now, finally, deal with the moves whose sources are constants.
-        for (ty, dst_reg, const_u64) in &const_bundles {
+        for (ty, dst_reg, const_val) in &const_bundles {
-            for inst in I::gen_constant(*dst_reg, *const_u64, *ty, |reg_class, ty| {
+            for inst in I::gen_constant(*dst_reg, *const_val as u128, *ty, |ty| {
-                self.alloc_tmp(reg_class, ty)
+                self.alloc_tmp(ty).only_reg().unwrap()
             })
             .into_iter()
             {
@@ -766,8 +807,8 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         debug!("about to lower function: {:?}", self.f);
 
         // Initialize the ABI object, giving it a temp if requested.
-        let maybe_tmp = if self.vcode.abi().temp_needed() {
+        let maybe_tmp = if let Some(temp_ty) = self.vcode.abi().temp_needed() {
-            Some(self.alloc_tmp(RegClass::I64, I64))
+            Some(self.alloc_tmp(temp_ty).only_reg().unwrap())
         } else {
             None
         };
@@ -848,11 +889,11 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         Ok((vcode, stack_map_info))
     }
 
-    fn put_value_in_reg(&mut self, val: Value) -> Reg {
+    fn put_value_in_regs(&mut self, val: Value) -> ValueRegs<Reg> {
-        debug!("put_value_in_reg: val {}", val,);
+        debug!("put_value_in_reg: val {}", val);
-        let mut reg = self.value_regs[val];
+        let mut regs = self.value_regs[val];
-        debug!(" -> reg {:?}", reg);
+        debug!(" -> regs {:?}", regs);
-        assert!(reg.is_valid());
+        assert!(regs.is_valid());
 
         self.value_lowered_uses[val] += 1;
 
@@ -864,12 +905,12 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         if let ValueDef::Result(i, 0) = self.f.dfg.value_def(val) {
             if self.f.dfg[i].opcode() == Opcode::GetPinnedReg {
                 if let Some(pr) = self.pinned_reg {
-                    reg = pr;
+                    regs = ValueRegs::one(pr);
                 }
             }
         }
 
-        reg
+        regs
     }
 
     /// Get the actual inputs for a value. This is the implementation for
@@ -944,8 +985,8 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
         self.vcode.abi()
     }
 
-    fn retval(&self, idx: usize) -> Writable<Reg> {
+    fn retval(&self, idx: usize) -> ValueRegs<Writable<Reg>> {
-        Writable::from_reg(self.retval_regs[idx])
+        writable_value_regs(self.retval_regs[idx])
     }
 
     fn get_vm_context(&self) -> Option<Reg> {
@@ -1050,23 +1091,19 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
         self.get_value_as_source_or_const(val)
     }
 
-    fn put_input_in_reg(&mut self, ir_inst: Inst, idx: usize) -> Reg {
+    fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg> {
         let val = self.f.dfg.inst_args(ir_inst)[idx];
         let val = self.f.dfg.resolve_aliases(val);
-        self.put_value_in_reg(val)
+        self.put_value_in_regs(val)
     }
 
-    fn get_output(&self, ir_inst: Inst, idx: usize) -> Writable<Reg> {
+    fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>> {
         let val = self.f.dfg.inst_results(ir_inst)[idx];
-        Writable::from_reg(self.value_regs[val])
+        writable_value_regs(self.value_regs[val])
     }
 
-    fn alloc_tmp(&mut self, rc: RegClass, ty: Type) -> Writable<Reg> {
+    fn alloc_tmp(&mut self, ty: Type) -> ValueRegs<Writable<Reg>> {
-        let v = self.next_vreg;
+        writable_value_regs(alloc_vregs(ty, &mut self.next_vreg, &mut self.vcode).unwrap())
-        self.next_vreg += 1;
-        let vreg = Reg::new_virtual(rc, v);
-        self.vcode.set_vreg_type(vreg.as_virtual_reg().unwrap(), ty);
-        Writable::from_reg(vreg)
     }
 
     fn emit(&mut self, mach_inst: I) {
@@ -1131,8 +1168,7 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
         if reg.is_virtual() {
             reg
         } else {
-            let rc = reg.get_class();
+            let new_reg = self.alloc_tmp(ty).only_reg().unwrap();
-            let new_reg = self.alloc_tmp(rc, ty);
             self.emit(I::gen_move(new_reg, reg, ty));
             new_reg.to_reg()
         }

cranelift/codegen/src/machinst/mod.rs

@@ -135,6 +135,8 @@ pub mod helpers;
 pub use helpers::*;
 pub mod inst_common;
 pub use inst_common::*;
+pub mod valueregs;
+pub use valueregs::*;
 
 /// A machine instruction.
 pub trait MachInst: Clone + Debug {
@@ -165,9 +167,9 @@ pub trait MachInst: Clone + Debug {
     fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self;
 
     /// Generate a constant into a reg.
-    fn gen_constant<F: FnMut(RegClass, Type) -> Writable<Reg>>(
+    fn gen_constant<F: FnMut(Type) -> Writable<Reg>>(
-        to_reg: Writable<Reg>,
+        to_regs: ValueRegs<Writable<Reg>>,
-        value: u64,
+        value: u128,
         ty: Type,
         alloc_tmp: F,
     ) -> SmallVec<[Self; 4]>;
@@ -180,9 +182,19 @@ pub trait MachInst: Clone + Debug {
     /// (e.g., add directly from or directly to memory), like x86.
     fn maybe_direct_reload(&self, reg: VirtualReg, slot: SpillSlot) -> Option<Self>;
 
-    /// Determine a register class to store the given Cranelift type.
+    /// Determine register class(es) to store the given Cranelift type, and the
-    /// May return an error if the type isn't supported by this backend.
+    /// Cranelift type actually stored in the underlying register(s).  May return
-    fn rc_for_type(ty: Type) -> CodegenResult<RegClass>;
+    /// an error if the type isn't supported by this backend.
+    ///
+    /// If the type requires multiple registers, then the list of registers is
+    /// returned in little-endian order.
+    ///
+    /// Note that the type actually stored in the register(s) may differ in the
+    /// case that a value is split across registers: for example, on a 32-bit
+    /// target, an I64 may be stored in two registers, each of which holds an
+    /// I32. The actually-stored types are used only to inform the backend when
+    /// generating spills and reloads for individual registers.
+    fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])>;
 
     /// Generate a jump to another target. Used during lowering of
     /// control flow.

cranelift/codegen/src/machinst/valueregs.rs

@@ -0,0 +1,185 @@
+//! Data structure for tracking the (possibly multiple) registers that hold one
+//! SSA `Value`.
+
+use regalloc::{RealReg, Reg, VirtualReg, Writable};
+use std::fmt::Debug;
+
+#[cfg(feature = "arm32")]
+const VALUE_REGS_PARTS: usize = 4;
+
+#[cfg(not(feature = "arm32"))]
+const VALUE_REGS_PARTS: usize = 2;
+
+/// Location at which a `Value` is stored in register(s): the value is located
+/// in one or more registers, depending on its width. A value may be stored in
+/// more than one register if the machine has no registers wide enough
+/// otherwise: for example, on a 32-bit architecture, we may store `I64` values
+/// in two registers, and `I128` values in four.
+///
+/// By convention, the register parts are kept in machine-endian order here.
+///
+/// N.B.: we cap the capacity of this at four (when any 32-bit target is
+/// enabled) or two (otherwise), and we use special in-band sentinal `Reg`
+/// values (`Reg::invalid()`) to avoid the need to carry a separate length. This
+/// allows the struct to be `Copy` (no heap or drop overhead) and be only 16 or
+/// 8 bytes, which is important for compiler performance.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct ValueRegs<R: Clone + Copy + Debug + PartialEq + Eq + InvalidSentinel> {
+    parts: [R; VALUE_REGS_PARTS],
+}
+
+/// A type with an "invalid" sentinel value.
+pub trait InvalidSentinel: Copy + Eq {
+    /// The invalid sentinel value.
+    fn invalid_sentinel() -> Self;
+    /// Is this the invalid sentinel?
+    fn is_invalid_sentinel(self) -> bool {
+        self == Self::invalid_sentinel()
+    }
+}
+impl InvalidSentinel for Reg {
+    fn invalid_sentinel() -> Self {
+        Reg::invalid()
+    }
+}
+impl InvalidSentinel for VirtualReg {
+    fn invalid_sentinel() -> Self {
+        VirtualReg::invalid()
+    }
+}
+impl InvalidSentinel for RealReg {
+    fn invalid_sentinel() -> Self {
+        RealReg::invalid()
+    }
+}
+impl InvalidSentinel for Writable<Reg> {
+    fn invalid_sentinel() -> Self {
+        Writable::from_reg(Reg::invalid_sentinel())
+    }
+}
+
+impl<R: Clone + Copy + Debug + PartialEq + Eq + InvalidSentinel> ValueRegs<R> {
+    /// Create an invalid Value-in-Reg.
+    pub fn invalid() -> Self {
+        ValueRegs {
+            parts: [R::invalid_sentinel(); VALUE_REGS_PARTS],
+        }
+    }
+
+    /// Is this Value-to-Reg mapping valid?
+    pub fn is_valid(self) -> bool {
+        !self.parts[0].is_invalid_sentinel()
+    }
+    /// Is this Value-to-Reg mapping invalid?
+    pub fn is_invalid(self) -> bool {
+        self.parts[0].is_invalid_sentinel()
+    }
+
+    /// Return the single register used for this value, if any.
+    pub fn only_reg(self) -> Option<R> {
+        if self.len() == 1 {
+            Some(self.parts[0])
+        } else {
+            None
+        }
+    }
+
+    /// Return an iterator over the registers storing this value.
+    pub fn regs(&self) -> &[R] {
+        &self.parts[0..self.len()]
+    }
+}
+
+#[cfg(feature = "arm32")]
+impl<R: Clone + Copy + Debug + PartialEq + Eq + InvalidSentinel> ValueRegs<R> {
+    /// Create a Value-in-R location for a value stored in one register.
+    pub fn one(reg: R) -> Self {
+        ValueRegs {
+            parts: [
+                reg,
+                R::invalid_sentinel(),
+                R::invalid_sentinel(),
+                R::invalid_sentinel(),
+            ],
+        }
+    }
+    /// Create a Value-in-R location for a value stored in two registers.
+    pub fn two(r1: R, r2: R) -> Self {
+        ValueRegs {
+            parts: [r1, r2, R::invalid_sentinel(), R::invalid_sentinel()],
+        }
+    }
+    /// Create a Value-in-R location for a value stored in four registers.
+    pub fn four(r1: R, r2: R, r3: R, r4: R) -> Self {
+        ValueRegs {
+            parts: [r1, r2, r3, r4],
+        }
+    }
+
+    /// Return the number of registers used.
+    pub fn len(self) -> usize {
+        // If rustc/LLVM is smart enough, this might even be vectorized...
+        (self.parts[0] != R::invalid_sentinel()) as usize
+            + (self.parts[1] != R::invalid_sentinel()) as usize
+            + (self.parts[2] != R::invalid_sentinel()) as usize
+            + (self.parts[3] != R::invalid_sentinel()) as usize
+    }
+
+    /// Map individual registers via a map function.
+    pub fn map<NewR, F>(self, f: F) -> ValueRegs<NewR>
+    where
+        NewR: Clone + Copy + Debug + PartialEq + Eq + InvalidSentinel,
+        F: Fn(R) -> NewR,
+    {
+        ValueRegs {
+            parts: [
+                f(self.parts[0]),
+                f(self.parts[1]),
+                f(self.parts[2]),
+                f(self.parts[3]),
+            ],
+        }
+    }
+}
+
+#[cfg(not(feature = "arm32"))]
+impl<R: Clone + Copy + Debug + PartialEq + Eq + InvalidSentinel> ValueRegs<R> {
+    /// Create a Value-in-R location for a value stored in one register.
+    pub fn one(reg: R) -> Self {
+        ValueRegs {
+            parts: [reg, R::invalid_sentinel()],
+        }
+    }
+    /// Create a Value-in-R location for a value stored in two registers.
+    pub fn two(r1: R, r2: R) -> Self {
+        ValueRegs { parts: [r1, r2] }
+    }
+
+    /// Return the number of registers used.
+    pub fn len(self) -> usize {
+        // If rustc/LLVM is smart enough, this might even be vectorized...
+        (self.parts[0] != R::invalid_sentinel()) as usize
+            + (self.parts[1] != R::invalid_sentinel()) as usize
+    }
+
+    /// Map individual registers via a map function.
+    pub fn map<NewR, F>(self, f: F) -> ValueRegs<NewR>
+    where
+        NewR: Clone + Copy + Debug + PartialEq + Eq + InvalidSentinel,
+        F: Fn(R) -> NewR,
+    {
+        ValueRegs {
+            parts: [f(self.parts[0]), f(self.parts[1])],
+        }
+    }
+}
+
+/// Create a writable ValueRegs.
+pub(crate) fn writable_value_regs(regs: ValueRegs<Reg>) -> ValueRegs<Writable<Reg>> {
+    regs.map(|r| Writable::from_reg(r))
+}
+
+/// Strip a writable ValueRegs down to a readonly ValueRegs.
+pub(crate) fn non_writable_value_regs(regs: ValueRegs<Writable<Reg>>) -> ValueRegs<Reg> {
+    regs.map(|r| r.to_reg())
+}