Merge pull request #1530 from bnjbvr/bbouvier-arm64-fixes

Pending arm64 fixes for Spidermonkey integration

cranelift/codegen/src/context.rs

@@ -180,7 +180,7 @@ impl Context {
         }
 
         if let Some(backend) = isa.get_mach_backend() {
-            let result = backend.compile_function(&mut self.func, self.want_disasm)?;
+            let result = backend.compile_function(&self.func, self.want_disasm)?;
             let info = result.code_info();
             self.mach_compile_result = Some(result);
             Ok(info)

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -3,9 +3,9 @@
 use crate::ir;
 use crate::ir::types;
 use crate::ir::types::*;
-use crate::ir::StackSlot;
+use crate::ir::{ArgumentExtension, StackSlot};
 use crate::isa;
-use crate::isa::aarch64::inst::*;
+use crate::isa::aarch64::{self, inst::*};
 use crate::machinst::*;
 use crate::settings;
 
@@ -58,7 +58,7 @@ static BALDRDASH_JIT_CALLEE_SAVED_GPR: &[bool] = &[
     /* 24 = */ false, false, false, false,
     // There should be 28, the pseudo stack pointer in this list, however the wasm stubs trash it
     // gladly right now.
-    /* 28 = */ false, false, true /* x30 = FP */, true /* x31 = SP */
+    /* 28 = */ false, false, true /* x30 = FP */, false /* x31 = SP */
 ];
 
 #[rustfmt::skip]
@@ -105,6 +105,7 @@ fn compute_arg_locs(call_conv: isa::CallConv, params: &[ir::AbiParam]) -> (Vec<A
     let mut next_vreg = 0;
     let mut next_stack: u64 = 0;
     let mut ret = vec![];
+
     for param in params {
         // Validate "purpose".
         match &param.purpose {
@@ -137,7 +138,7 @@ fn compute_arg_locs(call_conv: isa::CallConv, params: &[ir::AbiParam]) -> (Vec<A
                     _ => panic!("Unsupported vector-reg argument type"),
                 };
                 // Align.
-                assert!(size.is_power_of_two());
+                debug_assert!(size.is_power_of_two());
                 next_stack = (next_stack + size - 1) & !(size - 1);
                 ret.push(ABIArg::Stack(next_stack as i64, param.value_type));
                 next_stack += size;
@@ -159,7 +160,7 @@ impl ABISig {
         let (rets, _) = compute_arg_locs(sig.call_conv, &sig.returns);
 
         // Verify that there are no return values on the stack.
-        assert!(rets.iter().all(|a| match a {
+        debug_assert!(rets.iter().all(|a| match a {
             &ABIArg::Stack(..) => false,
             _ => true,
         }));
@@ -175,20 +176,22 @@ impl ABISig {
 
 /// AArch64 ABI object for a function body.
 pub struct AArch64ABIBody {
-    /// signature: arg and retval regs
+    /// Signature: arg and retval regs.
     sig: ABISig,
-    /// offsets to each stackslot
+    /// Offsets to each stackslot.
     stackslots: Vec<u32>,
-    /// total stack size of all stackslots
+    /// Total stack size of all stackslots.
     stackslots_size: u32,
-    /// clobbered registers, from regalloc.
+    /// Clobbered registers, from regalloc.
     clobbered: Set<Writable<RealReg>>,
-    /// total number of spillslots, from regalloc.
+    /// Total number of spillslots, from regalloc.
     spillslots: Option<usize>,
     /// Total frame size.
     frame_size: Option<u32>,
     /// Calling convention this function expects.
     call_conv: isa::CallConv,
+    /// The settings controlling this function's compilation.
+    flags: settings::Flags,
 }
 
 fn in_int_reg(ty: ir::Type) -> bool {
@@ -208,14 +211,14 @@ fn in_vec_reg(ty: ir::Type) -> bool {
 
 impl AArch64ABIBody {
     /// Create a new body ABI instance.
-    pub fn new(f: &ir::Function) -> Self {
+    pub fn new(f: &ir::Function, flags: settings::Flags) -> Self {
         debug!("AArch64 ABI: func signature {:?}", f.signature);
 
         let sig = ABISig::from_func_sig(&f.signature);
 
         let call_conv = f.signature.call_conv;
         // Only these calling conventions are supported.
-        assert!(
+        debug_assert!(
             call_conv == isa::CallConv::SystemV
                 || call_conv == isa::CallConv::Fast
                 || call_conv == isa::CallConv::Cold
@@ -231,7 +234,7 @@ impl AArch64ABIBody {
             let off = stack_offset;
             stack_offset += data.size;
             stack_offset = (stack_offset + 7) & !7;
-            assert_eq!(stackslot.as_u32() as usize, stackslots.len());
+            debug_assert_eq!(stackslot.as_u32() as usize, stackslots.len());
             stackslots.push(off);
         }
 
@@ -243,6 +246,20 @@ impl AArch64ABIBody {
             spillslots: None,
             frame_size: None,
             call_conv,
+            flags,
+        }
+    }
+
+    /// Returns the size of a function call frame (including return address and FP) for this
+    /// function's body.
+    fn frame_size(&self) -> i64 {
+        if self.call_conv.extends_baldrdash() {
+            let num_words = self.flags.baldrdash_prologue_words() as i64;
+            debug_assert!(num_words > 0, "baldrdash must set baldrdash_prologue_words");
+            debug_assert_eq!(num_words % 2, 0, "stack must be 16-aligned");
+            num_words * 8
+        } else {
+            16 // frame pointer + return address.
         }
     }
 }
@@ -314,17 +331,11 @@ fn is_callee_save(call_conv: isa::CallConv, r: RealReg) -> bool {
         match r.get_class() {
             RegClass::I64 => {
                 let enc = r.get_hw_encoding();
-                if BALDRDASH_JIT_CALLEE_SAVED_GPR[enc] {
+                return BALDRDASH_JIT_CALLEE_SAVED_GPR[enc];
-                    return true;
-                }
-                // Otherwise, fall through to preserve native ABI registers.
             }
             RegClass::V128 => {
                 let enc = r.get_hw_encoding();
-                if BALDRDASH_JIT_CALLEE_SAVED_FPU[enc] {
+                return BALDRDASH_JIT_CALLEE_SAVED_FPU[enc];
-                    return true;
-                }
-                // Otherwise, fall through to preserve native ABI registers.
             }
             _ => unimplemented!("baldrdash callee saved on non-i64 reg classes"),
         };
@@ -415,6 +426,10 @@ fn get_caller_saves_set(call_conv: isa::CallConv) -> Set<Writable<Reg>> {
 impl ABIBody for AArch64ABIBody {
     type I = Inst;
 
+    fn flags(&self) -> &settings::Flags {
+        &self.flags
+    }
+
     fn liveins(&self) -> Set<RealReg> {
         let mut set: Set<RealReg> = Set::empty();
         for &arg in &self.sig.args {
@@ -450,15 +465,71 @@ impl ABIBody for AArch64ABIBody {
     fn gen_copy_arg_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Inst {
         match &self.sig.args[idx] {
             &ABIArg::Reg(r, ty) => Inst::gen_move(into_reg, r.to_reg(), ty),
-            &ABIArg::Stack(off, ty) => load_stack(off + 16, into_reg, ty),
+            &ABIArg::Stack(off, ty) => load_stack(off + self.frame_size(), into_reg, ty),
         }
     }
 
-    fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Reg) -> Inst {
+    fn gen_copy_reg_to_retval(
+        &self,
+        idx: usize,
+        from_reg: Writable<Reg>,
+        ext: ArgumentExtension,
+    ) -> Vec<Inst> {
+        let mut ret = Vec::new();
         match &self.sig.rets[idx] {
-            &ABIArg::Reg(r, ty) => Inst::gen_move(Writable::from_reg(r.to_reg()), from_reg, ty),
+            &ABIArg::Reg(r, ty) => {
-            &ABIArg::Stack(off, ty) => store_stack(off + 16, from_reg, ty),
+                let from_bits = aarch64::lower::ty_bits(ty) as u8;
+                let dest_reg = Writable::from_reg(r.to_reg());
+                match (ext, from_bits) {
+                    (ArgumentExtension::Uext, n) if n < 64 => {
+                        ret.push(Inst::Extend {
+                            rd: dest_reg,
+                            rn: from_reg.to_reg(),
+                            signed: false,
+                            from_bits,
+                            to_bits: 64,
+                        });
                     }
+                    (ArgumentExtension::Sext, n) if n < 64 => {
+                        ret.push(Inst::Extend {
+                            rd: dest_reg,
+                            rn: from_reg.to_reg(),
+                            signed: true,
+                            from_bits,
+                            to_bits: 64,
+                        });
+                    }
+                    _ => ret.push(Inst::gen_move(dest_reg, from_reg.to_reg(), ty)),
+                };
+            }
+            &ABIArg::Stack(off, ty) => {
+                let from_bits = aarch64::lower::ty_bits(ty) as u8;
+                // Trash the from_reg; it should be its last use.
+                match (ext, from_bits) {
+                    (ArgumentExtension::Uext, n) if n < 64 => {
+                        ret.push(Inst::Extend {
+                            rd: from_reg,
+                            rn: from_reg.to_reg(),
+                            signed: false,
+                            from_bits,
+                            to_bits: 64,
+                        });
+                    }
+                    (ArgumentExtension::Sext, n) if n < 64 => {
+                        ret.push(Inst::Extend {
+                            rd: from_reg,
+                            rn: from_reg.to_reg(),
+                            signed: true,
+                            from_bits,
+                            to_bits: 64,
+                        });
+                    }
+                    _ => {}
+                };
+                ret.push(store_stack(off + self.frame_size(), from_reg.to_reg(), ty))
+            }
+        }
+        ret
     }
 
     fn gen_ret(&self) -> Inst {
@@ -527,7 +598,7 @@ impl ABIBody for AArch64ABIBody {
         store_stack(fp_off, from_reg, ty)
     }
 
-    fn gen_prologue(&mut self, flags: &settings::Flags) -> Vec<Inst> {
+    fn gen_prologue(&mut self) -> Vec<Inst> {
         let mut insts = vec![];
         if !self.call_conv.extends_baldrdash() {
             // stp fp (x29), lr (x30), [sp, #-16]!
@@ -555,10 +626,10 @@ impl ABIBody for AArch64ABIBody {
         let mut total_stacksize = self.stackslots_size + 8 * self.spillslots.unwrap() as u32;
         if self.call_conv.extends_baldrdash() {
             debug_assert!(
-                !flags.enable_probestack(),
+                !self.flags.enable_probestack(),
                 "baldrdash does not expect cranelift to emit stack probes"
             );
-            total_stacksize += flags.baldrdash_prologue_words() as u32 * 8;
+            total_stacksize += self.flags.baldrdash_prologue_words() as u32 * 8;
         }
         let total_stacksize = (total_stacksize + 15) & !15; // 16-align the stack.
 
@@ -635,7 +706,7 @@ impl ABIBody for AArch64ABIBody {
         insts
     }
 
-    fn gen_epilogue(&self, _flags: &settings::Flags) -> Vec<Inst> {
+    fn gen_epilogue(&self) -> Vec<Inst> {
         let mut insts = vec![];
 
         // Restore clobbered registers.

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -4,7 +4,7 @@ use crate::binemit::{CodeOffset, Reloc};
 use crate::ir::constant::ConstantData;
 use crate::ir::types::*;
 use crate::ir::TrapCode;
-use crate::isa::aarch64::inst::*;
+use crate::isa::aarch64::{inst::regs::PINNED_REG, inst::*};
 
 use regalloc::{Reg, RegClass, Writable};
 
@@ -1325,6 +1325,20 @@ impl<O: MachSectionOutput> MachInstEmit<O> for Inst {
                 }
                 _ => unimplemented!("{:?}", mem),
             },
+            &Inst::GetPinnedReg { rd } => {
+                let inst = Inst::Mov {
+                    rd,
+                    rm: xreg(PINNED_REG),
+                };
+                inst.emit(sink);
+            }
+            &Inst::SetPinnedReg { rm } => {
+                let inst = Inst::Mov {
+                    rd: Writable::from_reg(xreg(PINNED_REG)),
+                    rm,
+                };
+                inst.emit(sink);
+            }
         }
     }
 }
@@ -1333,6 +1347,7 @@ impl<O: MachSectionOutput> MachInstEmit<O> for Inst {
 mod test {
     use super::*;
     use crate::isa::test_utils;
+    use crate::settings;
 
     #[test]
     fn test_aarch64_binemit() {
@@ -4136,7 +4151,7 @@ mod test {
             "frintn d23, d24",
         ));
 
-        let rru = create_reg_universe();
+        let rru = create_reg_universe(&settings::Flags::new(settings::builder()));
         for (insn, expected_encoding, expected_printing) in insns {
             println!(
                 "AArch64: {:?}, {}, {}",

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -7,6 +7,7 @@ use crate::binemit::CodeOffset;
 use crate::ir::types::{B1, B16, B32, B64, B8, F32, F64, FFLAGS, I16, I32, I64, I8, IFLAGS};
 use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type};
 use crate::machinst::*;
+use crate::settings;
 
 use regalloc::Map as RegallocMap;
 use regalloc::{RealReg, RealRegUniverse, Reg, RegClass, SpillSlot, VirtualReg, Writable};
@@ -723,6 +724,16 @@ pub enum Inst {
         rd: Writable<Reg>,
         mem: MemArg,
     },
+
+    /// Sets the value of the pinned register to the given register target.
+    GetPinnedReg {
+        rd: Writable<Reg>,
+    },
+
+    /// Writes the value of the given source register to the pinned register.
+    SetPinnedReg {
+        rm: Reg,
+    },
 }
 
 fn count_zero_half_words(mut value: u64) -> usize {
@@ -1111,6 +1122,12 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
         &Inst::LoadAddr { rd, mem: _ } => {
             collector.add_def(rd);
         }
+        &Inst::GetPinnedReg { rd } => {
+            collector.add_def(rd);
+        }
+        &Inst::SetPinnedReg { rm } => {
+            collector.add_use(rm);
+        }
     }
 }
 
@@ -1675,6 +1692,12 @@ fn aarch64_map_regs(
             map_wr(d, rd);
             map_mem(u, mem);
         }
+        &mut Inst::GetPinnedReg { ref mut rd } => {
+            map_wr(d, rd);
+        }
+        &mut Inst::SetPinnedReg { ref mut rm } => {
+            map(u, rm);
+        }
     }
 }
 
@@ -1865,8 +1888,8 @@ impl MachInst for Inst {
         }
     }
 
-    fn reg_universe() -> RealRegUniverse {
+    fn reg_universe(flags: &settings::Flags) -> RealRegUniverse {
-        create_reg_universe()
+        create_reg_universe(flags)
     }
 }
 
@@ -2617,6 +2640,14 @@ impl ShowWithRRU for Inst {
                 }
                 _ => unimplemented!("{:?}", mem),
             },
+            &Inst::GetPinnedReg { rd } => {
+                let rd = rd.show_rru(mb_rru);
+                format!("get_pinned_reg {}", rd)
+            }
+            &Inst::SetPinnedReg { rm } => {
+                let rm = rm.show_rru(mb_rru);
+                format!("set_pinned_reg {}", rm)
+            }
         }
     }
 }

cranelift/codegen/src/isa/aarch64/inst/regs.rs

@@ -2,6 +2,7 @@
 
 use crate::isa::aarch64::inst::InstSize;
 use crate::machinst::*;
+use crate::settings;
 
 use regalloc::{RealRegUniverse, Reg, RegClass, RegClassInfo, Writable, NUM_REG_CLASSES};
 
@@ -10,6 +11,11 @@ use std::string::{String, ToString};
 //=============================================================================
 // Registers, the Universe thereof, and printing
 
+/// The pinned register on this architecture.
+/// It must be the same as Spidermonkey's HeapReg, as found in this file.
+/// https://searchfox.org/mozilla-central/source/js/src/jit/arm64/Assembler-arm64.h#103
+pub const PINNED_REG: u8 = 21;
+
 #[rustfmt::skip]
 const XREG_INDICES: [u8; 31] = [
     // X0 - X7
@@ -22,8 +28,12 @@ const XREG_INDICES: [u8; 31] = [
     47, 48,
     // X18
     60,
-    // X19 - X28
+    // X19, X20
-    49, 50, 51, 52, 53, 54, 55, 56, 57, 58,
+    49, 50,
+    // X21, put aside because it's the pinned register.
+    58,
+    // X22 - X28
+    51, 52, 53, 54, 55, 56, 57,
     // X29
     61,
     // X30
@@ -131,14 +141,13 @@ pub fn writable_spilltmp_reg() -> Writable<Reg> {
 }
 
 /// Create the register universe for AArch64.
-pub fn create_reg_universe() -> RealRegUniverse {
+pub fn create_reg_universe(flags: &settings::Flags) -> RealRegUniverse {
     let mut regs = vec![];
     let mut allocable_by_class = [None; NUM_REG_CLASSES];
 
-    // Numbering Scheme: we put V-regs first, then X-regs. The X-regs
+    // Numbering Scheme: we put V-regs first, then X-regs. The X-regs exclude several registers:
-    // exclude several registers: x18 (globally reserved for platform-specific
+    // x18 (globally reserved for platform-specific purposes), x29 (frame pointer), x30 (link
-    // purposes), x29 (frame pointer), x30 (link register), x31 (stack pointer
+    // register), x31 (stack pointer or zero register, depending on context).
-    // or zero register, depending on context).
 
     let v_reg_base = 0u8; // in contiguous real-register index space
     let v_reg_count = 32;
@@ -159,9 +168,12 @@ pub fn create_reg_universe() -> RealRegUniverse {
 
     let x_reg_base = 32u8; // in contiguous real-register index space
     let mut x_reg_count = 0;
+
+    let uses_pinned_reg = flags.enable_pinned_reg();
+
     for i in 0u8..32u8 {
         // See above for excluded registers.
-        if i == 15 || i == 18 || i == 29 || i == 30 || i == 31 {
+        if i == 15 || i == 18 || i == 29 || i == 30 || i == 31 || i == PINNED_REG {
             continue;
         }
         let reg = Reg::new_real(
@@ -188,13 +200,24 @@ pub fn create_reg_universe() -> RealRegUniverse {
     });
 
     // Other regs, not available to the allocator.
-    let allocable = regs.len();
+    let allocable = if uses_pinned_reg {
+        // The pinned register is not allocatable in this case, so record the length before adding
+        // it.
+        let len = regs.len();
+        regs.push((xreg(PINNED_REG).to_real_reg(), "x21/pinned_reg".to_string()));
+        len
+    } else {
+        regs.push((xreg(PINNED_REG).to_real_reg(), "x21".to_string()));
+        regs.len()
+    };
+
     regs.push((xreg(15).to_real_reg(), "x15".to_string()));
     regs.push((xreg(18).to_real_reg(), "x18".to_string()));
     regs.push((fp_reg().to_real_reg(), "fp".to_string()));
     regs.push((link_reg().to_real_reg(), "lr".to_string()));
     regs.push((zero_reg().to_real_reg(), "xzr".to_string()));
     regs.push((stack_reg().to_real_reg(), "sp".to_string()));
+
     // FIXME JRS 2020Feb06: unfortunately this pushes the number of real regs
     // to 65, which is potentially inconvenient from a compiler performance
     // standpoint.  We could possibly drop back to 64 by "losing" a vector

cranelift/codegen/src/isa/aarch64/lower.rs

@@ -1936,9 +1936,16 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(ctx: &mut C, insn: IRInst) {
             }
         }
 
-        Opcode::GetPinnedReg
+        Opcode::GetPinnedReg => {
-        | Opcode::SetPinnedReg
+            let rd = output_to_reg(ctx, outputs[0]);
-        | Opcode::Spill
+            ctx.emit(Inst::GetPinnedReg { rd });
+        }
+        Opcode::SetPinnedReg => {
+            let rm = input_to_reg(ctx, inputs[0], NarrowValueMode::None);
+            ctx.emit(Inst::SetPinnedReg { rm });
+        }
+
+        Opcode::Spill
         | Opcode::Fill
         | Opcode::FillNop
         | Opcode::Regmove
@@ -2358,7 +2365,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(ctx: &mut C, insn: IRInst) {
 
 //=============================================================================
 // Helpers for instruction lowering.
-fn ty_bits(ty: Type) -> usize {
+
+/// Returns the size (in bits) of a given type.
+pub fn ty_bits(ty: Type) -> usize {
     match ty {
         B1 => 1,
         B8 | I8 => 8,

cranelift/codegen/src/isa/aarch64/mod.rs

@@ -32,11 +32,11 @@ impl AArch64Backend {
         AArch64Backend { triple, flags }
     }
 
-    fn compile_vcode(&self, func: &Function, flags: &settings::Flags) -> VCode<inst::Inst> {
+    /// This performs lowering to VCode, register-allocates the code, computes block layout and
-        // This performs lowering to VCode, register-allocates the code, computes
+    /// finalizes branches. The result is ready for binary emission.
-        // block layout and finalizes branches. The result is ready for binary emission.
+    fn compile_vcode(&self, func: &Function, flags: settings::Flags) -> VCode<inst::Inst> {
-        let abi = Box::new(abi::AArch64ABIBody::new(func));
+        let abi = Box::new(abi::AArch64ABIBody::new(func, flags));
-        compile::compile::<AArch64Backend>(func, self, abi, flags)
+        compile::compile::<AArch64Backend>(func, self, abi)
     }
 }
 
@@ -47,12 +47,12 @@ impl MachBackend for AArch64Backend {
         want_disasm: bool,
     ) -> CodegenResult<MachCompileResult> {
         let flags = self.flags();
-        let vcode = self.compile_vcode(func, flags);
+        let vcode = self.compile_vcode(func, flags.clone());
         let sections = vcode.emit();
         let frame_size = vcode.frame_size();
 
         let disasm = if want_disasm {
-            Some(vcode.show_rru(Some(&create_reg_universe())))
+            Some(vcode.show_rru(Some(&create_reg_universe(flags))))
         } else {
             None
         };
@@ -77,7 +77,7 @@ impl MachBackend for AArch64Backend {
     }
 
     fn reg_universe(&self) -> RealRegUniverse {
-        create_reg_universe()
+        create_reg_universe(&self.flags)
     }
 }
 

cranelift/codegen/src/machinst/abi.rs

@@ -1,6 +1,6 @@
 //! ABI definitions.
 
-use crate::ir::StackSlot;
+use crate::ir::{ArgumentExtension, StackSlot};
 use crate::machinst::*;
 use crate::settings;
 
@@ -12,6 +12,9 @@ pub trait ABIBody {
     /// The instruction type for the ISA associated with this ABI.
     type I: VCodeInst;
 
+    /// Get the settings controlling this function's compilation.
+    fn flags(&self) -> &settings::Flags;
+
     /// Get the liveins of the function.
     fn liveins(&self) -> Set<RealReg>;
 
@@ -31,9 +34,13 @@ pub trait ABIBody {
     /// register.
     fn gen_copy_arg_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Self::I;
 
-    /// Generate an instruction which copies a source register to a return
+    /// Generate an instruction which copies a source register to a return value slot.
-    /// value slot.
+    fn gen_copy_reg_to_retval(
-    fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Reg) -> Self::I;
+        &self,
+        idx: usize,
+        from_reg: Writable<Reg>,
+        ext: ArgumentExtension,
+    ) -> Vec<Self::I>;
 
     /// Generate a return instruction.
     fn gen_ret(&self) -> Self::I;
@@ -82,13 +89,13 @@ pub trait ABIBody {
     /// `store_retval`, and spillslot accesses.)  `self` is mutable so that we
     /// can store information in it which will be useful when creating the
     /// epilogue.
-    fn gen_prologue(&mut self, flags: &settings::Flags) -> Vec<Self::I>;
+    fn gen_prologue(&mut self) -> Vec<Self::I>;
 
     /// Generate an epilogue, post-regalloc. Note that this must generate the
     /// actual return instruction (rather than emitting this in the lowering
     /// logic), because the epilogue code comes before the return and the two are
     /// likely closely related.
-    fn gen_epilogue(&self, flags: &settings::Flags) -> Vec<Self::I>;
+    fn gen_epilogue(&self) -> Vec<Self::I>;
 
     /// Returns the full frame size for the given function, after prologue emission has run. This
     /// comprises the spill space, incoming argument space, alignment padding, etc.

cranelift/codegen/src/machinst/compile.rs

@@ -2,7 +2,6 @@
 
 use crate::ir::Function;
 use crate::machinst::*;
-use crate::settings;
 use crate::timing;
 
 use log::debug;
@@ -14,7 +13,6 @@ pub fn compile<B: LowerBackend>(
     f: &Function,
     b: &B,
     abi: Box<dyn ABIBody<I = B::MInst>>,
-    flags: &settings::Flags,
 ) -> VCode<B::MInst>
 where
     B::MInst: ShowWithRRU,
@@ -22,7 +20,7 @@ where
     // This lowers the CL IR.
     let mut vcode = Lower::new(f, abi).lower(b);
 
-    let universe = &B::MInst::reg_universe();
+    let universe = &B::MInst::reg_universe(vcode.flags());
 
     debug!("vcode from lowering: \n{}", vcode.show_rru(Some(universe)));
 
@@ -47,7 +45,7 @@ where
 
     // Reorder vcode into final order and copy out final instruction sequence
     // all at once. This also inserts prologues/epilogues.
-    vcode.replace_insns_from_regalloc(result, flags);
+    vcode.replace_insns_from_regalloc(result);
 
     vcode.remove_redundant_branches();
 

cranelift/codegen/src/machinst/lower.rs

@@ -6,8 +6,8 @@ use crate::entity::SecondaryMap;
 use crate::inst_predicates::has_side_effect;
 use crate::ir::instructions::BranchInfo;
 use crate::ir::{
-    Block, ExternalName, Function, GlobalValueData, Inst, InstructionData, MemFlags, Opcode,
+    ArgumentExtension, Block, ExternalName, Function, GlobalValueData, Inst, InstructionData,
-    Signature, SourceLoc, Type, Value, ValueDef,
+    MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef,
 };
 use crate::machinst::{ABIBody, BlockIndex, VCode, VCodeBuilder, VCodeInst};
 use crate::num_uses::NumUses;
@@ -102,9 +102,9 @@ pub trait LowerBackend {
 
 /// Machine-independent lowering driver / machine-instruction container. Maintains a correspondence
 /// from original Inst to MachInsts.
-pub struct Lower<'a, I: VCodeInst> {
+pub struct Lower<'func, I: VCodeInst> {
     /// The function to lower.
-    f: &'a Function,
+    f: &'func Function,
 
     /// Lowered machine instructions.
     vcode: VCodeBuilder<I>,
@@ -116,7 +116,7 @@ pub struct Lower<'a, I: VCodeInst> {
     value_regs: SecondaryMap<Value, Reg>,
 
     /// Return-value vregs.
-    retval_regs: Vec<Reg>,
+    retval_regs: Vec<(Reg, ArgumentExtension)>,
 
     /// Next virtual register number to allocate.
     next_vreg: u32,
@@ -142,9 +142,9 @@ enum GenerateReturn {
     No,
 }
 
-impl<'a, I: VCodeInst> Lower<'a, I> {
+impl<'func, I: VCodeInst> Lower<'func, I> {
     /// Prepare a new lowering context for the given IR function.
-    pub fn new(f: &'a Function, abi: Box<dyn ABIBody<I = I>>) -> Lower<'a, I> {
+    pub fn new(f: &'func Function, abi: Box<dyn ABIBody<I = I>>) -> Lower<'func, I> {
         let mut vcode = VCodeBuilder::new(abi);
 
         let num_uses = NumUses::compute(f).take_uses();
@@ -190,7 +190,7 @@ impl<'a, I: VCodeInst> Lower<'a, I> {
             next_vreg += 1;
             let regclass = I::rc_for_type(ret.value_type);
             let vreg = Reg::new_virtual(regclass, v);
-            retval_regs.push(vreg);
+            retval_regs.push((vreg, ret.extension));
             vcode.set_vreg_type(vreg.as_virtual_reg().unwrap(), ret.value_type);
         }
 
@@ -220,10 +220,13 @@ impl<'a, I: VCodeInst> Lower<'a, I> {
     }
 
     fn gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn) {
-        for (i, reg) in self.retval_regs.iter().enumerate() {
+        for (i, (reg, ext)) in self.retval_regs.iter().enumerate() {
-            let insn = self.vcode.abi().gen_copy_reg_to_retval(i, *reg);
+            let reg = Writable::from_reg(*reg);
+            let insns = self.vcode.abi().gen_copy_reg_to_retval(i, reg, *ext);
+            for insn in insns {
                 self.vcode.push(insn);
             }
+        }
         let inst = match gen_ret_inst {
             GenerateReturn::Yes => self.vcode.abi().gen_ret(),
             GenerateReturn::No => self.vcode.abi().gen_epilogue_placeholder(),
@@ -516,7 +519,7 @@ impl<'a, I: VCodeInst> Lower<'a, I> {
     }
 }
 
-impl<'a, I: VCodeInst> LowerCtx for Lower<'a, I> {
+impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
     type I = I;
 
     /// Get the instdata for a given IR instruction.
@@ -640,7 +643,7 @@ impl<'a, I: VCodeInst> LowerCtx for Lower<'a, I> {
 
     /// Get the register for a return value.
     fn retval(&self, idx: usize) -> Writable<Reg> {
-        Writable::from_reg(self.retval_regs[idx])
+        Writable::from_reg(self.retval_regs[idx].0)
     }
 
     /// Get the target for a call instruction, as an `ExternalName`.

cranelift/codegen/src/machinst/mod.rs

@@ -102,6 +102,7 @@ use crate::ir::condcodes::IntCC;
 use crate::ir::{Function, Type};
 use crate::result::CodegenResult;
 use crate::settings::Flags;
+
 use alloc::boxed::Box;
 use alloc::vec::Vec;
 use core::fmt::Debug;
@@ -189,7 +190,7 @@ pub trait MachInst: Clone + Debug {
     fn with_block_offsets(&mut self, my_offset: CodeOffset, targets: &[CodeOffset]);
 
     /// Get the register universe for this backend.
-    fn reg_universe() -> RealRegUniverse;
+    fn reg_universe(flags: &Flags) -> RealRegUniverse;
 
     /// Align a basic block offset (from start of function).  By default, no
     /// alignment occurs.

cranelift/codegen/src/machinst/vcode.rs

@@ -299,6 +299,11 @@ impl<I: VCodeInst> VCode<I> {
         }
     }
 
+    /// Returns the flags controlling this function's compilation.
+    pub fn flags(&self) -> &settings::Flags {
+        self.abi.flags()
+    }
+
     /// Get the IR-level type of a VReg.
     pub fn vreg_type(&self, vreg: VirtualReg) -> Type {
         self.vreg_types[vreg.get_index()]
@@ -329,11 +334,7 @@ impl<I: VCodeInst> VCode<I> {
     /// Take the results of register allocation, with a sequence of
     /// instructions including spliced fill/reload/move instructions, and replace
     /// the VCode with them.
-    pub fn replace_insns_from_regalloc(
+    pub fn replace_insns_from_regalloc(&mut self, result: RegAllocResult<Self>) {
-        &mut self,
-        result: RegAllocResult<Self>,
-        flags: &settings::Flags,
-    ) {
         self.final_block_order = compute_final_block_order(self);
 
         // Record the spillslot count and clobbered registers for the ABI/stack
@@ -355,7 +356,7 @@ impl<I: VCodeInst> VCode<I> {
 
             if *block == self.entry {
                 // Start with the prologue.
-                final_insns.extend(self.abi.gen_prologue(flags).into_iter());
+                final_insns.extend(self.abi.gen_prologue().into_iter());
             }
 
             for i in start..end {
@@ -371,7 +372,7 @@ impl<I: VCodeInst> VCode<I> {
                 // with the epilogue.
                 let is_ret = insn.is_term() == MachTerminator::Ret;
                 if is_ret {
-                    final_insns.extend(self.abi.gen_epilogue(flags).into_iter());
+                    final_insns.extend(self.abi.gen_epilogue().into_iter());
                 } else {
                     final_insns.push(insn.clone());
                 }