AArch64: Migrate calls and returns to ISLE. (#4788)

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -225,12 +225,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
                         slots: smallvec![
                             ABIArgSlot::Reg {
                                 reg: lower_reg.to_real_reg().unwrap(),
-                                ty: param.value_type,
+                                ty: reg_types[0],
                                 extension: param.extension,
                             },
                             ABIArgSlot::Reg {
                                 reg: upper_reg.to_real_reg().unwrap(),
-                                ty: param.value_type,
+                                ty: reg_types[1],
                                 extension: param.extension,
                             },
                         ],

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -2818,3 +2818,11 @@
       (let ((dst WritableReg (temp_writable_reg $I8X16))
             (_ Unit (emit (MInst.IntToFpu op dst src))))
        dst))
+
+;;;; Helpers for Emitting Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl gen_call (SigRef ExternalName RelocDistance ValueSlice) InstOutput)
+(extern constructor gen_call gen_call)
+
+(decl gen_call_indirect (SigRef Value ValueSlice) InstOutput)
+(extern constructor gen_call_indirect gen_call_indirect)

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -2016,3 +2016,17 @@
 
 (rule (lower (get_return_address))
       (aarch64_link))
+
+;;;; Rules for calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (call (func_ref_data sig_ref extname dist) inputs))
+      (gen_call sig_ref extname dist inputs))
+
+(rule (lower (call_indirect sig_ref val inputs))
+      (gen_call_indirect sig_ref val inputs))
+
+;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; N.B.: the Ret itself is generated by the ABI.
+(rule (lower (return args))
+      (lower_return (range 0 (value_slice_len args)) args))

cranelift/codegen/src/isa/aarch64/lower/isle.rs

@@ -2,6 +2,7 @@
 
 // Pull in the ISLE generated code.
 pub mod generated_code;
+use generated_code::Context;
 
 // Types that the generated ISLE code uses via `use super::*`.
 use super::{
@@ -14,6 +15,7 @@ use super::{
 use crate::isa::aarch64::inst::{FPULeftShiftImm, FPURightShiftImm};
 use crate::isa::aarch64::lower::{lower_address, lower_splat_const};
 use crate::isa::aarch64::settings::Flags as IsaFlags;
+use crate::machinst::valueregs;
 use crate::machinst::{isle::*, InputSourceInst};
 use crate::settings::Flags;
 use crate::{
@@ -22,10 +24,11 @@ use crate::{
         immediates::*, types::*, AtomicRmwOp, ExternalName, Inst, InstructionData, MemFlags,
         TrapCode, Value, ValueList,
     },
+    isa::aarch64::abi::{AArch64Caller, AArch64MachineDeps},
     isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm},
     isa::aarch64::lower::{writable_vreg, writable_xreg, xreg},
     isa::unwind::UnwindInst,
-    machinst::{ty_bits, InsnOutput, Lower, VCodeConstant, VCodeConstantData},
+    machinst::{ty_bits, InsnOutput, Lower, MachInst, VCodeConstant, VCodeConstantData},
 };
 use regalloc2::PReg;
 use std::boxed::Box;
@@ -69,8 +72,13 @@ pub struct SinkableAtomicLoad {
     atomic_addr: Value,
 }
 
-impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
+impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
+    isle_prelude_method_helpers!(AArch64Caller);
+}
+
+impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
     isle_prelude_methods!();
+    isle_prelude_caller_methods!(AArch64MachineDeps, AArch64Caller);
 
     fn sign_return_address_disabled(&mut self) -> Option<()> {
         if self.isa_flags.sign_return_address() {

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -5,7 +5,6 @@ use crate::binemit::CodeOffset;
 use crate::ir::types::*;
 use crate::ir::Inst as IRInst;
 use crate::ir::{InstructionData, Opcode};
-use crate::isa::aarch64::abi::*;
 use crate::isa::aarch64::inst::*;
 use crate::isa::aarch64::settings as aarch64_settings;
 use crate::machinst::lower::*;
@@ -469,29 +468,7 @@ pub(crate) fn lower_insn_to_regs(
             }
         }
 
-        Opcode::Return => {
+        Opcode::Return => implemented_in_isle(ctx),
-            for (i, input) in inputs.iter().enumerate() {
-                // N.B.: according to the AArch64 ABI, the top bits of a register
-                // (above the bits for the value's type) are undefined, so we
-                // need not extend the return values.
-                let src_regs = put_input_in_regs(ctx, *input);
-                let retval_regs = ctx.retval(i);
-
-                assert_eq!(src_regs.len(), retval_regs.len());
-                let ty = ctx.input_ty(insn, i);
-                let (_, tys) = Inst::rc_for_type(ty)?;
-
-                src_regs
-                    .regs()
-                    .iter()
-                    .zip(retval_regs.regs().iter())
-                    .zip(tys.iter())
-                    .for_each(|((&src, &dst), &ty)| {
-                        ctx.emit(Inst::gen_move(dst, src, ty));
-                    });
-            }
-            // N.B.: the Ret itself is generated by the ABI.
-        }
 
         Opcode::Ifcmp | Opcode::Ffcmp => {
             // An Ifcmp/Ffcmp must always be seen as a use of a brif/brff or trueif/trueff
@@ -577,52 +554,7 @@ pub(crate) fn lower_insn_to_regs(
 
         Opcode::SymbolValue => implemented_in_isle(ctx),
 
-        Opcode::Call | Opcode::CallIndirect => {
+        Opcode::Call | Opcode::CallIndirect => implemented_in_isle(ctx),
-            let caller_conv = ctx.abi().call_conv();
-            let (mut abi, inputs) = match op {
-                Opcode::Call => {
-                    let (extname, dist) = ctx.call_target(insn).unwrap();
-                    let extname = extname.clone();
-                    let sig = ctx.call_sig(insn).unwrap();
-                    assert!(inputs.len() == sig.params.len());
-                    assert!(outputs.len() == sig.returns.len());
-                    (
-                        AArch64Caller::from_func(sig, &extname, dist, caller_conv, flags)?,
-                        &inputs[..],
-                    )
-                }
-                Opcode::CallIndirect => {
-                    let ptr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64);
-                    let sig = ctx.call_sig(insn).unwrap();
-                    assert!(inputs.len() - 1 == sig.params.len());
-                    assert!(outputs.len() == sig.returns.len());
-                    (
-                        AArch64Caller::from_ptr(sig, ptr, op, caller_conv, flags)?,
-                        &inputs[1..],
-                    )
-                }
-                _ => unreachable!(),
-            };
-
-            abi.emit_stack_pre_adjust(ctx);
-            assert!(inputs.len() == abi.num_args());
-            let mut arg_regs = vec![];
-            for input in inputs {
-                arg_regs.push(put_input_in_regs(ctx, *input))
-            }
-            for (i, arg_regs) in arg_regs.iter().enumerate() {
-                abi.emit_copy_regs_to_buffer(ctx, i, *arg_regs);
-            }
-            for (i, arg_regs) in arg_regs.iter().enumerate() {
-                abi.emit_copy_regs_to_arg(ctx, i, *arg_regs);
-            }
-            abi.emit_call(ctx);
-            for (i, output) in outputs.iter().enumerate() {
-                let retval_regs = get_output_reg(ctx, *output);
-                abi.emit_copy_retval_to_regs(ctx, i, retval_regs);
-            }
-            abi.emit_stack_post_adjust(ctx);
-        }
 
         Opcode::GetPinnedReg => {
             let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();

cranelift/codegen/src/isa/s390x/inst.isle

@@ -2940,8 +2940,8 @@
 (rule (abi_ext_ty (ArgumentExtension.Sext) _) $I64)
 
 ;; Copy a return value to a set of registers.
-(decl copy_to_regs (WritableValueRegs Value) Unit)
+(decl s390x_copy_to_regs (WritableValueRegs Value) Unit)
-(rule (copy_to_regs (only_writable_reg reg) val @ (value_type ty))
+(rule (s390x_copy_to_regs (only_writable_reg reg) val @ (value_type ty))
       (emit_mov ty reg val))
 
 

cranelift/codegen/src/isa/s390x/lower.isle

@@ -4047,13 +4047,13 @@
 ;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (return args))
-      (lower_return (range 0 (value_slice_len args)) args))
+      (s390x_lower_return (range 0 (value_slice_len args)) args))
 
-(decl lower_return (Range ValueSlice) InstOutput)
+(decl s390x_lower_return (Range ValueSlice) InstOutput)
-(rule (lower_return (range_empty) _) (output_none))
+(rule (s390x_lower_return (range_empty) _) (output_none))
-(rule (lower_return (range_unwrap head tail) args)
+(rule (s390x_lower_return (range_unwrap head tail) args)
-      (let ((_ Unit (copy_to_regs (retval head) (value_slice_get args head))))
+      (let ((_ Unit (s390x_copy_to_regs (retval head) (value_slice_get args head))))
-         (lower_return tail args)))
+         (s390x_lower_return tail args)))
 
 
 ;;;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

cranelift/codegen/src/isa/s390x/lower/isle.rs

@@ -22,7 +22,7 @@ use crate::{
     isa::unwind::UnwindInst,
     isa::CallConv,
     machinst::abi_impl::ABIMachineSpec,
-    machinst::{InsnOutput, Lower, VCodeConstant, VCodeConstantData},
+    machinst::{InsnOutput, Lower, MachInst, VCodeConstant, VCodeConstantData},
 };
 use regalloc2::PReg;
 use smallvec::{smallvec, SmallVec};

cranelift/codegen/src/isa/x64/inst.isle

@@ -1596,31 +1596,6 @@
 
 ;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; Generate a move between two registers.
-(decl gen_move (Type WritableReg Reg) MInst)
-(extern constructor gen_move gen_move)
-
-;; Copy a return value to a set of registers.
-(decl copy_to_regs (WritableValueRegs Value) Unit)
-(rule (copy_to_regs dsts val @ (value_type ty))
-      (let ((srcs ValueRegs (put_in_regs val)))
-        (copy_to_regs_range ty (value_regs_range srcs) dsts srcs)))
-
-;; Helper for `copy_to_regs` that uses a range to index into the reg/value
-;; vectors. Fails for the empty range.
-(decl copy_to_regs_range (Type Range WritableValueRegs ValueRegs) Unit)
-
-(rule (copy_to_regs_range ty (range_singleton idx) dsts srcs)
-      (let ((dst WritableReg (writable_regs_get dsts idx))
-            (src Reg (value_regs_get srcs idx)))
-        (emit (gen_move ty dst src))))
-
-(rule (copy_to_regs_range ty (range_unwrap head tail) dsts srcs)
-      (let ((dst WritableReg (writable_regs_get dsts head))
-            (src Reg (value_regs_get srcs head))
-            (_ Unit (emit (gen_move ty dst src))))
-        (copy_to_regs_range ty tail dsts srcs)))
-
 ;; Helper for constructing a LoadExtName instruction.
 (decl load_ext_name (ExternalName i64) Reg)
 (rule (load_ext_name extname offset)

cranelift/codegen/src/isa/x64/lower.isle

@@ -1441,13 +1441,6 @@
 (rule (lower (return args))
       (lower_return (range 0 (value_slice_len args)) args))
 
-(decl lower_return (Range ValueSlice) InstOutput)
-(rule (lower_return (range_empty) _) (output_none))
-(rule (lower_return (range_unwrap head tail) args)
-      (let ((_ Unit (copy_to_regs (retval head) (value_slice_get args head))))
-        (lower_return tail args)))
-
-
 ;;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (icmp cc a @ (value_type (fits_in_64 ty)) b))

cranelift/codegen/src/isa/x64/lower/isle.rs

@@ -92,6 +92,7 @@ pub(crate) fn lower_branch(
 
 impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
     isle_prelude_methods!();
+    isle_prelude_caller_methods!(X64ABIMachineSpec, X64Caller);
 
     #[inline]
     fn operand_size_of_type_32_64(&mut self, ty: Type) -> OperandSize {
@@ -708,54 +709,6 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
         MachAtomicRmwOp::from(*op)
     }
 
-    #[inline]
-    fn gen_move(&mut self, ty: Type, dst: WritableReg, src: Reg) -> MInst {
-        MInst::gen_move(dst, src, ty)
-    }
-
-    fn gen_call(
-        &mut self,
-        sig_ref: SigRef,
-        extname: ExternalName,
-        dist: RelocDistance,
-        args @ (inputs, off): ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = self.lower_ctx.abi().call_conv();
-        let sig = &self.lower_ctx.dfg().signatures[sig_ref];
-        let num_rets = sig.returns.len();
-        let abi = ABISig::from_func_sig::<X64ABIMachineSpec>(sig, self.flags).unwrap();
-        let caller = X64Caller::from_func(sig, &extname, dist, caller_conv, self.flags).unwrap();
-
-        assert_eq!(
-            inputs.len(&self.lower_ctx.dfg().value_lists) - off,
-            sig.params.len()
-        );
-
-        self.gen_call_common(abi, num_rets, caller, args)
-    }
-
-    fn gen_call_indirect(
-        &mut self,
-        sig_ref: SigRef,
-        val: Value,
-        args @ (inputs, off): ValueSlice,
-    ) -> InstOutput {
-        let caller_conv = self.lower_ctx.abi().call_conv();
-        let ptr = self.put_in_reg(val);
-        let sig = &self.lower_ctx.dfg().signatures[sig_ref];
-        let num_rets = sig.returns.len();
-        let abi = ABISig::from_func_sig::<X64ABIMachineSpec>(sig, self.flags).unwrap();
-        let caller =
-            X64Caller::from_ptr(sig, ptr, Opcode::CallIndirect, caller_conv, self.flags).unwrap();
-
-        assert_eq!(
-            inputs.len(&self.lower_ctx.dfg().value_lists) - off,
-            sig.params.len()
-        );
-
-        self.gen_call_common(abi, num_rets, caller, args)
-    }
-
     #[inline]
     fn preg_rbp(&mut self) -> PReg {
         regs::rbp().to_real_reg().unwrap().into()
@@ -1062,63 +1015,7 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
 }
 
 impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
-    fn abi_arg_slot_regs(&mut self, arg: &ABIArg) -> Option<WritableValueRegs> {
+    isle_prelude_method_helpers!(X64Caller);
-        match arg {
-            &ABIArg::Slots { ref slots, .. } => match slots.len() {
-                1 => {
-                    let a = self.temp_writable_reg(slots[0].get_type());
-                    Some(WritableValueRegs::one(a))
-                }
-                2 => {
-                    let a = self.temp_writable_reg(slots[0].get_type());
-                    let b = self.temp_writable_reg(slots[1].get_type());
-                    Some(WritableValueRegs::two(a, b))
-                }
-                _ => panic!("Expected to see one or two slots only from {:?}", arg),
-            },
-            _ => None,
-        }
-    }
-
-    fn gen_call_common(
-        &mut self,
-        abi: ABISig,
-        num_rets: usize,
-        mut caller: X64Caller,
-        (inputs, off): ValueSlice,
-    ) -> InstOutput {
-        caller.emit_stack_pre_adjust(self.lower_ctx);
-
-        assert_eq!(
-            inputs.len(&self.lower_ctx.dfg().value_lists) - off,
-            abi.num_args()
-        );
-        let mut arg_regs = vec![];
-        for i in 0..abi.num_args() {
-            let input = inputs
-                .get(off + i, &self.lower_ctx.dfg().value_lists)
-                .unwrap();
-            arg_regs.push(self.lower_ctx.put_value_in_regs(input));
-        }
-        for (i, arg_regs) in arg_regs.iter().enumerate() {
-            caller.emit_copy_regs_to_buffer(self.lower_ctx, i, *arg_regs);
-        }
-        for (i, arg_regs) in arg_regs.iter().enumerate() {
-            caller.emit_copy_regs_to_arg(self.lower_ctx, i, *arg_regs);
-        }
-        caller.emit_call(self.lower_ctx);
-
-        let mut outputs = InstOutput::new();
-        for i in 0..num_rets {
-            let ret = abi.get_ret(i);
-            let retval_regs = self.abi_arg_slot_regs(&ret).unwrap();
-            caller.emit_copy_retval_to_regs(self.lower_ctx, i, retval_regs.clone());
-            outputs.push(valueregs::non_writable_value_regs(retval_regs));
-        }
-        caller.emit_stack_post_adjust(self.lower_ctx);
-
-        outputs
-    }
 }
 
 // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we

cranelift/codegen/src/machinst/isle.rs

@@ -994,6 +994,129 @@ macro_rules! isle_prelude_methods {
         fn preg_to_reg(&mut self, preg: PReg) -> Reg {
             preg.into()
         }
+
+        #[inline]
+        fn gen_move(&mut self, ty: Type, dst: WritableReg, src: Reg) -> MInst {
+            MInst::gen_move(dst, src, ty)
+        }
+    };
+}
+
+/// Helpers specifically for machines that use ABICaller.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! isle_prelude_caller_methods {
+    ($abispec:ty, $abicaller:ty) => {
+        fn gen_call(
+            &mut self,
+            sig_ref: SigRef,
+            extname: ExternalName,
+            dist: RelocDistance,
+            args @ (inputs, off): ValueSlice,
+        ) -> InstOutput {
+            let caller_conv = self.lower_ctx.abi().call_conv();
+            let sig = &self.lower_ctx.dfg().signatures[sig_ref];
+            let num_rets = sig.returns.len();
+            let abi = ABISig::from_func_sig::<$abispec>(sig, self.flags).unwrap();
+            let caller =
+                <$abicaller>::from_func(sig, &extname, dist, caller_conv, self.flags).unwrap();
+
+            assert_eq!(
+                inputs.len(&self.lower_ctx.dfg().value_lists) - off,
+                sig.params.len()
+            );
+
+            self.gen_call_common(abi, num_rets, caller, args)
+        }
+
+        fn gen_call_indirect(
+            &mut self,
+            sig_ref: SigRef,
+            val: Value,
+            args @ (inputs, off): ValueSlice,
+        ) -> InstOutput {
+            let caller_conv = self.lower_ctx.abi().call_conv();
+            let ptr = self.put_in_reg(val);
+            let sig = &self.lower_ctx.dfg().signatures[sig_ref];
+            let num_rets = sig.returns.len();
+            let abi = ABISig::from_func_sig::<$abispec>(sig, self.flags).unwrap();
+            let caller =
+                <$abicaller>::from_ptr(sig, ptr, Opcode::CallIndirect, caller_conv, self.flags)
+                    .unwrap();
+
+            assert_eq!(
+                inputs.len(&self.lower_ctx.dfg().value_lists) - off,
+                sig.params.len()
+            );
+
+            self.gen_call_common(abi, num_rets, caller, args)
+        }
+    };
+}
+
+/// Helpers for the above ISLE prelude implementations. Meant to go
+/// inside the `impl` for the context type, not the trait impl.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! isle_prelude_method_helpers {
+    ($abicaller:ty) => {
+        fn gen_call_common(
+            &mut self,
+            abi: ABISig,
+            num_rets: usize,
+            mut caller: $abicaller,
+            (inputs, off): ValueSlice,
+        ) -> InstOutput {
+            caller.emit_stack_pre_adjust(self.lower_ctx);
+
+            assert_eq!(
+                inputs.len(&self.lower_ctx.dfg().value_lists) - off,
+                abi.num_args()
+            );
+            let mut arg_regs = vec![];
+            for i in 0..abi.num_args() {
+                let input = inputs
+                    .get(off + i, &self.lower_ctx.dfg().value_lists)
+                    .unwrap();
+                arg_regs.push(self.lower_ctx.put_value_in_regs(input));
+            }
+            for (i, arg_regs) in arg_regs.iter().enumerate() {
+                caller.emit_copy_regs_to_buffer(self.lower_ctx, i, *arg_regs);
+            }
+            for (i, arg_regs) in arg_regs.iter().enumerate() {
+                caller.emit_copy_regs_to_arg(self.lower_ctx, i, *arg_regs);
+            }
+            caller.emit_call(self.lower_ctx);
+
+            let mut outputs = InstOutput::new();
+            for i in 0..num_rets {
+                let ret = abi.get_ret(i);
+                let retval_regs = self.abi_arg_slot_regs(&ret).unwrap();
+                caller.emit_copy_retval_to_regs(self.lower_ctx, i, retval_regs.clone());
+                outputs.push(valueregs::non_writable_value_regs(retval_regs));
+            }
+            caller.emit_stack_post_adjust(self.lower_ctx);
+
+            outputs
+        }
+
+        fn abi_arg_slot_regs(&mut self, arg: &ABIArg) -> Option<WritableValueRegs> {
+            match arg {
+                &ABIArg::Slots { ref slots, .. } => match slots.len() {
+                    1 => {
+                        let a = self.temp_writable_reg(slots[0].get_type());
+                        Some(WritableValueRegs::one(a))
+                    }
+                    2 => {
+                        let a = self.temp_writable_reg(slots[0].get_type());
+                        let b = self.temp_writable_reg(slots[1].get_type());
+                        Some(WritableValueRegs::two(a, b))
+                    }
+                    _ => panic!("Expected to see one or two slots only from {:?}", arg),
+                },
+                _ => None,
+            }
+        }
     };
 }
 

cranelift/codegen/src/machinst/lower.rs

@@ -11,8 +11,8 @@ use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit};
 use crate::ir::{
     types::{FFLAGS, IFLAGS},
     ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, Function, GlobalValue,
-    GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, Signature, Type, Value,
+    GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, Type, Value, ValueDef,
-    ValueDef, ValueLabelAssignments, ValueLabelStart,
+    ValueLabelAssignments, ValueLabelStart,
 };
 use crate::ir::{ExternalName, RelSourceLoc};
 use crate::machinst::{
@@ -1026,34 +1026,6 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         &self.f.dfg[ir_inst]
     }
 
-    /// Get the target for a call instruction, as an `ExternalName`. Returns a tuple
-    /// providing this name and the "relocation distance", i.e., whether the backend
-    /// can assume the target will be "nearby" (within some small offset) or an
-    /// arbitrary address. (This comes from the `colocated` bit in the CLIF.)
-    pub fn call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)> {
-        match &self.f.dfg[ir_inst] {
-            &InstructionData::Call { func_ref, .. }
-            | &InstructionData::FuncAddr { func_ref, .. } => {
-                let funcdata = &self.f.dfg.ext_funcs[func_ref];
-                let dist = funcdata.reloc_distance();
-                Some((&funcdata.name, dist))
-            }
-            _ => None,
-        }
-    }
-
-    /// Get the signature for a call or call-indirect instruction.
-    pub fn call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature> {
-        match &self.f.dfg[ir_inst] {
-            &InstructionData::Call { func_ref, .. } => {
-                let funcdata = &self.f.dfg.ext_funcs[func_ref];
-                Some(&self.f.dfg.signatures[funcdata.signature])
-            }
-            &InstructionData::CallIndirect { sig_ref, .. } => Some(&self.f.dfg.signatures[sig_ref]),
-            _ => None,
-        }
-    }
-
     /// Get the symbol name, relocation distance estimate, and offset for a
     /// symbol_value instruction.
     pub fn symbol_value<'b>(

cranelift/codegen/src/prelude.isle

@@ -963,6 +963,37 @@
 (decl real_reg_to_writable_reg (RealReg) WritableReg)
 (extern constructor real_reg_to_writable_reg real_reg_to_writable_reg)
 
+;; Generate a move between two registers.
+(decl gen_move (Type WritableReg Reg) MInst)
+(extern constructor gen_move gen_move)
+
+;; Copy a return value to a set of registers.
+(decl copy_to_regs (WritableValueRegs Value) Unit)
+(rule (copy_to_regs dsts val @ (value_type ty))
+      (let ((srcs ValueRegs (put_in_regs val)))
+        (copy_to_regs_range ty (value_regs_range srcs) dsts srcs)))
+
+;; Helper for `copy_to_regs` that uses a range to index into the reg/value
+;; vectors. Fails for the empty range.
+(decl copy_to_regs_range (Type Range WritableValueRegs ValueRegs) Unit)
+
+(rule (copy_to_regs_range ty (range_singleton idx) dsts srcs)
+      (let ((dst WritableReg (writable_regs_get dsts idx))
+            (src Reg (value_regs_get srcs idx)))
+        (emit (gen_move ty dst src))))
+
+(rule (copy_to_regs_range ty (range_unwrap head tail) dsts srcs)
+      (let ((dst WritableReg (writable_regs_get dsts head))
+            (src Reg (value_regs_get srcs head))
+            (_ Unit (emit (gen_move ty dst src))))
+        (copy_to_regs_range ty tail dsts srcs)))
+
+(decl lower_return (Range ValueSlice) InstOutput)
+(rule (lower_return (range_empty) _) (output_none))
+(rule (lower_return (range_unwrap head tail) args)
+      (let ((_ Unit (copy_to_regs (retval head) (value_slice_get args head))))
+        (lower_return tail args)))
+
 ;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (convert Inst Value def_inst)

cranelift/filetests/filetests/isa/aarch64/call-pauth.clif

@@ -14,8 +14,8 @@ block0(v0: i64):
 ;   stp fp, lr, [sp, #-16]!
 ;   mov fp, sp
 ; block0:
-;   ldr x5, 8 ; b 12 ; data TestCase(%g) + 0
+;   ldr x4, 8 ; b 12 ; data TestCase(%g) + 0
-;   blr x5
+;   blr x4
 ;   ldp fp, lr, [sp], #16
 ;   autiasp ; ret
 

cranelift/filetests/filetests/isa/aarch64/call.clif

@@ -14,8 +14,8 @@ block0(v0: i64):
 ;   stp fp, lr, [sp, #-16]!
 ;   mov fp, sp
 ; block0:
-;   ldr x5, 8 ; b 12 ; data TestCase(%g) + 0
+;   ldr x4, 8 ; b 12 ; data TestCase(%g) + 0
-;   blr x5
+;   blr x4
 ;   ldp fp, lr, [sp], #16
 ;   ret
 
@@ -30,8 +30,8 @@ block0(v0: i32):
 ;   stp fp, lr, [sp, #-16]!
 ;   mov fp, sp
 ; block0:
-;   ldr x5, 8 ; b 12 ; data TestCase(%g) + 0
+;   ldr x4, 8 ; b 12 ; data TestCase(%g) + 0
-;   blr x5
+;   blr x4
 ;   ldp fp, lr, [sp], #16
 ;   ret
 
@@ -54,8 +54,8 @@ block0(v0: i32):
 ;   stp fp, lr, [sp, #-16]!
 ;   mov fp, sp
 ; block0:
-;   ldr x5, 8 ; b 12 ; data TestCase(%g) + 0
+;   ldr x4, 8 ; b 12 ; data TestCase(%g) + 0
-;   blr x5
+;   blr x4
 ;   ldp fp, lr, [sp], #16
 ;   ret
 
@@ -91,8 +91,8 @@ block0(v0: i8):
 ;   movz x6, #42
 ;   movz x7, #42
 ;   strb w15, [sp]
-;   ldr x15, 8 ; b 12 ; data TestCase(%g) + 0
+;   ldr x14, 8 ; b 12 ; data TestCase(%g) + 0
-;   blr x15
+;   blr x14
 ;   add sp, sp, #16
 ;   virtual_sp_offset_adjust -16
 ;   ldp fp, lr, [sp], #16
@@ -140,26 +140,26 @@ block0:
 ;   mov fp, sp
 ;   sub sp, sp, #48
 ; block0:
-;   ldr x9, 8 ; b 12 ; data TestCase(%g0) + 0
+;   ldr x8, 8 ; b 12 ; data TestCase(%g0) + 0
-;   blr x9
+;   blr x8
-;   str q0, [sp]
-;   ldr x11, 8 ; b 12 ; data TestCase(%g1) + 0
-;   blr x11
-;   str q0, [sp, #16]
-;   ldr x13, 8 ; b 12 ; data TestCase(%g1) + 0
-;   blr x13
 ;   str q0, [sp, #32]
-;   ldr x15, 8 ; b 12 ; data TestCase(%g2) + 0
+;   ldr x9, 8 ; b 12 ; data TestCase(%g1) + 0
-;   blr x15
+;   blr x9
-;   ldr q0, [sp]
+;   str q0, [sp, #16]
-;   ldr x1, 8 ; b 12 ; data TestCase(%g3) + 0
+;   ldr x10, 8 ; b 12 ; data TestCase(%g1) + 0
-;   blr x1
+;   blr x10
-;   ldr q0, [sp, #16]
+;   str q0, [sp]
-;   ldr x3, 8 ; b 12 ; data TestCase(%g4) + 0
+;   ldr x12, 8 ; b 12 ; data TestCase(%g2) + 0
-;   blr x3
+;   blr x12
 ;   ldr q0, [sp, #32]
-;   ldr x5, 8 ; b 12 ; data TestCase(%g4) + 0
+;   ldr x14, 8 ; b 12 ; data TestCase(%g3) + 0
-;   blr x5
+;   blr x14
+;   ldr q0, [sp, #16]
+;   ldr x0, 8 ; b 12 ; data TestCase(%g4) + 0
+;   blr x0
+;   ldr q0, [sp]
+;   ldr x2, 8 ; b 12 ; data TestCase(%g4) + 0
+;   blr x2
 ;   add sp, sp, #48
 ;   ldp fp, lr, [sp], #16
 ;   ret
@@ -184,26 +184,26 @@ block0:
 ;   mov fp, sp
 ;   sub sp, sp, #48
 ; block0:
+;   ldr x8, 8 ; b 12 ; data TestCase(%g0) + 0
+;   blr x8
+;   str q0, [sp, #32]
 ;   ldr x9, 8 ; b 12 ; data TestCase(%g0) + 0
 ;   blr x9
-;   str q0, [sp]
-;   ldr x11, 8 ; b 12 ; data TestCase(%g0) + 0
-;   blr x11
 ;   str q0, [sp, #16]
-;   ldr x13, 8 ; b 12 ; data TestCase(%g0) + 0
+;   ldr x10, 8 ; b 12 ; data TestCase(%g0) + 0
-;   blr x13
+;   blr x10
-;   str q0, [sp, #32]
+;   str q0, [sp]
-;   ldr x15, 8 ; b 12 ; data TestCase(%g1) + 0
+;   ldr x12, 8 ; b 12 ; data TestCase(%g1) + 0
-;   blr x15
+;   blr x12
-;   ldr q0, [sp]
-;   ldr x1, 8 ; b 12 ; data TestCase(%g2) + 0
-;   blr x1
-;   ldr q0, [sp, #16]
-;   ldr x3, 8 ; b 12 ; data TestCase(%g2) + 0
-;   blr x3
 ;   ldr q0, [sp, #32]
-;   ldr x5, 8 ; b 12 ; data TestCase(%g2) + 0
+;   ldr x14, 8 ; b 12 ; data TestCase(%g2) + 0
-;   blr x5
+;   blr x14
+;   ldr q0, [sp, #16]
+;   ldr x0, 8 ; b 12 ; data TestCase(%g2) + 0
+;   blr x0
+;   ldr q0, [sp]
+;   ldr x2, 8 ; b 12 ; data TestCase(%g2) + 0
+;   blr x2
 ;   add sp, sp, #48
 ;   ldp fp, lr, [sp], #16
 ;   ret
@@ -232,26 +232,26 @@ block0:
 ;   mov fp, sp
 ;   sub sp, sp, #48
 ; block0:
-;   ldr x9, 8 ; b 12 ; data TestCase(%g0) + 0
+;   ldr x8, 8 ; b 12 ; data TestCase(%g0) + 0
-;   blr x9
+;   blr x8
-;   str q0, [sp]
-;   ldr x11, 8 ; b 12 ; data TestCase(%g1) + 0
-;   blr x11
-;   str q0, [sp, #16]
-;   ldr x13, 8 ; b 12 ; data TestCase(%g2) + 0
-;   blr x13
 ;   str q0, [sp, #32]
-;   ldr x15, 8 ; b 12 ; data TestCase(%g3) + 0
+;   ldr x9, 8 ; b 12 ; data TestCase(%g1) + 0
-;   blr x15
+;   blr x9
-;   ldr q0, [sp]
+;   str q0, [sp, #16]
-;   ldr x1, 8 ; b 12 ; data TestCase(%g4) + 0
+;   ldr x10, 8 ; b 12 ; data TestCase(%g2) + 0
-;   blr x1
+;   blr x10
-;   ldr q0, [sp, #16]
+;   str q0, [sp]
-;   ldr x3, 8 ; b 12 ; data TestCase(%g5) + 0
+;   ldr x12, 8 ; b 12 ; data TestCase(%g3) + 0
-;   blr x3
+;   blr x12
 ;   ldr q0, [sp, #32]
-;   ldr x5, 8 ; b 12 ; data TestCase(%g6) + 0
+;   ldr x14, 8 ; b 12 ; data TestCase(%g4) + 0
-;   blr x5
+;   blr x14
+;   ldr q0, [sp, #16]
+;   ldr x0, 8 ; b 12 ; data TestCase(%g5) + 0
+;   blr x0
+;   ldr q0, [sp]
+;   ldr x2, 8 ; b 12 ; data TestCase(%g6) + 0
+;   blr x2
 ;   add sp, sp, #48
 ;   ldp fp, lr, [sp], #16
 ;   ret
@@ -283,8 +283,8 @@ block0(v0: i64):
 ;   movz x0, #42
 ;   movz x2, #42
 ;   mov x1, x7
-;   ldr x10, 8 ; b 12 ; data TestCase(%f11) + 0
+;   ldr x9, 8 ; b 12 ; data TestCase(%f11) + 0
-;   blr x10
+;   blr x9
 ;   ldp fp, lr, [sp], #16
 ;   ret
 
@@ -315,8 +315,8 @@ block0(v0: i64):
 ;   movz x3, #42
 ;   movz x0, #42
 ;   mov x2, x7
-;   ldr x10, 8 ; b 12 ; data TestCase(%f12) + 0
+;   ldr x9, 8 ; b 12 ; data TestCase(%f12) + 0
-;   blr x10
+;   blr x9
 ;   ldp fp, lr, [sp], #16
 ;   ret
 
@@ -347,8 +347,8 @@ block0(v0: i64):
 ;   movz x2, #42
 ;   movz x0, #42
 ;   mov x1, x7
-;   ldr x10, 8 ; b 12 ; data TestCase(%f13) + 0
+;   ldr x9, 8 ; b 12 ; data TestCase(%f13) + 0
-;   blr x10
+;   blr x9
 ;   ldp fp, lr, [sp], #16
 ;   ret
 
@@ -477,8 +477,8 @@ block0(v0: i64):
 ;   mov fp, sp
 ; block0:
 ;   mov x8, x0
-;   ldr x5, 8 ; b 12 ; data TestCase(%g) + 0
+;   ldr x4, 8 ; b 12 ; data TestCase(%g) + 0
-;   blr x5
+;   blr x4
 ;   mov x0, x8
 ;   ldp fp, lr, [sp], #16
 ;   ret

cranelift/filetests/filetests/isa/aarch64/constants.clif

@@ -164,7 +164,7 @@ block0:
 }
 
 ; block0:
-;   orr x0, xzr, #4294967295
+;   movn x0, #0
 ;   ret
 
 function %f() -> i32 {

cranelift/filetests/filetests/isa/aarch64/reftypes.clif

@@ -69,13 +69,13 @@ block3(v7: r64, v8: r64):
 ; block0:
 ;   str x1, [sp, #16]
 ;   str x0, [sp, #8]
-;   ldr x3, 8 ; b 12 ; data TestCase(%f) + 0
+;   ldr x2, 8 ; b 12 ; data TestCase(%f) + 0
-;   blr x3
+;   blr x2
-;   mov x9, sp
+;   mov x8, sp
 ;   ldr x11, [sp, #8]
-;   str x11, [x9]
+;   str x11, [x8]
-;   and w7, w0, #1
+;   and w6, w0, #1
-;   cbz x7, label1 ; b label3
+;   cbz x6, label1 ; b label3
 ; block1:
 ;   b label2
 ; block2:
@@ -89,8 +89,8 @@ block3(v7: r64, v8: r64):
 ;   ldr x1, [sp, #16]
 ;   b label5
 ; block5:
-;   mov x4, sp
+;   mov x3, sp
-;   ldr x2, [x4]
+;   ldr x2, [x3]
 ;   add sp, sp, #32
 ;   ldp fp, lr, [sp], #16
 ;   ret