diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index d2f289bc45..aa659c15b8 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -2900,11 +2900,8 @@ (rule 1 (emit_memcpy dst src (len_minus_one len)) (emit_side_effect (mvc (memarg_pair dst) (memarg_pair src) len))) (rule (emit_memcpy dst src len) - (let ((libcall LibCallInfo (lib_call_info_memcpy)) - (_ Unit (lib_accumulate_outgoing_args_size libcall)) - (_ Unit (emit_mov $I64 (writable_gpr 2) (load_addr dst))) - (_ Unit (emit_mov $I64 (writable_gpr 3) (load_addr src))) - (_ Unit (emit_imm $I64 (writable_gpr 4) len))) + (let ((libcall LibCallInfo (lib_call_info_memcpy (load_addr dst) (load_addr src) (imm $I64 len))) + (_ Unit (lib_accumulate_outgoing_args_size libcall))) (emit_side_effect (lib_call libcall)))) ;; Prepare a stack copy of a single (oversized) argument. @@ -2923,49 +2920,53 @@ ;; Copy a single argument/return value to its slots. ;; For oversized arguments, set the slot to the buffer address. -(decl copy_to_arg (LaneOrder i64 ABIArg Value) Unit) -(rule 2 (copy_to_arg lo base (abi_arg_only_slot slot) val) - (copy_val_to_arg_slot lo base slot val)) -(rule 1 (copy_to_arg _ base (abi_arg_struct_pointer slot offset _) _) +(decl copy_to_arg (CallArgListBuilder LaneOrder i64 ABIArg Value) Unit) +(rule 2 (copy_to_arg uses lo base (abi_arg_only_slot slot) val) + (copy_reg_to_arg_slot uses lo base slot (prepare_arg_val slot val))) +(rule 1 (copy_to_arg uses lo base (abi_arg_struct_pointer slot offset _) _) (let ((ptr Reg (load_addr (memarg_stack_off base offset)))) - (copy_reg_to_arg_slot base slot ptr))) -(rule 0 (copy_to_arg _ base (abi_arg_implicit_pointer slot offset _) _) + (copy_reg_to_arg_slot uses lo base slot ptr))) +(rule 0 (copy_to_arg uses lo base (abi_arg_implicit_pointer slot offset _) _) (let ((ptr Reg (load_addr (memarg_stack_off base offset)))) - (copy_reg_to_arg_slot base slot ptr))) + (copy_reg_to_arg_slot uses lo base slot ptr))) ;; Copy a single argument/return value from its slots. -(decl copy_from_arg (LaneOrder i64 ABIArg) ValueRegs) -(rule (copy_from_arg lo base (abi_arg_only_slot slot)) - (value_reg (copy_reg_from_arg_slot lo base slot))) +(decl copy_from_arg (CallRetList LaneOrder i64 ABIArg) ValueRegs) +(rule (copy_from_arg defs lo base (abi_arg_only_slot slot)) + (value_reg (copy_reg_from_arg_slot defs lo base slot))) -;; Copy one component of an argument/return value to its slot. -(decl copy_val_to_arg_slot (LaneOrder i64 ABIArgSlot Value) Unit) -(rule (copy_val_to_arg_slot lo _ (ABIArgSlot.Reg reg ty (ArgumentExtension.None)) val) - (emit_mov ty (real_reg_to_writable_reg reg) (abi_vec_elt_rev lo ty val))) -(rule (copy_val_to_arg_slot _ _ (ABIArgSlot.Reg reg _ (ArgumentExtension.Uext)) val) - (emit_put_in_reg_zext64 (real_reg_to_writable_reg reg) val)) -(rule (copy_val_to_arg_slot _ _ (ABIArgSlot.Reg reg _ (ArgumentExtension.Sext)) val) - (emit_put_in_reg_sext64 (real_reg_to_writable_reg reg) val)) -(rule (copy_val_to_arg_slot _ base (ABIArgSlot.Stack offset ty (ArgumentExtension.None)) val) - (emit_arg_store ty val (memarg_stack_off base offset))) -(rule (copy_val_to_arg_slot _ base (ABIArgSlot.Stack offset _ (ArgumentExtension.Uext)) val) - (emit_arg_store $I64 (put_in_reg_zext64 val) (memarg_stack_off base offset))) -(rule (copy_val_to_arg_slot _ base (ABIArgSlot.Stack offset _ (ArgumentExtension.Sext)) val) - (emit_arg_store $I64 (put_in_reg_sext64 val) (memarg_stack_off base offset))) +;; Place one component of an argument/return value into a register. +;; Copy reference values into registers of integer type. +;; Zero- or sign-extend as required by the ABI. +(decl prepare_arg_val (ABIArgSlot Value) Reg) +(rule 1 (prepare_arg_val (ABIArgSlot.Reg _ $R64 (ArgumentExtension.None)) val) + (copy_reg $I64 val)) +(rule (prepare_arg_val (ABIArgSlot.Reg _ _ (ArgumentExtension.None)) val) + val) +(rule (prepare_arg_val (ABIArgSlot.Reg _ _ (ArgumentExtension.Uext)) val) + (put_in_reg_zext64 val)) +(rule (prepare_arg_val (ABIArgSlot.Reg _ _ (ArgumentExtension.Sext)) val) + (put_in_reg_sext64 val)) +(rule (prepare_arg_val (ABIArgSlot.Stack _ _ (ArgumentExtension.None)) val) + val) +(rule (prepare_arg_val (ABIArgSlot.Stack _ _ (ArgumentExtension.Uext)) val) + (put_in_reg_zext64 val)) +(rule (prepare_arg_val (ABIArgSlot.Stack _ _ (ArgumentExtension.Sext)) val) + (put_in_reg_sext64 val)) ;; Copy one component of an argument/return value to its slot, where the ;; value is already extended and present in a register. -(decl copy_reg_to_arg_slot (i64 ABIArgSlot Reg) Unit) -(rule (copy_reg_to_arg_slot _ (ABIArgSlot.Reg reg ty ext) src) - (emit_mov (abi_ext_ty ext ty) (real_reg_to_writable_reg reg) src)) -(rule (copy_reg_to_arg_slot base (ABIArgSlot.Stack offset ty ext) src) +(decl copy_reg_to_arg_slot (CallArgListBuilder LaneOrder i64 ABIArgSlot Reg) Unit) +(rule (copy_reg_to_arg_slot uses lo _ (ABIArgSlot.Reg reg ty ext) src) + (args_builder_push uses (abi_vec_elt_rev lo ty src) reg)) +(rule (copy_reg_to_arg_slot _ _ base (ABIArgSlot.Stack offset ty ext) src) (emit_arg_store (abi_ext_ty ext ty) src (memarg_stack_off base offset))) ;; Copy one component of an argument/return value from its slot. -(decl copy_reg_from_arg_slot (LaneOrder i64 ABIArgSlot) Reg) -(rule (copy_reg_from_arg_slot lo _ (ABIArgSlot.Reg reg ty ext)) - (abi_vec_elt_rev lo ty (copy_reg (abi_ext_ty ext ty) (real_reg_to_reg reg)))) -(rule (copy_reg_from_arg_slot _ base (ABIArgSlot.Stack offset ty ext)) +(decl copy_reg_from_arg_slot (CallRetList LaneOrder i64 ABIArgSlot) Reg) +(rule (copy_reg_from_arg_slot defs lo _ (ABIArgSlot.Reg reg ty ext)) + (abi_vec_elt_rev lo ty (defs_lookup defs reg))) +(rule (copy_reg_from_arg_slot _ _ base (ABIArgSlot.Stack offset ty ext)) (emit_arg_load (abi_ext_ty ext ty) (memarg_stack_off base offset))) ;; Helper to compute the type of an implicitly extended argument/return value. @@ -3574,25 +3575,54 @@ ;; Helpers for generating `call` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; List of argument registers for a call instruction. +(type CallArgList extern (enum)) + +;; Partial (mutable) argument list in the process of being created. +(type CallArgListBuilder extern (enum)) + +;; Create a new empty instruction sequence builder. +(decl args_builder_new () CallArgListBuilder) +(extern constructor args_builder_new args_builder_new) + +;; Push an instruction to a sequence under construction. +(decl args_builder_push (CallArgListBuilder Reg RealReg) Unit) +(extern constructor args_builder_push args_builder_push) + +;; Complete the sequence under construction. +(decl args_builder_finish (CallArgListBuilder) CallArgList) +(extern constructor args_builder_finish args_builder_finish) + +;; List of return registers for a call instnuction. +(type CallRetList extern (enum)) + +;; Initialize return register list. +(decl defs_init (Sig) CallRetList) +(extern constructor defs_init defs_init) + +;; Look up return register in list. +(decl defs_lookup (CallRetList RealReg) Reg) +(extern constructor defs_lookup defs_lookup) + (decl abi_sig (SigRef) Sig) (extern constructor abi_sig abi_sig) -(decl abi_call_info (Sig ExternalName Opcode) BoxCallInfo) +(decl abi_call_info (Sig ExternalName CallArgList CallRetList Opcode) BoxCallInfo) (extern constructor abi_call_info abi_call_info) -(decl abi_call_ind_info (Sig Reg Opcode) BoxCallIndInfo) +(decl abi_call_ind_info (Sig Reg CallArgList CallRetList Opcode) BoxCallIndInfo) (extern constructor abi_call_ind_info abi_call_ind_info) (decl writable_link_reg () WritableReg) (rule (writable_link_reg) (writable_gpr 14)) -(decl abi_call (Sig ExternalName Opcode) SideEffectNoResult) -(rule (abi_call abi name opcode) - (call_impl (writable_link_reg) (abi_call_info abi name opcode))) +(decl abi_call (Sig ExternalName CallArgList CallRetList Opcode) SideEffectNoResult) +(rule (abi_call abi name uses defs opcode) + (call_impl (writable_link_reg) (abi_call_info abi name uses defs opcode))) -(decl abi_call_ind (Sig Reg Opcode) SideEffectNoResult) -(rule (abi_call_ind abi target opcode) - (call_ind_impl (writable_link_reg) (abi_call_ind_info abi target opcode))) +(decl abi_call_ind (Sig Reg CallArgList CallRetList Opcode) SideEffectNoResult) +(rule (abi_call_ind abi target uses defs opcode) + (call_ind_impl (writable_link_reg) (abi_call_ind_info abi target uses defs opcode))) (decl abi_accumulate_outgoing_args_size (Sig) Unit) (extern constructor abi_accumulate_outgoing_args_size abi_accumulate_outgoing_args_size) @@ -3605,10 +3635,10 @@ (type LibCallInfo extern (enum)) -(decl lib_call_info_memcpy () LibCallInfo) +(decl lib_call_info_memcpy (Reg Reg Reg) LibCallInfo) (extern constructor lib_call_info_memcpy lib_call_info_memcpy) -(decl lib_call_info_tls_get_offset (SymbolReloc) LibCallInfo) +(decl lib_call_info_tls_get_offset (WritableReg Reg Reg SymbolReloc) LibCallInfo) (extern constructor lib_call_info_tls_get_offset lib_call_info_tls_get_offset) (decl lib_call_info (LibCallInfo) BoxCallInfo) diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs index cf5c34a65f..706c50d4d9 100644 --- a/cranelift/codegen/src/isa/s390x/inst/mod.rs +++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs @@ -39,8 +39,8 @@ pub use crate::isa::s390x::lower::isle::generated_code::{ #[derive(Clone, Debug)] pub struct CallInfo { pub dest: ExternalName, - pub uses: SmallVec<[Reg; 8]>, - pub defs: SmallVec<[Writable; 8]>, + pub uses: CallArgList, + pub defs: CallRetList, pub clobbers: PRegSet, pub opcode: Opcode, pub caller_callconv: CallConv, @@ -53,8 +53,8 @@ pub struct CallInfo { #[derive(Clone, Debug)] pub struct CallIndInfo { pub rn: Reg, - pub uses: SmallVec<[Reg; 8]>, - pub defs: SmallVec<[Writable; 8]>, + pub uses: CallArgList, + pub defs: CallRetList, pub clobbers: PRegSet, pub opcode: Opcode, pub caller_callconv: CallConv, @@ -1004,15 +1004,23 @@ fn s390x_get_operands VReg>(inst: &Inst, collector: &mut OperandC } &Inst::Call { link, ref info } => { collector.reg_def(link); - collector.reg_uses(&*info.uses); - collector.reg_defs(&*info.defs); + for u in &info.uses { + collector.reg_fixed_use(u.vreg, u.preg); + } + for d in &info.defs { + collector.reg_fixed_def(d.vreg, d.preg); + } collector.reg_clobbers(info.clobbers); } &Inst::CallInd { link, ref info } => { collector.reg_def(link); collector.reg_use(info.rn); - collector.reg_uses(&*info.uses); - collector.reg_defs(&*info.defs); + for u in &info.uses { + collector.reg_fixed_use(u.vreg, u.preg); + } + for d in &info.defs { + collector.reg_fixed_def(d.vreg, d.preg); + } collector.reg_clobbers(info.clobbers); } &Inst::Args { ref args } => { diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle index 965d17902e..270f035353 100644 --- a/cranelift/codegen/src/isa/s390x/lower.isle +++ b/cranelift/codegen/src/isa/s390x/lower.isle @@ -2283,12 +2283,11 @@ ;; Helper to perform a call to the __tls_get_offset library routine. (decl lib_call_tls_get_offset (Reg Reg SymbolReloc) Reg) (rule (lib_call_tls_get_offset got got_offset symbol) - (let ((libcall LibCallInfo (lib_call_info_tls_get_offset symbol)) + (let ((tls_offset WritableReg (temp_writable_reg $I64)) + (libcall LibCallInfo (lib_call_info_tls_get_offset tls_offset got got_offset symbol)) (_ Unit (lib_accumulate_outgoing_args_size libcall)) - (_ Unit (emit_mov $I64 (writable_gpr 12) got)) - (_ Unit (emit_mov $I64 (writable_gpr 2) got_offset)) (_ Unit (emit_side_effect (lib_call libcall)))) - (copy_reg $I64 (writable_gpr 2)))) + tls_offset)) ;; Helper to extract the current thread pointer from %a0/%a1. (decl thread_pointer () Reg) @@ -3905,34 +3904,39 @@ (rule 1 (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args)) (let ((abi Sig (abi_sig sig_ref)) (_ Unit (abi_accumulate_outgoing_args_size abi)) - (_ InstOutput (lower_call_args abi (range 0 (abi_num_args abi)) args)) - (_ InstOutput (side_effect (abi_call abi name (Opcode.Call))))) - (lower_call_rets abi (range 0 (abi_num_rets abi)) (output_builder_new)))) + (uses CallArgList (lower_call_args abi (range 0 (abi_num_args abi)) args)) + (defs CallRetList (defs_init abi)) + (_ InstOutput (side_effect (abi_call abi name uses defs (Opcode.Call))))) + (lower_call_rets abi defs (range 0 (abi_num_rets abi)) (output_builder_new)))) ;; Direct call to an out-of-range function (implicitly via pointer). (rule (lower (call (func_ref_data sig_ref name _) args)) (let ((abi Sig (abi_sig sig_ref)) (_ Unit (abi_accumulate_outgoing_args_size abi)) - (_ InstOutput (lower_call_args abi (range 0 (abi_num_args abi)) args)) + (uses CallArgList (lower_call_args abi (range 0 (abi_num_args abi)) args)) + (defs CallRetList (defs_init abi)) (target Reg (load_symbol_reloc (SymbolReloc.Absolute name 0))) - (_ InstOutput (side_effect (abi_call_ind abi target (Opcode.Call))))) - (lower_call_rets abi (range 0 (abi_num_rets abi)) (output_builder_new)))) + (_ InstOutput (side_effect (abi_call_ind abi target uses defs (Opcode.Call))))) + (lower_call_rets abi defs (range 0 (abi_num_rets abi)) (output_builder_new)))) ;; Indirect call. (rule (lower (call_indirect sig_ref ptr args)) (let ((abi Sig (abi_sig sig_ref)) (target Reg (put_in_reg ptr)) (_ Unit (abi_accumulate_outgoing_args_size abi)) - (_ InstOutput (lower_call_args abi (range 0 (abi_num_args abi)) args)) - (_ InstOutput (side_effect (abi_call_ind abi target (Opcode.CallIndirect))))) - (lower_call_rets abi (range 0 (abi_num_rets abi)) (output_builder_new)))) + (uses CallArgList (lower_call_args abi (range 0 (abi_num_args abi)) args)) + (defs CallRetList (defs_init abi)) + (_ InstOutput (side_effect (abi_call_ind abi target uses defs (Opcode.CallIndirect))))) + (lower_call_rets abi defs (range 0 (abi_num_rets abi)) (output_builder_new)))) ;; Lower function arguments. -(decl lower_call_args (Sig Range ValueSlice) InstOutput) +(decl lower_call_args (Sig Range ValueSlice) CallArgList) (rule (lower_call_args abi range args) - (let ((_ InstOutput (lower_call_args_buffer abi range args)) - (_ InstOutput (lower_call_args_slots abi range args))) - (lower_call_ret_arg abi))) + (let ((uses CallArgListBuilder (args_builder_new)) + (_ InstOutput (lower_call_args_buffer abi range args)) + (_ InstOutput (lower_call_args_slots abi uses range args)) + (_ InstOutput (lower_call_ret_arg abi uses))) + (args_builder_finish uses))) ;; Lower function arguments (part 1): prepare buffer copies. (decl lower_call_args_buffer (Sig Range ValueSlice) InstOutput) @@ -3943,31 +3947,31 @@ (lower_call_args_buffer abi tail args))) ;; Lower function arguments (part 2): set up registers / stack slots. -(decl lower_call_args_slots (Sig Range ValueSlice) InstOutput) -(rule (lower_call_args_slots abi (range_empty) _) (output_none)) -(rule (lower_call_args_slots abi (range_unwrap head tail) args) - (let ((_ Unit (copy_to_arg (abi_lane_order abi) +(decl lower_call_args_slots (Sig CallArgListBuilder Range ValueSlice) InstOutput) +(rule (lower_call_args_slots abi _ (range_empty) _) (output_none)) +(rule (lower_call_args_slots abi uses (range_unwrap head tail) args) + (let ((_ Unit (copy_to_arg uses (abi_lane_order abi) 0 (abi_get_arg abi head) (value_slice_get args head)))) - (lower_call_args_slots abi tail args))) + (lower_call_args_slots abi uses tail args))) ;; Lower function arguments (part 3): implicit return-area pointer. -(decl lower_call_ret_arg (Sig) InstOutput) -(rule (lower_call_ret_arg (abi_no_ret_arg)) (output_none)) -(rule 1 (lower_call_ret_arg abi @ (abi_ret_arg (abi_arg_only_slot slot))) +(decl lower_call_ret_arg (Sig CallArgListBuilder) InstOutput) +(rule (lower_call_ret_arg (abi_no_ret_arg) _) (output_none)) +(rule 1 (lower_call_ret_arg abi @ (abi_ret_arg (abi_arg_only_slot slot)) uses) (let ((ret_arg Reg (load_addr (memarg_stack_off (abi_sized_stack_arg_space abi) 0))) - (_ Unit (copy_reg_to_arg_slot 0 slot ret_arg))) + (_ Unit (copy_reg_to_arg_slot uses (abi_lane_order abi) 0 slot ret_arg))) (output_none))) ;; Lower function return values by collecting them from registers / stack slots. -(decl lower_call_rets (Sig Range InstOutputBuilder) InstOutput) -(rule (lower_call_rets abi (range_empty) builder) (output_builder_finish builder)) -(rule (lower_call_rets abi (range_unwrap head tail) builder) - (let ((ret ValueRegs (copy_from_arg (abi_lane_order abi) +(decl lower_call_rets (Sig CallRetList Range InstOutputBuilder) InstOutput) +(rule (lower_call_rets abi _ (range_empty) builder) (output_builder_finish builder)) +(rule (lower_call_rets abi defs (range_unwrap head tail) builder) + (let ((ret ValueRegs (copy_from_arg defs (abi_lane_order abi) (abi_sized_stack_arg_space abi) (abi_get_ret abi head))) (_ Unit (output_builder_push builder ret))) - (lower_call_rets abi tail builder))) + (lower_call_rets abi defs tail builder))) ;;;; Rules for `get_{frame,stack}_pointer` and `get_return_address` ;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index 16ff77a3dc..1e6b3da5aa 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -17,17 +17,20 @@ use crate::machinst::{MachLabel, Reg}; use crate::settings::Flags; use crate::{ ir::{ - condcodes::*, immediates::*, types::*, AtomicRmwOp, Endianness, Inst, InstructionData, - KnownSymbol, LibCall, MemFlags, Opcode, TrapCode, Value, ValueList, + condcodes::*, immediates::*, types::*, ArgumentPurpose, AtomicRmwOp, Endianness, Inst, + InstructionData, KnownSymbol, LibCall, MemFlags, Opcode, TrapCode, Value, ValueList, }, isa::unwind::UnwindInst, isa::CallConv, machinst::abi::ABIMachineSpec, - machinst::{ArgPair, InsnOutput, Lower, MachInst, VCodeConstant, VCodeConstantData}, + machinst::{ + ArgPair, CallArgList, CallArgPair, CallRetList, CallRetPair, InsnOutput, Lower, MachInst, + VCodeConstant, VCodeConstantData, + }, }; use crate::{isle_common_prelude_methods, isle_lower_prelude_methods}; use regalloc2::PReg; -use smallvec::{smallvec, SmallVec}; +use smallvec::smallvec; use std::boxed::Box; use std::cell::Cell; use std::convert::TryFrom; @@ -37,6 +40,8 @@ use target_lexicon::Triple; /// Information describing a library call to be emitted. pub struct LibCallInfo { libcall: LibCall, + uses: CallArgList, + defs: CallRetList, tls_symbol: Option, } @@ -48,6 +53,7 @@ type BoxSymbolReloc = Box; type VecMInst = Vec; type VecMInstBuilder = Cell>; type VecArgPair = Vec; +type CallArgListBuilder = Cell; /// The main entry point for lowering with ISLE. pub(crate) fn lower( @@ -92,6 +98,69 @@ pub(crate) fn lower_branch( impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { isle_lower_prelude_methods!(); + #[inline] + fn args_builder_new(&mut self) -> CallArgListBuilder { + Cell::new(CallArgList::new()) + } + + #[inline] + fn args_builder_push( + &mut self, + builder: &CallArgListBuilder, + vreg: Reg, + preg: RealReg, + ) -> Unit { + let mut args = builder.take(); + args.push(CallArgPair { + vreg, + preg: preg.into(), + }); + builder.set(args); + } + + #[inline] + fn args_builder_finish(&mut self, builder: &CallArgListBuilder) -> CallArgList { + builder.take() + } + + fn defs_init(&mut self, abi: &Sig) -> CallRetList { + // Allocate writable registers for all retval regs, except for StructRet args. + let mut defs = smallvec![]; + for i in 0..self.lower_ctx.sigs()[*abi].num_rets() { + if let &ABIArg::Slots { + ref slots, purpose, .. + } = &self.lower_ctx.sigs()[*abi].get_ret(i) + { + if purpose == ArgumentPurpose::StructReturn { + continue; + } + for slot in slots { + match slot { + &ABIArgSlot::Reg { reg, ty, .. } => { + let value_regs = self.lower_ctx.alloc_tmp(ty); + defs.push(CallRetPair { + vreg: value_regs.only_reg().unwrap(), + preg: reg.into(), + }); + } + _ => {} + } + } + } + } + defs + } + + fn defs_lookup(&mut self, defs: &CallRetList, reg: RealReg) -> Reg { + let reg = Reg::from(reg); + for def in defs { + if def.preg == reg { + return def.vreg.to_reg(); + } + } + unreachable!() + } + fn abi_sig(&mut self, sig_ref: SigRef) -> Sig { self.lower_ctx.sigs().abi_sig_for_sig_ref(sig_ref) } @@ -108,13 +177,19 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> .accumulate_outgoing_args_size(off as u32); } - fn abi_call_info(&mut self, abi: &Sig, name: ExternalName, opcode: &Opcode) -> BoxCallInfo { - let (uses, defs, clobbers) = - self.lower_ctx.sigs()[*abi].call_uses_defs_clobbers::(); + fn abi_call_info( + &mut self, + abi: &Sig, + name: ExternalName, + uses: &CallArgList, + defs: &CallRetList, + opcode: &Opcode, + ) -> BoxCallInfo { + let clobbers = self.lower_ctx.sigs()[*abi].call_clobbers::(); Box::new(CallInfo { dest: name.clone(), - uses, - defs, + uses: uses.clone(), + defs: defs.clone(), clobbers, opcode: *opcode, caller_callconv: self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()), @@ -123,13 +198,19 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> }) } - fn abi_call_ind_info(&mut self, abi: &Sig, target: Reg, opcode: &Opcode) -> BoxCallIndInfo { - let (uses, defs, clobbers) = - self.lower_ctx.sigs()[*abi].call_uses_defs_clobbers::(); + fn abi_call_ind_info( + &mut self, + abi: &Sig, + target: Reg, + uses: &CallArgList, + defs: &CallRetList, + opcode: &Opcode, + ) -> BoxCallIndInfo { + let clobbers = self.lower_ctx.sigs()[*abi].call_clobbers::(); Box::new(CallIndInfo { rn: target, - uses, - defs, + uses: uses.clone(), + defs: defs.clone(), clobbers, opcode: *opcode, caller_callconv: self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()), @@ -137,16 +218,51 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> }) } - fn lib_call_info_memcpy(&mut self) -> LibCallInfo { + fn lib_call_info_memcpy(&mut self, dst: Reg, src: Reg, len: Reg) -> LibCallInfo { LibCallInfo { libcall: LibCall::Memcpy, + uses: smallvec![ + CallArgPair { + vreg: dst, + preg: gpr(2), + }, + CallArgPair { + vreg: src, + preg: gpr(3), + }, + CallArgPair { + vreg: len, + preg: gpr(4), + }, + ], + defs: smallvec![], tls_symbol: None, } } - fn lib_call_info_tls_get_offset(&mut self, tls_symbol: &SymbolReloc) -> LibCallInfo { + fn lib_call_info_tls_get_offset( + &mut self, + tls_offset: WritableReg, + got: Reg, + got_offset: Reg, + tls_symbol: &SymbolReloc, + ) -> LibCallInfo { LibCallInfo { libcall: LibCall::ElfTlsGetOffset, + uses: smallvec![ + CallArgPair { + vreg: got, + preg: gpr(12), + }, + CallArgPair { + vreg: got_offset, + preg: gpr(2), + }, + ], + defs: smallvec![CallRetPair { + vreg: tls_offset, + preg: gpr(2), + },], tls_symbol: Some(tls_symbol.clone()), } } @@ -162,26 +278,16 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> let caller_callconv = self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()); let callee_callconv = CallConv::for_libcall(&self.flags, caller_callconv); - // Uses and defs are defined by the particular libcall. - let (uses, defs): (SmallVec<[Reg; 8]>, SmallVec<[WritableReg; 8]>) = match info.libcall { - LibCall::Memcpy => ( - smallvec![gpr(2), gpr(3), gpr(4)], - smallvec![writable_gpr(2)], - ), - LibCall::ElfTlsGetOffset => (smallvec![gpr(2), gpr(12)], smallvec![writable_gpr(2)]), - _ => unreachable!(), - }; - - // Clobbers are defined by the calling convention. Remove deps from clobbers. + // Clobbers are defined by the calling convention. Remove defs from clobbers. let mut clobbers = S390xMachineDeps::get_regs_clobbered_by_call(callee_callconv); - for reg in &defs { - clobbers.remove(PReg::from(reg.to_reg().to_real_reg().unwrap())); + for reg in &info.defs { + clobbers.remove(PReg::from(reg.preg.to_real_reg().unwrap())); } Box::new(CallInfo { dest: ExternalName::LibCall(info.libcall), - uses, - defs, + uses: info.uses.clone(), + defs: info.defs.clone(), clobbers, opcode: Opcode::Call, caller_callconv, diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index e51a636145..a7d21dd59a 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -624,77 +624,6 @@ impl SigData { }) } - /// Return all uses (i.e, function args), defs (i.e., return values - /// and caller-saved registers), and clobbers for the callsite. - /// - /// FIXME: used only by s390x; remove once that backend moves to - /// `call_clobbers` and constraint-based calls. - pub fn call_uses_defs_clobbers( - &self, - ) -> (SmallVec<[Reg; 8]>, SmallVec<[Writable; 8]>, PRegSet) { - // Compute uses: all arg regs. - let mut uses = smallvec![]; - for arg in &self.args { - match arg { - &ABIArg::Slots { ref slots, .. } => { - for slot in slots { - match slot { - &ABIArgSlot::Reg { reg, .. } => { - uses.push(Reg::from(reg)); - } - _ => {} - } - } - } - &ABIArg::StructArg { ref pointer, .. } => { - if let Some(slot) = pointer { - match slot { - &ABIArgSlot::Reg { reg, .. } => { - uses.push(Reg::from(reg)); - } - _ => {} - } - } - } - &ABIArg::ImplicitPtrArg { ref pointer, .. } => match pointer { - &ABIArgSlot::Reg { reg, .. } => { - uses.push(Reg::from(reg)); - } - _ => {} - }, - } - } - - // Get clobbers: all caller-saves. These may include return value - // regs, which we will remove from the clobber set below. - let mut clobbers = M::get_regs_clobbered_by_call(self.call_conv); - - // Compute defs: all retval regs, and all caller-save - // (clobbered) regs, except for StructRet args. - let mut defs = smallvec![]; - for ret in &self.rets { - if let &ABIArg::Slots { - ref slots, purpose, .. - } = ret - { - if purpose == ir::ArgumentPurpose::StructReturn { - continue; - } - for slot in slots { - match slot { - &ABIArgSlot::Reg { reg, .. } => { - defs.push(Writable::from_reg(Reg::from(reg))); - clobbers.remove(PReg::from(reg)); - } - _ => {} - } - } - } - } - - (uses, defs, clobbers) - } - /// Return all clobbers for the callsite. pub fn call_clobbers(&self) -> PRegSet { // Get clobbers: all caller-saves. These may include return value diff --git a/cranelift/filetests/filetests/isa/s390x/call.clif b/cranelift/filetests/filetests/isa/s390x/call.clif index 7dfd72f031..53bc100ca2 100644 --- a/cranelift/filetests/filetests/isa/s390x/call.clif +++ b/cranelift/filetests/filetests/isa/s390x/call.clif @@ -35,8 +35,8 @@ block0(v0: i32): ; virtual_sp_offset_adjust 160 ; block0: ; llgfr %r2, %r2 -; bras %r1, 12 ; data %g + 0 ; lg %r3, 0(%r1) -; basr %r14, %r3 +; bras %r1, 12 ; data %g + 0 ; lg %r4, 0(%r1) +; basr %r14, %r4 ; lmg %r14, %r15, 272(%r15) ; br %r14 @@ -63,8 +63,8 @@ block0(v0: i32): ; virtual_sp_offset_adjust 160 ; block0: ; lgfr %r2, %r2 -; bras %r1, 12 ; data %g + 0 ; lg %r3, 0(%r1) -; basr %r14, %r3 +; bras %r1, 12 ; data %g + 0 ; lg %r4, 0(%r1) +; basr %r14, %r4 ; lmg %r14, %r15, 272(%r15) ; br %r14 @@ -107,8 +107,8 @@ block0(v0: i32): ; virtual_sp_offset_adjust 160 ; block0: ; llgfr %r2, %r2 -; bras %r1, 12 ; data %g + 0 ; lg %r3, 0(%r1) -; basr %r14, %r3 +; bras %r1, 12 ; data %g + 0 ; lg %r4, 0(%r1) +; basr %r14, %r4 ; lmg %r14, %r15, 272(%r15) ; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/reftypes.clif b/cranelift/filetests/filetests/isa/s390x/reftypes.clif index c3178c935b..793291ec7e 100644 --- a/cranelift/filetests/filetests/isa/s390x/reftypes.clif +++ b/cranelift/filetests/filetests/isa/s390x/reftypes.clif @@ -69,15 +69,15 @@ block3(v7: r64, v8: r64): ; aghi %r15, -184 ; virtual_sp_offset_adjust 160 ; block0: -; stg %r3, 176(%r15) ; stg %r2, 168(%r15) -; bras %r1, 12 ; data %f + 0 ; lg %r3, 0(%r1) -; basr %r14, %r3 -; la %r5, 160(%r15) +; stg %r3, 176(%r15) +; bras %r1, 12 ; data %f + 0 ; lg %r5, 0(%r1) +; basr %r14, %r5 +; la %r4, 160(%r15) ; lg %r3, 168(%r15) -; stg %r3, 0(%r5) -; lbr %r2, %r2 -; chi %r2, 0 +; stg %r3, 0(%r4) +; lbr %r5, %r2 +; chi %r5, 0 ; jgnlh label1 ; jg label3 ; block1: ; jg label2 diff --git a/cranelift/filetests/filetests/isa/s390x/struct-arg.clif b/cranelift/filetests/filetests/isa/s390x/struct-arg.clif index 4361c6c042..391fd20d03 100644 --- a/cranelift/filetests/filetests/isa/s390x/struct-arg.clif +++ b/cranelift/filetests/filetests/isa/s390x/struct-arg.clif @@ -103,22 +103,22 @@ block0(v0: i64, v1: i64, v2: i64): return v3 } -; stmg %r7, %r15, 56(%r15) +; stmg %r8, %r15, 64(%r15) ; aghi %r15, -1248 ; virtual_sp_offset_adjust 1248 ; block0: -; lgr %r7, %r2 -; lgr %r9, %r4 +; lgr %r8, %r2 +; lgr %r10, %r4 ; la %r2, 160(%r15) ; la %r3, 0(%r3) ; lghi %r4, 1024 ; brasl %r14, %Memcpy -; lgr %r4, %r9 +; lgr %r4, %r10 ; mvc 1184(63,%r15), 0(%r4) -; lgr %r2, %r7 ; la %r3, 160(%r15) ; la %r4, 1184(%r15) +; lgr %r2, %r8 ; brasl %r14, userextname0 -; lmg %r7, %r15, 1304(%r15) +; lmg %r8, %r15, 1312(%r15) ; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/tls_elf.clif b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif index e23fc14d04..20c08053ee 100644 --- a/cranelift/filetests/filetests/isa/s390x/tls_elf.clif +++ b/cranelift/filetests/filetests/isa/s390x/tls_elf.clif @@ -18,9 +18,9 @@ block0(v0: i32): ; bras %r1, 12 ; data userextname0@tlsgd ; lg %r2, 0(%r1) ; brasl %r14, %ElfTlsGetOffset:tls_gdcall:userextname0 ; ear %r3, %a0 -; sllg %r5, %r3, 32 -; ear %r5, %a1 -; agr %r2, %r5 +; sllg %r3, %r3, 32 +; ear %r3, %a1 +; agr %r2, %r3 ; lmg %r12, %r15, 256(%r15) ; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/vec-abi.clif b/cranelift/filetests/filetests/isa/s390x/vec-abi.clif index bd9bdb0e1e..a18d9c8fde 100644 --- a/cranelift/filetests/filetests/isa/s390x/vec-abi.clif +++ b/cranelift/filetests/filetests/isa/s390x/vec-abi.clif @@ -13,8 +13,8 @@ block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16): ; aghi %r15, -160 ; virtual_sp_offset_adjust 160 ; block0: -; bras %r1, 12 ; data %callee_be + 0 ; lg %r4, 0(%r1) -; basr %r14, %r4 +; bras %r1, 12 ; data %callee_be + 0 ; lg %r5, 0(%r1) +; basr %r14, %r5 ; lmg %r14, %r15, 272(%r15) ; br %r14 @@ -39,19 +39,19 @@ block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16): ; std %f15, 216(%r15) ; block0: ; vpdi %v24, %v24, %v24, 4 -; vpdi %v17, %v25, %v25, 4 -; verllg %v25, %v17, 32 -; vpdi %v22, %v26, %v26, 4 -; verllg %v26, %v22, 32 -; verllf %v26, %v26, 16 -; vpdi %v29, %v27, %v27, 4 -; verllg %v31, %v29, 32 -; verllf %v1, %v31, 16 -; verllh %v27, %v1, 8 -; bras %r1, 12 ; data %callee_le + 0 ; lg %r4, 0(%r1) -; basr %r14, %r4 -; vpdi %v19, %v24, %v24, 4 -; verllg %v24, %v19, 32 +; vpdi %v16, %v25, %v25, 4 +; verllg %v25, %v16, 32 +; vpdi %v20, %v26, %v26, 4 +; verllg %v22, %v20, 32 +; verllf %v26, %v22, 16 +; vpdi %v27, %v27, %v27, 4 +; verllg %v28, %v27, 32 +; verllf %v30, %v28, 16 +; verllh %v27, %v30, 8 +; bras %r1, 12 ; data %callee_le + 0 ; lg %r5, 0(%r1) +; basr %r14, %r5 +; vpdi %v6, %v24, %v24, 4 +; verllg %v24, %v6, 32 ; ld %f8, 160(%r15) ; ld %f9, 168(%r15) ; ld %f10, 176(%r15) @@ -84,19 +84,19 @@ block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16): ; std %f15, 216(%r15) ; block0: ; vpdi %v24, %v24, %v24, 4 -; vpdi %v17, %v25, %v25, 4 -; verllg %v25, %v17, 32 -; vpdi %v22, %v26, %v26, 4 -; verllg %v26, %v22, 32 -; verllf %v26, %v26, 16 -; vpdi %v29, %v27, %v27, 4 -; verllg %v31, %v29, 32 -; verllf %v1, %v31, 16 -; verllh %v27, %v1, 8 -; bras %r1, 12 ; data %callee_be + 0 ; lg %r4, 0(%r1) -; basr %r14, %r4 -; vpdi %v19, %v24, %v24, 4 -; verllg %v24, %v19, 32 +; vpdi %v16, %v25, %v25, 4 +; verllg %v25, %v16, 32 +; vpdi %v20, %v26, %v26, 4 +; verllg %v22, %v20, 32 +; verllf %v26, %v22, 16 +; vpdi %v27, %v27, %v27, 4 +; verllg %v28, %v27, 32 +; verllf %v30, %v28, 16 +; verllh %v27, %v30, 8 +; bras %r1, 12 ; data %callee_be + 0 ; lg %r5, 0(%r1) +; basr %r14, %r5 +; vpdi %v6, %v24, %v24, 4 +; verllg %v24, %v6, 32 ; ld %f8, 160(%r15) ; ld %f9, 168(%r15) ; ld %f10, 176(%r15) @@ -120,8 +120,8 @@ block0(v0: i64x2, v1: i32x4, v2: i16x8, v3: i8x16): ; aghi %r15, -160 ; virtual_sp_offset_adjust 160 ; block0: -; bras %r1, 12 ; data %callee_le + 0 ; lg %r4, 0(%r1) -; basr %r14, %r4 +; bras %r1, 12 ; data %callee_le + 0 ; lg %r5, 0(%r1) +; basr %r14, %r5 ; lmg %r14, %r15, 272(%r15) ; br %r14