From 7a9479f77c31e60548e3e8ac0455c988f2e4f7ab Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 29 Jun 2022 23:22:50 +0200 Subject: [PATCH] ISLE: Migrate call and return instructions (#3785) This adds infrastructure to allow implementing call and return instructions in ISLE, and migrates the s390x back-end. To implement ABI details, this patch creates public accessors for `ABISig` and makes them accessible in ISLE. All actual code generation is then done in ISLE rules, following the information provided by that signature. [ Note that the s390x back end never requires multiple slots for a single argument - the infrastructure to handle this should already be present, however. ] To implement loops in ISLE rules, this patch uses regular tail recursion, employing a `Range` data structure holding a range of integers to be looped over. --- cranelift/codegen/src/isa/s390x/abi.rs | 59 +----- cranelift/codegen/src/isa/s390x/inst.isle | 116 +++++++++++ cranelift/codegen/src/isa/s390x/lower.isle | 68 +++++++ cranelift/codegen/src/isa/s390x/lower.rs | 64 +------ cranelift/codegen/src/isa/s390x/lower/isle.rs | 41 +++- cranelift/codegen/src/machinst/abi_impl.rs | 181 ++++++++++++------ cranelift/codegen/src/machinst/isle.rs | 97 +++++++++- cranelift/codegen/src/prelude.isle | 112 +++++++++++ .../filetests/isa/s390x/reftypes.clif | 12 +- 9 files changed, 566 insertions(+), 184 deletions(-) diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs index 080d54d300..ddbcafcadf 100644 --- a/cranelift/codegen/src/isa/s390x/abi.rs +++ b/cranelift/codegen/src/isa/s390x/abi.rs @@ -69,7 +69,6 @@ use crate::machinst::*; use crate::machinst::{RealReg, Reg, RegClass, Writable}; use crate::settings; use crate::{CodegenError, CodegenResult}; -use alloc::boxed::Box; use alloc::vec::Vec; use regalloc2::{PReg, PRegSet, VReg}; use smallvec::{smallvec, SmallVec}; @@ -80,9 +79,6 @@ use std::convert::TryFrom; /// Support for the S390x ABI from the callee side (within a function body). pub type S390xABICallee = ABICalleeImpl; -/// Support for the S390x ABI from the caller side (at a callsite). -pub type S390xABICaller = ABICallerImpl; - /// ABI Register usage fn in_int_reg(ty: Type) -> bool { @@ -616,57 +612,16 @@ impl ABIMachineSpec for S390xMachineDeps { } fn gen_call( - dest: &CallDest, - uses: SmallVec<[Reg; 8]>, - defs: SmallVec<[Writable; 8]>, - clobbers: PRegSet, - opcode: ir::Opcode, - tmp: Writable, + _dest: &CallDest, + _uses: SmallVec<[Reg; 8]>, + _defs: SmallVec<[Writable; 8]>, + _clobbers: PRegSet, + _opcode: ir::Opcode, + _tmp: Writable, _callee_conv: isa::CallConv, _caller_conv: isa::CallConv, ) -> SmallVec<[Inst; 2]> { - let mut insts = SmallVec::new(); - match &dest { - &CallDest::ExtName(ref name, RelocDistance::Near) => insts.push(Inst::Call { - link: writable_gpr(14), - info: Box::new(CallInfo { - dest: name.clone(), - uses, - defs, - clobbers, - opcode, - }), - }), - &CallDest::ExtName(ref name, RelocDistance::Far) => { - insts.push(Inst::LoadExtNameFar { - rd: tmp, - name: Box::new(name.clone()), - offset: 0, - }); - insts.push(Inst::CallInd { - link: writable_gpr(14), - info: Box::new(CallIndInfo { - rn: tmp.to_reg(), - uses, - defs, - clobbers, - opcode, - }), - }); - } - &CallDest::Reg(reg) => insts.push(Inst::CallInd { - link: writable_gpr(14), - info: Box::new(CallIndInfo { - rn: *reg, - uses, - defs, - clobbers, - opcode, - }), - }), - } - - insts + unreachable!(); } fn gen_memcpy( diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index 3954edd012..23a2631c6b 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -1124,6 +1124,11 @@ (decl memarg_symbol (ExternalName i32 MemFlags) MemArg) (extern constructor memarg_symbol memarg_symbol) +;; Create a MemArg refering to a stack address formed by +;; adding a base (relative to SP) and an offset. +(decl memarg_stack_off (i64 i64) MemArg) +(extern constructor memarg_stack_off memarg_stack_off) + ;; Form the sum of two offset values, and check that the result is ;; a valid `MemArg::Symbol` offset (i.e. is even and fits into i32). (decl pure memarg_symbol_offset_sum (i64 i64) i32) @@ -1736,6 +1741,16 @@ (_ Unit (emit (MInst.LoadAddr dst mem)))) dst)) +;; Helper for emitting `MInst.Call` instructions. +(decl call_impl (WritableReg BoxCallInfo) SideEffectNoResult) +(rule (call_impl reg info) + (SideEffectNoResult.Inst (MInst.Call reg info))) + +;; Helper for emitting `MInst.CallInd` instructions. +(decl call_ind_impl (WritableReg BoxCallIndInfo) SideEffectNoResult) +(rule (call_ind_impl reg info) + (SideEffectNoResult.Inst (MInst.CallInd reg info))) + ;; Helper for emitting `MInst.Jump` instructions. (decl jump_impl (MachLabel) SideEffectNoResult) (rule (jump_impl target) @@ -1890,6 +1905,81 @@ (emit (MInst.Load64 dst addr))) +;; Helpers for accessing argument / return value slots ;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl emit_side_effect (SideEffectNoResult) Unit) +(rule (emit_side_effect (SideEffectNoResult.Inst inst)) (emit inst)) + +(decl emit_arg_store (Type Reg MemArg) Unit) +(rule (emit_arg_store $I8 reg mem) (emit_side_effect (store8 reg mem))) +(rule (emit_arg_store $I16 reg mem) (emit_side_effect (store16 reg mem))) +(rule (emit_arg_store $I32 reg mem) (emit_side_effect (store32 reg mem))) +(rule (emit_arg_store $I64 reg mem) (emit_side_effect (store64 reg mem))) +(rule (emit_arg_store $R64 reg mem) (emit_side_effect (store64 reg mem))) +(rule (emit_arg_store $F32 reg mem) (emit_side_effect (fpu_store32 reg mem))) +(rule (emit_arg_store $F64 reg mem) (emit_side_effect (fpu_store64 reg mem))) + +(decl emit_arg_load (Type MemArg) Reg) +(rule (emit_arg_load $I8 mem) (zext32_mem $I8 mem)) +(rule (emit_arg_load $I16 mem) (zext32_mem $I16 mem)) +(rule (emit_arg_load $I32 mem) (load32 mem)) +(rule (emit_arg_load $I64 mem) (load64 mem)) +(rule (emit_arg_load $R64 mem) (load64 mem)) +(rule (emit_arg_load $F32 mem) (fpu_load64 mem)) +(rule (emit_arg_load $F64 mem) (fpu_load64 mem)) + +;; Copy a single argument/return value to its slots. +(decl copy_to_arg (i64 ABIArg Value) Unit) +(rule (copy_to_arg base (abi_arg_only_slot slot) val) + (copy_val_to_arg_slot base slot val)) + +;; Copy a single argument/return value from its slots. +(decl copy_from_arg (i64 ABIArg) ValueRegs) +(rule (copy_from_arg base (abi_arg_only_slot slot)) + (value_reg (copy_reg_from_arg_slot base slot))) + +;; Copy one component of an argument/return value to its slot. +(decl copy_val_to_arg_slot (i64 ABIArgSlot Value) Unit) +(rule (copy_val_to_arg_slot _ (ABIArgSlot.Reg reg ty (ArgumentExtension.None)) val) + (emit_mov ty (real_reg_to_writable_reg reg) val)) +(rule (copy_val_to_arg_slot _ (ABIArgSlot.Reg reg _ (ArgumentExtension.Uext)) val) + (emit_put_in_reg_zext64 (real_reg_to_writable_reg reg) val)) +(rule (copy_val_to_arg_slot _ (ABIArgSlot.Reg reg _ (ArgumentExtension.Sext)) val) + (emit_put_in_reg_sext64 (real_reg_to_writable_reg reg) val)) +(rule (copy_val_to_arg_slot base (ABIArgSlot.Stack offset ty (ArgumentExtension.None)) val) + (emit_arg_store ty val (memarg_stack_off base offset))) +(rule (copy_val_to_arg_slot base (ABIArgSlot.Stack offset _ (ArgumentExtension.Uext)) val) + (emit_arg_store $I64 (put_in_reg_zext64 val) (memarg_stack_off base offset))) +(rule (copy_val_to_arg_slot base (ABIArgSlot.Stack offset _ (ArgumentExtension.Sext)) val) + (emit_arg_store $I64 (put_in_reg_sext64 val) (memarg_stack_off base offset))) + +;; Copy one component of an argument/return value to its slot, where the +;; value is already extended and present in a register. +(decl copy_reg_to_arg_slot (i64 ABIArgSlot Reg) Unit) +(rule (copy_reg_to_arg_slot _ (ABIArgSlot.Reg reg ty ext) src) + (emit_mov (abi_ext_ty ext ty) (real_reg_to_writable_reg reg) src)) +(rule (copy_reg_to_arg_slot base (ABIArgSlot.Stack offset ty ext) src) + (emit_arg_store (abi_ext_ty ext ty) src (memarg_stack_off base offset))) + +;; Copy one component of an argument/return value from its slot. +(decl copy_reg_from_arg_slot (i64 ABIArgSlot) Reg) +(rule (copy_reg_from_arg_slot _ (ABIArgSlot.Reg reg ty ext)) + (copy_reg (abi_ext_ty ext ty) (real_reg_to_reg reg))) +(rule (copy_reg_from_arg_slot base (ABIArgSlot.Stack offset ty ext)) + (emit_arg_load (abi_ext_ty ext ty) (memarg_stack_off base offset))) + +;; Helper to compute the type of an implicitly extended argument/return value. +(decl abi_ext_ty (ArgumentExtension Type) Type) +(rule (abi_ext_ty (ArgumentExtension.None) ty) ty) +(rule (abi_ext_ty (ArgumentExtension.Uext) _) $I64) +(rule (abi_ext_ty (ArgumentExtension.Sext) _) $I64) + +;; Copy a return value to a set of registers. +(decl copy_to_regs (WritableValueRegs Value) Unit) +(rule (copy_to_regs (only_writable_reg reg) val @ (value_type ty)) + (emit_mov ty reg val)) + + ;; Helpers for generating immediate values ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Move immediate value into destination register. (Non-SSA form.) @@ -2512,6 +2602,32 @@ (casloop_emit ib ty flags aligned_addr val))) +;; Helpers for generating `call` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl abi_sig (SigRef) ABISig) +(extern constructor abi_sig abi_sig) + +(decl abi_call_info (ABISig ExternalName Opcode) BoxCallInfo) +(extern constructor abi_call_info abi_call_info) + +(decl abi_call_ind_info (ABISig Reg Opcode) BoxCallIndInfo) +(extern constructor abi_call_ind_info abi_call_ind_info) + +(decl writable_link_reg () WritableReg) +(rule (writable_link_reg) (writable_gpr 14)) + +(decl abi_call (ABISig ExternalName Opcode) SideEffectNoResult) +(rule (abi_call abi name opcode) + (call_impl (writable_link_reg) (abi_call_info abi name opcode))) + +(decl abi_call_ind (ABISig Reg Opcode) SideEffectNoResult) +(rule (abi_call_ind abi target opcode) + (call_ind_impl (writable_link_reg) (abi_call_ind_info abi target opcode))) + +(decl abi_accumulate_outgoing_args_size (ABISig) Unit) +(extern constructor abi_accumulate_outgoing_args_size abi_accumulate_outgoing_args_size) + + ;; Helpers for generating `clz` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Count leading zeroes. For a zero input, return the specified value. diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle index e0b7edf029..c5a60fa086 100644 --- a/cranelift/codegen/src/isa/s390x/lower.isle +++ b/cranelift/codegen/src/isa/s390x/lower.isle @@ -2215,3 +2215,71 @@ (side_effect (trap_if_impl (mask_as_cond 3) trap_code))) +;;;; Rules for `return` and `fallthrough_return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (return args)) + (lower_return (range 0 (value_slice_len args)) args)) + +(rule (lower (fallthrough_return args)) + (lower_return (range 0 (value_slice_len args)) args)) + +(decl lower_return (Range ValueSlice) InstOutput) +(rule (lower_return (range_empty) _) (output_none)) +(rule (lower_return (range_unwrap head tail) args) + (let ((_ Unit (copy_to_regs (retval head) (value_slice_get args head)))) + (lower_return tail args))) + + +;;;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Direct call to an in-range function. +(rule (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args)) + (let ((abi ABISig (abi_sig sig_ref)) + (_1 Unit (abi_accumulate_outgoing_args_size abi)) + (_2 InstOutput (lower_call_args abi (range 0 (abi_num_args abi)) args)) + (_3 InstOutput (side_effect (abi_call abi name (Opcode.Call))))) + (lower_call_rets abi (range 0 (abi_num_rets abi)) (output_builder_new)))) + +;; Direct call to an out-of-range function (implicitly via pointer). +(rule (lower (call (func_ref_data sig_ref name _) args)) + (let ((abi ABISig (abi_sig sig_ref)) + (_1 Unit (abi_accumulate_outgoing_args_size abi)) + (_2 InstOutput (lower_call_args abi (range 0 (abi_num_args abi)) args)) + (target Reg (load_ext_name_far name 0)) + (_3 InstOutput (side_effect (abi_call_ind abi target (Opcode.Call))))) + (lower_call_rets abi (range 0 (abi_num_rets abi)) (output_builder_new)))) + +;; Indirect call. +(rule (lower (call_indirect sig_ref ptr args)) + (let ((abi ABISig (abi_sig sig_ref)) + (target Reg (put_in_reg ptr)) + (_1 Unit (abi_accumulate_outgoing_args_size abi)) + (_2 InstOutput (lower_call_args abi (range 0 (abi_num_args abi)) args)) + (_3 InstOutput (side_effect (abi_call_ind abi target (Opcode.CallIndirect))))) + (lower_call_rets abi (range 0 (abi_num_rets abi)) (output_builder_new)))) + +;; Lower function arguments by loading them into registers / stack slots. +(decl lower_call_args (ABISig Range ValueSlice) InstOutput) +(rule (lower_call_args abi (range_empty) _) (lower_call_ret_arg abi)) +(rule (lower_call_args abi (range_unwrap head tail) args) + (let ((idx usize (abi_copy_to_arg_order abi head)) + (_ Unit (copy_to_arg 0 (abi_get_arg abi idx) + (value_slice_get args idx)))) + (lower_call_args abi tail args))) + +;; Lower the implicit return-area pointer argument, if present. +(decl lower_call_ret_arg (ABISig) InstOutput) +(rule (lower_call_ret_arg (abi_no_ret_arg)) (output_none)) +(rule (lower_call_ret_arg abi @ (abi_ret_arg (abi_arg_only_slot slot))) + (let ((ret_arg Reg (load_addr (memarg_stack_off (abi_stack_arg_space abi) 0))) + (_ Unit (copy_reg_to_arg_slot 0 slot ret_arg))) + (output_none))) + +;; Lower function return values by collecting them from registers / stack slots. +(decl lower_call_rets (ABISig Range InstOutputBuilder) InstOutput) +(rule (lower_call_rets abi (range_empty) builder) (output_builder_finish builder)) +(rule (lower_call_rets abi (range_unwrap head tail) builder) + (let ((ret ValueRegs (copy_from_arg (abi_stack_arg_space abi) (abi_get_ret abi head))) + (_ Unit (output_builder_push builder ret))) + (lower_call_rets abi tail builder))) + diff --git a/cranelift/codegen/src/isa/s390x/lower.rs b/cranelift/codegen/src/isa/s390x/lower.rs index 39c7531392..fd33f8e1f3 100644 --- a/cranelift/codegen/src/isa/s390x/lower.rs +++ b/cranelift/codegen/src/isa/s390x/lower.rs @@ -28,13 +28,6 @@ fn sign_extend_to_u64(value: u64, from_bits: u8) -> u64 { } } -/// Lower an instruction input to a reg. -fn put_input_in_reg>(ctx: &mut C, input: InsnInput) -> Reg { - ctx.put_input_in_regs(input.insn, input.input) - .only_reg() - .unwrap() -} - //============================================================================= // Lowering: comparisons @@ -69,9 +62,6 @@ fn lower_insn_to_regs>( isa_flags: &s390x_settings::Flags, ) -> CodegenResult<()> { let op = ctx.data(insn).opcode(); - let inputs: SmallVec<[InsnInput; 4]> = (0..ctx.num_inputs(insn)) - .map(|i| InsnInput { insn, input: i }) - .collect(); let outputs: SmallVec<[InsnOutput; 2]> = (0..ctx.num_outputs(insn)) .map(|i| InsnOutput { insn, output: i }) .collect(); @@ -190,6 +180,10 @@ fn lower_insn_to_regs>( | Opcode::ResumableTrapnz | Opcode::Trapif | Opcode::Debugtrap + | Opcode::Call + | Opcode::CallIndirect + | Opcode::FallthroughReturn + | Opcode::Return | Opcode::StackAddr | Opcode::FuncAddr | Opcode::SymbolValue => implemented_in_isle(), @@ -227,56 +221,6 @@ fn lower_insn_to_regs>( unimplemented!("Pinned register support not implemented!"); } - Opcode::Call | Opcode::CallIndirect => { - let caller_conv = ctx.abi().call_conv(); - let (mut abi, inputs) = match op { - Opcode::Call => { - let (extname, dist) = ctx.call_target(insn).unwrap(); - let extname = extname.clone(); - let sig = ctx.call_sig(insn).unwrap(); - assert!(inputs.len() == sig.params.len()); - assert!(outputs.len() == sig.returns.len()); - ( - S390xABICaller::from_func(sig, &extname, dist, caller_conv, flags)?, - &inputs[..], - ) - } - Opcode::CallIndirect => { - let ptr = put_input_in_reg(ctx, inputs[0]); - let sig = ctx.call_sig(insn).unwrap(); - assert!(inputs.len() - 1 == sig.params.len()); - assert!(outputs.len() == sig.returns.len()); - ( - S390xABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?, - &inputs[1..], - ) - } - _ => unreachable!(), - }; - - assert!(inputs.len() == abi.num_args()); - for (i, input) in inputs.iter().enumerate() { - let arg_reg = put_input_in_reg(ctx, *input); - abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg)); - } - abi.emit_call(ctx); - for (i, output) in outputs.iter().enumerate() { - let retval_reg = get_output_reg(ctx, *output).only_reg().unwrap(); - abi.emit_copy_retval_to_regs(ctx, i, ValueRegs::one(retval_reg)); - } - abi.accumulate_outgoing_args_size(ctx); - } - - Opcode::FallthroughReturn | Opcode::Return => { - for (i, input) in inputs.iter().enumerate() { - let reg = put_input_in_reg(ctx, *input); - let retval_reg = ctx.retval(i).only_reg().unwrap(); - let ty = ctx.input_ty(insn, i); - ctx.emit(Inst::gen_move(retval_reg, reg, ty)); - } - // N.B.: the Ret itself is generated by the ABI. - } - Opcode::RawBitcast | Opcode::Splat | Opcode::Swizzle diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index ded692c834..c7132c6ed0 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -6,7 +6,7 @@ pub mod generated_code; // Types that the generated ISLE code uses via `use super::*`. use super::{ CallIndInfo, CallInfo, Cond, Inst as MInst, MachLabel, MemArg, MemFlags, Opcode, Reg, - UImm16Shifted, UImm32Shifted, + S390xMachineDeps, UImm16Shifted, UImm32Shifted, }; use crate::isa::s390x::settings::Flags as IsaFlags; use crate::machinst::isle::*; @@ -69,6 +69,40 @@ where { isle_prelude_methods!(); + fn abi_sig(&mut self, sig_ref: SigRef) -> ABISig { + let sig = &self.lower_ctx.dfg().signatures[sig_ref]; + ABISig::from_func_sig::(sig, self.flags).unwrap() + } + + fn abi_accumulate_outgoing_args_size(&mut self, abi: &ABISig) -> Unit { + let off = abi.stack_arg_space() + abi.stack_ret_space(); + self.lower_ctx + .abi() + .accumulate_outgoing_args_size(off as u32); + } + + fn abi_call_info(&mut self, abi: &ABISig, name: ExternalName, opcode: &Opcode) -> BoxCallInfo { + let (uses, defs, clobbers) = abi.call_uses_defs_clobbers::(); + Box::new(CallInfo { + dest: name.clone(), + uses, + defs, + clobbers, + opcode: *opcode, + }) + } + + fn abi_call_ind_info(&mut self, abi: &ABISig, target: Reg, opcode: &Opcode) -> BoxCallIndInfo { + let (uses, defs, clobbers) = abi.call_uses_defs_clobbers::(); + Box::new(CallIndInfo { + rn: target, + uses, + defs, + clobbers, + opcode: *opcode, + }) + } + #[inline] fn allow_div_traps(&mut self, _: Type) -> Option<()> { if !self.flags.avoid_div_traps() { @@ -432,6 +466,11 @@ where MemArg::reg_plus_off(reg, off, flags) } + #[inline] + fn memarg_stack_off(&mut self, base: i64, off: i64) -> MemArg { + MemArg::reg_plus_off(super::stack_reg(), base + off, MemFlags::trusted()) + } + #[inline] fn memarg_symbol(&mut self, name: ExternalName, offset: i32, flags: MemFlags) -> MemArg { MemArg::Symbol { diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 91e277e016..b2a11e2d76 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -520,7 +520,7 @@ pub trait ABIMachineSpec { /// ABI information shared between body (callee) and caller. #[derive(Clone)] -struct ABISig { +pub struct ABISig { /// Argument locations (regs or stack slots). Stack offsets are relative to /// SP on entry to function. args: Vec, @@ -533,15 +533,21 @@ struct ABISig { stack_ret_space: i64, /// Index in `args` of the stack-return-value-area argument. stack_ret_arg: Option, + /// Specific order for copying into arguments at callsites. We must be + /// careful to copy into StructArgs first, because we need to be able + /// to invoke memcpy() before we've loaded other arg regs (see above). + copy_to_arg_order: SmallVec<[usize; 8]>, /// Calling convention used. call_conv: isa::CallConv, } impl ABISig { - fn from_func_sig( + pub fn from_func_sig( sig: &ir::Signature, flags: &settings::Flags, ) -> CodegenResult { + let sig = ensure_struct_return_ptr_is_returned(sig); + // Compute args and retvals from signature. Handle retvals first, // because we may need to add a return-area arg to the args. let (rets, stack_ret_space, _) = M::compute_arg_locs( @@ -560,14 +566,30 @@ impl ABISig { need_stack_return_area, )?; + let mut copy_to_arg_order = SmallVec::new(); + for (i, arg) in args.iter().enumerate() { + // Struct args. + if arg.is_struct_arg() { + copy_to_arg_order.push(i); + } + } + for (i, arg) in args.iter().enumerate() { + // Non-struct args. Skip an appended return-area arg for multivalue + // returns, if any. + if !arg.is_struct_arg() && i < sig.params.len() { + copy_to_arg_order.push(i); + } + } + log::trace!( - "ABISig: sig {:?} => args = {:?} rets = {:?} arg stack = {} ret stack = {} stack_ret_arg = {:?}", + "ABISig: sig {:?} => args = {:?} rets = {:?} arg stack = {} ret stack = {} stack_ret_arg = {:?} copy_to_arg_order = {:?}", sig, args, rets, stack_arg_space, stack_ret_space, - stack_ret_arg + stack_ret_arg, + copy_to_arg_order, ); Ok(ABISig { @@ -576,9 +598,99 @@ impl ABISig { stack_arg_space, stack_ret_space, stack_ret_arg, + copy_to_arg_order, call_conv: sig.call_conv, }) } + + /// Return all uses (i.e, function args), defs (i.e., return values + /// and caller-saved registers), and clobbers for the callsite. + pub fn call_uses_defs_clobbers( + &self, + ) -> (SmallVec<[Reg; 8]>, SmallVec<[Writable; 8]>, PRegSet) { + // Compute uses: all arg regs. + let mut uses = smallvec![]; + for arg in &self.args { + if let &ABIArg::Slots { ref slots, .. } = arg { + for slot in slots { + match slot { + &ABIArgSlot::Reg { reg, .. } => { + uses.push(Reg::from(reg)); + } + _ => {} + } + } + } + } + + // Get clobbers: all caller-saves. These may include return value + // regs, which we will remove from the clobber set below. + let mut clobbers = M::get_regs_clobbered_by_call(self.call_conv); + + // Compute defs: all retval regs, and all caller-save (clobbered) regs. + let mut defs = smallvec![]; + for ret in &self.rets { + if let &ABIArg::Slots { ref slots, .. } = ret { + for slot in slots { + match slot { + &ABIArgSlot::Reg { reg, .. } => { + defs.push(Writable::from_reg(Reg::from(reg))); + clobbers.remove(PReg::from(reg)); + } + _ => {} + } + } + } + } + + (uses, defs, clobbers) + } + + /// Specific order for copying into arguments at callsites. + pub fn copy_to_arg_order(&self, idx: usize) -> usize { + self.copy_to_arg_order[idx] + } + + /// Get the number of arguments expected. + pub fn num_args(&self) -> usize { + if self.stack_ret_arg.is_some() { + self.args.len() - 1 + } else { + self.args.len() + } + } + + /// Get information specifying how to pass one argument. + pub fn get_arg(&self, idx: usize) -> ABIArg { + self.args[idx].clone() + } + + /// Get total stack space required for arguments. + pub fn stack_arg_space(&self) -> i64 { + self.stack_arg_space + } + + /// Get the number of return values expected. + pub fn num_rets(&self) -> usize { + self.rets.len() + } + + /// Get information specifying how to pass one return value. + pub fn get_ret(&self, idx: usize) -> ABIArg { + self.rets[idx].clone() + } + + /// Get total stack space required for return values. + pub fn stack_ret_space(&self) -> i64 { + self.stack_ret_space + } + + /// Get information specifying how to pass the implicit pointer + /// to the return-value area on the stack, if required. + pub fn get_ret_arg(&self) -> Option { + let ret_arg = self.stack_ret_arg?; + Some(self.args[ret_arg].clone()) + } } /// ABI object for a function body. @@ -1357,47 +1469,6 @@ impl ABICallee for ABICalleeImpl { } } -fn abisig_to_uses_defs_clobbers( - sig: &ABISig, -) -> (SmallVec<[Reg; 8]>, SmallVec<[Writable; 8]>, PRegSet) { - // Compute uses: all arg regs. - let mut uses = smallvec![]; - for arg in &sig.args { - if let &ABIArg::Slots { ref slots, .. } = arg { - for slot in slots { - match slot { - &ABIArgSlot::Reg { reg, .. } => { - uses.push(Reg::from(reg)); - } - _ => {} - } - } - } - } - - // Get clobbers: all caller-saves. These may include return value - // regs, which we will remove from the clobber set below. - let mut clobbers = M::get_regs_clobbered_by_call(sig.call_conv); - - // Compute defs: all retval regs, and all caller-save (clobbered) regs. - let mut defs = smallvec![]; - for ret in &sig.rets { - if let &ABIArg::Slots { ref slots, .. } = ret { - for slot in slots { - match slot { - &ABIArgSlot::Reg { reg, .. } => { - defs.push(Writable::from_reg(Reg::from(reg))); - clobbers.remove(PReg::from(reg)); - } - _ => {} - } - } - } - } - - (uses, defs, clobbers) -} - /// ABI object for a callsite. pub struct ABICallerImpl { /// CLIF-level signature, possibly normalized. @@ -1442,7 +1513,7 @@ impl ABICallerImpl { ) -> CodegenResult> { let ir_sig = ensure_struct_return_ptr_is_returned(sig); let sig = ABISig::from_func_sig::(&ir_sig, flags)?; - let (uses, defs, clobbers) = abisig_to_uses_defs_clobbers::(&sig); + let (uses, defs, clobbers) = sig.call_uses_defs_clobbers::(); Ok(ABICallerImpl { ir_sig, sig, @@ -1468,7 +1539,7 @@ impl ABICallerImpl { ) -> CodegenResult> { let ir_sig = ensure_struct_return_ptr_is_returned(sig); let sig = ABISig::from_func_sig::(&ir_sig, flags)?; - let (uses, defs, clobbers) = abisig_to_uses_defs_clobbers::(&sig); + let (uses, defs, clobbers) = sig.call_uses_defs_clobbers::(); Ok(ABICallerImpl { ir_sig, sig, @@ -1629,21 +1700,7 @@ impl ABICaller for ABICallerImpl { } fn get_copy_to_arg_order(&self) -> SmallVec<[usize; 8]> { - let mut ret = SmallVec::new(); - for (i, arg) in self.sig.args.iter().enumerate() { - // Struct args. - if arg.is_struct_arg() { - ret.push(i); - } - } - for (i, arg) in self.sig.args.iter().enumerate() { - // Non-struct args. Skip an appended return-area arg for multivalue - // returns, if any. - if !arg.is_struct_arg() && i < self.ir_sig.params.len() { - ret.push(i); - } - } - ret + self.sig.copy_to_arg_order.clone() } fn emit_copy_retval_to_regs>( diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index abdfbe0509..3ae82da927 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -1,14 +1,14 @@ use crate::ir::{types, Inst, Value, ValueList}; -use crate::machinst::{get_output_reg, InsnOutput, LowerCtx, Reg, Writable}; +use crate::machinst::{get_output_reg, InsnOutput, LowerCtx}; use alloc::boxed::Box; use alloc::vec::Vec; use smallvec::SmallVec; use std::cell::Cell; pub use super::MachLabel; -pub use crate::ir::{ExternalName, FuncRef, GlobalValue, SigRef}; +pub use crate::ir::{ArgumentExtension, ExternalName, FuncRef, GlobalValue, SigRef}; pub use crate::isa::unwind::UnwindInst; -pub use crate::machinst::RelocDistance; +pub use crate::machinst::{ABIArg, ABIArgSlot, ABISig, RealReg, Reg, RelocDistance, Writable}; pub type Unit = (); pub type ValueSlice = (ValueList, usize); @@ -17,10 +17,12 @@ pub type ValueArray3 = [Value; 3]; pub type WritableReg = Writable; pub type VecReg = Vec; pub type ValueRegs = crate::machinst::ValueRegs; +pub type WritableValueRegs = crate::machinst::ValueRegs; pub type InstOutput = SmallVec<[ValueRegs; 2]>; pub type InstOutputBuilder = Cell; pub type VecMachLabel = Vec; pub type BoxExternalName = Box; +pub type Range = (usize, usize); /// Helper macro to define methods in `prelude.isle` within `impl Context for /// ...` for each backend. These methods are shared amongst all backends. @@ -572,6 +574,95 @@ macro_rules! isle_prelude_methods { let data = VCodeConstantData::U64(value.to_le_bytes()); self.lower_ctx.use_constant(data) } + + fn range(&mut self, start: usize, end: usize) -> Range { + (start, end) + } + + fn range_empty(&mut self, r: Range) -> Option<()> { + if r.0 >= r.1 { + Some(()) + } else { + None + } + } + + fn range_unwrap(&mut self, r: Range) -> Option<(usize, Range)> { + if r.0 < r.1 { + Some((r.0, (r.0 + 1, r.1))) + } else { + None + } + } + + fn retval(&mut self, i: usize) -> WritableValueRegs { + self.lower_ctx.retval(i) + } + + fn only_writable_reg(&mut self, regs: WritableValueRegs) -> Option { + regs.only_reg() + } + + fn abi_copy_to_arg_order(&mut self, abi: &ABISig, idx: usize) -> usize { + abi.copy_to_arg_order(idx) + } + + fn abi_num_args(&mut self, abi: &ABISig) -> usize { + abi.num_args() + } + + fn abi_get_arg(&mut self, abi: &ABISig, idx: usize) -> ABIArg { + abi.get_arg(idx) + } + + fn abi_num_rets(&mut self, abi: &ABISig) -> usize { + abi.num_rets() + } + + fn abi_get_ret(&mut self, abi: &ABISig, idx: usize) -> ABIArg { + abi.get_ret(idx) + } + + fn abi_ret_arg(&mut self, abi: &ABISig) -> Option { + abi.get_ret_arg() + } + + fn abi_no_ret_arg(&mut self, abi: &ABISig) -> Option<()> { + if let Some(_) = abi.get_ret_arg() { + None + } else { + Some(()) + } + } + + fn abi_stack_arg_space(&mut self, abi: &ABISig) -> i64 { + abi.stack_arg_space() + } + + fn abi_stack_ret_space(&mut self, abi: &ABISig) -> i64 { + abi.stack_ret_space() + } + + fn abi_arg_only_slot(&mut self, arg: &ABIArg) -> Option { + match arg { + &ABIArg::Slots { ref slots, .. } => { + if slots.len() == 1 { + Some(slots[0]) + } else { + None + } + } + _ => None, + } + } + + fn real_reg_to_reg(&mut self, reg: RealReg) -> Reg { + Reg::from(reg) + } + + fn real_reg_to_writable_reg(&mut self, reg: RealReg) -> WritableReg { + Writable::from_reg(Reg::from(reg)) + } }; } diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index e0729dc3cb..13ba4b4528 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -37,6 +37,7 @@ (type ValueList (primitive ValueList)) (type ValueRegs (primitive ValueRegs)) +(type WritableValueRegs (primitive WritableValueRegs)) ;; Instruction lowering result: a vector of `ValueRegs`. (type InstOutput (primitive InstOutput)) @@ -635,6 +636,117 @@ (decl reloc_distance_near () RelocDistance) (extern extractor reloc_distance_near reloc_distance_near) +;;;; Helpers for tail recursion loops ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; A range of integers to loop through. +(type Range (primitive Range)) + +;; Create a new range from `start` through `end` (exclusive). +(decl range (usize usize) Range) +(extern constructor range range) + +;; Extractor to test whether a range is empty. +(decl range_empty () Range) +(extern extractor range_empty range_empty) + +;; Extractor to return the first value in the range, and a sub-range +;; containing the remaining values. +(decl range_unwrap (usize Range) Range) +(extern extractor range_unwrap range_unwrap) + +;;;; Helpers for generating returns ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; The (writable) register(s) that will contain the n'th return value. +(decl retval (usize) WritableValueRegs) +(extern constructor retval retval) + +;; Extractor to check for the special case that a `WritableValueRegs` +;; contains only a single register. +(decl only_writable_reg (WritableReg) WritableValueRegs) +(extern extractor only_writable_reg only_writable_reg) + +;;;; Helpers for generating calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Type to hold information about a function call signature. +(type ABISig extern (enum)) + +;; Information how to pass one argument or return value. +(type ABIArg extern (enum)) + +;; Information how to pass a single slot of one argument or return value. +(type ABIArgSlot extern + (enum + (Reg + (reg RealReg) + (ty Type) + (extension ArgumentExtension)) + (Stack + (offset i64) + (ty Type) + (extension ArgumentExtension)) +)) + +;; Physical register that may hold an argument or return value. +(type RealReg (primitive RealReg)) + +;; Instruction on whether and how to extend an argument value. +(type ArgumentExtension extern + (enum + (None) + (Uext) + (Sext) +)) + +;; Specific order for copying into arguments at callsites. +(decl abi_copy_to_arg_order (ABISig usize) usize) +(extern constructor abi_copy_to_arg_order abi_copy_to_arg_order) + +;; Get the number of arguments expected. +(decl abi_num_args (ABISig) usize) +(extern constructor abi_num_args abi_num_args) + +;; Get information specifying how to pass one argument. +(decl abi_get_arg (ABISig usize) ABIArg) +(extern constructor abi_get_arg abi_get_arg) + +;; Get the number of return values expected. +(decl abi_num_rets (ABISig) usize) +(extern constructor abi_num_rets abi_num_rets) + +;; Get information specifying how to pass one return value. +(decl abi_get_ret (ABISig usize) ABIArg) +(extern constructor abi_get_ret abi_get_ret) + +;; Get information specifying how to pass the implicit pointer +;; to the return-value area on the stack, if required. +(decl abi_ret_arg (ABIArg) ABISig) +(extern extractor abi_ret_arg abi_ret_arg) + +;; Succeeds if no implicit return-value area pointer is required. +(decl abi_no_ret_arg () ABISig) +(extern extractor abi_no_ret_arg abi_no_ret_arg) + +;; Size of the argument area. +(decl abi_stack_arg_space (ABISig) i64) +(extern constructor abi_stack_arg_space abi_stack_arg_space) + +;; Size of the return-value area. +(decl abi_stack_ret_space (ABISig) i64) +(extern constructor abi_stack_ret_space abi_stack_ret_space) + +;; Extractor to detect the special case where an argument or +;; return value only requires a single slot to be passed. +(decl abi_arg_only_slot (ABIArgSlot) ABIArg) +(extern extractor abi_arg_only_slot abi_arg_only_slot) + +;; Convert a real register number into a virtual register. +(decl real_reg_to_reg (RealReg) Reg) +(extern constructor real_reg_to_reg real_reg_to_reg) + +;; Convert a real register number into a writable virtual register. +(decl real_reg_to_writable_reg (RealReg) WritableReg) +(extern constructor real_reg_to_writable_reg real_reg_to_writable_reg) + ;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (convert Inst Value def_inst) diff --git a/cranelift/filetests/filetests/isa/s390x/reftypes.clif b/cranelift/filetests/filetests/isa/s390x/reftypes.clif index 8314f33a0a..f86a9c1eef 100644 --- a/cranelift/filetests/filetests/isa/s390x/reftypes.clif +++ b/cranelift/filetests/filetests/isa/s390x/reftypes.clif @@ -71,13 +71,13 @@ block3(v7: r64, v8: r64): ; block0: ; stg %r3, 176(%r15) ; stg %r2, 168(%r15) -; bras %r1, 12 ; data %f + 0 ; lg %r5, 0(%r1) -; basr %r14, %r5 +; bras %r1, 12 ; data %f + 0 ; lg %r4, 0(%r1) +; basr %r14, %r4 ; la %r3, 160(%r15) ; lg %r5, 168(%r15) ; stg %r5, 0(%r3) -; llcr %r4, %r2 -; chi %r4, 0 +; llcr %r3, %r2 +; chi %r3, 0 ; jgnlh label1 ; jg label3 ; block1: ; jg label2 @@ -92,8 +92,8 @@ block3(v7: r64, v8: r64): ; lg %r3, 176(%r15) ; jg label5 ; block5: -; la %r4, 160(%r15) -; lg %r4, 0(%r4) +; la %r5, 160(%r15) +; lg %r4, 0(%r5) ; lmg %r14, %r15, 296(%r15) ; br %r14