diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index e95b0dc081..69d59f20b1 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -1506,6 +1506,13 @@ r (OperandSize.Size32))) +;;;; Helpers for Emitting Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl gen_call (SigRef ExternalName RelocDistance ValueSlice) InstOutput) +(extern constructor gen_call gen_call) + +(decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) +(extern constructor gen_call_indirect gen_call_indirect) ;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 4359c77fce..5087e8c5e3 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -2922,3 +2922,11 @@ (rule (lower (has_type (and (fits_in_64 ty) (ty_int _)) (atomic_rmw flags op address input))) (x64_atomic_rmw_seq ty op (to_amode flags address (zero_offset)) input)) + +;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (call (func_ref_data sig_ref extname dist) inputs)) + (gen_call sig_ref extname dist inputs)) + +(rule (lower (call_indirect sig_ref val inputs)) + (gen_call_indirect sig_ref val inputs)) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index e6563538fb..5f1db2aab8 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -925,53 +925,12 @@ fn lower_insn_to_regs>( | Opcode::FuncAddr | Opcode::SymbolValue | Opcode::FallthroughReturn - | Opcode::Return => { + | Opcode::Return + | Opcode::Call + | Opcode::CallIndirect => { implemented_in_isle(ctx); } - Opcode::Call | Opcode::CallIndirect => { - let caller_conv = ctx.abi().call_conv(); - let (mut abi, inputs) = match op { - Opcode::Call => { - let (extname, dist) = ctx.call_target(insn).unwrap(); - let sig = ctx.call_sig(insn).unwrap(); - assert_eq!(inputs.len(), sig.params.len()); - assert_eq!(outputs.len(), sig.returns.len()); - ( - X64ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?, - &inputs[..], - ) - } - - Opcode::CallIndirect => { - let ptr = put_input_in_reg(ctx, inputs[0]); - let sig = ctx.call_sig(insn).unwrap(); - assert_eq!(inputs.len() - 1, sig.params.len()); - assert_eq!(outputs.len(), sig.returns.len()); - ( - X64ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?, - &inputs[1..], - ) - } - - _ => unreachable!(), - }; - - abi.emit_stack_pre_adjust(ctx); - assert_eq!(inputs.len(), abi.num_args()); - for i in abi.get_copy_to_arg_order() { - let input = inputs[i]; - let arg_regs = put_input_in_regs(ctx, input); - abi.emit_copy_regs_to_arg(ctx, i, arg_regs); - } - abi.emit_call(ctx); - for (i, output) in outputs.iter().enumerate() { - let retval_regs = get_output_reg(ctx, *output); - abi.emit_copy_retval_to_regs(ctx, i, retval_regs); - } - abi.emit_stack_post_adjust(ctx); - } - Opcode::Trapif | Opcode::Trapff => { let trap_code = ctx.data(insn).trap_code().unwrap(); diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 6a81d09425..8344ae0c61 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -6,7 +6,7 @@ use crate::{ ir::AtomicRmwOp, machinst::{InputSourceInst, Reg, Writable}, }; -use generated_code::MInst; +use generated_code::{Context, MInst}; // Types that the generated ISLE code uses via `use super::*`. use super::{is_int_or_ref_ty, is_mergeable_load, lower_to_amode}; @@ -21,13 +21,14 @@ use crate::{ settings::Flags, unwind::UnwindInst, x64::{ + abi::{X64ABICaller, X64ABIMachineSpec}, inst::{args::*, regs, CallInfo}, settings::Flags as IsaFlags, }, }, machinst::{ - isle::*, InsnInput, InsnOutput, LowerCtx, MachAtomicRmwOp, MachInst, VCodeConstant, - VCodeConstantData, + isle::*, valueregs, ABICaller, InsnInput, InsnOutput, LowerCtx, MachAtomicRmwOp, MachInst, + VCodeConstant, VCodeConstantData, }, }; use smallvec::SmallVec; @@ -59,7 +60,7 @@ where }) } -impl generated_code::Context for IsleContext<'_, C, Flags, IsaFlags, 6> +impl Context for IsleContext<'_, C, Flags, IsaFlags, 6> where C: LowerCtx, { @@ -591,6 +592,107 @@ where fn gen_move(&mut self, ty: Type, dst: WritableReg, src: Reg) -> MInst { MInst::gen_move(dst, src, ty) } + + fn gen_call( + &mut self, + sig_ref: SigRef, + extname: ExternalName, + dist: RelocDistance, + args @ (inputs, off): ValueSlice, + ) -> InstOutput { + let caller_conv = self.lower_ctx.abi().call_conv(); + let sig = &self.lower_ctx.dfg().signatures[sig_ref]; + let num_rets = sig.returns.len(); + let abi = ABISig::from_func_sig::(sig, self.flags).unwrap(); + let caller = X64ABICaller::from_func(sig, &extname, dist, caller_conv, self.flags).unwrap(); + + assert_eq!( + inputs.len(&self.lower_ctx.dfg().value_lists) - off, + sig.params.len() + ); + + self.gen_call_common(abi, num_rets, caller, args) + } + + fn gen_call_indirect( + &mut self, + sig_ref: SigRef, + val: Value, + args @ (inputs, off): ValueSlice, + ) -> InstOutput { + let caller_conv = self.lower_ctx.abi().call_conv(); + let ptr = self.put_in_reg(val); + let sig = &self.lower_ctx.dfg().signatures[sig_ref]; + let num_rets = sig.returns.len(); + let abi = ABISig::from_func_sig::(sig, self.flags).unwrap(); + let caller = + X64ABICaller::from_ptr(sig, ptr, Opcode::CallIndirect, caller_conv, self.flags) + .unwrap(); + + assert_eq!( + inputs.len(&self.lower_ctx.dfg().value_lists) - off, + sig.params.len() + ); + + self.gen_call_common(abi, num_rets, caller, args) + } +} + +impl IsleContext<'_, C, Flags, IsaFlags, 6> +where + C: LowerCtx, +{ + fn abi_arg_slot_regs(&mut self, arg: &ABIArg) -> Option { + match arg { + &ABIArg::Slots { ref slots, .. } => match slots.len() { + 1 => { + let a = self.temp_writable_reg(slots[0].get_type()); + Some(WritableValueRegs::one(a)) + } + 2 => { + let a = self.temp_writable_reg(slots[0].get_type()); + let b = self.temp_writable_reg(slots[1].get_type()); + Some(WritableValueRegs::two(a, b)) + } + _ => panic!("Expected to see one or two slots only from {:?}", arg), + }, + _ => None, + } + } + + fn gen_call_common( + &mut self, + abi: ABISig, + num_rets: usize, + mut caller: X64ABICaller, + (inputs, off): ValueSlice, + ) -> InstOutput { + caller.emit_stack_pre_adjust(self.lower_ctx); + + assert_eq!( + inputs.len(&self.lower_ctx.dfg().value_lists) - off, + abi.num_args() + ); + for i in caller.get_copy_to_arg_order() { + let input = inputs + .get(off + i, &self.lower_ctx.dfg().value_lists) + .unwrap(); + let arg_regs = self.lower_ctx.put_value_in_regs(input); + caller.emit_copy_regs_to_arg(self.lower_ctx, i, arg_regs); + } + caller.emit_call(self.lower_ctx); + + let mut outputs = InstOutput::new(); + for i in 0..num_rets { + let ret = abi.get_ret(i); + let retval_regs = self.abi_arg_slot_regs(&ret).unwrap(); + caller.emit_copy_retval_to_regs(self.lower_ctx, i, retval_regs.clone()); + outputs.push(valueregs::non_writable_value_regs(retval_regs)); + } + caller.emit_stack_post_adjust(self.lower_ctx); + + outputs + } } // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 3d110e3a92..9336113022 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -165,6 +165,16 @@ pub enum ABIArgSlot { }, } +impl ABIArgSlot { + /// The type of the value that will be stored in this slot. + pub fn get_type(&self) -> ir::Type { + match self { + ABIArgSlot::Reg { ty, .. } => *ty, + ABIArgSlot::Stack { ty, .. } => *ty, + } + } +} + /// An ABIArg is composed of one or more parts. This allows for a CLIF-level /// Value to be passed with its parts in more than one location at the ABI /// level. For example, a 128-bit integer may be passed in two 64-bit registers, diff --git a/cranelift/filetests/filetests/isa/x64/call-conv.clif b/cranelift/filetests/filetests/isa/x64/call-conv.clif index 4a14f53315..8f83231888 100644 --- a/cranelift/filetests/filetests/isa/x64/call-conv.clif +++ b/cranelift/filetests/filetests/isa/x64/call-conv.clif @@ -292,11 +292,11 @@ block0: ; pushq %rbp ; movq %rsp, %rbp ; block0: -; movl $1, %edx +; movl $1, %eax ; subq %rsp, $16, %rsp ; virtual_sp_offset_adjust 16 ; lea 0(%rsp), %rdi -; call *%rdx +; call *%rax ; movq 0(%rsp), %rdx ; addq %rsp, $16, %rsp ; virtual_sp_offset_adjust -16 @@ -318,8 +318,8 @@ block0: ; movq %r13, 0(%rsp) ; block0: ; movq %rdi, %r13 -; movl $1, %r9d -; call *%r9 +; movl $1, %edx +; call *%rdx ; movq %r13, %rdi ; movl %edx, 0(%rdi) ; movq 0(%rsp), %r13 @@ -342,17 +342,17 @@ block0: ; movq %rbx, 0(%rsp) ; block0: ; movq %rdi, %rbx -; movl $1, %r11d +; movl $1, %r8d ; subq %rsp, $16, %rsp ; virtual_sp_offset_adjust 16 ; lea 0(%rsp), %rdi -; call *%r11 -; movq 0(%rsp), %rcx +; call *%r8 +; movq 0(%rsp), %r11 ; addq %rsp, $16, %rsp ; virtual_sp_offset_adjust -16 ; movq %rbx, %rdi ; movq %rdx, 0(%rdi) -; movl %ecx, 8(%rdi) +; movl %r11d, 8(%rdi) ; movq 0(%rsp), %rbx ; addq %rsp, $16, %rsp ; movq %rbp, %rsp @@ -373,8 +373,8 @@ block0: ; movq %r12, 0(%rsp) ; block0: ; movq %rdi, %r12 -; movl $1, %edi -; call *%rdi +; movl $1, %r9d +; call *%r9 ; movq %r12, %rdi ; movq %rax, 0(%rdi) ; movl %edx, 8(%rdi) @@ -399,8 +399,8 @@ block0(v0: f32, v1: i64, v2: i32, v3: f32): ; movq %rbx, 0(%rsp) ; block0: ; movq %rdx, %rbx -; movl $1, %eax -; call *%rax +; movl $1, %r8d +; call *%r8 ; movq %rbx, %r10 ; movq %rax, 0(%r10) ; movl %edx, 8(%r10) diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index cc1c8e444e..838df0fee4 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -785,15 +785,15 @@ block0(v0: i128, v1: i128): ; subq %rsp, $16, %rsp ; virtual_sp_offset_adjust 16 ; lea 0(%rsp), %r8 -; load_ext_name %g+0, %rax -; call *%rax -; movq 0(%rsp), %r8 -; movq 8(%rsp), %r10 +; load_ext_name %g+0, %r9 +; call *%r9 +; movq 0(%rsp), %rcx +; movq 8(%rsp), %r8 ; addq %rsp, $16, %rsp ; virtual_sp_offset_adjust -16 ; movq %r12, %r9 -; movq %r8, 0(%r9) -; movq %r10, 8(%r9) +; movq %rcx, 0(%r9) +; movq %r8, 8(%r9) ; movq 0(%rsp), %r12 ; addq %rsp, $16, %rsp ; movq %rbp, %rsp