x64: Migrate call and call_indirect to ISLE (#4542)

https://github.com/bytecodealliance/wasmtime/pull/4542

cranelift/codegen/src/isa/x64/inst.isle

@@ -1506,6 +1506,13 @@
                   r
                   (OperandSize.Size32)))
 
+;;;; Helpers for Emitting Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl gen_call (SigRef ExternalName RelocDistance ValueSlice) InstOutput)
+(extern constructor gen_call gen_call)
+
+(decl gen_call_indirect (SigRef Value ValueSlice) InstOutput)
+(extern constructor gen_call_indirect gen_call_indirect)
 
 ;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 

cranelift/codegen/src/isa/x64/lower.isle

@@ -2922,3 +2922,11 @@
 (rule (lower (has_type (and (fits_in_64 ty) (ty_int _))
                   (atomic_rmw flags op address input)))
       (x64_atomic_rmw_seq ty op (to_amode flags address (zero_offset)) input))
+
+;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (call (func_ref_data sig_ref extname dist) inputs))
+      (gen_call sig_ref extname dist inputs))
+
+(rule (lower (call_indirect sig_ref val inputs))
+      (gen_call_indirect sig_ref val inputs))

cranelift/codegen/src/isa/x64/lower.rs

@@ -925,53 +925,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         | Opcode::FuncAddr
         | Opcode::SymbolValue
         | Opcode::FallthroughReturn
-        | Opcode::Return => {
+        | Opcode::Return
+        | Opcode::Call
+        | Opcode::CallIndirect => {
             implemented_in_isle(ctx);
         }
 
-        Opcode::Call | Opcode::CallIndirect => {
-            let caller_conv = ctx.abi().call_conv();
-            let (mut abi, inputs) = match op {
-                Opcode::Call => {
-                    let (extname, dist) = ctx.call_target(insn).unwrap();
-                    let sig = ctx.call_sig(insn).unwrap();
-                    assert_eq!(inputs.len(), sig.params.len());
-                    assert_eq!(outputs.len(), sig.returns.len());
-                    (
-                        X64ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?,
-                        &inputs[..],
-                    )
-                }
-
-                Opcode::CallIndirect => {
-                    let ptr = put_input_in_reg(ctx, inputs[0]);
-                    let sig = ctx.call_sig(insn).unwrap();
-                    assert_eq!(inputs.len() - 1, sig.params.len());
-                    assert_eq!(outputs.len(), sig.returns.len());
-                    (
-                        X64ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?,
-                        &inputs[1..],
-                    )
-                }
-
-                _ => unreachable!(),
-            };
-
-            abi.emit_stack_pre_adjust(ctx);
-            assert_eq!(inputs.len(), abi.num_args());
-            for i in abi.get_copy_to_arg_order() {
-                let input = inputs[i];
-                let arg_regs = put_input_in_regs(ctx, input);
-                abi.emit_copy_regs_to_arg(ctx, i, arg_regs);
-            }
-            abi.emit_call(ctx);
-            for (i, output) in outputs.iter().enumerate() {
-                let retval_regs = get_output_reg(ctx, *output);
-                abi.emit_copy_retval_to_regs(ctx, i, retval_regs);
-            }
-            abi.emit_stack_post_adjust(ctx);
-        }
-
         Opcode::Trapif | Opcode::Trapff => {
             let trap_code = ctx.data(insn).trap_code().unwrap();
 

cranelift/codegen/src/isa/x64/lower/isle.rs

@@ -6,7 +6,7 @@ use crate::{
     ir::AtomicRmwOp,
     machinst::{InputSourceInst, Reg, Writable},
 };
-use generated_code::MInst;
+use generated_code::{Context, MInst};
 
 // Types that the generated ISLE code uses via `use super::*`.
 use super::{is_int_or_ref_ty, is_mergeable_load, lower_to_amode};
@@ -21,13 +21,14 @@ use crate::{
         settings::Flags,
         unwind::UnwindInst,
         x64::{
+            abi::{X64ABICaller, X64ABIMachineSpec},
             inst::{args::*, regs, CallInfo},
             settings::Flags as IsaFlags,
         },
     },
     machinst::{
-        isle::*, InsnInput, InsnOutput, LowerCtx, MachAtomicRmwOp, MachInst, VCodeConstant,
+        isle::*, valueregs, ABICaller, InsnInput, InsnOutput, LowerCtx, MachAtomicRmwOp, MachInst,
-        VCodeConstantData,
+        VCodeConstant, VCodeConstantData,
     },
 };
 use smallvec::SmallVec;
@@ -59,7 +60,7 @@ where
     })
 }
 
-impl<C> generated_code::Context for IsleContext<'_, C, Flags, IsaFlags, 6>
+impl<C> Context for IsleContext<'_, C, Flags, IsaFlags, 6>
 where
     C: LowerCtx<I = MInst>,
 {
@@ -591,6 +592,107 @@ where
     fn gen_move(&mut self, ty: Type, dst: WritableReg, src: Reg) -> MInst {
         MInst::gen_move(dst, src, ty)
     }
+
+    fn gen_call(
+        &mut self,
+        sig_ref: SigRef,
+        extname: ExternalName,
+        dist: RelocDistance,
+        args @ (inputs, off): ValueSlice,
+    ) -> InstOutput {
+        let caller_conv = self.lower_ctx.abi().call_conv();
+        let sig = &self.lower_ctx.dfg().signatures[sig_ref];
+        let num_rets = sig.returns.len();
+        let abi = ABISig::from_func_sig::<X64ABIMachineSpec>(sig, self.flags).unwrap();
+        let caller = X64ABICaller::from_func(sig, &extname, dist, caller_conv, self.flags).unwrap();
+
+        assert_eq!(
+            inputs.len(&self.lower_ctx.dfg().value_lists) - off,
+            sig.params.len()
+        );
+
+        self.gen_call_common(abi, num_rets, caller, args)
+    }
+
+    fn gen_call_indirect(
+        &mut self,
+        sig_ref: SigRef,
+        val: Value,
+        args @ (inputs, off): ValueSlice,
+    ) -> InstOutput {
+        let caller_conv = self.lower_ctx.abi().call_conv();
+        let ptr = self.put_in_reg(val);
+        let sig = &self.lower_ctx.dfg().signatures[sig_ref];
+        let num_rets = sig.returns.len();
+        let abi = ABISig::from_func_sig::<X64ABIMachineSpec>(sig, self.flags).unwrap();
+        let caller =
+            X64ABICaller::from_ptr(sig, ptr, Opcode::CallIndirect, caller_conv, self.flags)
+                .unwrap();
+
+        assert_eq!(
+            inputs.len(&self.lower_ctx.dfg().value_lists) - off,
+            sig.params.len()
+        );
+
+        self.gen_call_common(abi, num_rets, caller, args)
+    }
+}
+
+impl<C> IsleContext<'_, C, Flags, IsaFlags, 6>
+where
+    C: LowerCtx<I = MInst>,
+{
+    fn abi_arg_slot_regs(&mut self, arg: &ABIArg) -> Option<WritableValueRegs> {
+        match arg {
+            &ABIArg::Slots { ref slots, .. } => match slots.len() {
+                1 => {
+                    let a = self.temp_writable_reg(slots[0].get_type());
+                    Some(WritableValueRegs::one(a))
+                }
+                2 => {
+                    let a = self.temp_writable_reg(slots[0].get_type());
+                    let b = self.temp_writable_reg(slots[1].get_type());
+                    Some(WritableValueRegs::two(a, b))
+                }
+                _ => panic!("Expected to see one or two slots only from {:?}", arg),
+            },
+            _ => None,
+        }
+    }
+
+    fn gen_call_common(
+        &mut self,
+        abi: ABISig,
+        num_rets: usize,
+        mut caller: X64ABICaller,
+        (inputs, off): ValueSlice,
+    ) -> InstOutput {
+        caller.emit_stack_pre_adjust(self.lower_ctx);
+
+        assert_eq!(
+            inputs.len(&self.lower_ctx.dfg().value_lists) - off,
+            abi.num_args()
+        );
+        for i in caller.get_copy_to_arg_order() {
+            let input = inputs
+                .get(off + i, &self.lower_ctx.dfg().value_lists)
+                .unwrap();
+            let arg_regs = self.lower_ctx.put_value_in_regs(input);
+            caller.emit_copy_regs_to_arg(self.lower_ctx, i, arg_regs);
+        }
+        caller.emit_call(self.lower_ctx);
+
+        let mut outputs = InstOutput::new();
+        for i in 0..num_rets {
+            let ret = abi.get_ret(i);
+            let retval_regs = self.abi_arg_slot_regs(&ret).unwrap();
+            caller.emit_copy_retval_to_regs(self.lower_ctx, i, retval_regs.clone());
+            outputs.push(valueregs::non_writable_value_regs(retval_regs));
+        }
+        caller.emit_stack_post_adjust(self.lower_ctx);
+
+        outputs
+    }
 }
 
 // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we

cranelift/codegen/src/machinst/abi_impl.rs

@@ -165,6 +165,16 @@ pub enum ABIArgSlot {
     },
 }
 
+impl ABIArgSlot {
+    /// The type of the value that will be stored in this slot.
+    pub fn get_type(&self) -> ir::Type {
+        match self {
+            ABIArgSlot::Reg { ty, .. } => *ty,
+            ABIArgSlot::Stack { ty, .. } => *ty,
+        }
+    }
+}
+
 /// An ABIArg is composed of one or more parts. This allows for a CLIF-level
 /// Value to be passed with its parts in more than one location at the ABI
 /// level. For example, a 128-bit integer may be passed in two 64-bit registers,

cranelift/filetests/filetests/isa/x64/call-conv.clif

@@ -292,11 +292,11 @@ block0:
 ;   pushq   %rbp
 ;   movq    %rsp, %rbp
 ; block0:
-;   movl    $1, %edx
+;   movl    $1, %eax
 ;   subq    %rsp, $16, %rsp
 ;   virtual_sp_offset_adjust 16
 ;   lea     0(%rsp), %rdi
-;   call    *%rdx
+;   call    *%rax
 ;   movq    0(%rsp), %rdx
 ;   addq    %rsp, $16, %rsp
 ;   virtual_sp_offset_adjust -16
@@ -318,8 +318,8 @@ block0:
 ;   movq    %r13, 0(%rsp)
 ; block0:
 ;   movq    %rdi, %r13
-;   movl    $1, %r9d
+;   movl    $1, %edx
-;   call    *%r9
+;   call    *%rdx
 ;   movq    %r13, %rdi
 ;   movl    %edx, 0(%rdi)
 ;   movq    0(%rsp), %r13
@@ -342,17 +342,17 @@ block0:
 ;   movq    %rbx, 0(%rsp)
 ; block0:
 ;   movq    %rdi, %rbx
-;   movl    $1, %r11d
+;   movl    $1, %r8d
 ;   subq    %rsp, $16, %rsp
 ;   virtual_sp_offset_adjust 16
 ;   lea     0(%rsp), %rdi
-;   call    *%r11
+;   call    *%r8
-;   movq    0(%rsp), %rcx
+;   movq    0(%rsp), %r11
 ;   addq    %rsp, $16, %rsp
 ;   virtual_sp_offset_adjust -16
 ;   movq    %rbx, %rdi
 ;   movq    %rdx, 0(%rdi)
-;   movl    %ecx, 8(%rdi)
+;   movl    %r11d, 8(%rdi)
 ;   movq    0(%rsp), %rbx
 ;   addq    %rsp, $16, %rsp
 ;   movq    %rbp, %rsp
@@ -373,8 +373,8 @@ block0:
 ;   movq    %r12, 0(%rsp)
 ; block0:
 ;   movq    %rdi, %r12
-;   movl    $1, %edi
+;   movl    $1, %r9d
-;   call    *%rdi
+;   call    *%r9
 ;   movq    %r12, %rdi
 ;   movq    %rax, 0(%rdi)
 ;   movl    %edx, 8(%rdi)
@@ -399,8 +399,8 @@ block0(v0: f32, v1: i64, v2: i32, v3: f32):
 ;   movq    %rbx, 0(%rsp)
 ; block0:
 ;   movq    %rdx, %rbx
-;   movl    $1, %eax
+;   movl    $1, %r8d
-;   call    *%rax
+;   call    *%r8
 ;   movq    %rbx, %r10
 ;   movq    %rax, 0(%r10)
 ;   movl    %edx, 8(%r10)

cranelift/filetests/filetests/isa/x64/i128.clif

@@ -785,15 +785,15 @@ block0(v0: i128, v1: i128):
 ;   subq    %rsp, $16, %rsp
 ;   virtual_sp_offset_adjust 16
 ;   lea     0(%rsp), %r8
-;   load_ext_name %g+0, %rax
+;   load_ext_name %g+0, %r9
-;   call    *%rax
+;   call    *%r9
-;   movq    0(%rsp), %r8
+;   movq    0(%rsp), %rcx
-;   movq    8(%rsp), %r10
+;   movq    8(%rsp), %r8
 ;   addq    %rsp, $16, %rsp
 ;   virtual_sp_offset_adjust -16
 ;   movq    %r12, %r9
-;   movq    %r8, 0(%r9)
+;   movq    %rcx, 0(%r9)
-;   movq    %r10, 8(%r9)
+;   movq    %r8, 8(%r9)
 ;   movq    0(%rsp), %r12
 ;   addq    %rsp, $16, %rsp
 ;   movq    %rbp, %rsp