[s390x, abi_impl] Support struct args using explicit pointers (#4585)

This adds support for StructArgument on s390x.  The ABI for this
platform requires that the address of the buffer holding the copy
of the struct argument is passed from caller to callee as hidden
pointer, using a register or overflow stack slot.

To implement this, I've added an optional "pointer" filed to
ABIArg::StructArg, and code to handle the pointer both in common
abi_impl code and the s390x back-end.

One notable change necessary to make this work involved the
"copy_to_arg_order" mechanism.  Currently, for struct args
we only need to copy the data (and that need to happen before
setting up any other args), while for non-struct args we only
need to set up the appropriate registers or stack slots.
This order is ensured by sorting the arguments appropriately
into a "copy_to_arg_order" list.

However, for struct args with explicit pointers we need to *both*
copy the data (again, before everything else), *and* set up a
register or stack slot.  Since we now need to touch the argument
twice, we cannot solve the ordering problem by a simple sort.
Instead, the abi_impl common code now provided *two* callbacks,
emit_copy_regs_to_buffer and emit_copy_regs_to_arg, and expects
the back end to first call copy..to_buffer for all args, and
then call copy.._to_arg for all args.  This required updates
to all back ends.

In the s390x back end, in addition to the new ABI code, I'm now
adding code to actually copy the struct data, using the MVC
instruction (for small buffers) or a memcpy libcall (for larger
buffers).  This also requires a bit of new infrastructure:
- MVC is the first memory-to-memory instruction we use, which
  needed a bit of memory argument tweaking
- We also need to set up the infrastructure to emit libcalls.

(This implements the first half of issue #4565.)

cranelift/codegen/src/isa/s390x/inst/args.rs

@@ -145,6 +145,66 @@ impl MemArg {
     }
 }
 
+/// A memory argument for an instruction with two memory operands.
+/// We cannot use two instances of MemArg, because we do not have
+/// two free temp registers that would be needed to reload two
+/// addresses in the general case.  Also, two copies of MemArg would
+/// increase the size of Inst beyond its current limit.  Use this
+/// simplified form instead that never needs any reloads, and suffices
+/// for all current users.
+#[derive(Clone, Debug)]
+pub struct MemArgPair {
+    pub base: Reg,
+    pub disp: UImm12,
+    pub flags: MemFlags,
+}
+
+impl MemArgPair {
+    /// Convert a MemArg to a MemArgPair if possible.
+    pub fn maybe_from_memarg(mem: &MemArg) -> Option<MemArgPair> {
+        match mem {
+            &MemArg::BXD12 {
+                base,
+                index,
+                disp,
+                flags,
+            } => {
+                if index != zero_reg() {
+                    None
+                } else {
+                    Some(MemArgPair { base, disp, flags })
+                }
+            }
+            &MemArg::RegOffset { reg, off, flags } => {
+                if off < 0 {
+                    None
+                } else {
+                    let disp = UImm12::maybe_from_u64(off as u64)?;
+                    Some(MemArgPair {
+                        base: reg,
+                        disp,
+                        flags,
+                    })
+                }
+            }
+            _ => None,
+        }
+    }
+
+    pub(crate) fn can_trap(&self) -> bool {
+        !self.flags.notrap()
+    }
+
+    /// Edit registers with allocations.
+    pub fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
+        MemArgPair {
+            base: allocs.next(self.base),
+            disp: self.disp,
+            flags: self.flags,
+        }
+    }
+}
+
 //=============================================================================
 // Instruction sub-components (conditions, branches and branch targets):
 // definitions

cranelift/codegen/src/isa/s390x/inst/emit.rs

@@ -297,6 +297,35 @@ pub fn mem_imm16_emit(
     }
 }
 
+pub fn mem_mem_emit(
+    dst: &MemArgPair,
+    src: &MemArgPair,
+    len_minus_one: u8,
+    opcode_ss: u8,
+    add_trap: bool,
+    sink: &mut MachBuffer<Inst>,
+    state: &mut EmitState,
+) {
+    if add_trap && (dst.can_trap() || src.can_trap()) {
+        let srcloc = state.cur_srcloc();
+        if srcloc != SourceLoc::default() {
+            sink.add_trap(TrapCode::HeapOutOfBounds);
+        }
+    }
+
+    put(
+        sink,
+        &enc_ss_a(
+            opcode_ss,
+            dst.base,
+            dst.disp.bits(),
+            src.base,
+            src.disp.bits(),
+            len_minus_one,
+        ),
+    );
+}
+
 pub fn mem_vrx_emit(
     rd: Reg,
     mem: &MemArg,
@@ -853,6 +882,31 @@ fn enc_siy(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 6] {
     enc
 }
 
+/// SSa-type instructions.
+///
+///   47     39 31 27 15 11
+///   opcode  l b1 d1 b2 d2
+///       40 32 28 16 12  0
+///
+///
+fn enc_ss_a(opcode: u8, b1: Reg, d1: u32, b2: Reg, d2: u32, l: u8) -> [u8; 6] {
+    let b1 = machreg_to_gpr(b1) & 0x0f;
+    let d1_lo = (d1 & 0xff) as u8;
+    let d1_hi = ((d1 >> 8) & 0x0f) as u8;
+    let b2 = machreg_to_gpr(b2) & 0x0f;
+    let d2_lo = (d2 & 0xff) as u8;
+    let d2_hi = ((d2 >> 8) & 0x0f) as u8;
+
+    let mut enc: [u8; 6] = [0; 6];
+    enc[0] = opcode;
+    enc[1] = l;
+    enc[2] = b1 << 4 | d1_hi;
+    enc[3] = d1_lo;
+    enc[4] = b2 << 4 | d2_hi;
+    enc[5] = d2_lo;
+    enc
+}
+
 /// VRIa-type instructions.
 ///
 ///   47      39 35 31 15 11  7
@@ -2025,6 +2079,16 @@ impl MachInstEmit for Inst {
                 };
                 mem_imm16_emit(imm, &mem, opcode, true, sink, emit_info, state);
             }
+            &Inst::Mvc {
+                ref dst,
+                ref src,
+                len_minus_one,
+            } => {
+                let dst = dst.with_allocs(&mut allocs);
+                let src = src.with_allocs(&mut allocs);
+                let opcode = 0xd2; // MVC
+                mem_mem_emit(&dst, &src, len_minus_one, opcode, true, sink, state);
+            }
 
             &Inst::LoadMultiple64 { rt, rt2, ref mem } => {
                 let mem = mem.with_allocs(&mut allocs);

cranelift/codegen/src/isa/s390x/inst/emit_tests.rs

@@ -5862,6 +5862,24 @@ fn test_s390x_binemit() {
         "stgrl %r1, label1",
     ));
 
+    insns.push((
+        Inst::Mvc {
+            dst: MemArgPair {
+                base: gpr(2),
+                disp: UImm12::maybe_from_u64(0x345).unwrap(),
+                flags: MemFlags::trusted(),
+            },
+            src: MemArgPair {
+                base: gpr(8),
+                disp: UImm12::maybe_from_u64(0x9ab).unwrap(),
+                flags: MemFlags::trusted(),
+            },
+            len_minus_one: 255,
+        },
+        "D2FF234589AB",
+        "mvc 837(255,%r2), 2475(%r8)",
+    ));
+
     insns.push((
         Inst::LoadMultiple64 {
             rt: writable_gpr(8),

cranelift/codegen/src/isa/s390x/inst/mod.rs

@@ -140,6 +140,7 @@ impl Inst {
             | Inst::StoreRev16 { .. }
             | Inst::StoreRev32 { .. }
             | Inst::StoreRev64 { .. }
+            | Inst::Mvc { .. }
             | Inst::LoadMultiple64 { .. }
             | Inst::StoreMultiple64 { .. }
             | Inst::Mov32 { .. }
@@ -600,6 +601,12 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandC
         | &Inst::StoreImm64SExt16 { ref mem, .. } => {
             memarg_operands(mem, collector);
         }
+        &Inst::Mvc {
+            ref dst, ref src, ..
+        } => {
+            collector.reg_use(dst.base);
+            collector.reg_use(src.base);
+        }
         &Inst::LoadMultiple64 {
             rt, rt2, ref mem, ..
         } => {
@@ -1763,6 +1770,22 @@ impl Inst {
 
                 format!("{}{} {}, {}", mem_str, op, mem, imm)
             }
+            &Inst::Mvc {
+                ref dst,
+                ref src,
+                len_minus_one,
+            } => {
+                let dst = dst.with_allocs(allocs);
+                let src = src.with_allocs(allocs);
+                format!(
+                    "mvc {}({},{}), {}({})",
+                    dst.disp.pretty_print_default(),
+                    len_minus_one,
+                    show_reg(dst.base),
+                    src.disp.pretty_print_default(),
+                    show_reg(src.base)
+                )
+            }
             &Inst::LoadMultiple64 { rt, rt2, ref mem } => {
                 let mem = mem.with_allocs(allocs);
                 let (mem_str, mem) = mem_finalize_for_show(&mem, state, false, true, false, false);