Legalize ABI arguments to call and return instructions.

The type signatures of functions can change when they are legalized for
a specific ABI. This means that all call and return instructions need to
be rewritten to use the correct arguments.

- Fix arguments to call instructions.
- Fix arguments to return instructions.

TBD:

- Fix return values from call instructions.

cranelift/filetests/isa/riscv/abi.cton

@@ -7,27 +7,26 @@ isa riscv
 
 function f(i32) {
     sig0 = signature(i32) -> i32
-
+    ; check: sig0 = signature(i32 [%x10]) -> i32 [%x10]
-; check: sig0 = signature(i32 [%x10]) -> i32 [%x10]
 
     sig1 = signature(i64) -> b1
-; check: sig1 = signature(i32 [%x10], i32 [%x11]) -> b1 [%x10]
+    ; check: sig1 = signature(i32 [%x10], i32 [%x11]) -> b1 [%x10]
 
-; The i64 argument must go in an even-odd register pair.
+    ; The i64 argument must go in an even-odd register pair.
     sig2 = signature(f32, i64) -> f64
-; check: sig2 = signature(f32 [%f10], i32 [%x12], i32 [%x13]) -> f64 [%f10]
+    ; check: sig2 = signature(f32 [%f10], i32 [%x12], i32 [%x13]) -> f64 [%f10]
 
-; Spilling into the stack args.
+    ; Spilling into the stack args.
     sig3 = signature(f64, f64, f64, f64, f64, f64, f64, i64) -> f64
-; check: sig3 = signature(f64 [%f10], f64 [%f11], f64 [%f12], f64 [%f13], f64 [%f14], f64 [%f15], f64 [%f16], i32 [0], i32 [4]) -> f64 [%f10]
+    ; check: sig3 = signature(f64 [%f10], f64 [%f11], f64 [%f12], f64 [%f13], f64 [%f14], f64 [%f15], f64 [%f16], i32 [0], i32 [4]) -> f64 [%f10]
 
-; Splitting vectors.
+    ; Splitting vectors.
     sig4 = signature(i32x4)
-; check: sig4 = signature(i32 [%x10], i32 [%x11], i32 [%x12], i32 [%x13])
+    ; check: sig4 = signature(i32 [%x10], i32 [%x11], i32 [%x12], i32 [%x13])
 
-; Splitting vectors, then splitting ints.
+    ; Splitting vectors, then splitting ints.
     sig5 = signature(i64x4)
-; check: sig5 = signature(i32 [%x10], i32 [%x11], i32 [%x12], i32 [%x13], i32 [%x14], i32 [%x15], i32 [%x16], i32 [%x17])
+    ; check: sig5 = signature(i32 [%x10], i32 [%x11], i32 [%x12], i32 [%x13], i32 [%x14], i32 [%x15], i32 [%x16], i32 [%x17])
 
 ebb0(v0: i32):
     return_reg v0
@@ -35,62 +34,75 @@ ebb0(v0: i32):
 
 function int_split_args(i64) -> i64 {
 ebb0(v0: i64):
-; check: $ebb0($(v0l=$VX): i32, $(v0h=$VX): i32):
+    ; check: $ebb0($(v0l=$VX): i32, $(v0h=$VX): i32):
-; check: iconcat_lohi $v0l, $v0h
+    ; check: iconcat_lohi $v0l, $v0h
     v1 = iadd_imm v0, 1
-    return v0
+    ; check: $(v1l=$V), $(v1h=$VX) = isplit_lohi $v1
+    ; check: return $v1l, $v1h
+    return v1
 }
 
-function int_ext(i8, i8 sext, i8 uext) -> i8 {
+function int_ext(i8, i8 sext, i8 uext) -> i8 uext {
 ebb0(v1: i8, v2: i8, v3: i8):
-; check: $ebb0($v1: i8, $(v2x=$VX): i32, $(v3x=$VX): i32):
+    ; check: $ebb0($v1: i8, $(v2x=$VX): i32, $(v3x=$VX): i32):
-; check: ireduce.i8 $v2x
+    ; check: ireduce.i8 $v2x
-; check: ireduce.i8 $v3x
+    ; check: ireduce.i8 $v3x
+    ; check: $(v1x=$V) = uextend.i32 $v1
+    ; check: return $v1x
+    return v1
 }
 
-function vector_split_args(i64x4) -> i64 {
+function vector_split_args(i64x4) -> i64x4 {
 ebb0(v0: i64x4):
-; check: $ebb0($(v0al=$VX): i32, $(v0ah=$VX): i32, $(v0bl=$VX): i32, $(v0bh=$VX): i32, $(v0cl=$VX): i32, $(v0ch=$VX): i32, $(v0dl=$VX): i32, $(v0dh=$VX): i32):
+    ; check: $ebb0($(v0al=$VX): i32, $(v0ah=$VX): i32, $(v0bl=$VX): i32, $(v0bh=$VX): i32, $(v0cl=$VX): i32, $(v0ch=$VX): i32, $(v0dl=$VX): i32, $(v0dh=$VX): i32):
-; check: $(v0a=$V) = iconcat_lohi $v0al, $v0ah
+    ; check: $(v0a=$V) = iconcat_lohi $v0al, $v0ah
-; check: $(v0b=$V) = iconcat_lohi $v0bl, $v0bh
+    ; check: $(v0b=$V) = iconcat_lohi $v0bl, $v0bh
-; check: $(v0ab=$V) = vconcat $v0a, $v0b
+    ; check: $(v0ab=$V) = vconcat $v0a, $v0b
-; check: $(v0c=$V) = iconcat_lohi $v0cl, $v0ch
+    ; check: $(v0c=$V) = iconcat_lohi $v0cl, $v0ch
-; check: $(v0d=$V) = iconcat_lohi $v0dl, $v0dh
+    ; check: $(v0d=$V) = iconcat_lohi $v0dl, $v0dh
-; check: $(v0cd=$V) = vconcat $v0c, $v0d
+    ; check: $(v0cd=$V) = vconcat $v0c, $v0d
-; check: $(v0abcd=$V) = vconcat $v0ab, $v0cd
+    ; check: $(v0abcd=$V) = vconcat $v0ab, $v0cd
     v1 = iadd v0, v0
-    return v0
+    ; check: $(v1ab=$V), $(v1cd=$VX) = vsplit
+    ; check: $(v1a=$V), $(v1b=$VX) = vsplit $v1ab
+    ; check: $(v1al=$V), $(v1ah=$VX) = isplit_lohi $v1a
+    ; check: $(v1bl=$V), $(v1bh=$VX) = isplit_lohi $v1b
+    ; check: $(v1c=$V), $(v1d=$VX) = vsplit $v1cd
+    ; check: $(v1cl=$V), $(v1ch=$VX) = isplit_lohi $v1c
+    ; check: $(v1dl=$V), $(v1dh=$VX) = isplit_lohi $v1d
+    ; check: return $v1al, $v1ah, $v1bl, $v1bh, $v1cl, $v1ch, $v1dl, $v1dh
+    return v1
 }
 
 function parse_encoding(i32 [%x5]) -> i32 [%x10] {
-; check: function parse_encoding(i32 [%x5]) -> i32 [%x10] {
+    ; check: function parse_encoding(i32 [%x5]) -> i32 [%x10] {
 
     sig0 = signature(i32 [%x10]) -> i32 [%x10]
-; check: sig0 = signature(i32 [%x10]) -> i32 [%x10]
+    ; check: sig0 = signature(i32 [%x10]) -> i32 [%x10]
 
     sig1 = signature(i32 [%x10], i32 [%x11]) -> b1 [%x10]
-; check: sig1 = signature(i32 [%x10], i32 [%x11]) -> b1 [%x10]
+    ; check: sig1 = signature(i32 [%x10], i32 [%x11]) -> b1 [%x10]
 
     sig2 = signature(f32 [%f10], i32 [%x12], i32 [%x13]) -> f64 [%f10]
-; check: sig2 = signature(f32 [%f10], i32 [%x12], i32 [%x13]) -> f64 [%f10]
+    ; check: sig2 = signature(f32 [%f10], i32 [%x12], i32 [%x13]) -> f64 [%f10]
 
-; Arguments on stack where not necessary
+    ; Arguments on stack where not necessary
     sig3 = signature(f64 [%f10], i32 [0], i32 [4]) -> f64 [%f10]
-; check: sig3 = signature(f64 [%f10], i32 [0], i32 [4]) -> f64 [%f10]
+    ; check: sig3 = signature(f64 [%f10], i32 [0], i32 [4]) -> f64 [%f10]
 
-; Stack argument before register argument
+    ; Stack argument before register argument
     sig4 = signature(f32 [72], i32 [%x10])
-; check: sig4 = signature(f32 [72], i32 [%x10])
+    ; check: sig4 = signature(f32 [72], i32 [%x10])
 
-; Return value on stack
+    ; Return value on stack
     sig5 = signature() -> f32 [0]
-; check: sig5 = signature() -> f32 [0]
+    ; check: sig5 = signature() -> f32 [0]
 
-; function + signature
+    ; function + signature
     fn15 = function bar(i32 [%x10]) -> b1 [%x10]
-; check: sig6 = signature(i32 [%x10]) -> b1 [%x10]
+    ; check: sig6 = signature(i32 [%x10]) -> b1 [%x10]
-; nextln: fn0 = sig6 bar
+    ; nextln: fn0 = sig6 bar
 
 ebb0(v0: i32):
-    return_reg v0
+    return v0
 }

lib/cretonne/src/legalizer.rs

@@ -14,9 +14,10 @@
 //! from the encoding recipes, and solved later by the register allocator.
 
 use abi::{legalize_abi_value, ValueConversion};
-use ir::{Function, Cursor, DataFlowGraph, InstructionData, Opcode, InstBuilder, Ebb, Type, Value,
+use ir::{Function, Cursor, DataFlowGraph, InstructionData, Opcode, Inst, InstBuilder, Ebb, Type,
-         ArgumentType};
+         Value, Signature, SigRef, ArgumentType};
 use ir::condcodes::IntCC;
+use ir::instructions::CallInfo;
 use isa::{TargetIsa, Legalize};
 
 /// Legalize `func` for `isa`.
@@ -36,6 +37,21 @@ pub fn legalize_function(func: &mut Function, isa: &TargetIsa) {
         let mut prev_pos = pos.position();
 
         while let Some(inst) = pos.next_inst() {
+            let opcode = func.dfg[inst].opcode();
+
+            // Check for ABI boundaries that need to be converted to the legalized signature.
+            if opcode.is_call() && handle_call_abi(&mut func.dfg, &mut pos) {
+                // Go back and legalize the inserted argument conversion instructions.
+                pos.set_position(prev_pos);
+                continue;
+            }
+
+            if opcode.is_return() && handle_return_abi(&mut func.dfg, &mut pos, &func.signature) {
+                // Go back and legalize the inserted return value conversion instructions.
+                pos.set_position(prev_pos);
+                continue;
+            }
+
             match isa.encode(&func.dfg, &func.dfg[inst]) {
                 Ok(encoding) => *func.encodings.ensure(inst) = encoding,
                 Err(action) => {
@@ -201,3 +217,232 @@ fn convert_from_abi(dfg: &mut DataFlowGraph,
         }
     }
 }
+
+/// Convert `value` to match an ABI signature by inserting instructions at `pos`.
+///
+/// This may require expanding the value to multiple ABI arguments. The conversion process is
+/// recursive and controlled by the `put_arg` closure. When a candidate argument value is presented
+/// to the closure, it will perform one of two actions:
+///
+/// 1. If the suggested argument has an acceptable value type, consume it by adding it to the list
+///    of arguments and return `None`.
+/// 2. If the suggested argument doesn't have the right value type, don't change anything, but
+///    return the `ArgumentType` that is needed.
+///
+fn convert_to_abi<PutArg>(dfg: &mut DataFlowGraph,
+                          pos: &mut Cursor,
+                          value: Value,
+                          put_arg: &mut PutArg)
+    where PutArg: FnMut(&mut DataFlowGraph, Value) -> Option<ArgumentType>
+{
+    // Start by invoking the closure to either terminate the recursion or get the argument type
+    // we're trying to match.
+    let arg_type = match put_arg(dfg, value) {
+        None => return,
+        Some(t) => t,
+    };
+
+    let ty = dfg.value_type(value);
+    match legalize_abi_value(ty, &arg_type) {
+        ValueConversion::IntSplit => {
+            let (lo, hi) = dfg.ins(pos).isplit_lohi(value);
+            convert_to_abi(dfg, pos, lo, put_arg);
+            convert_to_abi(dfg, pos, hi, put_arg);
+        }
+        ValueConversion::VectorSplit => {
+            let (lo, hi) = dfg.ins(pos).vsplit(value);
+            convert_to_abi(dfg, pos, lo, put_arg);
+            convert_to_abi(dfg, pos, hi, put_arg);
+        }
+        ValueConversion::IntBits => {
+            assert!(!ty.is_int());
+            let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
+            let arg = dfg.ins(pos).bitcast(abi_ty, value);
+            convert_to_abi(dfg, pos, arg, put_arg);
+        }
+        ValueConversion::Sext(abi_ty) => {
+            let arg = dfg.ins(pos).sextend(abi_ty, value);
+            convert_to_abi(dfg, pos, arg, put_arg);
+        }
+        ValueConversion::Uext(abi_ty) => {
+            let arg = dfg.ins(pos).uextend(abi_ty, value);
+            convert_to_abi(dfg, pos, arg, put_arg);
+        }
+    }
+}
+
+/// Check if a sequence of arguments match a desired sequence of argument types.
+fn check_arg_types<Args>(dfg: &DataFlowGraph, args: Args, types: &[ArgumentType]) -> bool
+    where Args: IntoIterator<Item = Value>
+{
+    let mut n = 0;
+    for arg in args {
+        match types.get(n) {
+            Some(&ArgumentType { value_type, .. }) => {
+                if dfg.value_type(arg) != value_type {
+                    return false;
+                }
+            }
+            None => return false,
+        }
+        n += 1
+    }
+
+    // Also verify that the number of arguments matches.
+    n == types.len()
+}
+
+/// Check if the arguments of the call `inst` match the signature.
+///
+/// Returns `None` if the signature matches and no changes are needed, or `Some(sig_ref)` if the
+/// signature doesn't match.
+fn check_call_signature(dfg: &DataFlowGraph, inst: Inst) -> Option<SigRef> {
+    // Extract the signature and argument values.
+    let (sig_ref, args) = match dfg[inst].analyze_call(&dfg.value_lists) {
+        CallInfo::Direct(func, args) => (dfg.ext_funcs[func].signature, args),
+        CallInfo::Indirect(sig_ref, args) => (sig_ref, args),
+        CallInfo::NotACall => panic!("Expected call, got {:?}", dfg[inst]),
+    };
+    let sig = &dfg.signatures[sig_ref];
+
+    if check_arg_types(dfg, args.iter().cloned(), &sig.argument_types[..]) &&
+       check_arg_types(dfg, dfg.inst_results(inst), &sig.return_types[..]) {
+        // All types check out.
+        None
+    } else {
+        // Call types need fixing.
+        Some(sig_ref)
+    }
+}
+
+/// Insert ABI conversion code for the arguments to the call or return instruction at `pos`.
+///
+/// - `abi_args` is the number of arguments that the ABI signature requires.
+/// - `get_abi_type` is a closure that can provide the desired `ArgumentType` for a given ABI
+///   argument number in `0..abi_args`.
+///
+fn legalize_inst_arguments<ArgType>(dfg: &mut DataFlowGraph,
+                                    pos: &mut Cursor,
+                                    abi_args: usize,
+                                    mut get_abi_type: ArgType)
+    where ArgType: FnMut(&DataFlowGraph, usize) -> ArgumentType
+{
+    let inst = pos.current_inst().expect("Cursor must point to a call instruction");
+
+    // Lift the value list out of the call instruction so we modify it.
+    let mut vlist = dfg[inst].take_value_list().expect("Call must have a value list");
+
+    // The value list contains all arguments to the instruction, including the callee on an
+    // indirect call which isn't part of the call arguments that must match the ABI signature.
+    // Figure out how many fixed values are at the front of the list. We won't touch those.
+    let fixed_values = dfg[inst].opcode().constraints().fixed_value_arguments();
+    let have_args = vlist.len(&dfg.value_lists) - fixed_values;
+
+    // Grow the value list to the right size and shift all the existing arguments to the right.
+    // This lets us write the new argument values into the list without overwriting the old
+    // arguments.
+    //
+    // Before:
+    //
+    //    <-->              fixed_values
+    //        <-----------> have_args
+    //   [FFFFOOOOOOOOOOOOO]
+    //
+    // After grow_at():
+    //
+    //    <-->                     fixed_values
+    //               <-----------> have_args
+    //        <------------------> abi_args
+    //   [FFFF-------OOOOOOOOOOOOO]
+    //               ^
+    //               old_arg_offset
+    //
+    // After writing the new arguments:
+    //
+    //    <-->                     fixed_values
+    //        <------------------> abi_args
+    //   [FFFFNNNNNNNNNNNNNNNNNNNN]
+    //
+    vlist.grow_at(fixed_values, abi_args - have_args, &mut dfg.value_lists);
+    let old_arg_offset = fixed_values + abi_args - have_args;
+
+    let mut abi_arg = 0;
+    for old_arg in 0..have_args {
+        let old_value = vlist.get(old_arg_offset + old_arg, &dfg.value_lists).unwrap();
+        convert_to_abi(dfg,
+                       pos,
+                       old_value,
+                       &mut |dfg, arg| {
+            let abi_type = get_abi_type(dfg, abi_arg);
+            if dfg.value_type(arg) == abi_type.value_type {
+                // This is the argument type we need.
+                vlist.as_mut_slice(&mut dfg.value_lists)[fixed_values + abi_arg] = arg;
+                abi_arg += 1;
+                None
+            } else {
+                // Nope, `arg` needs to be converted.
+                Some(abi_type)
+            }
+        });
+    }
+
+    // Put the modified value list back.
+    dfg[inst].put_value_list(vlist);
+}
+
+/// Insert ABI conversion code before and after the call instruction at `pos`.
+///
+/// Instructions inserted before the call will compute the appropriate ABI values for the
+/// callee's new ABI-legalized signature. The function call arguments are rewritten in place to
+/// match the new signature.
+///
+/// Instructions will be inserted after the call to convert returned ABI values back to the
+/// original return values. The call's result values will be adapted to match the new signature.
+///
+/// Returns `true` if any instructions were inserted.
+fn handle_call_abi(dfg: &mut DataFlowGraph, pos: &mut Cursor) -> bool {
+    let inst = pos.current_inst().expect("Cursor must point to a call instruction");
+
+    // Start by checking if the argument types already match the signature.
+    let sig_ref = match check_call_signature(dfg, inst) {
+        None => return false,
+        Some(s) => s,
+    };
+
+    // OK, we need to fix the call arguments to match the ABI signature.
+    let abi_args = dfg.signatures[sig_ref].argument_types.len();
+    legalize_inst_arguments(dfg,
+                            pos,
+                            abi_args,
+                            |dfg, abi_arg| dfg.signatures[sig_ref].argument_types[abi_arg]);
+
+    // TODO: Convert return values.
+
+    // Yes, we changed stuff.
+    true
+}
+
+/// Insert ABI conversion code before and after the call instruction at `pos`.
+///
+/// Return `true` if any instructions were inserted.
+fn handle_return_abi(dfg: &mut DataFlowGraph, pos: &mut Cursor, sig: &Signature) -> bool {
+    let inst = pos.current_inst().expect("Cursor must point to a return instruction");
+
+    // Check if the returned types already match the signature.
+    let fixed_values = dfg[inst].opcode().constraints().fixed_value_arguments();
+    if check_arg_types(dfg,
+                       dfg[inst]
+                           .arguments(&dfg.value_lists)
+                           .iter()
+                           .skip(fixed_values)
+                           .cloned(),
+                       &sig.return_types[..]) {
+        return false;
+    }
+
+    let abi_args = sig.return_types.len();
+    legalize_inst_arguments(dfg, pos, abi_args, |_, abi_arg| sig.return_types[abi_arg]);
+
+    // Yes, we changed stuff.
+    true
+}