diff --git a/cranelift/filetests/isa/riscv/split-args.cton b/cranelift/filetests/isa/riscv/split-args.cton new file mode 100644 index 0000000000..1cecb85c10 --- /dev/null +++ b/cranelift/filetests/isa/riscv/split-args.cton @@ -0,0 +1,40 @@ +; Test the legalization of EBB arguments that are split. +test legalizer +isa riscv + +; regex: V=vx?\d+ + +function simple(i64, i64) -> i64 { +ebb0(v1: i64, v2: i64): +; check: $ebb0($(v1l=$V): i32, $(v1h=$V): i32, $(v2l=$V): i32, $(v2h=$V): i32): + jump ebb1(v1) + ; check: jump $ebb1($v1l, $v1h) + +ebb1(v3: i64): +; check: $ebb1($(v3l=$V): i32, $(v3h=$V): i32): + v4 = band v3, v2 + ; check: $(v4l=$V) = band $v3l, $v2l + ; check: $(v4h=$V) = band $v3h, $v2h + return v4 + ; check: return $v4l, $v4h +} + +function multi(i64) -> i64 { +ebb1(v1: i64): +; check: $ebb1($(v1l=$V): i32, $(v1h=$V): i32): + jump ebb2(v1, v1) + ; check: jump $ebb2($v1l, $v1l, $v1h, $v1h) + +ebb2(v2: i64, v3: i64): +; check: $ebb2($(v2l=$V): i32, $(v3l=$V): i32, $(v2h=$V): i32, $(v3h=$V): i32): + jump ebb3(v2) + ; check: jump $ebb3($v2l, $v2h) + +ebb3(v4: i64): +; check: $ebb3($(v4l=$V): i32, $(v4h=$V): i32): + v5 = band v4, v3 + ; check: $(v5l=$V) = band $v4l, $v3l + ; check: $(v5h=$V) = band $v4h, $v3h + return v5 + ; check: return $v5l, $v5h +} diff --git a/lib/cretonne/meta/base/instructions.py b/lib/cretonne/meta/base/instructions.py index 66dc0e9090..ade9b540bd 100644 --- a/lib/cretonne/meta/base/instructions.py +++ b/lib/cretonne/meta/base/instructions.py @@ -44,7 +44,7 @@ jump = Instruction( EBB arguments. The number and types of arguments must match the destination EBB. """, - ins=(EBB, args), is_terminator=True) + ins=(EBB, args), is_branch=True, is_terminator=True) brz = Instruction( 'brz', r""" diff --git a/lib/cretonne/src/ir/layout.rs b/lib/cretonne/src/ir/layout.rs index 83edbea76f..c34d6a0aba 100644 --- a/lib/cretonne/src/ir/layout.rs +++ b/lib/cretonne/src/ir/layout.rs @@ -588,7 +588,8 @@ impl<'f> DoubleEndedIterator for Insts<'f> { /// When new instructions are added, the cursor can either append them to an EBB or insert them /// before the current instruction. pub struct Cursor<'f> { - layout: &'f mut Layout, + /// Borrowed function layout. Public so it can be re-borrowed from this cursor. + pub layout: &'f mut Layout, pos: CursorPosition, } diff --git a/lib/cretonne/src/legalizer/split.rs b/lib/cretonne/src/legalizer/split.rs index 7b36c4997e..d58c7ed285 100644 --- a/lib/cretonne/src/legalizer/split.rs +++ b/lib/cretonne/src/legalizer/split.rs @@ -65,26 +65,103 @@ //! instructions. These loops will remain in the program. use flowgraph::ControlFlowGraph; -use ir::{DataFlowGraph, Cursor, Value, Opcode, ValueDef, InstructionData, InstBuilder}; +use ir::{DataFlowGraph, Ebb, Cursor, Value, Type, Opcode, ValueDef, InstructionData, InstBuilder}; +use std::iter; /// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values /// if possible. pub fn isplit(dfg: &mut DataFlowGraph, - _cfg: &ControlFlowGraph, + cfg: &ControlFlowGraph, pos: &mut Cursor, value: Value) -> (Value, Value) { - split_value(dfg, pos, value, Opcode::Iconcat) + split_any(dfg, cfg, pos, value, Opcode::Iconcat) } /// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if /// possible. pub fn vsplit(dfg: &mut DataFlowGraph, - _cfg: &ControlFlowGraph, + cfg: &ControlFlowGraph, pos: &mut Cursor, value: Value) -> (Value, Value) { - split_value(dfg, pos, value, Opcode::Vconcat) + split_any(dfg, cfg, pos, value, Opcode::Vconcat) +} + +/// After splitting an EBB argument, we need to go back and fix up all of the predecessor +/// instructions. This is potentially a recursive operation, but we don't implement it recursively +/// since that could use up too muck stack. +/// +/// Instead, the repairs are deferred and placed on a work list in stack form. +struct Repair { + concat: Opcode, + // The argument type after splitting. + split_type: Type, + // The destination EBB whose arguments have been split. + ebb: Ebb, + // Number of the original EBB argument which has been replaced by the low part. + num: usize, + // Number of the new EBB argument which represents the high part after the split. + hi_num: usize, +} + +/// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode. +fn split_any(dfg: &mut DataFlowGraph, + cfg: &ControlFlowGraph, + pos: &mut Cursor, + value: Value, + concat: Opcode) + -> (Value, Value) { + let saved_pos = pos.position(); + let mut repairs = Vec::new(); + let result = split_value(dfg, pos, value, concat, &mut repairs); + + // We have split the value requested, and now we may need to fix some EBB predecessors. + while let Some(repair) = repairs.pop() { + for &(_, inst) in cfg.get_predecessors(repair.ebb) { + let branch_opc = dfg[inst].opcode(); + assert!(branch_opc.is_branch(), + "Predecessor not a branch: {}", + dfg.display_inst(inst)); + let fixed_args = branch_opc.constraints().fixed_value_arguments(); + let mut args = dfg[inst].take_value_list().expect("Branches must have value lists."); + let num_args = args.len(&dfg.value_lists); + // Get the old value passed to the EBB argument we're repairing. + let old_arg = args.get(fixed_args + repair.num, &dfg.value_lists) + .expect("Too few branch arguments"); + + // It's possible that the CFG's predecessor list has duplicates. Detect them here. + if dfg.value_type(old_arg) == repair.split_type { + dfg[inst].put_value_list(args); + continue; + } + + // Split the old argument, possibly causing more repairs to be scheduled. + pos.goto_inst(inst); + let (lo, hi) = split_value(dfg, pos, old_arg, repair.concat, &mut repairs); + + // The `lo` part replaces the original argument. + *args.get_mut(fixed_args + repair.num, &mut dfg.value_lists).unwrap() = lo; + + // The `hi` part goes at the end. Since multiple repairs may have been scheduled to the + // same EBB, there could be multiple arguments missing. + if num_args > fixed_args + repair.hi_num { + *args.get_mut(fixed_args + repair.hi_num, &mut dfg.value_lists).unwrap() = hi; + } else { + // We need to append one or more arguments. If we're adding more than one argument, + // there must be pending repairs on the stack that will fill in the correct values + // instead of `hi`. + args.extend(iter::repeat(hi).take(1 + fixed_args + repair.hi_num - num_args), + &mut dfg.value_lists); + } + + // Put the value list back after manipulating it. + dfg[inst].put_value_list(args); + } + } + + pos.set_position(saved_pos); + result } /// Split a single value using the integer or vector semantics given by the `concat` opcode. @@ -96,7 +173,8 @@ pub fn vsplit(dfg: &mut DataFlowGraph, fn split_value(dfg: &mut DataFlowGraph, pos: &mut Cursor, value: Value, - concat: Opcode) + concat: Opcode, + repairs: &mut Vec) -> (Value, Value) { let value = dfg.resolve_copies(value); let mut reuse = None; @@ -112,14 +190,56 @@ fn split_value(dfg: &mut DataFlowGraph, } } } - ValueDef::Arg(_ebb, _num) => {} + ValueDef::Arg(ebb, num) => { + // This is an EBB argument. We can split the argument value unless this is the entry + // block. + if pos.layout.entry_block() != Some(ebb) { + // We are going to replace the argument at `num` with two new arguments. + // Determine the new value types. + let ty = dfg.value_type(value); + let split_type = match concat { + Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"), + Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"), + _ => panic!("Unhandled concat opcode: {}", concat), + }; + + // Since the `repairs` stack potentially contains other argument numbers for `ebb`, + // avoid shifting and renumbering EBB arguments. It could invalidate other + // `repairs` entries. + // + // Replace the original `value` with the low part, and append the high part at the + // end of the argument list. + let lo = dfg.replace_ebb_arg(value, split_type); + let hi_num = dfg.num_ebb_args(ebb); + let hi = dfg.append_ebb_arg(ebb, split_type); + reuse = Some((lo, hi)); + + + // Now the original value is dangling. Insert a concatenation instruction that can + // compute it from the two new arguments. This also serves as a record of what we + // did so a future call to this function doesn't have to redo the work. + // + // Note that it is safe to move `pos` here since `reuse` was set above, so we don't + // need to insert a split instruction before returning. + pos.goto_top(ebb); + pos.next_inst(); + let concat_inst = dfg.ins(pos).Binary(concat, ty, lo, hi).0; + let concat_val = dfg.first_result(concat_inst); + dfg.change_to_alias(value, concat_val); + + // Finally, splitting the EBB argument is not enough. We also have to repair all + // of the predecessor instructions that branch here. + add_repair(concat, split_type, ebb, num, hi_num, repairs); + } + } } // Did the code above succeed in finding values we can reuse? if let Some(pair) = reuse { pair } else { - // No, we'll just have to insert the requested split instruction at `pos`. + // No, we'll just have to insert the requested split instruction at `pos`. Note that `pos` + // has not been moved by the EBB argument code above when `reuse` is `None`. match concat { Opcode::Iconcat => dfg.ins(pos).isplit(value), Opcode::Vconcat => dfg.ins(pos).vsplit(value), @@ -127,3 +247,19 @@ fn split_value(dfg: &mut DataFlowGraph, } } } + +// Add a repair entry to the work list. +fn add_repair(concat: Opcode, + split_type: Type, + ebb: Ebb, + num: usize, + hi_num: usize, + repairs: &mut Vec) { + repairs.push(Repair { + concat: concat, + split_type: split_type, + ebb: ebb, + num: num, + hi_num: hi_num, + }); +}