b95508c51a
This function is responsible for 2.2% of all heap allocation (calls) in CL. This change avoids all of them in the (presumably) common case where none of the parameters require splitting. It also slightly reduces the compiler's instruction count.
394 lines
14 KiB
Rust
394 lines
14 KiB
Rust
//! Value splitting.
|
|
//!
|
|
//! Some value types are too large to fit in registers, so they need to be split into smaller parts
|
|
//! that the ISA can operate on. There's two dimensions of splitting, represented by two
|
|
//! complementary instruction pairs:
|
|
//!
|
|
//! - `isplit` and `iconcat` for splitting integer types into smaller integers.
|
|
//! - `vsplit` and `vconcat` for splitting vector types into smaller vector types with the same
|
|
//! lane types.
|
|
//!
|
|
//! There is no floating point splitting. If an ISA doesn't support `f64` values, they probably
|
|
//! have to be bit-cast to `i64` and possibly split into two `i32` values that fit in registers.
|
|
//! This breakdown is handled by the ABI lowering.
|
|
//!
|
|
//! When legalizing a single instruction, it is wrapped in splits and concatenations:
|
|
//!
|
|
//! ```clif
|
|
//! v1 = bxor.i64 v2, v3
|
|
//! ```
|
|
//!
|
|
//! becomes:
|
|
//!
|
|
//! ```clif
|
|
//! v20, v21 = isplit v2
|
|
//! v30, v31 = isplit v3
|
|
//! v10 = bxor.i32 v20, v30
|
|
//! v11 = bxor.i32 v21, v31
|
|
//! v1 = iconcat v10, v11
|
|
//! ```
|
|
//!
|
|
//! This local expansion approach still leaves the original `i64` values in the code as operands on
|
|
//! the `split` and `concat` instructions. It also creates a lot of redundant code to clean up as
|
|
//! values are constantly split and concatenated.
|
|
//!
|
|
//! # Optimized splitting
|
|
//!
|
|
//! We can eliminate a lot of the splitting code quite easily. Whenever we need to split a value,
|
|
//! first check if the value is defined by the corresponding concatenation. If so, then just use
|
|
//! the two concatenation inputs directly:
|
|
//!
|
|
//! ```clif
|
|
//! v4 = iadd_imm.i64 v1, 1
|
|
//! ```
|
|
//!
|
|
//! becomes, using the expanded code from above:
|
|
//!
|
|
//! ```clif
|
|
//! v40, v5 = iadd_imm_cout.i32 v10, 1
|
|
//! v6 = bint.i32
|
|
//! v41 = iadd.i32 v11, v6
|
|
//! v4 = iconcat v40, v41
|
|
//! ```
|
|
//!
|
|
//! This means that the `iconcat` instructions defining `v1` and `v4` end up with no uses, so they
|
|
//! can be trivially deleted by a dead code elimination pass.
|
|
//!
|
|
//! # EBB arguments
|
|
//!
|
|
//! If all instructions that produce an `i64` value are legalized as above, we will eventually end
|
|
//! up with no `i64` values anywhere, except for EBB arguments. We can work around this by
|
|
//! iteratively splitting EBB arguments too. That should leave us with no illegal value types
|
|
//! anywhere.
|
|
//!
|
|
//! It is possible to have circular dependencies of EBB arguments that are never used by any real
|
|
//! instructions. These loops will remain in the program.
|
|
|
|
use crate::cursor::{Cursor, CursorPosition, FuncCursor};
|
|
use crate::flowgraph::{BasicBlock, ControlFlowGraph};
|
|
use crate::ir::{self, Ebb, Inst, InstBuilder, InstructionData, Opcode, Type, Value, ValueDef};
|
|
use core::iter;
|
|
use smallvec::SmallVec;
|
|
use std::vec::Vec;
|
|
|
|
/// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values
|
|
/// if possible.
|
|
pub fn isplit(
|
|
func: &mut ir::Function,
|
|
cfg: &ControlFlowGraph,
|
|
pos: CursorPosition,
|
|
srcloc: ir::SourceLoc,
|
|
value: Value,
|
|
) -> (Value, Value) {
|
|
split_any(func, cfg, pos, srcloc, value, Opcode::Iconcat)
|
|
}
|
|
|
|
/// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if
|
|
/// possible.
|
|
pub fn vsplit(
|
|
func: &mut ir::Function,
|
|
cfg: &ControlFlowGraph,
|
|
pos: CursorPosition,
|
|
srcloc: ir::SourceLoc,
|
|
value: Value,
|
|
) -> (Value, Value) {
|
|
split_any(func, cfg, pos, srcloc, value, Opcode::Vconcat)
|
|
}
|
|
|
|
/// After splitting an EBB argument, we need to go back and fix up all of the predecessor
|
|
/// instructions. This is potentially a recursive operation, but we don't implement it recursively
|
|
/// since that could use up too muck stack.
|
|
///
|
|
/// Instead, the repairs are deferred and placed on a work list in stack form.
|
|
struct Repair {
|
|
concat: Opcode,
|
|
// The argument type after splitting.
|
|
split_type: Type,
|
|
// The destination EBB whose arguments have been split.
|
|
ebb: Ebb,
|
|
// Number of the original EBB argument which has been replaced by the low part.
|
|
num: usize,
|
|
// Number of the new EBB argument which represents the high part after the split.
|
|
hi_num: usize,
|
|
}
|
|
|
|
/// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode.
|
|
fn split_any(
|
|
func: &mut ir::Function,
|
|
cfg: &ControlFlowGraph,
|
|
pos: CursorPosition,
|
|
srcloc: ir::SourceLoc,
|
|
value: Value,
|
|
concat: Opcode,
|
|
) -> (Value, Value) {
|
|
let mut repairs = Vec::new();
|
|
let pos = &mut FuncCursor::new(func).at_position(pos).with_srcloc(srcloc);
|
|
let result = split_value(pos, value, concat, &mut repairs);
|
|
|
|
perform_repairs(pos, cfg, repairs);
|
|
|
|
result
|
|
}
|
|
|
|
pub fn split_ebb_params(func: &mut ir::Function, cfg: &ControlFlowGraph, ebb: Ebb) {
|
|
let pos = &mut FuncCursor::new(func).at_top(ebb);
|
|
let ebb_params = pos.func.dfg.ebb_params(ebb);
|
|
|
|
// Add further splittable types here.
|
|
fn type_requires_splitting(ty: Type) -> bool {
|
|
ty == ir::types::I128
|
|
}
|
|
|
|
// A shortcut. If none of the param types require splitting, exit now. This helps because
|
|
// the loop below necessarily has to copy the ebb params into a new vector, so it's better to
|
|
// avoid doing so when possible.
|
|
if !ebb_params
|
|
.iter()
|
|
.any(|ebb_param| type_requires_splitting(pos.func.dfg.value_type(*ebb_param)))
|
|
{
|
|
return;
|
|
}
|
|
|
|
let mut repairs = Vec::new();
|
|
for (num, ebb_param) in ebb_params.to_vec().into_iter().enumerate() {
|
|
if !type_requires_splitting(pos.func.dfg.value_type(ebb_param)) {
|
|
continue;
|
|
}
|
|
|
|
split_ebb_param(pos, ebb, num, ebb_param, Opcode::Iconcat, &mut repairs);
|
|
}
|
|
|
|
perform_repairs(pos, cfg, repairs);
|
|
}
|
|
|
|
fn perform_repairs(pos: &mut FuncCursor, cfg: &ControlFlowGraph, mut repairs: Vec<Repair>) {
|
|
// We have split the value requested, and now we may need to fix some EBB predecessors.
|
|
while let Some(repair) = repairs.pop() {
|
|
for BasicBlock { inst, .. } in cfg.pred_iter(repair.ebb) {
|
|
let branch_opc = pos.func.dfg[inst].opcode();
|
|
debug_assert!(
|
|
branch_opc.is_branch(),
|
|
"Predecessor not a branch: {}",
|
|
pos.func.dfg.display_inst(inst, None)
|
|
);
|
|
let num_fixed_args = branch_opc.constraints().num_fixed_value_arguments();
|
|
let mut args = pos.func.dfg[inst]
|
|
.take_value_list()
|
|
.expect("Branches must have value lists.");
|
|
let num_args = args.len(&pos.func.dfg.value_lists);
|
|
// Get the old value passed to the EBB argument we're repairing.
|
|
let old_arg = args
|
|
.get(num_fixed_args + repair.num, &pos.func.dfg.value_lists)
|
|
.expect("Too few branch arguments");
|
|
|
|
// It's possible that the CFG's predecessor list has duplicates. Detect them here.
|
|
if pos.func.dfg.value_type(old_arg) == repair.split_type {
|
|
pos.func.dfg[inst].put_value_list(args);
|
|
continue;
|
|
}
|
|
|
|
// Split the old argument, possibly causing more repairs to be scheduled.
|
|
pos.goto_inst(inst);
|
|
let (lo, hi) = split_value(pos, old_arg, repair.concat, &mut repairs);
|
|
|
|
// The `lo` part replaces the original argument.
|
|
*args
|
|
.get_mut(num_fixed_args + repair.num, &mut pos.func.dfg.value_lists)
|
|
.unwrap() = lo;
|
|
|
|
// The `hi` part goes at the end. Since multiple repairs may have been scheduled to the
|
|
// same EBB, there could be multiple arguments missing.
|
|
if num_args > num_fixed_args + repair.hi_num {
|
|
*args
|
|
.get_mut(
|
|
num_fixed_args + repair.hi_num,
|
|
&mut pos.func.dfg.value_lists,
|
|
)
|
|
.unwrap() = hi;
|
|
} else {
|
|
// We need to append one or more arguments. If we're adding more than one argument,
|
|
// there must be pending repairs on the stack that will fill in the correct values
|
|
// instead of `hi`.
|
|
args.extend(
|
|
iter::repeat(hi).take(1 + num_fixed_args + repair.hi_num - num_args),
|
|
&mut pos.func.dfg.value_lists,
|
|
);
|
|
}
|
|
|
|
// Put the value list back after manipulating it.
|
|
pos.func.dfg[inst].put_value_list(args);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Split a single value using the integer or vector semantics given by the `concat` opcode.
|
|
///
|
|
/// If the value is defined by a `concat` instruction, just reuse the operand values of that
|
|
/// instruction.
|
|
///
|
|
/// Return the two new values representing the parts of `value`.
|
|
fn split_value(
|
|
pos: &mut FuncCursor,
|
|
value: Value,
|
|
concat: Opcode,
|
|
repairs: &mut Vec<Repair>,
|
|
) -> (Value, Value) {
|
|
let value = pos.func.dfg.resolve_aliases(value);
|
|
let mut reuse = None;
|
|
|
|
match pos.func.dfg.value_def(value) {
|
|
ValueDef::Result(inst, num) => {
|
|
// This is an instruction result. See if the value was created by a `concat`
|
|
// instruction.
|
|
if let InstructionData::Binary { opcode, args, .. } = pos.func.dfg[inst] {
|
|
debug_assert_eq!(num, 0);
|
|
if opcode == concat {
|
|
reuse = Some((args[0], args[1]));
|
|
}
|
|
}
|
|
}
|
|
ValueDef::Param(ebb, num) => {
|
|
// This is an EBB parameter. We can split the parameter value unless this is the entry
|
|
// block.
|
|
if pos.func.layout.entry_block() != Some(ebb) {
|
|
reuse = Some(split_ebb_param(pos, ebb, num, value, concat, repairs));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Did the code above succeed in finding values we can reuse?
|
|
if let Some(pair) = reuse {
|
|
pair
|
|
} else {
|
|
// No, we'll just have to insert the requested split instruction at `pos`. Note that `pos`
|
|
// has not been moved by the EBB argument code above when `reuse` is `None`.
|
|
match concat {
|
|
Opcode::Iconcat => pos.ins().isplit(value),
|
|
Opcode::Vconcat => pos.ins().vsplit(value),
|
|
_ => panic!("Unhandled concat opcode: {}", concat),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn split_ebb_param(
|
|
pos: &mut FuncCursor,
|
|
ebb: Ebb,
|
|
param_num: usize,
|
|
value: Value,
|
|
concat: Opcode,
|
|
repairs: &mut Vec<Repair>,
|
|
) -> (Value, Value) {
|
|
// We are going to replace the parameter at `num` with two new arguments.
|
|
// Determine the new value types.
|
|
let ty = pos.func.dfg.value_type(value);
|
|
let split_type = match concat {
|
|
Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"),
|
|
Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"),
|
|
_ => panic!("Unhandled concat opcode: {}", concat),
|
|
};
|
|
|
|
// Since the `repairs` stack potentially contains other parameter numbers for
|
|
// `ebb`, avoid shifting and renumbering EBB parameters. It could invalidate other
|
|
// `repairs` entries.
|
|
//
|
|
// Replace the original `value` with the low part, and append the high part at the
|
|
// end of the argument list.
|
|
let lo = pos.func.dfg.replace_ebb_param(value, split_type);
|
|
let hi_num = pos.func.dfg.num_ebb_params(ebb);
|
|
let hi = pos.func.dfg.append_ebb_param(ebb, split_type);
|
|
|
|
// Now the original value is dangling. Insert a concatenation instruction that can
|
|
// compute it from the two new parameters. This also serves as a record of what we
|
|
// did so a future call to this function doesn't have to redo the work.
|
|
//
|
|
// Note that it is safe to move `pos` here since `reuse` was set above, so we don't
|
|
// need to insert a split instruction before returning.
|
|
pos.goto_first_inst(ebb);
|
|
pos.ins()
|
|
.with_result(value)
|
|
.Binary(concat, split_type, lo, hi);
|
|
|
|
// Finally, splitting the EBB parameter is not enough. We also have to repair all
|
|
// of the predecessor instructions that branch here.
|
|
add_repair(concat, split_type, ebb, param_num, hi_num, repairs);
|
|
|
|
(lo, hi)
|
|
}
|
|
|
|
// Add a repair entry to the work list.
|
|
fn add_repair(
|
|
concat: Opcode,
|
|
split_type: Type,
|
|
ebb: Ebb,
|
|
num: usize,
|
|
hi_num: usize,
|
|
repairs: &mut Vec<Repair>,
|
|
) {
|
|
repairs.push(Repair {
|
|
concat,
|
|
split_type,
|
|
ebb,
|
|
num,
|
|
hi_num,
|
|
});
|
|
}
|
|
|
|
/// Strip concat-split chains. Return a simpler way of computing the same value.
|
|
///
|
|
/// Given this input:
|
|
///
|
|
/// ```clif
|
|
/// v10 = iconcat v1, v2
|
|
/// v11, v12 = isplit v10
|
|
/// ```
|
|
///
|
|
/// This function resolves `v11` to `v1` and `v12` to `v2`.
|
|
fn resolve_splits(dfg: &ir::DataFlowGraph, value: Value) -> Value {
|
|
let value = dfg.resolve_aliases(value);
|
|
|
|
// Deconstruct a split instruction.
|
|
let split_res;
|
|
let concat_opc;
|
|
let split_arg;
|
|
if let ValueDef::Result(inst, num) = dfg.value_def(value) {
|
|
split_res = num;
|
|
concat_opc = match dfg[inst].opcode() {
|
|
Opcode::Isplit => Opcode::Iconcat,
|
|
Opcode::Vsplit => Opcode::Vconcat,
|
|
_ => return value,
|
|
};
|
|
split_arg = dfg.inst_args(inst)[0];
|
|
} else {
|
|
return value;
|
|
}
|
|
|
|
// See if split_arg is defined by a concatenation instruction.
|
|
if let ValueDef::Result(inst, _) = dfg.value_def(split_arg) {
|
|
if dfg[inst].opcode() == concat_opc {
|
|
return dfg.inst_args(inst)[split_res];
|
|
}
|
|
}
|
|
|
|
value
|
|
}
|
|
|
|
/// Simplify the arguments to a branch *after* the instructions leading up to the branch have been
|
|
/// legalized.
|
|
///
|
|
/// The branch argument repairs performed by `split_any()` above may be performed on branches that
|
|
/// have not yet been legalized. The repaired arguments can be defined by actual split
|
|
/// instructions in that case.
|
|
///
|
|
/// After legalizing the instructions computing the value that was split, it is likely that we can
|
|
/// avoid depending on the split instruction. Its input probably comes from a concatenation.
|
|
pub fn simplify_branch_arguments(dfg: &mut ir::DataFlowGraph, branch: Inst) {
|
|
let mut new_args = SmallVec::<[Value; 32]>::new();
|
|
|
|
for &arg in dfg.inst_args(branch) {
|
|
let new_arg = resolve_splits(dfg, arg);
|
|
new_args.push(new_arg);
|
|
}
|
|
|
|
dfg.inst_args_mut(branch).copy_from_slice(&new_args);
|
|
}
|