cranelift: Port most of simple_preopt.rs over to the peepmatic DSL
This ports all of the identity, no-op, simplification, and canonicalization related optimizations over from being hand-coded to the `peepmatic` DSL. This does not handle the branch-to-branch optimizations or most of the divide-by-constant optimizations.
This commit is contained in:
Nick Fitzgerald
@@ -10,13 +10,12 @@ use crate::divconst_magic_numbers::{MS32, MS64, MU32, MU64};
|
||||
use crate::flowgraph::ControlFlowGraph;
|
||||
use crate::ir::{
|
||||
condcodes::{CondCode, IntCC},
|
||||
dfg::ValueDef,
|
||||
immediates,
|
||||
instructions::{Opcode, ValueList},
|
||||
types::{I16, I32, I64, I8},
|
||||
instructions::Opcode,
|
||||
types::{I32, I64},
|
||||
Block, DataFlowGraph, Function, Inst, InstBuilder, InstructionData, Type, Value,
|
||||
};
|
||||
use crate::isa::TargetIsa;
|
||||
use crate::peepmatic::ValueOrInst;
|
||||
use crate::timing;
|
||||
|
||||
#[inline]
|
||||
@@ -183,12 +182,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
|
||||
|
||||
// U32 div by 1: identity
|
||||
// U32 rem by 1: zero
|
||||
DivRemByConstInfo::DivU32(n1, 1) | DivRemByConstInfo::RemU32(n1, 1) => {
|
||||
if is_rem {
|
||||
pos.func.dfg.replace(inst).iconst(I32, 0);
|
||||
} else {
|
||||
replace_single_result_with_alias(&mut pos.func.dfg, inst, n1);
|
||||
}
|
||||
DivRemByConstInfo::DivU32(_, 1) | DivRemByConstInfo::RemU32(_, 1) => {
|
||||
unreachable!("unsigned division and remainder by one is handled in `preopt.peepmatic`");
|
||||
}
|
||||
|
||||
// U32 div, rem by a power-of-2
|
||||
@@ -203,7 +198,10 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
|
||||
let mask = (1u64 << k) - 1;
|
||||
pos.func.dfg.replace(inst).band_imm(n1, mask as i64);
|
||||
} else {
|
||||
pos.func.dfg.replace(inst).ushr_imm(n1, k as i64);
|
||||
unreachable!(
|
||||
"unsigned division by a power of two is handled in \
|
||||
`preopt.peepmatic`"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -253,12 +251,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
|
||||
|
||||
// U64 div by 1: identity
|
||||
// U64 rem by 1: zero
|
||||
DivRemByConstInfo::DivU64(n1, 1) | DivRemByConstInfo::RemU64(n1, 1) => {
|
||||
if is_rem {
|
||||
pos.func.dfg.replace(inst).iconst(I64, 0);
|
||||
} else {
|
||||
replace_single_result_with_alias(&mut pos.func.dfg, inst, n1);
|
||||
}
|
||||
DivRemByConstInfo::DivU64(_, 1) | DivRemByConstInfo::RemU64(_, 1) => {
|
||||
unreachable!("unsigned division and remainder by one is handled in `preopt.peepmatic`");
|
||||
}
|
||||
|
||||
// U64 div, rem by a power-of-2
|
||||
@@ -273,7 +267,9 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
|
||||
let mask = (1u64 << k) - 1;
|
||||
pos.func.dfg.replace(inst).band_imm(n1, mask as i64);
|
||||
} else {
|
||||
pos.func.dfg.replace(inst).ushr_imm(n1, k as i64);
|
||||
unreachable!(
|
||||
"unsigned division by a power of two is handled in `preopt.peepmatic`"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -326,12 +322,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
|
||||
|
||||
// S32 div by 1: identity
|
||||
// S32 rem by 1: zero
|
||||
DivRemByConstInfo::DivS32(n1, 1) | DivRemByConstInfo::RemS32(n1, 1) => {
|
||||
if is_rem {
|
||||
pos.func.dfg.replace(inst).iconst(I32, 0);
|
||||
} else {
|
||||
replace_single_result_with_alias(&mut pos.func.dfg, inst, n1);
|
||||
}
|
||||
DivRemByConstInfo::DivS32(_, 1) | DivRemByConstInfo::RemS32(_, 1) => {
|
||||
unreachable!("signed division and remainder by one is handled in `preopt.peepmatic`");
|
||||
}
|
||||
|
||||
DivRemByConstInfo::DivS32(n1, d) | DivRemByConstInfo::RemS32(n1, d) => {
|
||||
@@ -401,12 +393,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
|
||||
|
||||
// S64 div by 1: identity
|
||||
// S64 rem by 1: zero
|
||||
DivRemByConstInfo::DivS64(n1, 1) | DivRemByConstInfo::RemS64(n1, 1) => {
|
||||
if is_rem {
|
||||
pos.func.dfg.replace(inst).iconst(I64, 0);
|
||||
} else {
|
||||
replace_single_result_with_alias(&mut pos.func.dfg, inst, n1);
|
||||
}
|
||||
DivRemByConstInfo::DivS64(_, 1) | DivRemByConstInfo::RemS64(_, 1) => {
|
||||
unreachable!("division and remaineder by one are handled in `preopt.peepmatic`");
|
||||
}
|
||||
|
||||
DivRemByConstInfo::DivS64(n1, d) | DivRemByConstInfo::RemS64(n1, d) => {
|
||||
@@ -468,340 +456,6 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn resolve_imm64_value(dfg: &DataFlowGraph, value: Value) -> Option<immediates::Imm64> {
|
||||
if let ValueDef::Result(candidate_inst, _) = dfg.value_def(value) {
|
||||
if let InstructionData::UnaryImm {
|
||||
opcode: Opcode::Iconst,
|
||||
imm,
|
||||
} = dfg[candidate_inst]
|
||||
{
|
||||
return Some(imm);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Try to transform [(x << N) >> N] into a (un)signed-extending move.
|
||||
/// Returns true if the final instruction has been converted to such a move.
|
||||
fn try_fold_extended_move(
|
||||
pos: &mut FuncCursor,
|
||||
inst: Inst,
|
||||
opcode: Opcode,
|
||||
arg: Value,
|
||||
imm: immediates::Imm64,
|
||||
) -> bool {
|
||||
if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) {
|
||||
if let InstructionData::BinaryImm {
|
||||
opcode: Opcode::IshlImm,
|
||||
arg: prev_arg,
|
||||
imm: prev_imm,
|
||||
} = &pos.func.dfg[arg_inst]
|
||||
{
|
||||
if imm != *prev_imm {
|
||||
return false;
|
||||
}
|
||||
|
||||
let dest_ty = pos.func.dfg.ctrl_typevar(inst);
|
||||
if dest_ty != pos.func.dfg.ctrl_typevar(arg_inst) || !dest_ty.is_int() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let imm_bits: i64 = imm.into();
|
||||
let ireduce_ty = match (dest_ty.lane_bits() as i64).wrapping_sub(imm_bits) {
|
||||
8 => I8,
|
||||
16 => I16,
|
||||
32 => I32,
|
||||
_ => return false,
|
||||
};
|
||||
let ireduce_ty = ireduce_ty.by(dest_ty.lane_count()).unwrap();
|
||||
|
||||
// This becomes a no-op, since ireduce_ty has a smaller lane width than
|
||||
// the argument type (also the destination type).
|
||||
let arg = *prev_arg;
|
||||
let narrower_arg = pos.ins().ireduce(ireduce_ty, arg);
|
||||
|
||||
if opcode == Opcode::UshrImm {
|
||||
pos.func.dfg.replace(inst).uextend(dest_ty, narrower_arg);
|
||||
} else {
|
||||
pos.func.dfg.replace(inst).sextend(dest_ty, narrower_arg);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Apply basic simplifications.
|
||||
///
|
||||
/// This folds constants with arithmetic to form `_imm` instructions, and other minor
|
||||
/// simplifications.
|
||||
///
|
||||
/// Doesn't apply some simplifications if the native word width (in bytes) is smaller than the
|
||||
/// controlling type's width of the instruction. This would result in an illegal instruction that
|
||||
/// would likely be expanded back into an instruction on smaller types with the same initial
|
||||
/// opcode, creating unnecessary churn.
|
||||
fn simplify(pos: &mut FuncCursor, inst: Inst, native_word_width: u32) {
|
||||
match pos.func.dfg[inst] {
|
||||
InstructionData::Binary { opcode, args } => {
|
||||
if let Some(mut imm) = resolve_imm64_value(&pos.func.dfg, args[1]) {
|
||||
let new_opcode = match opcode {
|
||||
Opcode::Iadd => Opcode::IaddImm,
|
||||
Opcode::Imul => Opcode::ImulImm,
|
||||
Opcode::Sdiv => Opcode::SdivImm,
|
||||
Opcode::Udiv => Opcode::UdivImm,
|
||||
Opcode::Srem => Opcode::SremImm,
|
||||
Opcode::Urem => Opcode::UremImm,
|
||||
Opcode::Band => Opcode::BandImm,
|
||||
Opcode::Bor => Opcode::BorImm,
|
||||
Opcode::Bxor => Opcode::BxorImm,
|
||||
Opcode::Rotl => Opcode::RotlImm,
|
||||
Opcode::Rotr => Opcode::RotrImm,
|
||||
Opcode::Ishl => Opcode::IshlImm,
|
||||
Opcode::Ushr => Opcode::UshrImm,
|
||||
Opcode::Sshr => Opcode::SshrImm,
|
||||
Opcode::Isub => {
|
||||
imm = imm.wrapping_neg();
|
||||
Opcode::IaddImm
|
||||
}
|
||||
Opcode::Ifcmp => Opcode::IfcmpImm,
|
||||
_ => return,
|
||||
};
|
||||
let ty = pos.func.dfg.ctrl_typevar(inst);
|
||||
if ty.bytes() <= native_word_width {
|
||||
pos.func
|
||||
.dfg
|
||||
.replace(inst)
|
||||
.BinaryImm(new_opcode, ty, imm, args[0]);
|
||||
|
||||
// Repeat for BinaryImm simplification.
|
||||
simplify(pos, inst, native_word_width);
|
||||
}
|
||||
} else if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[0]) {
|
||||
let new_opcode = match opcode {
|
||||
Opcode::Iadd => Opcode::IaddImm,
|
||||
Opcode::Imul => Opcode::ImulImm,
|
||||
Opcode::Band => Opcode::BandImm,
|
||||
Opcode::Bor => Opcode::BorImm,
|
||||
Opcode::Bxor => Opcode::BxorImm,
|
||||
Opcode::Isub => Opcode::IrsubImm,
|
||||
_ => return,
|
||||
};
|
||||
let ty = pos.func.dfg.ctrl_typevar(inst);
|
||||
if ty.bytes() <= native_word_width {
|
||||
pos.func
|
||||
.dfg
|
||||
.replace(inst)
|
||||
.BinaryImm(new_opcode, ty, imm, args[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
InstructionData::Unary { opcode, arg } => {
|
||||
if let Opcode::AdjustSpDown = opcode {
|
||||
if let Some(imm) = resolve_imm64_value(&pos.func.dfg, arg) {
|
||||
// Note this works for both positive and negative immediate values.
|
||||
pos.func.dfg.replace(inst).adjust_sp_down_imm(imm);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
InstructionData::BinaryImm { opcode, arg, imm } => {
|
||||
let ty = pos.func.dfg.ctrl_typevar(inst);
|
||||
|
||||
let mut arg = arg;
|
||||
let mut imm = imm;
|
||||
match opcode {
|
||||
Opcode::IaddImm
|
||||
| Opcode::ImulImm
|
||||
| Opcode::BorImm
|
||||
| Opcode::BandImm
|
||||
| Opcode::BxorImm => {
|
||||
// Fold binary_op(C2, binary_op(C1, x)) into binary_op(binary_op(C1, C2), x)
|
||||
if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) {
|
||||
if let InstructionData::BinaryImm {
|
||||
opcode: prev_opcode,
|
||||
arg: prev_arg,
|
||||
imm: prev_imm,
|
||||
} = &pos.func.dfg[arg_inst]
|
||||
{
|
||||
if opcode == *prev_opcode && ty == pos.func.dfg.ctrl_typevar(arg_inst) {
|
||||
let lhs: i64 = imm.into();
|
||||
let rhs: i64 = (*prev_imm).into();
|
||||
let new_imm = match opcode {
|
||||
Opcode::BorImm => lhs | rhs,
|
||||
Opcode::BandImm => lhs & rhs,
|
||||
Opcode::BxorImm => lhs ^ rhs,
|
||||
Opcode::IaddImm => lhs.wrapping_add(rhs),
|
||||
Opcode::ImulImm => lhs.wrapping_mul(rhs),
|
||||
_ => panic!("can't happen"),
|
||||
};
|
||||
let new_imm = immediates::Imm64::from(new_imm);
|
||||
let new_arg = *prev_arg;
|
||||
pos.func
|
||||
.dfg
|
||||
.replace(inst)
|
||||
.BinaryImm(opcode, ty, new_imm, new_arg);
|
||||
imm = new_imm;
|
||||
arg = new_arg;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Opcode::UshrImm | Opcode::SshrImm => {
|
||||
if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width
|
||||
&& try_fold_extended_move(pos, inst, opcode, arg, imm)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
_ => {}
|
||||
};
|
||||
|
||||
// Replace operations that are no-ops.
|
||||
match (opcode, imm.into()) {
|
||||
(Opcode::IaddImm, 0)
|
||||
| (Opcode::ImulImm, 1)
|
||||
| (Opcode::SdivImm, 1)
|
||||
| (Opcode::UdivImm, 1)
|
||||
| (Opcode::BorImm, 0)
|
||||
| (Opcode::BandImm, -1)
|
||||
| (Opcode::BxorImm, 0)
|
||||
| (Opcode::RotlImm, 0)
|
||||
| (Opcode::RotrImm, 0)
|
||||
| (Opcode::IshlImm, 0)
|
||||
| (Opcode::UshrImm, 0)
|
||||
| (Opcode::SshrImm, 0) => {
|
||||
// Alias the result value with the original argument.
|
||||
replace_single_result_with_alias(&mut pos.func.dfg, inst, arg);
|
||||
}
|
||||
(Opcode::ImulImm, 0) | (Opcode::BandImm, 0) => {
|
||||
// Replace by zero.
|
||||
pos.func.dfg.replace(inst).iconst(ty, 0);
|
||||
}
|
||||
(Opcode::BorImm, -1) => {
|
||||
// Replace by minus one.
|
||||
pos.func.dfg.replace(inst).iconst(ty, -1);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
InstructionData::IntCompare { opcode, cond, args } => {
|
||||
debug_assert_eq!(opcode, Opcode::Icmp);
|
||||
if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[1]) {
|
||||
if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width {
|
||||
pos.func.dfg.replace(inst).icmp_imm(cond, args[0], imm);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
InstructionData::CondTrap { .. }
|
||||
| InstructionData::Branch { .. }
|
||||
| InstructionData::Ternary {
|
||||
opcode: Opcode::Select,
|
||||
..
|
||||
} => {
|
||||
// Fold away a redundant `bint`.
|
||||
let condition_def = {
|
||||
let args = pos.func.dfg.inst_args(inst);
|
||||
pos.func.dfg.value_def(args[0])
|
||||
};
|
||||
if let ValueDef::Result(def_inst, _) = condition_def {
|
||||
if let InstructionData::Unary {
|
||||
opcode: Opcode::Bint,
|
||||
arg: bool_val,
|
||||
} = pos.func.dfg[def_inst]
|
||||
{
|
||||
let args = pos.func.dfg.inst_args_mut(inst);
|
||||
args[0] = bool_val;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
struct BranchOptInfo {
|
||||
br_inst: Inst,
|
||||
cmp_arg: Value,
|
||||
args: ValueList,
|
||||
new_opcode: Opcode,
|
||||
}
|
||||
|
||||
/// Fold comparisons into branch operations when possible.
|
||||
///
|
||||
/// This matches against operations which compare against zero, then use the
|
||||
/// result in a `brz` or `brnz` branch. It folds those two operations into a
|
||||
/// single `brz` or `brnz`.
|
||||
fn branch_opt(pos: &mut FuncCursor, inst: Inst) {
|
||||
let mut info = if let InstructionData::Branch {
|
||||
opcode: br_opcode,
|
||||
args: ref br_args,
|
||||
..
|
||||
} = pos.func.dfg[inst]
|
||||
{
|
||||
let first_arg = {
|
||||
let args = pos.func.dfg.inst_args(inst);
|
||||
args[0]
|
||||
};
|
||||
|
||||
let icmp_inst = if let ValueDef::Result(icmp_inst, _) = pos.func.dfg.value_def(first_arg) {
|
||||
icmp_inst
|
||||
} else {
|
||||
return;
|
||||
};
|
||||
|
||||
if let InstructionData::IntCompareImm {
|
||||
opcode: Opcode::IcmpImm,
|
||||
arg: cmp_arg,
|
||||
cond: cmp_cond,
|
||||
imm: cmp_imm,
|
||||
} = pos.func.dfg[icmp_inst]
|
||||
{
|
||||
let cmp_imm: i64 = cmp_imm.into();
|
||||
if cmp_imm != 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
// icmp_imm returns non-zero when the comparison is true. So, if
|
||||
// we're branching on zero, we need to invert the condition.
|
||||
let cond = match br_opcode {
|
||||
Opcode::Brz => cmp_cond.inverse(),
|
||||
Opcode::Brnz => cmp_cond,
|
||||
_ => return,
|
||||
};
|
||||
|
||||
let new_opcode = match cond {
|
||||
IntCC::Equal => Opcode::Brz,
|
||||
IntCC::NotEqual => Opcode::Brnz,
|
||||
_ => return,
|
||||
};
|
||||
|
||||
BranchOptInfo {
|
||||
br_inst: inst,
|
||||
cmp_arg,
|
||||
args: br_args.clone(),
|
||||
new_opcode,
|
||||
}
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
return;
|
||||
};
|
||||
|
||||
info.args.as_mut_slice(&mut pos.func.dfg.value_lists)[0] = info.cmp_arg;
|
||||
if let InstructionData::Branch { ref mut opcode, .. } = pos.func.dfg[info.br_inst] {
|
||||
*opcode = info.new_opcode;
|
||||
} else {
|
||||
panic!();
|
||||
}
|
||||
}
|
||||
|
||||
enum BranchOrderKind {
|
||||
BrzToBrnz(Value),
|
||||
BrnzToBrz(Value),
|
||||
@@ -945,14 +599,20 @@ fn branch_order(pos: &mut FuncCursor, cfg: &mut ControlFlowGraph, block: Block,
|
||||
}
|
||||
|
||||
/// The main pre-opt pass.
|
||||
pub fn do_preopt(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &dyn TargetIsa) {
|
||||
pub fn do_preopt<'func, 'isa>(
|
||||
func: &'func mut Function,
|
||||
cfg: &mut ControlFlowGraph,
|
||||
isa: &'isa dyn TargetIsa,
|
||||
) {
|
||||
let _tt = timing::preopt();
|
||||
|
||||
let mut pos = FuncCursor::new(func);
|
||||
let native_word_width = isa.pointer_bytes();
|
||||
let mut preopt = crate::peepmatic::preopt(isa);
|
||||
|
||||
while let Some(block) = pos.next_block() {
|
||||
while let Some(inst) = pos.next_inst() {
|
||||
// Apply basic simplifications.
|
||||
simplify(&mut pos, inst, native_word_width as u32);
|
||||
preopt.apply_all(&mut pos, ValueOrInst::Inst(inst));
|
||||
let inst = pos.current_inst().unwrap();
|
||||
|
||||
// Try to transform divide-by-constant into simpler operations.
|
||||
if let Some(divrem_info) = get_div_info(inst, &pos.func.dfg) {
|
||||
@@ -960,7 +620,6 @@ pub fn do_preopt(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &dyn Targ
|
||||
continue;
|
||||
}
|
||||
|
||||
branch_opt(&mut pos, inst);
|
||||
branch_order(&mut pos, cfg, block, inst);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user