cranelift: Port most of simple_preopt.rs over to the peepmatic DSL

This ports all of the identity, no-op, simplification, and canonicalization
related optimizations over from being hand-coded to the `peepmatic` DSL. This
does not handle the branch-to-branch optimizations or most of the
divide-by-constant optimizations.

cranelift/codegen/src/ir/dfg.rs

@@ -234,11 +234,7 @@ impl DataFlowGraph {
 
     /// Get the type of a value.
     pub fn value_type(&self, v: Value) -> Type {
-        match self.values[v] {
-            ValueData::Inst { ty, .. }
-            | ValueData::Param { ty, .. }
-            | ValueData::Alias { ty, .. } => ty,
-        }
+        self.values[v].ty()
     }
 
     /// Get the definition of a value.
@@ -383,9 +379,14 @@ pub enum ValueDef {
 impl ValueDef {
     /// Unwrap the instruction where the value was defined, or panic.
     pub fn unwrap_inst(&self) -> Inst {
+        self.inst().expect("Value is not an instruction result")
+    }
+
+    /// Get the instruction where the value was defined, if any.
+    pub fn inst(&self) -> Option<Inst> {
         match *self {
-            Self::Result(inst, _) => inst,
-            _ => panic!("Value is not an instruction result"),
+            Self::Result(inst, _) => Some(inst),
+            _ => None,
         }
     }
 
@@ -428,6 +429,16 @@ enum ValueData {
     Alias { ty: Type, original: Value },
 }
 
+impl ValueData {
+    fn ty(&self) -> Type {
+        match *self {
+            ValueData::Inst { ty, .. }
+            | ValueData::Param { ty, .. }
+            | ValueData::Alias { ty, .. } => ty,
+        }
+    }
+}
+
 /// Instructions.
 ///
 impl DataFlowGraph {

cranelift/codegen/src/ir/function.rs

@@ -308,6 +308,30 @@ impl Function {
         // function, assume it is not a leaf.
         self.dfg.signatures.is_empty()
     }
+
+    /// Replace the `dst` instruction's data with the `src` instruction's data
+    /// and then remove `src`.
+    ///
+    /// `src` and its result values should not be used at all, as any uses would
+    /// be left dangling after calling this method.
+    ///
+    /// `src` and `dst` must have the same number of resulting values, and
+    /// `src`'s i^th value must have the same type as `dst`'s i^th value.
+    pub fn transplant_inst(&mut self, dst: Inst, src: Inst) {
+        debug_assert_eq!(
+            self.dfg.inst_results(dst).len(),
+            self.dfg.inst_results(src).len()
+        );
+        debug_assert!(self
+            .dfg
+            .inst_results(dst)
+            .iter()
+            .zip(self.dfg.inst_results(src))
+            .all(|(a, b)| self.dfg.value_type(*a) == self.dfg.value_type(*b)));
+
+        self.dfg[dst] = self.dfg[src].clone();
+        self.layout.remove_inst(src);
+    }
 }
 
 /// Additional annotations for function display.

cranelift/codegen/src/ir/instructions.rs

@@ -11,9 +11,7 @@ use core::fmt::{self, Display, Formatter};
 use core::ops::{Deref, DerefMut};
 use core::str::FromStr;
 
-use crate::ir;
-use crate::ir::types;
-use crate::ir::{Block, FuncRef, JumpTable, SigRef, Type, Value};
+use crate::ir::{self, trapcode::TrapCode, types, Block, FuncRef, JumpTable, SigRef, Type, Value};
 use crate::isa;
 
 use crate::bitset::BitSet;
@@ -257,6 +255,30 @@ impl InstructionData {
         }
     }
 
+    /// If this is a trapping instruction, get its trap code. Otherwise, return
+    /// `None`.
+    pub fn trap_code(&self) -> Option<TrapCode> {
+        match *self {
+            Self::CondTrap { code, .. }
+            | Self::FloatCondTrap { code, .. }
+            | Self::IntCondTrap { code, .. }
+            | Self::Trap { code, .. } => Some(code),
+            _ => None,
+        }
+    }
+
+    /// If this is a trapping instruction, get an exclusive reference to its
+    /// trap code. Otherwise, return `None`.
+    pub fn trap_code_mut(&mut self) -> Option<&mut TrapCode> {
+        match self {
+            Self::CondTrap { code, .. }
+            | Self::FloatCondTrap { code, .. }
+            | Self::IntCondTrap { code, .. }
+            | Self::Trap { code, .. } => Some(code),
+            _ => None,
+        }
+    }
+
     /// Return information about a call instruction.
     ///
     /// Any instruction that can call another function reveals its call signature here.

cranelift/codegen/src/lib.rs

@@ -101,6 +101,7 @@ mod licm;
 mod nan_canonicalization;
 mod num_uses;
 mod partition_slice;
+mod peepmatic;
 mod postopt;
 mod predicates;
 mod redundant_reload_remover;

cranelift/codegen/src/peepmatic.rs

@@ -0,0 +1,847 @@
+//! Glue for working with `peepmatic`-generated peephole optimizers.
+
+use crate::cursor::{Cursor, FuncCursor};
+use crate::ir::{
+    dfg::DataFlowGraph,
+    entities::{Inst, Value},
+    immediates::{Imm64, Uimm64},
+    instructions::{InstructionData, Opcode},
+    types, InstBuilder,
+};
+use crate::isa::TargetIsa;
+use cranelift_codegen_shared::condcodes::IntCC;
+use peepmatic_runtime::{
+    cc::ConditionCode,
+    instruction_set::InstructionSet,
+    operator::Operator,
+    part::{Constant, Part},
+    paths::Path,
+    r#type::{BitWidth, Kind, Type},
+    PeepholeOptimizations, PeepholeOptimizer,
+};
+use std::boxed::Box;
+use std::convert::{TryFrom, TryInto};
+use std::ptr;
+use std::sync::atomic::{AtomicPtr, Ordering};
+
+/// Get the `preopt.peepmatic` peephole optimizer.
+pub(crate) fn preopt<'a, 'b>(
+    isa: &'b dyn TargetIsa,
+) -> PeepholeOptimizer<'static, 'a, &'b dyn TargetIsa> {
+    static SERIALIZED: &[u8] = include_bytes!("preopt.serialized");
+
+    // Once initialized, this must never be re-assigned. The initialized value
+    // is semantically "static data" and is intentionally leaked for the whole
+    // program's lifetime.
+    static DESERIALIZED: AtomicPtr<PeepholeOptimizations> = AtomicPtr::new(ptr::null_mut());
+
+    // If `DESERIALIZED` has already been initialized, then just use it.
+    let ptr = DESERIALIZED.load(Ordering::SeqCst);
+    if let Some(peep_opts) = unsafe { ptr.as_ref() } {
+        return peep_opts.optimizer(isa);
+    }
+
+    // Otherwise, if `DESERIALIZED` hasn't been initialized, then we need to
+    // deserialize the peephole optimizations and initialize it. However,
+    // another thread could be doing the same thing concurrently, so there is a
+    // race to see who initializes `DESERIALIZED` first, and we need to be
+    // prepared to both win or lose that race.
+    let peep_opts = PeepholeOptimizations::deserialize(SERIALIZED)
+        .expect("should always be able to deserialize `preopt.serialized`");
+    let peep_opts = Box::into_raw(Box::new(peep_opts));
+
+    // Only update `DESERIALIZE` if it is still null, attempting to perform the
+    // one-time transition from null -> non-null.
+    if DESERIALIZED
+        .compare_and_swap(ptr::null_mut(), peep_opts, Ordering::SeqCst)
+        .is_null()
+    {
+        // We won the race to initialize `DESERIALIZED`.
+        debug_assert_eq!(DESERIALIZED.load(Ordering::SeqCst), peep_opts);
+        let peep_opts = unsafe { &*peep_opts };
+        return peep_opts.optimizer(isa);
+    }
+
+    // We lost the race to initialize `DESERIALIZED`. Drop our no-longer-needed
+    // instance of `peep_opts` and get the pointer to the instance that won the
+    // race.
+    let _ = unsafe { Box::from_raw(peep_opts) };
+    let peep_opts = DESERIALIZED.load(Ordering::SeqCst);
+    let peep_opts = unsafe { peep_opts.as_ref().unwrap() };
+    peep_opts.optimizer(isa)
+}
+
+/// Either a `Value` or an `Inst`.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ValueOrInst {
+    Value(Value),
+    Inst(Inst),
+}
+
+impl ValueOrInst {
+    /// Get the underlying `Value` if any.
+    pub fn value(&self) -> Option<Value> {
+        match *self {
+            Self::Value(v) => Some(v),
+            Self::Inst(_) => None,
+        }
+    }
+
+    /// Get the underlying `Inst` if any.
+    pub fn inst(&self) -> Option<Inst> {
+        match *self {
+            Self::Inst(i) => Some(i),
+            Self::Value(_) => None,
+        }
+    }
+
+    /// Unwrap the underlying `Value`, panicking if it is not a `Value.
+    pub fn unwrap_value(&self) -> Value {
+        self.value().unwrap()
+    }
+
+    /// Unwrap the underlying `Inst`, panicking if it is not a `Inst.
+    pub fn unwrap_inst(&self) -> Inst {
+        self.inst().unwrap()
+    }
+
+    /// Is this a `Value`?
+    pub fn is_value(&self) -> bool {
+        self.value().is_some()
+    }
+
+    /// Is this an `Inst`?
+    pub fn is_inst(&self) -> bool {
+        self.inst().is_some()
+    }
+
+    fn resolve_inst(&self, dfg: &DataFlowGraph) -> Option<Inst> {
+        match *self {
+            ValueOrInst::Inst(i) => Some(i),
+            ValueOrInst::Value(v) => dfg.value_def(v).inst(),
+        }
+    }
+
+    fn result_bit_width(&self, dfg: &DataFlowGraph) -> u8 {
+        match *self {
+            ValueOrInst::Value(v) => dfg.value_type(v).bits().try_into().unwrap(),
+            ValueOrInst::Inst(inst) => {
+                let result = dfg.first_result(inst);
+                dfg.value_type(result).bits().try_into().unwrap()
+            }
+        }
+    }
+
+    fn to_constant(&self, pos: &mut FuncCursor) -> Option<Constant> {
+        let inst = self.resolve_inst(&pos.func.dfg)?;
+        match pos.func.dfg[inst] {
+            InstructionData::UnaryImm {
+                opcode: Opcode::Iconst,
+                imm,
+            } => {
+                let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap();
+                let x: i64 = imm.into();
+                Some(Constant::Int(x as u64, width))
+            }
+            InstructionData::UnaryBool {
+                opcode: Opcode::Bconst,
+                imm,
+            } => {
+                let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap();
+                Some(Constant::Bool(imm, width))
+            }
+            _ => None,
+        }
+    }
+}
+
+impl From<Value> for ValueOrInst {
+    fn from(v: Value) -> ValueOrInst {
+        ValueOrInst::Value(v)
+    }
+}
+
+impl From<Inst> for ValueOrInst {
+    fn from(i: Inst) -> ValueOrInst {
+        ValueOrInst::Inst(i)
+    }
+}
+
+/// Get the fixed bit width of `bit_width`, or if it is polymorphic, the bit
+/// width of `root`.
+fn bit_width(dfg: &DataFlowGraph, bit_width: BitWidth, root: Inst) -> u8 {
+    bit_width.fixed_width().unwrap_or_else(|| {
+        let tyvar = dfg.ctrl_typevar(root);
+        let ty = dfg.compute_result_type(root, 0, tyvar).unwrap();
+        u8::try_from(ty.bits()).unwrap()
+    })
+}
+
+/// Convert the constant `c` into an instruction.
+fn const_to_value<'a>(builder: impl InstBuilder<'a>, c: Constant, root: Inst) -> Value {
+    match c {
+        Constant::Bool(b, width) => {
+            let width = bit_width(builder.data_flow_graph(), width, root);
+            let ty = match width {
+                1 => types::B1,
+                8 => types::B8,
+                16 => types::B16,
+                32 => types::B32,
+                64 => types::B64,
+                128 => types::B128,
+                _ => unreachable!(),
+            };
+            builder.bconst(ty, b)
+        }
+        Constant::Int(x, width) => {
+            let width = bit_width(builder.data_flow_graph(), width, root);
+            let ty = match width {
+                1 | 8 => types::I8,
+                16 => types::I16,
+                32 => types::I32,
+                64 => types::I64,
+                128 => types::I128,
+                _ => unreachable!(),
+            };
+            builder.iconst(ty, x as i64)
+        }
+    }
+}
+
+fn part_to_inst(pos: &mut FuncCursor, root: Inst, part: Part<ValueOrInst>) -> Option<Inst> {
+    match part {
+        Part::Instruction(ValueOrInst::Inst(inst)) => Some(inst),
+        Part::Instruction(ValueOrInst::Value(v)) => {
+            let inst = pos.func.dfg.value_def(v).inst()?;
+            if pos.func.dfg.inst_results(inst).len() == 1 {
+                Some(inst)
+            } else {
+                None
+            }
+        }
+        Part::Constant(c) => {
+            let v = const_to_value(pos.ins(), c, root);
+            let inst = pos.func.dfg.value_def(v).unwrap_inst();
+            Some(inst)
+        }
+        Part::ConditionCode(_) => None,
+    }
+}
+
+fn part_to_value(pos: &mut FuncCursor, root: Inst, part: Part<ValueOrInst>) -> Option<Value> {
+    match part {
+        Part::Instruction(ValueOrInst::Inst(inst)) => {
+            pos.func.dfg.inst_results(inst).first().copied()
+        }
+        Part::Instruction(ValueOrInst::Value(v)) => Some(v),
+        Part::Constant(c) => Some(const_to_value(pos.ins(), c, root)),
+        Part::ConditionCode(_) => None,
+    }
+}
+
+impl Opcode {
+    fn to_peepmatic_operator(&self) -> Option<Operator> {
+        macro_rules! convert {
+            ( $( $op:ident $(,)* )* ) => {
+                match self {
+                    $( Self::$op => Some(Operator::$op), )*
+                    _ => None,
+                }
+            }
+        }
+
+        convert!(
+            AdjustSpDown,
+            AdjustSpDownImm,
+            Band,
+            BandImm,
+            Bconst,
+            Bint,
+            Bor,
+            BorImm,
+            Brnz,
+            Brz,
+            Bxor,
+            BxorImm,
+            Iadd,
+            IaddImm,
+            Icmp,
+            IcmpImm,
+            Iconst,
+            Ifcmp,
+            IfcmpImm,
+            Imul,
+            ImulImm,
+            Ireduce,
+            IrsubImm,
+            Ishl,
+            IshlImm,
+            Isub,
+            Rotl,
+            RotlImm,
+            Rotr,
+            RotrImm,
+            Sdiv,
+            SdivImm,
+            Select,
+            Sextend,
+            Srem,
+            SremImm,
+            Sshr,
+            SshrImm,
+            Trapnz,
+            Trapz,
+            Udiv,
+            UdivImm,
+            Uextend,
+            Urem,
+            UremImm,
+            Ushr,
+            UshrImm,
+        )
+    }
+}
+
+impl TryFrom<Constant> for Imm64 {
+    type Error = &'static str;
+
+    fn try_from(c: Constant) -> Result<Self, Self::Error> {
+        match c {
+            Constant::Int(x, _) => Ok(Imm64::from(x as i64)),
+            Constant::Bool(..) => Err("cannot create Imm64 from Constant::Bool"),
+        }
+    }
+}
+
+impl Into<Constant> for Imm64 {
+    #[inline]
+    fn into(self) -> Constant {
+        let x: i64 = self.into();
+        Constant::Int(x as _, BitWidth::SixtyFour)
+    }
+}
+
+impl Into<Part<ValueOrInst>> for Imm64 {
+    #[inline]
+    fn into(self) -> Part<ValueOrInst> {
+        let c: Constant = self.into();
+        c.into()
+    }
+}
+
+fn part_to_imm64(pos: &mut FuncCursor, part: Part<ValueOrInst>) -> Imm64 {
+    return match part {
+        Part::Instruction(x) => match x.to_constant(pos).unwrap_or_else(|| cannot_convert()) {
+            Constant::Int(x, _) => (x as i64).into(),
+            Constant::Bool(..) => cannot_convert(),
+        },
+        Part::Constant(Constant::Int(x, _)) => (x as i64).into(),
+        Part::ConditionCode(_) | Part::Constant(Constant::Bool(..)) => cannot_convert(),
+    };
+
+    #[inline(never)]
+    #[cold]
+    fn cannot_convert() -> ! {
+        panic!("cannot convert part into `Imm64`")
+    }
+}
+
+impl Into<Constant> for Uimm64 {
+    #[inline]
+    fn into(self) -> Constant {
+        let x: u64 = self.into();
+        Constant::Int(x, BitWidth::SixtyFour)
+    }
+}
+
+impl Into<Part<ValueOrInst>> for Uimm64 {
+    #[inline]
+    fn into(self) -> Part<ValueOrInst> {
+        let c: Constant = self.into();
+        c.into()
+    }
+}
+
+fn peepmatic_to_intcc(cc: ConditionCode) -> IntCC {
+    match cc {
+        ConditionCode::Eq => IntCC::Equal,
+        ConditionCode::Ne => IntCC::NotEqual,
+        ConditionCode::Slt => IntCC::SignedLessThan,
+        ConditionCode::Sle => IntCC::SignedGreaterThanOrEqual,
+        ConditionCode::Sgt => IntCC::SignedGreaterThan,
+        ConditionCode::Sge => IntCC::SignedLessThanOrEqual,
+        ConditionCode::Ult => IntCC::UnsignedLessThan,
+        ConditionCode::Uge => IntCC::UnsignedGreaterThanOrEqual,
+        ConditionCode::Ugt => IntCC::UnsignedGreaterThan,
+        ConditionCode::Ule => IntCC::UnsignedLessThanOrEqual,
+        ConditionCode::Of => IntCC::Overflow,
+        ConditionCode::Nof => IntCC::NotOverflow,
+    }
+}
+
+fn intcc_to_peepmatic(cc: IntCC) -> ConditionCode {
+    match cc {
+        IntCC::Equal => ConditionCode::Eq,
+        IntCC::NotEqual => ConditionCode::Ne,
+        IntCC::SignedLessThan => ConditionCode::Slt,
+        IntCC::SignedGreaterThanOrEqual => ConditionCode::Sle,
+        IntCC::SignedGreaterThan => ConditionCode::Sgt,
+        IntCC::SignedLessThanOrEqual => ConditionCode::Sge,
+        IntCC::UnsignedLessThan => ConditionCode::Ult,
+        IntCC::UnsignedGreaterThanOrEqual => ConditionCode::Uge,
+        IntCC::UnsignedGreaterThan => ConditionCode::Ugt,
+        IntCC::UnsignedLessThanOrEqual => ConditionCode::Ule,
+        IntCC::Overflow => ConditionCode::Of,
+        IntCC::NotOverflow => ConditionCode::Nof,
+    }
+}
+
+fn get_immediate(dfg: &DataFlowGraph, inst: Inst, i: usize) -> Part<ValueOrInst> {
+    return match dfg[inst] {
+        InstructionData::BinaryImm { imm, .. } if i == 0 => imm.into(),
+        InstructionData::BranchIcmp { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::BranchInt { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntCompare { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntCompareImm { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntCompareImm { imm, .. } if i == 1 => imm.into(),
+        InstructionData::IntCond { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntCondTrap { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntSelect { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::UnaryBool { imm, .. } if i == 0 => {
+            Constant::Bool(imm, BitWidth::Polymorphic).into()
+        }
+        InstructionData::UnaryImm { imm, .. } if i == 0 => imm.into(),
+        ref otherwise => unsupported(otherwise),
+    };
+
+    #[inline(never)]
+    #[cold]
+    fn unsupported(data: &InstructionData) -> ! {
+        panic!("unsupported instruction data: {:?}", data)
+    }
+}
+
+fn get_argument(dfg: &DataFlowGraph, inst: Inst, i: usize) -> Option<Value> {
+    dfg.inst_args(inst).get(i).copied()
+}
+
+fn peepmatic_ty_to_ir_ty(ty: Type, dfg: &DataFlowGraph, root: Inst) -> types::Type {
+    match (ty.kind, bit_width(dfg, ty.bit_width, root)) {
+        (Kind::Int, 1) | (Kind::Int, 8) => types::I8,
+        (Kind::Int, 16) => types::I16,
+        (Kind::Int, 32) => types::I32,
+        (Kind::Int, 64) => types::I64,
+        (Kind::Int, 128) => types::I128,
+        (Kind::Bool, 1) => types::B1,
+        (Kind::Bool, 8) => types::I8,
+        (Kind::Bool, 16) => types::I16,
+        (Kind::Bool, 32) => types::I32,
+        (Kind::Bool, 64) => types::I64,
+        (Kind::Bool, 128) => types::I128,
+        _ => unreachable!(),
+    }
+}
+
+impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
+    type Context = FuncCursor<'b>;
+
+    type Instruction = ValueOrInst;
+
+    fn replace_instruction(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        old: ValueOrInst,
+        new: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("replace {:?} with {:?}", old, new);
+        let old_inst = old.unwrap_inst();
+
+        // Try to convert `new` to an instruction, because we prefer replacing
+        // an old instruction with a new one wholesale. However, if the
+        // replacement cannot be converted to an instruction (e.g. the
+        // right-hand side is a block/function parameter value) then we change
+        // the old instruction's result to an alias of the new value.
+        match part_to_inst(pos, old_inst, new) {
+            Some(new_inst) => {
+                pos.func.transplant_inst(old_inst, new_inst);
+                debug_assert_eq!(pos.current_inst(), Some(old_inst));
+                old_inst.into()
+            }
+            None => {
+                let new_value = part_to_value(pos, old_inst, new).unwrap();
+
+                let old_results = pos.func.dfg.detach_results(old_inst);
+                let old_results = old_results.as_slice(&pos.func.dfg.value_lists);
+                assert_eq!(old_results.len(), 1);
+                let old_value = old_results[0];
+
+                pos.func.dfg.change_to_alias(old_value, new_value);
+                pos.func.dfg.replace(old_inst).nop();
+
+                new_value.into()
+            }
+        }
+    }
+
+    fn get_part_at_path(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        path: Path,
+    ) -> Option<Part<ValueOrInst>> {
+        // The root is path [0].
+        debug_assert!(!path.0.is_empty());
+        debug_assert_eq!(path.0[0], 0);
+
+        let mut part = Part::Instruction(root);
+        for p in path.0[1..].iter().copied() {
+            let inst = part.as_instruction()?.resolve_inst(&pos.func.dfg)?;
+            let operator = pos.func.dfg[inst].opcode().to_peepmatic_operator()?;
+
+            if p < operator.immediates_arity() {
+                part = get_immediate(&pos.func.dfg, inst, p as usize);
+                continue;
+            }
+
+            let arg = p - operator.immediates_arity();
+            let arg = arg as usize;
+            let value = get_argument(&pos.func.dfg, inst, arg)?;
+            part = Part::Instruction(value.into());
+        }
+
+        log::trace!("get_part_at_path({:?}) = {:?}", path, part);
+        Some(part)
+    }
+
+    fn operator(&self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst) -> Option<Operator> {
+        let inst = value_or_inst.resolve_inst(&pos.func.dfg)?;
+        pos.func.dfg[inst].opcode().to_peepmatic_operator()
+    }
+
+    fn make_inst_1(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        operator: Operator,
+        r#type: Type,
+        a: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("make_inst_1: {:?}({:?})", operator, a);
+
+        let root = root.unwrap_inst();
+        match operator {
+            Operator::AdjustSpDown => {
+                let a = part_to_value(pos, root, a).unwrap();
+                pos.ins().adjust_sp_down(a).into()
+            }
+            Operator::AdjustSpDownImm => {
+                let c = a.unwrap_constant();
+                let imm = Imm64::try_from(c).unwrap();
+                pos.ins().adjust_sp_down_imm(imm).into()
+            }
+            Operator::Bconst => {
+                let c = a.unwrap_constant();
+                const_to_value(pos.ins(), c, root).into()
+            }
+            Operator::Bint => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                pos.ins().bint(ty, a).into()
+            }
+            Operator::Brnz => {
+                let a = part_to_value(pos, root, a).unwrap();
+
+                // NB: branching instructions must be the root of an
+                // optimization's right-hand side, so we get the destination
+                // block and arguments from the left-hand side's root. Peepmatic
+                // doesn't currently represent labels or varargs.
+                let block = pos.func.dfg[root].branch_destination().unwrap();
+                let args = pos.func.dfg.inst_args(root)[1..].to_vec();
+
+                pos.ins().brnz(a, block, &args).into()
+            }
+            Operator::Brz => {
+                let a = part_to_value(pos, root, a).unwrap();
+
+                // See the comment in the `Operator::Brnz` match argm.
+                let block = pos.func.dfg[root].branch_destination().unwrap();
+                let args = pos.func.dfg.inst_args(root)[1..].to_vec();
+
+                pos.ins().brz(a, block, &args).into()
+            }
+            Operator::Iconst => {
+                let a = a.unwrap_constant();
+                const_to_value(pos.ins(), a, root).into()
+            }
+            Operator::Ireduce => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                pos.ins().ireduce(ty, a).into()
+            }
+            Operator::Sextend => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                pos.ins().sextend(ty, a).into()
+            }
+            Operator::Trapnz => {
+                let a = part_to_value(pos, root, a).unwrap();
+
+                // NB: similar to branching instructions (see comment in the
+                // `Operator::Brnz` match arm) trapping instructions must be the
+                // root of an optimization's right-hand side, and we get the
+                // trap code from the root of the left-hand side. Peepmatic
+                // doesn't currently represent trap codes.
+                let code = pos.func.dfg[root].trap_code().unwrap();
+
+                pos.ins().trapnz(a, code).into()
+            }
+            Operator::Trapz => {
+                let a = part_to_value(pos, root, a).unwrap();
+                // See comment in the `Operator::Trapnz` match arm.
+                let code = pos.func.dfg[root].trap_code().unwrap();
+                pos.ins().trapz(a, code).into()
+            }
+            Operator::Uextend => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                pos.ins().uextend(ty, a).into()
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn make_inst_2(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        operator: Operator,
+        _: Type,
+        a: Part<ValueOrInst>,
+        b: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("make_inst_2: {:?}({:?}, {:?})", operator, a, b);
+
+        let root = root.unwrap_inst();
+        match operator {
+            Operator::Band => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().band(a, b).into()
+            }
+            Operator::BandImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().band_imm(b, a).into()
+            }
+            Operator::Bor => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().bor(a, b).into()
+            }
+            Operator::BorImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().bor_imm(b, a).into()
+            }
+            Operator::Bxor => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().bxor(a, b).into()
+            }
+            Operator::BxorImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().bxor_imm(b, a).into()
+            }
+            Operator::Iadd => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().iadd(a, b).into()
+            }
+            Operator::IaddImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().iadd_imm(b, a).into()
+            }
+            Operator::Ifcmp => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().ifcmp(a, b).into()
+            }
+            Operator::IfcmpImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().ifcmp_imm(b, a).into()
+            }
+            Operator::Imul => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().imul(a, b).into()
+            }
+            Operator::ImulImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().imul_imm(b, a).into()
+            }
+            Operator::IrsubImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().irsub_imm(b, a).into()
+            }
+            Operator::Ishl => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().ishl(a, b).into()
+            }
+            Operator::IshlImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().ishl_imm(b, a).into()
+            }
+            Operator::Isub => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().isub(a, b).into()
+            }
+            Operator::Rotl => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().rotl(a, b).into()
+            }
+            Operator::RotlImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().rotl_imm(b, a).into()
+            }
+            Operator::Rotr => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().rotr(a, b).into()
+            }
+            Operator::RotrImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().rotr_imm(b, a).into()
+            }
+            Operator::Sdiv => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().sdiv(a, b).into()
+            }
+            Operator::SdivImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().sdiv_imm(b, a).into()
+            }
+            Operator::Srem => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().srem(a, b).into()
+            }
+            Operator::SremImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().srem_imm(b, a).into()
+            }
+            Operator::Sshr => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().sshr(a, b).into()
+            }
+            Operator::SshrImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().sshr_imm(b, a).into()
+            }
+            Operator::Udiv => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().udiv(a, b).into()
+            }
+            Operator::UdivImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().udiv_imm(b, a).into()
+            }
+            Operator::Urem => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().urem(a, b).into()
+            }
+            Operator::UremImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().urem_imm(b, a).into()
+            }
+            Operator::Ushr => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().ushr(a, b).into()
+            }
+            Operator::UshrImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                pos.ins().ushr_imm(b, a).into()
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn make_inst_3(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        operator: Operator,
+        _: Type,
+        a: Part<ValueOrInst>,
+        b: Part<ValueOrInst>,
+        c: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("make_inst_3: {:?}({:?}, {:?}, {:?})", operator, a, b, c);
+
+        let root = root.unwrap_inst();
+        match operator {
+            Operator::Icmp => {
+                let cond = a.unwrap_condition_code();
+                let cond = peepmatic_to_intcc(cond);
+                let b = part_to_value(pos, root, b).unwrap();
+                let c = part_to_value(pos, root, c).unwrap();
+                pos.ins().icmp(cond, b, c).into()
+            }
+            Operator::IcmpImm => {
+                let cond = a.unwrap_condition_code();
+                let cond = peepmatic_to_intcc(cond);
+                let imm = part_to_imm64(pos, b);
+                let c = part_to_value(pos, root, c).unwrap();
+                pos.ins().icmp_imm(cond, c, imm).into()
+            }
+            Operator::Select => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let c = part_to_value(pos, root, c).unwrap();
+                pos.ins().select(a, b, c).into()
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn instruction_to_constant(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        value_or_inst: ValueOrInst,
+    ) -> Option<Constant> {
+        value_or_inst.to_constant(pos)
+    }
+
+    fn instruction_result_bit_width(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        value_or_inst: ValueOrInst,
+    ) -> u8 {
+        value_or_inst.result_bit_width(&pos.func.dfg)
+    }
+
+    fn native_word_size_in_bits(&self, _pos: &mut FuncCursor<'b>) -> u8 {
+        self.pointer_bits()
+    }
+}

cranelift/codegen/src/preopt.peepmatic

@@ -0,0 +1,193 @@
+;; Apply basic simplifications.
+;;
+;; This folds constants with arithmetic to form `_imm` instructions, and other
+;; minor simplifications.
+;;
+;; Doesn't apply some simplifications if the native word width (in bytes) is
+;; smaller than the controlling type's width of the instruction. This would
+;; result in an illegal instruction that would likely be expanded back into an
+;; instruction on smaller types with the same initial opcode, creating
+;; unnecessary churn.
+
+;; Binary instructions whose second argument is constant.
+(=> (when (iadd $x $C)
+      (fits-in-native-word $C))
+    (iadd_imm $C $x))
+(=> (when (imul $x $C)
+      (fits-in-native-word $C))
+    (imul_imm $C $x))
+(=> (when (sdiv $x $C)
+      (fits-in-native-word $C))
+    (sdiv_imm $C $x))
+(=> (when (udiv $x $C)
+      (fits-in-native-word $C))
+    (udiv_imm $C $x))
+(=> (when (srem $x $C)
+      (fits-in-native-word $C))
+    (srem_imm $C $x))
+(=> (when (urem $x $C)
+      (fits-in-native-word $C))
+    (urem_imm $C $x))
+(=> (when (band $x $C)
+      (fits-in-native-word $C))
+    (band_imm $C $x))
+(=> (when (bor $x $C)
+      (fits-in-native-word $C))
+    (bor_imm $C $x))
+(=> (when (bxor $x $C)
+      (fits-in-native-word $C))
+    (bxor_imm $C $x))
+(=> (when (rotl $x $C)
+      (fits-in-native-word $C))
+    (rotl_imm $C $x))
+(=> (when (rotr $x $C)
+      (fits-in-native-word $C))
+    (rotr_imm $C $x))
+(=> (when (ishl $x $C)
+      (fits-in-native-word $C))
+    (ishl_imm $C $x))
+(=> (when (ushr $x $C)
+      (fits-in-native-word $C))
+    (ushr_imm $C $x))
+(=> (when (sshr $x $C)
+      (fits-in-native-word $C))
+    (sshr_imm $C $x))
+(=> (when (isub $x $C)
+      (fits-in-native-word $C))
+    (iadd_imm $(neg $C) $x))
+(=> (when (ifcmp $x $C)
+      (fits-in-native-word $C))
+    (ifcmp_imm $C $x))
+(=> (when (icmp $cond $x $C)
+      (fits-in-native-word $C))
+    (icmp_imm $cond $C $x))
+
+;; Binary instructions whose first operand is constant.
+(=> (when (iadd $C $x)
+      (fits-in-native-word $C))
+    (iadd_imm $C $x))
+(=> (when (imul $C $x)
+      (fits-in-native-word $C))
+    (imul_imm $C $x))
+(=> (when (band $C $x)
+      (fits-in-native-word $C))
+    (band_imm $C $x))
+(=> (when (bor $C $x)
+      (fits-in-native-word $C))
+    (bor_imm $C $x))
+(=> (when (bxor $C $x)
+      (fits-in-native-word $C))
+    (bxor_imm $C $x))
+(=> (when (isub $C $x)
+      (fits-in-native-word $C))
+    (irsub_imm $C $x))
+
+;; Unary instructions whose operand is constant.
+(=> (adjust_sp_down $C) (adjust_sp_down_imm $C))
+
+;; Fold `(binop_imm $C1 (binop_imm $C2 $x))` into `(binop_imm $(binop $C2 $C1) $x)`.
+(=> (iadd_imm $C1 (iadd_imm $C2 $x)) (iadd_imm $(iadd $C1 $C2) $x))
+(=> (imul_imm $C1 (imul_imm $C2 $x)) (imul_imm $(imul $C1 $C2) $x))
+(=> (bor_imm $C1 (bor_imm $C2 $x)) (bor_imm $(bor $C1 $C2) $x))
+(=> (band_imm $C1 (band_imm $C2 $x)) (band_imm $(band $C1 $C2) $x))
+(=> (bxor_imm $C1 (bxor_imm $C2 $x)) (bxor_imm $(bxor $C1 $C2) $x))
+
+;; Remove operations that are no-ops.
+(=> (iadd_imm 0 $x) $x)
+(=> (imul_imm 1 $x) $x)
+(=> (sdiv_imm 1 $x) $x)
+(=> (udiv_imm 1 $x) $x)
+(=> (bor_imm 0 $x) $x)
+(=> (band_imm -1 $x) $x)
+(=> (bxor_imm 0 $x) $x)
+(=> (rotl_imm 0 $x) $x)
+(=> (rotr_imm 0 $x) $x)
+(=> (ishl_imm 0 $x) $x)
+(=> (ushr_imm 0 $x) $x)
+(=> (sshr_imm 0 $x) $x)
+
+;; Replace with zero.
+(=> (imul_imm 0 $x) 0)
+(=> (band_imm 0 $x) 0)
+
+;; Replace with negative 1.
+(=> (bor_imm -1 $x) -1)
+
+;; Transform `[(x << N) >> N]` into a (un)signed-extending move.
+;;
+;; i16 -> i8 -> i16
+(=> (when (ushr_imm 8 (ishl_imm 8 $x))
+      (bit-width $x 16))
+    (uextend{i16} (ireduce{i8} $x)))
+(=> (when (sshr_imm 8 (ishl_imm 8 $x))
+      (bit-width $x 16))
+    (sextend{i16} (ireduce{i8} $x)))
+;; i32 -> i8 -> i32
+(=> (when (ushr_imm 24 (ishl_imm 24 $x))
+      (bit-width $x 32))
+    (uextend{i32} (ireduce{i8} $x)))
+(=> (when (sshr_imm 24 (ishl_imm 24 $x))
+      (bit-width $x 32))
+    (sextend{i32} (ireduce{i8} $x)))
+;; i32 -> i16 -> i32
+(=> (when (ushr_imm 16 (ishl_imm 16 $x))
+      (bit-width $x 32))
+    (uextend{i32} (ireduce{i16} $x)))
+(=> (when (sshr_imm 16 (ishl_imm 16 $x))
+      (bit-width $x 32))
+    (sextend{i32} (ireduce{i16} $x)))
+;; i64 -> i8 -> i64
+(=> (when (ushr_imm 56 (ishl_imm 56 $x))
+      (bit-width $x 64))
+    (uextend{i64} (ireduce{i8} $x)))
+(=> (when (sshr_imm 56 (ishl_imm 56 $x))
+      (bit-width $x 64))
+    (sextend{i64} (ireduce{i8} $x)))
+;; i64 -> i16 -> i64
+(=> (when (ushr_imm 48 (ishl_imm 48 $x))
+      (bit-width $x 64))
+    (uextend{i64} (ireduce{i16} $x)))
+(=> (when (sshr_imm 48 (ishl_imm 48 $x))
+      (bit-width $x 64))
+    (sextend{i64} (ireduce{i16} $x)))
+;; i64 -> i32 -> i64
+(=> (when (ushr_imm 32 (ishl_imm 32 $x))
+      (bit-width $x 64))
+    (uextend{i64} (ireduce{i32} $x)))
+(=> (when (sshr_imm 32 (ishl_imm 32 $x))
+      (bit-width $x 64))
+    (sextend{i64} (ireduce{i32} $x)))
+
+;; Fold away redundant `bint` instructions that accept both integer and boolean
+;; arguments.
+(=> (select (bint $x) $y $z) (select $x $y $z))
+(=> (brz (bint $x)) (brz $x))
+(=> (brnz (bint $x)) (brnz $x))
+(=> (trapz (bint $x)) (trapz $x))
+(=> (trapnz (bint $x)) (trapnz $x))
+
+;; Fold comparisons into branch operations when possible.
+;;
+;; This matches against operations which compare against zero, then use the
+;; result in a `brz` or `brnz` branch. It folds those two operations into a
+;; single `brz` or `brnz`.
+(=> (brnz (icmp_imm ne 0 $x)) (brnz $x))
+(=> (brz (icmp_imm ne 0 $x)) (brz $x))
+(=> (brnz (icmp_imm eq 0 $x)) (brz $x))
+(=> (brz (icmp_imm eq 0 $x)) (brnz $x))
+
+;; Division and remainder by constants.
+;;
+;; TODO: this section is incomplete, and a bunch of related optimizations are
+;; still hand-coded in `simple_preopt.rs`.
+
+;; (Division by one is handled above.)
+
+;; Remainder by one is zero.
+(=> (urem_imm 1 $x) 0)
+(=> (srem_imm 1 $x) 0)
+
+;; Division by a power of two -> shift right.
+(=> (when (udiv_imm $C $x)
+          (is-power-of-two $C))
+    (ushr_imm $(log2 $C) $x))

cranelift/codegen/src/simple_preopt.rs

@@ -10,13 +10,12 @@ use crate::divconst_magic_numbers::{MS32, MS64, MU32, MU64};
 use crate::flowgraph::ControlFlowGraph;
 use crate::ir::{
     condcodes::{CondCode, IntCC},
-    dfg::ValueDef,
-    immediates,
-    instructions::{Opcode, ValueList},
-    types::{I16, I32, I64, I8},
+    instructions::Opcode,
+    types::{I32, I64},
     Block, DataFlowGraph, Function, Inst, InstBuilder, InstructionData, Type, Value,
 };
 use crate::isa::TargetIsa;
+use crate::peepmatic::ValueOrInst;
 use crate::timing;
 
 #[inline]
@@ -183,12 +182,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
 
         // U32 div by 1: identity
         // U32 rem by 1: zero
-        DivRemByConstInfo::DivU32(n1, 1) | DivRemByConstInfo::RemU32(n1, 1) => {
-            if is_rem {
-                pos.func.dfg.replace(inst).iconst(I32, 0);
-            } else {
-                replace_single_result_with_alias(&mut pos.func.dfg, inst, n1);
-            }
+        DivRemByConstInfo::DivU32(_, 1) | DivRemByConstInfo::RemU32(_, 1) => {
+            unreachable!("unsigned division and remainder by one is handled in `preopt.peepmatic`");
         }
 
         // U32 div, rem by a power-of-2
@@ -203,7 +198,10 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
                 let mask = (1u64 << k) - 1;
                 pos.func.dfg.replace(inst).band_imm(n1, mask as i64);
             } else {
-                pos.func.dfg.replace(inst).ushr_imm(n1, k as i64);
+                unreachable!(
+                    "unsigned division by a power of two is handled in \
+                     `preopt.peepmatic`"
+                );
             }
         }
 
@@ -253,12 +251,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
 
         // U64 div by 1: identity
         // U64 rem by 1: zero
-        DivRemByConstInfo::DivU64(n1, 1) | DivRemByConstInfo::RemU64(n1, 1) => {
-            if is_rem {
-                pos.func.dfg.replace(inst).iconst(I64, 0);
-            } else {
-                replace_single_result_with_alias(&mut pos.func.dfg, inst, n1);
-            }
+        DivRemByConstInfo::DivU64(_, 1) | DivRemByConstInfo::RemU64(_, 1) => {
+            unreachable!("unsigned division and remainder by one is handled in `preopt.peepmatic`");
         }
 
         // U64 div, rem by a power-of-2
@@ -273,7 +267,9 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
                 let mask = (1u64 << k) - 1;
                 pos.func.dfg.replace(inst).band_imm(n1, mask as i64);
             } else {
-                pos.func.dfg.replace(inst).ushr_imm(n1, k as i64);
+                unreachable!(
+                    "unsigned division by a power of two is handled in `preopt.peepmatic`"
+                );
             }
         }
 
@@ -326,12 +322,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
 
         // S32 div by 1: identity
         // S32 rem by 1: zero
-        DivRemByConstInfo::DivS32(n1, 1) | DivRemByConstInfo::RemS32(n1, 1) => {
-            if is_rem {
-                pos.func.dfg.replace(inst).iconst(I32, 0);
-            } else {
-                replace_single_result_with_alias(&mut pos.func.dfg, inst, n1);
-            }
+        DivRemByConstInfo::DivS32(_, 1) | DivRemByConstInfo::RemS32(_, 1) => {
+            unreachable!("signed division and remainder by one is handled in `preopt.peepmatic`");
         }
 
         DivRemByConstInfo::DivS32(n1, d) | DivRemByConstInfo::RemS32(n1, d) => {
@@ -401,12 +393,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
 
         // S64 div by 1: identity
         // S64 rem by 1: zero
-        DivRemByConstInfo::DivS64(n1, 1) | DivRemByConstInfo::RemS64(n1, 1) => {
-            if is_rem {
-                pos.func.dfg.replace(inst).iconst(I64, 0);
-            } else {
-                replace_single_result_with_alias(&mut pos.func.dfg, inst, n1);
-            }
+        DivRemByConstInfo::DivS64(_, 1) | DivRemByConstInfo::RemS64(_, 1) => {
+            unreachable!("division and remaineder by one are handled in `preopt.peepmatic`");
         }
 
         DivRemByConstInfo::DivS64(n1, d) | DivRemByConstInfo::RemS64(n1, d) => {
@@ -468,340 +456,6 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
     }
 }
 
-#[inline]
-fn resolve_imm64_value(dfg: &DataFlowGraph, value: Value) -> Option<immediates::Imm64> {
-    if let ValueDef::Result(candidate_inst, _) = dfg.value_def(value) {
-        if let InstructionData::UnaryImm {
-            opcode: Opcode::Iconst,
-            imm,
-        } = dfg[candidate_inst]
-        {
-            return Some(imm);
-        }
-    }
-    None
-}
-
-/// Try to transform [(x << N) >> N] into a (un)signed-extending move.
-/// Returns true if the final instruction has been converted to such a move.
-fn try_fold_extended_move(
-    pos: &mut FuncCursor,
-    inst: Inst,
-    opcode: Opcode,
-    arg: Value,
-    imm: immediates::Imm64,
-) -> bool {
-    if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) {
-        if let InstructionData::BinaryImm {
-            opcode: Opcode::IshlImm,
-            arg: prev_arg,
-            imm: prev_imm,
-        } = &pos.func.dfg[arg_inst]
-        {
-            if imm != *prev_imm {
-                return false;
-            }
-
-            let dest_ty = pos.func.dfg.ctrl_typevar(inst);
-            if dest_ty != pos.func.dfg.ctrl_typevar(arg_inst) || !dest_ty.is_int() {
-                return false;
-            }
-
-            let imm_bits: i64 = imm.into();
-            let ireduce_ty = match (dest_ty.lane_bits() as i64).wrapping_sub(imm_bits) {
-                8 => I8,
-                16 => I16,
-                32 => I32,
-                _ => return false,
-            };
-            let ireduce_ty = ireduce_ty.by(dest_ty.lane_count()).unwrap();
-
-            // This becomes a no-op, since ireduce_ty has a smaller lane width than
-            // the argument type (also the destination type).
-            let arg = *prev_arg;
-            let narrower_arg = pos.ins().ireduce(ireduce_ty, arg);
-
-            if opcode == Opcode::UshrImm {
-                pos.func.dfg.replace(inst).uextend(dest_ty, narrower_arg);
-            } else {
-                pos.func.dfg.replace(inst).sextend(dest_ty, narrower_arg);
-            }
-            return true;
-        }
-    }
-    false
-}
-
-/// Apply basic simplifications.
-///
-/// This folds constants with arithmetic to form `_imm` instructions, and other minor
-/// simplifications.
-///
-/// Doesn't apply some simplifications if the native word width (in bytes) is smaller than the
-/// controlling type's width of the instruction. This would result in an illegal instruction that
-/// would likely be expanded back into an instruction on smaller types with the same initial
-/// opcode, creating unnecessary churn.
-fn simplify(pos: &mut FuncCursor, inst: Inst, native_word_width: u32) {
-    match pos.func.dfg[inst] {
-        InstructionData::Binary { opcode, args } => {
-            if let Some(mut imm) = resolve_imm64_value(&pos.func.dfg, args[1]) {
-                let new_opcode = match opcode {
-                    Opcode::Iadd => Opcode::IaddImm,
-                    Opcode::Imul => Opcode::ImulImm,
-                    Opcode::Sdiv => Opcode::SdivImm,
-                    Opcode::Udiv => Opcode::UdivImm,
-                    Opcode::Srem => Opcode::SremImm,
-                    Opcode::Urem => Opcode::UremImm,
-                    Opcode::Band => Opcode::BandImm,
-                    Opcode::Bor => Opcode::BorImm,
-                    Opcode::Bxor => Opcode::BxorImm,
-                    Opcode::Rotl => Opcode::RotlImm,
-                    Opcode::Rotr => Opcode::RotrImm,
-                    Opcode::Ishl => Opcode::IshlImm,
-                    Opcode::Ushr => Opcode::UshrImm,
-                    Opcode::Sshr => Opcode::SshrImm,
-                    Opcode::Isub => {
-                        imm = imm.wrapping_neg();
-                        Opcode::IaddImm
-                    }
-                    Opcode::Ifcmp => Opcode::IfcmpImm,
-                    _ => return,
-                };
-                let ty = pos.func.dfg.ctrl_typevar(inst);
-                if ty.bytes() <= native_word_width {
-                    pos.func
-                        .dfg
-                        .replace(inst)
-                        .BinaryImm(new_opcode, ty, imm, args[0]);
-
-                    // Repeat for BinaryImm simplification.
-                    simplify(pos, inst, native_word_width);
-                }
-            } else if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[0]) {
-                let new_opcode = match opcode {
-                    Opcode::Iadd => Opcode::IaddImm,
-                    Opcode::Imul => Opcode::ImulImm,
-                    Opcode::Band => Opcode::BandImm,
-                    Opcode::Bor => Opcode::BorImm,
-                    Opcode::Bxor => Opcode::BxorImm,
-                    Opcode::Isub => Opcode::IrsubImm,
-                    _ => return,
-                };
-                let ty = pos.func.dfg.ctrl_typevar(inst);
-                if ty.bytes() <= native_word_width {
-                    pos.func
-                        .dfg
-                        .replace(inst)
-                        .BinaryImm(new_opcode, ty, imm, args[1]);
-                }
-            }
-        }
-
-        InstructionData::Unary { opcode, arg } => {
-            if let Opcode::AdjustSpDown = opcode {
-                if let Some(imm) = resolve_imm64_value(&pos.func.dfg, arg) {
-                    // Note this works for both positive and negative immediate values.
-                    pos.func.dfg.replace(inst).adjust_sp_down_imm(imm);
-                }
-            }
-        }
-
-        InstructionData::BinaryImm { opcode, arg, imm } => {
-            let ty = pos.func.dfg.ctrl_typevar(inst);
-
-            let mut arg = arg;
-            let mut imm = imm;
-            match opcode {
-                Opcode::IaddImm
-                | Opcode::ImulImm
-                | Opcode::BorImm
-                | Opcode::BandImm
-                | Opcode::BxorImm => {
-                    // Fold binary_op(C2, binary_op(C1, x)) into binary_op(binary_op(C1, C2), x)
-                    if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) {
-                        if let InstructionData::BinaryImm {
-                            opcode: prev_opcode,
-                            arg: prev_arg,
-                            imm: prev_imm,
-                        } = &pos.func.dfg[arg_inst]
-                        {
-                            if opcode == *prev_opcode && ty == pos.func.dfg.ctrl_typevar(arg_inst) {
-                                let lhs: i64 = imm.into();
-                                let rhs: i64 = (*prev_imm).into();
-                                let new_imm = match opcode {
-                                    Opcode::BorImm => lhs | rhs,
-                                    Opcode::BandImm => lhs & rhs,
-                                    Opcode::BxorImm => lhs ^ rhs,
-                                    Opcode::IaddImm => lhs.wrapping_add(rhs),
-                                    Opcode::ImulImm => lhs.wrapping_mul(rhs),
-                                    _ => panic!("can't happen"),
-                                };
-                                let new_imm = immediates::Imm64::from(new_imm);
-                                let new_arg = *prev_arg;
-                                pos.func
-                                    .dfg
-                                    .replace(inst)
-                                    .BinaryImm(opcode, ty, new_imm, new_arg);
-                                imm = new_imm;
-                                arg = new_arg;
-                            }
-                        }
-                    }
-                }
-
-                Opcode::UshrImm | Opcode::SshrImm => {
-                    if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width
-                        && try_fold_extended_move(pos, inst, opcode, arg, imm)
-                    {
-                        return;
-                    }
-                }
-
-                _ => {}
-            };
-
-            // Replace operations that are no-ops.
-            match (opcode, imm.into()) {
-                (Opcode::IaddImm, 0)
-                | (Opcode::ImulImm, 1)
-                | (Opcode::SdivImm, 1)
-                | (Opcode::UdivImm, 1)
-                | (Opcode::BorImm, 0)
-                | (Opcode::BandImm, -1)
-                | (Opcode::BxorImm, 0)
-                | (Opcode::RotlImm, 0)
-                | (Opcode::RotrImm, 0)
-                | (Opcode::IshlImm, 0)
-                | (Opcode::UshrImm, 0)
-                | (Opcode::SshrImm, 0) => {
-                    // Alias the result value with the original argument.
-                    replace_single_result_with_alias(&mut pos.func.dfg, inst, arg);
-                }
-                (Opcode::ImulImm, 0) | (Opcode::BandImm, 0) => {
-                    // Replace by zero.
-                    pos.func.dfg.replace(inst).iconst(ty, 0);
-                }
-                (Opcode::BorImm, -1) => {
-                    // Replace by minus one.
-                    pos.func.dfg.replace(inst).iconst(ty, -1);
-                }
-                _ => {}
-            }
-        }
-
-        InstructionData::IntCompare { opcode, cond, args } => {
-            debug_assert_eq!(opcode, Opcode::Icmp);
-            if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[1]) {
-                if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width {
-                    pos.func.dfg.replace(inst).icmp_imm(cond, args[0], imm);
-                }
-            }
-        }
-
-        InstructionData::CondTrap { .. }
-        | InstructionData::Branch { .. }
-        | InstructionData::Ternary {
-            opcode: Opcode::Select,
-            ..
-        } => {
-            // Fold away a redundant `bint`.
-            let condition_def = {
-                let args = pos.func.dfg.inst_args(inst);
-                pos.func.dfg.value_def(args[0])
-            };
-            if let ValueDef::Result(def_inst, _) = condition_def {
-                if let InstructionData::Unary {
-                    opcode: Opcode::Bint,
-                    arg: bool_val,
-                } = pos.func.dfg[def_inst]
-                {
-                    let args = pos.func.dfg.inst_args_mut(inst);
-                    args[0] = bool_val;
-                }
-            }
-        }
-
-        _ => {}
-    }
-}
-
-struct BranchOptInfo {
-    br_inst: Inst,
-    cmp_arg: Value,
-    args: ValueList,
-    new_opcode: Opcode,
-}
-
-/// Fold comparisons into branch operations when possible.
-///
-/// This matches against operations which compare against zero, then use the
-/// result in a `brz` or `brnz` branch. It folds those two operations into a
-/// single `brz` or `brnz`.
-fn branch_opt(pos: &mut FuncCursor, inst: Inst) {
-    let mut info = if let InstructionData::Branch {
-        opcode: br_opcode,
-        args: ref br_args,
-        ..
-    } = pos.func.dfg[inst]
-    {
-        let first_arg = {
-            let args = pos.func.dfg.inst_args(inst);
-            args[0]
-        };
-
-        let icmp_inst = if let ValueDef::Result(icmp_inst, _) = pos.func.dfg.value_def(first_arg) {
-            icmp_inst
-        } else {
-            return;
-        };
-
-        if let InstructionData::IntCompareImm {
-            opcode: Opcode::IcmpImm,
-            arg: cmp_arg,
-            cond: cmp_cond,
-            imm: cmp_imm,
-        } = pos.func.dfg[icmp_inst]
-        {
-            let cmp_imm: i64 = cmp_imm.into();
-            if cmp_imm != 0 {
-                return;
-            }
-
-            // icmp_imm returns non-zero when the comparison is true. So, if
-            // we're branching on zero, we need to invert the condition.
-            let cond = match br_opcode {
-                Opcode::Brz => cmp_cond.inverse(),
-                Opcode::Brnz => cmp_cond,
-                _ => return,
-            };
-
-            let new_opcode = match cond {
-                IntCC::Equal => Opcode::Brz,
-                IntCC::NotEqual => Opcode::Brnz,
-                _ => return,
-            };
-
-            BranchOptInfo {
-                br_inst: inst,
-                cmp_arg,
-                args: br_args.clone(),
-                new_opcode,
-            }
-        } else {
-            return;
-        }
-    } else {
-        return;
-    };
-
-    info.args.as_mut_slice(&mut pos.func.dfg.value_lists)[0] = info.cmp_arg;
-    if let InstructionData::Branch { ref mut opcode, .. } = pos.func.dfg[info.br_inst] {
-        *opcode = info.new_opcode;
-    } else {
-        panic!();
-    }
-}
-
 enum BranchOrderKind {
     BrzToBrnz(Value),
     BrnzToBrz(Value),
@@ -945,14 +599,20 @@ fn branch_order(pos: &mut FuncCursor, cfg: &mut ControlFlowGraph, block: Block,
 }
 
 /// The main pre-opt pass.
-pub fn do_preopt(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &dyn TargetIsa) {
+pub fn do_preopt<'func, 'isa>(
+    func: &'func mut Function,
+    cfg: &mut ControlFlowGraph,
+    isa: &'isa dyn TargetIsa,
+) {
     let _tt = timing::preopt();
+
     let mut pos = FuncCursor::new(func);
-    let native_word_width = isa.pointer_bytes();
+    let mut preopt = crate::peepmatic::preopt(isa);
+
     while let Some(block) = pos.next_block() {
         while let Some(inst) = pos.next_inst() {
-            // Apply basic simplifications.
-            simplify(&mut pos, inst, native_word_width as u32);
+            preopt.apply_all(&mut pos, ValueOrInst::Inst(inst));
+            let inst = pos.current_inst().unwrap();
 
             // Try to transform divide-by-constant into simpler operations.
             if let Some(divrem_info) = get_div_info(inst, &pos.func.dfg) {
@@ -960,7 +620,6 @@ pub fn do_preopt(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &dyn Targ
                 continue;
             }
 
-            branch_opt(&mut pos, inst);
             branch_order(&mut pos, cfg, block, inst);
         }
     }