T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

peepmatic: Do not use paths in linear IR

Browse Source 
Rather than using paths from the root instruction to the instruction we are
matching against or checking if it is constant or whatever, use temporary
variables. When we successfully match an instruction's opcode, we simultaneously
define these temporaries for the instruction's operands. This is similar to how
open-coding these matches in Rust would use `match` expressions with pattern
matching to bind the operands to variables at the same time.

This saves about 1.8% of instructions retired when Peepmatic is enabled.
This commit is contained in:
Nick Fitzgerald
2020-09-23 15:53:30 -07:00
parent 447c3e71a6
commit c015d69eb8
14 changed files with 633 additions and 844 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
cranelift/peepmatic/src/dot_fmt.rs
View File

@@ -7,7 +7,6 @@ use peepmatic_runtime::{
cc::ConditionCode,
integer_interner::{IntegerId, IntegerInterner},
linear,
paths::{PathId, PathInterner},
};
use std::convert::{TryFrom, TryInto};
use std::fmt::Debug;
@@ -15,7 +14,7 @@ use std::io::{self, Write};
use std::num::{NonZeroU16, NonZeroU32};
#[derive(Debug)]
pub(crate) struct PeepholeDotFmt<'a>(pub(crate) &'a PathInterner, pub(crate) &'a IntegerInterner);
pub(crate) struct PeepholeDotFmt<'a>(pub(crate) &'a IntegerInterner);
impl<TOperator> DotFmt<linear::MatchResult, linear::MatchOp, Box<[linear::Action<TOperator>]>>
for PeepholeDotFmt<'_>
@@ -44,7 +43,7 @@ where
write!(w, "{}", cc)
}
linear::MatchOp::IntegerValue { .. } => {
let x = self.1.lookup(IntegerId(
let x = self.0.lookup(IntegerId(
NonZeroU16::new(x.get().try_into().unwrap()).unwrap(),
));
write!(w, "{}", x)
@@ -61,17 +60,16 @@ where
write!(w, r#"<font face="monospace">"#)?;
let p = p(self.0);
match op {
Opcode { path } => write!(w, "opcode @ {}", p(path))?,
IsConst { path } => write!(w, "is-const? @ {}", p(path))?,
IsPowerOfTwo { path } => write!(w, "is-power-of-two? @ {}", p(path))?,
BitWidth { path } => write!(w, "bit-width @ {}", p(path))?,
FitsInNativeWord { path } => write!(w, "fits-in-native-word @ {}", p(path))?,
Eq { path_a, path_b } => write!(w, "{} == {}", p(path_a), p(path_b))?,
IntegerValue { path } => write!(w, "integer-value @ {}", p(path))?,
BooleanValue { path } => write!(w, "boolean-value @ {}", p(path))?,
ConditionCode { path } => write!(w, "condition-code @ {}", p(path))?,
Opcode(id) => write!(w, "opcode $lhs{}", id.0)?,
IsConst(id) => write!(w, "is-const? $lhs{}", id.0)?,
IsPowerOfTwo(id) => write!(w, "is-power-of-two? $lhs{}", id.0)?,
BitWidth(id) => write!(w, "bit-width $lhs{}", id.0)?,
FitsInNativeWord(id) => write!(w, "fits-in-native-word $lhs{}", id.0)?,
Eq(a, b) => write!(w, "$lhs{} == $lhs{}", a.0, b.0)?,
IntegerValue(id) => write!(w, "integer-value $lhs{}", id.0)?,
BooleanValue(id) => write!(w, "boolean-value $lhs{}", id.0)?,
ConditionCode(id) => write!(w, "condition-code $lhs{}", id.0)?,
Nop => write!(w, "nop")?,
}
@@ -91,11 +89,9 @@ where
write!(w, r#"<font face="monospace">"#)?;
let p = p(self.0);
for a in actions.iter() {
match a {
GetLhs { path } => write!(w, "get-lhs @ {}<br/>", p(path))?,
GetLhs { lhs } => write!(w, "get-lhs $lhs{}<br/>", lhs.0)?,
UnaryUnquote { operator, operand } => {
write!(w, "eval {:?} $rhs{}<br/>", operator, operand.0)?
}
@@ -107,7 +103,7 @@ where
MakeIntegerConst {
value,
bit_width: _,
} => write!(w, "make {}<br/>", self.1.lookup(*value))?,
} => write!(w, "make {}<br/>", self.0.lookup(*value))?,
MakeBooleanConst {
value,
bit_width: _,
@@ -142,13 +138,3 @@ where
writeln!(w, "</font>")
}
}
fn p<'a>(paths: &'a PathInterner) -> impl Fn(&PathId) -> String + 'a {
move |path: &PathId| {
let mut s = vec![];
for b in paths.lookup(*path).0 {
s.push(b.to_string());
}
s.join(".")
}
}

9
cranelift/peepmatic/src/lib.rs
View File

@@ -142,11 +142,10 @@ where
sort_lexicographically(&mut opts);
let automata = automatize(&opts);
let paths = opts.paths;
let integers = opts.integers;
if let Ok(path) = std::env::var("PEEPMATIC_DOT") {
let f = dot_fmt::PeepholeDotFmt(&paths, &integers);
let f = dot_fmt::PeepholeDotFmt(&integers);
if let Err(e) = automata.write_dot_file(&f, &path) {
panic!(
"failed to write GraphViz Dot file to PEEPMATIC_DOT={}; error: {}",
@@ -155,11 +154,7 @@ where
}
}
Ok(PeepholeOptimizations {
paths,
integers,
automata,
})
Ok(PeepholeOptimizations { integers, automata })
}
#[cfg(test)]

245
cranelift/peepmatic/src/linear_passes.rs
View File

@@ -1,9 +1,6 @@
//! Passes over the linear IR.
use peepmatic_runtime::{
linear,
paths::{PathId, PathInterner},
};
use peepmatic_runtime::linear;
use std::cmp::Ordering;
use std::fmt::Debug;
use std::hash::Hash;
@@ -33,13 +30,12 @@ where
{
let linear::Optimizations {
ref mut optimizations,
ref paths,
..
} = opts;
// NB: we *cannot* use an unstable sort here, because we want deterministic
// compilation of optimizations to automata.
optimizations.sort_by(|a, b| compare_optimization_generality(paths, a, b));
optimizations.sort_by(compare_optimization_generality);
debug_assert!(is_sorted_by_generality(opts));
}
@@ -52,17 +48,14 @@ where
{
let linear::Optimizations {
ref mut optimizations,
ref paths,
..
} = opts;
// NB: we *cannot* use an unstable sort here, same as above.
optimizations
.sort_by(|a, b| compare_optimizations(paths, a, b, |a_len, b_len| a_len.cmp(&b_len)));
optimizations.sort_by(|a, b| compare_optimizations(a, b, |a_len, b_len| a_len.cmp(&b_len)));
}
fn compare_optimizations<TOperator>(
paths: &PathInterner,
a: &linear::Optimization<TOperator>,
b: &linear::Optimization<TOperator>,
compare_lengths: impl Fn(usize, usize) -> Ordering,
@@ -71,7 +64,7 @@ where
TOperator: Copy + Debug + Eq + Hash,
{
for (a, b) in a.matches.iter().zip(b.matches.iter()) {
let c = compare_match_op_generality(paths, a.operation, b.operation);
let c = compare_match_op_generality(a.operation, b.operation);
if c != Ordering::Equal {
return c;
}
@@ -91,86 +84,62 @@ where
}
fn compare_optimization_generality<TOperator>(
paths: &PathInterner,
a: &linear::Optimization<TOperator>,
b: &linear::Optimization<TOperator>,
) -> Ordering
where
TOperator: Copy + Debug + Eq + Hash,
{
compare_optimizations(paths, a, b, |a_len, b_len| {
compare_optimizations(a, b, |a_len, b_len| {
// If they shared equivalent prefixes, then compare lengths and invert the
// result because longer patterns are less general than shorter patterns.
a_len.cmp(&b_len).reverse()
})
}
fn compare_match_op_generality(
paths: &PathInterner,
a: linear::MatchOp,
b: linear::MatchOp,
) -> Ordering {
fn compare_match_op_generality(a: linear::MatchOp, b: linear::MatchOp) -> Ordering {
use linear::MatchOp::*;
match (a, b) {
(Opcode { path: a }, Opcode { path: b }) => compare_paths(paths, a, b),
(Opcode { .. }, _) => Ordering::Less,
(_, Opcode { .. }) => Ordering::Greater,
(Opcode(a), Opcode(b)) => a.cmp(&b),
(Opcode(_), _) => Ordering::Less,
(_, Opcode(_)) => Ordering::Greater,
(IntegerValue { path: a }, IntegerValue { path: b }) => compare_paths(paths, a, b),
(IntegerValue { .. }, _) => Ordering::Less,
(_, IntegerValue { .. }) => Ordering::Greater,
(IntegerValue(a), IntegerValue(b)) => a.cmp(&b),
(IntegerValue(_), _) => Ordering::Less,
(_, IntegerValue(_)) => Ordering::Greater,
(BooleanValue { path: a }, BooleanValue { path: b }) => compare_paths(paths, a, b),
(BooleanValue { .. }, _) => Ordering::Less,
(_, BooleanValue { .. }) => Ordering::Greater,
(BooleanValue(a), BooleanValue(b)) => a.cmp(&b),
(BooleanValue(_), _) => Ordering::Less,
(_, BooleanValue(_)) => Ordering::Greater,
(ConditionCode { path: a }, ConditionCode { path: b }) => compare_paths(paths, a, b),
(ConditionCode { .. }, _) => Ordering::Less,
(_, ConditionCode { .. }) => Ordering::Greater,
(ConditionCode(a), ConditionCode(b)) => a.cmp(&b),
(ConditionCode(_), _) => Ordering::Less,
(_, ConditionCode(_)) => Ordering::Greater,
(IsConst { path: a }, IsConst { path: b }) => compare_paths(paths, a, b),
(IsConst { .. }, _) => Ordering::Less,
(_, IsConst { .. }) => Ordering::Greater,
(IsConst(a), IsConst(b)) => a.cmp(&b),
(IsConst(_), _) => Ordering::Less,
(_, IsConst(_)) => Ordering::Greater,
(
Eq {
path_a: pa1,
path_b: pb1,
},
Eq {
path_a: pa2,
path_b: pb2,
},
) => compare_paths(paths, pa1, pa2).then(compare_paths(paths, pb1, pb2)),
(Eq { .. }, _) => Ordering::Less,
(_, Eq { .. }) => Ordering::Greater,
(Eq(a1, b1), Eq(a2, b2)) => a1.cmp(&a2).then(b1.cmp(&b2)),
(Eq(..), _) => Ordering::Less,
(_, Eq(..)) => Ordering::Greater,
(IsPowerOfTwo { path: a }, IsPowerOfTwo { path: b }) => compare_paths(paths, a, b),
(IsPowerOfTwo { .. }, _) => Ordering::Less,
(_, IsPowerOfTwo { .. }) => Ordering::Greater,
(IsPowerOfTwo(a), IsPowerOfTwo(b)) => a.cmp(&b),
(IsPowerOfTwo(_), _) => Ordering::Less,
(_, IsPowerOfTwo(_)) => Ordering::Greater,
(BitWidth { path: a }, BitWidth { path: b }) => compare_paths(paths, a, b),
(BitWidth { .. }, _) => Ordering::Less,
(_, BitWidth { .. }) => Ordering::Greater,
(BitWidth(a), BitWidth(b)) => a.cmp(&b),
(BitWidth(_), _) => Ordering::Less,
(_, BitWidth(_)) => Ordering::Greater,
(FitsInNativeWord { path: a }, FitsInNativeWord { path: b }) => compare_paths(paths, a, b),
(FitsInNativeWord { .. }, _) => Ordering::Less,
(_, FitsInNativeWord { .. }) => Ordering::Greater,
(FitsInNativeWord(a), FitsInNativeWord(b)) => a.cmp(&b),
(FitsInNativeWord(_), _) => Ordering::Less,
(_, FitsInNativeWord(_)) => Ordering::Greater,
(Nop, Nop) => Ordering::Equal,
}
}
fn compare_paths(paths: &PathInterner, a: PathId, b: PathId) -> Ordering {
if a == b {
Ordering::Equal
} else {
let a = paths.lookup(a);
let b = paths.lookup(b);
a.0.cmp(&b.0)
}
}
/// Are the given optimizations sorted from least to most general?
pub(crate) fn is_sorted_by_generality<TOperator>(opts: &linear::Optimizations<TOperator>) -> bool
where
@@ -178,7 +147,7 @@ where
{
opts.optimizations
.windows(2)
.all(|w| compare_optimization_generality(&opts.paths, &w[0], &w[1]) <= Ordering::Equal)
.all(|w| compare_optimization_generality(&w[0], &w[1]) <= Ordering::Equal)
}
/// Are the given optimizations sorted lexicographically?
@@ -189,8 +158,7 @@ where
TOperator: Copy + Debug + Eq + Hash,
{
opts.optimizations.windows(2).all(|w| {
compare_optimizations(&opts.paths, &w[0], &w[1], |a_len, b_len| a_len.cmp(&b_len))
<= Ordering::Equal
compare_optimizations(&w[0], &w[1], |a_len, b_len| a_len.cmp(&b_len)) <= Ordering::Equal
})
}
@@ -309,10 +277,7 @@ where
mod tests {
use super::*;
use crate::ast::*;
use peepmatic_runtime::{
linear::{bool_to_match_result, Else, MatchOp::*, MatchResult},
paths::*,
};
use peepmatic_runtime::linear::{bool_to_match_result, Else, LhsId, MatchOp::*, MatchResult};
use peepmatic_test_operator::TestOperator;
use std::num::NonZeroU32;
@@ -374,7 +339,6 @@ mod tests {
eprintln!("after = {:#?}", before);
let linear::Optimizations {
mut paths,
mut integers,
optimizations,
} = opts;
@@ -389,9 +353,8 @@ mod tests {
})
.collect();
let mut p = |p: &[u8]| paths.intern(Path::new(&p));
let mut i = |i: u64| Ok(integers.intern(i).into());
let expected = $make_expected(&mut p, &mut i);
let expected = $make_expected(&mut i);
assert_eq!(expected, actual);
}
@@ -452,7 +415,6 @@ mod tests {
eprintln!("after = {:#?}", before);
let linear::Optimizations {
mut paths,
mut integers,
optimizations,
} = opts;
@@ -467,9 +429,8 @@ mod tests {
})
.collect();
let mut p = |p: &[u8]| paths.intern(Path::new(&p));
let mut i = |i: u64| Ok(integers.intern(i).into());
let expected = $make_expected(&mut p, &mut i);
let expected = $make_expected(&mut i);
assert_eq!(expected, actual);
}
@@ -488,55 +449,43 @@ mod tests {
(=> (iadd $x 42) 0)
(=> (iadd $x (iadd $y $z)) 0)
",
|p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![
|i: &mut dyn FnMut(u64) -> MatchResult| vec![
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Iadd.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Iadd.into())),
(Nop, Err(Else)),
(Opcode { path: p(&[0, 1]) }, Ok(TestOperator::Iadd.into())),
(Opcode(LhsId(2)), Ok(TestOperator::Iadd.into())),
(Nop, Err(Else)),
(Nop, Err(Else)),
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Iadd.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Iadd.into())),
(Nop, Err(Else)),
(IntegerValue { path: p(&[0, 1]) }, i(42))
(IntegerValue(LhsId(2)), i(42))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Iadd.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Iadd.into())),
(Nop, Err(Else)),
(IsConst { path: p(&[0, 1]) }, bool_to_match_result(true)),
(
IsPowerOfTwo { path: p(&[0, 1]) },
bool_to_match_result(true)
)
(IsConst(LhsId(2)), bool_to_match_result(true)),
(IsPowerOfTwo(LhsId(2)), bool_to_match_result(true))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Iadd.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Iadd.into())),
(Nop, Err(Else)),
(IsConst { path: p(&[0, 1]) }, bool_to_match_result(true)),
(
BitWidth { path: p(&[0, 0]) },
Ok(NonZeroU32::new(32).unwrap())
)
(IsConst(LhsId(2)), bool_to_match_result(true)),
(BitWidth(LhsId(1)), Ok(NonZeroU32::new(32).unwrap()))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Iadd.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Iadd.into())),
(Nop, Err(Else)),
(IsConst { path: p(&[0, 1]) }, bool_to_match_result(true))
(IsConst(LhsId(2)), bool_to_match_result(true))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Iadd.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Iadd.into())),
(Nop, Err(Else)),
(
Eq {
path_a: p(&[0, 1]),
path_b: p(&[0, 0]),
},
bool_to_match_result(true)
)
(Eq(LhsId(2), LhsId(1)), bool_to_match_result(true))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Iadd.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Iadd.into())),
(Nop, Err(Else)),
(Nop, Err(Else)),
],
@@ -554,36 +503,36 @@ mod tests {
(=> (imul 2 $x) (ishl $x 1))
(=> (imul $x 2) (ishl $x 1))
",
|p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![
|i: &mut dyn FnMut(u64) -> MatchResult| vec![
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Imul.into())),
(IntegerValue { path: p(&[0, 0]) }, i(2)),
(Opcode(LhsId(0)), Ok(TestOperator::Imul.into())),
(IntegerValue(LhsId(1)), i(2)),
(Nop, Err(Else))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Imul.into())),
(IntegerValue { path: p(&[0, 0]) }, i(1)),
(Opcode(LhsId(0)), Ok(TestOperator::Imul.into())),
(IntegerValue(LhsId(1)), i(1)),
(Nop, Err(Else))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Imul.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Imul.into())),
(Nop, Err(Else)),
(IntegerValue { path: p(&[0, 1]) }, i(2))
(IntegerValue(LhsId(2)), i(2))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Imul.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Imul.into())),
(Nop, Err(Else)),
(IntegerValue { path: p(&[0, 1]) }, i(1))
(IntegerValue(LhsId(2)), i(1))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Iadd.into())),
(IntegerValue { path: p(&[0, 0]) }, i(0)),
(Opcode(LhsId(0)), Ok(TestOperator::Iadd.into())),
(IntegerValue(LhsId(1)), i(0)),
(Nop, Err(Else))
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Iadd.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Iadd.into())),
(Nop, Err(Else)),
(IntegerValue { path: p(&[0, 1]) }, i(0))
(IntegerValue(LhsId(2)), i(0))
]
]
);
@@ -597,32 +546,20 @@ mod tests {
(=> (bor (bor $x $y) $y)
(bor $x $y))
",
|p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![
|i: &mut dyn FnMut(u64) -> MatchResult| vec![
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Bor.into())),
(Opcode { path: p(&[0, 0]) }, Ok(TestOperator::Bor.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Bor.into())),
(Opcode(LhsId(1)), Ok(TestOperator::Bor.into())),
(Nop, Err(Else)),
(Eq(LhsId(3), LhsId(2)), bool_to_match_result(true)),
(Nop, Err(Else)),
(
Eq {
path_a: p(&[0, 1]),
path_b: p(&[0, 0, 0]),
},
bool_to_match_result(true)
),
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Bor.into())),
(Opcode { path: p(&[0, 0]) }, Ok(TestOperator::Bor.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Bor.into())),
(Opcode(LhsId(1)), Ok(TestOperator::Bor.into())),
(Nop, Err(Else)),
(Nop, Err(Else)),
(
Eq {
path_a: p(&[0, 1]),
path_b: p(&[0, 0, 1]),
},
bool_to_match_result(true)
),
(Eq(LhsId(4), LhsId(2)), bool_to_match_result(true)),
],
]
);
@@ -636,39 +573,21 @@ mod tests {
(=> (bor (bor $x $y) $y)
(bor $x $y))
",
|p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![
|i: &mut dyn FnMut(u64) -> MatchResult| vec![
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Bor.into())),
(Opcode { path: p(&[0, 0]) }, Ok(TestOperator::Bor.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Bor.into())),
(Opcode(LhsId(1)), Ok(TestOperator::Bor.into())),
(Nop, Err(Else)),
(Eq(LhsId(3), LhsId(2)), bool_to_match_result(true)),
(Nop, Err(Else)),
(
Eq {
path_a: p(&[0, 1]),
path_b: p(&[0, 0, 0]),
},
bool_to_match_result(true)
),
],
vec![
(Opcode { path: p(&[0]) }, Ok(TestOperator::Bor.into())),
(Opcode { path: p(&[0, 0]) }, Ok(TestOperator::Bor.into())),
(Opcode(LhsId(0)), Ok(TestOperator::Bor.into())),
(Opcode(LhsId(1)), Ok(TestOperator::Bor.into())),
(Nop, Err(Else)),
(Eq(LhsId(3), LhsId(2)), Err(Else)),
(Nop, Err(Else)),
(
Eq {
path_a: p(&[0, 1]),
path_b: p(&[0, 0, 0]),
},
Err(Else),
),
(
Eq {
path_a: p(&[0, 1]),
path_b: p(&[0, 0, 1]),
},
bool_to_match_result(true)
),
(Eq(LhsId(4), LhsId(2)), bool_to_match_result(true)),
],
]
);

293
cranelift/peepmatic/src/linearize.rs
View File

@@ -49,14 +49,8 @@
//!
//! Here are the general principles that linearization should adhere to:
//!
//! * Actions should be pushed as early in the optimization's match chain as
//! they can be. This means the tail has fewer side effects, and is therefore
//! more likely to be share-able with other optimizations in the automata that
//! we build.
//!
//! * RHS actions cannot reference matches from the LHS until they've been
//! defined. And finally, an RHS operation's operands must be defined before
//! the RHS operation itself. In general, definitions must come before uses!
//! * Don't match on a subtree until we know it exists. That is, match on
//! parents before matching on children.
//!
//! * Shorter match chains are better! This means fewer tests when matching
//! left-hand sides, and a more-compact, more-cache-friendly automata, and
@@ -76,13 +70,15 @@
//! precondition.
//!
//! Within matching the pattern structure, we emit matching operations in a
//! pre-order traversal of the pattern. This ensures that we've already matched
//! an operation before we consider its operands, and therefore we already know
//! the operands exist. See `PatternPreOrder` for details.
//! breadth-first traversal of the pattern. This ensures that we've already
//! matched an operation before we consider its operands, and therefore we
//! already know the operands exist. It also lets us fuse "what opcode does this
//! instruction have?" and "define temporary variables for this instruction's
//! operands" into a single operation. See `PatternBfs` for details.
//!
//! As we define the match operations for a pattern, we remember the path where
//! each LHS id first occurred. These will later be reused when building the RHS
//! actions. See `LhsIdToPath` for details.
//! actions. See `LhsCanonicalizer` for details.
//!
//! After we've generated the match operations and expected result of those
//! match operations, then we generate the right-hand side actions. The
@@ -94,12 +90,8 @@
//! linear optimization translation function.
use crate::ast::*;
use crate::traversals::Dfs;
use peepmatic_runtime::{
integer_interner::IntegerInterner,
linear,
paths::{Path, PathId, PathInterner},
};
use crate::traversals::{Bfs, Dfs};
use peepmatic_runtime::{integer_interner::IntegerInterner, linear};
use std::collections::BTreeMap;
use std::convert::TryInto;
use std::fmt::Debug;
@@ -114,22 +106,19 @@ where
TOperator: Copy + Debug + Eq + Hash + Into<NonZeroU32>,
{
let mut optimizations = vec![];
let mut paths = PathInterner::new();
let mut integers = IntegerInterner::new();
for opt in &opts.optimizations {
let lin_opt = linearize_optimization(&mut paths, &mut integers, opt);
let lin_opt = linearize_optimization(&mut integers, opt);
optimizations.push(lin_opt);
}
linear::Optimizations {
optimizations,
paths,
integers,
}
}
/// Translate an AST optimization into a linear optimization!
fn linearize_optimization<TOperator>(
paths: &mut PathInterner,
integers: &mut IntegerInterner,
opt: &Optimization<TOperator>,
) -> linear::Optimization<TOperator>
@@ -138,22 +127,21 @@ where
{
let mut matches: Vec<linear::Match> = vec![];
let mut lhs_id_to_path = LhsIdToPath::new();
let mut lhs_canonicalizer = LhsCanonicalizer::new();
// We do a pre-order traversal of the LHS because we don't know whether a
// child actually exists to match on until we've matched its parent, and we
// don't want to emit matching operations on things that might not exist!
let mut patterns = PatternPreOrder::new(&opt.lhs.pattern);
while let Some((path, pattern)) = patterns.next(paths) {
// We do a breadth-first traversal of the LHS because we don't know whether
// a child actually exists to match on until we've matched its parent, and
// we don't want to emit matching operations on things that might not exist!
for (id, pattern) in PatternBfs::new(&opt.lhs.pattern) {
// Create the matching parts of an `Match` for this part of the
// pattern.
let (operation, expected) = pattern.to_linear_match_op(integers, &lhs_id_to_path, path);
let (operation, expected) = pattern.to_linear_match_op(integers, &lhs_canonicalizer, id);
matches.push(linear::Match {
operation,
expected,
});
lhs_id_to_path.remember_path_to_pattern_ids(pattern, path);
lhs_canonicalizer.remember_linear_id(pattern, id);
// Some operations require type ascriptions for us to infer the correct
// bit width of their results: `ireduce`, `sextend`, `uextend`, etc.
@@ -165,7 +153,7 @@ where
let expected = Ok(unsafe { NonZeroU32::new_unchecked(w as u32) });
matches.push(linear::Match {
operation: linear::MatchOp::BitWidth { path },
operation: linear::MatchOp::BitWidth(id),
expected,
});
}
@@ -175,7 +163,7 @@ where
// Now that we've added all the matches for the LHS pattern, add the
// matches for its preconditions.
for pre in &opt.lhs.preconditions {
matches.push(pre.to_linear_match(&lhs_id_to_path));
matches.push(pre.to_linear_match(&lhs_canonicalizer));
}
assert!(!matches.is_empty());
@@ -183,7 +171,7 @@ where
// Finally, generate the RHS-building actions and attach them to the first match.
let mut rhs_builder = RhsBuilder::new(&opt.rhs);
let mut actions = vec![];
rhs_builder.add_rhs_build_actions(integers, &lhs_id_to_path, &mut actions);
rhs_builder.add_rhs_build_actions(integers, &lhs_canonicalizer, &mut actions);
linear::Optimization { matches, actions }
}
@@ -222,55 +210,46 @@ where
}
}
/// A pre-order, depth-first traversal of left-hand side patterns.
/// A breadth-first traversal of left-hand side patterns.
///
/// Keeps track of the path to each pattern, and yields it along side the
/// Keeps track of the `LhsId` of each pattern, and yields it along side the
/// pattern AST node.
struct PatternPreOrder<'a, TOperator> {
last_child: Option<u8>,
path: Vec<u8>,
dfs: Dfs<'a, TOperator>,
///
/// We use a breadth-first traversal because we fuse "which opcode is this?" and
/// "assign operands to temporaries" into a single linear match operation. A
/// breadth-first traversal aligns with "match this opcode, and on success bind
/// all of its operands to temporaries". Fusing these operations into one is
/// important for attaining similar performance as an open-coded Rust `match`
/// expression, which would also fuse these operations via pattern matching.
struct PatternBfs<'a, TOperator> {
next_id: u16,
bfs: Bfs<'a, TOperator>,
}
impl<'a, TOperator> PatternPreOrder<'a, TOperator>
impl<'a, TOperator> PatternBfs<'a, TOperator>
where
TOperator: Copy + Debug + Eq + Hash,
{
fn new(pattern: &'a Pattern<'a, TOperator>) -> Self {
Self {
last_child: None,
path: vec![],
dfs: Dfs::new(pattern),
next_id: 0,
bfs: Bfs::new(pattern),
}
}
}
fn next(&mut self, paths: &mut PathInterner) -> Option<(PathId, &'a Pattern<'a, TOperator>)> {
use crate::traversals::TraversalEvent as TE;
impl<'a, TOperator> Iterator for PatternBfs<'a, TOperator>
where
TOperator: 'a + Copy + Debug + Eq + Hash,
{
type Item = (linear::LhsId, &'a Pattern<'a, TOperator>);
fn next(&mut self) -> Option<Self::Item> {
loop {
match self.dfs.next()? {
(TE::Enter, DynAstRef::Pattern(pattern)) => {
let last_child = self.last_child.take();
self.path.push(match last_child {
None => 0,
Some(c) => {
assert!(
c < std::u8::MAX,
"operators must have less than or equal u8::MAX arity"
);
c + 1
}
});
let path = paths.intern(Path(&self.path));
return Some((path, pattern));
}
(TE::Exit, DynAstRef::Pattern(_)) => {
self.last_child = Some(
self.path
.pop()
.expect("should always have a non-empty path during traversal"),
);
}
_ => {}
if let DynAstRef::Pattern(pattern) = self.bfs.next()? {
let id = linear::LhsId(self.next_id);
self.next_id = self.next_id.checked_add(1).unwrap();
return Some((id, pattern));
}
}
}
@@ -278,42 +257,37 @@ where
/// A map from left-hand side identifiers to the path in the left-hand side
/// where they first occurred.
struct LhsIdToPath<'a, TOperator> {
id_to_path: BTreeMap<&'a str, PathId>,
struct LhsCanonicalizer<'a, TOperator> {
id_to_linear: BTreeMap<&'a str, linear::LhsId>,
_marker: PhantomData<&'a TOperator>,
}
impl<'a, TOperator> LhsIdToPath<'a, TOperator> {
/// Construct a new, empty `LhsIdToPath`.
impl<'a, TOperator> LhsCanonicalizer<'a, TOperator> {
/// Construct a new, empty `LhsCanonicalizer`.
fn new() -> Self {
Self {
id_to_path: Default::default(),
id_to_linear: Default::default(),
_marker: PhantomData,
}
}
/// Have we already seen the given identifier?
fn get_first_occurrence(&self, id: &Id) -> Option<PathId> {
self.id_to_path.get(id.name()).copied()
/// Get the canonical `linear::LhsId` for the given variable, if any.
fn get(&self, id: &Id) -> Option<linear::LhsId> {
self.id_to_linear.get(id.name()).copied()
}
/// Get the path within the left-hand side pattern where we first saw the
/// given AST id.
///
/// ## Panics
///
/// Panics if the given AST id has not already been canonicalized.
fn unwrap_first_occurrence(&self, id: &Id) -> PathId {
self.id_to_path[id.name()]
}
/// Remember the path to any LHS ids used in the given pattern.
fn remember_path_to_pattern_ids(&mut self, pattern: &'a Pattern<'a, TOperator>, path: PathId) {
/// Remember the canonical `linear::LhsId` for any variables or constants
/// used in the given pattern.
fn remember_linear_id(
&mut self,
pattern: &'a Pattern<'a, TOperator>,
linear_id: linear::LhsId,
) {
match pattern {
// If this is the first time we've seen an identifier defined on the
// left-hand side, remember it.
Pattern::Variable(Variable { id, .. }) | Pattern::Constant(Constant { id, .. }) => {
self.id_to_path.entry(id.name()).or_insert(path);
self.id_to_linear.entry(id.name()).or_insert(linear_id);
}
_ => {}
}
@@ -366,11 +340,11 @@ where
fn add_rhs_build_actions(
&mut self,
integers: &mut IntegerInterner,
lhs_id_to_path: &LhsIdToPath<TOperator>,
lhs_canonicalizer: &LhsCanonicalizer<TOperator>,
actions: &mut Vec<linear::Action<TOperator>>,
) {
while let Some(rhs) = self.rhs_post_order.next() {
actions.push(self.rhs_to_linear_action(integers, lhs_id_to_path, rhs));
actions.push(self.rhs_to_linear_action(integers, lhs_canonicalizer, rhs));
let id = linear::RhsId(self.rhs_span_to_id.len().try_into().unwrap());
self.rhs_span_to_id.insert(rhs.span(), id);
}
@@ -379,7 +353,7 @@ where
fn rhs_to_linear_action(
&self,
integers: &mut IntegerInterner,
lhs_id_to_path: &LhsIdToPath<TOperator>,
lhs_canonicalizer: &LhsCanonicalizer<TOperator>,
rhs: &Rhs<TOperator>,
) -> linear::Action<TOperator> {
match rhs {
@@ -401,8 +375,8 @@ where
linear::Action::MakeConditionCode { cc: *cc }
}
Rhs::Variable(Variable { id, .. }) | Rhs::Constant(Constant { id, .. }) => {
let path = lhs_id_to_path.unwrap_first_occurrence(id);
linear::Action::GetLhs { path }
let lhs = lhs_canonicalizer.get(id).unwrap();
linear::Action::GetLhs { lhs }
}
Rhs::Unquote(unq) => match unq.operands.len() {
1 => linear::Action::UnaryUnquote {
@@ -461,16 +435,16 @@ where
TOperator: Copy + Debug + Eq + Hash + Into<NonZeroU32>,
{
/// Convert this precondition into a `linear::Match`.
fn to_linear_match(&self, lhs_id_to_path: &LhsIdToPath<TOperator>) -> linear::Match {
fn to_linear_match(&self, lhs_canonicalizer: &LhsCanonicalizer<TOperator>) -> linear::Match {
match self.constraint {
Constraint::IsPowerOfTwo => {
let id = match &self.operands[0] {
ConstraintOperand::Constant(Constant { id, .. }) => id,
_ => unreachable!("checked in verification"),
};
let path = lhs_id_to_path.unwrap_first_occurrence(&id);
let id = lhs_canonicalizer.get(&id).unwrap();
linear::Match {
operation: linear::MatchOp::IsPowerOfTwo { path },
operation: linear::MatchOp::IsPowerOfTwo(id),
expected: linear::bool_to_match_result(true),
}
}
@@ -480,7 +454,7 @@ where
| ConstraintOperand::Variable(Variable { id, .. }) => id,
_ => unreachable!("checked in verification"),
};
let path = lhs_id_to_path.unwrap_first_occurrence(&id);
let id = lhs_canonicalizer.get(&id).unwrap();
let width = match &self.operands[1] {
ConstraintOperand::ValueLiteral(ValueLiteral::Integer(Integer {
@@ -495,7 +469,7 @@ where
let expected = Ok(unsafe { NonZeroU32::new_unchecked(width as u32) });
linear::Match {
operation: linear::MatchOp::BitWidth { path },
operation: linear::MatchOp::BitWidth(id),
expected,
}
}
@@ -505,9 +479,9 @@ where
| ConstraintOperand::Variable(Variable { id, .. }) => id,
_ => unreachable!("checked in verification"),
};
let path = lhs_id_to_path.unwrap_first_occurrence(&id);
let id = lhs_canonicalizer.get(&id).unwrap();
linear::Match {
operation: linear::MatchOp::FitsInNativeWord { path },
operation: linear::MatchOp::FitsInNativeWord(id),
expected: linear::bool_to_match_result(true),
}
}
@@ -527,52 +501,46 @@ where
fn to_linear_match_op(
&self,
integers: &mut IntegerInterner,
lhs_id_to_path: &LhsIdToPath<TOperator>,
path: PathId,
lhs_canonicalizer: &LhsCanonicalizer<TOperator>,
linear_lhs_id: linear::LhsId,
) -> (linear::MatchOp, linear::MatchResult)
where
TOperator: Into<NonZeroU32>,
{
match self {
Pattern::ValueLiteral(ValueLiteral::Integer(Integer { value, .. })) => (
linear::MatchOp::IntegerValue { path },
linear::MatchOp::IntegerValue(linear_lhs_id),
Ok(integers.intern(*value as u64).into()),
),
Pattern::ValueLiteral(ValueLiteral::Boolean(Boolean { value, .. })) => (
linear::MatchOp::BooleanValue { path },
linear::MatchOp::BooleanValue(linear_lhs_id),
linear::bool_to_match_result(*value),
),
Pattern::ValueLiteral(ValueLiteral::ConditionCode(ConditionCode { cc, .. })) => {
let cc = *cc as u32;
debug_assert!(cc != 0, "no `ConditionCode` variants are zero");
let expected = Ok(unsafe { NonZeroU32::new_unchecked(cc) });
(linear::MatchOp::ConditionCode { path }, expected)
(linear::MatchOp::ConditionCode(linear_lhs_id), expected)
}
Pattern::Constant(Constant { id, .. }) => {
if let Some(path_b) = lhs_id_to_path.get_first_occurrence(id) {
debug_assert!(path != path_b);
if let Some(linear_lhs_id2) = lhs_canonicalizer.get(id) {
debug_assert!(linear_lhs_id != linear_lhs_id2);
(
linear::MatchOp::Eq {
path_a: path,
path_b,
},
linear::MatchOp::Eq(linear_lhs_id, linear_lhs_id2),
linear::bool_to_match_result(true),
)
} else {
(
linear::MatchOp::IsConst { path },
linear::MatchOp::IsConst(linear_lhs_id),
linear::bool_to_match_result(true),
)
}
}
Pattern::Variable(Variable { id, .. }) => {
if let Some(path_b) = lhs_id_to_path.get_first_occurrence(id) {
debug_assert!(path != path_b);
if let Some(linear_lhs_id2) = lhs_canonicalizer.get(id) {
debug_assert!(linear_lhs_id != linear_lhs_id2);
(
linear::MatchOp::Eq {
path_a: path,
path_b,
},
linear::MatchOp::Eq(linear_lhs_id, linear_lhs_id2),
linear::bool_to_match_result(true),
)
} else {
@@ -581,7 +549,7 @@ where
}
Pattern::Operation(op) => {
let expected = Ok(op.operator.into());
(linear::MatchOp::Opcode { path }, expected)
(linear::MatchOp::Opcode(linear_lhs_id), expected)
}
}
}
@@ -592,7 +560,7 @@ mod tests {
use super::*;
use peepmatic_runtime::{
integer_interner::IntegerId,
linear::{bool_to_match_result, Action::*, Else, MatchOp::*},
linear::{bool_to_match_result, Action::*, Else, LhsId, MatchOp::*, RhsId},
r#type::{BitWidth, Kind, Type},
unquote::UnquoteOperator,
};
@@ -628,20 +596,16 @@ mod tests {
panic!("should verify OK")
}
let mut paths = PathInterner::new();
let mut p = |p: &[u8]| paths.intern(Path::new(&p));
let mut integers = IntegerInterner::new();
let mut i = |i: u64| integers.intern(i);
#[allow(unused_variables)]
let make_expected: fn(
&mut dyn FnMut(&[u8]) -> PathId,
&mut dyn FnMut(u64) -> IntegerId,
) -> (Vec<linear::Match>, Vec<linear::Action<_>>) = $make_expected;
let expected = make_expected(&mut p, &mut i);
let actual = linearize_optimization(&mut paths, &mut integers, &opts.optimizations[0]);
let expected = make_expected(&mut i);
let actual = linearize_optimization(&mut integers, &opts.optimizations[0]);
assert_eq!(expected.0, actual.matches);
assert_eq!(expected.1, actual.actions);
}
@@ -655,10 +619,10 @@ mod tests {
(is-power-of-two $C))
(ishl $x $(log2 $C)))
",
|p, i| (
|i| (
vec![
linear::Match {
operation: Opcode { path: p(&[0]) },
operation: Opcode(LhsId(0)),
expected: Ok(TestOperator::Imul.into()),
},
linear::Match {
@@ -666,20 +630,20 @@ mod tests {
expected: Err(Else),
},
linear::Match {
operation: IsConst { path: p(&[0, 1]) },
operation: IsConst(LhsId(2)),
expected: bool_to_match_result(true),
},
linear::Match {
operation: IsPowerOfTwo { path: p(&[0, 1]) },
operation: IsPowerOfTwo(LhsId(2)),
expected: bool_to_match_result(true),
},
],
vec![
GetLhs { path: p(&[0, 0]) },
GetLhs { path: p(&[0, 1]) },
GetLhs { lhs: LhsId(1) },
GetLhs { lhs: LhsId(2) },
UnaryUnquote {
operator: UnquoteOperator::Log2,
operand: linear::RhsId(1)
operand: RhsId(1)
},
MakeBinaryInst {
operator: TestOperator::Ishl,
@@ -687,31 +651,31 @@ mod tests {
kind: Kind::Int,
bit_width: BitWidth::Polymorphic
},
operands: [linear::RhsId(0), linear::RhsId(2)]
operands: [RhsId(0), RhsId(2)]
}
],
),
);
linearizes_to!(variable_pattern_id_optimization, "(=> $x $x)", |p, i| (
linearizes_to!(variable_pattern_id_optimization, "(=> $x $x)", |i| (
vec![linear::Match {
operation: Nop,
expected: Err(Else),
}],
vec![GetLhs { path: p(&[0]) }],
vec![GetLhs { lhs: LhsId(0) }],
));
linearizes_to!(constant_pattern_id_optimization, "(=> $C $C)", |p, i| (
linearizes_to!(constant_pattern_id_optimization, "(=> $C $C)", |i| (
vec![linear::Match {
operation: IsConst { path: p(&[0]) },
operation: IsConst(LhsId(0)),
expected: bool_to_match_result(true),
}],
vec![GetLhs { path: p(&[0]) }],
vec![GetLhs { lhs: LhsId(0) }],
));
linearizes_to!(boolean_literal_id_optimization, "(=> true true)", |p, i| (
linearizes_to!(boolean_literal_id_optimization, "(=> true true)", |i| (
vec![linear::Match {
operation: BooleanValue { path: p(&[0]) },
operation: BooleanValue(LhsId(0)),
expected: bool_to_match_result(true),
}],
vec![MakeBooleanConst {
@@ -720,9 +684,9 @@ mod tests {
}],
));
linearizes_to!(number_literal_id_optimization, "(=> 5 5)", |p, i| (
linearizes_to!(number_literal_id_optimization, "(=> 5 5)", |i| (
vec![linear::Match {
operation: IntegerValue { path: p(&[0]) },
operation: IntegerValue(LhsId(0)),
expected: Ok(i(5).into()),
}],
vec![MakeIntegerConst {
@@ -734,26 +698,26 @@ mod tests {
linearizes_to!(
operation_id_optimization,
"(=> (iconst $C) (iconst $C))",
|p, i| (
|i| (
vec![
linear::Match {
operation: Opcode { path: p(&[0]) },
operation: Opcode(LhsId(0)),
expected: Ok(TestOperator::Iconst.into()),
},
linear::Match {
operation: IsConst { path: p(&[0, 0]) },
operation: IsConst(LhsId(1)),
expected: bool_to_match_result(true),
},
],
vec![
GetLhs { path: p(&[0, 0]) },
GetLhs { lhs: LhsId(1) },
MakeUnaryInst {
operator: TestOperator::Iconst,
r#type: Type {
kind: Kind::Int,
bit_width: BitWidth::Polymorphic,
},
operand: linear::RhsId(0),
operand: RhsId(0),
},
],
),
@@ -762,10 +726,10 @@ mod tests {
linearizes_to!(
redundant_bor,
"(=> (bor $x (bor $x $y)) (bor $x $y))",
|p, i| (
|i| (
vec![
linear::Match {
operation: Opcode { path: p(&[0]) },
operation: Opcode(LhsId(0)),
expected: Ok(TestOperator::Bor.into()),
},
linear::Match {
@@ -773,14 +737,11 @@ mod tests {
expected: Err(Else),
},
linear::Match {
operation: Opcode { path: p(&[0, 1]) },
operation: Opcode(LhsId(2)),
expected: Ok(TestOperator::Bor.into()),
},
linear::Match {
operation: Eq {
path_a: p(&[0, 1, 0]),
path_b: p(&[0, 0]),
},
operation: Eq(LhsId(3), LhsId(1)),
expected: bool_to_match_result(true),
},
linear::Match {
@@ -789,17 +750,15 @@ mod tests {
},
],
vec![
GetLhs { path: p(&[0, 0]) },
GetLhs {
path: p(&[0, 1, 1]),
},
GetLhs { lhs: LhsId(1) },
GetLhs { lhs: LhsId(4) },
MakeBinaryInst {
operator: TestOperator::Bor,
r#type: Type {
kind: Kind::Int,
bit_width: BitWidth::Polymorphic,
},
operands: [linear::RhsId(0), linear::RhsId(1)],
operands: [RhsId(0), RhsId(1)],
},
],
),
@@ -809,9 +768,9 @@ mod tests {
large_integers,
// u64::MAX
"(=> 18446744073709551615 0)",
|p, i| (
|i| (
vec![linear::Match {
operation: IntegerValue { path: p(&[0]) },
operation: IntegerValue(LhsId(0)),
expected: Ok(i(std::u64::MAX).into()),
}],
vec![MakeIntegerConst {
@@ -824,14 +783,14 @@ mod tests {
linearizes_to!(
ireduce_with_type_ascription,
"(=> (ireduce{i32} $x) 0)",
|p, i| (
|i| (
vec![
linear::Match {
operation: Opcode { path: p(&[0]) },
operation: Opcode(LhsId(0)),
expected: Ok(TestOperator::Ireduce.into()),
},
linear::Match {
operation: linear::MatchOp::BitWidth { path: p(&[0]) },
operation: linear::MatchOp::BitWidth(LhsId(0)),
expected: Ok(NonZeroU32::new(32).unwrap()),
},
linear::Match {

39
cranelift/peepmatic/src/traversals.rs
View File

@@ -1,6 +1,7 @@
//! Traversals over the AST.
use crate::ast::*;
use std::collections::VecDeque;
use std::fmt::Debug;
use std::hash::Hash;
@@ -103,6 +104,44 @@ where
}
}
/// A breadth-first traversal of an AST
///
/// This implementation is not recursive, and exposes an `Iterator` interface
/// that yields `DynAstRef` items.
///
/// The traversal can walk a whole set of `Optimization`s or just a subtree of
/// the AST, because the `new` constructor takes anything that can convert into
/// a `DynAstRef`.
#[derive(Clone, Debug)]
pub struct Bfs<'a, TOperator> {
queue: VecDeque<DynAstRef<'a, TOperator>>,
}
impl<'a, TOperator> Bfs<'a, TOperator>
where
TOperator: Copy + Debug + Eq + Hash,
{
/// Construct a new `Bfs` traversal starting at the given `start` AST node.
pub fn new(start: impl Into<DynAstRef<'a, TOperator>>) -> Self {
let mut queue = VecDeque::with_capacity(16);
queue.push_back(start.into());
Bfs { queue }
}
}
impl<'a, TOperator> Iterator for Bfs<'a, TOperator>
where
TOperator: Copy + Debug + Eq + Hash,
{
type Item = DynAstRef<'a, TOperator>;
fn next(&mut self) -> Option<Self::Item> {
let node = self.queue.pop_front()?;
node.child_nodes(&mut self.queue);
Some(node)
}
}
#[cfg(test)]
mod tests {
use super::*;