From c82326a1ae8c63d1d32f4b784fae16c101f164e3 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 1 May 2020 15:30:37 -0700 Subject: [PATCH 01/22] peepmatic: Introduce the `peepmatic-automata` crate The `peepmatic-automata` crate builds and queries finite-state transducer automata. A transducer is a type of automata that has not only an input that it accepts or rejects, but also an output. While regular automata check whether an input string is in the set that the automata accepts, a transducer maps the input strings to values. A regular automata is sort of a compressed, immutable set, and a transducer is sort of a compressed, immutable key-value dictionary. A [trie] compresses a set of strings or map from a string to a value by sharing prefixes of the input string. Automata and transducers can compress even better: they can share both prefixes and suffixes. [*Index 1,600,000,000 Keys with Automata and Rust* by Andrew Gallant (aka burntsushi)][burntsushi-blog-post] is a top-notch introduction. If you're looking for a general-purpose transducers crate in Rust you're probably looking for [the `fst` crate][fst-crate]. While this implementation is fully generic and has no dependencies, its feature set is specific to `peepmatic`'s needs: * We need to associate extra data with each state: the match operation to evaluate next. * We can't provide the full input string up front, so this crate must support incremental lookups. This is because the peephole optimizer is computing the input string incrementally and dynamically: it looks at the current state's match operation, evaluates it, and then uses the result as the next character of the input string. * We also support incremental insertion and output when building the transducer. This is necessary because we don't want to emit output values that bind a match on an optimization's left-hand side's pattern (for example) until after we've succeeded in matching it, which might not happen until we've reached the n^th state. * We need to support generic output values. The `fst` crate only supports `u64` outputs, while we need to build up an optimization's right-hand side instructions. This implementation is based on [*Direct Construction of Minimal Acyclic Subsequential Transducers* by Mihov and Maurel][paper]. That means that keys must be inserted in lexicographic order during construction. [trie]: https://en.wikipedia.org/wiki/Trie [burntsushi-blog-post]: https://blog.burntsushi.net/transducers/#ordered-maps [fst-crate]: https://crates.io/crates/fst [paper]: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.24.3698&rep=rep1&type=pdf --- .../peepmatic/crates/automata/Cargo.toml | 18 + .../peepmatic/crates/automata/src/dot.rs | 273 +++++ .../peepmatic/crates/automata/src/lib.rs | 1024 +++++++++++++++++ .../crates/automata/src/output_impls.rs | 93 ++ .../crates/automata/src/serde_impls.rs | 195 ++++ 5 files changed, 1603 insertions(+) create mode 100644 cranelift/peepmatic/crates/automata/Cargo.toml create mode 100644 cranelift/peepmatic/crates/automata/src/dot.rs create mode 100644 cranelift/peepmatic/crates/automata/src/lib.rs create mode 100644 cranelift/peepmatic/crates/automata/src/output_impls.rs create mode 100644 cranelift/peepmatic/crates/automata/src/serde_impls.rs diff --git a/cranelift/peepmatic/crates/automata/Cargo.toml b/cranelift/peepmatic/crates/automata/Cargo.toml new file mode 100644 index 0000000000..27c89230af --- /dev/null +++ b/cranelift/peepmatic/crates/automata/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "peepmatic-automata" +version = "0.1.0" +authors = ["Nick Fitzgerald "] +edition = "2018" + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +serde = { version = "1.0.106", optional = true } + +[features] +# Enable support for generating GraphViz Dot files that can be used to visually +# render an automaton. +# +# https://en.wikipedia.org/wiki/DOT_%28graph_description_language%29 +dot = [] diff --git a/cranelift/peepmatic/crates/automata/src/dot.rs b/cranelift/peepmatic/crates/automata/src/dot.rs new file mode 100644 index 0000000000..bc102d1f81 --- /dev/null +++ b/cranelift/peepmatic/crates/automata/src/dot.rs @@ -0,0 +1,273 @@ +//! Helpers for generating [GraphViz +//! Dot](https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf) files to visually +//! render automata. +//! +//! **This module only exists when the `"dot"` cargo feature is enabled.** + +use crate::{Automaton, Output, State}; +use std::fmt::{Debug, Display}; +use std::fs; +use std::hash::Hash; +use std::io::{self, Write}; +use std::path::Path; + +/// Format the user-provided bits of an `Automaton` for Graphviz Dot output. +/// +/// There are two provided implementations of `DotFmt`: +/// +/// * [`DebugDotFmt`][crate::dot::DebugDotFmt] -- format each type parameter +/// with its `std::fmt::Debug` implementation. +/// +/// * [`DisplayDotFmt`][crate::dot::DisplayDotFmt] -- format each type parameter +/// with its `std::fmt::Display` implementation. +/// +/// You can also implement this trait yourself if your type parameters don't +/// implement `Debug` or `Display`, or if you want to format them in some other +/// way. +pub trait DotFmt { + /// Format a transition edge: `from ---input---> to`. + /// + /// This will be inside an [HTML + /// label](https://www.graphviz.org/doc/info/shapes.html#html), so you may + /// use balanced HTML tags. + fn fmt_transition( + &self, + w: &mut impl Write, + from: Option<&TState>, + input: &TAlphabet, + to: Option<&TState>, + ) -> io::Result<()>; + + /// Format the custom data associated with a state. + /// + /// This will be inside an [HTML + /// label](https://www.graphviz.org/doc/info/shapes.html#html), so you may + /// use balanced HTML tags. + fn fmt_state(&self, w: &mut impl Write, state: &TState) -> io::Result<()>; + + /// Format a transition's output or the final output of a final state. + /// + /// This will be inside an [HTML + /// label](https://www.graphviz.org/doc/info/shapes.html#html), so you may + /// use balanced HTML tags. + fn fmt_output(&self, w: &mut impl Write, output: &TOutput) -> io::Result<()>; +} + +impl Automaton +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + /// Write this `Automaton` out as a [GraphViz + /// Dot](https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf) file at the + /// given path. + /// + /// The `formatter` parameter controls how `TAlphabet`, `TState`, and + /// `TOutput` are rendered. See the [`DotFmt`][crate::dot::DotFmt] trait for + /// details. + /// + /// **This method only exists when the `"dot"` cargo feature is enabled.** + pub fn write_dot_file( + &self, + formatter: &impl DotFmt, + path: impl AsRef, + ) -> io::Result<()> { + let mut file = fs::File::create(path)?; + self.write_dot(formatter, &mut file)?; + Ok(()) + } + + /// Write this `Automaton` out to the given write-able as a [GraphViz + /// Dot](https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf) file. + /// + /// The `formatter` parameter controls how `TAlphabet`, `TState`, and + /// `TOutput` are rendered. See the [`DotFmt`][crate::dot::DotFmt] trait for + /// details. + /// + /// **This method only exists when the `"dot"` cargo feature is enabled.** + pub fn write_dot( + &self, + formatter: &impl DotFmt, + w: &mut impl Write, + ) -> io::Result<()> { + writeln!(w, "digraph {{")?; + writeln!(w, " rankdir = \"LR\";")?; + writeln!(w, " nodesep = 2;")?; + + // Fake state for the incoming arrow to the start state. + writeln!(w, " \"\" [shape = none];")?; + + // Each state, its associated custom data, and its final output. + for (i, state_data) in self.state_data.iter().enumerate() { + write!( + w, + r#" state_{i} [shape = {shape}, label = <")?; + if let Some(final_output) = self.final_states.get(&State(i as u32)) { + write!(w, r#"")?; + } + writeln!(w, "
{i}
"#, + i = i, + shape = if self.final_states.contains_key(&State(i as u32)) { + "doublecircle" + } else { + "circle" + } + )?; + if let Some(state_data) = state_data { + formatter.fmt_state(w, state_data)?; + } else { + write!(w, "(no state data)")?; + } + write!(w, "
"#)?; + formatter.fmt_output(w, final_output)?; + write!(w, "
>];")?; + } + + // Fake transition to the start state. + writeln!(w, r#" "" -> state_{};"#, self.start_state.0)?; + + // Transitions between states and their outputs. + for (from, transitions) in self.transitions.iter().enumerate() { + for (input, (to, output)) in transitions { + write!( + w, + r#" state_{from} -> state_{to} [label = <
Input:"#, + from = from, + to = to.0, + )?; + formatter.fmt_transition( + w, + self.state_data[from].as_ref(), + input, + self.state_data[to.0 as usize].as_ref(), + )?; + write!( + w, + r#"
Output:"#, + )?; + formatter.fmt_output(w, output)?; + writeln!(w, "
>];")?; + } + } + + writeln!(w, "}}")?; + Ok(()) + } +} + +/// Format an `Automaton`'s `TAlphabet`, `TState`, and `TOutput` with their +/// `std::fmt::Debug` implementations. +#[derive(Debug)] +pub struct DebugDotFmt; + +impl DotFmt for DebugDotFmt +where + TAlphabet: Debug, + TState: Debug, + TOutput: Debug, +{ + fn fmt_transition( + &self, + w: &mut impl Write, + _from: Option<&TState>, + input: &TAlphabet, + _to: Option<&TState>, + ) -> io::Result<()> { + write!(w, r#"{:?}"#, input) + } + + fn fmt_state(&self, w: &mut impl Write, state: &TState) -> io::Result<()> { + write!(w, r#"{:?}"#, state) + } + + fn fmt_output(&self, w: &mut impl Write, output: &TOutput) -> io::Result<()> { + write!(w, r#"{:?}"#, output) + } +} + +/// Format an `Automaton`'s `TAlphabet`, `TState`, and `TOutput` with their +/// `std::fmt::Display` implementations. +#[derive(Debug)] +pub struct DisplayDotFmt; + +impl DotFmt for DisplayDotFmt +where + TAlphabet: Display, + TState: Display, + TOutput: Display, +{ + fn fmt_transition( + &self, + w: &mut impl Write, + _from: Option<&TState>, + input: &TAlphabet, + _to: Option<&TState>, + ) -> io::Result<()> { + write!(w, "{}", input) + } + + fn fmt_state(&self, w: &mut impl Write, state: &TState) -> io::Result<()> { + write!(w, "{}", state) + } + + fn fmt_output(&self, w: &mut impl Write, output: &TOutput) -> io::Result<()> { + write!(w, "{}", output) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Builder; + + #[test] + fn test_write_dot() { + let mut builder = Builder::::new(); + + // Insert "mon" -> 1 + let mut insertion = builder.insert(); + insertion.next('m', 1).next('o', 0).next('n', 0); + insertion.finish(); + + // Insert "sat" -> 6 + let mut insertion = builder.insert(); + insertion.next('s', 6).next('a', 0).next('t', 0); + insertion.finish(); + + // Insert "sun" -> 0 + let mut insertion = builder.insert(); + insertion.next('s', 0).next('u', 0).next('n', 0); + insertion.finish(); + + let automata = builder.finish(); + + let expected = r#" +digraph { + rankdir = "LR"; + nodesep = 2; + "" [shape = none]; + state_0 [shape = doublecircle, label = <
0
(no state data)
0
>]; + state_1 [shape = circle, label = <
1
(no state data)
>]; + state_2 [shape = circle, label = <
2
(no state data)
>]; + state_3 [shape = circle, label = <
3
(no state data)
>]; + state_4 [shape = circle, label = <
4
(no state data)
>]; + state_5 [shape = circle, label = <
5
(no state data)
>]; + "" -> state_5; + state_1 -> state_0 [label = <
Input:'n'
Output:0
>]; + state_2 -> state_1 [label = <
Input:'o'
Output:0
>]; + state_3 -> state_0 [label = <
Input:'t'
Output:0
>]; + state_4 -> state_3 [label = <
Input:'a'
Output:6
>]; + state_4 -> state_1 [label = <
Input:'u'
Output:0
>]; + state_5 -> state_2 [label = <
Input:'m'
Output:1
>]; + state_5 -> state_4 [label = <
Input:'s'
Output:0
>]; +} +"#; + + let mut buf = vec![]; + automata.write_dot(&DebugDotFmt, &mut buf).unwrap(); + let actual = String::from_utf8(buf).unwrap(); + eprintln!("{}", actual); + assert_eq!(expected.trim(), actual.trim()); + } +} diff --git a/cranelift/peepmatic/crates/automata/src/lib.rs b/cranelift/peepmatic/crates/automata/src/lib.rs new file mode 100644 index 0000000000..6ed09a66e3 --- /dev/null +++ b/cranelift/peepmatic/crates/automata/src/lib.rs @@ -0,0 +1,1024 @@ +//! Finite-state transducer automata. +//! +//! A transducer is a type of automata that has not only an input that it +//! accepts or rejects, but also an output. While regular automata check whether +//! an input string is in the set that the automata accepts, a transducer maps +//! the input strings to values. A regular automata is sort of a compressed, +//! immutable set, and a transducer is sort of a compressed, immutable key-value +//! dictionary. A [trie] compresses a set of strings or map from a string to a +//! value by sharing prefixes of the input string. Automata and transducers can +//! compress even better: they can share both prefixes and suffixes. [*Index +//! 1,600,000,000 Keys with Automata and Rust* by Andrew Gallant (aka +//! burntsushi)][burntsushi-blog-post] is a top-notch introduction. +//! +//! If you're looking for a general-purpose transducers crate in Rust you're +//! probably looking for [the `fst` crate][fst-crate]. While this implementation +//! is fully generic and has no dependencies, its feature set is specific to +//! `peepmatic`'s needs: +//! +//! * We need to associate extra data with each state: the match operation to +//! evaluate next. +//! +//! * We can't provide the full input string up front, so this crate must +//! support incremental lookups. This is because the peephole optimizer is +//! computing the input string incrementally and dynamically: it looks at the +//! current state's match operation, evaluates it, and then uses the result as +//! the next character of the input string. +//! +//! * We also support incremental insertion and output when building the +//! transducer. This is necessary because we don't want to emit output values +//! that bind a match on an optimization's left-hand side's pattern (for +//! example) until after we've succeeded in matching it, which might not +//! happen until we've reached the n^th state. +//! +//! * We need to support generic output values. The `fst` crate only supports +//! `u64` outputs, while we need to build up an optimization's right-hand side +//! instructions. +//! +//! This implementation is based on [*Direct Construction of Minimal Acyclic +//! Subsequential Transducers* by Mihov and Maurel][paper]. That means that keys +//! must be inserted in lexicographic order during construction. +//! +//! [trie]: https://en.wikipedia.org/wiki/Trie +//! [burntsushi-blog-post]: https://blog.burntsushi.net/transducers/#ordered-maps +//! [fst-crate]: https://crates.io/crates/fst +//! [paper]: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.24.3698&rep=rep1&type=pdf + +#![deny(missing_debug_implementations)] +#![deny(missing_docs)] + +mod output_impls; + +#[cfg(feature = "serde")] +mod serde_impls; + +#[cfg(feature = "dot")] +pub mod dot; + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::convert::TryInto; +use std::hash::Hash; +use std::iter; +use std::mem; + +/// An output type for a transducer automata. +/// +/// Not every type can be the output of a transducer. For correctness (not +/// memory safety) each type that implements this trait must satisfy the +/// following laws: +/// +/// 1. `concat(empty(), x) == x` -- concatenating something with the empty +/// instance produces that same something. +/// +/// 2. `prefix(a, b) == prefix(b, a)` -- taking the prefix of two instances is +/// commutative. +/// +/// 3. `prefix(empty(), x) == empty()` -- the prefix of any value and the empty +/// instance is the empty instance. +/// +/// 4. `difference(concat(a, b), a) == b` -- concatenating a prefix value and +/// then removing it is the identity function. +/// +/// ## Example +/// +/// Here is an example implementation for unsigned integers: +/// +/// ``` +/// use peepmatic_automata::Output; +/// +/// #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +/// struct MyInt(u64); +/// +/// impl Output for MyInt { +/// // The empty value is zero. +/// fn empty() -> Self { +/// MyInt(0) +/// } +/// +/// // The prefix of two values is their min. +/// fn prefix(a: &MyInt, b: &MyInt) -> Self { +/// std::cmp::min(*a, *b) +/// } +/// +/// // The difference is subtraction. +/// fn difference(a: &MyInt, b: &MyInt) -> Self { +/// MyInt(a.0 - b.0) +/// } +/// +/// // Concatenation is addition. +/// fn concat(a: &MyInt, b: &MyInt) -> Self { +/// MyInt(a.0 + b.0) +/// } +/// } +/// +/// // Law 1 +/// assert_eq!( +/// MyInt::concat(&MyInt::empty(), &MyInt(5)), +/// MyInt(5), +/// ); +/// +/// // Law 2 +/// assert_eq!( +/// MyInt::prefix(&MyInt(3), &MyInt(5)), +/// MyInt::prefix(&MyInt(5), &MyInt(3)) +/// ); +/// +/// // Law 3 +/// assert_eq!( +/// MyInt::prefix(&MyInt::empty(), &MyInt(5)), +/// MyInt::empty() +/// ); +/// +/// // Law 4 +/// assert_eq!( +/// MyInt::difference(&MyInt::concat(&MyInt(2), &MyInt(3)), &MyInt(2)), +/// MyInt(3), +/// ); +/// ``` +pub trait Output: Sized + Eq + Hash + Clone { + /// Construct the empty instance. + fn empty() -> Self; + + /// Is this the empty instance? + /// + /// The default implementation constructs the empty instance and then checks + /// if `self` is equal to it. Override this default if you can provide a + /// better implementation. + fn is_empty(&self) -> bool { + *self == Self::empty() + } + + /// Get the shared prefix of two instances. + /// + /// This must be commutative. + fn prefix(a: &Self, b: &Self) -> Self; + + /// When `b` is a prefix of `a`, get the remaining suffix of `a` that is not + /// shared with `b`. + fn difference(a: &Self, b: &Self) -> Self; + + /// Concatenate `a` and `b`. + fn concat(a: &Self, b: &Self) -> Self; +} + +/// A builder for a transducer automata. +/// +/// ## Type Parameters +/// +/// Generic over the following parameters: +/// +/// * `TAlphabet` -- the alphabet of the input strings. If your input keys are +/// `String`s, this would be `char`. If your input keys are arbitrary byte +/// strings, this would be `u8`. +/// +/// * `TState` -- extra, custom data associated with each state. This isn't used +/// by the automata itself, but you can use it to annotate states with extra +/// information for your own purposes. +/// +/// * `TOutput` -- the output type. See [the `Output` trait][crate::Output] for +/// the requirements that any output type must fulfill. +/// +/// ## Insertions +/// +/// Insertions *must* happen in lexicographic order. Failure to do this, or +/// inserting duplicates, will trigger panics. +/// +/// ## Example +/// +/// ``` +/// use peepmatic_automata::Builder; +/// +/// let mut builder = Builder::::new(); +/// +/// // Insert "mon" -> 1 +/// let mut insertion = builder.insert(); +/// insertion +/// .next(b'm', 1) +/// .next(b'o', 0) +/// .next(b'n', 0); +/// insertion.finish(); +/// +/// // Insert "sat" -> 6 +/// let mut insertion = builder.insert(); +/// insertion +/// .next(b's', 6) +/// .next(b'a', 0) +/// .next(b't', 0); +/// insertion.finish(); +/// +/// // Insert "sun" -> 0 +/// let mut insertion = builder.insert(); +/// insertion +/// .next(b's', 0) +/// .next(b'u', 0) +/// .next(b'n', 0); +/// insertion.finish(); +/// +/// let automata = builder.finish(); +/// +/// assert_eq!(automata.get(b"sun"), Some(0)); +/// assert_eq!(automata.get(b"mon"), Some(1)); +/// assert_eq!(automata.get(b"sat"), Some(6)); +/// +/// assert!(automata.get(b"tues").is_none()); +/// ``` +#[derive(Debug, Clone)] +pub struct Builder +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + inner: Option>, +} + +impl Builder +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + /// Make a new builder to start constructing a new transducer automata. + pub fn new() -> Self { + let mut inner = BuilderInner { + frozen: vec![], + wip: BTreeMap::new(), + wip_state_id_counter: 0, + unfinished: vec![], + already_frozen: HashMap::new(), + last_insertion_finished: true, + }; + + // Create the start state. + let id = inner.new_wip_state(); + inner.unfinished.push(id); + + Builder { inner: Some(inner) } + } + + fn inner(&mut self) -> &mut BuilderInner { + self.inner + .as_mut() + .expect("cannot use `Builder` anymore after calling `finish` on it") + } + + /// Start building a new key/value insertion. + /// + /// Insertions are built up incrementally, and a full entry is created from + /// a series of `TAlphabet` and `TOutput` pairs passed to + /// [`InsertionBuilder::next`][crate::InsertionBuilder::next]. + /// + /// ## Panics + /// + /// Panics if [`finish`][crate::InsertionBuilder::finish] was not called on + /// the last `InsertionBuilder` returned from this method. + pub fn insert(&mut self) -> InsertionBuilder { + let inner = self.inner(); + assert!( + inner.last_insertion_finished, + "did not call `finish` on the last `InsertionBuilder`" + ); + inner.last_insertion_finished = false; + InsertionBuilder { + inner: inner, + index: 0, + output: TOutput::empty(), + } + } + + /// Finish building this transducer and return the constructed `Automaton`. + /// + /// ## Panics + /// + /// Panics if this builder is empty, and has never had anything inserted + /// into it. + /// + /// Panics if the last insertion's + /// [`InsertionBuilder`][crate::InsertionBuilder] did not call its + /// [finish][crate::InsertionBuilder::finish] method. + pub fn finish(&mut self) -> Automaton { + let mut inner = self + .inner + .take() + .expect("cannot use `Builder` anymore after calling `finish` on it"); + assert!(inner.last_insertion_finished); + + let wip_start = inner.unfinished[0]; + + // Freeze everything! We're done! + let wip_to_frozen = inner.freeze_from(0); + assert!(inner.wip.is_empty()); + assert!(inner.unfinished.is_empty()); + + // Now transpose our states and transitions into our packed, + // struct-of-arrays representation that we use inside `Automaton`. + let FrozenStateId(s) = wip_to_frozen[&wip_start]; + let start_state = State(s); + let mut state_data = vec![None; inner.frozen.len()]; + let mut transitions = (0..inner.frozen.len()) + .map(|_| BTreeMap::new()) + .collect::>(); + let mut final_states = BTreeMap::new(); + + assert!((inner.frozen.len() as u64) < (std::u32::MAX as u64)); + for (i, state) in inner.frozen.into_iter().enumerate() { + assert!(state_data[i].is_none()); + assert!(transitions[i].is_empty()); + + state_data[i] = state.state_data; + + for (input, (FrozenStateId(to_state), output)) in state.transitions { + assert!((to_state as usize) < transitions.len()); + transitions[i].insert(input, (State(to_state), output)); + } + + if state.is_final { + final_states.insert(State(i as u32), state.final_output); + } else { + assert!(state.final_output.is_empty()); + } + } + + let automata = Automaton { + state_data, + transitions, + final_states, + start_state, + }; + + #[cfg(debug_assertions)] + { + if let Err(msg) = automata.check_representation() { + panic!("Automaton::check_representation failed: {}", msg); + } + } + + automata + } +} + +/// A state in an automaton. +/// +/// Only use a `State` with the automaton that it came from! Mixing and matching +/// states between automata will result in bogus results and/or panics! +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Ord, PartialOrd)] +pub struct State(u32); + +#[derive(Clone, Debug)] +struct BuilderInner +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + // The `i`th entry maps `FrozenStateId(i)` to its state. + frozen: Vec>, + + // Our mutable, work-in-progress states. + wip: BTreeMap>, + + // A counter for WIP state ids. + wip_state_id_counter: u32, + + // A stack of our work-in-progress states. + unfinished: Vec, + + // A map from `WipState`s that we've already frozen to their canonical, + // de-duplicated frozen state. This is used for hash-consing frozen states + // so that we share suffixes in the automata. + already_frozen: HashMap, FrozenStateId>, + + // The the last `InsertionBuilder` have its `finish` method invoked? + last_insertion_finished: bool, +} + +impl BuilderInner +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + fn new_wip_state(&mut self) -> WipStateId { + let id = WipStateId(self.wip_state_id_counter); + self.wip_state_id_counter += 1; + let old = self.wip.insert( + id, + WipState { + state_data: None, + transitions: BTreeMap::new(), + is_final: false, + final_output: TOutput::empty(), + }, + ); + debug_assert!(old.is_none()); + id + } + + fn freeze_from(&mut self, index: usize) -> BTreeMap { + assert!(index <= self.unfinished.len()); + + let mut wip_to_frozen = BTreeMap::new(); + + if index == self.unfinished.len() { + // Nothing to freeze. + return wip_to_frozen; + } + + // Freeze `self.inner.unfinished[self.index + 1..]` from the end + // back. We're essentially hash-consing each state. + for _ in (index..self.unfinished.len()).rev() { + let wip_id = self.unfinished.pop().unwrap(); + let mut wip = self.wip.remove(&wip_id).unwrap(); + + // Update transitions to any state we just froze in an earlier + // iteration of this loop. + wip.update_transitions(&wip_to_frozen); + + // Get or create the canonical frozen state for this WIP state. + // + // Note: we're not using the entry API here because this way we can + // avoid cloning `wip`, which would be more costly than the double + // lookup we're doing instead. + let frozen_id = if let Some(id) = self.already_frozen.get(&wip) { + *id + } else { + let id = FrozenStateId(self.frozen.len().try_into().unwrap()); + self.frozen.push(FrozenState { + state_data: wip.state_data.clone(), + transitions: wip + .transitions + .clone() + .into_iter() + .map(|(input, (id, output))| { + let id = match id { + WipOrFrozenStateId::Frozen(id) => id, + WipOrFrozenStateId::Wip(_) => panic!( + "when we are freezing a WIP state, it should never have \ + any transitions to another WIP state" + ), + }; + (input, (id, output)) + }) + .collect(), + is_final: wip.is_final, + final_output: wip.final_output.clone(), + }); + self.already_frozen.insert(wip, id); + id + }; + + // Record the id for this newly frozen state, so that other states + // which referenced it when it wasn't frozen can reference it as a + // frozen state. + wip_to_frozen.insert(wip_id, frozen_id); + } + + // Update references to newly frozen states from the rest of the + // unfinished stack that we didn't freeze. + for wip_id in &self.unfinished { + self.wip + .get_mut(wip_id) + .unwrap() + .update_transitions(&wip_to_frozen); + } + + wip_to_frozen + } +} + +/// A builder for a new entry in a transducer automata. +#[derive(Debug)] +pub struct InsertionBuilder<'a, TAlphabet, TState, TOutput> +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + inner: &'a mut BuilderInner, + + // The index within `inner.unfinished` where we will transition out of next. + index: usize, + + // Any leftover output from the last transition that we need to roll over + // into the next transition. + output: TOutput, +} + +impl<'a, TAlphabet, TState, TOutput> InsertionBuilder<'a, TAlphabet, TState, TOutput> +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + /// Insert the next character of input for this entry, and the associated + /// output that should be emitted along with it. + /// + /// In general, you want to add all of your output on the very first `next` + /// call, and use [`Output::empty()`][crate::Output::empty] for all the + /// rest. This enables the most tail-sharing of suffixes, which leads to the + /// most compact automatas. + /// + /// However, there are times when you *cannot* emit output yet, as it + /// depends on having moved throught he automata further. For example, with + /// `peepmatic` we cannot bind something from an optimization's left-hand + /// side's pattern until after we know it exists, which only happens after + /// we've moved some distance through the automata. + pub fn next(&mut self, input: TAlphabet, output: TOutput) -> &mut Self { + assert!(self.index < self.inner.unfinished.len()); + + if output.is_empty() { + // Leave `self.output` as it is. + } else if self.output.is_empty() { + self.output = output; + } else { + self.output = TOutput::concat(&self.output, &output); + } + + let wip_id = self.inner.unfinished[self.index]; + let wip = self.inner.wip.get_mut(&wip_id).unwrap(); + + match wip.transitions.get_mut(&input) { + Some((WipOrFrozenStateId::Frozen(_), _)) => { + panic!("out of order insertion: wip->frozen edge in shared prefix") + } + + // We're still in a shared prefix with the last insertion. That + // means that the state we are transitioning to must be the next + // state in `unfinished`. All we have to do is make sure the + // transition's output is the common prefix of the this insertion + // and the last, and push any excess suffix output out to other + // transition edges. + Some((WipOrFrozenStateId::Wip(next_id), out)) => { + let next_id = *next_id; + assert_eq!(next_id, self.inner.unfinished[self.index + 1]); + + // Find the common prefix of `out` and `self.output`. + let prefix = TOutput::prefix(&self.output, out); + + // Carry over this key's suffix for the next input's transition. + self.output = TOutput::difference(&self.output, &prefix); + + let rest = TOutput::difference(out, &prefix); + *out = prefix; + + let next_wip = self.inner.wip.get_mut(&next_id).unwrap(); + + // Push the leftover suffix of `out` along its other + // transitions. As a small optimization, only iterate over the + // edges if there is a non-empty value to push out along them. + if !rest.is_empty() { + if next_wip.is_final { + next_wip.final_output = TOutput::concat(&rest, &next_wip.final_output); + } + for (_input, (_state, output)) in &mut next_wip.transitions { + *output = TOutput::concat(&rest, output); + } + } + } + + // We've diverged from the shared prefix with the last + // insertion. Freeze the last insertion's unshared suffix and create + // a new WIP state for us to transition into. + None => { + self.inner.freeze_from(self.index + 1); + + let output = mem::replace(&mut self.output, TOutput::empty()); + + let new_id = self.inner.new_wip_state(); + self.inner.unfinished.push(new_id); + self.inner + .wip + .get_mut(&wip_id) + .unwrap() + .transitions + .insert(input, (WipOrFrozenStateId::Wip(new_id), output)); + } + } + + self.index += 1; + assert!(self.index < self.inner.unfinished.len()); + + self + } + + /// Finish this insertion. + /// + /// Failure to call this method before this `InsertionBuilder` is dropped + /// means that the insertion is *not* committed in the builder, and future + /// calls to [`InsertionBuilder::next`][crate::InsertionBuilder::next] will + /// panic! + pub fn finish(self) { + assert!(!self.inner.unfinished.is_empty()); + assert_eq!( + self.index, + self.inner.unfinished.len() - 1, + "out of order insertion" + ); + + let wip_id = *self.inner.unfinished.last().unwrap(); + let wip = self.inner.wip.get_mut(&wip_id).unwrap(); + wip.is_final = true; + wip.final_output = self.output; + + self.inner.last_insertion_finished = true; + } + + /// Set the optional, custom data for the current state. + /// + /// If you assign different state data to two otherwise-identical states + /// within the same shared *prefix* during insertion, it is implementation + /// defined which state and custom state data is kept. + /// + /// For *suffixes*, assigning different state data to two + /// otehrwise-identical states will result in the duplication of those + /// states: they won't get de-duplicated. + pub fn set_state_data(&mut self, data: TState) -> &mut Self { + assert!(self.index < self.inner.unfinished.len()); + let id = self.inner.unfinished[self.index]; + self.inner.wip.get_mut(&id).unwrap().state_data = Some(data); + self + } + + /// Get the current state's optional, custom data, if any. + /// + /// For shared prefixes, this may return state data that was assigned to an + /// equivalent state that was added earlier in the build process. + pub fn get_state_data(&self) -> Option<&TState> { + let id = self.inner.unfinished[self.index]; + self.inner.wip.get(&id).unwrap().state_data.as_ref() + } +} + +/// The id of an immutable, frozen state. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct FrozenStateId(u32); + +/// The id of a mutable, work-in-progress state. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct WipStateId(u32); + +/// The id of either a frozen or a WIP state. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +enum WipOrFrozenStateId { + Wip(WipStateId), + Frozen(FrozenStateId), +} + +/// A frozen, immutable state inside a `Builder`. +/// +/// These states are from earlier in the lexicographic sorting on input keys, +/// and have already been processed. +#[derive(Clone, Debug, Hash)] +struct FrozenState +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + state_data: Option, + transitions: BTreeMap, + is_final: bool, + final_output: TOutput, +} + +/// A mutable, work-in-progress state inside a `Builder`. +/// +/// These states only exist for the last-inserted and currently-being-inserted +/// input keys. As soon as we find the end of their shared prefix, the last +/// key's unshared suffix is frozen, and then only the currently-being-inserted +/// input key has associated WIP states. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct WipState +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + state_data: Option, + transitions: BTreeMap, + is_final: bool, + final_output: TOutput, +} + +impl WipState +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + /// Given that we froze some old, WIP state, update any transitions out of + /// this WIP state so they point to the new, frozen state. + fn update_transitions(&mut self, wip_to_frozen: &BTreeMap) { + for (to, _) in self.transitions.values_mut() { + if let WipOrFrozenStateId::Wip(w) = *to { + if let Some(f) = wip_to_frozen.get(&w) { + *to = WipOrFrozenStateId::Frozen(*f); + } + } + } + } +} + +/// A finite-state transducer automata. +/// +/// These are constructed via [`Builder`][crate::Builder]. +/// +/// An `Automaton` is immutable: new entries cannot be inserted and existing +/// entries cannot be removed. +/// +/// To query an `Automaton`, there are two APIs: +/// +/// 1. [`get`][crate::Automaton::get] -- a high-level method to get the associated +/// output value of a full input sequence. +/// +/// 2. [`query`][crate::Automaton::query] -- a low-level method to +/// incrementally query the automata. It does not require that you have the +/// full input sequence on hand all at once, only the next character. It also +/// allows you to process the output as it it built up, rather than only at +/// giving you the final, complete output value. +#[derive(Debug, Clone)] +pub struct Automaton +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + // The `i`th entry is `State(i)`'s associated custom data. + state_data: Vec>, + + // The `i`th entry contains `State(i)`'s transitions. + transitions: Vec>, + + // Keeps track of which states are final, and if so, what their final output + // is. + final_states: BTreeMap, + + // The starting state. + start_state: State, +} + +impl Automaton +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + /// Get the output value associated with the given input sequence. + /// + /// Returns `None` if the input sequence is not a member of this + /// `Automaton`'s keys. Otherwise, returns `Some(output)`. + pub fn get<'a>(&self, input: impl IntoIterator) -> Option + where + TAlphabet: 'a, + { + let mut query = self.query(); + let mut output = TOutput::empty(); + + for inp in input { + let this_out = query.next(inp)?; + output = TOutput::concat(&output, &this_out); + } + + let final_output = query.finish()?; + Some(TOutput::concat(&output, final_output)) + } + + /// Create a low-level query. + /// + /// This allows you to incrementally query this `Automaton`, without + /// providing the full input sequence ahead of time, and also incrementally + /// build up the output. + /// + /// See [`Query`][crate::Query] for details. + pub fn query(&self) -> Query { + Query { + automata: self, + current_state: self.start_state, + } + } + + /// Check that the internal representaton is OK. + /// + /// Checks that we don't have any transitions to unknown states, that there + /// aren't any cycles, that ever path through the automata eventually ends + /// in a final state, etc. + /// + /// This property is `debug_assert!`ed in `Builder::finish`, and checked + /// when deserializing an `Automaton`. + /// + /// Returns `true` if the representation is okay, `false` otherwise. + fn check_representation(&self) -> Result<(), &'static str> { + macro_rules! bail_if { + ($condition:expr, $msg:expr) => { + if $condition { + return Err($msg); + } + }; + } + + bail_if!( + self.state_data.len() != self.transitions.len(), + "different number of states and transition sets" + ); + bail_if!( + self.final_states.is_empty(), + "the set of final states is empty" + ); + + bail_if!( + (self.start_state.0 as usize) >= self.transitions.len(), + "the start state is not a valid state" + ); + + for (f, _out) in &self.final_states { + bail_if!( + (f.0 as usize) >= self.transitions.len(), + "one of the final states is not a valid state" + ); + } + + // Walk the state transition graph and ensure that + // + // 1. there are no cycles, and + // + // 2. every path ends in a final state. + let mut on_stack = HashSet::new(); + let mut stack = vec![ + (Traversal::Stop, self.start_state), + (Traversal::Start, self.start_state), + ]; + loop { + match stack.pop() { + None => break, + Some((Traversal::Start, state)) => { + let is_new = on_stack.insert(state); + debug_assert!(is_new); + + let mut has_any_transitions = false; + for (_input, (to_state, _output)) in &self.transitions[state.0 as usize] { + has_any_transitions = true; + + // A transition to a state that we walked through to get + // here means that there is a cycle. + bail_if!( + on_stack.contains(to_state), + "there is a cycle in the state transition graph" + ); + + stack.extend( + iter::once((Traversal::Stop, *to_state)) + .chain(iter::once((Traversal::Start, *to_state))), + ); + } + + if !has_any_transitions { + // All paths must end in a final state. + bail_if!( + !self.final_states.contains_key(&state), + "a path through the state transition graph does not end in a final state" + ); + } + } + Some((Traversal::Stop, state)) => { + debug_assert!(on_stack.contains(&state)); + on_stack.remove(&state); + } + } + } + + return Ok(()); + + enum Traversal { + Start, + Stop, + } + } +} + +/// A low-level query of an `Automaton`. +/// +/// This allows you to incrementally query an `Automaton`, without providing the +/// full input sequence ahead of time, and also to incrementally build up the +/// output. +/// +/// The typical usage pattern is: +/// +/// * First, a series of [`next`][crate::Query::next] calls that each provide +/// one character of the input sequence. +/// +/// If this query is still on a path towards a known entry of the +/// automata, then `Some` is returned with the partial output of the +/// transition that was just taken. Otherwise, `None` is returned, signifying +/// that the input string has been rejected by the automata. +/// +/// You may also inspect the current state's associated custom data, if any, +/// in between `next` calls via the +/// [`current_state_data`][crate::Query::current_state_data] method. +/// +/// * When the input sequence is exhausted, call +/// [`is_in_final_state`][crate::Query::is_in_final_state] to determine if this +/// query is in a final state of the automata. If it is not, then the +/// input string has been rejected by the automata. +/// +/// * Given that the input sequence is exhausted, you may call +/// [`finish`][crate::Query::finish] to get the final bit of partial output. +#[derive(Debug, Clone)] +pub struct Query<'a, TAlphabet, TState, TOutput> +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + automata: &'a Automaton, + current_state: State, +} + +impl<'a, TAlphabet, TState, TOutput> Query<'a, TAlphabet, TState, TOutput> +where + TAlphabet: Clone + Eq + Hash + Ord, + TState: Clone + Eq + Hash, + TOutput: Output, +{ + /// Get the current state in the automaton that this query is at. + pub fn current_state(&self) -> State { + self.current_state + } + + /// Move this query to the given state in the automaton. + /// + /// This can be used to implement backtracking, if you can also reset your + /// output to the way it was when you previously visited the given `State`. + /// + /// Only use a `State` that came from this query's automaton! Mixing and + /// matching states between automata will result in bogus results and/or + /// panics! + pub fn go_to_state(&mut self, state: State) { + assert!((state.0 as usize) < self.automata.transitions.len()); + debug_assert_eq!( + self.automata.state_data.len(), + self.automata.transitions.len() + ); + self.current_state = state; + } + + /// Does the query's current state have a transition on the given input? + /// + /// Regardless whether a transition on the given input exists for the + /// current state or not, the query remains in the current state. + pub fn has_transition_on(&self, input: &TAlphabet) -> bool { + let State(i) = self.current_state; + self.automata.transitions[i as usize].contains_key(input) + } + + /// Transition to the next state given the next input character, and return + /// the partial output for that transition. + /// + /// If `None` is returned, then the input sequence has been rejected by the + /// automata, and this query remains in its current state. + #[inline] + pub fn next(&mut self, input: &TAlphabet) -> Option<&'a TOutput> { + let State(i) = self.current_state; + match self.automata.transitions[i as usize].get(input) { + None => None, + Some((next_state, output)) => { + self.current_state = *next_state; + Some(output) + } + } + } + + /// Get the current state's associated custom data, if any. + /// + /// See also + /// [`InsertionBuilder::set_state_data`][crate::InsertionBuilder::set_state_data]. + #[inline] + pub fn current_state_data(&self) -> Option<&'a TState> { + let State(i) = self.current_state; + self.automata.state_data[i as usize].as_ref() + } + + /// Is this query currently in a final state? + #[inline] + pub fn is_in_final_state(&self) -> bool { + self.automata.final_states.contains_key(&self.current_state) + } + + /// Given that the input sequence is exhausted, get the final bit of partial + /// output. + /// + /// Returns `None` if this query is not currently in a final state, meaning + /// that the automata has rejected this input sequence. You can check + /// whether that is the case or not with the + /// [`is_in_final_state`][crate::Query::is_in_final_state] method. + pub fn finish(self) -> Option<&'a TOutput> { + self.automata.final_states.get(&self.current_state) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/cranelift/peepmatic/crates/automata/src/output_impls.rs b/cranelift/peepmatic/crates/automata/src/output_impls.rs new file mode 100644 index 0000000000..6701f7d867 --- /dev/null +++ b/cranelift/peepmatic/crates/automata/src/output_impls.rs @@ -0,0 +1,93 @@ +use crate::Output; +use std::cmp; +use std::hash::Hash; + +impl Output for u64 { + fn empty() -> Self { + 0 + } + + fn prefix(a: &Self, b: &Self) -> Self { + cmp::min(*a, *b) + } + + fn difference(a: &Self, b: &Self) -> Self { + a - b + } + + fn concat(a: &Self, b: &Self) -> Self { + a + b + } +} + +impl Output for Vec +where + T: Clone + Eq + Hash, +{ + fn empty() -> Self { + vec![] + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn prefix(a: &Self, b: &Self) -> Self { + a.iter() + .cloned() + .zip(b.iter().cloned()) + .take_while(|(a, b)| a == b) + .map(|(a, _)| a) + .collect() + } + + fn difference(a: &Self, b: &Self) -> Self { + let i = a + .iter() + .zip(b.iter()) + .position(|(a, b)| a != b) + .unwrap_or(cmp::min(a.len(), b.len())); + a[i..].to_vec() + } + + fn concat(a: &Self, b: &Self) -> Self { + let mut c = a.clone(); + c.extend(b.iter().cloned()); + c + } +} + +#[cfg(test)] +mod tests { + use crate::Output; + use std::fmt::Debug; + + // Assert the laws that `Output` requires for correctness. `a` and `b` + // should be two different instances of an `Output` type. + fn assert_laws(a: O, b: O) + where + O: Clone + Debug + Output, + { + // Law 1 + assert_eq!(O::concat(&O::empty(), &a), a.clone()); + + // Law 2 + assert_eq!(O::prefix(&b, &a), O::prefix(&a, &b)); + + // Law 3 + assert_eq!(O::prefix(&O::empty(), &a), O::empty()); + + // Law 4 + assert_eq!(O::difference(&O::concat(&a, &b), &a), b); + } + + #[test] + fn impl_for_u64() { + assert_laws(3, 5); + } + + #[test] + fn impl_for_vec() { + assert_laws(vec![0, 1, 2, 3], vec![0, 2, 4, 6]); + } +} diff --git a/cranelift/peepmatic/crates/automata/src/serde_impls.rs b/cranelift/peepmatic/crates/automata/src/serde_impls.rs new file mode 100644 index 0000000000..f7543626e5 --- /dev/null +++ b/cranelift/peepmatic/crates/automata/src/serde_impls.rs @@ -0,0 +1,195 @@ +//! `serde::Serialize` and `serde::Deserialize` implementations for `Automaton`. +//! +//! Rather than prefix each serialized field with which field it is, we always +//! serialize fields in alphabetical order. Make sure to maintain this if you +//! add or remove fields! +//! +//! Each time you add/remove a field, or change serialization in any other way, +//! make sure to bump `SERIALIZATION_VERSION`. + +use crate::{Automaton, Output, State}; +use serde::{ + de::{self, Deserializer, SeqAccess, Visitor}, + ser::SerializeTupleStruct, + Deserialize, Serialize, Serializer, +}; +use std::collections::BTreeMap; +use std::fmt; +use std::hash::Hash; +use std::marker::PhantomData; + +const SERIALIZATION_VERSION: u32 = 1; + +impl Serialize for State { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(self.0) + } +} + +impl<'de> Deserialize<'de> for State { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(State(deserializer.deserialize_u32(U32Visitor)?)) + } +} + +struct U32Visitor; + +impl<'de> Visitor<'de> for U32Visitor { + type Value = u32; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("an integer between `0` and `2^32 - 1`") + } + + fn visit_u8(self, value: u8) -> Result + where + E: de::Error, + { + Ok(u32::from(value)) + } + + fn visit_u32(self, value: u32) -> Result + where + E: de::Error, + { + Ok(value) + } + + fn visit_u64(self, value: u64) -> Result + where + E: de::Error, + { + use std::u32; + if value <= u64::from(u32::MAX) { + Ok(value as u32) + } else { + Err(E::custom(format!("u32 out of range: {}", value))) + } + } +} + +impl Serialize for Automaton +where + TAlphabet: Serialize + Clone + Eq + Hash + Ord, + TState: Serialize + Clone + Eq + Hash, + TOutput: Serialize + Output, +{ + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let Automaton { + final_states, + start_state, + state_data, + transitions, + } = self; + + let mut s = serializer.serialize_tuple_struct("Automaton", 5)?; + s.serialize_field(&SERIALIZATION_VERSION)?; + s.serialize_field(final_states)?; + s.serialize_field(start_state)?; + s.serialize_field(state_data)?; + s.serialize_field(transitions)?; + s.end() + } +} + +impl<'de, TAlphabet, TState, TOutput> Deserialize<'de> for Automaton +where + TAlphabet: 'de + Deserialize<'de> + Clone + Eq + Hash + Ord, + TState: 'de + Deserialize<'de> + Clone + Eq + Hash, + TOutput: 'de + Deserialize<'de> + Output, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_tuple_struct( + "Automaton", + 5, + AutomatonVisitor { + phantom: PhantomData, + }, + ) + } +} + +struct AutomatonVisitor<'de, TAlphabet, TState, TOutput> +where + TAlphabet: 'de + Deserialize<'de> + Clone + Eq + Hash + Ord, + TState: 'de + Deserialize<'de> + Clone + Eq + Hash, + TOutput: 'de + Deserialize<'de> + Output, +{ + phantom: PhantomData<&'de (TAlphabet, TState, TOutput)>, +} + +impl<'de, TAlphabet, TState, TOutput> Visitor<'de> + for AutomatonVisitor<'de, TAlphabet, TState, TOutput> +where + TAlphabet: 'de + Deserialize<'de> + Clone + Eq + Hash + Ord, + TState: 'de + Deserialize<'de> + Clone + Eq + Hash, + TOutput: 'de + Deserialize<'de> + Output, +{ + type Value = Automaton; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("Automaton") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + match seq.next_element::()? { + Some(v) if v == SERIALIZATION_VERSION => {} + Some(v) => { + return Err(de::Error::invalid_value( + de::Unexpected::Unsigned(v as u64), + &self, + )); + } + None => return Err(de::Error::invalid_length(0, &"Automaton expects 5 elements")), + } + + let final_states = match seq.next_element::>()? { + Some(x) => x, + None => return Err(de::Error::invalid_length(1, &"Automaton expects 5 elements")), + }; + + let start_state = match seq.next_element::()? { + Some(x) => x, + None => return Err(de::Error::invalid_length(2, &"Automaton expects 5 elements")), + }; + + let state_data = match seq.next_element::>>()? { + Some(x) => x, + None => return Err(de::Error::invalid_length(3, &"Automaton expects 5 elements")), + }; + + let transitions = match seq.next_element::>>()? { + Some(x) => x, + None => return Err(de::Error::invalid_length(4, &"Automaton expects 5 elements")), + }; + + let automata = Automaton { + final_states, + start_state, + state_data, + transitions, + }; + + // Ensure that the deserialized automata is well-formed. + automata + .check_representation() + .map_err(|msg| de::Error::custom(msg))?; + + Ok(automata) + } +} From 0f03a974759a866eea9994262e0170f51f2ed241 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 1 May 2020 15:33:55 -0700 Subject: [PATCH 02/22] peepmatic: Introduce the `peepmatic-macro` crate This crate provides the derive macros used by `peepmatic`, notable AST-related derives that enumerate child AST nodes, and operator-related derives that provide helpers for type checking. --- cranelift/peepmatic/crates/macro/Cargo.toml | 15 + .../peepmatic/crates/macro/src/child_nodes.rs | 110 ++++++ .../crates/macro/src/into_dyn_ast_ref.rs | 23 ++ cranelift/peepmatic/crates/macro/src/lib.rs | 156 +++++++++ .../peepmatic/crates/macro/src/operator.rs | 325 ++++++++++++++++++ cranelift/peepmatic/crates/macro/src/span.rs | 50 +++ 6 files changed, 679 insertions(+) create mode 100644 cranelift/peepmatic/crates/macro/Cargo.toml create mode 100644 cranelift/peepmatic/crates/macro/src/child_nodes.rs create mode 100644 cranelift/peepmatic/crates/macro/src/into_dyn_ast_ref.rs create mode 100644 cranelift/peepmatic/crates/macro/src/lib.rs create mode 100644 cranelift/peepmatic/crates/macro/src/operator.rs create mode 100644 cranelift/peepmatic/crates/macro/src/span.rs diff --git a/cranelift/peepmatic/crates/macro/Cargo.toml b/cranelift/peepmatic/crates/macro/Cargo.toml new file mode 100644 index 0000000000..e13cffd988 --- /dev/null +++ b/cranelift/peepmatic/crates/macro/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "peepmatic-macro" +version = "0.1.0" +authors = ["Nick Fitzgerald "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +proc-macro2 = "1.0.9" +quote = "1.0.3" +syn = { version = "1.0.16", features = ['extra-traits'] } + +[lib] +proc_macro = true diff --git a/cranelift/peepmatic/crates/macro/src/child_nodes.rs b/cranelift/peepmatic/crates/macro/src/child_nodes.rs new file mode 100644 index 0000000000..c71ac409a0 --- /dev/null +++ b/cranelift/peepmatic/crates/macro/src/child_nodes.rs @@ -0,0 +1,110 @@ +use quote::quote; +use syn::DeriveInput; +use syn::{parse_quote, GenericParam, Generics, Result}; + +pub fn derive_child_nodes(input: &DeriveInput) -> Result { + let children = get_child_nodes(&input.data)?; + let name = &input.ident; + let generics = add_trait_bounds(input.generics.clone()); + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + Ok(quote! { + impl #impl_generics ChildNodes<'a, 'a> for #name #ty_generics #where_clause { + fn child_nodes(&'a self, children: &mut impl Extend>) { + #children + } + } + }) +} + +fn get_child_nodes(data: &syn::Data) -> Result { + match data { + syn::Data::Struct(s) => { + let mut fields = vec![]; + + match &s.fields { + syn::Fields::Named(n) => { + for f in n.named.iter() { + let opts = crate::PeepmaticOpts::from_attrs(&mut f.attrs.clone())?; + + if opts.skip_child { + continue; + } + + let field_name = f.ident.as_ref().unwrap(); + if opts.flatten { + fields.push(quote! { + self.#field_name.iter().map(DynAstRef::from) + }); + } else { + fields.push(quote! { + std::iter::once(DynAstRef::from(&self.#field_name)) + }); + } + } + } + syn::Fields::Unnamed(u) => { + for (i, f) in u.unnamed.iter().enumerate() { + let opts = crate::PeepmaticOpts::from_attrs(&mut f.attrs.clone())?; + if opts.skip_child { + continue; + } + if opts.flatten { + return Err(syn::Error::new( + u.paren_token.span, + "#[peepmatic(flatten)] is only allowed with named fields", + )); + } + fields.push(quote! { + std::iter::once(DynAstRef::from(&self.#i)) + }); + } + } + syn::Fields::Unit => {} + } + + Ok(match fields.as_slice() { + [] => quote! { let _ = children; }, + [f, rest @ ..] => { + let rest = rest.iter().map(|f| { + quote! { + .chain(#f) + } + }); + quote! { + children.extend( #f #( #rest )* ); + } + } + }) + } + syn::Data::Enum(e) => { + let mut match_arms = vec![]; + for v in e.variants.iter() { + match v.fields { + syn::Fields::Unnamed(ref u) if u.unnamed.len() == 1 => { + let variant = &v.ident; + match_arms.push(quote! { + Self::#variant(x) => children.extend(Some(x.into())), + }); + } + _ => panic!("#[derive(ChildNodes)] only supports enums whose variants all ahve a single unnamed field") + } + } + Ok(quote! { + match self { + #( #match_arms )* + } + }) + } + syn::Data::Union(_) => panic!("#[derive(ChildNodes)] is not supported on unions"), + } +} + +fn add_trait_bounds(mut generics: Generics) -> Generics { + for param in &mut generics.params { + if let GenericParam::Type(type_param) = param { + type_param.bounds.push(parse_quote!(ChildNodes<'a, 'a>)); + } + } + generics +} diff --git a/cranelift/peepmatic/crates/macro/src/into_dyn_ast_ref.rs b/cranelift/peepmatic/crates/macro/src/into_dyn_ast_ref.rs new file mode 100644 index 0000000000..4cd993a72c --- /dev/null +++ b/cranelift/peepmatic/crates/macro/src/into_dyn_ast_ref.rs @@ -0,0 +1,23 @@ +use quote::quote; +use syn::DeriveInput; +use syn::Result; + +pub fn derive_into_dyn_ast_ref(input: &DeriveInput) -> Result { + let ty = &input.ident; + + let opts = crate::PeepmaticOpts::from_attrs(&mut input.attrs.clone())?; + if opts.no_into_dyn_node { + return Ok(quote! {}); + } + + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + + Ok(quote! { + impl #impl_generics From<&'a #ty #ty_generics> for DynAstRef<'a> #where_clause { + #[inline] + fn from(x: &'a #ty #ty_generics) -> Self { + Self::#ty(x) + } + } + }) +} diff --git a/cranelift/peepmatic/crates/macro/src/lib.rs b/cranelift/peepmatic/crates/macro/src/lib.rs new file mode 100644 index 0000000000..5375947c36 --- /dev/null +++ b/cranelift/peepmatic/crates/macro/src/lib.rs @@ -0,0 +1,156 @@ +extern crate proc_macro; + +use crate::proc_macro::TokenStream; +use proc_macro2::Span; +use quote::quote; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::DeriveInput; +use syn::Error; +use syn::{parse_macro_input, Ident, Result}; + +mod child_nodes; +mod into_dyn_ast_ref; +mod operator; +mod span; + +#[proc_macro_derive(PeepmaticOperator, attributes(peepmatic))] +pub fn operator(input: TokenStream) -> TokenStream { + operator::derive_operator(input) +} + +#[proc_macro_derive(Ast, attributes(peepmatic))] +pub fn derive_ast(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + + let span_impl = match span::derive_span(&input) { + Ok(s) => s, + Err(e) => return e.to_compile_error().into(), + }; + + let child_nodes_impl = match child_nodes::derive_child_nodes(&input) { + Ok(c) => c, + Err(e) => return e.to_compile_error().into(), + }; + + let into_dyn_ast_ref_impl = match into_dyn_ast_ref::derive_into_dyn_ast_ref(&input) { + Ok(n) => n, + Err(e) => return e.to_compile_error().into(), + }; + + let expanded = quote! { + #span_impl + #child_nodes_impl + #into_dyn_ast_ref_impl + }; + + TokenStream::from(expanded) +} + +#[derive(Default)] +pub(crate) struct PeepmaticOpts { + // `ChildNodes` options. + skip_child: bool, + flatten: bool, + + // `From<&'a Self> for DynAstRef<'a>` options. + no_into_dyn_node: bool, + + // Peepmatic operator options. + immediates_paren: syn::token::Paren, + immediates: Vec, + params_paren: syn::token::Paren, + params: Vec, + result: Option, +} + +impl Parse for PeepmaticOpts { + fn parse(input: ParseStream) -> Result { + enum Attr { + Immediates(syn::token::Paren, Vec), + Params(syn::token::Paren, Vec), + Result(syn::Ident), + NoIntoDynNode, + SkipChild, + Flatten, + } + + let attrs = Punctuated::<_, syn::token::Comma>::parse_terminated(input)?; + let mut ret = PeepmaticOpts::default(); + for attr in attrs { + match attr { + Attr::Immediates(paren, imms) => { + ret.immediates_paren = paren; + ret.immediates = imms; + } + Attr::Params(paren, ps) => { + ret.params_paren = paren; + ret.params = ps; + } + Attr::Result(r) => ret.result = Some(r), + Attr::NoIntoDynNode => ret.no_into_dyn_node = true, + Attr::SkipChild => ret.skip_child = true, + Attr::Flatten => ret.flatten = true, + } + } + + return Ok(ret); + + impl Parse for Attr { + fn parse(input: ParseStream) -> Result { + let attr: Ident = input.parse()?; + if attr == "immediates" { + let inner; + let paren = syn::parenthesized!(inner in input); + let imms = Punctuated::<_, syn::token::Comma>::parse_terminated(&inner)?; + return Ok(Attr::Immediates(paren, imms.into_iter().collect())); + } + if attr == "params" { + let inner; + let paren = syn::parenthesized!(inner in input); + let params = Punctuated::<_, syn::token::Comma>::parse_terminated(&inner)?; + return Ok(Attr::Params(paren, params.into_iter().collect())); + } + if attr == "result" { + let inner; + syn::parenthesized!(inner in input); + return Ok(Attr::Result(syn::Ident::parse(&inner)?)); + } + if attr == "skip_child" { + return Ok(Attr::SkipChild); + } + if attr == "no_into_dyn_node" { + return Ok(Attr::NoIntoDynNode); + } + if attr == "flatten" { + return Ok(Attr::Flatten); + } + return Err(Error::new(attr.span(), "unexpected attribute")); + } + } + } +} + +fn peepmatic_attrs(attrs: &mut Vec) -> TokenStream { + let mut ret = proc_macro2::TokenStream::new(); + let ident = syn::Path::from(syn::Ident::new("peepmatic", Span::call_site())); + for i in (0..attrs.len()).rev() { + if attrs[i].path != ident { + continue; + } + let attr = attrs.remove(i); + let group = match attr.tokens.into_iter().next().unwrap() { + proc_macro2::TokenTree::Group(g) => g, + _ => panic!("#[peepmatic(...)] expected"), + }; + ret.extend(group.stream()); + ret.extend(quote! { , }); + } + return ret.into(); +} + +impl PeepmaticOpts { + pub(crate) fn from_attrs(attrs: &mut Vec) -> syn::Result { + syn::parse(peepmatic_attrs(attrs)) + } +} diff --git a/cranelift/peepmatic/crates/macro/src/operator.rs b/cranelift/peepmatic/crates/macro/src/operator.rs new file mode 100644 index 0000000000..0d53a1219d --- /dev/null +++ b/cranelift/peepmatic/crates/macro/src/operator.rs @@ -0,0 +1,325 @@ +//! Implementation of the `#[peepmatic]` macro for the `Operator` AST node. + +use crate::proc_macro::TokenStream; +use crate::PeepmaticOpts; +use proc_macro2::{Ident, Span}; +use quote::quote; +use syn::DeriveInput; +use syn::Error; +use syn::{parse_macro_input, Result}; + +pub fn derive_operator(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + + let variants = match get_enum_variants(&input) { + Ok(v) => v, + Err(e) => return e.to_compile_error().into(), + }; + + let arity = match create_arity(&variants) { + Ok(a) => a, + Err(e) => return e.to_compile_error().into(), + }; + + let num_operators = variants.len(); + let type_methods = create_type_methods(&variants); + let parse_impl = create_parse_impl(&input.ident, &variants); + let display_impl = create_display_impl(&input.ident, &variants); + let try_from_u32_impl = create_try_from_u32_impl(&input.ident, &variants); + let ident = &input.ident; + + let expanded = quote! { + impl #ident { + #arity + #type_methods + + /// Get the total number of different operators. + pub const fn num_operators() -> usize { + #num_operators + } + } + + #display_impl + #try_from_u32_impl + #parse_impl + }; + + // eprintln!("{}", expanded); + TokenStream::from(expanded) +} + +fn get_enum_variants(input: &DeriveInput) -> Result> { + let en = match &input.data { + syn::Data::Enum(en) => en, + syn::Data::Struct(_) => { + panic!("can only put #[peepmatic] on an enum; found it on a struct") + } + syn::Data::Union(_) => panic!("can only put #[peepmatic] on an enum; found it on a union"), + }; + en.variants + .iter() + .cloned() + .map(|mut variant| { + Ok(OperatorVariant { + opts: PeepmaticOpts::from_attrs(&mut variant.attrs)?, + syn: variant, + }) + }) + .collect() +} + +struct OperatorVariant { + syn: syn::Variant, + opts: PeepmaticOpts, +} + +fn create_arity(variants: &[OperatorVariant]) -> Result { + let mut imm_arities = vec![]; + let mut params_arities = vec![]; + + for v in variants { + let variant = &v.syn.ident; + + let imm_arity = v.opts.immediates.len(); + if imm_arity > std::u8::MAX as usize { + return Err(Error::new( + v.opts.immediates_paren.span, + "cannot have more than u8::MAX immediates", + )); + } + let imm_arity = imm_arity as u8; + + imm_arities.push(quote! { + Self::#variant => #imm_arity, + }); + + let params_arity = v.opts.params.len(); + if params_arity > std::u8::MAX as usize { + return Err(Error::new( + v.opts.params_paren.span, + "cannot have more than u8::MAX params", + )); + } + let params_arity = params_arity as u8; + + params_arities.push(quote! { + Self::#variant => #params_arity, + }); + } + + Ok(quote! { + /// Get the number of immediates that this operator has. + pub fn immediates_arity(&self) -> u8 { + match *self { + #( #imm_arities )* + } + } + + /// Get the number of parameters that this operator takes. + pub fn params_arity(&self) -> u8 { + match *self { + #( #params_arities )* + } + } + }) +} + +fn create_type_methods(variants: &[OperatorVariant]) -> impl quote::ToTokens { + let mut result_types = vec![]; + let mut imm_types = vec![]; + let mut param_types = vec![]; + + for v in variants { + let variant = &v.syn.ident; + + let result_ty = v.opts.result.as_ref().unwrap_or_else(|| { + panic!( + "must define #[peepmatic(result(..))] on operator `{}`", + variant + ) + }); + result_types.push(quote! { + Self::#variant => { + context.#result_ty(span) + } + }); + + let imm_tys = match &v.opts.immediates[..] { + [] => quote! {}, + [ty, rest @ ..] => { + let rest = rest.iter().map(|ty| { + quote! { .chain(::std::iter::once(context.#ty(span))) } + }); + quote! { + types.extend(::std::iter::once(context.#ty(span))#( #rest )*); + } + } + }; + imm_types.push(quote! { + Self::#variant => { + #imm_tys + } + }); + + let param_tys = match &v.opts.params[..] { + [] => quote! {}, + [ty, rest @ ..] => { + let rest = rest.iter().map(|ty| { + quote! { .chain(::std::iter::once(context.#ty(span))) } + }); + quote! { + types.extend(::std::iter::once(context.#ty(span))#( #rest )*); + } + } + }; + param_types.push(quote! { + Self::#variant => { + #param_tys + } + }); + } + + quote! { + /// Get the result type of this operator. + #[cfg(feature = "construct")] + pub fn result_type<'a, C>( + &self, + context: &mut C, + span: wast::Span, + ) -> C::TypeVariable + where + C: 'a + TypingContext<'a>, + { + match *self { + #( #result_types )* + } + } + + /// Get the immediate types of this operator. + #[cfg(feature = "construct")] + pub fn immediate_types<'a, C>( + &self, + context: &mut C, + span: wast::Span, + types: &mut impl Extend, + ) + where + C: 'a + TypingContext<'a>, + { + match *self { + #( #imm_types )* + } + } + + /// Get the parameter types of this operator. + #[cfg(feature = "construct")] + pub fn param_types<'a, C>( + &self, + context: &mut C, + span: wast::Span, + types: &mut impl Extend, + ) + where + C: 'a + TypingContext<'a>, + { + match *self { + #( #param_types )* + } + } + } +} + +fn snake_case(s: &str) -> String { + let mut t = String::with_capacity(s.len() + 1); + for (i, ch) in s.chars().enumerate() { + if i != 0 && ch.is_uppercase() { + t.push('_'); + } + t.extend(ch.to_lowercase()); + } + t +} + +fn create_parse_impl(ident: &syn::Ident, variants: &[OperatorVariant]) -> impl quote::ToTokens { + let token_defs = variants.iter().map(|v| { + let tok = snake_case(&v.syn.ident.to_string()); + let tok = Ident::new(&tok, Span::call_site()); + quote! { + wast::custom_keyword!(#tok); + } + }); + + let parses = variants.iter().map(|v| { + let tok = snake_case(&v.syn.ident.to_string()); + let tok = Ident::new(&tok, Span::call_site()); + let ident = &v.syn.ident; + quote! { + if p.peek::<#tok>() { + p.parse::<#tok>()?; + return Ok(Self::#ident); + } + } + }); + + let expected = format!("expected {}", ident); + + quote! { + #[cfg(feature = "construct")] + impl<'a> wast::parser::Parse<'a> for #ident { + fn parse(p: wast::parser::Parser<'a>) -> wast::parser::Result { + #( #token_defs )* + + #( #parses )* + + Err(p.error(#expected)) + } + } + } +} + +fn create_display_impl(ident: &syn::Ident, variants: &[OperatorVariant]) -> impl quote::ToTokens { + let displays = variants.iter().map(|v| { + let variant = &v.syn.ident; + let snake = snake_case(&v.syn.ident.to_string()); + quote! { + Self::#variant => write!(f, #snake), + } + }); + + quote! { + impl std::fmt::Display for #ident { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + #( #displays )* + } + } + } + } +} + +fn create_try_from_u32_impl( + ident: &syn::Ident, + variants: &[OperatorVariant], +) -> impl quote::ToTokens { + let matches = variants.iter().map(|v| { + let variant = &v.syn.ident; + quote! { + x if Self::#variant as u32 == x => Ok(Self::#variant), + } + }); + + let error_msg = format!("value is not an `{}`", ident); + + quote! { + impl std::convert::TryFrom for #ident { + type Error = &'static str; + + fn try_from(value: u32) -> Result { + match value { + #( #matches )* + _ => Err(#error_msg) + } + } + } + } +} diff --git a/cranelift/peepmatic/crates/macro/src/span.rs b/cranelift/peepmatic/crates/macro/src/span.rs new file mode 100644 index 0000000000..f994920dff --- /dev/null +++ b/cranelift/peepmatic/crates/macro/src/span.rs @@ -0,0 +1,50 @@ +use quote::quote; +use syn::DeriveInput; +use syn::{parse_quote, GenericParam, Generics, Result}; + +pub fn derive_span(input: &DeriveInput) -> Result { + let ty = &input.ident; + + let body = match &input.data { + syn::Data::Struct(_) => quote! { self.span }, + syn::Data::Enum(e) => { + let variants = e.variants.iter().map(|v| match v.fields { + syn::Fields::Unnamed(ref fields) if fields.unnamed.len() == 1 => { + let variant = &v.ident; + quote! { #ty::#variant(x) => x.span(), } + } + _ => panic!( + "derive(Ast) on enums only supports variants with a single, unnamed field" + ), + }); + quote! { + match self { + #( #variants )* + } + } + } + syn::Data::Union(_) => panic!("derive(Ast) can only be used with structs and enums, not unions"), + }; + + let generics = add_span_trait_bounds(input.generics.clone()); + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + Ok(quote! { + impl #impl_generics Span for #ty #ty_generics #where_clause { + #[inline] + fn span(&self) -> wast::Span { + #body + } + } + }) +} + +// Add a bound `T: Span` to every type parameter `T`. +fn add_span_trait_bounds(mut generics: Generics) -> Generics { + for param in &mut generics.params { + if let GenericParam::Type(ref mut type_param) = *param { + type_param.bounds.push(parse_quote!(Span)); + } + } + generics +} From 197a9e88cbec2ec1f5bd6effa1a08d5eaa092e44 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 1 May 2020 15:36:49 -0700 Subject: [PATCH 03/22] peepmatic: Introduce the `peepmatic-runtime` crate The `peepmatic-runtime` crate contains everything required to use a `peepmatic`-generated peephole optimizer. In short: build times and code size. If you are just using a peephole optimizer, you shouldn't need the functions to construct it from scratch from the DSL (and the implied code size and compilation time), let alone even build it at all. You should just deserialize an already-built peephole optimizer, and then use it. That's all that is contained here in this crate. --- cranelift/peepmatic/crates/runtime/Cargo.toml | 24 + cranelift/peepmatic/crates/runtime/src/cc.rs | 90 +++ .../peepmatic/crates/runtime/src/error.rs | 44 ++ .../crates/runtime/src/instruction_set.rs | 136 +++++ .../crates/runtime/src/integer_interner.rs | 71 +++ cranelift/peepmatic/crates/runtime/src/lib.rs | 34 ++ .../peepmatic/crates/runtime/src/linear.rs | 219 ++++++++ .../peepmatic/crates/runtime/src/operator.rs | 295 ++++++++++ .../crates/runtime/src/optimizations.rs | 74 +++ .../peepmatic/crates/runtime/src/optimizer.rs | 512 ++++++++++++++++++ .../peepmatic/crates/runtime/src/part.rs | 128 +++++ .../peepmatic/crates/runtime/src/paths.rs | 242 +++++++++ .../peepmatic/crates/runtime/src/type.rs | 262 +++++++++ 13 files changed, 2131 insertions(+) create mode 100644 cranelift/peepmatic/crates/runtime/Cargo.toml create mode 100644 cranelift/peepmatic/crates/runtime/src/cc.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/error.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/instruction_set.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/integer_interner.rs create mode 100755 cranelift/peepmatic/crates/runtime/src/lib.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/linear.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/operator.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/optimizations.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/optimizer.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/part.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/paths.rs create mode 100644 cranelift/peepmatic/crates/runtime/src/type.rs diff --git a/cranelift/peepmatic/crates/runtime/Cargo.toml b/cranelift/peepmatic/crates/runtime/Cargo.toml new file mode 100644 index 0000000000..31c4655b25 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "peepmatic-runtime" +version = "0.1.0" +authors = ["Nick Fitzgerald "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bincode = "1.2.1" +bumpalo = "3.2.0" +log = "0.4.8" +peepmatic-automata = { version = "0.1.0", path = "../automata", features = ["serde"] } +peepmatic-macro = { version = "0.1.0", path = "../macro" } +serde = { version = "1.0.105", features = ["derive"] } +thiserror = "1.0.15" +wast = { version = "13.0.0", optional = true } + + +[features] +# Enable support for a few extra methods that are required by the `peepmatic` +# crate when constructing peephole optimizers, but are not needed when simply +# using already-constructed peephole optimizers. +construct = ["wast"] diff --git a/cranelift/peepmatic/crates/runtime/src/cc.rs b/cranelift/peepmatic/crates/runtime/src/cc.rs new file mode 100644 index 0000000000..69f44db2ae --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/cc.rs @@ -0,0 +1,90 @@ +//! Condition codes. + +use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; +use std::fmt; + +/// A condition code. +/// +/// This is a special kind of immediate for `icmp` instructions that dictate +/// which parts of the comparison result we care about. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[repr(u32)] +pub enum ConditionCode { + /// Equal. + Eq = 1, + + /// Not equal. + Ne, + + /// Signed less than. + Slt, + + /// Unsigned less than. + Ult, + + /// Signed greater than or equal. + Sge, + + /// Unsigned greater than or equal. + Uge, + + /// Signed greater than. + Sgt, + + /// Unsigned greater than. + Ugt, + + /// Signed less than or equal. + Sle, + + /// Unsigned less than or equal. + Ule, + + /// Overflow. + Of, + + /// No overflow. + Nof, +} + +impl TryFrom for ConditionCode { + type Error = &'static str; + + fn try_from(x: u32) -> Result { + Ok(match x { + x if Self::Eq as u32 == x => Self::Eq, + x if Self::Ne as u32 == x => Self::Ne, + x if Self::Slt as u32 == x => Self::Slt, + x if Self::Ult as u32 == x => Self::Ult, + x if Self::Sge as u32 == x => Self::Sge, + x if Self::Uge as u32 == x => Self::Uge, + x if Self::Sgt as u32 == x => Self::Sgt, + x if Self::Ugt as u32 == x => Self::Ugt, + x if Self::Sle as u32 == x => Self::Sle, + x if Self::Ule as u32 == x => Self::Ule, + x if Self::Of as u32 == x => Self::Of, + x if Self::Nof as u32 == x => Self::Nof, + _ => return Err("not a valid condition code value"), + }) + } +} + +impl fmt::Display for ConditionCode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Eq => write!(f, "eq"), + Self::Ne => write!(f, "ne"), + Self::Slt => write!(f, "slt"), + Self::Ult => write!(f, "ult"), + Self::Sge => write!(f, "sge"), + Self::Uge => write!(f, "uge"), + Self::Sgt => write!(f, "sgt"), + Self::Ugt => write!(f, "ugt"), + Self::Sle => write!(f, "sle"), + Self::Ule => write!(f, "ule"), + Self::Of => write!(f, "of"), + Self::Nof => write!(f, "nof"), + } + } +} diff --git a/cranelift/peepmatic/crates/runtime/src/error.rs b/cranelift/peepmatic/crates/runtime/src/error.rs new file mode 100644 index 0000000000..3453316238 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/error.rs @@ -0,0 +1,44 @@ +//! `Error` and `Result` types for this crate. + +use std::io; +use thiserror::Error; + +/// A result type containing `Ok(T)` or `Err(peepmatic_runtime::Error)`. +pub type Result = std::result::Result; + +/// Errors that `peepmatic_runtime` may generate. +#[derive(Debug, Error)] +#[error(transparent)] +pub struct Error { + #[from] + inner: Box, +} + +#[derive(Debug, Error)] +enum ErrorInner { + #[error(transparent)] + Io(#[from] io::Error), + + #[error(transparent)] + Bincode(#[from] bincode::Error), +} + +impl From for Error { + fn from(e: io::Error) -> Error { + let e: ErrorInner = e.into(); + e.into() + } +} + +impl From for Error { + fn from(e: bincode::Error) -> Error { + let e: ErrorInner = e.into(); + e.into() + } +} + +impl From for Error { + fn from(e: ErrorInner) -> Error { + Box::new(e).into() + } +} diff --git a/cranelift/peepmatic/crates/runtime/src/instruction_set.rs b/cranelift/peepmatic/crates/runtime/src/instruction_set.rs new file mode 100644 index 0000000000..1a501d4ce4 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/instruction_set.rs @@ -0,0 +1,136 @@ +//! Interfacing with actual instructions. + +use crate::operator::Operator; +use crate::part::{Constant, Part}; +use crate::paths::Path; +use crate::r#type::Type; +use std::fmt::Debug; + +/// A trait for interfacing with actual instruction sequences. +/// +/// This trait enables both: +/// +/// * `peepmatic-runtime` to be used by `cranelift-codegen` without a circular +/// dependency from `peepmatic-runtime` to `cranelift-codegen` to get access +/// to Cranelift's IR types, and +/// +/// * enables us to write local tests that exercise peephole optimizers on a +/// simple, testing-only instruction set without pulling in all of Cranelift. +/// +/// Finally, this should also make the task of adding support for Cranelift's +/// new `MachInst` and vcode backend easier, since all that needs to be done is +/// "just" implementing this trait. (And probably add/modify some +/// `peepmatic_runtime::operation::Operation`s as well). +pub trait InstructionSet<'a> { + /// Mutable context passed into all trait methods. Can be whatever you want! + /// + /// In practice, this is a `FuncCursor` for `cranelift-codegen`'s trait + /// implementation. + type Context; + + /// An instruction (or identifier for an instruction). + type Instruction: Copy + Debug + Eq; + + /// Replace the `old` instruction with `new`. + /// + /// `new` is either a `Part::Instruction` or a constant `Part::Boolean` or + /// `Part::Integer`. In the former case, it can directly replace `old`. In + /// the latter case, implementations of this trait should transparently + /// create an `iconst` or `bconst` instruction to wrap the given constant. + /// + /// `new` will never be `Part::ConditionCode`. + fn replace_instruction( + &self, + context: &mut Self::Context, + old: Self::Instruction, + new: Part, + ) -> Self::Instruction; + + /// Get the instruction, constant, or condition code at the given path. + /// + /// If there is no such entity at the given path (e.g. we run into a + /// function parameter and can't traverse the path any further) then `None` + /// should be returned. + fn get_part_at_path( + &self, + context: &mut Self::Context, + root: Self::Instruction, + path: Path, + ) -> Option>; + + /// Get the given instruction's operator. + /// + /// If the instruction's opcode does not have an associated + /// `peepmatic_runtime::operator::Operator` variant (i.e. that instruction + /// isn't supported by `peepmatic` yet) then `None` should be returned. + fn operator(&self, context: &mut Self::Context, instr: Self::Instruction) -> Option; + + /// Make a unary instruction. + /// + /// If the type is not given, then it should be inferred. + fn make_inst_1( + &self, + context: &mut Self::Context, + root: Self::Instruction, + operator: Operator, + r#type: Type, + a: Part, + ) -> Self::Instruction; + + /// Make a binary instruction. + /// + /// Operands are given as immediates first and arguments following + /// them. Condition codes are treated as immediates. So if we are creating + /// an `iadd_imm` instruction, then `a` will be the constant integer + /// immediate and `b` will be the instruction whose result is the dynamic + /// argument. + fn make_inst_2( + &self, + context: &mut Self::Context, + root: Self::Instruction, + operator: Operator, + r#type: Type, + a: Part, + b: Part, + ) -> Self::Instruction; + + /// Make a ternary instruction. + /// + /// Operands are given as immediates first and arguments following + /// them. Condition codes are treated as immediates. So if we are creating + /// an `icmp` instruction, then `a` will be the condition code, and `b` and + /// `c` will be instructions whose results are the dynamic arguments. + fn make_inst_3( + &self, + context: &mut Self::Context, + root: Self::Instruction, + operator: Operator, + r#type: Type, + a: Part, + b: Part, + c: Part, + ) -> Self::Instruction; + + /// Try to resolve the given instruction into a constant value. + /// + /// If we can tell that the instruction returns a constant value, then + /// return that constant value as either a `Part::Boolean` or + /// `Part::Integer`. Otherwise, return `None`. + fn instruction_to_constant( + &self, + context: &mut Self::Context, + inst: Self::Instruction, + ) -> Option; + + /// Get the bit width of the given instruction's result. + /// + /// Must be one of 1, 8, 16, 32, 64, or 128. + fn instruction_result_bit_width( + &self, + context: &mut Self::Context, + inst: Self::Instruction, + ) -> u8; + + /// Get the size of a native word in bits. + fn native_word_size_in_bits(&self, context: &mut Self::Context) -> u8; +} diff --git a/cranelift/peepmatic/crates/runtime/src/integer_interner.rs b/cranelift/peepmatic/crates/runtime/src/integer_interner.rs new file mode 100644 index 0000000000..93dd2e7183 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/integer_interner.rs @@ -0,0 +1,71 @@ +//! Interner for (potentially large) integer values. +//! +//! We support matching on integers that can be represented by `u64`, but only +//! support automata results that fit in a `u32`. So we intern the (relatively +//! few compared to the full range of `u64`) integers we are matching against +//! here and then reference them by `IntegerId`. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::convert::TryInto; + +/// An identifier for an interned integer. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct IntegerId(#[doc(hidden)] pub u32); + +/// An interner for integer values. +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct IntegerInterner { + // Note: we use `BTreeMap`s for deterministic serialization. + map: BTreeMap, + values: Vec, +} + +impl IntegerInterner { + /// Construct a new `IntegerInterner`. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Intern a value into this `IntegerInterner`, returning its canonical + /// `IntegerId`. + #[inline] + pub fn intern(&mut self, value: impl Into) -> IntegerId { + debug_assert_eq!(self.map.len(), self.values.len()); + + let value = value.into(); + + if let Some(id) = self.map.get(&value) { + return *id; + } + + let id = IntegerId(self.values.len().try_into().unwrap()); + + self.values.push(value); + self.map.insert(value, id); + debug_assert_eq!(self.map.len(), self.values.len()); + + id + } + + /// Get the id of an already-interned integer, or `None` if it has not been + /// interned. + pub fn already_interned(&self, value: impl Into) -> Option { + let value = value.into(); + self.map.get(&value).copied() + } + + /// Lookup a previously interned integer by id. + #[inline] + pub fn lookup(&self, id: IntegerId) -> u64 { + self.values[id.0 as usize] + } +} + +impl From for u32 { + #[inline] + fn from(id: IntegerId) -> u32 { + id.0 + } +} diff --git a/cranelift/peepmatic/crates/runtime/src/lib.rs b/cranelift/peepmatic/crates/runtime/src/lib.rs new file mode 100755 index 0000000000..ae45c42f2b --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/lib.rs @@ -0,0 +1,34 @@ +//! Runtime support for `peepmatic`'s peephole optimizers. +//! +//! This crate contains everything required to use a `peepmatic`-generated +//! peephole optimizer. +//! +//! ## Why is this a different crate from `peepmatic`? +//! +//! In short: build times and code size. +//! +//! If you are just using a peephole optimizer, you shouldn't need the functions +//! to construct it from scratch from the DSL (and the implied code size and +//! compilation time), let alone even build it at all. You should just +//! deserialize an already-built peephole optimizer, and then use it. +//! +//! That's all that is contained here in this crate. + +#![deny(missing_docs)] +#![deny(missing_debug_implementations)] + +pub mod cc; +pub mod error; +pub mod instruction_set; +pub mod integer_interner; +pub mod linear; +pub mod operator; +pub mod optimizations; +pub mod optimizer; +pub mod part; +pub mod paths; +pub mod r#type; + +pub use error::{Error, Result}; +pub use optimizations::PeepholeOptimizations; +pub use optimizer::PeepholeOptimizer; diff --git a/cranelift/peepmatic/crates/runtime/src/linear.rs b/cranelift/peepmatic/crates/runtime/src/linear.rs new file mode 100644 index 0000000000..006f18e0e5 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/linear.rs @@ -0,0 +1,219 @@ +//! A linear IR for optimizations. +//! +//! This IR is designed such that it should be easy to combine multiple linear +//! optimizations into a single automata. +//! +//! See also `src/linearize.rs` for the AST to linear IR translation pass. + +use crate::cc::ConditionCode; +use crate::integer_interner::{IntegerId, IntegerInterner}; +use crate::operator::{Operator, UnquoteOperator}; +use crate::paths::{PathId, PathInterner}; +use crate::r#type::{BitWidth, Type}; +use serde::{Deserialize, Serialize}; + +/// A set of linear optimizations. +#[derive(Debug)] +pub struct Optimizations { + /// The linear optimizations. + pub optimizations: Vec, + + /// The de-duplicated paths referenced by these optimizations. + pub paths: PathInterner, + + /// The integer literals referenced by these optimizations. + pub integers: IntegerInterner, +} + +/// A linearized optimization. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Optimization { + /// The chain of increments for this optimization. + pub increments: Vec, +} + +/// An increment is a matching operation, the expected result from that +/// operation to continue to the next increment, and the actions to take to +/// build up the LHS scope and RHS instructions given that we got the expected +/// result from this increment's matching operation. Each increment will +/// basically become a state and a transition edge out of that state in the +/// final automata. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Increment { + /// The matching operation to perform. + pub operation: MatchOp, + + /// The expected result of our matching operation, that enables us to + /// continue to the next increment. `None` is used for wildcard-style "else" + /// transitions. + pub expected: Option, + + /// Actions to perform, given that the operation resulted in the expected + /// value. + pub actions: Vec, +} + +/// A matching operation to be performed on some Cranelift instruction as part +/// of determining whether an optimization is applicable. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)] +pub enum MatchOp { + /// Switch on the opcode of an instruction. + Opcode { + /// The path to the instruction whose opcode we're switching on. + path: PathId, + }, + + /// Does an instruction have a constant value? + IsConst { + /// The path to the instruction (or immediate) that we're checking + /// whether it is constant or not. + path: PathId, + }, + + /// Is the constant value a power of two? + IsPowerOfTwo { + /// The path to the instruction (or immediate) that we are checking + /// whether it is a constant power of two or not. + path: PathId, + }, + + /// Switch on the bit width of a value. + BitWidth { + /// The path to the instruction (or immediate) whose result's bit width + /// we are checking. + path: PathId, + }, + + /// Does the value fit in our target architecture's native word size? + FitsInNativeWord { + /// The path to the instruction (or immediate) whose result we are + /// checking whether it fits in a native word or not. + path: PathId, + }, + + /// Are the instructions (or immediates) at the given paths the same? + Eq { + /// The path to the first instruction (or immediate). + path_a: PathId, + /// The path to the second instruction (or immediate). + path_b: PathId, + }, + + /// Switch on the constant integer value of an instruction. + IntegerValue { + /// The path to the instruction. + path: PathId, + }, + + /// Switch on the constant boolean value of an instruction. + BooleanValue { + /// The path to the instruction. + path: PathId, + }, + + /// Switch on a condition code. + ConditionCode { + /// The path to the condition code. + path: PathId, + }, + + /// No operation. Always evaluates to `None`. + /// + /// Exceedingly rare in real optimizations; nonetheless required to support + /// corner cases of the DSL, such as a LHS pattern that is nothing but a + /// variable pattern. + Nop, +} + +/// A canonicalized identifier for a left-hand side value that was bound in a +/// pattern. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct LhsId(pub u32); + +/// A canonicalized identifier for a right-hand side value. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct RhsId(pub u32); + +/// An action to perform when transitioning between states in the automata. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Action { + /// Implicitly define the n^th built up RHS instruction as something from + /// the left-hand side. + GetLhs { + /// The path to the instruction or value. + path: PathId, + }, + + /// Implicitly define the n^th RHS instruction as the result of the + /// compile-time evaluation off this unquote operation. + UnaryUnquote { + /// The unquote operator. + operator: UnquoteOperator, + /// The constant operand to this unquote. + operand: RhsId, + }, + + /// Implicitly define the n^th RHS instruction as the result of the + /// compile-time evaluation off this unquote operation. + BinaryUnquote { + /// The unquote operator. + operator: UnquoteOperator, + /// The constant operands to this unquote. + operands: [RhsId; 2], + }, + + /// Implicitly define the n^th RHS as an integer constant. + MakeIntegerConst { + /// The constant integer value. + value: IntegerId, + /// The bit width of this constant. + bit_width: BitWidth, + }, + + /// Implicitly define the n^th RHS as a boolean constant. + MakeBooleanConst { + /// The constant boolean value. + value: bool, + /// The bit width of this constant. + bit_width: BitWidth, + }, + + /// Implicitly defint the n^th RHS as a condition code. + MakeConditionCode { + /// The condition code. + cc: ConditionCode, + }, + + /// Implicitly define the n^th RHS instruction by making a unary + /// instruction. + MakeUnaryInst { + /// The operand for this instruction. + operand: RhsId, + /// The type of this instruction's result. + r#type: Type, + /// The operator for this instruction. + operator: Operator, + }, + + /// Implicitly define the n^th RHS instruction by making a binary + /// instruction. + MakeBinaryInst { + /// The opcode for this instruction. + operator: Operator, + /// The type of this instruction's result. + r#type: Type, + /// The operands for this instruction. + operands: [RhsId; 2], + }, + + /// Implicitly define the n^th RHS instruction by making a ternary + /// instruction. + MakeTernaryInst { + /// The opcode for this instruction. + operator: Operator, + /// The type of this instruction's result. + r#type: Type, + /// The operands for this instruction. + operands: [RhsId; 3], + }, +} diff --git a/cranelift/peepmatic/crates/runtime/src/operator.rs b/cranelift/peepmatic/crates/runtime/src/operator.rs new file mode 100644 index 0000000000..35e17a89f7 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/operator.rs @@ -0,0 +1,295 @@ +//! Operator definitions. + +use peepmatic_macro::PeepmaticOperator; +use serde::{Deserialize, Serialize}; + +/// An operator. +/// +/// These are a subset of Cranelift IR's operators. +/// +/// ## Caveats for Branching and Trapping Operators +/// +/// Branching operators are not fully modeled: we do not represent their label +/// and jump arguments. It is up to the interpreter doing the instruction +/// replacement to recognize when we are replacing one branch with another, and +/// copy over the extra information. +/// +/// Affected operations: `brz`, `brnz`, `trapz`, `trapnz`. +#[derive(PeepmaticOperator, Clone, Copy, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)] +#[repr(u32)] +pub enum Operator { + /// `adjust_sp_down` + #[peepmatic(params(iNN), result(void))] + AdjustSpDown = 1, + + /// `adjust_sp_down_imm` + #[peepmatic(immediates(iNN), result(void))] + AdjustSpDownImm, + + /// `band` + #[peepmatic(params(iNN, iNN), result(iNN))] + Band, + + /// `band_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + BandImm, + + /// `bconst` + #[peepmatic(immediates(b1), result(bNN))] + Bconst, + + /// `bint` + #[peepmatic(params(bNN), result(iNN))] + Bint, + + /// `bor` + #[peepmatic(params(iNN, iNN), result(iNN))] + Bor, + + /// `bor_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + BorImm, + + /// `brnz` + #[peepmatic(params(bool_or_int), result(void))] + Brnz, + + /// `brz` + #[peepmatic(params(bool_or_int), result(void))] + Brz, + + /// `bxor` + #[peepmatic(params(iNN, iNN), result(iNN))] + Bxor, + + /// `bxor_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + BxorImm, + + /// `iadd` + #[peepmatic(params(iNN, iNN), result(iNN))] + Iadd, + + /// `iadd_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + IaddImm, + + /// `icmp` + #[peepmatic(immediates(cc), params(iNN, iNN), result(b1))] + Icmp, + + /// `icmp_imm` + #[peepmatic(immediates(cc, iNN), params(iNN), result(b1))] + IcmpImm, + + /// `iconst` + #[peepmatic(immediates(iNN), result(iNN))] + Iconst, + + /// `ifcmp` + #[peepmatic(params(iNN, iNN), result(cpu_flags))] + Ifcmp, + + /// `ifcmp_imm` + #[peepmatic(immediates(iNN), params(iNN), result(cpu_flags))] + IfcmpImm, + + /// `imul` + #[peepmatic(params(iNN, iNN), result(iNN))] + Imul, + + /// `imul_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + ImulImm, + + /// `ireduce` + #[peepmatic(params(iNN), result(iMM))] + Ireduce, + + /// `irsub_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + IrsubImm, + + /// `ishl` + #[peepmatic(params(iNN, iNN), result(iNN))] + Ishl, + + /// `ishl_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + IshlImm, + + /// `isub` + #[peepmatic(params(iNN, iNN), result(iNN))] + Isub, + + /// `rotl` + #[peepmatic(params(iNN, iNN), result(iNN))] + Rotl, + + /// `rotl_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + RotlImm, + + /// `rotr` + #[peepmatic(params(iNN, iNN), result(iNN))] + Rotr, + + /// `rotr_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + RotrImm, + + /// `sdiv` + #[peepmatic(params(iNN, iNN), result(iNN))] + Sdiv, + + /// `sdiv_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + SdivImm, + + /// `select` + #[peepmatic(params(bool_or_int, any_t, any_t), result(any_t))] + Select, + + /// `sextend` + #[peepmatic(params(iNN), result(iMM))] + Sextend, + + /// `srem` + #[peepmatic(params(iNN, iNN), result(iNN))] + Srem, + + /// `srem_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + SremImm, + + /// `sshr` + #[peepmatic(params(iNN, iNN), result(iNN))] + Sshr, + + /// `sshr_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + SshrImm, + + /// `trapnz` + #[peepmatic(params(bool_or_int), result(void))] + Trapnz, + + /// `trapz` + #[peepmatic(params(bool_or_int), result(void))] + Trapz, + + /// `udiv` + #[peepmatic(params(iNN, iNN), result(iNN))] + Udiv, + + /// `udiv_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + UdivImm, + + /// `uextend` + #[peepmatic(params(iNN), result(iMM))] + Uextend, + + /// `urem` + #[peepmatic(params(iNN, iNN), result(iNN))] + Urem, + + /// `urem_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + UremImm, + + /// `ushr` + #[peepmatic(params(iNN, iNN), result(iNN))] + Ushr, + + /// `ushr_imm` + #[peepmatic(immediates(iNN), params(iNN), result(iNN))] + UshrImm, +} + +/// Compile-time unquote operators. +/// +/// These are used in the right-hand side to perform compile-time evaluation of +/// constants matched on the left-hand side. +#[derive(PeepmaticOperator, Clone, Copy, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)] +#[repr(u32)] +pub enum UnquoteOperator { + /// Compile-time `band` of two constant values. + #[peepmatic(params(iNN, iNN), result(iNN))] + Band, + + /// Compile-time `bor` of two constant values. + #[peepmatic(params(iNN, iNN), result(iNN))] + Bor, + + /// Compile-time `bxor` of two constant values. + #[peepmatic(params(iNN, iNN), result(iNN))] + Bxor, + + /// Compile-time `iadd` of two constant values. + #[peepmatic(params(iNN, iNN), result(iNN))] + Iadd, + + /// Compile-time `imul` of two constant values. + #[peepmatic(params(iNN, iNN), result(iNN))] + Imul, + + /// Take the base-2 log of a power of two integer. + #[peepmatic(params(iNN), result(iNN))] + Log2, + + /// Wrapping negation of an integer. + #[peepmatic(params(iNN), result(iNN))] + Neg, +} + +/// A trait to represent a typing context. +/// +/// This is used by the macro-generated operator methods that create the type +/// variables for their immediates, parameters, and results. This trait is +/// implemented by the concrete typing context in `peepmatic/src/verify.rs`. +#[cfg(feature = "construct")] +pub trait TypingContext<'a> { + /// A type variable. + type TypeVariable; + + /// Create a condition code type. + fn cc(&mut self, span: wast::Span) -> Self::TypeVariable; + + /// Create a boolean type with a polymorphic bit width. + /// + /// Each use of `bNN` by the same operator refers to the same type variable. + #[allow(non_snake_case)] + fn bNN(&mut self, span: wast::Span) -> Self::TypeVariable; + + /// Create an integer type with a polymorphic bit width. + /// + /// Each use of `iNN` by the same operator refers to the same type variable. + #[allow(non_snake_case)] + fn iNN(&mut self, span: wast::Span) -> Self::TypeVariable; + + /// Create an integer type with a polymorphic bit width. + /// + /// Each use of `iMM` by the same operator refers to the same type variable. + #[allow(non_snake_case)] + fn iMM(&mut self, span: wast::Span) -> Self::TypeVariable; + + /// Create the CPU flags type variable. + fn cpu_flags(&mut self, span: wast::Span) -> Self::TypeVariable; + + /// Create a boolean type of size one bit. + fn b1(&mut self, span: wast::Span) -> Self::TypeVariable; + + /// Create the void type, used as the result of operators that branch away, + /// or do not return anything. + fn void(&mut self, span: wast::Span) -> Self::TypeVariable; + + /// Create a type variable that may be either a boolean or an integer. + fn bool_or_int(&mut self, span: wast::Span) -> Self::TypeVariable; + + /// Create a type variable that can be any type T. + /// + /// Each use of `any_t` by the same operator refers to the same type + /// variable. + fn any_t(&mut self, span: wast::Span) -> Self::TypeVariable; +} diff --git a/cranelift/peepmatic/crates/runtime/src/optimizations.rs b/cranelift/peepmatic/crates/runtime/src/optimizations.rs new file mode 100644 index 0000000000..29dfa7f388 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/optimizations.rs @@ -0,0 +1,74 @@ +//! Compiled peephole optimizations. + +use crate::error::Result; +use crate::instruction_set::InstructionSet; +use crate::integer_interner::IntegerInterner; +use crate::linear::{Action, MatchOp}; +use crate::optimizer::PeepholeOptimizer; +use crate::paths::PathInterner; +use peepmatic_automata::Automaton; +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "construct")] +use std::fs; +#[cfg(feature = "construct")] +use std::path::Path; + +/// A compiled set of peephole optimizations. +/// +/// This is the compilation result of the `peepmatic` crate, after its taken a +/// bunch of optimizations written in the DSL and lowered and combined them. +#[derive(Debug, Serialize, Deserialize)] +pub struct PeepholeOptimizations { + /// The instruction paths referenced by the peephole optimizations. + pub paths: PathInterner, + + /// Not all integers we're matching on fit in the `u32` that we use as the + /// result of match operations. So we intern them and refer to them by id. + pub integers: IntegerInterner, + + /// The underlying automata for matching optimizations' left-hand sides, and + /// building up the corresponding right-hand side. + pub automata: Automaton, MatchOp, Vec>, +} + +impl PeepholeOptimizations { + /// Deserialize a `PeepholeOptimizations` from bytes. + pub fn deserialize(serialized: &[u8]) -> Result { + let peep_opt: Self = bincode::deserialize(serialized)?; + Ok(peep_opt) + } + + /// Serialize these peephole optimizations out to the file at the given path. + /// + /// Requires that the `"construct"` cargo feature is enabled. + #[cfg(feature = "construct")] + pub fn serialize_to_file(&self, path: &Path) -> Result<()> { + let file = fs::File::create(path)?; + bincode::serialize_into(file, self)?; + Ok(()) + } + + /// Create a new peephole optimizer instance from this set of peephole + /// optimizations. + /// + /// The peephole optimizer instance can be used to apply these peephole + /// optimizations. When checking multiple instructions for whether they can + /// be optimized, it is more performant to reuse a single peephole optimizer + /// instance, rather than create a new one for each instruction. Reusing the + /// peephole optimizer instance allows the reuse of a few internal + /// allocations. + pub fn optimizer<'peep, 'ctx, I>(&'peep self, instr_set: I) -> PeepholeOptimizer<'peep, 'ctx, I> + where + I: InstructionSet<'ctx>, + { + PeepholeOptimizer { + peep_opt: self, + instr_set, + left_hand_sides: vec![], + right_hand_sides: vec![], + actions: vec![], + backtracking_states: vec![], + } + } +} diff --git a/cranelift/peepmatic/crates/runtime/src/optimizer.rs b/cranelift/peepmatic/crates/runtime/src/optimizer.rs new file mode 100644 index 0000000000..535bb56219 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/optimizer.rs @@ -0,0 +1,512 @@ +//! An optimizer for a set of peephole optimizations. + +use crate::instruction_set::InstructionSet; +use crate::linear::{Action, MatchOp}; +use crate::operator::UnquoteOperator; +use crate::optimizations::PeepholeOptimizations; +use crate::part::{Constant, Part}; +use crate::r#type::{BitWidth, Type}; +use peepmatic_automata::State; +use std::convert::TryFrom; +use std::fmt::{self, Debug}; +use std::mem; + +/// A peephole optimizer instance that can apply a set of peephole +/// optimizations to instructions. +/// +/// These are created from a set of peephole optimizations with the +/// [`PeepholeOptimizer::instance`][crate::PeepholeOptimizer::instance] method. +/// +/// Reusing an instance when applying peephole optimizations to different +/// instruction sequences means that you reuse internal allocations that are +/// used to match left-hand sides and build up right-hand sides. +pub struct PeepholeOptimizer<'peep, 'ctx, I> +where + I: InstructionSet<'ctx>, +{ + pub(crate) peep_opt: &'peep PeepholeOptimizations, + pub(crate) instr_set: I, + pub(crate) left_hand_sides: Vec>, + pub(crate) right_hand_sides: Vec>, + pub(crate) actions: Vec, + pub(crate) backtracking_states: Vec<(State, usize)>, +} + +impl<'peep, 'ctx, I> Debug for PeepholeOptimizer<'peep, 'ctx, I> +where + I: InstructionSet<'ctx>, +{ + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let PeepholeOptimizer { + peep_opt, + instr_set: _, + left_hand_sides, + right_hand_sides, + actions, + backtracking_states, + } = self; + f.debug_struct("PeepholeOptimizer") + .field("peep_opt", peep_opt) + .field("instr_set", &"_") + .field("left_hand_sides", left_hand_sides) + .field("right_hand_sides", right_hand_sides) + .field("actions", actions) + .field("backtracking_states", backtracking_states) + .finish() + } +} + +impl<'peep, 'ctx, I> PeepholeOptimizer<'peep, 'ctx, I> +where + I: InstructionSet<'ctx>, +{ + fn eval_unquote_1(&self, operator: UnquoteOperator, a: Constant) -> Constant { + use Constant::*; + + macro_rules! map_int { + ( $c:expr , | $x:ident | $e:expr ) => { + match $c { + Int($x, w) => Int($e, w), + Bool(..) => panic!("not an integer"), + } + }; + } + + match operator { + UnquoteOperator::Log2 => map_int!(a, |x| x.trailing_zeros() as _), + UnquoteOperator::Neg => map_int!(a, |x| x.wrapping_neg()), + UnquoteOperator::Band + | UnquoteOperator::Bor + | UnquoteOperator::Bxor + | UnquoteOperator::Iadd + | UnquoteOperator::Imul => unreachable!("not a unary unquote operator: {:?}", operator), + } + } + + fn eval_unquote_2(&self, operator: UnquoteOperator, a: Constant, b: Constant) -> Constant { + use Constant::*; + + macro_rules! fold_ints { + ( $c1:expr , $c2:expr , | $x:ident , $y:ident | $e:expr ) => { + match ($c1, $c2) { + (Int($x, w1), Int($y, w2)) if w1 == w2 => Int($e, w1), + _ => panic!("not two integers of the same width"), + } + }; + } + + match operator { + UnquoteOperator::Band => fold_ints!(a, b, |x, y| x & y), + UnquoteOperator::Bor => fold_ints!(a, b, |x, y| x | y), + UnquoteOperator::Bxor => fold_ints!(a, b, |x, y| x ^ y), + UnquoteOperator::Iadd => fold_ints!(a, b, |x, y| x.wrapping_add(y)), + UnquoteOperator::Imul => fold_ints!(a, b, |x, y| x.wrapping_mul(y)), + UnquoteOperator::Log2 | UnquoteOperator::Neg => { + unreachable!("not a binary unquote operator: {:?}", operator) + } + } + } + + fn eval_actions(&mut self, context: &mut I::Context, root: I::Instruction) { + let mut actions = mem::replace(&mut self.actions, vec![]); + + for action in actions.drain(..) { + log::trace!("Evaluating action: {:?}", action); + match action { + Action::GetLhs { path } => { + let path = self.peep_opt.paths.lookup(path); + let lhs = self + .instr_set + .get_part_at_path(context, root, path) + .expect("should always get part at path OK by the time it is bound"); + self.right_hand_sides.push(lhs); + } + Action::UnaryUnquote { operator, operand } => { + let operand = self.right_hand_sides[operand.0 as usize]; + let operand = match operand { + Part::Instruction(i) => self + .instr_set + .instruction_to_constant(context, i) + .expect("cannot convert instruction to constant for unquote operand"), + Part::Constant(c) => c, + Part::ConditionCode(_) => { + panic!("cannot use a condition code as an unquote operand") + } + }; + let result = self.eval_unquote_1(operator, operand); + self.right_hand_sides.push(result.into()); + } + Action::BinaryUnquote { operator, operands } => { + let a = self.right_hand_sides[operands[0].0 as usize]; + let a = match a { + Part::Instruction(i) => self + .instr_set + .instruction_to_constant(context, i) + .expect("cannot convert instruction to constant for unquote operand"), + Part::Constant(c) => c, + Part::ConditionCode(_) => { + panic!("cannot use a condition code as an unquote operand") + } + }; + + let b = self.right_hand_sides[operands[1].0 as usize]; + let b = match b { + Part::Instruction(i) => self + .instr_set + .instruction_to_constant(context, i) + .expect("cannot convert instruction to constant for unquote operand"), + Part::Constant(c) => c, + Part::ConditionCode(_) => { + panic!("cannot use a condition code as an unquote operand") + } + }; + + let result = self.eval_unquote_2(operator, a, b); + self.right_hand_sides.push(result.into()); + } + Action::MakeIntegerConst { + value, + mut bit_width, + } => { + let value = self.peep_opt.integers.lookup(value); + if bit_width.is_polymorphic() { + bit_width = BitWidth::try_from( + self.instr_set.instruction_result_bit_width(context, root), + ) + .unwrap(); + } + self.right_hand_sides + .push(Constant::Int(value, bit_width).into()); + } + Action::MakeBooleanConst { + value, + mut bit_width, + } => { + if bit_width.is_polymorphic() { + bit_width = BitWidth::try_from( + self.instr_set.instruction_result_bit_width(context, root), + ) + .unwrap(); + } + self.right_hand_sides + .push(Constant::Bool(value, bit_width).into()); + } + Action::MakeConditionCode { cc } => { + self.right_hand_sides.push(Part::ConditionCode(cc)); + } + Action::MakeUnaryInst { + operator, + r#type: + Type { + kind, + mut bit_width, + }, + operand, + } => { + if bit_width.is_polymorphic() { + bit_width = BitWidth::try_from( + self.instr_set.instruction_result_bit_width(context, root), + ) + .unwrap(); + } + let ty = Type { kind, bit_width }; + let operand = self.right_hand_sides[operand.0 as usize]; + let inst = self + .instr_set + .make_inst_1(context, root, operator, ty, operand); + self.right_hand_sides.push(Part::Instruction(inst)); + } + Action::MakeBinaryInst { + operator, + r#type: + Type { + kind, + mut bit_width, + }, + operands, + } => { + if bit_width.is_polymorphic() { + bit_width = BitWidth::try_from( + self.instr_set.instruction_result_bit_width(context, root), + ) + .unwrap(); + } + let ty = Type { kind, bit_width }; + let a = self.right_hand_sides[operands[0].0 as usize]; + let b = self.right_hand_sides[operands[1].0 as usize]; + let inst = self + .instr_set + .make_inst_2(context, root, operator, ty, a, b); + self.right_hand_sides.push(Part::Instruction(inst)); + } + Action::MakeTernaryInst { + operator, + r#type: + Type { + kind, + mut bit_width, + }, + operands, + } => { + if bit_width.is_polymorphic() { + bit_width = BitWidth::try_from( + self.instr_set.instruction_result_bit_width(context, root), + ) + .unwrap(); + } + let ty = Type { kind, bit_width }; + let a = self.right_hand_sides[operands[0].0 as usize]; + let b = self.right_hand_sides[operands[1].0 as usize]; + let c = self.right_hand_sides[operands[2].0 as usize]; + let inst = self + .instr_set + .make_inst_3(context, root, operator, ty, a, b, c); + self.right_hand_sides.push(Part::Instruction(inst)); + } + } + } + + // Reuse the heap elements allocation. + self.actions = actions; + } + + fn eval_match_op( + &mut self, + context: &mut I::Context, + root: I::Instruction, + match_op: MatchOp, + ) -> Option { + use crate::linear::MatchOp::*; + + log::trace!("Evaluating match operation: {:?}", match_op); + let result = match match_op { + Opcode { path } => { + let path = self.peep_opt.paths.lookup(path); + let part = self.instr_set.get_part_at_path(context, root, path)?; + let inst = part.as_instruction()?; + self.instr_set.operator(context, inst).map(|op| op as u32) + } + IsConst { path } => { + let path = self.peep_opt.paths.lookup(path); + let part = self.instr_set.get_part_at_path(context, root, path)?; + let is_const = match part { + Part::Instruction(i) => { + self.instr_set.instruction_to_constant(context, i).is_some() + } + Part::ConditionCode(_) | Part::Constant(_) => true, + }; + Some(is_const as u32) + } + IsPowerOfTwo { path } => { + let path = self.peep_opt.paths.lookup(path); + let part = self.instr_set.get_part_at_path(context, root, path)?; + match part { + Part::Constant(c) => Some(c.as_int().unwrap().is_power_of_two() as u32), + Part::Instruction(i) => { + let c = self.instr_set.instruction_to_constant(context, i)?; + Some(c.as_int().unwrap().is_power_of_two() as u32) + } + Part::ConditionCode(_) => panic!("IsPowerOfTwo on a condition code"), + } + } + BitWidth { path } => { + let path = self.peep_opt.paths.lookup(path); + let part = self.instr_set.get_part_at_path(context, root, path)?; + let bit_width = match part { + Part::Instruction(i) => self.instr_set.instruction_result_bit_width(context, i), + Part::Constant(Constant::Int(_, w)) | Part::Constant(Constant::Bool(_, w)) => { + w.fixed_width().unwrap_or_else(|| { + self.instr_set.instruction_result_bit_width(context, root) + }) + } + Part::ConditionCode(_) => panic!("BitWidth on condition code"), + }; + Some(bit_width as u32) + } + FitsInNativeWord { path } => { + let native_word_size = self.instr_set.native_word_size_in_bits(context); + debug_assert!(native_word_size.is_power_of_two()); + + let path = self.peep_opt.paths.lookup(path); + let part = self.instr_set.get_part_at_path(context, root, path)?; + let fits = match part { + Part::Instruction(i) => { + let size = self.instr_set.instruction_result_bit_width(context, i); + size <= native_word_size + } + Part::Constant(c) => { + let root_width = self.instr_set.instruction_result_bit_width(context, root); + let size = c.bit_width(root_width); + size <= native_word_size + } + Part::ConditionCode(_) => panic!("FitsInNativeWord on condition code"), + }; + Some(fits as u32) + } + Eq { path_a, path_b } => { + let path_a = self.peep_opt.paths.lookup(path_a); + let part_a = self.instr_set.get_part_at_path(context, root, path_a)?; + let path_b = self.peep_opt.paths.lookup(path_b); + let part_b = self.instr_set.get_part_at_path(context, root, path_b)?; + let eq = match (part_a, part_b) { + (Part::Instruction(inst), Part::Constant(c1)) + | (Part::Constant(c1), Part::Instruction(inst)) => { + match self.instr_set.instruction_to_constant(context, inst) { + Some(c2) => c1 == c2, + None => false, + } + } + (a, b) => a == b, + }; + Some(eq as _) + } + IntegerValue { path } => { + let path = self.peep_opt.paths.lookup(path); + let part = self.instr_set.get_part_at_path(context, root, path)?; + match part { + Part::Constant(c) => { + let x = c.as_int()?; + self.peep_opt.integers.already_interned(x).map(|id| id.0) + } + Part::Instruction(i) => { + let c = self.instr_set.instruction_to_constant(context, i)?; + let x = c.as_int()?; + self.peep_opt.integers.already_interned(x).map(|id| id.0) + } + Part::ConditionCode(_) => panic!("IntegerValue on condition code"), + } + } + BooleanValue { path } => { + let path = self.peep_opt.paths.lookup(path); + let part = self.instr_set.get_part_at_path(context, root, path)?; + match part { + Part::Constant(c) => c.as_bool().map(|b| b as u32), + Part::Instruction(i) => { + let c = self.instr_set.instruction_to_constant(context, i)?; + c.as_bool().map(|b| b as u32) + } + Part::ConditionCode(_) => panic!("IntegerValue on condition code"), + } + } + ConditionCode { path } => { + let path = self.peep_opt.paths.lookup(path); + let part = self.instr_set.get_part_at_path(context, root, path)?; + part.as_condition_code().map(|cc| cc as u32) + } + MatchOp::Nop => None, + }; + log::trace!("Evaluated match operation: {:?} = {:?}", match_op, result); + result + } + + /// Attempt to apply a single peephole optimization to the given root + /// instruction. + /// + /// If an optimization is applied, then the `root` is replaced with the + /// optimization's right-hand side, and the root of the right-hand side is + /// returned as `Some`. + /// + /// If no optimization's left-hand side matches `root`, then `root` is left + /// untouched and `None` is returned. + pub fn apply_one( + &mut self, + context: &mut I::Context, + root: I::Instruction, + ) -> Option { + log::trace!("PeepholeOptimizer::apply_one"); + + self.backtracking_states.clear(); + self.actions.clear(); + self.left_hand_sides.clear(); + self.right_hand_sides.clear(); + + let mut r#final = None; + + let mut query = self.peep_opt.automata.query(); + loop { + log::trace!("Current state: {:?}", query.current_state()); + + if query.is_in_final_state() { + // If we're in a final state (which means an optimization is + // applicable) then record that fact, but keep going. We don't + // want to stop yet, because we might discover another, + // more-specific optimization that is also applicable if we keep + // going. And we always want to apply the most specific + // optimization that matches. + log::trace!("Found a match at state {:?}", query.current_state()); + r#final = Some((query.current_state(), self.actions.len())); + } + + // Anything following a `None` transition doesn't care about the + // result of this match operation, so if we partially follow the + // current non-`None` path, but don't ultimately find a matching + // optimization, we want to be able to backtrack to this state and + // then try taking the `None` transition. + if query.has_transition_on(&None) { + self.backtracking_states + .push((query.current_state(), self.actions.len())); + } + + let match_op = match query.current_state_data() { + None => break, + Some(op) => op, + }; + + let input = self.eval_match_op(context, root, *match_op); + + let actions = if let Some(actions) = query.next(&input) { + actions + } else if r#final.is_some() { + break; + } else if let Some((state, actions_len)) = self.backtracking_states.pop() { + query.go_to_state(state); + self.actions.truncate(actions_len); + query + .next(&None) + .expect("backtracking states always have `None` transitions") + } else { + break; + }; + + self.actions.extend(actions.iter().copied()); + } + + // If `final` is none, then we didn't encounter any final states, so + // there are no applicable optimizations. + let (final_state, actions_len) = match r#final { + Some(f) => f, + None => { + log::trace!("No optimizations matched"); + return None; + } + }; + + // Go to the last final state we saw, reset the LHS and RHS to how + // they were at the time we saw the final state, and process the + // final actions. + self.actions.truncate(actions_len); + query.go_to_state(final_state); + let final_actions = query.finish().expect("should be in a final state"); + self.actions.extend(final_actions.iter().copied()); + self.eval_actions(context, root); + + // And finally, the root of the RHS for this optimization is the + // last entry in `self.right_hand_sides`, so replace the old root + // instruction with this one! + let result = self.right_hand_sides.pop().unwrap(); + let new_root = self.instr_set.replace_instruction(context, root, result); + Some(new_root) + } + + /// Keep applying peephole optimizations to the given instruction until none + /// can be applied anymore. + pub fn apply_all(&mut self, context: &mut I::Context, mut inst: I::Instruction) { + loop { + if let Some(new_inst) = self.apply_one(context, inst) { + inst = new_inst; + } else { + break; + } + } + } +} diff --git a/cranelift/peepmatic/crates/runtime/src/part.rs b/cranelift/peepmatic/crates/runtime/src/part.rs new file mode 100644 index 0000000000..6190b8e980 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/part.rs @@ -0,0 +1,128 @@ +//! Parts of instructions. + +use crate::cc::ConditionCode; +use crate::r#type::BitWidth; +use std::fmt::Debug; + +/// A constant value. +/// +/// Whether an integer is interpreted as signed or unsigned depends on the +/// operations applied to it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Constant { + /// A boolean of the given width. + Bool(bool, BitWidth), + + /// An integer constant of the given width, + Int(u64, BitWidth), +} + +/// A part of an instruction, or a whole instruction itself. +/// +/// These are the different values that can be matched in an optimization's +/// left-hand side and then built up in its right-hand side. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Part +where + I: Copy + Debug + Eq, +{ + /// An instruction (or result of an instruction). + Instruction(I), + + /// A constant value. + Constant(Constant), + + /// A condition code. + ConditionCode(ConditionCode), +} + +impl From for Part +where + I: Copy + Debug + Eq, +{ + #[inline] + fn from(c: Constant) -> Part { + Part::Constant(c) + } +} + +impl From for Part +where + I: Copy + Debug + Eq, +{ + #[inline] + fn from(c: ConditionCode) -> Part { + Part::ConditionCode(c) + } +} + +macro_rules! accessors { + ( $( $variant:ident , $result:ty , $getter:ident , $is:ident , $unwrap:ident ; )* ) => { + $( + #[inline] + #[allow(missing_docs)] + pub fn $getter(&self) -> Option<$result> { + match *self { + Self::$variant(x, ..) => Some(x), + _ => None + } + } + + #[inline] + #[allow(missing_docs)] + pub fn $is(&self) -> bool { + self.$getter().is_some() + } + + #[inline] + #[allow(missing_docs)] + pub fn $unwrap(&self) -> $result { + self.$getter().expect(concat!("failed to unwrap `", stringify!($variant), "`")) + } + )* + } +} + +impl Constant { + /// If this is any kind of integer constant, get it as a 64-bit unsigned + /// integer. + pub fn as_int(&self) -> Option { + match *self { + Constant::Bool(..) => None, + Constant::Int(x, _) => Some(x), + } + } + + /// If this is any kind of boolean constant, get its value. + pub fn as_bool(&self) -> Option { + match *self { + Constant::Bool(b, _) => Some(b), + Constant::Int(..) => None, + } + } + + /// The number of bits required to represent this constant value's type. + pub fn bit_width(&self, root_width: u8) -> u8 { + match *self { + Constant::Bool(_, w) | Constant::Int(_, w) => { + if let Some(w) = w.fixed_width() { + w + } else { + debug_assert!(w.is_polymorphic()); + root_width + } + } + } + } +} + +impl Part +where + I: Copy + Debug + Eq, +{ + accessors! { + Instruction, I, as_instruction, is_instruction, unwrap_instruction; + Constant, Constant, as_constant, is_constant, unwrap_constant; + ConditionCode, ConditionCode, as_condition_code, is_condition_code, unwrap_condition_code; + } +} diff --git a/cranelift/peepmatic/crates/runtime/src/paths.rs b/cranelift/peepmatic/crates/runtime/src/paths.rs new file mode 100644 index 0000000000..a30aa81886 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/paths.rs @@ -0,0 +1,242 @@ +//! Representing paths through the dataflow graph. +//! +//! Paths are relative from a *root* instruction, which is the instruction we +//! are determining which, if any, optimizations apply. +//! +//! Paths are series of indices through each instruction's children as we +//! traverse down the graph from the root. Children are immediates followed by +//! arguments: `[imm0, imm1, ..., immN, arg0, arg1, ..., argN]`. +//! +//! ## Examples +//! +//! * `[0]` is the path to the root. +//! * `[0, 0]` is the path to the root's first child. +//! * `[0, 1]` is the path to the root's second child. +//! * `[0, 1, 0]` is the path to the root's second child's first child. +//! +//! ## Interning +//! +//! To avoid extra allocations, de-duplicate paths, and reference them via a +//! fixed-length value, we intern paths inside a `PathInterner` and then +//! reference them via `PathId`. + +// TODO: Make `[]` the path to the root, and get rid of this redundant leading +// zero that is currently in every single path. + +use serde::de::{Deserializer, SeqAccess, Visitor}; +use serde::ser::{SerializeSeq, Serializer}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::convert::TryInto; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; + +/// A path through the data-flow graph from the root instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Path<'a>(pub &'a [u8]); + +impl Path<'_> { + /// Construct a new path through the data-flow graph from the root + /// instruction. + pub fn new(path: &impl AsRef<[u8]>) -> Path { + Path(path.as_ref()) + } +} + +/// An identifier for an interned path. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct PathId(u32); + +/// An interner and de-duplicator for `Path`s. +/// +/// Can be serialized and deserialized while maintaining the same id to interned +/// path mapping. +#[derive(Debug, Default)] +pub struct PathInterner { + // A map from a path (whose owned data is inside `arena`) to the canonical + // `PathId` we assigned it when interning it. + map: HashMap, + + // A map from a `PathId` index to an unsafe, self-borrowed path pointing + // into `arena`. It is safe to given these out as safe `Path`s, as long as + // the lifetime is not longer than this `PathInterner`'s lifetime. + paths: Vec, + + // Bump allocation arena for path data. The bump arena ensures that these + // allocations never move, and are therefore safe for self-references. + arena: bumpalo::Bump, +} + +impl PathInterner { + /// Construct a new, empty `PathInterner`. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Intern a path into this `PathInterner`, returning its canonical + /// `PathId`. + /// + /// If we've already interned this path before, then the existing id we + /// already assigned to it is returned. If we've never seen this path + /// before, then it is copied into this `PathInterner` and a new id is + /// assigned to it. + #[inline] + pub fn intern<'a>(&mut self, path: Path<'a>) -> PathId { + let unsafe_path = unsafe { UnsafePath::from_path(&path) }; + if let Some(id) = self.map.get(&unsafe_path) { + return *id; + } + self.intern_new(path) + } + + #[inline(never)] + fn intern_new<'a>(&mut self, path: Path<'a>) -> PathId { + let id: u32 = self + .paths + .len() + .try_into() + .expect("too many paths interned"); + let id = PathId(id); + + let our_path = self.arena.alloc_slice_copy(&path.0); + let unsafe_path = unsafe { UnsafePath::from_slice(&our_path) }; + + self.paths.push(unsafe_path.clone()); + let old = self.map.insert(unsafe_path, id); + + debug_assert!(old.is_none()); + debug_assert_eq!(self.lookup(id), path); + debug_assert_eq!(self.intern(path), id); + + id + } + + /// Lookup a previously interned path by id. + #[inline] + pub fn lookup<'a>(&'a self, id: PathId) -> Path<'a> { + let unsafe_path = self + .paths + .get(id.0 as usize) + .unwrap_or_else(|| Self::lookup_failure()); + unsafe { unsafe_path.as_path() } + } + + #[inline(never)] + fn lookup_failure() -> ! { + panic!( + "no path for the given id; this can only happen when mixing `PathId`s with different \ + `PathInterner`s" + ) + } +} + +impl Serialize for PathInterner { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.paths.len()))?; + for p in &self.paths { + let p = unsafe { p.as_path() }; + seq.serialize_element(&p)?; + } + seq.end() + } +} + +impl<'de> Deserialize<'de> for PathInterner { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_seq(PathInternerVisitor { + marker: PhantomData, + }) + } +} + +struct PathInternerVisitor { + marker: PhantomData PathInterner>, +} + +impl<'de> Visitor<'de> for PathInternerVisitor { + type Value = PathInterner; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a `peepmatic_runtime::paths::PathInterner`") + } + + fn visit_seq(self, mut access: M) -> Result + where + M: SeqAccess<'de>, + { + const DEFAULT_CAPACITY: usize = 16; + let capacity = access.size_hint().unwrap_or(DEFAULT_CAPACITY); + + let mut interner = PathInterner { + map: HashMap::with_capacity(capacity), + paths: Vec::with_capacity(capacity), + arena: bumpalo::Bump::new(), + }; + + while let Some(path) = access.next_element::()? { + interner.intern(path); + } + + Ok(interner) + } +} + +/// An unsafe, unchecked borrow of a path. Not for use outside of +/// `PathInterner`! +#[derive(Clone, Debug)] +struct UnsafePath { + ptr: *const u8, + len: usize, +} + +impl PartialEq for UnsafePath { + fn eq(&self, rhs: &UnsafePath) -> bool { + unsafe { self.as_slice() == rhs.as_slice() } + } +} + +impl Eq for UnsafePath {} + +impl Hash for UnsafePath { + fn hash(&self, hasher: &mut H) + where + H: Hasher, + { + unsafe { self.as_slice().hash(hasher) } + } +} + +/// Safety: callers must ensure that the constructed values won't have unsafe +/// usages of `PartialEq`, `Eq`, or `Hash`. +impl UnsafePath { + unsafe fn from_path(p: &Path) -> Self { + Self::from_slice(&p.0) + } + + unsafe fn from_slice(s: &[u8]) -> Self { + UnsafePath { + ptr: s.as_ptr(), + len: s.len(), + } + } +} + +/// Safety: callers must ensure that `'a` does not outlive the lifetime of the +/// underlying data. +impl UnsafePath { + unsafe fn as_slice<'a>(&self) -> &'a [u8] { + std::slice::from_raw_parts(self.ptr, self.len) + } + + unsafe fn as_path<'a>(&self) -> Path<'a> { + Path(self.as_slice()) + } +} diff --git a/cranelift/peepmatic/crates/runtime/src/type.rs b/cranelift/peepmatic/crates/runtime/src/type.rs new file mode 100644 index 0000000000..e4a08c9851 --- /dev/null +++ b/cranelift/peepmatic/crates/runtime/src/type.rs @@ -0,0 +1,262 @@ +//! Types. + +use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; +use std::fmt; + +/// A bit width of a type. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum BitWidth { + /// Polymorphic over bit width, with the same width as the root of the + /// optimization's LHS and RHS. + Polymorphic = 0, + + /// A fixed bit width of 1. + One = 1, + + /// A fixed bit width of 8. + Eight = 8, + + /// A fixed bit width of 16. + Sixteen = 16, + + /// A fixed bit width of 32. + ThirtyTwo = 32, + + /// A fixed bit width of 64. + SixtyFour = 64, + + /// A fixed bit width of 128. + OneTwentyEight = 128, +} + +/// The kind of type we are looking at: either an integer kind or boolean kind. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Kind { + /// Integer kind. + Int, + + /// Boolean kind. + Bool, + + /// CPU flags kind. + CpuFlags, + + /// Void kind. + Void, +} + +/// A type a value or the result of an operation. +#[allow(missing_docs)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Type { + /// This type's kind. + pub kind: Kind, + + /// This type's bit width. + pub bit_width: BitWidth, +} + +impl TryFrom for BitWidth { + type Error = &'static str; + + #[inline] + fn try_from(x: u8) -> Result { + Ok(match x { + 1 => Self::One, + 8 => Self::Eight, + 16 => Self::Sixteen, + 32 => Self::ThirtyTwo, + 64 => Self::SixtyFour, + 128 => Self::OneTwentyEight, + _ => return Err("not a valid bit width"), + }) + } +} + +impl fmt::Display for Type { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.kind { + Kind::CpuFlags => return write!(f, "cpu-flags"), + Kind::Void => return write!(f, "void"), + Kind::Int => write!(f, "i")?, + Kind::Bool => write!(f, "b")?, + } + match self.bit_width { + BitWidth::Polymorphic => write!(f, "NN"), + otherwise => write!(f, "{}", otherwise as u8), + } + } +} + +impl BitWidth { + /// Is this a polymorphic bit width? + pub fn is_polymorphic(&self) -> bool { + matches!(self, BitWidth::Polymorphic) + } + + /// Get this width in bits, unless this is a polymorphic bit width. + pub fn fixed_width(&self) -> Option { + if self.is_polymorphic() { + None + } else { + Some(*self as u8) + } + } +} + +macro_rules! type_ctors { + ( $( $( #[$attr:meta] )* $name:ident ( $kind:ident , $width:ident ) ; )* ) => { + $( + $( #[$attr] )* + pub const fn $name() -> Self { + Type { + kind: Kind::$kind, + bit_width: BitWidth::$width, + } + } + )* + } +} + +impl Type { + type_ctors! { + /// Get the `i1` type. + i1(Int, One); + /// Get the `i8` type. + i8(Int, Eight); + /// Get the `i16` type. + i16(Int, Sixteen); + /// Get the `i32` type. + i32(Int, ThirtyTwo); + /// Get the `i64` type. + i64(Int, SixtyFour); + /// Get the `i128` type. + i128(Int, OneTwentyEight); + /// Get the `b1` type. + b1(Bool, One); + /// Get the `b8` type. + b8(Bool, Eight); + /// Get the `b16` type. + b16(Bool, Sixteen); + /// Get the `b32` type. + b32(Bool, ThirtyTwo); + /// Get the `b64` type. + b64(Bool, SixtyFour); + /// Get the `b128` type. + b128(Bool, OneTwentyEight); + /// Get the CPU flags type. + cpu_flags(CpuFlags, One); + /// Get the void type. + void(Void, One); + } +} + +#[cfg(feature = "construct")] +mod tok { + use wast::custom_keyword; + custom_keyword!(b1); + custom_keyword!(b8); + custom_keyword!(b16); + custom_keyword!(b32); + custom_keyword!(b64); + custom_keyword!(b128); + custom_keyword!(i1); + custom_keyword!(i8); + custom_keyword!(i16); + custom_keyword!(i32); + custom_keyword!(i64); + custom_keyword!(i128); +} + +#[cfg(feature = "construct")] +impl<'a> wast::parser::Parse<'a> for Type { + fn parse(p: wast::parser::Parser<'a>) -> wast::parser::Result { + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Bool, + bit_width: BitWidth::One, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Bool, + bit_width: BitWidth::Eight, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Bool, + bit_width: BitWidth::Sixteen, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Bool, + bit_width: BitWidth::ThirtyTwo, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Bool, + bit_width: BitWidth::SixtyFour, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Bool, + bit_width: BitWidth::OneTwentyEight, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Int, + bit_width: BitWidth::One, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Int, + bit_width: BitWidth::Eight, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Int, + bit_width: BitWidth::Sixteen, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Int, + bit_width: BitWidth::ThirtyTwo, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Int, + bit_width: BitWidth::SixtyFour, + }); + } + if p.peek::() { + p.parse::()?; + return Ok(Type { + kind: Kind::Int, + bit_width: BitWidth::OneTwentyEight, + }); + } + Err(p.error("expected an ascribed type")) + } +} From de9fc63009fe39a45623fa767ad10d906a7495aa Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 1 May 2020 15:41:06 -0700 Subject: [PATCH 04/22] peepmatic: Introduce the main `peepmatic` crate Peepmatic is a DSL for peephole optimizations and compiler for generating peephole optimizers from them. The user writes a set of optimizations in the DSL, and then `peepmatic` compiles the set of optimizations into an efficient peephole optimizer: ``` DSL ----peepmatic----> Peephole Optimizer ``` The generated peephole optimizer has all of its optimizations' left-hand sides collapsed into a compact automata that makes matching candidate instruction sequences fast. The DSL's optimizations may be written by hand or discovered mechanically with a superoptimizer like [Souper][]. Eventually, `peepmatic` should have a verifier that ensures that the DSL's optimizations are sound, similar to what [Alive][] does for LLVM optimizations. [Souper]: https://github.com/google/souper [Alive]: https://github.com/AliveToolkit/alive2 --- cranelift/peepmatic/Cargo.toml | 15 + cranelift/peepmatic/README.md | 423 +++++ .../peepmatic/examples/mul-by-pow2.peepmatic | 3 + cranelift/peepmatic/examples/preopt.peepmatic | 193 +++ .../examples/redundant-bor.peepmatic | 13 + .../peepmatic/examples/redundant-bor.png | Bin 0 -> 225727 bytes cranelift/peepmatic/examples/simple.peepmatic | 11 + cranelift/peepmatic/src/ast.rs | 508 ++++++ cranelift/peepmatic/src/automatize.rs | 31 + cranelift/peepmatic/src/dot_fmt.rs | 142 ++ cranelift/peepmatic/src/lib.rs | 165 ++ cranelift/peepmatic/src/linear_passes.rs | 444 +++++ cranelift/peepmatic/src/linearize.rs | 831 ++++++++++ cranelift/peepmatic/src/parser.rs | 932 +++++++++++ cranelift/peepmatic/src/traversals.rs | 278 ++++ cranelift/peepmatic/src/verify.rs | 1433 +++++++++++++++++ 16 files changed, 5422 insertions(+) create mode 100644 cranelift/peepmatic/Cargo.toml create mode 100644 cranelift/peepmatic/README.md create mode 100644 cranelift/peepmatic/examples/mul-by-pow2.peepmatic create mode 100644 cranelift/peepmatic/examples/preopt.peepmatic create mode 100644 cranelift/peepmatic/examples/redundant-bor.peepmatic create mode 100644 cranelift/peepmatic/examples/redundant-bor.png create mode 100644 cranelift/peepmatic/examples/simple.peepmatic create mode 100644 cranelift/peepmatic/src/ast.rs create mode 100644 cranelift/peepmatic/src/automatize.rs create mode 100644 cranelift/peepmatic/src/dot_fmt.rs create mode 100755 cranelift/peepmatic/src/lib.rs create mode 100644 cranelift/peepmatic/src/linear_passes.rs create mode 100644 cranelift/peepmatic/src/linearize.rs create mode 100644 cranelift/peepmatic/src/parser.rs create mode 100644 cranelift/peepmatic/src/traversals.rs create mode 100644 cranelift/peepmatic/src/verify.rs diff --git a/cranelift/peepmatic/Cargo.toml b/cranelift/peepmatic/Cargo.toml new file mode 100644 index 0000000000..843b06a1a2 --- /dev/null +++ b/cranelift/peepmatic/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "peepmatic" +version = "0.1.0" +authors = ["Nick Fitzgerald "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.27" +peepmatic-automata = { version = "0.1.0", path = "crates/automata", features = ["dot"] } +peepmatic-macro = { version = "0.1.0", path = "crates/macro" } +peepmatic-runtime = { version = "0.1.0", path = "crates/runtime", features = ["construct"] } +wast = "13.0.0" +z3 = { version = "0.5.0", features = ["static-link-z3"] } diff --git a/cranelift/peepmatic/README.md b/cranelift/peepmatic/README.md new file mode 100644 index 0000000000..550f6d310d --- /dev/null +++ b/cranelift/peepmatic/README.md @@ -0,0 +1,423 @@ + + + + + + +- [About](#about) +- [Example](#example) +- [A DSL for Optimizations](#a-dsl-for-optimizations) + - [Variables](#variables) + - [Constants](#constants) + - [Nested Patterns](#nested-patterns) + - [Preconditions and Unquoting](#preconditions-and-unquoting) + - [Bit Widths](#bit-widths) +- [Implementation](#implementation) + - [Parsing](#parsing) + - [Type Checking](#type-checking) + - [Linearization](#linearization) + - [Automatization](#automatization) +- [References](#references) +- [Acknowledgments](#acknowledgments) + + + +## About + +Peepmatic is a DSL for peephole optimizations and compiler for generating +peephole optimizers from them. The user writes a set of optimizations in the +DSL, and then `peepmatic` compiles the set of optimizations into an efficient +peephole optimizer: + +``` +DSL ----peepmatic----> Peephole Optimizer +``` + +The generated peephole optimizer has all of its optimizations' left-hand sides +collapsed into a compact automata that makes matching candidate instruction +sequences fast. + +The DSL's optimizations may be written by hand or discovered mechanically with a +superoptimizer like [Souper][]. Eventually, `peepmatic` should have a verifier +that ensures that the DSL's optimizations are sound, similar to what [Alive][] +does for LLVM optimizations. + +Currently, `peepmatic` is targeting peephole optimizers that operate on +Cranelift's clif intermediate representation. The intended next target is +Cranelift's new backend's "vcode" intermediate representation. Supporting +non-Cranelift targets is not a goal. + +[Cranelift]: https://github.com/bytecodealliance/wasmtime/tree/master/cranelift#readme +[Souper]: https://github.com/google/souper +[Alive]: https://github.com/AliveToolkit/alive2 + +## Example + +This snippet of our DSL describes optimizations for removing redundant +bitwise-or instructions that are no-ops: + +```lisp +(=> (bor $x (bor $x $y)) + (bor $x $y)) + +(=> (bor $y (bor $x $y)) + (bor $x $y)) + +(=> (bor (bor $x $y) $x) + (bor $x $y)) + +(=> (bor (bor $x $y) $y) + (bor $x $y)) +``` + +When compiled into a peephole optimizer automaton, they look like this: + +![](examples/redundant-bor.png) + +## A DSL for Optimizations + +A single peephole optimization has two parts: + +1. A **left-hand side** that describes candidate instruction sequences that the + optimization applies to. +2. A **right-hand side** that contains the new instruction sequence that + replaces old instruction sequences that the left-hand side matched. + +A left-hand side may bind sub-expressions to variables and the right-hand side +may contain those bound variables to reuse the sub-expressions. The operations +inside the left-hand and right-hand sides are a subset of clif operations. + +Let's take a look at an example: + +```lisp +(=> (imul $x 2) + (ishl $x 1)) +``` + +As you can see, the DSL uses S-expressions. (S-expressions are easy to parse and +we also have a bunch of nice parsing infrastructure for S-expressions already +for our [`wat`][wat] and [`wast`][wast] crates.) + +[wat]: https://crates.io/crates/wat +[wast]: https://crates.io/crates/wast + +The left-hand side of this optimization is `(imul $x 2)`. It matches integer +multiplication operations where a value is multiplied by the constant two. The +value multiplied by two is bound to the variable `$x`. + +The right-hand side of this optimization is `(ishl $x 1)`. It reuses the `$x` +variable that was bound in the left-hand side. + +This optimization replaces expressions of the form `x * 2` with `x << 1`. This +is sound because multiplication by two is the same as shifting left by one for +binary integers, and it is desirable because a shift-left instruction executes +in fewer cycles than a multiplication. + +The general form of an optimization is: + +```lisp +(=> ) +``` + +### Variables + +Variables begin with a dollar sign and are followed by lowercase letters, +numbers, hyphens, and underscores: `$x`, `$y`, `$my-var`, `$operand2`. + +Left-hand side patterns may contain variables that match any kind of +sub-expression and give it a name so that it may be reused in the right-hand +side. + +```lisp +;; Replace `x + 0` with simply `x`. +(=> (iadd $x 0) + $x) +``` + +Within a pattern, every occurrence of a variable with the same name must match +the same value. That is `(iadd $x $x)` matches `(iadd 1 1)` but does not match +`(iadd 1 2)`. This lets us write optimizations such as this: + +```lisp +;; Xor'ing a value with itself is always zero. +(=> (bxor $x $x) + (iconst 0)) +``` + +### Constants + +We've already seen specific integer literals and wildcard variables in patterns, +but we can also match any constant. These are written similar to variables, but +use uppercase letters rather than lowercase: `$C`, `$MY-CONST`, and `$OPERAND2`. + +For example, we can use constant patterns to combine an `iconst` and `iadd` into +a single `iadd_imm` instruction: + +```lisp +(=> (iadd (iconst $C) $x) + (iadd_imm $C $x)) +``` + +### Nested Patterns + +Patterns can also match nested operations with their own nesting: + +```lisp +(=> (bor $x (bor $x $y)) + (bor $x $y)) +``` + +### Preconditions and Unquoting + +Let's reconsider our first example optimization: + +```lisp +(=> (imul $x 2) + (ishl $x 1)) +``` + +This optimization is a little too specific. Here is another version of this +optimization that we'd like to support: + +```lisp +(=> (imul $x 4) + (ishl $x 2)) +``` + +We don't want to have to write out all instances of this general class of +optimizations! That would be a lot of repetition and could also bloat the size +of our generated peephole optimizer's matching automata. + +Instead, we can generalize this optimization by matching any multiplication by a +power of two constant `C` and replacing it with a shift left of `log2(C)`. + +First, rather than match `2` directly, we want to match any constant variable `C`: + +```lisp +(imul $x $C) +``` + +Note that variables begin with lowercase letters, while constants begin with +uppercase letters. Both the constant pattern `$C` and variable pattern `$x` will +match `5`, but only the variable pattern `$x` will match a whole sub-expression +like `(iadd 1 2)`. The constant pattern `$C` only matches constant values. + +Next, we augment our left-hand side's pattern with a **precondition** that the +constant `$C` must be a power of two. Preconditions are introduced by wrapping +a pattern in the `when` form: + +```lisp +;; Our new left-hand side, augmenting a pattern with a precondition! +(when + ;; The pattern matching multiplication by a constant value. + (imul $x $C) + + ;; The precondition that $C must be a power of two. + (is-power-of-two $C)) + ``` + +In the right-hand side, we use **unquoting** to perform compile-time evaluation +of `log2($C)`. Unquoting is done with the `$(...)` form: + +```lisp +;; Our new right-hand side, using unqouting to do compile-time evaluation of +;; constants that were matched and bound in the left-hand side! +(ishl $x $(log2 $C)) +``` + +Finally, here is the general optimization putting our new left-hand and +right-hand sides together: + +```lisp +(=> (when (imul $x $C) + (is-power-of-two $C)) + (ishl $x $(log2 $C))) +``` + +### Bit Widths + +Similar to how Cranelift's instructions are bit-width polymorphic, `peepmatic` +optimizations are also bit-width polymorphic. Unless otherwise specified, a +pattern will match expressions manipulating `i32`s just the same as expressions +manipulating `i64`s, etc... An optimization that doesn't constrain its pattern's +bit widths must be valid for all bit widths: + +* 1 +* 8 +* 16 +* 32 +* 64 +* 128 + +To constrain an optimization to only match `i32`s, for example, you can use the +`bit-width` precondition: + +```lisp +(=> (when (iadd $x $y) + (bit-width $x 32) + (bit-width $y 32)) + ...) +``` + +Alternatively, you can ascribe a type to an operation by putting the type inside +curly brackets after the operator, like this: + +```lisp +(=> (when (sextend{i64} (ireduce{i32} $x)) + (bit-width $x 64)) + (sshr (ishl $x 32) 32)) +``` + +## Implementation + +Peepmatic has roughly four phases: + +1. Parsing +2. Type Checking +3. Linearization +4. Automatization + +(I say "roughly" because there are a couple micro-passes that happen after +linearization and before automatization. But those are the four main phases.) + +### Parsing + +Parsing transforms the DSL source text into an abstract syntax tree (AST). + +We use [the `wast` crate][wast]. It gives us nicely formatted errors with source +context, as well as some other generally nice-to-have parsing infrastructure. + +Relevant source files: + +* `src/parser.rs` +* `src/ast.rs` + +[wast]: https://crates.io/crates/wast + +### Type Checking + +Type checking operates on the AST. It checks that types and bit widths in the +optimizations are all valid. For example, it ensures that the type and bit width +of an optimization's left-hand side is the same as its right-hand side, because +it doesn't make sense to replace an integer expression with a boolean +expression. + +After type checking is complete, certain AST nodes are assigned a type and bit +width, that are later used in linearization and when matching and applying +optimizations. + +We walk the AST and gather type constraints. Every constraint is associated with +a span in the source file. We hand these constraints off to Z3. In the case that +there are type errors (i.e. Z3 returns `unsat`), we get the constraints that are +in conflict with each othe via `z3::Solver::get_unsat_core` and report the type +errors to the user, with the source context, thanks to the constraints' +associated spans. + +Using Z3 not only makes implementing type checking easier than it otherwise +would be, but makes it that much easier to extend type checking with searching +for counterexample inputs in the future. That is, inputs for which the RHS is +not equivalent to the LHS, implying that the optimization is unsound. + +Relevant source files: + +* `src/verify.rs` + +### Linearization + +Linearization takes the AST of optimizations and converts each optimization into +a linear form. The goal is to make automaton construction easier in the +automatization step, as well as simplifying the language to make matching and +applying optimizations easier. + +Each optimization's left-hand side is converted into a sequence of + +* match operation, +* path to the instruction/value/immediate to which the operation is applied, and +* expected result of the operation. + +All match operations must have the expected result for the optimization to be +applicable to an instruction sequence. + +Each optimization's right-hand side is converted into a sequence of build +actions. These are commands that describe how to construct the right-hand side, +given that the left-hand side has been matched. + +Relevant source files: + +* `src/linearize.rs` +* `src/linear_passes.rs` +* `crates/runtime/src/linear.rs` + +### Automatization + +Automatization takes a set of linear optimizations and combines them into a +transducer automaton. This automaton is the final, compiled peephole +optimizations. The goal is to de-duplicate as much as we can from all the linear +optimizations, producing as compact and cache-friendly a representation as we +can. + +Plain automata can tell you whether it matches an input string. It can be +thought of as a compact representation of a set of strings. A transducer is a +type of automaton that doesn't just match input strings, but can map them to +output values. It can be thought of as a compact representation of a dictionary +or map. By using transducers, we de-duplicate not only the prefix and suffix of +the match operations, but also the right-hand side build actions. + +Each state in the emitted transducers is associated with a match operation and +path. The transitions out of that state are over the result of the match +operation. Each transition optionally accumulates some RHS build actions. By the +time we reach a final state, the RHS build actions are complete and can be +interpreted to apply the matched optimization. + +The relevant source files for constructing the transducer automaton are: + +* `src/automatize.rs` +* `crates/automata/src/lib.rs` + +The relevant source files for the runtime that interprets the transducers and +applies optimizations are: + +* `crates/runtime/src/optimizations.rs` +* `crates/runtime/src/optimizer.rs` + +## References + +I found these resources helpful when designing `peepmatic`: + +* [Extending tree pattern matching for application to peephole + optimizations](https://pure.tue.nl/ws/portalfiles/portal/125543109/Thesis_JanekvOirschot.pdf) + by van Oirschot + +* [Interpreted Pattern Match Execution for + MLIR](https://drive.google.com/drive/folders/1hb_sXbdMbIz95X-aaa6Vf5wSYRwsJuve) + by Jeff Niu + +* [Direct Construction of Minimal Acyclic Subsequential + Transducers](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.24.3698&rep=rep1&type=pdf) + by Mihov and Maurel + +* [Index 1,600,000,000 Keys with Automata and + Rust](https://blog.burntsushi.net/transducers/) and [the `fst` + crate](https://crates.io/crates/fst) by Andrew Gallant + +## Acknowledgments + +Thanks to [Jubi Taneja], [Dan Gohman], [John Regehr], and [Nuno Lopes] for their +input in design discussions and for sharing helpful resources! + +[Jubi Taneja]: https://www.cs.utah.edu/~jubi/ +[Dan Gohman]: https://github.com/sunfishcode +[John Regehr]: https://www.cs.utah.edu/~regehr/ +[Nuno Lopes]: http://web.ist.utl.pt/nuno.lopes/ diff --git a/cranelift/peepmatic/examples/mul-by-pow2.peepmatic b/cranelift/peepmatic/examples/mul-by-pow2.peepmatic new file mode 100644 index 0000000000..cca78a372f --- /dev/null +++ b/cranelift/peepmatic/examples/mul-by-pow2.peepmatic @@ -0,0 +1,3 @@ +(=> (when (imul $x $C) + (is-power-of-two $C)) + (ishl $x $(log2 $C))) diff --git a/cranelift/peepmatic/examples/preopt.peepmatic b/cranelift/peepmatic/examples/preopt.peepmatic new file mode 100644 index 0000000000..22523f0a01 --- /dev/null +++ b/cranelift/peepmatic/examples/preopt.peepmatic @@ -0,0 +1,193 @@ +;; Apply basic simplifications. +;; +;; This folds constants with arithmetic to form `_imm` instructions, and other +;; minor simplifications. +;; +;; Doesn't apply some simplifications if the native word width (in bytes) is +;; smaller than the controlling type's width of the instruction. This would +;; result in an illegal instruction that would likely be expanded back into an +;; instruction on smaller types with the same initial opcode, creating +;; unnecessary churn. + +;; Binary instructions whose second argument is constant. +(=> (when (iadd $x $C) + (fits-in-native-word $C)) + (iadd_imm $C $x)) +(=> (when (imul $x $C) + (fits-in-native-word $C)) + (imul_imm $C $x)) +(=> (when (sdiv $x $C) + (fits-in-native-word $C)) + (sdiv_imm $C $x)) +(=> (when (udiv $x $C) + (fits-in-native-word $C)) + (udiv_imm $C $x)) +(=> (when (srem $x $C) + (fits-in-native-word $C)) + (srem_imm $C $x)) +(=> (when (urem $x $C) + (fits-in-native-word $C)) + (urem_imm $C $x)) +(=> (when (band $x $C) + (fits-in-native-word $C)) + (band_imm $C $x)) +(=> (when (bor $x $C) + (fits-in-native-word $C)) + (bor_imm $C $x)) +(=> (when (bxor $x $C) + (fits-in-native-word $C)) + (bxor_imm $C $x)) +(=> (when (rotl $x $C) + (fits-in-native-word $C)) + (rotl_imm $C $x)) +(=> (when (rotr $x $C) + (fits-in-native-word $C)) + (rotr_imm $C $x)) +(=> (when (ishl $x $C) + (fits-in-native-word $C)) + (ishl_imm $C $x)) +(=> (when (ushr $x $C) + (fits-in-native-word $C)) + (ushr_imm $C $x)) +(=> (when (sshr $x $C) + (fits-in-native-word $C)) + (sshr_imm $C $x)) +(=> (when (isub $x $C) + (fits-in-native-word $C)) + (iadd_imm $(neg $C) $x)) +(=> (when (ifcmp $x $C) + (fits-in-native-word $C)) + (ifcmp_imm $C $x)) +(=> (when (icmp $cond $x $C) + (fits-in-native-word $C)) + (icmp_imm $cond $C $x)) + +;; Binary instructions whose first operand is constant. +(=> (when (iadd $C $x) + (fits-in-native-word $C)) + (iadd_imm $C $x)) +(=> (when (imul $C $x) + (fits-in-native-word $C)) + (imul_imm $C $x)) +(=> (when (band $C $x) + (fits-in-native-word $C)) + (band_imm $C $x)) +(=> (when (bor $C $x) + (fits-in-native-word $C)) + (bor_imm $C $x)) +(=> (when (bxor $C $x) + (fits-in-native-word $C)) + (bxor_imm $C $x)) +(=> (when (isub $C $x) + (fits-in-native-word $C)) + (irsub_imm $C $x)) + +;; Unary instructions whose operand is constant. +(=> (adjust_sp_down $C) (adjust_sp_down_imm $C)) + +;; Fold `(binop_imm $C1 (binop_imm $C2 $x))` into `(binop_imm $(binop $C2 $C1) $x)`. +(=> (iadd_imm $C1 (iadd_imm $C2 $x)) (iadd_imm $(iadd $C1 $C2) $x)) +(=> (imul_imm $C1 (imul_imm $C2 $x)) (imul_imm $(imul $C1 $C2) $x)) +(=> (bor_imm $C1 (bor_imm $C2 $x)) (bor_imm $(bor $C1 $C2) $x)) +(=> (band_imm $C1 (band_imm $C2 $x)) (band_imm $(band $C1 $C2) $x)) +(=> (bxor_imm $C1 (bxor_imm $C2 $x)) (bxor_imm $(bxor $C1 $C2) $x)) + +;; Remove operations that are no-ops. +(=> (iadd_imm 0 $x) $x) +(=> (imul_imm 1 $x) $x) +(=> (sdiv_imm 1 $x) $x) +(=> (udiv_imm 1 $x) $x) +(=> (bor_imm 0 $x) $x) +(=> (band_imm -1 $x) $x) +(=> (bxor_imm 0 $x) $x) +(=> (rotl_imm 0 $x) $x) +(=> (rotr_imm 0 $x) $x) +(=> (ishl_imm 0 $x) $x) +(=> (ushr_imm 0 $x) $x) +(=> (sshr_imm 0 $x) $x) + +;; Replace with zero. +(=> (imul_imm 0 $x) 0) +(=> (band_imm 0 $x) 0) + +;; Replace with negative 1. +(=> (bor_imm -1 $x) -1) + +;; Transform `[(x << N) >> N]` into a (un)signed-extending move. +;; +;; i16 -> i8 -> i16 +(=> (when (ushr_imm 8 (ishl_imm 8 $x)) + (bit-width $x 16)) + (uextend{i16} (ireduce{i8} $x))) +(=> (when (sshr_imm 8 (ishl_imm 8 $x)) + (bit-width $x 16)) + (sextend{i16} (ireduce{i8} $x))) +;; i32 -> i8 -> i32 +(=> (when (ushr_imm 24 (ishl_imm 24 $x)) + (bit-width $x 32)) + (uextend{i32} (ireduce{i8} $x))) +(=> (when (sshr_imm 24 (ishl_imm 24 $x)) + (bit-width $x 32)) + (sextend{i32} (ireduce{i8} $x))) +;; i32 -> i16 -> i32 +(=> (when (ushr_imm 16 (ishl_imm 16 $x)) + (bit-width $x 32)) + (uextend{i32} (ireduce{i16} $x))) +(=> (when (sshr_imm 16 (ishl_imm 16 $x)) + (bit-width $x 32)) + (sextend{i32} (ireduce{i16} $x))) +;; i64 -> i8 -> i64 +(=> (when (ushr_imm 56 (ishl_imm 56 $x)) + (bit-width $x 64)) + (uextend{i64} (ireduce{i8} $x))) +(=> (when (sshr_imm 56 (ishl_imm 56 $x)) + (bit-width $x 64)) + (sextend{i64} (ireduce{i8} $x))) +;; i64 -> i16 -> i64 +(=> (when (ushr_imm 48 (ishl_imm 48 $x)) + (bit-width $x 64)) + (uextend{i64} (ireduce{i16} $x))) +(=> (when (sshr_imm 48 (ishl_imm 48 $x)) + (bit-width $x 64)) + (sextend{i64} (ireduce{i16} $x))) +;; i64 -> i32 -> i64 +(=> (when (ushr_imm 32 (ishl_imm 32 $x)) + (bit-width $x 64)) + (uextend{i64} (ireduce{i32} $x))) +(=> (when (sshr_imm 32 (ishl_imm 32 $x)) + (bit-width $x 64)) + (sextend{i64} (ireduce{i32} $x))) + +;; Fold away redundant `bint` instructions that accept both integer and boolean +;; arguments. +(=> (select (bint $x) $y $z) (select $x $y $z)) +(=> (brz (bint $x)) (brz $x)) +(=> (brnz (bint $x)) (brnz $x)) +(=> (trapz (bint $x)) (trapz $x)) +(=> (trapnz (bint $x)) (trapnz $x)) + +;; Fold comparisons into branch operations when possible. +;; +;; This matches against operations which compare against zero, then use the +;; result in a `brz` or `brnz` branch. It folds those two operations into a +;; single `brz` or `brnz`. +(=> (brnz (icmp_imm ne 0 $x)) (brnz $x)) +(=> (brz (icmp_imm ne 0 $x)) (brz $x)) +(=> (brnz (icmp_imm eq 0 $x)) (brz $x)) +(=> (brz (icmp_imm eq 0 $x)) (brnz $x)) + +;; Division and remainder by constants. +;; +;; Note that this section is incomplete, and a bunch of related optimizations +;; are still hand-coded in `simple_preopt.rs`. + +;; (Division by one is handled above.) + +;; Remainder by one is zero. +(=> (urem_imm 1 $x) 0) +(=> (srem_imm 1 $x) 0) + +;; Division by a power of two -> shift right. +(=> (when (udiv_imm $C $x) + (is-power-of-two $C)) + (ushr_imm $(log2 $C) $x)) diff --git a/cranelift/peepmatic/examples/redundant-bor.peepmatic b/cranelift/peepmatic/examples/redundant-bor.peepmatic new file mode 100644 index 0000000000..d8d6f4a144 --- /dev/null +++ b/cranelift/peepmatic/examples/redundant-bor.peepmatic @@ -0,0 +1,13 @@ +;; Remove redundant bitwise OR instructions that are no-ops. + +(=> (bor $x (bor $x $y)) + (bor $x $y)) + +(=> (bor $y (bor $x $y)) + (bor $x $y)) + +(=> (bor (bor $x $y) $x) + (bor $x $y)) + +(=> (bor (bor $x $y) $y) + (bor $x $y)) diff --git a/cranelift/peepmatic/examples/redundant-bor.png b/cranelift/peepmatic/examples/redundant-bor.png new file mode 100644 index 0000000000000000000000000000000000000000..dab873e6cfec627af608c2d74acd0af76d80fde0 GIT binary patch literal 225727 zcmdqKcU+GB|37?Yb;-!ejEra@3GHE~w5O7CQIb_E+MOB6tTaTLXsb{pO+t%GL!yP! zFiKPVe!k9&>ifGNzx#gN_kZ`@=ks`6SC=}E<9NSc<2hdM!$oyfg=riMIT#GaG{v3U zG#QL3XBdo$p8rh3e^EFWn}vT&+P_O-8)KOM7n&aF!(c3BC~n)VZ5Q~x+T2&CD`;(9 z^87Pb&bYo-6<-&#({kJ~jSJ(mHE%t?Fm?J&i};H{m3Mur3ZGVY|FAJwQE-ko>LT0b zi7)v>SGkVQoiX`Z)P>MlUp2h!y%zp-H_~=LH0x|uSe^a558t#0zkN@yb>0*^ou{TF z_I5kIa5I0u)b+1KYc@^z`z`)C;5_FU+vxX#3$|O1e&?#RVzgA(*4o_5E{xHMZNF?hu)Jcg?cTL%ZO^yl zurb7M&++s3_n#*@vV7V*_=0RSudm%3uvqHU-GgreCg$DW|MF7O(aMKQ8;8WN3mXT| zRux}%>_bxCnTb<47PB+zc8a{(HnPC))f@|I-2_hy>p$cjKWQp|Zi-yIZkz#^Wkjm;begClG7W5nfyUHY1f>DUL~MHXoy zR~K$n%JEn%YWuT_H}898V#lKd=QVsGPFC;Y_ug_D>Tj=7n1{{g7nt{<{T> z;yW4xb&Te!6zf-47Wi-p>6-j*`_!8`+%tUe{r!LKzg!iw>i>}(dGFrpMVpT1wtdP@ zG|BU>I$5!|H>x0MByWRy_hX9 zA*R7I^U?^FxoZud6dB-O67k*l508WttzIoKGkJS^du4{RR)l z3>iGS#J>KTr*wBn0v8uoxXP_f`uF!elTA_DwM*;K;qtSms+kFyzOxywgPX=|s<6loU%K;ef|B-@EiT#wGh1C-rk&x zW0R6P_wAU-s9P8M&-A~G&~J_?=$BrV7r!@EwlfXuxhl_H=_AwNeeilyB}v<#fgg{j zu2C$BP`SmMmpS+`vmwZ~Chz{g*$*9ie)8reFbBW+`uT;Q{Ps>i-Qgr++UP`}qR*;u zgpmEul*Z{VR=Vt$vi+%iJ8(L~GF0Hq-=NK^ZEo;4E=%^R%iC$VzqZJ}F~}dm$(LN~;^@%T;@AD-qgiq2=E@{OLc!2*z&@$+6?Cq1qS1{)zu)U$chu601 zgu=n1;JtO)`0X9r8}Q{eeN5ra${8C(495SDOD`7g|81M8o4q^v_}+})FNMDb9En#v zapFW@*EfGu5N1C9Wy<0o_kVSNv}pX9+Pc=}O17rCxp_wK8;gSt`F?I5b2=_P2 z@5$&%uR4*l*u66?q%7|)FbBTyaDI?rb6w%NT>Nize0M+g@v&)^ zD_b+_TgFgVLS$4FzoI5;tyA+Z<9wt@Ffs{gu!dQ2!XJ6THhvQKm0b3&_dKT0&P-e& zZD)z6$(A`<@BHym>|?Tp9Dn0PhKKZMK;GvW>W~{2jn$3I zugDzg@*L=GlML6>w$SK*|+fSO#edE^g zrvf>PZL5|^G@e$Ea)M5kFMV(z=L(P9(Co!hibi&y+>%g{KRZnKjSO*-%p4wUwSLK^_dOAPE;$_Q zPAPvp?boXfhs*fUe_x~b2n8o&C96;)dfz2y_$9b>#ik6j7j%>-xNN+&=~!QE3`-k<|FwA>^^hC-jfQCYIT3DG?@qr-|rj>}Ks zz8_+NsaGx#x7-(@77~1VJp0$Qj=}*483V)YWRD}svcJj1_uQkrgGwi@iJXhc*Q`|$Q)QCt=~ zCS*AGA4EK}C!ysf9(ky$vIFVJhz@h8hv0ygBO z*>`vk^lV&fki?%{eQM837c6$6ZAM4XYNIsq;i1;yp=*u(Px=>oVCPn;1n??i6^0)4 zurWN0#)dsrY($+Yx*|_+2U;S<)?!JU-!B?tG#fIH$f9eP6bFekK2cP1>!oc2Z zjUQ_VW#i8}b9&!J?kcwHMZO--_#Qr{=YI0uwN}x4_3rz1E0{z1LJ1~C!7|Ms9(~&^ z!K5aCcfV%WrwH6kzw*}MC*5=7k@ydb`fAK8bCx)MXRB`Y5ijnkyQ2i?&b&CbmQXh~ zFS)C@#HsBW@)|g*^qzhq1w7Xg$IBT!PHmgL@1~^Er9 zP`anen!UXtPi`Pv&g9#>xXQ=JKFWSeJ@uon*c$f{z$+jikSCV$>nfndGDM#-DvZ{z zK%U{>Iaur?8EtODPW8RAKV96v$h`lCJKW`M^X=V_6$vvLb!uZ;waZh~u8x-2goB@+ zyERuN829zGh5((cZO!OEUSs#=s!eN+b4Vle{}KM+k*viPZ%tdY$pqz%uoE>L?O5F( zvlplCxVkV}FTv<=S=0%6<6FT|Cxka4gvczo18%VP?IyZ2IlK|#Tk#pC|20C1M)u`3trM!#qHT^UU{ z219=3ozb*rFx2z-|93DDAMt~BiHeeMtv#tSS%5hqI6$@CmBV-5$tod?0QHlK3_$HP(cQJ9HCLiu9QD47t&2v#IQ;J3%>y11 zs|YPuKRv019MO)A*fO3W-|&U`;P17+A80HLB5=Lw=zA~9@s~daduof`#qAM#93*9X z@Z;lSl@CpKf#t|`_Yx-pen!`&hJeswp3Hps@F8xrck4H)Q{QucR;S{}j+8}RL&im- z97bzJsvfS0529~@RJguPTe#u(<&VHj{%X!GT0H$u42I>>{}pbtSDc@==piEDo?*)R zBk%6z0nR0!`tjkOai%P4i&2eZYx1pqpNLd{ z_mAGXvz&sfuA@Ur(1U?kB-(d;rJq6;6k)5{hx%LT2f4y~(myKry*`4cq*z7=Q~dJk zYUz^`U#PSaP`I<>D(b4w2Gr`$5)4M$jDo|v()TMxjN1P|P$p6{SPq?+(u{s-y)pW2uMDjEM8m^5u+ zy|0#PUUXz|0Ft1!<2{~boY7=l+;yZBWQD0483hAenxeevKe7CG?&{rMZd2w_wbRdV zviH8LhTvZ^FTZL>HI^YCH(S=SlR|`)DB&#~HKvRp$E%^$47Atp*75{qZrGPjR8x z@jw(A-GN0%E8_PSihsF6s~!zzayo&^qT&c5+{Z-htcDJh!0(*->(wHtAe&Epe}4^7 zDbT4s&m8O*$?jjOCYOHO?#M$i_YXA}I~^n6W}>lh87Jr`u2R%QNGBbsSPL+=6B%Sp zyke{SbXu^}rQ;cGGe^t(&~+Kd?)o=kwXcbszrH1w@c9I?OVJy_7qK_>6lqp34`8WK zXM)D@xn~{&(x4rwaq4)(T9_Y?D%xKyvM@mzRP@H~-UX}c_szy4-2m2poRtUMcz6^y zG7qC3+IN-fb)?n~#^3H7#P8OV!h$ke61LSn(7C%(;@iDMtL{p(NPJYZai&Yyfo!&9 z+*aC9-!~Gs1eeqTCZH;&?EmbH`x1PdSR^Zj922%V%5c5oHWEjcoq|F_1yBRqfeN*c zCS(pdAWeNJ1_P}Nqxqdea5chCF@Oh6@72|*KpaU(zpiKOybmZ6+Wv2N5oDr#2v8nr?Ti4u6~gLyN;{OPG50fj+Q zkyz3E*ftTA{FMpm-3DI^gKmk{I*F3%*Xq)f3JFW9tEGGw+H@&C>QWq9u$#wJhI-{F zeo~0QgX$^w28)`h8re7a0oO@_pQ&JvN|9}WkZUAb(>k!0>^>Qv7$3onj2wSFdVN!r z&-&}@Jh}{F5CI{fT?iA&4SI(tkbp%4-Y3+O+SV! znIeqzlMZPh5K55Nfy|*^)^EMtK996JH^3it{*j>1`ecUPxw-L(F7ZiH*k}pSZ~c&E zUwMW;@dzq~60HZ`#Er67qTdRwI$s1jL|cZXEk%(uw0XOoVrgy0@6QCmLHq@53>mIl zH;>c~ei9tN(kd9a^fnNkLTneX^Ei1?k%qS0{PvCl#Uk46JV`;z`TT(Nl=zow7a+9X zP}E`<>2V0sV;Qf5+&0`e;K91m+0iSBchc?q8@otoMJVUh=lO`Dlk+NqEeNi1F?Ra( z(j4sIw9h!d%}|A?h$@H3rtc@>M;HgX_PXj+TeY5_)f9Aq>2bv;*cj6%j09a>9R(fT z8u1*uMrU(rB1$*}=~XCMuTA~;UcbEdDL?cR!>~K!6fQubl%l64&3vAb-7hl+6);a zTE4G<)_U-+fuWwvB9R3Qlg@fXjatO^mYa92iA*YF&AT>@aR5#LDcLGw*PulIbef; z51(4$*EhGUpHJqyOZN`wyf*dwLxNAmB5bnP0BFBLV~giMUxZ?e{4T(Q82{@3iDXov zbjOE_JB-#cral0Jk#ko$efVrX)z?&=oLaO(@>lvBdIM*NWBnz*ToL71Ji9K`04NK4 zWbkX=3^IGrxYtr7_~{wl6pDYc9KcY(MwwC9bbEwDd_RKrM+^rthA*&>Dei%~AgYNU z5wM2^Yj*)LO<6oUbYG3bw>atk?}_$kJXC$C7Gpi~x%3RL2_>Yhr|N|W(rvoI$hxL7 zl5#sqU9TPZDn^$HMmzcb{=o|To)RR060i;`cH6ga$F5nvC!Olx>l@buv?9p+Dnw(_ z#4dpE%HV=|^dKxlY8At>egwg~-qAn`Vb%R7&t7Hn`NhTF!B%DnJfg1@bs@6ObQNIC zH0*$25Wz=;8W+z!(DyyaVZC3eUC*ZoD??7gA~K=|MrvT)p$8~-#QJZFpV;p*_|4c1 zonPWdQYn36XG^)zJY%O5tj>4?Toj^TJ>vcKi;J^G0kp|w0G&|*{wJBL^LwS}Nl0DT zfKV&VzsjZS3MyqC-xG@tRy{cZZP>-9q@;vZUP>d?96Fn>p|;JFT%M(I0GOAHdGQmj zpzIxJTVvmhABfNBYY9B@YJuCde;C@UM#`z>U7%815|O2WxzGajh@TmrEZ0eaPHk1L zqZf|Nb@TYgvTP4Rh84Vl;v$*2J~<#%JJTI|3d{A3gVyeSN>Gw=iaTR;Wp=K^da^#e zJ&&&gvSB3IcOWM5xB@+D4!5;(*otv&^Pmi|Q~Ow@8hC?*)64UORjI9lI~gLbn;|tq zcF>@p`Lul`Xaw(o)ur{d#NJ%@@0KOboo@+N0-EMi2;f&5FA2r(1Q0-Ax(=>m4h<;d z{%5aJ&P5m3!p9pz+Yng0Rux!K3(CzFg$WD}wb4(%j75GOuu%xxSpdoujHqp>_fb({>ybQGyU>yJ^=@k}e_JE zM^(^f8>6mxG}g1cT)q^%IOtl4VP*CK-* z1Q&@Ne;p||*nCOzw?Jf|X43^m*0b<`Hm^KA@OKvAuSe&87B|?C(imI-Y%BWj^K&E! z_#0X&k`4vHCz->;bi(FZ-uM7zfSLmSgLGzJdBQ_P0X->KcpDz+5iQL4(PifdPfFTA zl^!&*{rv$?Q)p-@b!Vo~+Ra=rh@db(C#*d#|&m)uqf&;OWvE~Qhl-_97qD| z&-gAkQpLsJLj+upCPA)?r%d;5khDD-FBWY$qz(qP7mLOHX{a;3=~oU<5o(-Z&1G5( zba*b@O(+IC(eYgK$RJ(%)$pNCnHDg?T(BPNe1XWjF;l(7lio={|k272P&-ct`?{M*aYG6GZprBR zSR&F23n73}xXT8Nv{)xii}WsC%AzYvWF!H7Rmtc8T;SV!erldXW1v=(9i#2Rn-DoU zEPJe-+7Wa;dOi~N%e2(i*hn+#n4>E`v>WQ-_FhLISZe*U{gp)r>#|N0DdAYm$&kI4 z&TQBZY_&k@)KQZE3CzF{*f~5jSc%^G5Zt-8 zLgC=R?_bbo6`_1<{g+$p^8|wFKAwB@h{AE-2L0)v)&#a5|@uhk~f~1b#UO%YOjWMzDN}#`#EFQb*4@8TkeE|8)plT_4$HqSu9M3*o8J)I0 zTFp5oYcgAC$w*4pAx|%|HTEUd>ct~LwIw&#;`*oS^}jmr|j>2 z1njAP(GFe}tGNM~H3zZ5T{se?;_CrOO~1arxd699!zw^*cLD0iX{GiKr!}4GXy$&T z+X4wIlO-;{Khb&ujusV%ZB6$apQSJoLZMcBAU)8W2t6=4z(kX&9Ew$0ZvFGLS+(Yn zSg5n^@Zi2leTOJDqW5^}kH=KL-8|SC?xRmD7=l{b2tA4l_j-UBk`=K)1yDr-6AL9{ z5pk~LP2g(V|Ni9Oku{#P9v(DN9je7u&_T=rZo$uAp0w(1gu!wJRbUR%-eVz$~{ z%;FHngPm*zz!Cxdn39uw8z{}23o}===dSQH0UhdnZ6@nOF@ca*Wg!4g#`IBDZn}j< z=HRIvo=bvgSO{u`3e*(QL&c#}xy9GPmRL=-Y{o~xzE#RMykI1lw3J1oow9lx+09sv zQ@GWvun~|9ate@aKkzH31ClJkch$>t*Q`efRY7Et=ql5ig6I%eCUY%7e0 z+ezCja{4N|K+@(A1{Mw|i8K60OT9bIo`~YoG%=M2WrY=-izPQua6+Wti&V^-8G}eD zk{fo~ab^CsUte+}QI;00H`#f9`r-g6nc=w0eWT6D(volvfiuJiD>k7mQ?}8JL7+oR zG{rKvscyXCO?Ms_@oaa_qDx#tR?Xr5Up=IL?rg0|XK_;WzY`;|X=ypoot*iGW&@=A zn(kC0Snw=9w=6OwKnv~B9t`M4HIAc~*!ML@cHH3^wDH))@896;x7h6wuIC*h_b>k%M=mc)}<<9 z4O5JNZ(-q(Lt~k3TOO=}Mj8Ww_#Q$_1=^MZ3D6o5&wC8bZVrn(}0B z^b0KB6Q?hWYI89Ra?5zIMUNas8clTSD-9_`RBIU+7*H#CHZm9}`xuy>iX{y?1Y>K; zVswO1w_mfi3X?CiRhC(V0&Q!ZZHX{I%xM%d?p30>ivO7UB=oaUM~qpDaexk*jHF@< zX&{i)n0I?DR8Y-eKj~wi80^*0-KJQ>k2OOhrp}WAr23UPm@w?e!VC);3Vb73DQlpw zuOAF~=gNGML#Bgf6$|w+ zYRHNVe5B;)N49phVsdglxvVfK{gLYtEFBcf_Z~60hwOOPbQ&@oMIgC8WVkn^6b8^1 zh4FQ{BR$u0!yY&xR4YGayPLhd#Ff%nvP%@~kBC_`pbFiY_Cj)>D8MiaIh+eEk0vJ@ z0Z){HybwOFD}4T9xvaMlkXk4i)tG&y$~V`qd=s#-fN;s4<0&i2O<{fFVHCRp=bmbR z#Eul?%-(kY%s^r1=3NrtG*jpkFOPoW{@(TmR`jB+nBe0Gky+8m4Fu_EGzr$~X|QHQ zThqnl20n4p{Go;Ow>>GCY$Ptodgu}B!je}AiZ=}|40J}yy}_c8hD1#&1){IY@Fa%# z1e#gp!9X4jUlM9-b#8S`fz#kTgVR+_>2KAo+nEG>CXH#L1tR;mQSgD9H6k%`yDE%B z5SWAlvosoEyB9DAE0_(yiEkA7f_&*46|hnk0zAafRvjVk3YQ=m)8tZbvl=si(u(X$ zivFpnl4gM?Si#~&(;|honfLPBYDJ<_AA3& zlPFM+^gvrSt$%#xP@mIKM@UhnS;*d|LkxzKG6jG?Iw1+~#si<{bp*9K@v*dz)r{%P z10&E!@EVD+*A$lIskyZKz*-STEiVA3u|~}DPetr$La)oiR7nSOsAyObzCs6PZ(0|C zm65@KNXhp98!7UpZC`I$d^MBn*LT6~kmg5l0-sh2G>sTw)G??=VPLMc0qLkrxByx{ zX)Rc`tWW03ep~qdEL+{39}u@tH+P}Lmtug3cu6ltfLNo%5U`Z$;7*dU*FYB}mjSh9 z1yt34Cvh$bg!N0y=DzH2S#iyN4WF|e@{Z^lgQVQ7)8k8lb4a))X7&@Mnq@1@!~pXJ zpwmDS_KAs*8V(}8v9`aa=OO%E3gva0+D7~g<12Z&h&Q=pyKX6b_H46@i;Lum&*xey5)z=uLq;=tYSZl9yn*5RZ{T0* zKi?Yh8%FeD_17U%9R}zkB8Yk#TVHRoecQGs0Epr%3qm}KD9JIHyivofQ5H<6VmXV=!fJdW< zz0QK_w2Tf$;+JE{Vt5|smdf7s0yZeEGfg7CrYyd))+&OvPfebf%`I7w)oxP~cfAwmI ztgI|+Tgqc2(V`2ov$yKS8;S%?V&wiibMqTw>ov|9e8*sJXV3FOXcSY z8)@&`7iwr|sLY4w4t0$+OnIn#Z+BQz)1j9+Ik70Kn31{JyqY0D{o(F7-Pq=l8F~gI z*R%U)b*!r^WBZOBF*~mN@oN++>caQ?-HIxn|9}4GHsR z;W&m#g5}t~X`5G-;k*Z4cl5Y%VdEY(IVoI2 z#j_{r)22RmW;(h_duOMwySqC^Ka|bb7!N1&jTz<18ur4FJA7__>@2ifO?2n-NHrd` zd}TG+-n-4xxMbaF=iUbYh2oZ>C8ecY+}!tI%9h*AV$^-I92?F9$p!wxM<|>z;098D z=TA68WeN~EFo}|lQ7?sHd<*hh4h0^usfY@rq^Kx@AhG-Bf5gbJ8DfNkgQEoS-PFu1 za*2#XZbgM2LS;SnpTRJ>g{DD^D5(GNuI$k71L*)25jbze+$+9yLHN$amy z@c2KgJ6`3*>$h586RL&Z%l#TK5qK35+jZjJ_N%q=f3XGOEu5!t?`6?A$AcII2} zmAkZ|*i!(qXV0K~WPOE(O5orGsYS@JW}t6Lr{DKy&7Qqd5$560n}7QKEr!<*JWfn3 zco+Qo^=rrMHe=3WW=fER^$sNNHC~;2yEOo5b9qr(Z#u!SMIr`ZfAP% zV9RCUjA9tvkeiID>9==xze5w#0hTDW#~rxNgkqsX0?498vG67o;DcSidi5$YIE&}N zXrcBBp$~&3e#z5Q4=UW}FJHOF1eYe~Rmp$t8r*71N=jaJ_1@#hj|&>5J!v+n7oE4(P#uln%-ORid)rM*`PbLP zLcfWEnr~%ms|QNdP*=B4vMbEQ#H6Y2*;zbOrfok2S)>J@n3x!bic|q;w6Lfuk;^$R zU%p(5C46-7%|gF5dvq}bSph~VprD{oYL1(d52gZS@8HmaPGa!XCK`P2m?WIc{`sSI zq5kB81q+B}f-Qui7Bv0-^@Z{b)iN6RE(QZ)D79xI{zutl` zZ#eOJ8%bP`A3u&rWH5ThZs|Ns2sVF9$kes5Nx}#YMmFUrL}>FQPkoO9md>rIsj-PT zF?sUjK78aO^g^6a*}MJH91XD1a*W}g{P96&rIPQupbLgJ1HVr=Lq+14HcjW?!HB1} zt?@W8wF^X8lvQ5pcTt{rE!$dl|KF`^j57=|xp}UcmytD`Gnx)dg?pj7a4lI9j}fmH zxK1SD)_?l647FP)!Km8CDEJftBhD~I69WYw9vpC2^5qZ1a7JEfsTLNfvGeJ;7Jzvw z5@*hwq3$gM<8$G*d3i*e)S zn7t80Wj})ZgF$(I@cH$Nc1(`TLl_oG(Y+t(L^LyWiscIp(K zvVA*wTz-e4z?I+LK4sd1_1mCh>k^D&-GW>G#o2sicN91Wp{#LCnW6=#ufoSB|L^~} z4Z}>Ay{Au~rubiLoUt0A0ZzUr#yB&hIcZBXHAgvS21KLpBMpL3N)j>YuBz)s8Mr(J8&cUMMY|xHf>^U#lK@pp-;SF z%5BW%2%^K!y}TVgX`#seb5PLtOHT8KUlkS_x)hd-G3^AR1+$yH6oe6_9C3g89DtRi zWu0qaP!KH=O`o!o2%wKqr$-?Nne-dvp%SQWy-6z>xqHXNrPXuPr%n)|Lpat${ShEm z$il+HZ|&X>HVa)892}l5U$J5%P#}5rkb&L-oTGi2NX8Y5V;p14)~y7re0}Gf{8>dE zl@gw{01W0nbOHy=1M1w}IWx{E-C*t7wLk!&_xFu?{2)y&tw)F6E`M`FWGl+b!c9kS zK#7RMyx&dL?D4r6$%k3C5p{=}RB&)G06U#>RF34@iTLP4?j+g$+7F>Z4_Q)%^1E?g z7DEOCLPQ6ipCKIz1vU@B6xpJ6=ui}_^na0>9A_;Tl}@A|V0}&8ykp0COlt9>3cm@I z(1Pan7F$x^*qHN=G|z(v4>X`S`+0eV0&xfd24czHotr!x#~b%cCS>hYRfWWq*ol%C z3m5ibVqz)6)~2u19N@uYCY`zRD$+)4y(MNr?Xnzh>D9$jx~Sn44Ow4Cq%nr8JB{s7 zmxtNl4N6ht(tpfPKHlG|>0Z<&Q|Nsl9Iycj@9q4X8OfcXmv3#z2?@ zW}|I}2J9zJnzSD0NrA<0U`B`@a9430T2{v#n-$5!?9et)k)OWVBq4o=sQFC!r#k%vow zI+UUK%)#2gsH?GAf_q?$KYHxg#sjaWg3J=HprQ#}1CYkQZnbV~dGV|29e-y54&%`G zpRc9DAJHMx($ahb0%9=G6pMlH2p=!^qnO7--V-5*duZG=)c?f3BVx;zEkqrw)9kh2 z`4+>&?}CXK@Lx2#LuM0mz(8y#S_jRELIsXgMm)OAD;yJ$!&^~(C<9O+w!jKV!$5^@ z(~gYa*23wX-Z)%w7nA(au)|y4-f{Q#@li2rc!CB<8SLTV5g={J>R;6Zr)3wL6$Wa+5I9+v#fGt=y(@NH@p`#m zB*;zrCx#w0L3>OO9JqsF*Cy1|^+CQrTqM|>;MggtsFb_1jqk)s&sc!^h*Da&`tcw# zL`{)XcZ-YF0bmM1&Fb~Wkg2v>6j9*q+i)~nP#@a6x=bf{u$SPp!(wSWbrf`UG~12; zKA&J}VsfJ}e=f!b$^jvDaE3-DP+S9~RBQkK+XI7x3rLb+*zLp3mEoe)wGrz_;98*W zF0K7MrqB%gT)!ScA_(SQ-pLL3s$r~YSQ8^(DrU<XH;t@6X=C=xgcD!pd+eR&Wy5j?8B zj`*k?;<}{)w!H)TOBa;&BWM6uvaWr@nXFg@M_yr}DyYd@(8DV>7*wLC(?lCAkKZfh zjB|r{-~$t;<4RflmtWq7O(c(5*|(5|8yXwUfC$=GkJA3oVsYobBuuj3M#nyk)(@U1 z7Ibb)3=SyM!obtJ1A$c$gfQ<>V}?`jLrlpCBP!^T*{5?CLm`ZU^jL2|>BtzkNmTgR zvu6P~vj$!G-s{)OVD4Ux>qe`N8|vj5ibebyzpShg)?5@<{0UW@`d}Aa?ik=}B2Lzw zy(A@?;;`#Xdq;u7uBkl;0-gwq*Rl2SAuh$tZnxw7>Hg6U>Jy8;uD6aR1-4$O|yrC!P=pz^;g{bhXwR^WZKX z@9jsMZ26d+Kv)4>rUligy{AVH=Rh9*z{w?CF>>SKvYnLNm^P-T_y%{6!c=Vxy5zWL ztUARSBwbngiqiGfh|b@8fw3{)^_EUW;ZfYqos;g|2y7h#u1cKdl847TFl8W_t4E`# z9lKWP;TRz@L`YOr3*rJjI1s}+k1(Er8B@^1W3i@9ZbLo534SxRo`OqMaXg?&K zBa)Nth;gEv+}eNOz$5EMRT&uHJwy6OkKS`~a-w6vBz=GyT#Ji~duBk_%XQX`)q$<_ z4uzap5Qa4=>Z@iMy%=plS=mZzYI>+5C%am7epIFS%oOLCiu-hPcaOy+1|3Q~^|MOA z(a}*QSXzI|ytO1&KRWSc2l{)tjUZiT{=W})Z$NXQ`T{hi4zzZjljk@&nH}DuE7_>j4Q^o`fke7f3hb7r_XTs2~0Lm`M3lDUn)HfS52Qil`WQM!K zSGzB9URXB-T94PHMWY7hP{-uogR9Av&ZHk=eKB(&0d&H&%df39hTa@u?aED(LNo?G z44znqA(^@`!HtE*kGo3K(L4ui@y3I%=T}3vXL%SC7!nN^SevZ(=TlL3b@RRs{Hxpyd6iOjJOSiv_U?zP%G(_h7(*>gC5V zEur_-&|p6SO%@o$bdz!;V+WW7jd%7Or%!RMt-xRbk$IR26`e1B<+$NV_>6~@KmTfkda8xhaE@pCyIwgm0lQ)gHsC&!p_wgbK(ob>T z-W>qaOp+a+_2g>v2$6A>OZ+!`61eGM%+g=UIz7DK!h)tVjsOx;i*9D=>A9sC+B*q~ zgl@PUZO9k@*^Ykna@w;k*xG<;l(_4Y1pMo9SL=`f<9Cj#nGtU@AFpP?HI#;Hy0e zkd1oqLJ2E!=}qy3Losd3BUp(?KfxIC2gT$ESjV}&1uHD;)ON3)i&jUfqXWPY=_zO} z`0|Q`AK3&ChD6PyM~`l>{z}-S&3}w_lGzacN=QS1O|R=|I@i0k{P^R-Hb5 z`YR(&*XKe|9Wf15hM|zyJRd&9h$}nS*i2MJv}gg)f(%9gE!`+&YrlHs%2W`A-HPK8zS9D_@D$ah)#VuWR_CFKqM{~liBvp@*wu~$ zV8kHUakYaVkmoQZtzvdRxQ#4rO(j~Lmv`}jxGVQe=%-ND$WbF$OW`qN3>&CP#Jndw z2?0D0)N~OnniR017YY(7O|WN60c4hXraE-R5S7J9+BZ~QmSZz(7LFIJYL@$Ql#egc z@$7fLV7o1tMt|?jXFqHuWRTQk?C=v3+}?Bt9a!P%ttP)vWopE>Wa6`U^I*)Xs;W9* z7^EJg5>iwP#zK62d}dp?j&Cc*Z1GZKV`GGW{>P6WQ2_@QoUfx$ps%GC4-Dd{%#jcd z?SK{+BLoh+fQ(A98#)on3;w(9Tr>&*eJ<2OoED{cr+0SoMsIQBh}n}Ik2WeaZ{9o- ze{nCge4h>rY}}}G@!~~E`}VsYfDD*`18>>d2Zag4jp0!IsN!2$TWi5CEq`+204onR zxjG#J-VRuAdI?7`)}m_S`fUnhfJ$0}kDBaT@MiQVm-4th704ZnK>%fz6x01Z8pAAj z4hG{q_{)#lOk)ud5%NxeLc^gJKwaB`Su-76l=v=|>+8j5J;o=&ZJ^VW6aFE^I&zFd z1r_r!PSg@?A`*bOjj(2`pN%nezs$@`yc#SH+ldRh4!ey)#9S%poFtEtO3KE@h9>7E zyl$O3V!0GSq=W2Yt*bxp%9V7^mhds&A{PaE91Ye%qH0xDRsGkmt56154eHHI*ym}s zA9|6zC)?^a0|k8p9z&k~D8pOh%A+*0ZC1HTzdrmZAASzfmOwBV#&Hx87tS%L!4_jD zQXFWEP`gNJLK(~O_w!o`OO5M1ev9nIf3jMEE@+N<4t@y16$SMM9jGo-WRwTlXG2-i z!RbfHZjS#|-O#M}L`IVig`uWJf$C8|li!3>{v~h+@g^iXMvDmBPw*1k7Xf^IAAWTN{5`<6LtvS1-Ms2hIWJ*kmD=6r&6`P?Hi1q=d2 zC4-~TlwRyr0@EW^9`&yooP``tG(J)V6nLD=Y?io6*Wn4skVLtH#41GAAWHI-(KO9U zgAiL#S*Z`X7Xrfipz(9Wp_pm_AS7Xja31LhEmBr3Sw@71$wq@j1faDF+WyJr61QKr zn;@=h0^CA;dbIx~N9BhPq^7{5qx^?Ue!lw2iLE4Mp!ySopxFsvu#01YdJ2@KI~Eze zJLpU&%XeW+)Fh+}afi;-lixx2F&>_11*{apU0@#FHRk2$B^!ct_w9xILL7DH&I?oo zv3+z3j07_PE!a5g>S0Gh#F zHP{3x0ZeXF5SsqbHW-N1fB}JXA^9S0%0MYepMDc8b8DvEbRn&XE3m#IFfZH!FP4^Z z1Dq6^+`t1Wsj6-Y`X|ekm^oZvynX`p$me)%2HCPNF4q0I4MtWJW+3Roftpk6¥h z+9B!C83nafr4h(v*c+0Ym)8tpW~hTLRVj(EI@UlpC^dJh0gX-C_iP;MvLL!ANyA`t z+K&$Fz!tW`S&JNcdkEwaL%g`FZa9u{W6qeSIN!RxUJ-Hojm$$}FW|O`HB_$=;(#zw zh(Ojx45v8u#6xQ6#0(-0_qEh!W&%Qm!yAR$GqelD0^vX+%-%FR4a9;2hCFrfP7S(= zw>ZJuMS21BN6gf02pS*Swtv*hd2J6-mjrugW8SlqTn$SwFpW;Ciu9t5kPq{Rz7Ks$ zB}h^SRvaD3>%>ds)(aN0yY$+MndZ+4l4BY{&GfXAE}lp(zj6@Xm&TY2Dr3 zWcT63;To`_i4!J7VLSSul+$StTz~o^jA0GoC8t!xLD*^0XNH z(-D%CG=Lct+&qWTOL>SFIm9Bnk3a{f;f9RfuZy5;zJR}z^~c|#4(%!3e!5^Sg=3B+bE5o9sJwI-~hvU^XErl?wj5fLbpRW2(>8!upu5gUO?gI z=@&42ffzD*up9p;^DJ&}>N^|VVCWPDv~f~E z2?}APBMt}pKr+>L?!E`d#b(0aXy%tDXhdZY(iTabs2pxY+HrhpG7Usv1Tv(VyM6|m zLXeE3F7Yft*Q*->kvbGq*BV`B;P|K>=3XGlLLho8=S+AO{-X$?F_=corbabbme^_X zeRK2jn!^&tFRb5RJW@f?m zuq0i%+kmEHfLCBwdp|x}xtyJy9j%JvvE>*;C54>rBH8|XEbuXrzli_!EVNd%K^Y@&B-dK*0>T!r!`d}v?rOWxnNN5DY|M^q1q1Cuv4}yD4=m=KGXoZ# zHsW<7j(IraXfa^_feA*rnBh7EeTp<4vejXE84V2$eI8H^t&7HNA&tx618BShQV+1l z9Lq6?JwC;`|1m`ia7iAGRIqW#pj(pkM-!bK9H9t;DB!%ORt=Y$eQQB|%TsMFh~#4w zA?odQY+DSpRAK{fr!zSDP{%R2O5F?GftcMi9$CXpn_gSD@c?lW3#QB?D;OtH%i*aU zlAI#H$r?kFXw)ntc!|sho9V9OJdDl%vR7Bb*^3v)A)aH%e$!&?JY4xO94}fn$G}E_ z>gKiZIO0fe#A_0oARJS;U`kCGfk|%?@uov)R~Lv9K_J5pcN_tDnFq%e(}mu=$gqM} zufEpYav&QKMp*yPDHxmpV!?}h_%LBnZ{-rMAAq+k z#6Szg2zta24)7mAKOkMOd3F6}&^z)iFdS1DEE9|NCA5Bh3BvFwL`6|&oIs~h4UBM7 zkH{KZ84_)pHOV4n8Cm1xy}?D%n=u$vsBnGWEHc>a?4A^c$R%K$lp-B5Lya3pjpz^( zv}~GIMaw320Cki^B1%5;q$-S=YY-%8v_EQ2?(!212V44<(U12*EmlM!^22n(XU86l zp*P`v``|uXx4I2`kgEm&EsTRQ&OM$`?U13mz(8mlD`4A@iG)L{!yN_|X}`{cw1=al z9tP}a`S2(fC9caqz5*5*=q&3-!ok1qNNn9fG@Aj5tA=;SVMAcZ}gX#|cA$8aU3!HFIVdu6nbOt&K^u zK{7afOojnW0&_|B%GUs*e8|d#;oVPwuKQqswx%ZF;h(w;K83Nxs|cyS7-)nz6*6?E zH>^q;Pk>4zxP19%+l0Z|9{tsG|CFc91X9|e7)f+Lt2?O~fmAe`z@oeTY@SOQK$_=Tplr@Fa1Er-TNF$k!A z7xeWTrVjnn)6)@MTPXoNJw1^v2{aZ3;En=eGCQ_B2A)z2PSnzQVb-iN8~^U-_WjHL z*6duaw{PFV?4*e-l}N^KKQy>lNWpsC$XdM_eWB&sGnl`lX_Mc&Sn6@d#RvX0R++n zDcuOG!%hfw9oyTt z>9vcR#^>I`*_{}G9|2TNcul?8VwV`4F(AH|u3ptaww40~Ud3B@#pKl3aznrll0aHW zwTArA=WLNR!PR}tTvz=QT2iPI7_wUzG_KCp&dv_+@FB}iB5tIQN#qxp|zhJJrR6!$szP;jEzmcWBLSdNU@oxc59e~sfb8Gf z-!Fs~NpHc!`%|o18`&AXdob6GV?_p#Am{)CVT4((;NclH{d1&ZrV(uUwiZd>l(OT#OK+@uWayJ^2?eUz+1=Kp&tpui~FUaI_BJ z8#nkBsm!|!^_pR0$W`=)&bg{u%rY2hzf-6ED+@WOo>0zht8rADCK(8RK%W4~cv&2Q(-z!y)j%#J<4IY&T+iCm(e z-2t8Xh84gu{}eFqa9o5ObMLzVN_1b?6Xe!)^5`+XfN@0^N+^u#4R<n6 zjL$ubMqY+t{;!~b7ab9j z=0w+*`K|STE1fI14UA+`YvB#_y=jhQw0l6SjY5WeNVW(8*szdfKTkXqNicEfDjU|K z7?m>?#VaReL96Q!3BO!Ko)qCm>xd-0+w@+zBf zR$+$}aSl*aO@!E{4{UWV?3_!=C~?r~(O_=P6u-icF-X$!@L2RZTpA69o?N*PhQs0N zr;ku{ty;C57InXJ#B#0g`46lWSa33h5%#2m>BjDFq5PtzNNeXxz7 znFl2&XKj7>=utWJ43h5f3(;A_PZ!A-j;-a|5dN?|7Lwz$#zW6chuav9Lu5BqPcIX* zs42q>Chi~r75j=5-nHW3ouy78P__p?J_0zbq0idP#N-fI3?flth1OC|Xc6u(wpT#6 zKt1yiGhbRXA1`f`9cbH1nTr_#kmR~$tnrU`ab(YzWipwlfM5pg%P#xz$ovf_`RToY z^yCE5>!%1OBjT9LB_#B*O6uFTokN5lgt_5rm<+>)ro$65`eKOdgG>dej&oVDeEAm8 zcRHI%n);$eh2ti+C*n;RG%!Z<9sdaixyH|Xb^czIkt>3MPO?RD#Ox@dWjsG#*BR{i zQ{s0zK}2vqgPcT02%^0U z(2fGLXpEgEu?mwM$TZ)epdb)n&Ro>4U%!4uh{8G6!CyE+)8DikiO2P118n0$Ne05>{Yv0M&)`)d{i1Ru+*BT~5-{RsAU;9TEwp83UlG~!NZ25A{7~DKwVl^Ajm;>}28WSUE!+U$y2^*UiD8CP&*4;x@WnC8B zHJgVya2kp9D|_)aZQj)r`2M@z@|&PHbcSNa#clZidawodNWMFO6OdV4Q(*UI)5ZTvj%RcdN28-WzggX>Nwxsue57 z<86L9&~=_AYhSo}ebfTZ&QOCjM>PwsH6e1YE+1+j6W0*1fo-WR~QtYtM1;bsD?|q6jA4ZbS0h~|9f6I?#ocg=Yx5u?#&U6zT=jq|`3V%(^gvy@CLx&E@ z!|H09j|*|_Ffrj}YUp2l@!~}tJchQOo*YyMGK;yT?Pf6QzI<6>DXE!+PhKV~y9CzC z3`vfeGp}ud@0$Jg?M%qN;trksm$RZvesy=h0w7rjiMR^y0%Yw!MknsyzwdhX?06uA za~@l_HO!gv>>`#3Ur~k^uU_4bcZzudT-eM(SO`wMxa|xa-!KS{2(^iL>Cj}X%rdcV z{9Oe^wN}rKKi8gVi@>f=>XXAp8%u3o+7 z?R^F%V6&PUCnhPj*x89QmLdef7S2XT3(T4|tFEc(pS^qc0`YTTrjy|c zu`$5y?!#_Kd4SOTgwXsySbOuZoYOY`|1L_hv?yEBqDb~Vp~xLg))GP}+1HR=T9g(` zLiV&Mk~L%B+9YEJ#fo!#r% zR+e{SbeLw39zB=|p-z5GZ3QkTAp)qs~jZyjX1=AMDu zOuKbm_GzY~P;&@&@^%J81`NpHTt#f(K6+t|uQkcfK|6LBFf%a#oSAX!R()!dRJ6Dp zoRw&3h6uiw=*T)kDbb|+yy|>A+7SAI#E)?3l>Pg2`VF@-)k*60#C`i}`na<2n)55Z zEMOo>4Nh?bYt5E96oxdI29f7$8U}5Oq3(q}UzOc~U8|Q5Pa~fIBeo7Tqn2T?!^;mJ zKhC8a5-?K_hE(2>Gm1yE6=iR`End1bAMvB=z4bGTbWE@dHvSd?=e7@h z@1_3rS8WucjRu`McUC$&YSJvJP!RAkh!@4ruNug+$ziMpKf$mAJ;Z~y-fy+Cl%_g# zEPB1TW_a2SReh*5Y|U+8zv34ANTQ=6@Ad0iixw?f)1F((WmT7hSpK>k>IH*r?w zd$^tPu1ODec(xPo#qk+T24s6R(1AaE5kWL^A0=olj9IDpO*$<7FjWF;O9mromEiKz zZrZf&uwldGpOtE-uKRHaUCc})5>vOXTUVFfg-M??^B2wO^0a|}HHje#j5Re9t{Hpl zl$5<&ovIS!TlrPt$3|#oW)`3NL{WtQwc6s`F7-+tu->|L zt2?vY;Dj4k*VT!v_bZ(xuwtqTPIs-H$|bdH*XFK45RQHZgmeSz{sRW=ANV^*w5GYW zbz`_1U2FB0f6zWL6zTgfbgu%7c;%iwdt|RppPuyXh0bJFXGmC>ox8g+;-3k5un#ko z8Ul|q2(#-qZ8GB@S(*$QH0aHPvkk(HY%gtUIjkziR{t4e9rjUht0-WN6kl^|_Gs0r zmclsFv6$JMr)RNs=)BcD5V>Q={=X8<;n+GUdJ~Q(My0F_2+)k$Sh;|mn)DY+Y7zf# z+@4b#WFQQQusv>=)5V|}f8B_qPkp(9!lXY*I&1=P6;ZksJXQ`o951qFzt;6L*k=S= zueoz|P`i48^1yJ+>pd3?j}}}X8F-C`t)|p=@kD(TXHIfjvn1up=WV- zMT&bFqKjk_qO(HIiX!jj%TP22i8gT73O}a58no2h5hr_Y)-tBLIj zH=SVD38g`AbVHS0A>mSDsR|o*nXbpAC6)Q}=QD?+@%3oiEmz@)+Z;G>pgpZ*ib{?H z?HaXj5-lUw-oEXM%O4P=WTD)tv(xC}Tfbw+YFw)mC~94QpInF76@cea(=c4`F93pR zL{(yOG3m9UydmIoYtJPe5o_5=W~adKrpCq!r0F!OBU!S9yu3U=1X_2J`6)Ugc_|L= zNp>}UhKgoZRt=HQ7k~NCQjy2}HweIbti;4jBY=Jh!(fjrcs=6crAw*`q@>lvl|OEc zIzC+=;wXs0aRA{bcrk|#_xarQBGsNwg9gErtN9et^3_$VRr3JeK!?C5x-)=}bumCF zQ1$fmRQR1aV+~tSXTgGx9m~V?Xm_dwcwk8T3O28{c+s>=?1|Q+k7?jx zQ9|g%#KozSBjxe~(n&g|p-JPfmv@u-iLLUSSnQS~zAjl8lovUGctj?YrI$ZEFFaIf z+H-ly(3pwwt+#LAp2)c5rnW0u3rL8YfPo?nM$=>F%-Y-^D5uqcDq(B)`OResKnZ_r zU#RVr$(4n=Sd`ST%%}0JF$oFg-e#t{Ar6+2jzM+%RAqdnGO{l<^<0>)qPOaNc+yoi zs~|*zib?(V`SD8qQE0Lr)24+2@&{F6GyATX*;LNsJ>fO7I@7%eg zQKybQ`39`1G~cv-y&pc5-9S*#cg&lxJxZ5^>O~KJfNn_y&WYHa8#ZisL}O>7ZhN9X zszOy<3TxfDvmUGKdSYgy)v2MHu}Mjq5SaHCoX4)63 z4fy_Snmn4p=@j5r;&jydRf3&6ggB^t`0!!<&Yi8jYg;BD`sX2Xr{mWY+-7BMo&Ml~ zA(Xyd|5n|a_FU`-DAh3B?RMqr)l7Q%w6R@y>&0Eaa`|#{Z*@y0>@g9jjqlbF-x`_4!p8uvm9w&FA{E-y*G@#(jC4c^J(Pj4E5 zU1~%)5K!ahUCqo?d$jz09VqSfQBiFq#s)D=csJ^P37)LdKI7sne_=+K=};DuRpXso z@)vc9`?P5-y<4-ex*9zK>J4vt=FFMaW{Ig%dC?ja%jVi@<8$VKg)@;cBcmGPjhDB# za{Bb?$+y;i!|0oIM(vx?fOFi#YvIC6u;#IQ_KcWZ*5=^R6?MJ6y_qfC)5^-K$kN?q zHeQr!kYgDfEsyEbNz8HchJfE78^|2#&>vO7#aK7=2quxlq@?sacN#hGbCKF_m8-_! z?;Az_M>CTY92vZ8SJNH=Bi;GTzHV+!=$si>@C0WUYKyJwF?Ti#g{Fj4D1W&A8&KCukxLd;iw<9s$HQ%i3gq z@?Hb@vUIAVs&5&yd-qDHz&wChQE_oqt}v+|K>Ypj6Jf`^l;5>?CT?JFWTN%0kBBhw z*0c0Se^e+Sd`A{MVN2Ce?Ci0qu0m3INalo5FN)Ci<#f!ZBX8EJ1+VMxvh`iZIJjyG zl;HMoOaiA(OjHp5`CJkFv>P^bO**cUhc%~t*RGol!{g|~(%{n7cJLi7yFGvPN|VhzGi`TB)duaDkHC6~$MBdkB^dXZsk!;Hjb-Kc7e4#g{W4raFoDy= z(lSXoe!l6X;*THgsV>+!L`zc^P;(M&73Nws(W=^}EA`#r2btl#oRP8BJUl!ENp>{d z@UHfMt=>f9H`IPjWL$j;w8$G!i$x5eH15zrXX(GzM%WB_$~yF1XmWQ>P3t`eQE9l&!*MZP;N# zb-+a6kt6NTt*B@WOmEnAWjuLvu8aYrw8$JlGX#*ikZaW0UF`=1PZ{=-rn zcH3^zbxP3v1vzE=#}9>&9y)Sl0|`03iQftZe7|ao*bNqrL)zD=QwNXXm4El|+o!*E z*lkyvWY^j#6rjCo3aLe?)br&fz`rRJ6zFYp?v|3zs5q0dO}!MPgrLovHy>cg4P0xK#GJFyo(phMyP<%Ig4euC%v*EK;hsul<8VJ#A`~ zRv}BGcI$Utl~+D-FJcU=D7kMBn9;H|qBIkj)ZyOJIUe86e*M*t3k$>PtXm^pqHsa# z|8wwhnVPq1Y91HXrn0XgSV|sxDic)4om=cr`KDSiCdP$9hE8FoX|QFdO*m0d6!t&xqNCeS1d6+U*aa}EFEoaKe`rX z;-utZ;wSz>;JiZ9pa~{cPR}AL&;Mmt6eb2kg&&Rxy)Pxy(<|S+ndv%eRLe<~{rdG& z7{7KI`;aN-mlyPHmsB3`n#SeR#pSc_FqcvD!J*E26;1R;m<=&Aqs~E)tjRX#TsZ~p zJ$yJE2w>YNTdC6=1{^Pur$PKa~XD4pBTsiJq zyT%D&G8bC-G^V120e|f*BUKlVEa$Q=%bnc*fX&tpKs@Q*)@|EV6cB2~MUVTIl$2by z(eT1UL@U-~hUN6et8l=t)Tgk)ObQM;{$2|fwE5>Q2v=w`g0p_o;#+#nkRU z*Fv`IZQU{63f+4pQ<_un-)|yB7#H76nvd0@7Q>LZ17crxh5mxG?@F3H#j>ZWa2+$I z!O}I#Nu-S$G`FQOb)tyCAb90Q;)sKdF}?}k$8H`j2g8wYb^H9XZsRU2ugS85EF(7j zf-3@9^qQxX3>-duc%GRuUZeC8R?|CjTBJi^F54H{ON%UmvM{;GR1v zyEhycgtxK0KtB+L$SNb4&+uN(*fJyjrw_fBe^ys3ef7D~{;1FPM zw3%E1a{_VZ(r$G`=zyG*6I`7lNi*ft+DO#&9 zhFJO|!wN3IdB+%)*6g`wuHON;%dzpcIPZ5u*q6X<2Cj3SKB>3~KqoajTN?@5<*rSL zO^Y5q41@-biZT>>h;-@rl0;jAULu}e;kqO$C{XGyA&4;jwhIG7UFDxysl54bz|%}O zG-l79RZwJAr{Sq@yH;oFb#hvgOeIA>p&kQy?E1I-%=D8rgk?`mYIgl&EoVak;+M<7 z0c;8scHG~8g7q0j7<}^rgf}g?bi%j`>Ws# zO#{^x*scfVnYlbXzvO+te+$Fx#*a6`U6=~nR7_bE`(#sC$LN*xJ}zFeYs044~qN`33PP6^K`jalV9T0E*&inP|hcW(vVW}!5rRlT6z{}VRp9cm0| zU>RC&=e4l<-f#-YI$Vmh7BW^vfwZArzkYMCr2G!ux&?!EIdHAKH|aWkn!ldoD78FX zJ^R5w{g%09C=je!_wJ1din>J1gekLT)q!+!jgErs>HQPaKue?>YAQv9d}9;QjRZ|hK? zTv<)Rs0XLAigrnUe^PB8;nx_GIR^YBk>w%chu*z= zcRe9I%h*(Hap^nrixWCqXrr>1albc93#t2jIBQWO~1f1ewsRGRI?NFBIp)dVo4eE2+O>ji)} zJUKK~djV5s3xH)XA@&JXjetq#ydGWUTgy+pH*wP6!E8fC&dr-;1@}>ULX@WDr@gi7 zZ9WL$yqj-|{t*}(Ya$=RwVjqH+v*G-2gW2`)wk}qZRn*L6ZuY33C)?KwP3*lK0rvA zKA}M)bFPgXUeR;wrcEFIh<-pky^_v2a&>ej@-j?o;uWXi!-e1FX2LZ<579F8rsA}c zSi`()f2eylj8o0zpG9PdCPq!ZupN0MaGhx)yl6TFT=VA5v$M1P09L|)oH}(%_wZ@7 zQZBEr%~tK(ub(=AhM=f^`|e%SPMv0o^?qc`;z?he~GmABQpF{f7^ME|x(qmiEB@z;(mSFx{a}_d0g`_*{m6 zk>4fFBGn!_c1#Hal$<+o&>*SJrN~?QaXNasI-GcKDl1`FZEaI@Cmo#nTG#2>-u?T> zyzCLs>V21^kR%OzE$xh_R+X=(wx|9G`yS(qdzK8IH?O11@Zmaodet%S2~!L7kc!BC z>QoIM2Bqs|4gQ-P@>YmY4YA|vATwNQAocTFMELOLm`%m6vY4zJJ6KJ?Ahxmuv8T2`J>{_>CYN#npB=-*WYW>g??Yk z$}+BB*TP*+OdRPlF>Uy<_xjgY7MitaAsh`mvNws7F#^Iw^@KVGX$N2Mz@jPn|wJ{qkiEKvi$7S&!emsSWimR1qU8Qt{R4 zG;O->)G4cR+X*r7u##yY=8rm8xG7z9m{VZfv7@d)Ziz6rVLIGr%~}gf{I1$LrT+S) zY!$&E@{>3bqqiuSO>CAor1!3&m6tK;PTj}HN4_TMYRirkKD?YHFBLB_kZfCor>Ox{ z@X_uRBc~Y8@Mgu(q1sB*h&NvmYm#Y!eP(pO^6b24nuyqfXApipD5@ITEI)OBCwX%5 zvSrQSEr;yt*@t+-Oo?RzC?LjWg?{Qmd#uQ6;2*W7H#vKt%E9;^-Ym#SPgf|AF}4^K z-bkoL0po?e2~^p8`0>Sa?hQY-AT;saelCvXk?3SqArQwtu}~Y|Y_%M_k|rtdA)g7o z<7N*WHmn|_qx!*a*q-&_3=2?0F(fHsThc#8Xqv^}m{9WaPUhVM)5jnHQ<3;bL^#Es zT;T1U%~(8SkpZz;RL|*L>ZV6?N+4r5G$i`G=KBeoNd}YcAdp5)cXj4%x@hsHucbM@ zQC0lc)zR@i{8$ts2rx+h!EHHI-C%-@|Fi&Jzj@Ev5M_)(yQ0` z+iD?DXh+*_E`^7w3j<3e5H5^Jhrns^se^t0{?!y1q(|hjlG|CfS|OpK{-L42C5+n= zVrg5iUOk5d=bk=a^f(;)*6W;I;&U$(-f=$v`}=#0X+Yj^)9VjT%o>wC&*j+r;r2_{ zMYcq(fL$|xIIeHsYE(__J9f0AKC|s%+pAZ_ZnY=4HZ-)ff|aJ=%>!>Sj`+?1Qo?EAyQ%;8<>Y{XD9WKFOl@@B4D{g#tad^rNP3XmzQ2 zC?NBmKVL=pqA0W;kCH!e$`merfBV)X(s7dSm7@kFuxlM4`uWMjJXFH;`IY?XH+PQ@ zm@NZ9Vh$c$D;SEcK&DM6T|#!Y%Ds9n5#T(BgCQEV!jIzAlR0y)Q0>HG;-#%SZEqM_ z+I+eTPohw?Y13w>)m&Ao%7z^$YAAY-8DpS;78w09*F5e@yOj1_Q`Vp8m_K7;u|%Ti zCyPeLEJ2IhR{!jaMFuBNoajdwDW;iV!dI@3FD_2Qay!xJ?ttHZOM{@+F)&z75tPqc z5cdLy)??BnHA6!~NfMM*SRpcDX$hfPTX*c(jii-`fAN6Tc`_T!;wr>gCR2rTr1A-< z8g=XTLlS#4#SN8(d+iy`+G8IF0#gP*yR(g^Zxod%i zvCDJPAC*np4kCHn0QBtH(|paEHSQiB#dk_|L!a)B zy7VMJUt3>4NFvb1i!fdTX5TBn_B{G^=Uu+hFx<4VYEMXRNbo~e)~T>X3=!iDKHfBX znnhJ~JhGx_bV~aiev_l=AC_x1?b(~d;CO7@1+DPR)Kmp!q%d#KE^)VUFbOCF=?oMXylq<} zn!7ZpnnN-(Q;c^hP5n_^6~W zOo4Q?%FU@{Z&=pHj~`=`u8hA!Se?v%FSxlk{LRjR%4aJ{Ap+m95EMYU3K$%S9|MP2 zZ$=V$%$XC;AW#j$h0V%B%U|KbdEc?@x|@O)6#ZrN=hdqf1xt+<_H3;s*Il>|G}G0$ z>IanXx#V4S!YKi~H`0?-I(QO=dCqK`O~iM6{b|~S395Nx=Y9gnu>0v0-r9;X^um{h9&(B@EzUJr}WuGOsv+boF0$AF&x)Nh%rWeQL@1 znJ>HU+O{o}^ zmoBNpTx5!mixm49(a`*icLGWQ8CyjGNhSxLe)}nRm1Iq54DHR3w(M<*^7(9eI z^zd+sTerUI)HTLM90n(+Vc2(CN#jw=IcJ$K8Vc#mzp(UPGjGV)v4*6Ysa$bPhv*Ls zHR@>f9~i1hlQje|6X<1HeKL_;y-2V^(2J)JLPB*pZ>i|f&G}z%3L&5}FnYtPn#%ljhuruL&qUg0r8_?ak86ZtFIu>59sVMdIXw7m})y#~L4Zjt5z_d2@Yv zMkpuw4sdPh*i=*$d|o^6vk(^q%|EO+ZvMO=G(=J5fWcC8`rNBPQ;XNfN6M7tvK+0I z=PTuErp)bQ+@7(H@AF*`e}Iy9heWBB8; zO%5D9`1tA5>OgH_G-Z5~eqw)n`{8$Ur<*37LVpb=ZN?EvWTEWX5RVadP_%-m&m4ZE zFh4Ts&-%Qh^+x5!mM@ey(hoHl_^ljPDxZy>Jh?fcGK7!4yq1wpx+LD<-MgEK_;{xT z^9=v-X1m-QC`Tkh|6+n{gHzSte*@IpohOB_S+hxh+q3x#7Gxr4)Ky5xp>&ig5^K~=uPnmcLtmIGtvkW+UJBu!h2p~{MABGE z`*xlrlNAqarhqR|*zEj^3EC#8$FE+!QfoTq=){_0+vmo44QyD?R*Xe!IJKg}nl)U_ zW5K1m6<9gO9>{$NfA5LO%jiKNqMZBW$s_Xb+gT2I(P~!>Tl%pZK;x%FCUk&FjD0dx z=QlH5;C3I=2-3#5g=HCKOptIGG$``kh;L{SyLnTn33RgeM-K3 z!SnW*6@$T875Am)vDptAQ+03U^e{Qd@%SN34)CnW&;@Si7VGeo80mr|BlWq=d(<2B z9Z^e|3b&TmM>uM#y*hJi*IA8mijT9Hg@}e6 z4x;PCk$%x;wHpK4nT@?Caw+yZ#zrMX$-R{diwU)l@a+H0%d%D(xgP3aAi7KVOh*ntb0V^TZ zw^XJXqK3*H+?Vei=habNqtGSm-qeZcDDpgixIa(ruhq!;pN-$LpWWHjzKeU{h(+<3 zchC3sta8tjz!WXiApdn6VBaWuO{qN?iYdU!Cm=jrOQ0$SoD~JD6E|?xfbNB_UuJN7 zPlVA|V@u!v_}+1hN?8fMp%uz~;>1AQ>1c!16k_=Ezd3y7rAwDES~lSZ&DH!yPa+NV z`}~Nrt#oyUcSV1|hrlikfFY-*FQPjl1{_W-?#%A%}rIP(1hK=piq@l3b-T*PyzX>PD&Q4RF|$0H7thWCc zrjxVmKb}9}71@QsftqW2SXt%4MClqkJ=+)&;ZI;S29HXlC$@;|v^7~(#^YueL&g1t z@F42QA8SK1ClFnGTSkH$`ga}2)Dh|(k!{3iDZIeU2WRv_0zqfaEFIOqMVzjXir_pk zUB>PnYQBYEMrvDe;^tyJ1b~ER*Saf+0`Kn(ege*!di#h8nbkboN!Q7#0@)PEpUNxE zru5_)bEYuMyE<7qh%!JT^dA@jg=>)b#px2`OT`eyPm5cKWun+gnZ{@}H@`Yn7-dPd z<@C*pNwd~l+$6CyAv*CCShymmmcgmIGmuPkk58Ub^&4i`ALtx;1GhTMKkKx42l~#nD;+ z{PT}=70EGk?KvBx6!3LY)2>dDa~anM<#`mZQ2-e>lW309S(S4qyK4hecp71(3#2k+ z=+#itEW1lb@n5n3U_rGG!~*)#LA zhxhMa4O?jUqTe?sfw+&*DF3vau7rzVFx z`*Unl?ok%I_#~aE5Iv}e>SND%`!j`u$GYOOB6gwE@*NFrW09!~jOP;fT6lP8gq|&# zb^C}4XJq4p>v+GgcnFEdZ|gmF?0L6?Jtkb#faZu(-qUp|-2zQO)Fs@>E`h31qD^v} zZ#LcNuCA_9o*%)Ve&R$|Qu;3`n#&pxf?k-r4ttwmzNRc8HseFqt*_GeK-(c zbFQ{>)*jv-qafZD9fXs}ScYJz^MRQp{v0}i3NSVrh9)|C{<5OJf*IoeP91^@O)i=^ zokU6jxnoE?lwF({P|O`g3qGGXcI+ZaPHpgqCBO!h>`Xc+yhyVWu_(z?#R`OjkXfe_ zeA~tE*)xG!lz39~@uToFI&6FC$>C)&7D4b^^6!>2#wz$FZ2>}SVWaXXYhVZFhu$)Y zduM|6p8|*Hh|u>%7w-+n4n1%Y5XdR$!lCpGF#E9##39@^3|aR#p}(>b7u< zX~gAOJ08s{pC5GzPF1vlWY+^EFv}BEIus_`D%*xyHiTSw^YFX@DITp|9Ic6KI0R3q zG9-I}P|cBVE!X~(wp>z4jUBiegDZj5XdK}U(etp9vKGlZShOeVxXf|gnm6B*Jvc5b z=C9-XUY+)OtSaw_e@i!tQ`d(iWr2Z|Aeo4a$;(UMO*m*kWPrdBfgN&0RUvc>DlVHa z9gbsw%aMZz7WgkxD=w8e>Tz8{92y!eER(r=Ox$>O#)lS+42~@Xrk{h^mp9u9*Y>y% z(ewymkKevsOE=DnqpoV_)c{W1H=$vKbuktD@y0}Ma?*7nw{@{D_gR`(QiJD)gxG@ahUQT9i?OM z@|Le}0ay&KAe*dq@fxg5@6O@@tJ0zkLM$~f)}!atKYj0%(zN9<*Yb?nPyVC8>3gVm z`LN6<6g5w}M~9iWY*`ao#6z#)k##@306Npm_3&AfPxI)D7niAUB;IDuN2m-$7!yNb zN^oe>R?=7+#!UXC5{fgHdWn8I%h1BS@1$15P6|Oe_7I0HKg>n46L=x`9hQIfvB{tn zY(>Lj-fuOvnE&=&>rS2OA@hV#d9I}esSc4c=-Q&WbN!hgb^kUoO?-(ExgTEV4%xS4 zfV2|PCXvd^Qz+;kfg0`y$)5upyVDaN%z^PK;}%v@qgX9`(@+!%wjnKKdNWdB&_rFr zfjrTZZ$nAYj*^6;v>HH}=%ZIA^Hx=)kEK-+b=_6*Xuw#eUc4CZhT{5cOUE-l4T30Y z`EQ23(Yh<-XiE9JS|llH?+cn}VwyQ%$dD}BijL6rFU<3XrwzR5Z#h{UzM!#hidRpJ zn=}_3ozEUShG6A$5TR9?+ClUU>@fnKD{*XRl@TDGOYWxT|n`sB8W=hpBjc ziP_S-(yFr7$V z3cqyaR@sr)yVi3COP%UJxUd?fx4w*wO+JbJzYR3H7)$|{k)GfvSR-}7k{BeOrItOo z)TVW7o9^A%#>f!!9%S0KZW@lAnLMvuuii?8NaEz1u3wYyA-{}#b=}%*CofDatx(Do zPUclc7}b`eC`rAZT&ff6sU4v}KYvVyryYm1SN)OC=cw$6kfbI*p>4!YAa`F(H%#Db z{b4SM807I7wolF#3Gl-jHRm|KYy;F zc=IU5h}Q)VBs0{@0r=7TwPO{)wJ-@y?X{MBQ0cy?)T&y3nzy9#U2p+ z#|w3IOO?q4r%ki|KLrw$nhp4EQKX-pdH&3H#JF4LrXCvqydATR9w9zPn*>S|l|3}g z2OUZCmun0^Rf*cdF2_uLM^Pb>gOwk@;ODoUCCq_9DV z?}5el5hd`0U9q0&EY~hvK0l?8Qhxvob%Y5_4hc6ji+x>8oe!`jrH&J7TWm{s;H$8i z$4_kbN6RZaK7xzs#M<&;eFx;<#3y0sc=(cOT-q#D|aTVotXO}+s`~941^fV3(t(8XSQL|P9 zuYb-^1Qpe9; zW@s37ePQF=oSZ&jhSJLBD#Fz=Ap)0DobL1oXS&cEUrZVSwK}lUW_~pF?B>*Nwe_{Z zT+m$5TF_6lX0}(F7Z(?slDyOq3sl2jJbkJ+y5ShzQu{N7i-NtpyyUev<&Kh9A6w1P zu_`JKsQ7#~-p6Y5jvZw=A8pRHUbJMW>&T@M4la2OXU&``W<#kNOifi8-zk-uu$$s% zFKbYWSAE)pV}k|`RGe#dt+MFGEAr$@`gx^-n9Xj@{FA02qCucg_Za|0s>YF7}NYH_peVoyq*FK>E$4(?RO3j=zMw zd0f@Ej^@;S@(9kI)+KZ)}XAoWtLiy2R5^IC4_o$tqscO)f zb98WS^v3b=<2R`m(`KdD;bRZfQHKEo`qC;CYQ3;QKlBqm3y%)>rUQl9?UW9knh)9K zcHlTiw1_2EtRL%G4n9UNjhQ)^qAYYC3|)F|ZUbbSivY%nPuQvHmC1`9c?^2^wk+#< zd;98oC*$%HV@_$;*l=p@j8^}Kp7J_8BGXB~+bwIa5eE|y= zJ}1m9`>KthL1;>R-l=`OObzug+E8`Lz}UE9wATN&;)Ixi%U|p%_ML6OV5;jM)o0L6 zg3OSz;A}}sa8|3~Ngw)*>{jM1op-qU%PWmddrbyDBV0Y)UK^KC?+#^-2-eGKvzPs@LS<8nuQ_!8KTA$b z|IKVe&n;6{?5MIz0uF3TZ(BwtWq*xHMt_%TK#XfJxN&gXwDvWIG~V{&0{y1x9B-kD zn9-oXHPfCZ3EHnE*7863AiL8|J>A_EFn{HRzLhjbH3a0>nz{ZIL@rm%>B-ChE)z)q z`|l6C!y_S9c|{&Hq^$pAVxr4FU2bbgX5ObF6_50LpMfMkxg?`gq7CgsV%zDstv2Z; zjW2-*`a*c)+uRP*37Y8$j%dkd1c|;-5#kI{uut;F7`=G%WF=I)+!h?NZr#_JtXX;K zhLb+Ni8el8Vkpk|r?=d$`h{E^w$DXy4@PGT>!6T=-(m2ClI4Eo^)FsI&UsHS;-cA& zM_NWorU1QCV81(;{N?K8_n&7q_I;mh6BZgOEtc&Wek!f;M&+k)n5il%S2)DKKou-~ zZywifC#s|98ltcA%b%Gl;;sj-%==EXZCV`^qy{q0W{`+H4k0-Rm3#ETRiIQF9z}X^ z9X)z_Q%`hcd@$0s$Snhcarw3?iYoeS;ahQaNlbuQT5`gta=v>Hk~VY-{iA=kFf0S> z733Z^lCc6HBQ{B^Fh ziDOg?Ma7vqn}pZ|VDb64x3r~+rGx&fbudoV6e5Gu-qZ3?mh=99Fm(I8UiGi3eOL@D zS86kv?^=X&MjS?1&r%Vk03hpX)@yFuy?Y~~VixG*-T9h^CChrqy~G_BzHwb+WZRVT zHh1|o%H~d|4z|itQ=m{FpxNfmqGr8OZbKj@nHu>lH}$zR>rpAZRu2xDC*s{!>jg;# zBqLjV-_j$x}P+whsDUOs1TZ?OFA?2RQ0pW@SZL$4OTT z{?d=x+$?+|HPl9pyH&9(($^AM;w2r|~vwy(L)v;i0xx?|hM;wyim1FG3_2C5`7 z`LMt&&TdGzuQU9r7F+IW%=P~kTd$yr^ybO}mGc-CUp zj$4y3Mp?~&UYjWmVW>WK3C?L(uhyZ6zKExF6^rxqOeKq%10x~T8m~{B7*3!DQBYXm zITek$gUVlNnnsV95Lj;Q9DT!R@JPd=$nN3vvBgeuhqug0fXxfc%ab0g82FL(jtMeO zVdcuc7s7$LZA1-z`0$~$K>|(MGw?{&z_khHoYrmHK!VJJE=-5RfF{Xe1K7U2zW>C! z0ZQK|Ckg|kkCv*Sq^xYd_D+XjqCh?Ncj?`Sj!I=^8{2!AI(6z)%(weocb!p>g|(&6 zoadO|>w+>=xAOGZ@~c@gkpAPPuK|qL(#k9BHY*y#6J7#=86D^pr!qPJK55pRL-jLK z`qq1S5~*AGthi1g4mJMT)ysf3tqcUdB9wsD*Gw!wn7s;Es>YG5`mu;cDhNVQQ8j44 zC*@@K6IkA)y$QnC{zbt-G|}BaX7T;NsWr0>4ynHQ(TO?xT&~P|WYiLZmG%P&k3z#R zQ`5J|tbO4(lV{%PD7{eLj5+Y}qY_zEcTHv4XIv@QKi=+k@6w}_)6Vg179$Te;a%FC zdM(|2a;Bo94xYYBGXm9BqHD4#BR#_LR>2Kf<~|XhHGRBw!}KSRax6w4v$*i(RjYEQ zAH~gg7{G+CFgoo_n6l`tsq@kgARs54dPV;?`Z$7wO0SN%8);qE#e4xY?XAYl3t z6x_K|LLrUBwGQ2!!`n`WD9brGrkfPGWc{hP4&13RUUg^42QIbk`tQT@`x|{3VWCHC z%|9`x4a5HY>D{2LOqYC`S6HY;12kcO^|dj+J>hN& zf9HM5<;#1zSPnmwl``XANs8|rL{q#@*npomuI@YkoCq(>4UmdfvK*}&X_!S)9~q?C zJ`WogZ^Z1-aOg6XlCsnIO}aB}OPb$mwcQ=SaJ=D2navOhuIrnJ7W{b=X+Z4RB0tdN z?|Wi!y#4*34IAm)_11{3*NHBPA<)wgG zyTp6OLR*JRC77Av;@wooC;s9^r7+3-UTI=PL=_V$IhM;Is|F0MUR_HB7Vdfrz1C)v zR1MgCd+&3hhPU&so73PRav<0?l|$Z~m={w0PxNFvJ5_{OL00r9@JuSabQ!8#{*~}2 zxF6=>P}azUqxHo*J7UBw-xZ>nCjK)W4n_<8fTF5bd#F!3r3dVI)Kq8M9ZKk-XjTlV zH}dlK){E+^v^IuhqG;#`_AO?fbSQ`Az`}Vyt?)KyA!bC%!Uvj|84M?89`$IW*Ixoyxl&hyK?*ic4!Pmy9RGxHLGY zh>aWH_X+o%pI}I}JaB7!6#(wvliTv7;1dsg&U`*aP1rRPQ^RCxLGdvYcKrr6uNT$V zrQzJ4kI5;~ly*kx+UoSd2jPV~*lOdti0756G_l1f=J4fU+i^NK$w{ntlcznj+YpEzW$rfxCJ(B9=eZUlQ_RE1)4Xv?d)5 z+4ODj$nkv{;Uy-d#}uaRu^aOZZ@=lRJGS=^m5*UU$(tM#CKt$sZ5p*|^&K&yJ`+^6 zc%6d1h)PT#d0cCdHJnTbY4$}nTrynSRE19WBiyw>x6Jy!Kp7WfUgD#rXCrbR?QTi8 z8BeCiNezNMkEO_vRLMk4tge?#yi9wwh;SInC1Jdeff8o6lKM07jZs|(Y}vA{xb@c;s zUB7wrj*hr+tE?cS`+$K1FG`kUgd}a$cIewQEUA56;9!g6_6ZM<$PpxR;;dOt$rJ2r zH*BvEk(~p?yJvCK5}ZT%#B@5>I&NFjH_tkevQj89#y*1`3sxUvZV%HD+<`-a{}b2K zA6&iY^QAd#BSmlK?@8COSUbe~$TLqF^)hF2q#CgKJ}so7(OzlXU~? z<&0r=sv-PO2}4cha@C1Ejb@D>`VrySxJ01f=_WOe0Y1W0gk~m7FMw0!9@i(yI1oAIC!!O;=y84TcBJO=dC*} zl!->%IyAXpOxrC|yH7vf7|gcP#WIt#0ZZz{6)x2gc0}Ed9Xg~*vCTJp z!WW|HZinE(l|SLXGX5kFqt5hJx06iomhmaj;0RAgV~(C%XJoq$Q|X6)9~gjvYbeBU z2N&Samj=b#{Aqm3CaRMHXE^~J`9gI-VobuGJ^94E?Xh!$?EJyCySZ2;k){Qia6x)O zrUe_XzGmL0L+8#Hk^kDmBGNh;zQZ`wrrPn*YsvN=3=Sf65-gO;pupthxfg;{{ZkHa zGLLX@=Ms`2OVcm+*=M-}~Y)G5^b#W%4sLz#5p461j z9d*=9JXWk-#lfs5ou_JxH>qB|xsbe)JkrxusW&l&TwCNx!GN*biCO7Fk;8={>~|;r zb)bVpuhob*8x8p5dHMOO08hqV^}4R;QuKM%TlRqjGrpRHyliH-!l!r5t>C{@Q2=bO zFE~r?=5bcTn}~{ZWQ0OOH`#a(j`A0uDEr9c{`OImHngibPyTKB}f5WLV{TM)z8_ zirx5x4Oiu&fJa)iA_~aQm0NYFHdCqfho14CcJ;+~LgD#oUld(0u49&x$f%+Mq*aGs zaI3cYc>WHbi;}KS>It>H!a*BkqzO*SgG=0aR8)Zg8mBHlo5$i@_*f!!99l)J^FD6t z2~wr_$Y7e+`+pi$A^JMGv{R(Rf)C}wdf6UCri$xXD;!n=5ZObWx9nRN7ACXZXtWgf z&D9FnrZrHK(`^Q$-tfhx_j}^Ri7^y23O~H63cskRv5Xq&(c`{SWO-$ znMv0JlE6qo`8rheG9HCvr5S<|_W6rFO68nkhupd&NN13GkUj#LKU`pDdOqtvmTn@j z1h_l%(FH?AQAx3<3`e&4`eO4_;+6FIEM`8Td{<(LFm3AU>XoNUKL{rPc89SI;g~&Z ztZF;&sfJVlg#J8k0mx#F#M(hkRRh?vl@Vj$4x2A!J(icWKX>k26^zWxl$}ZCjFWKt z^HW%qGk}s#d}X|PO?ZY=zLhqJe&Qh}dA^RyiiW0@R#ItPg-f=gRNm!ZswG+<6}JrC zz(+LvLenynI^v|}TgntD9#3>}N}45I-3pN$IA-Ds6aoRLytC<{??Tcim0^R8>F^ zd2uHQ-I}5`Dg5}*Y6@(x?|hoR>-*?hqnF`K%97q`>D(RId?geN4_%!A$>PiLk#Pk4 zdFf2#{GpG`*Vx%vHVgH#jJ1NBb4h&5I5W(kf6do70IN9Rd?r4cRxKVdjC7r|Y!CpF z93f4Ne@{OS7x7=~_BU3EO>M_EM(|Zph>$kv%BDWNLKOwLe0$k!s(jYq5^E8@cS!W~ z6Emx&(lGvT>2k&^F%Vtc&hS94yRN#Tz$yFyTCw2U?e~xZR>BBMPaL}E=E<#=tYE0K zCz*76INL`&6&xvQ?Ktq6xX}~@gc?|B)r&s&%VnR|k$yrZvj|q8>$GRqoLdJPv3C!Z z@_Dz&_6sibfaE^HR7OEcZ1Pp|3Oi5rqmc$fUi$6ZB~jrO>(#mc&h3SV7=dYj;@uu9 z({DGf3Gx@hroM<1+<@y{kZp4P#OH3d*_$2AaJ{Y?eehbRBZuFbOqD~n!^Kh``Z{)~ zh4Gd@=Q6un=w&_tre!57w|8#~batka8C!4ppiG{S38&y9_G(^Kzik~SDO>31TU$nI zLOS54FYI60D*|v9!>9`il0@weUAjCow$eGj1zasYB&OcTb_ej(i&Dntp88qtT13;B zs3`As?9++xPBQb5y0Z^rY%*R#xQ?fdRcrV$0OCdWPw?O!7(xS<0A!zA(Dwr|U!}a4 z_YCx1upk&Q6>SO3UR|1EmVfyC;?0|#FHWGft11}Yd)O_-G2w>W=T^?~8geul24WF; z?kZ<%S5GgM&Z)hg|7gD z4!AB|&Hzp(4C$hWFIjc2s{H}s^4jZ53x9Y4=%}g1zL$l)IxDrZ^7r^bU%an5bOU}F zLnz8yWNrX=F^96y?ldN0FSMs2PNqsC;N(;Li}p;5SNH0JOS^K(4rdT^-3o0Ej0I?_ zmhacn7AO7d=|l{sf$XF2g_hewKCI@raC(#=rlwQxpXxwPTgQ+&X?O#DeQ5j(E53Vq z_15HS)mFM0rIZd&a8^-B;mh14nI=R0lpc1%Dt|FE{_8DI?!%DUk@*CE&igJxqrI=4 zb>U9ygUI z509)JDr4)+Q7^0f%{1caX_rZa)?x})+`T%C#}f7n=0FRP8}HQf((QB9ODkV8rJby> zYVF#q%g$;BT-e!3x0$7-CK_%QW9hT8Aa1g7Qsu?0#Qn{!js8$3Vn<_Lu<`v}J0#_` zZ*GXqCRk}9S@Fg$zhSUF#NsfiUA8&=e1K#Gh7X(8VS8P@juW*~hTbAId#mfbtVvM2 z`p^E5f+xR9Q-=WRlUHs6GA?;w-Elzxjan1#LUta8P~4W z<hq(!?YM=3VAJblxDO#GW-e~fXXN!p z)5b4dYAx8u$45#jiJJ&uSrA@cL((MokyT8puw#zy|tYl=hM6o8?nZqTiffA5O5hqqnrF-FGhEy!=s$4Qr-^?MeJ!Gq9nf1 z56g_^%$&IfXestFP}4=;wo|N~noL0w;e)9tsv}3{_6%<|>gdxJ3uvA)q>PdAbX`u; zF4B7Z1-DLX?sEoYmIk@EAINiluZMIca@R!k-$f=6@pH*rS69t~9}{`~eJM*E2X!-g z7HFtS`l$c$yY#U={2+Q2cfCO8)(1Oh0cyHL&-eEJyLMpZ7^YS!^e&(9BHJnQ-oA~R zvh3xKZl(%2n(A_EV!iz@U|M&_3hN-Hf2UN0L%@E^I~Qoh*GQP6W&M9u^???pQw%|BBO{o z(V{U@5eK7kvMo(=E3(!!l36f91-9?f#U79#Gg_%17hP~*!jrlJwDzIJ=T}d<@7CI~ zX;UDIoChVOuwP(+swvozO)_av9Gy}rnVYK#i%Sa6`}f~#CSEswgUU)LbVy&lb&$hh zsd%IbPRq7j!6a7*5T$YnGkT4=DTyGhzn*9gQ{r?1e49pNJ#GSf?!yqg*vx5uw9WTI zT1Q{?k#A)$*s3ivZZ7l~il8EPSPpgN(^-X;BV4cF*&?%$VT}DDNF5wBSVl#8#gu-Q zk<7o688qyfJ$cS?;Q-%;QVTH20GS`!HhvLrQ(D&raOi1XjT}q7^ZtYCP8!r%wg#6Y zkAyh1MuOqiMLp;o;-|xPnR7#Kz;P`zG9*-Pj4B_~`R5nbR96ow%wGg;4EtiX0-9LW zNVSIMi|(xs$gZY`oGmTmPvKBgnfjx~ar%4zNG-s8(Wg%`w$v%frDe`(rP>70?Clg^ zTh6((h(J!==s^w9iPa7Sa{-7YL|H1Bj1YEAwU!DQH;+FXzm zz-zHdUaq~lu#c?x=bl3C2;%|kE&iPHbR8&RuuXF|;3oBW*?u8$o54sM% zcw%hjWWq>!+0!|Vl<|8-r$Ka8lO9$qWb!WSgpS^E*zCv4)S-ctEXLu?6n6kja%<~g2ctJnkB|R87@rMdvzzTAh7)|-(x?9) z^eHb4`WUZ%KeQR_HGWWg88>g%qa13Yy0_g87^xT+%VeS=r1#Rt*sVp1XQIwuP{gJ; zmdxoI(4u*B&9i!AT07DxN#+ z7UWn>DCAqq%bBMkOfcAesD&Oes3P%6U1S7A56i@XQE><7UwUDO0)9gVZwP_fVhlS= z^hFjO8eo8T-H2;X`u&;<6{i}{n*_5nTCOG>h#g5a2N}Giv@{l*6%8UafnB_`HHa5N zCH(VGWZyhxrCiA%95#`YIAFr^P7%Ke91@q1(y-f%hgn$oP^gZci;EQBF*G*m>WpEwj&_(M zwH0O=nsfzH5Iv)g$&56*ic=~+8Q@$L`dY{52-Q(OCo`_ou~%ebypa94nWa6&ibOUv z=`^5>LIL|wQs7xuTIxX*qA>ci|D2<@?C)AnS>EpO7HUHpCEK6na8?9Q4vhE8f`uq2FB+!Kv=-D`~qH2@;eTI@t6^ zBWcYbf=KJAOrsE{$ms%aCLivIXRze+6*&4nyPZ6H>QoFOK6bMHdp1 z=vr@=aEF7;gz)ri27#?fRo|(c*7NP%{Q2J)GrZ}Wv@%s12E70UZV(+LJkvu z7JFKTyaomb%kVUMgb35C5JH1fLyyJdMra79O8qvj;AB5tpG2aAOyFk7nZ690lVf4= zh#(=;7IPYn=GMbH3M%WD?8BloL_ zbj>`(0L*exLq24>o}6vS^$e@%cj=AtbIn9|HtGZ9M}15%H9@jfp}chRj2ZhIU2Agy zM(@y$Q=UR-rzzLwmQ~(ig#kS542xRQl@?@KuRMI0x!HQd(c6u#)N)Mp@>m zQ!ulZy3esKh<}9Hty@mcK8})hqXtHMvG(&#F^-s6_iy!ZW{1che2d&CdZ?`6ioqHK z?qWK8aR1%A*+~cI?-_DE=z{DV#_m;-lG~V#k_tXS%rC$uBno#6otI6BQ9loGgFgFQ zrmwo7uc>xu-1#tMl1=T#H>F5`IzVeeA`bLdX}8BA{!d|xh{+!GI7+9bOf0qzE%^N` z_e`+p?ctiCxXALRreCJc>8gP*Y&QpDg6A=LW(9-6*O2+2-nEL7$)iCFHNJXId%d&x zboMSBSaFDRr2Itf(S$zX)Tz~E$|P)pcir(A3qmdVFbeK59d8ZVx7_VP0w>sP zVBbkZr4p(a71w3Xst3>|J6_DE$KYe|nG~wIJw}B(+ z0x8r^hAxtk_3dB?{lPB?07CS#3$23>PyOD0!EG212`ww#+GY{qL?GB?Ib2iX8Nv}m zc;tSK>gwtOS4lnJ1y73E*49xWHff|S8C8Ns&bUjL;aCW!#E#f62L~0&6Vkm(KL#s( z_${n{n=mSTFrPu(uw$3#2vK-Zce$M;Z*ntGT{aWdziz~y-;XW0R^m2QlzQl$p=cze zdKU+|z&mkhtSX?IVC3+^>pwK}59=Umh--|xstt&u65q{Q%6EkUfi9QHWYE|=MmnZ) z873*x1b{p>hPKgQ$avxLmE>0z(!OpiGB!LgE-ZRoqJ%u zrN4^73o_tF*k5*oT=Jxl`!nYF?4+aHOgD7$>W?o(F-ZL9)#tCj6Lgk8=$69MeRz2~ zy9FFDGFb-<=*qnX;wOWbo_^xF7XE$gMLJOl^HFw0vJ$}x2+9TPB1%e#bpK0=v zs!W|=t8#RHz)_{LFwS`&t@bs8>{L#>)#YR58VOq6AZ-F+KAF5nm2v9vj)Gd;gu6~m zm8pIzY~WIS)*$j+(_{stA#LynyjJEth6;%?*$! z9S{}NrkzanTDx}b`ok3_+=;M;q9p^Wi8#UI*N{08*aUz-D7tbg(*}?GGw^2w^mc^n zA$}w^D#sus_YzTgeIK9QxB132rXFJW){h#Xw3fDa`a>Ep{L>u?0Y+)Cx2}`Fj7*dA z3=(^uR0f(+GLabO4NBBv!~x#;L755lf3%%FvR*v!ZV%GD=0j?@s%_h7>5HpHClgJI z(1cI}o=}Al(2Wd- z0n#|x3{P^wMLAGze7E0Q5Q+1`gac@L>9Y(vL?y)54|D^!`jHgD5vFe(=?uZAFO zt7)RT@ZIowKjNY3mzhXB(u*QiRH9&S0Jr!dm}yc?#5Z<~Fy7?M|4N(r+HmN(iyrv| zOAks((PN7J0DK&(lIOYdP6ywx49m5uPPrB2%nS**fly3PVS`QLmOElR!tB#6pn)Gw zAemJ!bqP}}iBkR$i*lpZDW?4?j6ph58YA3`CWNCucH$o!r2mwgpA?GQM<>bHzRs0z znOqizDDnIZCv}#<$z3)+VT6>ajd_j1_<~=4*z6TYwrmD$C()vo#@G8r_J@QVYfe}p zQ9?cN7uyt&xQa;{bUBLRyKUeq>Iko8wRH3~zM0gXU?+hqRkDDjM4!%p1|p2~V@lf) zP=J4`qcd#~N~nmVZ*C{|C7wUappX<*oDyM;@A#{Gedkp7MXZDOk*+Pip!DU)3>uuV zvj6F1mPSSCq~so-_9)R}4gfUCJu&2VWD>q#fGBv3gFJm=ZDrYsle6yHpzN`y!Xd=O z{~Y}}@LaWS0XOea3wW}^qS48H#1sJWQB8i_$i_7zsvG$h(2G~IPuj)yhPdPu2ebDm ze5Kxm1dcr8WBgq?bVf-pErM2Sh>|$4Y1~Fm`LXF(07~d2 ztHOaTm7BD9Zz3EDAwVQ~@n~h9f>0Aq`S|?ILCy~E|58=sA4Izhm!gsU(5M4k8nqe$ zcsJ86i2=D|!Yc~zJ?!ERzYn{ec)YMj3I+Zgc?jCv<4CzA68)&YW;X_p8xZ+3+SuHc z&iMc4Gcl;*qY^W{T)TrgH{e@d;%)#nDnUxr8i*rnsYl7tO3gEyhcWEba{H zdIudhQEg!w*DioyO>?=-AOn5B>jS8>j10SG(V0O=yY30?Pf2<;OI_VJyu9N5tEEKk zE((d#(f4BpP^lFP^~4DRCH20^fCT5ct*lQ+_I z!=y`P#|H1q4jBqjB@P|g%Dg(65h;V@xFUuSvfb#YFjU^72nBPwUJp`Ko``uGM>!`a z!?45P;(`teK^c@zD{o!so+-J!4Mr{vh9vZf;X`G{8Av{E*oPa!c~xjCs9dkiPl2jZ zT<&y?jo&#%OH)%oD#uh!vAuGa^y6XZ(n80^%cZK2*;T-DVMi&H7jwbPb1Ur0{pE|% zt{hNO-*yxy-S7URRynp@+Nw_s&E&a2gXEf;X}?OWxOOhMCmG?={{J7WRr%&>i@4Jz-WU)jxOD$8hG|8ZnsI9G%mP8V4hq-?N-^p7XfxVL(c3wZ`CF2N&>vtLTk0F5+r19d2@kpazKH+#+;EgaqP@KKg*0N8S97|7LDJ4Xntwclk@ z5sq#W&(N2Fp|o9P`u2$1zgHm8ky;Qa10sw6Epsl;AJu&wN?C+>0-2BWa`cMToDACM zTr#?DX!VA}p67VSfJL+2xQeXC5KDxQB5^xuttYF8QvuW@%@vT>5^?l8>EHnV0KN9) zZ|~-TVFlIX+1sa0<=tmpY$b6q8L2`QTGadY&8H}gSL@9ni_DsQo4p9c0DI|N-LS{58E-p93X0iN( zhC7_hUw*K{SF15zj>w&k@-d|gyvtOLjC&=3M}g}`FLLXP?}i0vcVxv+;8y?(Xt*YQ ze*5tgRt@T(F|9Rf*q;Sv!&gar#*P4i z!A2@D0K!E7W@TojIor>$y0nEZ738_*9#qMSv3U5f6)#SjYKgl^8vnuOOy#G!GU5y* zA5>G;rVTeDQ0iZ)hEPRg%SKNcLV92T;$r-m=W&P5eG;BUjp&e&h$2TlvTBPf`gbmS z4}-WE=qYDk_g#I6(^U^oknWtuEZSoF_P1H5tNfz{c+>Rnsr~I3aTdsDaIB|f0Kh!% zuvSl`*ETdlnI<#dWLlC(_7~2ImDpM^DJvwfbyHkD8L4S)Et6g z=06oef~uJtC6$SmwYyrvhZX#Sa0JIHMry@3zy89u{IF3(ssaQ9(lhI@o+~QLO?ef98@Hpd`}vvAR+NY} zQZ6n0hi_+5Fo?>c{F6i9tZK<0y*RtF0Io!n$*(qwQ0Bj;An}qHrAGewjFIPE`h7rz z!0lifbzI}Vi@P%+RRc&d^EKYf64fB$Z&YgyGL_r5q2ad%N^&Elo#SsT7Y!wt>;%@A zYbeyy+t1XTm%aqvg?4%EKh09lqAr`xib9b@SD);diW9V5^6aHy(%7+UM3E<&O~4~& z6Hj)z$46;76`o&fp_s?FBcCfa2ICc63vn#bT9YM9_F##V25N)a04%&AALrF)K8|q~ zrp!Hy_IY+Fgg}vUhXVA&lmZZ9D}97@OhkBo3w0M1^19?OCZ|v+O;vb_Dv~F=TnHK9 z(-bE$4OtP}3lZ6L{>kITh$k8I#b@-S7+rk&786aw5stOQ_`VpxFFZ!y`~ zQ);*D-<*F<9UYB5A}&YA-;QJLF&&szYQ`+zS9jSdXmgfOdJ#HYid!M0bK?M@%5Ak_ z7*Nk_yQjjT@UjsIE2ULQ^xgksaVgA-fJvY#p^^(1hGJ$y?`XQ^xv$dKzUv`SI^A7LOYexzV*$f!>uZ zvtx@Ef;Y56qdSf94)+yOG}0jq20UR?u03ims3{um^I9mBTwYq2g$tANdkExYfqrY31}DyAk$A==sbo1=&bT_}% zEDJ&>EddQz)o<4;Bb=liZ5i~CA^_6EG<2vyB;XdJ-DUuW6{nt-qs9JlW+S4;K@~{~ z*m5XfAQ*L~q`qGk=woJNc+gll{&p1LZ91Ta9wjS~4r0Nuxzo5&0S#w+>EWSd!E@7* z@$~V@L5lFtyG9`!y!a;jJ;+P7C>1oRvkSOFMi%6GOQry!*hMZvBcbI zeI2;dM%&hZ1Oy~Z|C{-;dUph_PRc2()hbXo zZR!OD7rhyDf6~vcCmQ9>@AG`vf@%Jf9^4$X;pVmq-?N;#Rn#sMik3 z4?d`dXoAyy=hbwNPj>C9q7i#~D``MwS^Lo|b;-o_cVJ0pZU=s0$K8<~GFU@fu&Rth zEROl)llIJdzW04!mxT8P`jJs&c!*b}hNC`wCV*LnQa~_HZSE8_6pJ`q`4|EUWA`a5 zcIFgL#OzS-$;X9h0@3P}JgPw<*;21x|9HS$<8tI``bfOx)0`-YYiug&hHzsR@A2c6 zjbx<*?La6^*i5Z2$*s7v4$xZ)QtT}nNY6%>W7nV4vOg$9f~1>ma5 zT@hPB_aFl!kSK!x-3KjP61s~4B4|uSibH|o;hc4cxMcgW+UN3(v5e}#H&B1G?C&3E zie}_(N#`h50E$U+Y&duXwHNs>Gk@ewhY|~>qaz4kzkVpnWHAIZQLNu_BND!laUDD&4K#HCPT}p^-C(D(%I$=42OKCE5s#?B!2$1O zz;l@{qT`VX2=6{~z{X*1IXG!lnNpz>4|WgPx9R}9-Yb5*dRtgX28c-?$Th;o36H-P zaz7($QkBbQ5JEV7R67@!C3_+I^d{7Ff@2C;gxP7TOYmKOFus`D&x|q9K)&js#doHG z)Q=0`EIF2*MLZ>qlXwAKlStL(-vSXvY5*Vqi_-72rb^8x(^Aqu?i3i~Ri*}+f7{?t6S@ySuvs6RsxYN=QjXI}Q6ka%)Wfi=2QZQ38#T|BQ#c zN2b)C3+w$gs2_Q}Y*cU-sp}xeAE+o7IIXd3KnJq!MGn?zGfm!KWd-GBb)}C(cRSwOT5x-{YY{a*Y{dQ;GT|H78(jy^XtJwZw{Oun#PG zz^rBnXpr@5y9H0LI{YW?r>w+x`|?i8kimjQhLXp2PEA|QpU}VBGl%Le83MF#Kjd(Z zfZL#;{&4?9F230!Vn24@-I05U*=6IA6QGIHj&ZRRgz{@uhe*FED1j{#(I&;K`ik{r@@rH=uSlCvW!OB!xgt;FBn z<5II$tu(ZeHrMsViwC~?s^sUK0_{`*4kjQxy8d$@L*(1`&GY@>=b(>BgHkULnn@YO zuRkswAAPrR-{7+5@Fbx`14NeRUXQl;kXa<0qoidk{YaitSkBX@6v(Et2B2GT?Uf0Q z@?5`s`LfP{jP4K~NC`z&ao8GAv2bZ5Bcy)d5hD_cezD_TKCEIz5p_ugd>Cv|fO zmZq1!>2mMw(~ z4J}0~Xf9ls(~gcNPO<(lKqk#iadV`QtycHBH7EhC&>I+~eU(3wgJx_)NhDlDd=3YU zLOr2D(Bs>$ARf|8GkyBpZ3W%%ELV`?8^y8-7BR368iUjw?gd7D$+`)V&%ElnlkWFH zOpNSR?pavbmW=v&8O6N>v)nb3>gG9fUwh~A+a4Yssv9(eg|gRpuhi0T!|5p;f33Wu z`yZe1c;BCJ5Dw4;*N(lnsZ+uj_rpUcJ;m%Sqp2>GqA2lEPzqv@e=37c@+8xsAF!J0 zduV+wgcwE4bjPp~lGam}_M9_KrG@eDtAgXXNLE2%rcbU(S^9^_QC)lDZl@=X0Hgc7 z?1A6~V%-6OHU2?kh0txebm8aEud^X%mhD)m(=A2P&Vq>!4{-1b9cMPLB3Vx5>qWoL zGGtE5Dk?D@O@aHUD4Eg|;sr64x9P?3yGz_}5txtPzkpD$s)3QV&IVbS6KAnEv@_s| zI+Z*<*Z0Wu(+<8!J!DS8Q8pN;MOIP|aZB%DE4Hlk{txisLJ29dioAZV7Q$|)6IZVu zy*w^i9G?2ui4%W*W&Z-!A|W{tY)G)xgJ6VYY3?ly9NJUM_P{#{np(WryhV#N$X7N$ zy8PYbyGv?MCcC!8S7^i5tq9;^vh5G3Y704Js~2TmlZcgZ|fZ5!_5hz@VgojV;XV7!B` zyv;6`NI1rv3%Nn%OLt(@CIvK{6zsO{VVfaqWMn5fgcgERSpX3Yp%-8x4R-j>^`C=o z(}^P(e^DqMWA~Py?nzqxCuY`spA|GjAp^#qe248H;5#-)7itgcN;wA7IIKp0(wbed zL@>z_8g})qVZbAe$+p4NBE0UB??0b&{EyO%+#ZX&h-!6Rn99ipCG_zaYMn!PN~Nxx z&(cuTB&7rttgb#-|<64&r9s7{1o0b)t1h(5wT zbkIw^CV6^lBbY`KIah-LLDVHkvj~HeI_}%9ojadoS=0v9@ibMfk^gg#l{P4_%qwq$ z@eonyXqS!ofV+5o9&YZ;ebY5G25i&lM{8A4T!}Qwi9`N*9+|Az=Ah9}xBwVY4sIpG z<5*;`I%D?koZzryS*NaEZ7l1`(W9#uo;Kc~saibU^I`CTr9a2alcf?5fvRU9t&~B$ zk1jBY=PIwa`rKv)7`73Co>{1bG_V7Dm~}LX@f#4*Uz_cq_tYhMKRS9nSVJWddZ0z@ zKtJ%+w*(u;3qwF!lI|}pnN%=t^7={!HSN4@=~78^*KbF(Hv}JhnE}gMg!O4)Fw{ZU ze|YZI%&Pws$>zWL1j5CHP=Jep*q|p@l$YZktD+g3wDK@*Jq9R>sipP_lQoe?+7+H| z#*UopnqtuiBw?p2+oeuWotYoNZ>(T`ju0}HRZxJOggpA5xTL0y8xQpJD?dJgMH7m8 z#8ljtH1Y49xjx<=w)jD)8O$k3gD>S zI$oYN$P`l1!>xmxog5GuNy9a=eH!|1ZE)92SFMJfPaso}4HiS=;{$Sr&L(4#z1X93 z?#O=6Nd#Cz@`=>coShGL@89oWm|ODBORt})hp$X6G@E7lx3{FvxlNc{y%Dvpu%XW#EF!2ANHqdK;;w%?Wt!FIi#N3np3a_b`OeFV!}b-ASuj^e(YR;(Lxh ztoLg-$^&B)!k%_57B;o1+Q$;ZoR)>FQdm+_e|ZkzqjQ49%vJL~UPDWwEm78x-LJzGcOv!)boiV^}_hSzM8rd17dgH5-Nugeo?gwE6 zN_~lFQ5b*w?Cc?$0|f&AMnf z(Q@y)EkpS{33clvum9n|LT0Qakx%&J51I|b+!kMzZ+#rN=2HJ^bDM44v*;DJN3aIA zGF?+ndAqpO8~+xMUhN8(6ym8tkck_+DV|k2Ne^>s!At9 zt8gOdR!jPOr=>bMChhlB|CY(DKyD2Y^Q_EANzfZ-#mfoNlOST z84KFvdT)n_9ww4lI`zlI>M2B(8g1IN;qxc#U)nGKF*xz8H{aa3YnSQ%*FQzi9b+~Z z9eB)Y0n|oUuP!V4GD=~^XCD6wlQ>1k3ufZqNasS;daTFA(&BH%A81(+cKyJy=j;rU zu^BXVRQ}c6;!cEDj4v8NNNJk@GieBfj`w^Dr27n{&XYV~&S6=3l$0z(eSz-g1~dfn zzrj=viB2_m9>!0EB|ZSews>(iK4V<@z({N7;g;Q1#Zh)aZXa?!95&)s>)-SaHD z=&l}zMPC3hLlBR$|0EqqMlhqqHI%(jFzk{33 zYPR=_BP0rH*708jXoqznt49*f{~B=j=5YRz>VO|^L{pfy!Y!6h%KIMdvgI7hZ>bb- zt^es=hLrg1py<{H45g?S-xkhY91ya-EdGe)+5IiWTXuomxD3`&^)n#7=xT=L+ID;m?5 zC($^-gP#OrB!3wA=e3fD=@;_kSNX?+F}(=|e^;d>xet;rF-2;l83fgmo~)y#@FVh1 z1A?k1_Xym$!HmN^Kg;8WxCS&a zHsQir{M9Jrsk!!j^$tl2c|$dWW-_syfvwYyI)j1K7)BukIN5_jMh;laDHO8pneC~F zs(!yy)(wZ=MPK>7*SFdMta-uf?c~3B|Jt}2IK0wn4IDTy3niW^tdQ-d2dvRa+x7fFy+M9 zR-WIe2mj*5<;h$T^-5d{O~?zRd*0E~M=ud_ZzcEAmKR6Gk1<;KCZXKFCv#YuB|!~! zk;t{Zw$4K;Vf_y|aFp(Q&|lmcmSl(l>H*(Ir0n zee%c1;amz0k6{Y>C?*)>1yFU%@o2Wb zXtsz=^o~33Ue73QEPM-z5yMBW<0^NXK^{+M;xmMQuEOCZbHJ0l890GKPs}1Esiby5 zdR5CX@SS(M+S}V33{v0?A6`=!IUuJYgzJk9bPQX`nTR>%j&l5JuBIaT$Y&5~Jxl=Y z$b1&Hv*%IpJ31n2Z^8ThAOG3&>5Z~nqUV$$KE4w%l%mwAtGtaCo8#BDMBL=I8vDFB z!P7H@(q8lLa0%>p4`sLqX-_rwU4!_UM;)8&yo{oQu^+JTqR~&ANHh=s? zY)i(r7Or(p_X~b? z#0UvV7+U$XsVTUhY5&fhD+%N=4DzelmxM$rZI=z3ix=>feNKN}JLmz>8~F1xU+h!T zNw(ITfwM`44aPXA1ji5#Y?@||Z{BR{x(D&WEeB0>lXnI|h-hR5F({AJw8wG7K+c^| zN`N8+Q}+A`!B0x!q5^CJL@Cpa)1u@MVC2O@q|m%8d@HRycl5MshcaHn`aXGudUF_(EEW{+BJ9WW~K1Eg5o$?1enHLj{{Nx~Fe;iKBPxWR*{_=QqR5l-xo`SX)_S7ziK z;>#{9LVz0XfV4L|Q7QvoV~0>xd@xqGEb5IsA2?SAMZCfwmXB)hFa0m6kgbVBx#D;7 zw)myO_+V8WB3i~5t=-KFE3;$=;O%{>*~+W~VaezCutJ-hYEetq7O|f}E^C6JUN9aBJqn{7z7#`lDks4052i; z=k2|`y;qP_g}IxE@q4QWkqbK1Gdnfd6gSEM+&!W{8lpiN; z++sqb?I)ov2fFp9vnWT&+7WaED5#7*iPp=>wgL<(7NX9_7sACHf}D^njoTp{n>!@j zK6AFod`hw$tY^h#@&pv}q$%h0ny>ic?DY}`VzZ~)OX84$B(P7b~CePs_k>eWB z>HJQYsntNh;VfHr3mCN<4c%Hnn()yIkA!Sokx-gd0q;`C9wK;?0AA4HFE zvHIVfR4ti4MpJW7vrJJ)Oqo;uDiuqt|sfZIR$1T=N@+HC=Y zW&VpE5N%lw-|ymg-wj4hKv3ay_j)LV!uAvVCS9a(Gh7RM1QrJnI9ldWLraZG)YE_i z8v`RoCHOy~dv)Sf*V3Ecz~g~&T4Jl!8^OnN8ONaq<4^m9dg+HJC;m0*Na-77`HmfN zgob?}bZnLaV|U-Ybt_jE)D%wIbS;Ghidf$IYIs3NGOK+T9Ea&YeI}{kg0z#En7@~X z4;M!o78}823#DFF53xuI47&|ums}#IHj3QD`%>OfIyYuSVGxthHuIs~`ux+_!+2awl#KD1TI!<-djBzLKSN&Uj@e<$q!9hJKrj zHiYg}5HaAklxH?0r_GKxTvn(!`4Y;m9y&o{8&Y5{tT?{Itybx*%yK zt)YZq3~6w{mUOGzvcK3OjrEhlXgMbaIF&{)b&3%b#lR~5KvCn|x4xrH&Su;Y)xIbt zvVoBEWvFORe0;J0kXbq}!$FJuWE%^E8$%3mU0ZgV6MK*q;Id1nG_}!_ObrDM+8Ul8 zELSghS}H5?BdB#__U;7$tn~yQe(kk8b}#+&9n8P8GeRd^9xV_wqQyo@Fyeykq*Q8m$9XWV?q#&7_uQTG`Q+(DsTaBkos3= zxx6e;CU@}k3Dbw0^0EM_wWm&EJJ2SqQ%>5lvo$gYJZt=yZ#Cim;g)2!%l36XoHLxk zB{oE8=d5kUfSsw_H%Z>n+@#r(ml+BKlwA`5$`c7gv!lntCSk z-^j^HtrTqa?@aGYN@DPcp>AS{Pcy_48JA!qL={s;k{g@gxkl9@&4XMeDZi4BAK?%w$Lnry6;Lk+W2SaiJP}6+@tUb(W)(;lP05>-R4`w?%%(Tte>ztiTQ+@Y*T%jUQOk* zkW7w0I{>m?%&_6FzN$WxQbR>57>!oSTR)Gh&EzZ%K`pJBpc;`dQw~`)+>{T2rcU{>C%gBS+;C{>V3VsvTy6WE5Jiw#VQQ~7X5NV@E z>ge#|R`2_n=1yvHO7lUkm_JD{AmJO`V?yiu6274F;Ki+?ZsRXn6yki>U7PrN!Jy)R zE}eLDQY#4Mu|*vBXitE2RV&PbYd}qdae$l*#yM7cn41*`s~BG`c{=N0L2J>y5=w+r zp=Z^$*?jXqcgyN6KW_CNFvZfz_gXe$kZKXKx*WgJ?JOSq&}fSFoTAp5MFc{~$f%gS z-S=zsVOwrKl{*bZEGG~PpRoMq311OSin=o9Bxfc$1p^l`4_ujKAcBlGC9YchM0{~t z_ph{_T7z1R$A{O2ykE6R6=J)hTd75h|M4Km$@88qnvWIQ7*H_!|T60j;O>*IR`=_+2Ar9m~Q)3k*wpsAPTOVTgc zxy&T6uspEjhjtY)y!o6 zL`3Y&{*i0eAU3}0S1oxd_R4a}`FH9ye-pW_q1tLjmE{s()54l8z8qe@TDT^$A)(-@ zw=*YGUF9Af#}qz<(6?u(rr9PRZyZIW!GhzNMwM$_5H|;PQD!vB+MaIt6xT`yp=1@V z)ij_$ZL`4#(DZze6C`La7C95`BxF?zRijS0N@Ab$i?c2c3~=E+-2p!2n6{!pN5&;t z@?zsF#ZmO5JaYk?dSmEcYYC-#&19~L9yQ^711MIu8P7Ymf5uyD6&Bg&mzTj=tw>MCypkb;} zF$H2rmevuV&#=~y2seI;Go~s*1z@4BtkJ>K-ZGu|fY+W(aOo=6@aLyV+||(0r@K@z z>(-r8fr{I&6mS9y$VIHM*!NZ8YlN*Sl|PBKjUky}GpHe|DLxfdR8T*Fi-QqWfP0008_05#$(1@xrbegHnkyJkv9{1XEpIa5Q9mt0h2>nZ@)raBCm)dgn{uV zP5jAQ8^ZHjQu5o8iv+FF{dR{b&m3;9ly~cv9MzL4NDhBr~Bc+t%*DcqY z20(Nh*Mj7^LcJK;v!7`OE#^!a_iE|{l#YXcn1i3Njyt7TO}ib&`X~|c`S~?;GiEz~ z2$o9AqTYntTutIW0(g%(mpgo17D;cS$$IXEJIIL`NXX9!KG&N3p`%0(zchj*QP zybz`)I41`qfzyH&vNiqW*nP!sgQvTaMW58p-pr#mk?*(y8?*YF5YFnf=YcrEu%rM!OOTPSeQ!2uvXBv1l(VkR7-8?Kg;fD9_?}mz&4n$)n z%NLp>ZhJMFG|>D#)J#YvyJ{NQ*t}ibKMX=?5Nf1qDdp<1`-BI;ODS>OuPUZjgD|1G{oVb=*yIJU}YYGw5iyE$TpLy z3(cY;X%Lb%pI#i*Qx(}nRrqxM^Gq^4p**Ego`S%K3EnVj4?vENu2IGi_+}2Z03TpD zZOjQf5pRy)yLUR#Tk)FF4f$5^5mUJF_(72sHsqVE9_iRa2~?mAGZ+9KR z%0lzD`R?-)zCd+x5DlU_DAGk5sB)g|I4GYypp{ewMIJspu@No^%& zo2YH7NEJ)D0WZCrx<`kE2pSUIueCMdCVZE2h!n(t^&E?B@i@Wqdu66bn4H4xK-uj? zkJYjRHWbWba%A0wuX98NJnsdwfV7tPk4AxEz!htN0MrcWx)N5*nuxDqZs*GXy< zK#oP`CD3)RhWiHUGn6ZnTs|!y#P6Lc*Gu7=L6SU^$5pc2NK*F_}ze{1n%M zTx?DFXI7zDp-Mp?Q-#aM%ZfU`(_>Ai6=Qa@StLS}KJh_wSWmyndEnOI@WVdAZuhCLA!($m=(K_@z2M8j&r=#}{k z>lCWH!9*FKPC+bvk#)s0)j@uX^i!u&(Uri-!-1c~;-1_pf zyL=^a_45&Xuwc1$D~y~O3kM=kIbNWRh?{N7`xw-d22CTGpm|!ZPQf3XI0@`;ep$(T zl|s!`|AZs-SoLx zdX9|}_Ecq(>mYWq`3(#)zw?Mc4OYQ(+_*6B_>!N0ehEyE{yVzsngl1(WqQL#jjB@w zflKQ&%fo^pi#D!NWo}TT(po|h*zlG^)2eErOHhU-+*pSZDFsa)FM|0S$#dI|1U8du zf_N#;1Pe|OD)S6yXJ^*bk2;*Q2!fCYe7$kWUsv`v!szR1Zger)$@Ybch;FGn`X_`E zYi4fRR~gxbqM2kPQde+{sXW@dM&6vP^Z>5crTc0$r#bsk9`qt%n-NvtNGGN`{T z`JG*?>+dorVI!xRC!h`mOuIpS$xV<88vE8Gcm?Sd6P1_z@isvFDze+MoX-aQBsCm8FzFWYKx+Uw!~1Al_%~4X*cqN^0O0ipyKI0S z9yoe*!KPQfTdEMz9Lz?7E0H@S+i57%ZH;GVV^xBOmX|GBG?Z#m!IB|~L9_*#eaK_W z@ruMIk-|WJg_<%v(r6dWv2c+{dd&+3WFgp)2`Ff%RwvOa z=KdJ@Pt66|2W{xAYtEmT$sTsiiz`HYp5LM$uD1Bg>!~oW)m;masZ0@RDt?&h8l)Yf z&%mmn5~?F|t=v`{osBhif%T@_*3Lv)lB|cHQ;!B;zVMI+SzCcQ?njs0H&S;=(ZAcf zrh!g!|D0hzBdNxIj`4AGo%ti$I>{6WsymLrWYFLmi0{~a^(NDzqNzaf1pl*ltW$8{ zdlY+fu9jf%18hEO%^7iI z8$jU@C=o3rW%NRmnq;x!C`C!WG+{IiQYSI>1>1{2b#sz)a|dOie>;0?u#iS^?2LK? z*sCdBx6Xkc_|pk@e#Uc6ee=Jtd3c;C&Run}tQq^`sE(0`c6Ywxp1SM(L$yzR z|IO>2LfzH`cKUi(euFNbeKKv(2Y=3p8Z^7lt&*~>1>3WV($};OoG^L)AO4BICKW9B z>sIfrZJvusA6&uze|r-}BuAg-{aWhW#gTulnpR_4FG zE^Ht^&{wHxqIjXTNJniIlaQb>eBwi0@;>4*cjgGO(z-XTyx$huYZaLwQ|L)wkk&!A z=7@F9oIUG?BuCsAQ(k8~oey$N?h^ z(h2S4hOS^X-u@#*Pe?U2@nthhXk9#u@``5bB&||CR$Ku1!d4g^K{%9)0Bfnsz{um` z;*hDgbc(3LU#N4jJc)cZ3#T!0r!`~*8+|})@I=y=wjt-rlbRwEuhX)W)4IrH4Z@yg z+!&@pCal3t`Ev=(NEK+rPhjz zWGFNKb#6#QRUu#~0<%G_t(%6)Mo2anG$%47!n~+6&NDR(!t|xdly+21(XD!=Z<46# zAOwyemh5)a>OY*Sx^-)}-VafTt>tbi!1sRGWy7$=KCVxD8hLuM%A&betN=-fl%TP< zmAKz!w177Qahrx#gX^IM7b1=6OvCNoG8NomN37rqP~}uARS9#H=2brO0I%@onKoDf z=~u~+R9?1J@jauIkI66ZwYabqz=LpZ328MxSV8a-P}&)jhxKDR5nNN>M*kT6%Vn1S z=^vwTK_c>~$*u1o1wB3%CWMKN#O|v2@KYoLFGm2NlqbYWzy;;5cBKgPTi1n9Dy|8* z-R4Bq+Z4^df<}DxH2Mj7scLF?xXI7g!z0qaun#1!W-zoK@!i|@a9Xs6`#t`ZtD4$* zkX*=5@=g=BMK>&QHH32v?YZlLkt1GxHJxj9D5!Lagu6swX{~HVu#OgcFt}cj*aM*AwWIqVSEah{F^K5mGHW%1*`yBIX9H46m%eDIUL#wsfsD@BHM>QDVg$DPL2y2&d;@$h_M8H1puP~ zf7G30YUjlx2222*QD8n2+W@J?!afl(#pEbO z-pg1+6(vjoa_}VQoGZK>vVAHj0UOH>=l8XGcVaA3Gu4QYp!HSc(%gT%=#Aa}H((zX z{HXT|fBm*u-=LDlAVPfkFxqs6!X6#?xPS}kL zmTVe=L<0|aS6L3z{E6aY;5!5VDF;pFODrIST(5KW_|jKs zG!s|+I&8dz2L4XZe;35XwQ`;+=vl&1{w}G#=(RwvJzvTZ~WJ& zd;H$2G#Q$Zr`>VfbE7FBX%Y2G3H3Ow@8n`U{Pm-`Dj8AQLJV0UE?eF&(bO1;#wL~` z2trSaAX6WsQhvh^=4v3GCzsy)RnQUET!+{^Kursp>eRVkzZL|{)6kk4n1%quxXdUgZ2t*nS>ZzM~@Fw!S(~_J znE4ZtHrnO^LCwfU2!cU@iB1uwfTo!B~GS-D(D~3i)>hpE8u7_Y`^K%4Q=J_93(c_ADgc#r|86Eqq?0;A^V#8?;rHpr*iK z^TWUgg0puCe~#K9IY1vB=dK@Y{@(Z4Vn>a-F+9ZTdEA%tkAsI{+qQXS7Vy#;tnOj? z`5O{d%Ru+u|7KaQ*Y4cS8zIsZY+RdC2(?(f?JxOyj2y=zQjcRKfy@F`b`6YtNx9JG zD_aj)J)x{L-}&T-54mZ2<6d4~`e>^CWH!lyqk+-= z1G`T-5fZ#C7!*#bZ~z{;x5J7v!w~`&Sx}|4y2FZ1y%3wR7lDqlk{U!?8n$|yx^9tc znz#r48~G)WSk2XrlYuzY8O}3dX1ZYLALUB0)zSj68RT4yt5-IlJL*sLGt<9k%M zu_kbc2ktZf5z``&X{_u3ULNl=xFQFk|P0 z&VuZ$rMx;n75d$Jbob>8sfGkf$O|6eTjt3?+}KWx;bdpIV$uhR8iE$<3k=jW=Z$Mt z+27MvQ(j4z%B8Upbha&@suc97VJK$Yedj(mWgxRLcYAsuNMPDo*2O>vrMb5@5j0s~ z;w-xup|}heYDG95G&jK0g*xY?m?AhYl7oSY*hWrSz2{(DCyD|wv7kRh9_CZBujDO= zOD7|BL{~Y!ggGH0qS)d)Al!m@agf5A5Dlg~cK;bGdKIXjyuw$;?f*v@OPAG)@Jj9G z4irfV2C9OBVjTrZ8WL09o)RHs+=$W>KN3!<)@2k%9|_RwKvAPL6jHKs-Iz6{cpJl{ zMd?}~-02{)T3 zeMqdQf+u$$O?z(DrA6roh-y=<%gmYiDmWxU};S|_Yp7^cg)8*qp7_{mQWJo(}D5~Ts$v+y#(FiV_59%TT0-_K~yy}cd z694x%J~yHYfbE^Ha9Mw1(oaB-;Y;$U*8rg{4-2b4_S5mp;d9Z$TfP5+=9Aa6L@k)C zc`?^@NNE$qLlr7uQ}XdkALM6=)q%biZA)#)5e2jb6zfzy8RbAE#aGn7%0AHVd!V>- z;X;O@Qcj|NDVcB-G7O}qldJTLbFViCe;8B70m1-a*1$1iK0osjM|CBYv+@RPg(j$Y zu{%p1fi|4EaA7D{SDTcmYpt{+YVS~|j4NqxoK7FYfWZwW%^@N+DUU!mcErrc$)Ha;r~=JFgZtqy*eUd`207vxXPp(a{D0gvNjyi zC@K{1-kn#y<-NxR$aM6!stOJ!fi&J5iqyh6ng*YwUl~0RWmpxP|42keWBL|B z&KgmMQrG$&k9brlnk)zQQFAgbX19MQ4hx!Xc?!xni69aQv#csg!!qz3z{u5cd_bWt zKN#9{a)gF6pJ~%wiVu)>ZfGQCe(q|2!{ZB6{{(c;n-6v&FPEa+3!(kXw%3aHO?-iF zcQqnIU3X?WS2MYb_(zH{YGDn@YMT0^kM=Grqg#ogNy(^G1p07;etro3yj}-8S`IbN za&~dL&FK`}jsroRttRH@~(5*WUYn- zSMnGHG!1}fy!Jl~)IM?=1(FFW>oBq~Y5OAQH*?#+<+iD)CK}w;#36ite0v`&z18Xg zO$poKm^sFF*U$=ZklP=sHt@=oN(PN77{`8EgWiD#{qc^e{(RWWEN0Z|F#3a4$H%%D zw*jgiq=prK0PPAfB!Lx4*XX)7V=J*m;IZ`V0{7=$KUVkrMns&`xK3g+892cM4Fi@F zbsg4xcRGrBKVA<#OE~?yqCceeH%q(K*mN#WXs2QUX;!8*)BumOGj7i?R8RTs+)|)= zz?0Q9cCt;N`b2{=pMN!k>QoY5`pdD3-!Az4iGrh(#0XU*2PWwk_yO=d)9JV0R#Ng& zylMV%#E8K_TpF6uoS@5w7)cF#{!4a&)W+35d+wazM-^W5;;GNH?k=$xHd-v9&`**i zi-#4N$6Zc-a1}&SR=K)0U<9JLL5&(MnEy!Ak0Pe{a+>@PD!DV1$?Kfs$yw;`)>5&E z>m<0-vz2F;r+KeinO!sk7D98MTGB|M0%S3~!(SPFG)naJCvY}i9-0Ow z9LnrX^f$Og+?YM<@?SL0p!d3g&od-*BN0LkAf%FVs zy!bX$EyVIv`f$}Gq!{Gev{yhpgsbB7(w)`$&7Dv{{y%^j z1^?W*I@Q|N7iQSglc1;fI7(1cNM#ycA~g_Cw5|8g&sTeh zjzKsMVYw0n6>Ei@JV$B)8F5G2FDn1%i2LsqY@b86S4!l?526wvA)81MRT>04LIZZ2 zDRm0O4#Wd~UzB+nvAj-%Aa=yZUJHJ$ZPN`$gnM+LFqCACXK8ER2Af^FO!}$;TG=!b zrvM~Ed2MX)Iru~~@m`GmPeB4v0d?7^tM*UdJSKyECUYL8Bg2JBMzs{+poZ1(9D1<- zuaS8_L*1*Nf(J;$&lpLWY2vZU|#36pRo@^Yl5V2zR0sep^6OhsOjabEcO*b8PD8>pYW zge=voh~908!Pm58alVP4J)5@iZo1Hrx=C-4Cka}3I=Jj!O9|J>22MMMRF;{X6t+Zl zE;6s>@AJQ3XRMb2VB5*2IFQI3YbHnY$d+vYA-MjhSU|=wb|`KbDVP73XHj%7=Rs?s6_O1~%48lnSMen^_->dbLkv-LOYCbicg^j7g zD>~|(G{)8i`FE zGI-J0UVhqa+o6YC0cZniTexGohRx|i+rSeim)xy|KK9}H80ZlRXHa(tCy`A$&iVz{ zN{ivGHMl78TyE}43YeM+rJePL+iW=}!&ep;wBWa_f^UIk9mFGD4KyVEFQ(F)a1nO; zlM+_MId_Bo7)H|BWeS)OvT9QC@13@T5A7Jx!Rst2UwuiZ0p)-X)TZ6 zrxFui485D!@E^# z?%^hovdHsP6DpqREFFIIj^M-^B24muRFg>=%c!Gi1twLDR>sgkbNqgF{>W7B&D;>q z(}JBzOp^l~O5HUBu)X(A-}{g0g=nm0nGOU`-!}#4{sT*&7E1i5U^)NsuH;1}uUOt5 z$_Y6-X3jR7Kzz_N93+U#h=_9>G&8%L+DxxgqBHd1+0@~;lJ|X331sCobAu8qA$YBW zQ&U0n`aswU7r>0D#O3Nzb#svPESaQX>iBkuYkNREQwFvy;^ooLC}33x=bczsGmZ<` zXouShbTl$}1@E)mdEFm29lmzyP+wcVtGTM>k?@JMcR|CU^VeH9_M*GIn`}m@iJHzwNA0I ztua%PJw;0mghFlDXwAAUySnZ2XRy1xBbU>OK~5B0BpxEpU@-r zOotP77+gHH;CkYq_aD0W7|sc7qVeGg;dS!=(L5`VOIUe8{}UKI_JKwz{ZLD4trUv+8A-Q=*r7)s&9DUWXAZHzcFo6e{t$-QWsS|b^<_V54k zxWU10g4Afm$%q;x*SIQgGmPKuf2Dx($qju2XwEY~UfWzYoz+rb3fR^_i9ly7u*U%9 z&U)?DtsWoaZXp>Btq}0$5PBW^+rcz({Qq^;xXD5oR(QnsYWMu}!Y4FUBpm}iOT(X{ zKl&(&7ELp?Ojb1Rx8#eh7uPfGUfdUfbqe=Fj+hcIGvYt=+}X1!uVgJ-31KCB**P#Q zneb&gQzBK3p6%2 zA;Jl?W+9cpkAE3yzuphuo;r0oOCFvPiwoZ|z5>-ChdJt`!Ayp*(Qzv)XdA_LiskVXU7QSDm6LeYBeuhDA>{41tA_m;|BAy0^WK5 zRV`(099Dd{2Z}KiZ38++EiuL7ulNTgLb9%kY1?G%n42btJ_8t3Ahj|HpC*f&(vtF3 zE3@;q0OJa%BH^0jJTcMt+6GBdi!mh4#A=h&0*`%QxVa$C0*Fb|8|ZFmXSBMGV6HW< zHW~ZBN@^)t;;!jq5zwi@L#c>GA<)CwV%3Yq2eeoTo{ttoh$$&+@1V!*om|bUknD^r zp>86^+i5t1Ma%>VRr+Lfv;+22)Qt^Y-+&;{ycSYb?L&zxntR*3^w#w7>lcWLd^;^l zG3}`r=maA1&By(kQ9rnI)mb0(7Z?VP&l*#btcskiH-sbt4A0J>@ByGXlnPlAUK#WN za)D5$!|WIgs_9`L%x1H}aKJBl^^qfsV&9g~p9V|gblf6iGs)_x3O8@{Nu7%b9y_mC zu8n-<`9Hg_J%FB2LqpfN8sJMXDX#iV#IeY?4Tn~ec7DB_qa}`IW|P>MGr3;rKKdM@ zX4(%wY#E$wa@RUj^*UHTp<2PMq$xeN8)uYkrFHFJy!nHJnkI%c3%?sZpmn=;RRzys zl~3xktyz4U_RNFiJSAeB10roIiER*h1}1|nm6N(^1vXAr0xE@ZI=At9lge38tTM*?XdcQo-D135wgMst`I> zAbqc(!5?wKq!bd&#lQ(~=M-I_G#s*Vr^)JClyX1#NPJwJHbRGvtulavrw~-HUaNpB zR|jN$1acDt2qMN7NauYzTF53KF~DJ`9+kB|G}#Ygc@ec=`{mpjl;=H`f3Gkdq&Xv@)+QEl)SmkR@0 z{CMk9N%0qey*a@={J-JWv9uOV2=SK|9Hm)z`jt>hNR5IM`0MXrNGB{zv<7|j34W1% zmj-db0!)^QmDg|-s-G$!cn@EILxy&5Q|S)&^t{-oQ4RHqYOS;{Vc8>pH%q}`T-=Ke zK&wl@B_fh9pm+5pHws|8fr2nM%KcV@ZRIVhR1e|zu_qw$;+M27Sl-#8H86xQ=BQ-D zl&N=6 ze?VzbeLxz|$U?gt6ZPJADkM z(hQ70qkODSY3{@{zXQYmg!M3F(*i~R(rQVKk^^PUD_0Kz0c&WO2XC+)rglfEVYA&y zZjGU=y!2+h#b=KIJHu09T$f+#L%eBlP>6z6h#ckE9@kkOBr=XcvXEi5k`&r1Quwqg zQb5F&mJxbshvhRr#_4bt+LkQI3p6ZQx;lJRO$Z+X?AU%q-hk= zxFqIex3P5O|I3ar1mPR;>nzU}!D&}H~Pw(Bm zs|y0$a`< zED!7)0?))b0rTb+E?xiVUSQ0nsib3#Qb@T?L?h|E$HFv~`ldb0dT6kRDwL`E74Vd{ z$_o8GY(pK{AySeOiC5#UBRW%i#kZ+g`1glhLGez^OaRG-lkL!~RVEy{A-_Dt{vFs| z?|^CXe*=|rVzm*4iKT%o725n*BM=-8cA{wmFja}3SLUTlXacJMhDA|ICVjp&4*pK) z6Dt`R9`hS{(~my0|4)3%dSE}eHWYfiL0l7dU=@KWB0VlwJ-LpCGL_$mmrAY|G9;v)JUCFLM!C9h*A zNB@1R)nmT_BOkLqG`L)4?Ta$s7u!Y*U(CwQyX4C6rMZMO(p~)~Bg1%VN%1SxuJiu7 z)|;YcVt~C@V$8mMYdPXm*`dR)vo45~Pdpg6!aXyfP2_)oVBviTK(Ihb7;t-XL#?SV zX=9SxKXY-xO3y$dQhw=}f?ncU*of{A$HdTDKA@IedQgO)SZ>1GMeez9BNW`KLKFLY_y zjogi&lL7>Z024EfF(}~t^DM{`tuP;t4MTA4qb(^)A*=S@{e724SOC%GOp0C}{MyI2 z+_ggzUeSP$xyZ+a8Q$Ex-&#hRfrk~k+kPM z?oIPdi&kTqdr@n2s3CY++HBV9S(*_R#e)?_1@)JOC~d#!9dH{hA-ekYqHg%vQkzHv zsu36daA~k7Px()9F%^wgT%{_$mK@wtsG3x5sNRUJ%!p^cw|fxBLBl;DTG@o*IG?>A z_(wAu#5CMNVwj3Bs5;#i&6(K6O2{z_rTr15Y}z_Zho6h4nNqupN_B< zJDotCtz-IYnLEc&AS^--BueOF?BvD)orPxRQIoCzh6w$Kq#7JLj);aYabdvQo^@&= zGbe+?Id+-`Xz{PGeU|%wjGGnBjSyr=8c)Ul%d@q`U9vG|-}pVU zg@HBg@IaV3;(i=)q&zYE_(b(wORaYS!b0gvuCsy_ucsXhR7W^0JTY z-aRAs=g?=61pZ^hUFPLJWlWx2b&6jqp$y*VGD5U|w@Pe*+|cQSr_!~C{#~_tb?pHV z9L~6SZMRVK2TWze($#AQgQBz6ZK%aCE*?*$+UU_2a=w8LFFnpYiFPSSzIo)x5#H#u zP3X)>VlGUOqYr7gV@zlKN^C1RXt}EyG!zNYB}&bANKwH)tgdz6pllYq*==a0I(Svh z`h{$>Z#`}g9?xiLD#Is9!^{#Wkn%Y(6rfN(E3kA*Dl7hDTEVk{7s<_dYGrIXPT~cm zYK`(jHp`Twq0E!zADecchE$Xt4I-u_p0Vlu%HJ8dC9Xs1dnwDo-NAo+W@RDM)p~a> z?KCi@bPLWy#Y@5VxAvUrz~`ug08G-A46u0kV^EFGi&}0E_xPH?CRG@S_gb>L{~svF z*wEgZQZkH+$@yIG18me_-Wfci6{Qjoo7Nkuc?Cv(g21LL-9Ga}_4fZOAbbRfUn)l|0 z*4=dWbrifWVG06b%!1RntCO((5WK+I2JGaFW)9^s4uW)?hsw(~Ow!u;12}y+`l}q` zfZt7P0=T3^OswgmjXcFe)Me9%4vGdzV*$}3OO8GB75R@=#3FWRI8r!l(!@%GMR2EB zi_ia}8)ac{VN0MfGVtthCI{5;N4SJFk6A_le|_ zH2@M>FraX0%P2;=0>^{Q%WzJspx*k?o7Ir@D0V1!SvOOkRjMs#-#oKHoI6-_D;kGu zvPEIqNUZl8Z;WkwVcW)ymC=wPvZ6I=4KK?5S2ahV<zy8i+osQve9vcXplFAv<;_@%%W=$sM4AmKd{6-gCcHX^)>OMxq z^uPhvm;KfN=qjv9vu3uj_OBNmU=#dQB%uyS#`bp}Y4!jRi2;F?42DvQ_vdIJDWI}Z z!w_Bc5KKwmfI$py+2X?NN{oO{#oQO!oq927r7l0s`p*rsdTWUwFBaJ575Ez2Q{X!` z75Wxhaz*~J3md`QSSP{mAaLfR7k9)h`txcfnoR9aqC%+LDgKxTt2Y3twUAg*qWJzJ zObI1)&_xH~yx*=T^J4{#aE+mP=P=nEF&#E|qNYKNV@KQC)ipG{EQJ%NfJF}_V>l@J z8YifhvVmq%$pse`oP&xK+D(s|PBpK^6GHSVJbI*SWuT`x^xS`l>2MD`>l%|4Z-C81 z&OFE5fJQo~D#AxSZcFzu7_AOc=GugTyuXRCR@!(0_3$E98+yOD2p<~ka#P@TxfNZ% zxWJxXMu-R#=wnZ9@qmvz9?b-?2Ft18*l?NpHgGujrrrRuUjl!4*=jiyVF_z%BiZ4- z1c?cj8QMi8uDqb%5F6EAiF#3El0c%Kpe3TOwafzJ7d~iI;`aZf{I~$8*K;9zB#RuFjQP?7{7tN2#b=-+ku!=8G3E23)n)RauElL!J`|>)9vf zSn7`EMjCaT?}jDbh5|?6fy-t(0ujN8>SKr?)E+X%M8(kd;D`V7?Ut4nzl8Un=986q zRr}Da5YJ*YM^dnoJwBBfO_7qH%B$NKbKxGeBuoxUv>mmKZHt3Jh4}YQGVC!QqS(A| zf=^;M?gzTr6g~*^mdV}b7f(7C?RtDV0HYR11DLrGB}{P>4jY@dXd!8R;ZIGh8#~d~ z3+EudchPCphy%4D8-Z$oPSdqpK{a9QOyHh!YN0=t7lCR*tPY;B2(W$2{$oggTFT{% z9ctpdamY5|-3%~=OrwmQcjGZEO&wHVH1NhR`T>E`&`KkUp~OkBvviV>+SdN| zVwIg8RO+4wCil}A4e>pc)QtWKbJ73&Sek**>tPGEBVd=@8c?2smN=6o?U}O;AzT$< zquMQm>ru?LG7w%#JQ#%{pEnpktSAT|^fW~#o5AIH(CG*ntjq3w`-=RZ`OV-!;bo@@I%@fd-?jym4vxq5#O1xjzZ{ov_tbxr zlgXtY#vXJO>0li;VLXe(1#n|$5c-m&*zztWE)%=Q&B~&53j@C8Jvmd@)i)0-dXQB5 z_eA|Y{12xj!T&K&Q1T6hwlinqI*d+?s^8drhS-FR^hE1<)&~zS1edMkbGr2DLyPKI zx}G5{u8_3B+hOD9g6TGxgNb+2KFHcg(AlB3pL7Dy2L&2}DdUbSe&>)+rNL46| z$E6P5Nz_|5g+v3F;+7;4fLs#oMiNo6W*S;ipwKnxBmd{8+=mVcGaxA$;hcwnIm&zj z$haDH)?#kf(0-%Q$yT&Jj8f>B;|9%-x^JGDeeoL1k$qFVzKB2RA=7#>2B$Y zFJQa@(p{DKBgZW%CN&;SBf;c6U0~9JW3;tgd`aK{co>>~=(nRAQUOxL3Nk(&`zP1R zsLwlBD4*{P>@rqPlX%gfUjI)Q|yf zFJPj)V1>L>Fd0Iqu~~LF;?Yq?X+`k+Czk9ypL^p*CG^`c-fO%Rvl!jRJZ+$-C#MKOE~5+} zX;J4}S=Te|vW7Zgp3l7h92cwiE|~kqFT7rZt>XfH3AIyjwn0v&d1=TulB}`A`f(tx zY}gQ_ff1HJzufNO%&UzY>s6`ZHt<{L z+VfBEd!gz}e@5nv=-y+?ryrgBw%vmloezI@ZOe;e&UKFeXtICG?s`?zC%L)LuT*72 z;LtCZ-+P>vHFB+;eU0r?f4vuUe94cc{vY33_QtQH>h=t}u;2~dPP)}#%a}iJt;?>G z{o}qSM?W%p-52K$&w?3s!#?&KPSv9D`0~q_sYKho+rTBEJ=CIxppQNGudSagCCADvo*&_5me+i&+>|E&H?RkTtYuwFzQ1XJEZa)@O1vfX1Q)bF?EZ&k6;>L@9QW{vvT zd+k*C{XfFq1Fq-&{r`VONEwB6M5L@TIviOMr9_0%FpF?d_DE)=Xb>UU6wyH9IChc| z85yCDQAr|EMku4-{mS_qr_bmA`~BU%=tmT-Re=9JPUZEBfaG-;4CZJw-oK z7m8W5LhOi(eBw$!;w2IbXS(;xS)`twSn$3T71;`E_o~YGukVLQ>g28^AGahYh_oKi zc4L9nntO|i&aM)M?AKq1Ml=Q7!;mi#Me&_`H$GnzsaQW6Tdq-kuLhC*#?!xJQm~CW z-p;-y`s&Tt=f9W4`&0yep?sB*gMfnxdtz3Lt{&}sL$j5(#wI2y#1!P6TNbyK%25zF zno)O~=8?i;TcZHWsK}=O`CZZj!ZvMkr=SIny4NG|z=2w#M_@Dvb-C!A&b+9VD|mFIh}F3v{0Y<>nU>aXrX?*0 znxD2_wIcgPfuQl%(7XF)*vB>ibfm?sBoOz6T#*we%s9$kQf$|DY{ZSb-dod+yH}sA{3jk;Fxgv|@0^ipm_x_hhVU%bB1c4i@1Pn60(*2e1FYy)>!GOzxg^nMe+oSi4 zX$LXS3K8Vt5u0Hj-Bc9JIC}*sN&NE03_^HlVSmR}Xp7%+HW&aAa16Sq{Pc#&{Sev1 zAk0>TZEVQWU8s(!r`J7@7J*3vRHEI?%&w(;q^jl~|BOG;da;6!M<~IGn{E z`>sqP@h7DKb?uHO08%*vo2|_2xU4*ON^SzC&;`H863vZb@2V12loL!WtMHi_@TWdzKw?}=AlN7Vh%T5zYz7v za(y+~@zNf7HK}x88pIk3q5{i%?AW}h-a~*3=$9dZ_uf)n3U4FbCzy9_ZKv@|_?%)# z^o&0%){HZy&$nS1K9G@+zLP#z!eu4Fjyn7uC(B)ltrjj2sKIonOI(M(4!Lvq^Qga~ zQb%5&cX}wW%%uD<-kNe_~>sfW?IQjoYO)9hoXsbZh@TdHa^AK&%Qu27CP=`XYiU7Jf zZdWv4F(BLu#;QWE%M)Bo>AJ3JGk?GnI>suknZwQ#JM5qad1k@?Xo=t&7_Bf zF5Z}7U->49R(bvSv<~14#tD~bA0S`RD?tcY+3ss33VRtwLNg>afhcF?rr_LYV7B$I zyVz`H91=1+mfzJ^!UPd5h|F`xn)~hr}+=;}&GaUtJ5zT3b}YJ{>gD9`Gh40h@)7 z)JS-nUmyr&m>c;UGHDO!c5{z;#v}*Rs4k~jN%x+<1wuqx3pmjv&WTrrrEh^wMV4~s zj>C%=Jx_PPOr0dtw;{YFX;pRsEk(!akGcurLjf)dVh%wV08!;CXW|D~t*ubiv?p!)YGq<(v(HF`x&-(}(! zAsM-_&4+p%>5_kysd~u0hgYE*g+kb)8-Mh8@LBZpXU>fNevWQGEF|VYt36Rs{^KsJ zqyuQGNFeElR}bn^Fi-=vRr-9_7-g5g>Fvjl{g9^a=(4t4As{lOe|g~!^M zAxJjbbm;J$t8Fsfu%gtcQA1`9@XWcKCk;EHJA2kF`>UoKD=+_o6tTjfvgGzKN z4!k!GjM$*%-C7hdeMgLF^km_y_WottW+Vnufw)s{^K)W@l-xRsm3>2-5h2CJG$!uB zvNhSd?YLRd5>JivR7I-0j0QJYZ^h0%adE-i`p76o7PhhNG55tXV%~{Zb}pMkCjb}L zBlGI$fv++h+^OW^7c5#-RRIY3l=_~haqeA{w1J>|pmq*p3_{!76M>b3LluSZq9~9H zZJOLY=i{R!;a@WDWTy~C?=04CN{UWv61@%D1-arUIKcas*0>0+$Yi% zbA+du6`fhZHVxb!Qm~$QGh`HT^WOe6l2_n8rc1?V1LFObky+O~SrG zge-Lw`1~l#F@M&~{G0hPUG{Y!?}A~)Ei)T zJam6`XXYQUL%aCKwHZ-=TWljZWd%*|zB{|zAmu3`(e^}#zkm4GUsXPT30HO6bMWB7 z!;KftyQ+JLi(b3n$m1jAPQ9A@C@%WdgWX43E$H}n?j&cMyG=xoP3zbDN;!$w3=}{_ zyyDbU^IgU#4bod%FckVhG;I|nBRJH*e%SXy^M6TF(78a`rUSnST|73~ujW(LyZV$a z%I-gw|Benw>G^Q<<^neyTY@SB?;5AK2G8eo|BA!hbn9AK9!Ska5Dxtz=OJmERamNT z7RE(Fp>u?{^s3bo(Al5sG3e|}3oZp%m0aZBSoY4k)6Zg9{%P-0h#Eh{j5u=I zq{EyB9?x3ZSi;YZwme@~5@dd^`Lp*+tR2fK6#ZCY(AYytIw#5TpjV!WYF#wtzWR#y{jaB&ep@J$)X}Mwm1*eOJGK_xjsw_ zY$h0Tb3cxBN9vYR0f4+r}7Ik*`k8IIP0ZMP{bbfpW(~i?2e& z$dxDz!u4%kMQ!n!UhUc1PYciw+|%736u*2Bnh~ZSLS25bO5bmarrMYVxt>zk#v^$d8B z1eHV&KnY`WcE_Gm{9Vkk4xqS1)jouhAcg-T$E`qJtoi!OCY{cYXT8HZv4ZP&Z+Z)8p&>yguLP@>pqE^2^*io(E{G!j9hrlHOXBJ)0mAue zVzw3f^rGKZ!v2PZpu}``1k{%~H zZRZz|-{?5oX6C{K&XQc-4H`IYv!K+|gjIl)y#u|;2^ zC;*2dv7}|V-m19z^VhFy!=Vco74ZfW0*E4=wzFr`9c5}Zu_g#teDx0p4n{suz4bkS zW53Uzy%WX<^!qcLUxp-P__NOUWJtB-e13kmVTepqIuDjt*#DUGuK4q?9pchPg+Q+N z-og2Y+GoC=^7-2SG9a}J^qv<(ENV+EMg<~Oz0w-hy0_u!Q+J8lq4rcSfyBUIswzCH z(U#xK2#CYY8t7C0C5uX~50hpX*ANWC6rg99D*{;fQ`G3mHJUBfkFN`l!E6I2<6h#R zaO+WbzUqUa$ZC`%z4L(u%l5tbeI&1qREy@+jf$*g?BPM%V}BRnC6yy5E+ybGhphPJ z@N|OeAtxIUwe`?%ojNHBjfz3*=8-NFao73`sl3rb%W7(9T!2Ff(@mte6SJ$R2{sOz zV~0Fe6!$XTk?+=JdGHnVQ=A|~1BE>x@lR^sb?er7+&Iq#5ypkh4HA9*%^|BbBfzuQ zZQW|@wYvHwfLK1Dny+G^DS`XqZ7CdXO0KCN^-qxA9_NBDrkqLV!9EO2*yi>3*C57VE}(vibL({R^^E2Zgis6udTWojm5H!@3xBcFUE%pl zXxX|k>E|_KD_6AXL&%8{e4XyFO*)>D>fq@VwSgN~_n!ycOoX*n&h!Ea+T)ZXj_B>3 zw^LagQzH#y&3La8(==kl7EY4RRpzUNx^f0xlH&m%sx&1=`c6z3aqo zff9;NnP8PZ6djmK&qZ$kwXhe$$1Pj6YL$Z>(pru+9V6{f4-wJQaPX-khJHeR4%9;A zm21yL^9%OhsM^nSUrJJf4AKVhY(zEe+JiA_R!_Y9TE+Jxk5d^sa-{L>dXbk~wr^jX z-r9;kPCxAhusb66MO`3yW9C#%UN^Gz0? zZxxIa*t}b}O>43}`rPmH9;6O8j7*BK!l0M{T!>6<_nnRv?|!PgVQjC=FZsZ0p9|vR zZ$Y}0>Vmy}vrXB!Y12FRRtBaLnSfRv03S&vWO8)Nb4bEv(bh(Htuki{(J`FuSQd zr9b1iw?5JO8B4O1B&~3hu3Nv}pm&JQZ(1=r?8zriD=KY-{x^h`1tIcVvu4d1XZ$EM zwo%V{IuD2prq%_PvW$Qv3gOZ_JF{Km^Y=FDtG(M-3_2%{sK3PsUWFOUIg|PILEG2& z@5TDbyx7li{shnA#1R3^)J92Ks_X3Nc(LD11JlU(__W}dN}nSjeGtDA3{BfZ=BO`N z&>ekV2M1$=dnkDPsDN`=AnNltcdY2rH2sWI3ex9yo7b5po%GNVg}*2Y$w=W%4b`f! zzf^jjer!pH2mtc^b8cN3b&ij|L~ld?gI9GOdVkKYDs;j2mClx-HypWw3nH{iEA9i(ysVp(uuKwBl*vE?ltxXFgvAj%e!O?7R|pf;%c& z^%7pGm_(VqE~nc`L}F+j)Q>^7llINzg`hRr^RAMy243!-q9_6dr|zN;rAB19u!PM> zPyq#~jAYi)39mkAz}_c7R4NMJ2>02lD^mh=*J3zJ7%N9q!3>6TI7a_?1{Qbr0c3`E<`zSq1|W2q z5vm)A#C6Y}Jr1>(IqV4dem~Z!ZRsNG%XGIyRq$mDy$z3(k1Wjz-a1oVDKY3g7us6y z9WkM(CGl^e8#GNjkfLy&a(G0p>7XG)=;f_Ee07HX{A=W&Vs`O8*&;^w=vO$zO=P3z zfr%p$%CK>!k3(%>cxmM_N9xVy9mbB3~m$IFX`&l z0aI*dBGm&gCH_5uGoL%3UWg@R4t@Oj3+{kFkuOFdJ4*X!vgX4=)C@#HcUR8`)xbWi_NnJtffQ|m_4 zwvpjS|M-J*&)+gD0NfwAE$&gb4S+YPRCTCg;LxZ*!|R4 zi3X(}qW6cIth9wAd+2*QFG<|R(Pm}RdePdR=LYO!@iaa^AQU2n(hBq>!@hqt8L1-q z=ag5PjE9!y7ubZ}yC0qq!rxtUyJMn~QPLQzrvCLuO{sW+;6a(iZAIT0*R*M1H))Xt+4`xSFcH|zwU%=KNpoAG219iJ0m{jq6INcIN@AbcA% zznVHat-Ut(km0VVdc|3ba%~mcLtH-8dTp8rY)WhgoX=C_SKhCWJs1lB04$%w7Y@6* z14vS`2Wb}Yw^-2zi6Q~>{t@R!%O=@%6k`XkAI39?_^4)N4>*}8Qv2;eOfM;uBz@cl z8wU7@ZJ3t+oAj(gXo!6Ctl}W}4mIjmO?uJHB?0Whjb>(|+9kgrAUQ(9mpC zYU~>wUC@v7@DdVMLSEt9u(Y_F%?c`s+jLJeu!A;{5xk>Z7cs{eQbn9VgPcG8B)0n& zwqdZF$GVOt)LCl*EOLOzsXOD3w_~Gh?mD9>9T(~j6?oJ z^*M9rI-YLT!$R{{!dCxA4Q;qGFfx$nyc(V;8`-h#c<7uUaC?!Iz$skj-N=|Av{=SX z!Z&SN$=^Djg>!^ld5Hdr|ORZc-u@yLINx_%24TF{~t3imF?i}^8 z{2)Yv(KqG*d$noXHtxN@RV~gG{&84Lt1brfCQY8KhUehes0%w&Ob`C9o4a+Nk!N0T z^hiT*_KASr(uydDNvc5$6};a1p4*qLiz6VuzVtX-#dSyT^h#}ysTXEop60OVu#Fo* zVA~w%Kno*kRXH?dMAGkayYAZMc6Mz{9ZFT*tVU4;I14u0e1Cd~b0*;{fKaB+%nj5|yw zAE~E6!e~gi8>$VNj{wdtmRlsfRDp3$0>B%P3L@{enW`uzHMmKc!P9qufA0uV$c zEJ*`A{`%O1sjnZoI=bTj%BP354WhLxG`sBflEHo84bLen<_1l=13}~@_{p5tX?AEI0lF+1V8 z%Fw*cw4fDu1$3I&y8y7iM15<_X)k}3D9pKC#-dWO-L~Lft82lRziv42AdgjHay@gi zJ|vSH%wo(+zym!3^ z+S($L0O<w8=`eyBD2eo}0QHQ6!) z@`o4bnJMs-%-H1c6`D?lQ~+Q|x`5CqDhdK6(Nd7gEAAk1(@A8!jCtTU3iZ2SFtS?7 z4}ysi-aZFouTad0c8OAH+!)QP-O{hPlV`sE%T$0Hk?6t9=FQ)C$m2SOJGsUdb7{*I zO!C5hoqS8bl%bU_2+_Je@vbUg7`i9{QMjbhfsosGJJe-_Yv<>?#F$oLbJ}gZ>-6P* zh=!NG^;-=QpdB;q zhL!)Qb;X*C6K&1Yj){d;7a{KE;*{(rR8NM;Fo*=b$TKBuSs}4k#G3e880mO)u`5Gm zNEsU#;e2n^BYe_gOr;vgY3kk5_={!L=u(|3eeStArcW@R_jPEc5;c8uSa`T-RAZ_u z$dTNwV&@1G#RZ$3n}FL3%YY)=37mZyEI*a~ECQ00v^Q6p{$C^s#%Wtnc44}j`sjp~ zufA{>f+oKEyw++MWJbEP{2Es4uoIN)ZVz_qY!d!Z_L5prz``SU+fAELfx)U6>+nZR0W)(7&V-_ zbzyqQlqcMfbX|>6j=cN*WPZ7nhi;o+F90M)F;GJh*A4+>ICmcnJnFb=YK;t7D5LZ= zoO~#tN;@20vjrI#MH0j|^3c7rgj&FwbnJ=tcF(P4@l&=Qc24Zi@<3aX(WYtcHS@!s zQ>KXB3>`1~_QE2JFfwGUw8JODJ*y}}P~UJH@p=K`R;HX^5EW0sxO=wj-t^}hD!U8* zFS{3*By8*Oi9W4I-nGkdbfPQxdMK~lF>~cSu(Nl1wCa;NWtWpKbq}7zfrj`JvYsYWTdwm!{P@F2)y&^-4HlYN&jg3ItQ!!@Tl>;NtNuP zHN_J!bBx;HxizZQcsejeVQHCckMC&IzCFXu5*-f4>Xl@w9XijgT>O`3e2kOfiSTe8 zWh|(cST4>CC`J^R0+qyo0R>a5@oLKDillPtPn084i4cG&Fm9g9J__ft}PRIa>Lf=O}AkZFTIP5 zrw}{o<+N~gOi0O^XIYT`A0?EUu>VlGGK`E7hNITxS6fCxL^1{du4w7R-L~8e{Woh9o@ub6FKvD1+sB9Q zDbc@CNhZ_SRa0!sd5`L>m-Wrq#v@dg$?0uDAal-e#>xvDIqnVnMk4vbZa#)6968au z>_dzM#Yg2b7qOb+OH7~TO1d@Y9wy3p&XUV%4=CiSDVF$H&V0P*;#L#9y8+fh1H5<= zKzVHIC{5LItk|&M=gpZ@pKis|biZ*awYoBe#VJ?_f{Jp znu7kdp`wN%EcWh#d;UmmiXnpG)*Sfzbas|UJMI`cF@=6xL-E6%aQv_LiY9HlTkEw^ zhw!XwpmtC{h8B7>IdS!0d)(c(<`#N`sIqI{&Ao}vs-omD zIB{sVR$aO{EBYAZ52emo`Z06TJD+lP0nZ3HteS!ES8In$0fPFg*P(n#r3vCSJbHpFf+4y;7| z71t}K434g-033=YxR}=kos=36BVu7 zBdzVmoFy$4O^ybt_v)=5_L@gC=5C{PkDkE~h_oi8yN;Rdp1Xu(j3a^zi^iSdi2AhA z`e82!OEP`k=+KlkDZDWL82(09s1ZyL3qK?s3c}V!=V4}MO+^$L7MkA^V@H7fug%1K zPP9vh-p}32%fgjDk}M^&)5c-Uuw>mmGy99yM?=Pr0yg>6j9<;~zWDQDS&unD^d+UY z+&l>TVgQa7jdI>GdKt3VMkxBkW!XzIt{+UNY@tnA-Sd!RR#DF1zHM)Q6UG;PN<%CH z46;zAn~UJIxVLpkzmE$f-mrG@i%JmEi;2=yo`?F4Xm$GTjF~eJ_m3%X9zJ}t(g*Lk z-bk#!~@HZ`e~lgZ9AMi(`lj_S{1K9xE$Nge1GSMR}! zC9A=oydgE*QOT4Y!q9V;NUvuJHsL3*n)(77Q-2BuHI%~1oFS?(%`$z@lHN9IJk}bC zIgOyO^M#QtK3}7m8mNfL^?A6zBuYu>gn&pGK+M!DL+V**`ki`hz5nuFrua~;4JTpB{QKwe z3Avr4LFq)BD?=ZdF?%V*FcuE&%=?>WGFz51dli`11x%QQ!;KIA7-(IQT})^MLjE;K znVQpgQ!r`(cetl$X+QYm+$=lKCTxvWm>O}`*%h`?=L|tHr6nzkB8pzx3udxh^>J}U zMNUj*2<=~P@AA^R6$OtZwaXl2=rS;^h9H`KXaZy=;@|TUE3W<=j=2j|CYS@3^Zz71 z4&41B-BEG8&o>QD_X~u-$Z#<^AeEV<(};!$$jdu-O;;;#ubm7IswEyN?3mer7MmW_ zuUAimFC4{vq2Z}9sOuzET#P-`KT^pn@(=tuB2JJ7880mjn3PxCnJm+{{_nrt#TzUs zz))t%nVBaoc4<@s>>WPBbc)Q2Zk?83-GenWgn&u>sR5f0y%BRw{MnDXJi*pp4c3S_ ziPjH$3~1*(@mW%%+WNFE!hywEErVAVExoc7ehq;d8M+}Xm0-Oni-akI?GeL?8~N+^ zFGVMqmkG&so19DD!oJi|W|NOdg(n0A1da?plJ^NcS)D6|&=SdSsCb{9tw-N=+(m*a zt4lJ(6ZWLXaD`>VsSf?Lc6jUu|COhf{8xWm$WPWl?ZHbX;dij>e1zt>(Zy?_b@$#5 z>Qz~DTrvR5$1-MhzQft437yEXfUNO3wE z+|%G%{`%=$`K8vo7W;BX1?k!3iWHJmFye|f{?&PT)ph4=cAgvM_51JpzMd`V$;%Qn z2yTngHjtN9^7^5Uul~$@$5`z*eFp8-MfNY5Wr)l|JatW5iI$bpKmCZ-nZt*N<_wt^ zrS3L4`qiq1|6W&-mj~ZZDqDP)kf5lMroL7C(1mEBv!VhF(6ipd{U&^hY~J+ghAx@a z8l-hy0wL4Jbk#L{h_{Y;x=8;<1p^CazelD4Q}ZQ*^GZdSMpQPZ7*;fK#(6M%S@Q3b z+pn&`=_M%-LO@w24pchDXO1NUuY;eJ{x^O$Y5Md!)26+)ov(`7MaRhL+pO8Mlezvb zg+$oXlgfDJV~;wwgNXyNCy-*(1I}9+Wv+KV#sg3fe6S0H*=Qcq+ptFapkbeJ=Iu&R z4(&vTOJQmH=1@b_Vdz68w4HXA<=O}gSG9U|OHEZITWs10T3Yv%*!k{DA|n0=STzG1 z5?NKtmevtZj!Ya_`h297<4He1)u`IH9fl24Q8;Vj=YpNpR0weA!cpV$z>nEK&)?5) zH+H?L<^~Qb5)#0{LFZNB+?*?cmjnhN)T;EuaftZm_{06T?%89=F=J(GmEKL z_RZk`!Xg^iH5m!S^Ix=?LFB@x(A)?_qI8w09$Mc-3;o?S6s_(GzN?gY z<8fBbmh@Cq1dM3sv@-}Jb-<4a0!H6bHU(VFB_SVB3XncwkLK-xP0YMxSc&ur{NIqRBV@ zr|+@X<^uIF8+G(QT7U!wHNevHKU^Aw-VEnAR=oU?|@b8$9 zcK(KNlckvpBPNG{vG6c3pgjs-zm`S*&kv^;c%a1TQ~^6l>?~!D{Soehz}ZghiT&pIFu8I27>-{uCKPLH`r}fVW6~b~VH}#wiI}+Z#}BBoV`g zZghowN?Rla70Rcrod@vd90R%ZBWbP9#8nFoT2Ejcx@$^bL*Dv7KVqd*^6%YvB-%l| zTUOoY&zqH7ACsoeE2c#_7@h!=5XB^e(?#@j2(1p}vvK-_qUK$EhaGD~iY4+IF#+nU zVf1v6=Pp@IfA10}@x3F$KCx6QW;bqT4a~?Dc7}f~+Wwb%(JVlA*(_Xms$w<2(02Uu z%W?D>tXv>@E3_u3C@m!Gz<(s57>{`z5&3!Dy7+EbU)TJzg271!Wy?d5@IGO2A;INU z#Y&Vq73t{2#Uo@jZ%14^x9EwUl;r9h`AxpTx z;ENNiV)L;^W617@??}6Vs}*M8i$=;3zOL!K0>`f?{7Dp*gn}0`ZGc6MU7u3g>O(zh zlz!w)#4CkfdR!g|BYY!&w-(JZ#{@LvE_<_R?QC`h{R=Lvy>zT6nW4qD_}d6 zu@I&t>1UNE%+H@?npfT|$?dqc!=|aXS3l0^q+4tKl6Dc*LZddWGZ^@1YKOjt4Xh2O z>^(AZf167g>z8<}p1Pu)wf}m9m_uC=LKw`k7IFK6D^oybot z%9mLM!xI;GS0v0=Q6^5z>-p_{+_w#Au$fASxncO|y1^XgwnJGBLqV8&_RXT$Ba3Kb z|CoT(eQZ9~!fsrCM@GW4icy8f`Yk_!7`w$=wr$(PF+*ZZJswrG{3JNWIM53@qN=c( zFN*vgw;4;J%m)Du6G3#y7L=9q&yO;~rdkZKBSNz%>sy0G{Kb0L6L~} z#BM7hBcJMg(&v!4^L9!}kx!i^uU@{)0lXsMd`thU0jWDh(O2|j%%P7{!iuSX%YS@A z^3!F=IUr0wf~YfQ1z6=GYNs1?2iLCnkP>+d=u@4hNDh7Tr9Da@RB=RCdZU3>yfXf-I8~f8#m%SwW@}zpaqswRaA(4q}XCs3y~q0>pK2iQBGrtMNw6xZci$A zZkTrSW_5*f3S;XvJ;TucIOR6%cR^M;pkGU#kvrW>SK)EecJ&Uw7wdd5NQABM`Ub5G zjm`LHe_BdR+BS!llDtJ)!0V$aOc(v`#DbJiI(mo2pNlODyIDm|XW1~JSwijPn(^nr zkP+P^i;rPDXm8b5`wV=EX`uOhjem(7jlVF&w(Y033M5coW)W4JvToAkFGW=!%4W+* zFhpsnqOjbmD8f!+bk>2-ef#~}BBc+i3l3odpjHAf_o?xsjmZqU1IZIOc2r_I=<4K1vv)8#B3|_@B%ZV+?Y?G*sJJC+2Ex3q2pzAY7N`i@N32$e0Z^~aWn%XUy>PCWbE1nLWyTmh%Zcp;3ufB{D*wxF` zV5Vh|`S{e0Mv||rkz*l>nyWASYyqjLgoUS2f-XoLLpJj_9});(B;uj?K4h z-TIXGh0Y(%+s_IydLwxex*WsL-Fxw&S{#Lc6)*y(cIeR0C%h8ixsixS91vM)0o&L0 z@5wLj2TrbqhaPeCh$X}bhz*Ul>}1-pjH?V}I4X$)Tv@yxCstzAQy%E>b4z>SOJZ7t z<6y+jbO{CZDpul{_0*KpDF-!j1@HQhz0kD&T5R3dc{k zUGGfj-WdwnGjq&@HWEWbEVc~!I>P#wBt^n8hVO35k(h>(Eg1`LHle0`hakOG(HB|e z>U8hCR01plUTAQ$)sVt|&>$&(jhk>WGBR4M(e5@zM`%k$MlN>*3ApIeR45M!V-L_f zcs;w?8lMym-T%~jj`r{%so!k*_w+xfjgo1s(xi-j%mqnjjWjf}|JB4t==r!eftr60 z(2E+1&SIqTS?+(Pm-+GEvMJkJdfj~2j_^hfZ~yh{*B@vIs|pYxQ*b3?pNoRT8P4#? zdL^XY3J?QxX@lnK68pq$EXglo@L`=%PCG`(3CFII;TexK!rndO`|{Hkd?U#qgw)YN z!69Nep(xgkGRcek!*x)2b=O;;i=Ld>4LtDUhqm4Czqe}GVRV4_02PTF=M4pIr}+&? zm0=UO(uAcsWDd}x+*yJGu(7|t z;9D-)w-S3p*4z5HWJkCB4cl?SKeG_&a?+F*wkAHp3 zh?TsUwN1J^bcrvPNZKlM@UXE`4n*PEpFgUI z?!nIPM$b4OK1w}`KyQe={#S#ZBes-SSoT#L7re^WY*3j857Z-{K!g_pyFWDP)k}}j z2b)=>WDi#baFESK!C$`MW}069t5+JRJ~E4ox&I{_=;T(^1GHm-><&5Plx0HwbEe2@ zWZ?@1m>%n-A%8yVvtsMB<%5}cB{_LOZ=pNi$JBV?$aZlYPTa;X=8L^| z-a7}wq9CY2GCXka1ZyssrgW;JKRz;&8a`o8{$PyO_~ zrtP?=7#$%ePK}=QTiOyCBYk`b&4Zs$;vZ#`YA?Hm_Wh+t-R-Us=3(wTPsk)eBFsR%C#e z&YCiLayI0EvW!MWY37pt{WM)=6$Nq8x}wscg`+Ku+AMC;@{mJR(1G8lPOYU7ob^Pf zy2=8`aWb8YGl84Ag%Do?6r^FQ|Y73itz* zt6`%bBR=L5qBPM{zqfWpwqpx8XMsB;HB-JzC>8&B`gpof*?%0V(SA&!s!55m>c^~& z7G6yv<8N}5NO%Gyh|&io(@b0yP0Sbn!tdGsHEUvOiS$qkyCO#TTAvI>?=ps7dqBckQmC6ai?2PZ1szyDSB{jekK?Y19g?P_?2 z%+0X4-jEVWDWCp2_w2-wBD>#V$v=DB6#8`VZB^00z0W*AkJs-06FL!P8S8e*$472v zw59m)`g`S1%g(K`Si)LPkMgo$cta3qT{W(bWZ+xW^6mJJ2IyUDi-n7%8QvopqmkV#MHKIf;RU`Ns_N%1lj3O40GAFFvxom}*aM2=pPAl%)ZI<=a3lh$zp=&kry1 z0zqV&y4#zg^+(FwPqnG~aMy+Zy@4FLocchozd!K37e1iqm>7vh;(Xf9JI-O><7rwK zhf_Z<^ON+0=1-RVh3y3F<`Vt5Y?e%OvD@+jaFqPui+NTch#}^A5L)%e z8(2sa#B{x8O>KS>5p5eDpyf6~3!j|xq5rMR)3P^0nY)sNDVgrbg}_1fB2K!1^n;^x zxI|bd`$eY6dA%nP4tMPQm6Eer#ZuLahZ2>cvfyd%<I zZ09v-#Nv*6$hvtw`t8~s=(ek=>#gi+wSS)Zbn@jg%L5q_Fc{5C4N(t*N|KHj$hj;) zt0Z^xqww2sPW?e!I#mPz^8WN@$Cb6&?sAeV{xdc!~T@+9OC43tWS#cFe2QS7hp&UR*tG<0!Y!J+hr=DgTH1Td>EK|iqlhKv1#OoiRHA6=(DjVyGIf+QmN3!H6F0Mr zkEHZrpdv}F7L-2EKAt775QecBiG771l`v#Z+}kH+jTcoPpb>MiS5ie%)J#Zo;&_;a zC>~qndB{zu2T9L_k*UrIA(H|rqog#sx1C`xI$aVsmnQJh-X3yQiKtl=rTl()9s~O+LqAJx0$(eH*H`$jb{$S ziAA3z)_3I8tjd&k#~WM>beu^%s&&X>xmj|!cc~tB^>Q9Cpr&NJ0OCwMM(^3#BVA3# zMv`|lpr1)Y*eRL9Tfv5JH2L^)$=&r$S`(1ckG*r4OvHL-HY38euoZ+uA96GXI)Xk zWeZBok~6!hWAyTq9ZEz$-lrsKlP&HMxP4DgTD&CC#<%p6CokQBjuLAjz<_qt>zhZu zZLrL<C#-U`jn188!|e8zSDISFbVrsyr<6ni!u( zJ{vcd9tLV5hrP!j6-}W=D7DAJshyyf@3=qeDfoLAq2~Zp=s@2~$Ub#3E)T)z{{7>U z2TyEv-$%Rl5R1bVfdji@Kn5~k?&!RaGR9B-dJ!f4qjpwD1_tuEP z^Om+B%&jSnDq#(6KWKiY1WQ29!@KM|UF|E+L*5a@a`2;VoU`q4BjbG4V2__!`fa2< z0uS`qBtCQ^oUo>olM^>pXn)DY#WI9xtq-3%?+%L>F5*+qqWd+xhl`0}fHP61|9nP~(qtK=R@!;;-ZMtu019{L2i1 zn)pTN3%+p*3LJET1C19ZOhNmAOj^$Fggp%g{C@xu4M&ypVX3Kw7ZF*1@WB}i{vc{k zN)I64+4JBtWf~{{m->2I zIVJ+CNL03rVr1Tm3767rQ3JxlFJo{?LWv{)04_umYt&ec2C>AYyQb?MoH9TUL>r2bpjpS=i)KfEcxyL**)wRv<5;p_PhlwRn*zK_&Z-=ONwmXkmbt zmc48(D-iIgZNht06!I@L?K14jiN+CzJfFFQtSZ4Yt+4$2|D0^^T<&z_@MzBW5fOAjkUbFUD=#Lb6r`=O};Be5S&sC25*q6Z$T0=(~ieK}cV3K|8Cc zJP_NAGYftb@m*Kd?+7VBmzB9AwRY>IT4xyzUd;q}9J(#QSaIkD=&dq7%(IlQhI~31 zo!RZ(7F81rwd@<^;!5Iz5*@dArRmu3viap~pl?N79p{+#ZHj*}aB>c?<1eIyO4b!R zw{VX-mA+tPkR=lYST9h4GRPW4?1!2B7MLLo=Y)w9Ex5K(>~VQ6?}}qh5LH%5??sXd zm*nGS>qp>nUusUo%}{K2Dl^QJ}IiZ7F)$gN%GGdJ4Vxna?}!FkST z4)Bju<4`&|u85BHd{$O7wq)rYevd)}X(i_tRWrl8)M=lIlr!>b5i1(o8fe93avfD8 ztJ<-jk+HEmXWj{)a&rnu={q`Mqo|Fr5l6Hiin8bt=C%E@*h1XrmmoYOt+b2>~ev=<3Cd@oSf=3 zONnN9pnL{eU^EpzFE1DxJ5jxX(pq7NlaQ2?)99r8azdeXAJHzKA#7orx$|X!6Fxj& zEOamemqM(xuTYi?b0OF#%2L*RWT!(I3x!Ogbzr(S(!R(}DX@_EIVup6pbD*Y`}Py7 z=HMr!`@)13ptts)MGz2eDW9Jl7I?1=dn@r>GWTE)_g?hqlbwD{j`C;BqQR#RZ{JEf z-qP}K^Vt&3jf`qCj7B(RoyRXv>@BNjUpfyRrv&+dZi}EDM;@9BQoY2zB^T?TlD{wF zI~p;I5hG6CYP6}tm>@zmlP+DdK4gxNQc6M`1paRZwAd%pR!)1>CZm0}q`h!9)_`@i zE*o*Rt}G%6hNV95hX9PpS!ywk#w;R*DuI@CCHqoLu=0dvyF=)Rc@(^76@a1eB-hCh z2|qjbfz(@ro9?Qfocr`?H_gF6%NNI+GR4kUzmKMBMIvsuYV{Dz(oZW8Hp8DMYGnK^ zIx≧71Y(JZuVI@M=oRRfrd=C~PIC7l|x4`ne0_-bkhcGbXCBC`4qdkc%1Jf5E_5 z6Rogj97!t#PP%`PI!1%GO|mb&nT98z94hnMN9^cBxt#pScp1G!T*zQ;{0=~;_6|KF zeYX+wC7dnI&9!^qzJ0^YSJE@i&&P=867wSKTyI&9th67!!iTURpmw_7+Z_EecSK$N zYuk_jb$JhnPQ-oyS4_9iA{GUS%=V)%`HVm2V#u(uEF+2}nZm-k5y>-?7!!=%8E=2? zwgB5Pwzld7Bud}w81JgbO(`=@F^j<0gib2wAw=-TAvsntfn(vv!ayW5Czs%3BX55< z$ywRC%Mz>%LYE*yKIqu=VQWO~%L)4eO%}y33VA8Bi zfFIF|gIpvlK%tL}b1p3(w%*YLFS^W|IY10Q7Y6w5l8Gcn-bR`x1ACDiEr&VD2n~qd z4#21*XEODa;g1?968C0b1kNy35wzM~+byrn&_BcJ*hKB8r8iPNk9vwjs~He#2q$@S zSRDvNY=>#{eb7UTjfO*_A7wbjb1?0d1*{C`OozM=)iyADnNo}xdtJ^~!EJ#(C?gGz zXVFV1FfN?Zqd7aD=12konC%eEySCi*30t?M;)5i$jDuf@02HRJGZsTPb(5iyQOvEg z;FKx~OZocIgR-h<^FrSQjC5EM7+naxWr0&fkPqG_#I+E@V*N!(DzWo2#ZK?Hx^Pmg z$%+ABUp^i@;K~}4{5%*EfLDW7Ez0J@SV$5V&`2Pp#g%DuX3vKDxN+3hdlm!i!i=d^ zByg7z5rWU)zc8z}q#F0V1Z3<+Ytn6IcAt|?oDCz7Oa!PB$3HhhQ}e}wXgL6`ROIj@ zpo)A_7T1kyen1z^MA%UgTXxYTj<}6i&wrg1^-4JPBbw|a34ZHiciJ-P`mbORp>et1%W+G<`F80c!x!+*{z$qOw(n# ztlPf57H0>E$vxwIQT;T7zDVMJkR)4MH2Sb$$c`RjVkaW2j&gX)-B#kY z~oXCyTG;;5?pJ^w})v&8EvL-+Knt@OdpyyfLUtBE$mv{R4DXe%1( z=45zqTLRqqQI+F{ZRd3{?XMHF4Eaj5?Co5dnR&I7u@~ICSHGJZ8V?i;DvyJ$;Cg5& zra=m2B--^+T(tZ6ezxXz+s9x}8Rl%#}7z{74p|k_1W) zKDV6nN!Bl)=vii0+}et`#U#j$zOt{3wVVB{zbHk(JuV9z7F((zxT!#_GIyLcd!^|^ zupy=`qPLMLh;$!&fM1xz+)&2xB7fRjEuSTRYoA@?%%X6bcHY|s6bJMHDSUkbMvD!4 z?Hi)e*0x1GPOCneem&otS~qeviAh8;M+-lmV@pa_Bze+U{oUe7w?{Ql0~j-5d_(5x zmCz&wQ4Gqk)7d~{4_ET&1!CAc*Q(TvLYz8=92odxyAj`bK${6;khevr5`N`mceTNh z(qC}$BoM7av5bbshDjlH^m;FCLJamr;KNz4y4>tRdVv9n1~j@d5|53eh7Dr&MpCkd6xMJ zFoX`pxvi0r1vEM>xajGj%2X5(zxJ~m(xuVPoJ1W~7I)>Fd1?Om+YDl6Nc_6c(B>|m zK7PCm4k9ci*%=}Y!rnbPROeSt9siXpvGZ`P3){xA{>YrOpuIy8(6fpJDhXHi;lo_? z7B0?JWdQglxi`v?R!KcZcodroc3K39pb!>_apnHzzT)v`3du_@BUR_MH|8n|K=UQu zfP~cGGahx(&#HU_s2^)lMy#Stn*DGoxy!;Uv!hyiEoGM4WC<}r3N@CZ3Txx!o#o5w ze+%a>9f!9?UX6f17)59tZ@WZO{0lfDDKH$gfz*RmQRt?S8WR!DAkA`@bAMgPs*%k4 zS&RNgkQJ1zm3P8QeYH~8_h~b<_6|8lr+_nnCKBJl$tA=S4D2vfD<&>3KZZz>o@<$* zl_(_({j49rlL5B}nxAFdvkdtd_Ts&9n>HIWJpLk#zIySwoUi9j(>wwrycMI8JjR6- zvQ_c5boRVoi?t^Q3$4;B08~U1grlm_mbs^Vb*EXAQOg`s@2`8xKIBwb2@*w0Ogjh5 z$hEu;t}FY~UeiD$ou@zK1dw!!6Xf_2aZKP@NAMOLyT7je0Y2ar?pR7R-n z4SAk?YQzRPfGDC#W%_N0O4or2#;*Eo<7`Tk*(Fp#d6~y~K}a7jvM?u9%`X3{+hcC@ zqmJt8&1}}HchYZKuipCoZZuEj^U7g1vcaaUT@%cDqks}+Ix>wFfG-B9jRZX4p=msh zZ1u7rPp)I^Z1TJc$p0TAmJGKgx&OQ{0q&bsGg-&-?&aq+-|V@(wcLH zT`k};##L(xiO_Lk>c)CEk}PJ{8zWvGJ-symet_Cfi80YJs~2WZMfbN}Ta*v9$C<&l z29fJKQ@jyyGmZzx-Pu6+@Or8$Br^!$i6Ou_a+hU$T+*6}C{h?`Kjcr#ZZ(pBdMBgL zbov!|fvuMdAW4XT74mbynO&2zxLRbukqmNQ@U8{4n!iB#m=f{K)qpV!%l`$eIQ_tg z`jktOlZ7sH*wjn_^$-H*C_8-KuFFCvbsM`8Nt3ypI?4R27OFvFu@r@iPvh}XmIFy6 zVYOqyyBi1?&*K+El5Aa`i*+i8?^vmmX?ZSiWWRm$*h~;J^3|#+WPRr^+K!GMUBuM9 zqNfD|t)hkizW;#J52z%xP&6B3I&A(5l!F5R^q6$%?l~4*Zka~vzmbf)@pO5ig*Zx z*f^LUuNKCAS=M!cpJnWWguax0Dl*{?jvcFNkpvy6iEWzNErKuTZew8FP@qbdyH>q= zRjA$)ayK@jL`lI`C_p|(Dn{7~0UUSK#SnVk|NL+q%dfMcrk@EweX{JttXw%T@s3v! z-u4~{RRCUJ(=1m~+eB{oF8v0vTwwSoW$=(-z}A`N#K=}S#lB45K+Du+#*_|O$HyXFN!OW$m$}nwQzDW&3f;6XmlfiLH_*HE8-mr7_k6b;I}R8)ezK2 zqNXYC5o}I=a;7`}rFTo7m`{SBWzN^NVVs7NWJ8U7!%@_lU zlBN_Fs^SG>+|#s&z_MhLk=$*cx$2$mt-&3l22NyMR|m1_NR1-DohQ`c{=+XrOVP5( zP@Jdq6#+p(Pc^$~1zOS&gJAJ>q+gUi+^a>T$#Vc$6b~2%R`ty;-;LcC_TVej7M<65 z3Ym#*{aWLLsRXt30VA9#bEHR(#4lErI_ZMExW%Qjm1e7Um*2!{f|C&4aBHs6>aYh` ze$o9j5?V7cUlgY@{sn%?eazb<+`s}@f`KQVn1OzUQ$q&M%PqnACn^!NHEV!fEbA_> z+6U4{c&Khjf}W3(@OVykIWJ|FArgw<&Z>15kFM4gP@V6I3eSSKDl8F1x{2=n4%AbQ zNZ!w0;G0RPVCU(NexVK!4xeQ&GmsW7`l4>8T1VjuJulZcb$aHi<>shlDkjm<<3re5 z@AY%MO|)BewvB8>XuXCfr%6U25yFtY>34pqQX@(Ge9UTfHMJ!E8(dB>7n%%hWxIq* zZz2#V*F1(tn4j7L1y>8Fl*fg`h}fnu1J)c47ZFRg5ROxdWU`}0>#Mi_`ZWT_xdy~$ zLV+i}9svu>s3|gT|Eot==jL5{VMy<9e!$!UO%-;_yU%5csj|1Yfh$CbkcfuBc;ak@ zqWxHuC6l~rF*=tpQyD7)_ASn|4N$^L1uzv zaRJ+$yrQS|Esi7IfHh_2RhYyBycIL7wG3IJvi^EH_qV+cF;kUKl-y`t6Q@4p*UB&w z!g!j}t}gBnb8*m)9@_E}d3eu$A6d;J3GK@)^ydN#a>q`aVl7zdG!6tVM=P?i1>xtN z72h4K-3Jt!-6CBc@DAjEFqL?T((8uqAFRg;(!Rp#kw96|C{b!OG#h-f zWQzV`b`^KxML95^J-eu({`0XoJ}1$|pmXQWLmpE$Qt>owe*1wSdz7T~$l}8bqs;c7WTqA@g0`m^Kb5 z+5C8W@Np=qOWaU7+1W$?y3L&WycEi4@y0W=WIRV0_0`kjJ1)=I^w?}Jm%Mr)DVT%{ ztjTgSFxEj9w~~ria)CJyt+=9q**C^xqFddpcT>d1h%l;_?`Z!!rCJ8vx_BK0u4c$g z)tsNJfq&8tK<-0DE%QGFCIAGIG*xac)F5@F(KK9Y`rWhmBZnH?#YHM(p{f$Q4t6U9 zZxno5RK;lKPEKRt*pSpK{WFbwpq41n5_x{2u*CLj#m{Av7sq1?)(cJ)(=OfATZTPK z$SlaV4SVKNh_#ht8*n3l_!O2jt7$nb1jT&+5GFp8#89V3ja5BOY@@yNx@ww>&?wrz zZ>61&(zgoDx5T5eUga+Sn@j@I+6^!4bte8koq$a50mMEJAUb^%ej5Dlauo`2vDfc| zDRrp~jRB$$;MJr~6T!HwX6i(iiaZ-!PJmArRGD8rTkTAP=(sps z)-~Z$l41GevCH-n&PDvMmC)J8fR*?hC%6Fye1EWHyt_O9b{JWT!=T}(IAQ$(H~Tm_ zX%j+nfj4>Xb1y#u;JLU_uW)Tbq)-QC=uB$Qjp??6(paP_e zrgzP4rk)K^v`Ds-aFDYf3i6?;si2}_;Nnm&%qKfgPDoT0r=K7*C#T7Q4x|8tE~Ra- zDc7!`cLQqn5Gk`2Bm{_u)6%P*@ow^IWCyb}S5X(s7+(4YIK2V#%B}bDCjr?? z4jje#MNk%@n?_Hc-T}ed;R#pk!3*S{)N2@a;A{0{K5=y}nt#)5_FFb>*Z{94HFd(C zfCA1qwp%}JJod1MlRZ2LUsi{0w0`q{cze^Z9{0BG|I8FJ&!U9P^R!5YkRdW85mJ;f zv$9f}NXA7nWU3HKh9Xmu${ejFsbok4r81QiP3rj^v99HL^MCW-w#Rng_jcXuYU%eo z&+qpb_I*G0Bj@c~(e{h(Qzj@G^JA1&W{(N`5z$VNeQKbv;DzT*n;UoMA!eFDD}MX@ zT8$?sJsYDD_?~wGMa*M9gi>=PS?lA$rjIy^$AW{!KAHy{y}T7S>MCk4ZWIIKbmpE@ zr`k7xVS#ByUL~)rcI}E**B;}X#y~IaNJI2-02e8*76%m&1WHeN-@Bu)p zX`sKBV^6rqDpzo?V;~7a1`(h{X2omVuCa6LX;T>y0qq8njN&!n?bse_>KI8~5O$#z zkO$u7_}tqyZJr5i$R7Fr;)2Db|BEdJ!I<^P+D4(kDO6<5CVE?(hui(x zzw08_bmGORRkI4)@jFi$RZ*>3kq7{>0FQL8qpMj0*T>0!Z$8V z4b)t2EW)@I8yg&L&sek@ZN8}4Xd_2|HlF3Q?NMVykYtxTtg{w$hBgK(aCcZ{8EWH? z;OX>W&yBii<8%Tlaf0SuiXg&A##KxV#4d~66~$OBy4ca<$7dP&F!SDXL6~_!)6lv9DI~Sej69%e)cdIBY9@y@!1C)xGMBwc0 z+sAvB?wfAmmycGg-s7^V#nt}R0uaq@47_ALs8A3dSpom2VU;O?Q#*)) z1^VqaLvyllCF~Of+w_Wbj5?N8*j?1aF^B7u}Pia=m`by>@b z_WJr3_%&1^ZU~SKBO&{S9zANL+ss5^p;`9RQ);64u+P0@j)7t}{yE)y?dp9HC) zvJ&Kc8DnKXgAwFcwD_3*0VOMI50uwfhP_y-u zzfWjG%KV2J3frUPtb}i$^>)+o$NWyoQ`JrNAkr~3i8tSc|mfoiJ# zT3l9?Cm1RYd4*7P8@1x91?$^2L?9qjx@_&%BPUAx30D6WMT2mN*v0wg#`f4h6eMaa z1(eKCQ<#btYt1WmD~ba9m#~c)>*ig$>>mJyco6xx0G!eZP>qDtA62DIMK1Q1+gULt z6W&CfrhS&1GjA8Xhsj*D){-#Eg}@$FQ7CKzf<+#7ch~nsG0z}CA)9@|piAd&6>Zjk z*O+1Qqe7)@ZSI2FRUS>5GJk8;22ZCK>+lGQt&L`(`w`%t_tS#{0hEa%4`S{G`Des@Y4 z+YsJ!?Bd1Q-PTsG)6mEmh07gcG%C01Fd`_z$ESFnc4s_c1ALkIhS3AQp%Ifv6mAvQ z*c#O&GE%WM6+iUuzj3@_WY^ZpG0H&?vz6EoDIz#lQ4p1GG41ZvG)9(=XlWh0B?yXS zNP!;!{#43dXcWKo7`~Z~EI&UV%b!{zikmqz9eX&3Aoh6EzHrGxek?H%#gsT+u@hwV z>eZbgTxdQyAE}E!0vT=uvx@rr?>ZubraoKE68qMJRw{enuYL>6g4L~&H{-Y;`9Tyj%9E#0t0>NY z=q2dmvbPF^WN21fCcU_S7`)r8GwOgE-(ykuWdmt#EPUlTZ|RNcRYgurc|oE zbnVz^9}ri4OhHK(w!TPrBmB*)YxVKrS?yK`_ku5q7%_-Dn@pKZGHB!Bo|fQ;bA~#i zPI}6yEp42)^5YE&?+1}>0Jdgm1m9!h$~_)V<-9s7H4zp8If0*jx!-DKw611D&twXj zbRZiykZHQ%8X$)+T3PGI$_h29Ld2Jrhk&E>ls|8bu{MVcY}I7bN-7{q4H4M$^)r40 z>df0yw{M1@K|RqX%pB5+Z(ZqoEOaJCSV$qY&wX~kC(L~TZmyybp&#(ej8JW`-9h}m z#fAZmg4p*s7ADRR9R#&jgT{@&8rmdv>)F$5ZOn=#yV?w|mliwHo<(910(i5Ri!lpLsXt_#9L?tv7DzLXdt~I;Kkl|E{$kUi(*~Mt1`3>Rq*KBCmP|p_ZdhP z#RBh6A2(>m`elE-tf1#hPQLjX`7~D4BftG%J-2jjvKAd0q;eZMUcZZ8b`n?pcAhn9 zSsPS49=B+7_rV<;?9q5O#IK2WEPi<_BZ@nMG3J4wwCmNYK3uy@1T!m{p|Rh0FZ~5? zM@HuAv>tRJ&V4FR3@mjc_ey4Lb%URN;OM9dmBowXNc3Lv^kj_>KaJ)$Kx+!FG&OMA zB!dbKwOct?j-5DBIh(-0!>PWDmjXJuT4LeEAT=Oq{;ucUJ1}NJwJxeax?5ob!OkTL zLhbLh8`92i1{G#)SU>@TL`ls_;u7t)>u!BtY~zSI;*N|{*0Vkv4_|X>wka6X02EZe z1s1=K#un#FR#H%^+v+<{+q=hQsad;bJNlrXA~jV}gd;|aA)Z=1`6>!K+=>sq3mQlp z4;Xms_3IxlyOys&1&)#jX=!SW7{oa0JJ0ueehsz)Ob-#zUCTaqEiZmF&3gB4m`yVz zgIo3eYIxuyzsNT@W%&0~ge)*2StR6^Dfm8U^cMctikwulW=*~M^E3JlTyULM>f__H z;%i`Tu1S@o{@2B8*IfL=7)nGKoiPN1yxza%#9Ck->Biur!urp_!EIGMF_&J$nPoZ>(;&<6BCp9+r^*No03<2H^|$L86|uHHY^?og^In| z<6=>4$bJ+@550DZ$)t>g4EbKxXj?{D*1D#(Hs(b?j`QgHCfH*7?AbeBzMMtqz>2em zmBqbx_8#xY^=hLJ8>p4iY#15F}CBvoXKCoiv|AnTor*%^MVXxU_Q z(xfFv>&T1KZngt6j}0JX$)T&OGR=D%O0W)(4Nl{BQqp&Kv*RzPaz&z7?6gD_K$|A+ z1Tv#Rb(4;SLPJBt{(*Ko>pu|3Fj7^Ajz9<)gewaJEF9VqzUa{&5Y2Z0W~K9#_pS9f z=9QCMJstGXl$kHJ|M<)7Q|dG$uESIJI`5N_^~A(PL_xh4lmf;mS|6HC+Ia|xp1*$$ z>{04^9Ublx5%(M>VMII6G>WT`DtRu$;7DGl=@eEknLJgCtCGAp!eyye0LhQmwfW1n zCK&KB18RDCaYHGrQD+FdC^HyG9eVOrGP1I&Dj+s-L34D09s0Lu+`RdBL!A$~=bX7g zHDUme#P9CI%SrmN9jP^-sbu>&!e6|YH_Ht`eTU7}bqY-0l4d(s z qcqrICmxtn}T+h_B1rb5aQf)uIY>KV{i1UtG*wfIkF6=uJb7eH_dIWhc^av^l z(%Keq*>pfkm@WHxGa|TlD?q-*re2IGM93nDgtvG1T+@Q>6g789*HXn{!^gt8hBBo= zoE=FL#b;s^a3xjAt&RVc{SHfp6=yUS(G?K&x{MO2;#uEx*553N-T6UnOZ5?p6O7EX z8`0Ljb6F9IGp&;L=1qRGP13Vx<*}azjHQB7l`*r91|(YsG!J%;$W;3R7};R=wf4jdFA>Yy9Kp&@P};pwsh69H-Bn za|{Y2^@;DF6pOT;j0sC>r-xS34qdc>{q-u#%b?a+r-7l{8_mE#4dz56p1u`1fHR? zVR{};!oaBenr@aguPJ>D9^@hfz$ph@&-l^LNApZSi6;7?6xPMt3s&U$>y+M1L| z?L3$1VfuP-rMcwEo%e-Fyqks>WPh#$3`y8Oi^Vx5!5qlST0uK}}!WxBh-Jf>}*C zy(+FXJg%iUOi;AI9RFyMo0*xoc-apYiIjT2LJqd(#6>uka+^hwwmEyWi1vSDuCeSz z34N=IE1Rp{jENN>6BA;s(unZzvWums5UJ7^4p|#PO%_wWJds=G^MqyaBjEK2W4H0Z zEqwm=Tcnhe9)G`rmw_{X;~6{i#-HaEJ%|y9kKz2QE*~!{TKF~%coDAOYxas2K{81; zqU-+J?I<%q>_l@)5P$S|roKmGP0gM#!ZiV3M)IIf6v8k{B2tHY8OqrpEC;!3A^d_(Q51}zp*0EA%t8&UPn-prdH7FOi@M&Wl1!V-6q4&x_G@bUO1FtcP@pKoZ4^BMVB6*N4gDNjZIKwvRemL z(lMld9cgS4^ync5;AJRo+ePWu8NV+&I%LMkG!a4IeJnJz1=m@I*pLqTcKJ$AjQg26Vn98zAjbulrDoWo&B};0@oRpHKn$5DXdq$> z?GFV`G$o?10GhL376pp&%i^dmAU z20Fq8P$~L)K!vnJ;;l**1@*U)59l3V9>_u#5h?LEh+QSvG-4A?+Gu1X8z{dJ#(Der z2=Te~Uig?zTgRCPSw!xlLaZ+xJnmu0kX32E?H~AcY{t+a(*Cwky+Z55*ig-9XzVbb z2Rt?!+*kBj6b53HOii<*VBzq8k6OdHvv+y$2=-Mb%rO58zC~pDC}VX%y?W#h`sW!0 znK0loTF>0b9=q0lNzb@5!oX3Wi?fsHkWz zwDr!zAttQKWAeO+JqmL?-6m_z?M4(k3 zcok#gc>nsPWzKWk4vMo8Q4Pf*4Xhg>jj8sxP*{orLV_$r1MpZpoSkSe^pU5ax%x7) z@3iUD{f-?IlZ%n$d=@Rvx9Hcm?`{Zr;3(;LK~JdJrNE~mK^^gZpN^%omU?+ya}U68ecPHN3J;OippnUR9Jgj|`;858Hbs~r&!NU-qy z;oCfMZU!XFf@>?=`7O+VH=!gGVck4{LjoM7_%ynt4rF4{OUfQc5q*eZKUmcL9apjs zB{tvWf?Ev|^E)6Qo3Pf;ySnFpiMn8X4#x*1fk}-DJ~XWKNUoRCEd8J$nf}BHk6f$s4i!vGz@gVu_m)N z(!GuFCo)k_JG8jJKlYh|3m|kB#VQ1q*h{P#H^IZEIDdg`4wUW#woSNGdZa(p4=^_m z#rdn3yag$h!PB>FJIKhGhHw-q^xW3cT#xtfttG(YR~Qm9b$b7v(890pBqk)lrB*`- z&R;c>XfxfAIg#~bFDb!%Z;aE|@AVueJ1=f*n{coaf_XY^Buz~B@{VDFyjKt_2xeL6E!;HkacTe0 zylcsvvO_>t5Ar-Wv8keUBjcNBHt*0MicbUjA@QGs10AS71NS)2i_CIM)d^lL;vOAs z=XBGWGDnwS<~?Y273mLYH}mM1tY~R^WNvwyGRvprg4g1H3= zP#WJ-tS$LXH3(!KCzU#M14txjU1UlH;k+fFIYefSyOyg@T*NPP9V^Z-RH)4`gk%20 zP6XOD<9${Vf^hj5Is5{*_JF^sKMa1bB zeLGr#d+k;^1d{Z>Fdy~?AVmOBA-nNQ6?Z7^ttZITZ-1;ea3(}J2M)_nMjMunt0)p@ zB(*S@OttWszjFa1J^@a0BT9S9J|R%H27ukm`;)bt>47Rox*;|N7h*^UdFu7nQzOmU zCBkc0V35QQGxz3qO;K9drPv1;HN;a(hlWB1t+@HZ>Qb@w2&~y!i5e+=n%_eG?#-;Y z#d2E-1kxnPULfYT^e0W)75+BP3Wrz}UQ+B~l1KggK*TUmokZ&xUCo|C_-s#PZQE!YN`yVlcb~Z^ zATjA;vrHATEQk<9cdFWJ!R(3&HiH%-O9p>d6+266)~)*O*dGv{_vy)wz4+Sz&o@E` zETe3wAl1@h-{i6(sF_QAw_unQ=}$rOc@0mf2IM&rx$E9%d{}Za*u?Xz%UgjL?xawu zT)u?SulLhyX0YDGcR3U?&`LMi3V=#Rt0@cBcMhk5K{F$Dxoq+jMGOQ^>phW<-K)uH zn)PO4xqz{3UJ_jbfl>x-Wf~b>>LXpvACY`hp~*#CNco9ttR!M)2chJrr+mpC@Qlkh zb4wm0Qz2U3|4A87*J2xj@ZQ1>E>yF~zN_F}^$L(<{_(J}!@dbBj_P>Bozm*b#m?UZ|0{B4n0FpgoX)I{1 zsP6GlX@VqUGXqTwO^>PfaJCnhMVMb%Mn$mhgLYkZo&S{9?z|rWN0yk_RA2{c#ofec z-m9K=qlays;>nCuQAYjzu~U$VKh#x_Jo>X{se?+o@$O{^pLTGs-QA?86!fCBLH;Z5 zhoIz4UFpM?@VNo|#`#BRVgIXv{B2#ukMA>opZfi040X1^DI8jHqbwpnNd+!e;T)y> zT}cv3xx+Mg((kfErJasWOFV5vkVLYN)7Cj0->0fHA6%)nCw@V(w@n<0e|gne9VO^pddwGnY7puqgi6#7O}@7elywATK-Bf zt!N%EsZSoS+OgB8w}KVO{YU;cg%A&;!a)$V8KklOSjQgXDL`Gj8A%tIvm1mpx}2)w ziqABD_}hjP{VX7lSS5BaOS+*8HI`R1Z<7@zPX-Vq?BMIpz869G< zJ|ZQZj~g5>G?3eQ+I_XHtyoQK6nt`eviiOC3h zRQ{|&2p~a(mTsS&zF(*577fZNm$^AtMvNF_q5I$6mnF27{VElRY+t1)5weCYDm7#p z4gjde+g_|2B?G7eZn=ZOvnhggU%U7_I=wV!l$N8j1-x{Cb=f7LCbmNWO=u7XL&il03b6FTEnhxfeM3{ptIQ{9V2_D`y#G3}&^o76V#V|(V zPUUe2z!Emt<^5mKo+4@G$cei&vp=$FOi~(~=ajW;3$q^c&+DtIs&aPx7^C2aEl%`- z17e16fJk4kAq0c60&~NxY-Mj4&w5I4R5CSUG-xpZX%V!fpBlJm$%+SmWdvh!g(*Vi zEoVvv1ly7$C-QqybIG1dLay}5Q2XI;8{M>(*8rj=2|!{Q;EHUla&RbV|3H_eB;b7l z$1~t8;~elJ;ZT&K+9xrm++Cmd9-;Cr4*%9ZR`-v|BqRxfbuWM6xTV3wQ4}o2kLaWW zL<0t9WUxzi%gZE&=$%EVjuhpNZ?q>mAAS8Evqq10zu;2gEDP0mq#J>jD>sXAas}%O ze>-3TR8Ip6kq+3Qnh6vKq_;`HJP^G6U1JJH64Ppd&NiaBk|k@j?o%qO5+UV&{R#6N zr#%u}v$_(p<FDU>(|Bl}Z2cAN&vU}OGIdgol3H%ma2jZd;?h@a}e4Tu# z?`TDfwIfrg!|M9>n;*S9;PH`t`&bl-;;k(vr*Pb6ILUmC9F}F!7l*l;iWp4S*{f+s!Y?C#1{*KsyU7Y4E)Ms!y38rwreJf2^mcJt zVQ%g#?$NB^oh_bVlxFT|%>OXTpYe3qpFeAcbgq`9P-K3UntYT2%od4d8x5^A6(gAw ztTtk|UBZl6w1?b^E!;xc7l1AKA5e`#h5|XxuG1`*ZdJ$b|4c;IBW{m|XC2KbA(?8GL5wPDHX?4!`Ty)rD!K%g}1JgBihh z4^~K$ZqavsO(f?#K|h4AdiH|=5~6$Cmu3Od?Jl`njJr97^JwA8=8;x;`~hPp^iG{Z(EjvO^r8XvC!|tZ6h-?B1h#|2PYd!#rb2IY`YU(0sXWeO*EJ! zlYzO$Ss$OBZ-w3klXo4$)sRG+quK&2h&qyAlFqA;@o}^o>T5%Xt;wj#)e-M}GQMCM zfLda(0BpY#F29C?!d1j6g7ed)s3;hqDER&P1q`b<`9zpTKkLCFcsf36OK+5%LUd4q zA{-S5zSV3+pbUD0d|TGBa1}E&4rK4=p~gwtf!v_Yp(3#=am@2{3o6l4e+_^cBP*0@+&J^xE zdo@acW<0e{x$A;Ap(d@xIogOHKBz=@ryq)${NPQqO-*a^?=AfO zbe-p?1qHyz7~J`|qDQ;y59TM+mLTcO^1LWYNSHQ{=OBJ}t=7Ky%;$Cs`=Glqm+@mp zo*Qs>H-V9iX-S<|UGWGvCK*j=wYG!H)IyD0rOW^<_Q5Sg(c`H$t(G3V>oJg(b?>0? za5LJ1n|M`eakmDT&PbhE0{i0hewl|21NgkXda{s;3acxsiRom?2KWYP>XaMwO|ZLG zp0yg+r(IYw+u5zW5Y8jOxTm-xym2lBG&^YN6vR4JkPNx^{N5GmAO@2|Ud5i$(YWMd zIX18P2ip3O*pd79?NfJb=D?gfYD`{lexnCK9DCp!F7N zThZdq)Z*i91k|PP>Wa!~;4RJY7U0vj;e_1r3#rBW!zPfGtP_^mEdXqpv79{31}pgX zWn4sgVH`Y>+SDX8EHCJHf9WobThaiV&j5eUD=ZvM#X~bDx|ZY!r&hV)U+ZxK(%Jt7 zzbnQVKv4O+UTLF@b(4W?>erC7%W?g-k;&Oq$FFt8^N)5J?>*VSp1*5CLIy79qS2N5 z7kL_s$AASt{#KX=@25(rs)!t?*ol%+Nuz&Yn2>p_GFMJA{ZRjd0SKkylDYVDap@t6 zq?#|<4yM#s*nlqP?3^re7XKsScxCsIBc*?rQY1%?i9zMADE!Bjc_v-=|Is3Vl@|nq z6ToYrmkt?aPzdYZEvrZ%cSw7KN=KG`;HRu0(V%W<*RkWfi?QqX)F~`A3ns+=e`_@6 zp9uO`_!{PWmT0nE0mR>e+k4zJzCk@FLHu5G3pU(UXTVHmS)^S? z;V+GrtRQ%OW1#pV=R*jOn4JS zUz=HB#-L`jZqK-G_ehz9AJfoPZN(Ukp0sjjBQn12aNBtm|CwrES&Y(2;mEggw5Yfe zvHCtps1Xa}V4y6?KX7b&n)bZ1B>tVJjoL<6>e1s&uga{BfRN}cUtE3vpv+0vW2%Ws z;gB0eriG=ZM~|161~3a;)~@}Ci4*J636EN{Mk;Fw{i4S}f-U=j&MBo!`Xw~~R^c;F z+j`pFEXI9lv6u>L3uP`u7PDDACb`PpJ_n^PbZ{7q-EynH5n9(9J^Eh9ZV}6SKKRSm zf{L6@u>s;Nk?SD=!!_|99>JouKqCd=6rKgzwLKgF6 zu4GPn1F@mtRO;l8u%gm{Ppm=rwHhN|cBi$Z9qrt``z91WLPvsGkg(8Ee7U$XqLI-k znmsX$7*_i6(wOS$JR+wz7Vjl0{`@n`kJ%@Ji;!})YrE{vmfd-P60+grM3&St@Gy$2 z2$JBm9-tYPk>3{wX|39V8oRu%&GDtInx$vRwgG3U>{U4u+q3~gEu zb!Y-i=QVU7Ms{>tkS`W)E}bUkm<(N5ungL_0dyMwrbz3!-2k6Cz9;Eey#mF z`Y|dbeqeO};3{`X`ILC|V1yf?ZjQoBw>J$foX`)8tT-BIAx(x3T@bS(HDV{TmoPDM z2$=N1opLOMIKOiKTcR~BDUD)X_-J~==$@gnorD7F<73FGrk#8B8VfSR37o=kRoa`r zv^xS_)PDS_a#dbEdM#|FhpxSI0U4dR(}E!ABc%pclZF+XcGeIaoaR7wcqy~$49ncu%2=s+_qyLK>!qHM(fC@5$Qf2r^l%DNn%Q&gaR(1hTm)(Qex zIA`i|spb_I=b-5;|8E_Pz+;RkJo~g3LlKYU`@DxQ=!~oAop}6m@>xn|hc!H6G0|d- zWHVRpczAegOzuoC%$2$J9;J~J^z-8(OJ$muTQ!z7fuhiYYL)>(T)8)pAxVI}=^e+W z97;b*;d7k%F#Mawe2~*9i{YC!qng*7o-l{EjcAXvrlOGHM3?d*^gJ{DKntk5S74NsA*b!7)AS_0_Z^My^*CCfc zk~<`nOz+RXm+lg9{`?QKFu+V1B0h(Q^i@Mn7pP6hcl+0`U#Gu&x0#`00c7g>2J_{I z50A`xlvnBWiywo8qo|_~&R)Mh>uL%zExv|RV9RPF^vukX+ut~S_Vj5HV}#_&{45S8 z;c)UIiv!8*G&>2rY37Kc;apagD77Wa5|Y6MX@MSH&qyjQ&DzIfuDG;!#f1G3&O|h( z(nOm=>nNuPxQMXS#y&}<-JanWP3ur>k%!u8X;tpp#P9?NL58Gw`7*8orGX-ciqq}- z^%CRdCnCm~rC2Xr>WNb-r7XKY-qOhtxLkonMSM*Ca62(^|Gm`MZSHA;))XWYiYR%i zwrX``%m*Mes?Qxj*wM8^E6T;*lumPsnOXM_$#W2oaTEm{1$w7EkzP_keEejteyJGxumrZFfj8;+LD;k~Ox$vaT%y=+VCQCx;*cgqh%1ycl7d6k) zGGE_D>PyUc?vP~BU~A*-G76lS5;N-92sioq+8sMSx^6#MS62kavHM?Pz{J~pAe9u| zTojJa;$H#~yTFPVzDslHcDewoit-=gi9%J59=aNY8lKFhVSR_x0=V-YV&6y{=Ed!e zScaWQrLcvP9Xh7;X+O~nvPgmQ4VpwI7--TN8I)><0;R@VP#raNG!j3>`H)ctak&8E z9yxPnvF9tYimJl9$O`-V#{;V|E`g^{>oFAvswxb}{6&jIzD_E5Ae{rk_zMU3oIV7^ z72=mC{R>44t-W;SwA2?#(!A3szBEP;RLvO94#2Np{k8pxkdXX~S$q7&rcvq|yin=o zWLA`b2ms9VM0LI_noBE+04BAzB(J%u-!9!gx8<+(xdpwD7v-f^ls9L{YZO91nX>(j z_H7uac7=a}1}b7iz$Y)6;X?2-V$7KPHg@YP0=yjlp0$V8xgArHNb@kJ6b%3$j)^MS zl1jGL>)wpYui;H?T9`d+<9G4ln1+J|M6iMoPL{um0+d=*zAsQO3p#63Ssw7p;mgZ# zF6mh!6qy9@TU3iQSHwB-fTm$e!!Vs|!x&-SmG_y&8-5;im|tzanb_|O(G6ks+ks^5 z?m~jIP|il9|Dh#8G;+VWh2sY1Xm92f&wkTq>1Qqik;awC%22u3?NT9d_vor0aL)LQ zopTTBYj+|=B38qssS)$BOv7#j;DJdHeWp_Wv2PHOUUoiYXGed&5Cj5^kOY0uq>o%E z5-DJ)KAI*)sC?oFnQVpaweMtvWV~!BI24802d6KNY?L*wh&-faLD%)F-hgdwnq>I6 z{w)2>>r2%`ttZBw+I%r;c*O^imqlWwF z5X8{}<(&ABA!UuCi4tig7a>|+pG*Y^k4CXLud>=jHb^Ltg^@H;Vl*vY6O43FAOc+| zV9XRI`VfaH*l#x7yT$ou;=qR21$q)CXxpK`-h<>ew(1wGuVm zje_=o_n0L#--I0gVD;Y(4+>_TK7W2RJU1%IZ8So_>MH!zyeO$8#PR^k2z1_l3x$c2 zK|=ZzJlEscS8YGm&%dy$S|zN&Lc}1@^}M4MWvU>YVplc{tN=m=VNd1n|M`F7)L5p# zGp{M>77gGnO5-Q!E|~C_aW=?2u(y47hV85uA~X_;ecFr}qu{Io&%gr?hg%uNmy8&Z zQtRhy%Pw3OsAuc?Mg3Ea>}UE!Oa=&Kd};#3#`d`+JSVa&X*b^Bbq2ZE^2w7KGc<2v9)Ul zaLQ#Tpu&i6ZK^dD1>hPXZT?^ANAxEXwQ6#}->fs~VW5 z*zJI}?3eex0kacM54|wFW&7DHmM!~>l3b@_d!3);F@N37mwdT|%LuwxqYRQ(1o9S( z48G@WP(h=aGixb=++c+RRdi!ab#nl!#j+Uh7^)&!i!e@mBhyXFF=|nX66NnZ&zd>& z#pz?=5qTeh4oKt?-H)Pa%i5#Jbokn)#`@?+Twjya0N==8-u@6pg%jQN1yEzpRB0^Px*l=>N(OyKOjdmH!q zvxDD(74XXk880oLS>DZa+>}|f9wi-5Qv3J9p0+2?-GlfC&a6WZLVl-p<+>VOnoiRu zBJ;6h+jD8ihz!k*1q2a7aNax>8UzHXGoN|BBGmCIIq(y`zNX*VQg^T?yP6(uPvjS? zDWc5)8xYF{(pXeVitjxuDyUg}aX~UU)!lR>7tNoK60aVWl=$KipK+;dmArP8g+-)+ z4TSuo$DcTC9G87Qso=t@iV&tgsi&rPA$84f?)~#|3~)zDCLsmCY?%@T`$6_-EO!Zn zjQH@bbhm!ONs^|Ric_!gJA>j%z@JHb(QR8kto7ehDPA<92$?ZHiOfOh|MQF57Q701 zs`qdg%?=$ppfIe93?YubMa~8Tn_{F&g%;kvVlwSG_L_T!I+uPLh*ws1f)2m~AyuMk z|HTcE5@l)}Ah@{W@OcX%V^`hC2M->c@N0j4@1cLS0De^v>V|w>|3jlcPvl5}WzeGi zl$t+VZ-H{8+aCEo+5Fb?F#aYuU$K^_Q%yL zSLCWW?4&KJ4Y8s}hCQ*C69;@Lj#3%B|H?ppDfKP~eaOALcg00CWMdxGv)KIP?;2h3 z=pnD<^_l+EErcGS*wLB@&ZnUPfgHThsL8jJBj-TFShPk=e>2buOXb=eW z;RD(=Y&c-HgumLgWvxAFU#>1c^df8yunkoyP2;CnHlM=J;RQ^3 z!Pcv~xNw3ogrBI~=V8=u&4*__B-8+fay%gG#-Ki+%yH!zS>b;!o|4#H$j`T>Qe>== zrfhwN120x^O9XAp-^|w@`xB!AV|dpC3#G&9mqpp|b`#7WEdo z8W>zUcYL5ktI8-B5j9ielEHp7DtsZaHv(go5zmkJpP-Ff2M~m~;R?n?kP^oy+%%;{7K>Q8)RaCJEoaW0DQ@dv zeAu$o1_j~4S&A|S@b#zqk79rvDN2@lnnSk2HE2;%fl|v)$Q1DG*{_bDLOH-S`&HJA#e;1|7jzK@usWEs59fwSpf>3YZzQ7Y;Z*R%xZZTwCZ;mMa4D+p9 z%RO$E?s{~6Me-o$i{9G~d%bDaNo!*}>o=zp9vy1jW6@(ngXIUq?k)e^u*1s}`ww3_ z{>MYZTRXQun%+cpq5m$-pZ>aY{T~t-lZ9O>r%aU-TSBThvSz3-~)d+ zpbx}NqZ*`TQXQ}ofdj*9DBN+$ITVM7+BE@ znl$s%r}GH%4?d1*N-iv*?pVTsD3RR&OP8L7240G$-r?P$MJ=AID4N&tw>@tbHdW|U z*?+_Pc=+noY>L_`7)KgrW!8qexx@gqA4r!qFo8pc=U4!zPIX%knu2}cKp#~;S(*wx zPdredjV7ZaoUy3@0EvF1)vhdiF_oXns<%N|<3p2X4L~*6xh$0I+R>n^>4jr>e&h~n z8?LvZ3N>!C)YRzVEjCt_FPrZV^*+&e$V~1q(?bUs89NReaqw}K@+(a8e?SPnH!?C3 z5<3vLg)_8C@1@In{d!_nrZsBFM0(USAg<$JuZ^%c?mst5soSErUem7Adh?m1@6a|} zL6BEmTDp|1gXhAm$1bT$6VEId+-t@SW_Y04&e0@Jp@5rC6dS9cu7*Hju>RMQBlXIrxRXG(A z@IGJPAYe?5;m&5n4{XWzj3Cqtl_#3!-S*S>qRMlv*i6*V4UV}t=t&ZV4R@Y5@gtCrX1gB?p(Z}uEyYf z@l_>gRK;|vNux%7Xvsbx6H9=vvjuT{@#@uC#A@vDK7Xi zI;Y9Odky5hE4znRuU2g_;c_s(hHk4&rL10_8Zn3T^bF`ffnzS8#v1DU+_`gxS?Mk^ zRCKoxp+1AN0AbkcY z_@Oz)w_mL=8of!oPPP+|9dscwh@E=emKmh9QpJ6=NC_)6Eb0qUi{2ZRU}?) zefDI4SPP)%CPFlIoi!)7H|b)@iJBWtcQ=5YwgjyY5X1=#hfQORf&7b+W%)aR;?A-# z<*fd;M(S)?WQyi?mrZ=;sG(GkH{M>(CEHT^OdxLk8|)Y$)EPYZ@TBUlW*vxbMBJw& zvjl35c%9xm+79C1R+PwQ#7fAu9BLknfc8Uh^x!bZ&y04XpP;VX#GS|A_haKA;n;O8FZj8=AgfE`aGDn8{%XOQy%QewE-w#~}jfjSX_&U8RZ zW^!tL(|?W*x8SM{@Tty6F!e?uL?4z>ClU$M`RC6~X%hkgS9I>=k}_uT!Tb8CDPb?c zz$AiGpBoSu2l!OI^lavlo|RuCY7c35JQPy`>W~1)hZtfAt?v_LW`+T6J*eWJ%23Js z_h%^mop|Lh(WAG|8*+^AkY8As2;`o?KuBRuJmABCr$`MZ29S&|PykyqBbQ%Lz}DAm z5^N_AE8f&_o#qsO!|X=~PmphF#ihJ0=IlMxKM zO(%F4V32|cbHe-7$VhRld5s2H0xIBCBC@qHX&gd%ikOejt5(mTQKz4&H?I>q(*i~| zx-kh$j)*7Un?yQ0@tP2chFk{Pn|QGI2rMzNBs~oA*j-Z3l686#iMWImk>ddJ6G>Gj zeD3&}ifbt*E)&tRKSN0{f91+x7%4e-87mstKSvNr`8TFZ|Nhg0!}GVxRpJ94d0d5O ztr?1DZ!!5<$Af^`b?DGJ+*uQntBod*yyL`bN_|uQ+qFQH@|gXN=cWwaoL2dg%IZRO ztK%amp9)dh!=$7c03H2&+|2+!MuL;XAsLXV4yiI&rPtrbeTx={npQjkKpedOB!#M6 zX~%wblzdlE~Gkv~RIr{Z{I%;Yv|vs27N1`-YYZB(>USO59vbg+T`s0HG?^!E65 zb6jRjpY8=|AB2D*o4=0b@;=OAbOh>+UvgmzyXg+&Zh$Jmk9z)LYN`bSAOA~p^*9#< z1y*GZp}SlH4J}n-PIo10_$r#6i9Ul(kpIOuo;;*2^i5*WoL9EZ>VTmq0kvl4<_61i zi;bOJSv1M*NxsBs|G@3V-HYvzyNSNYlY2-ynRjP1v0xATfr+X9jXI1!__(^OV-Fpj znM~HbBu0{8tp}^V>+`e7Ni+Z9(7>7TIK+frBL+EV@s|>acM}H&Ft)!_bLcFd5ntgs zqxStNzUU@1%GP5^Z8ca;$r~s+322?a^sq!L>X+3ARI}+C6A{v#V?^=h*T=f*nEzCm z^SPre7tEjEnb!_h{wr|+8&VH-U`M@ooE<01g#12p_Ut2*h_;VlBMa{~o}3LVum&Gt zpQ>Mfm8E%;7u3H^FLjRIm{Vth=+?XDp;-QQ?D*^t<3LWp?9Q{C^WhlMv@ydNhMjJ0 z)EKl%k8y}cA<3UX-Yr?ZfG5dV_XJR; z;3}B6?Htg$DCFa7eTz$FgN?tFTzCb?YV8@~hXf67U9y>M$_ia)?Zr%NT zwrrWtds<^>mqaIV8Rhd$2EP7^`VxRF@(QbG822}n%IVs*Yaq=N7pW-aI5JNFsF=b( zHlb?M&{?yo;h^%Vq>U!6Tc0Due_+HViKDmQ+j|-3iY6-ou;Mb**F^6@Mg-w!C~3Pw zv)64i%nVRK?j1b&8B$EQ!GmWpsdk)R)1NZBZxvNnv-e!hGZv8%Idwo9e;xE;n2slq z3uRY>&!E*Ksd5tugDi&aLC_SZ=s@s=4>Wag#6j`;s4QTctvN)uI;+>sf<3jHbw5p^ z|B(499(l%6sq66ozKkYcVi!vC1=<_dUu6VN9s!s22~%%H4)RPZ#ydD%2Ow$Mq{&H| z6iBIVOwRM1*Q{DKg{O(uxdR#4-oYVXy@Qf5fplp!cNBV@*TJvCYKgYj9(HunM<39=fn@jWoBVc;A{V`WxPT3J;Yyr?2TFs20!b% zZ6KLq+2C4A{wXRQxjTFcslzKv{aRtB98!e($2mLap^?G&|M01%D6VE=%EWyjKT~yN zQ@DfxMx8yo^l_dn=XLma@quAh6-9F+;>OeDacM8HL!Ouw)%RCFl{O+qbJJvAC)xA; z%PVK4m1JAU!AW-_Wn#MU^Xu7Ul|-J|v@6yoeDJDdx&G|Fq2-;#1$oM_Q!w=#DaqCG zEyfbhI53$xIe|Rw)69kW@7=A+WS}*!=JGYESo z>_XDp@jHhs#yyW8e5~_nJ!yRWq)F4`{6=Sf{@hc}q2mn=r!o2WI9ohTI466wA*JQH()&r_ zPRnR=a$LhZ_Y~LUj~r_2=NFC>EGnY@$ormop!T!v+_LroGI)b=BV54Qg1~$0}+#)A^IWmJ)Tg z@7!7WCKfzuQrWh|*wKb|1f5ySsu!9EqvIpp4Lh|x`YPQ1lB7;j#Zo$BsIY!SrLf27 z#XFmH?D@Bc@=)l5hZF!k}VQ#7gmv@?b z?WTqSnQ9cP)3`Y@)N&JXObC>bPqlG-oLM0tjR4JW5@MM!Jr(8i23&QRi9L=#oiXCd zKYXfY6mg>ynXX%#n=+@OIM!wU(wK=(zy_L|QC(0LeQU3+tq-Bxm7Nlx4R#N*j~JV( zDE>4AW2N6=K-MrID&$w3je6Rrrb`iu+q0BUOBrb41hfq3d;1MQ@3`2Z-@V!Tc9upg z$8)S^AD&{fmGt^jp-Y8Ia*WrB%9YVJe(Z>^OS!!hqx;Flpn++_Vxps`==~mIoAF~PE^4rDZY;g-j2<9ptX z^h0v35Ro656k`nV(i7dAM0z$H>)pfi6$4ZD1>YS9GPPV3j#uRx3xqT8zMHTEh|Z*Xxp09i_a zY7$J7hh`xaEyYvtb^10_9um>Pe=pCVJ0?We;cIM> zf>E34Gx!__y0VrTN2-`{qz)6e8gPsMD~ilX;NVZV>H?u~0H9qHDG6=SczDNsmk)}j=+w!R zl;^ZXNBfoixq5Z_imr<5kU&b`_pfhUqam{-Uv2r7uk7!=u91#yk0+*oIsJEh!$0#3 z)tex=%#G6>6+J-ue2g0BEZ?1`scOI+mFOe^dl73dCJ;H`h&VAzsc2CVj2b(C$It$t z^Uv@OUyw+y+JE@)8NtH@)UaAfU?#|{X*+~agsaiv@dtXI=Mn}|!ui|Gwg1)E>`t0r zUsk}Jp>wK>JFC=`4V;?;M9X{Q+jO9(p~Ns@@Tjw20A#yyZiWaj}(pozFZG zjrmm?(}jGOB#7se&CS=1pp;*A z%Kf-Ef{n}4DY7m@cT(}Uyu95cfh_~*SGvhPl;RK+DZa}E!(a7U^an`&D+~Wom&T9r zxVa5_=5VvDpPt;trS%s~89Zx%PqpM30*TNv6k-+Q9~=>-eLDBF1IL z|AmAuu>2K&oW-$^xBAbaew`LI7zUrUzl-sG+n1yE)0*#e{D9}_2 z7R}kp0bk&>npj((sokXWF-ouKfBu)0y6R7Ruun%lsFPuy%#mijxyZF?5Aiv9jE1qzq|*DpbfX zTO3n9rAAs%jSuvxHl#O6&iX~jDNG-25iAcaFM2BhN>7u{F8~HucTx#c(Nk!Gwm0t; zBp>(Aolyj(HGnllsA;FCw2glOb5n~$)72s(8~`m;w?%o)U$u4qS!!$GFr@n@$jVzj+k4{NH3&o^*XG->0QCsF~7UbtQB z(xmQXmF+kHfT!S)odkq4P@ZUH_V`x|plH=cfH4D8SPi#aIro(5UfS>|eEPS$;;yl& z62!jFfvU=GyX7xJDjRJNkF3w?MeYAR%DK+EH4%S>a(zyS-?8mWr$T%mKadS%xSHJ1{1{sc=}5T#zM`JERXBZoB6JLQ%l-ZQb#{I!)p%7 zZdl0KBsQSfLXCrWbOs9~A5F)pP|{1-jT=Ila6G`wtQ>17t}4RV2&BmmADpxy^VgYH z%Db&^FpLiRHH$UD>yoh0J0;A_q|ziRxBz5BrvY>8%=7HEz@la!y0pq(rk;}AoPUw1}FE)xd zaBV071=FCWz~q%=r;4kKpy_l!U1wiCCww4iH8Ggw5tf@IEif+&L~>6)Zf{gs(kRzf zQ~OnKHf`B5kWNAXZs@OkBG--`I{-^EiM3Gra$?i!gHHx>cNxqyMyF<&6}9G9)v4H_ z&^pNlS;{OQ7CShkg!@s@7kN^Boa5i3KHr#`hVmYm&4iXC1<~$uOof1D-0N_O< z;_9MJOaP9?_~!yqzY-HRaP1=nhwagtDoWYVu;mTP@lQERjuzq!#+I*ep#^l1b)26&#lRWxR1&M}`bLnooZD>FNe3A1 zKOTQFMUPuMZq^b^Hox#mICn+wFMo@ z(T?(CFQwGA@jzFZ%*2yg%e~N2fih#DY79ZK*T=_4evmQcDcD^F68xuKv>Y9;!`tf_ zWDywzeDJzY(J9=U4Wx=cz_=*6HhSq67$_owXTXC9jk4p5%Aw7b*FvOcj@| zm@h!_>_*gpN$6a@fvHS|B7KmoPgy6gS<8Zbjp4|Bj?)8XGhif@lpxt&UL9eQQB(&q z6v|SpARsW_%cIx2-~dM{h;|?iY1*buFKK0TF8m8EMg0(UcaNDN`cxv1TW*T{_Ht?c z!s=&$bcYsa?3EPHP>r^(?u{+qE^bI(jCfmpsKaFhH>dgdaz1a2pE%J(a0GdHg^4pL z4R#D<0!+wXI+(xy<{Drl{f1E+f95vbH*f9=LWRiw`kKt`a=Ay4M(w=wKfQT#drqi; z7VCkh3#p8|YHQyGpK9|fhpuv=32xqF79U2JR;g@deZKddO!g9Z48KZiYwlENA{iYE zu8N``O7UotH4Nx+w0a!($v<0;G&^bG=+Uwn)BlO=n}+if;N6qSU5UW7QvK~d%eJ*H zfBZ2WOi8EH`Yn8DP*Wfy_}(I}mKRPKkMmw7CYp2@zpuAheJb(o6uoD7_6`_p(G%zY z8$|2h=kLCPS61ZEn#9-ZvXULeBq(3<*9U~-z5FlF_za9##VbdxRPp)cU@?CgNQ1#U zch$v6Z6r1M6W!eK6Vw5Tw)J)|mCe|KQVAG?9qK{OMpU~6NArA~uG;}NkgFrX12f*f zJ<)FT_T(>r()S%6y!H$i;Iiet+I*U9-(i!eE1RmTpMeT#+S9Dad02I=*du)xY!@_wt6 zSrNp8YJVlM&Q&Txr;Sa#mB-M9p+HS(6+O(uO##>A1uhgENY(3q zpE~sOA2lZ%8YW=ZfhflJ>z@n@3NutlS6h}-dIJm*knIE!AK~%Z`h(jzXf!t-vc0T?Dg;7knAvYY!WIs6@pbP0;B8xRFF zS2q83mx&vM&3E)kh7Y3r7lR$b{>qX|L!dpgQ}>#mtK(GL5E2e+ss3`1|*AAk;Hr zvqyJQ)!2m`6~2-nm{b|BbDb+Fm8S#v4R9z}498MDFQbb6EQm4`A;JuBeGfi<*qqDk zNAJh4J-aIX_84^x0glyf+O4eZEQb0hOw34BBwkA^>0aP+kPM zx+D zancGi(N=jkC;4V&WSm51X=Rl~wiS)t%bV-3d3sbVGIUuncM{EM)5`W}?u+NoIb`{p zsAS(G^O;7ftYjNhmEHBB1GVR}I;>xffjF)67Xk%)YFX_od#+aRP-8>fPX?N_A@ z6NpNxV@j{3z)}7-c?#PTjChbU0NQh!Xgv_&hIG=hF;V^v(E;)1&(j?&p^hMnDfZI< zr$qDCu;FnY0s}#r-k+TiPZ>-VIT-XEnBN2m3djKteYG*RGvKpcXaeHriTt+xs^rE? zJmC4}!jV|vjKS*I9L@Q!AoMu$G}(w4+0EfK;ap(f8#unH7T5pV^AWrIBw z6{7a$lpDwuXGiYmqAMh(ZZuj9-0y`F@`{)B#-{;}n%Rsm0x3;k7)bPsLJFV**#C7s zSjH3SsPH$O4L5mR&>2C@pq!;O7SLIEdlIP$(nw#&d;^mpvzg#M?|d7Si^eE{aS^F2 z{Ec0QoN_GAcc5Sxz;z;;-60id9bh@=6uJZfj^I3K#AcIfcHg{t6SdGOuB18VaJpP2 zLZsl>xU0EKEd>lF_#u!`630s*ZF_Z!uu7*23-Z>Ge7FZ4{)p{g%hG8e+*!DLVlqXBrD}GWV<&vf5L0~t6byAkFb^l ztoLzk0=Qx~DdD86q$foKNk2#&k;1o~Oix9DfbZXmn1d=a7cl%@9K*Vx2V=>_*n5FI8wi)45{@5cH z`5tW`q?kx;fyyosGBdKXJA*?4o*ppn&XxEaN9$IQeZs@~e?WbP=7(QUq_Sfbifz;f z?{L_&brkOoa9lE zeDZf$bfatLy6j_HrhzD5Q2W~oxZGu`{+`S0zF1OPjAyV@NT}SB2$#~OcC9u}m`aR* zsnUG@wOlu3_R8eY<6H`ZJ)+{HX@E*~7IsxwWZ2I@eDri&K1;?0Q5?YRx8P}6U+t-@ z_jjN9G$H<$1@lEBIhYpVXzxVpCXaF{Cs-IU7?3-u+Z4rMDo_|-weZF*JI*(vfG!4I zXNJsw99^2^hKu6TRu$KIa9Y7dR5^*L>S+2Yo^KKEJ=R|J7v%zfv~w;#(3Cpl&!onB zcx~I#zlw|l4Rr#NgABqsks-5=mP)b(2Oxv86EZe!j|jkppOhEBbLWrOxf6s5=wJXtxWAy{=;Rt6 zUdx&J6CDRi;)wx@{Y&~unP=_6N?U44iLdpI*tnq);IOD-P^s-os(MLPKNW>fcefoS zX9o1J5|bPT6z6*lV_eF~y$uI_ZuzSxnEQqIG>r|j5@)DFrg6A&Yv~w2Q_pwdqD289 zO4dlW1R|yf9uRL(q=o=}uQtCaZq-A#y+ic^ks>1&>0kggyMM=ynA{Y9z>xaG8&r%# za3xwa8tfRhYf}dXz$|^(IWR?Nul_7`s#M+}m>G7RqK)Tcb3f&UTQ1cw7vJFeSKHi= z7co*jA+d^ptL_YutrRCAR}^7qe9LI}UkBdR>@82Hgn{}3h@OL!RPx=46*1`5U@e|N zQccWC>t%M+!DLSYB$GjfOCzd;YbM9^#NIZCqia{zlJG!Asrc-~JuyuPOnZTmNG}6{ z7FgAJ1vZ1loNsrvUH9$Pd>75R#lkhC9-IpHGQg)whHNc&$nvpc3N?TQ2vRU6<2v1+ zblsT<3ECk=316|PhQ@h1Vs_(B5Cxm9?KIF5G$f}Em^&Cy>P+YV4Xas`Hm=GkfB*NQ z>7vNUX_7JW7oeyYB&eITBNU#b0mobS?~f4`C(wfEzA6ikslI!?y}iH2(t3>B!!WV) z_dligItkeQ;$?Y7%=O6s!`7L=<(#j7|Hfd(SY|O}rx`V59Xn-d##pi>imYYHlI)V5 zjDx`-P4*Hag(xChRK`$~nNX6g>ciR0EmZ3xl8+7TF(43z%-&t?6&WfwM?kjze4 z9}O5s2s0FD4 z-*S)}Q@FiCg*OVZ2!~o_3`@D&N>SJ`1vx$zb(leJj=Nc+IOa$7Zf(ZDL{fRO$R$tUi$eY$?W3&$vqE<_bcocz55h z!g-GibI#1h4WsD)Rd(#`3~QRy$&B5)u`GzoYFlfsK5$01zNbc&IOI0QEz#s zajD{-H$|vZwJYUDMpkI4=ewzasANfU7~*6h#=8xu7K9xm4BjEGOG^MG#q&Cxf)Q(5 zxane9+B3^6>dt3|g9}tdq|8KJgOEp-Oi`@mUT_P1^Y=+{Z_{=?<&mb9m(hKfq+kdJ zv-(d6nL}o>W{2C1Fg~H-nTz^mFv~FbD%P~|+N0u|+5?PH&+`KS#}gSIp+~gLPIYlh z1@KNiK7GvDv2myE`}Ipx(VTw$?wLN{VU1K6;BtqNk_oW8%Ue~&Xu<;r0q+F^4gT-| z))Q{r*Y@ONP9zb!^D>Fp86oKtuk6_gg3)TJ1C^%9UeJUa(EMWi&9}SIYqhSWxEjO$w$Z7k?zHZOQA?MRHm_to0l74aU&tkeE zx8ke5`26!-urRVZMlN1Nq-H-rH6Sh_Nkw0R1KDzHVhC?~fxAJx{_VRyJf3eo|Hdv3 zawJfI6$B#d?vb5Yo_M!d=BPX%ET{C09684KR2acvH!5ddD zH|5Ci_r-EU=2ZK>eN&0<7tj7xC>zlv7)~<D{E)!5Ns z`<&mn5}~thMc^6v8N46p?gvVIN)58B8vJ_sH{6Mbn!q*1Pre(YN*`>I-C(bT0~WFZ z{W|6Qb58&Gq_$7DmFw0Or(d#tJ6*+xpt-Md4$THCUexZj4ttYSj2@%?Hh?@7myO?C1>}7H(;x zKZY0ih7l{cfU=Ws9q&)6mQ#?q7}w|=KD>j1OO*4#CM`B~phBR7jdR$GT_bdScaikL znbra*S8`7L(LE$stYIQXkGWUJWbeWH9KZ?uayzEJJ+_V$aDm49XRX*ALvAder~U(f z6;!I;bIo~wtAg?8VLjJ>J&`fPEK)2da+h|7hhb-K9p;P0t>B8TH~uxf4CA-Y?G6pg z{lm(c+mXZ^Mk}TbC43_LUtXw|4}{=FLSyeoV%f-f&kk;)Hk5(b{qff`79|D7@l3wA z19)H*K?cz-#{u5--liv$7c;M#P&$As$8%0e(F55 zl-}ggqbVTmqGvFljaOXGzIFV8(TpUIC+>a#Zwuh9oTde3QwJHrhNc8SWA%(Wf09+J? z+pd{-$l~jP(SUuJB2^AJo&Tx~fB|XXRyifTN&Q z|NPyw%rDU6aLWY4m`&?dx2k97J1> zaZp1D40ytT?wDZJ*SJ1B&$;vG8-4T5`In1;6P^r)lHF#~HAh7Ig!5p`$ImA;a3fJ` z`EVZM$oB)MO5EZ`>i%Pu`0!#DP&|`6~Dh`X{M1lIN;E<3T-cO63tj$n2A(Qnc$D>Md!}}T9ztNLtPxa1| zX-QlG(oCc6KEk|zeL3?N98i=~k3*kOL&N32joN_k((cTDLfs2XzyGPvQgGrh`DfE# z^-N4kx(ePKQS(`|cY>vn@3$wi+(3WT*jl+UkYn@HVv7dCXrsJM+_dI(HYIU`U58^F z6mSJ6{5-jEgEQu>sYRsYL>Yl9Py-Iis_Z1&r_NwA*%b}mOx>I;`T(%;IGeI|Ef5S* z^hUUU2^|a!?$~CnTD{nJC%19=7)HPX=~>v*xY9gw>1@cp>t8WZa3G%V>^cPmKAmnJ zpS~+M=IxZCT%Q4F-+#Z_O^`yd4K7vLl4x03D6@7(9>Fvi1{LA zeNSF-R=LE!&^^xd_&^VGU2Ih7Cr)q|`i91>{I;Yn^;q0du_e=7=l{jM_-ul@W+`%i z`YBuJ6Q;&S4%XNn)8fv+_Od7L7oyJuZHA$;3y%6xcptts_&?bdLzsC86&CfS_*N(jlij?aH8DtZ;`3D6xmxJjdi z4KESegUi^e6v~8=@hc;0(WXsu;{V#lEy%0mOddk{4od)&GA*#q+`DL4{hNBu>N zO_CbtZ(hyaMed+L7=j?%oR}cYlJs!Mfk7$O4ae#^h&TI;k1M4L55dGyHwCnKjVG^& z#70G z-W+l2^Upq0sf;%B>uYDze#)y)j-GuCqBCdY=&Nm19a!}e(=g3bp$zBf(e=Zj|6UGK zPJ+^LEXO29UKshAYidDy8;e|GPd`*tcjG>5g-@@IJp5V$;a#-X4u5f%zK+Y|gBFJ1 zKp%n4HM!cRAGMqEb?4Zz9lN&-eUKLOCH@F#Z36xEq)%6yp17Di@=+Uvev}U6C--jd z0U2%gqTB6$n;7+oMBGTjuj;=9uu?K6m=T*9_kPN*?XL;tDp$kvFnJ~aLn!0~c3E6B zyZk8?DY*Ci{k5d&vuBG}HQAWF6uby39)7PaR&voj>ahkxPj#+>qP@0l+pEs)7z)b# z{H%^X2yR#C!R8Ql<06apEYJ*ofE$jnKU)!|` zh=D!_2)rMJgg3P(`M?6X?R~n4GFc-dc^|ol^*7}dq2aT)pN=WFOn*e4m%{?K%L~|a6r4rpCT7S7Nn?Su3*x&J zo$(3%9L1CkRCqXqb^zvLCUJwZ7Z<+`0@n=)<37zGTl6s}OY$qVgL_=i{JUVngn<)Jl<^jt57<*0y; zQEg3ARw3I@pXq?B6cD2(T>c)a_Mz;mEBeDh#Yz&Y6y>(a$jH!!)kz^A(!;qF9nPIP z_3+>(BAan>2Rb_LppA@f+8^k{lksljp704KU-A#aMhPLP1`qwOXEoV+q!PoyKOZeA zU$sT#;@ciPwR+9Op@K@gx_HVA2MX4yJi#QlYNHm5PfG+ZREwjba3%KeC&(ZSr1(dS*XiaBWBKa%TxRq_l#Nm& zhKZLOa3SqoC819SL-k{@xJ5A*1mNVT416aK?vfl4{TN>CimecjT<1S&!=ZSzvcdLm zhyRp}Af~f>k8#6;QYW6yZMFr*Cn)xj({6`Gp9@=i;_BpOoZBmaAQ+!)=>L~7>8d5! z$5vk3srcOsiA|Jj6n*4Az*&3Xb_{=Y%@2;2GqL@uiobqQIb3yT=@yu3Z;(v1#`ej4 zSRM@732ge&7lbAG(0Al3RHR@s^OLaLh~rt;?|M24g7yoht>nfzi`O(xDW_Q;7x<^* zicOoO2(^{C0)7AA*Jdt92&NHO3_#tR^kZ<%b#|ZBMg)M@PJ~QI8Zs|)N%={!%@@@A z+sWerl!x4w;&cG)UaLM^mnZYSIBtIC)WpA~oPZM?YJyWYMswACwhYM@}bv z^3O+?fM(TaKfh-Wb7^Z*T+?rqqXX+t2+9uVLpno- z?6f~}y)&LX*B^Arja8(3ziT^wKT%<3#Ee#sJM8hPeN z&&oeu477D&rKNxHI%kNj3)k& z?gnj!*V}YKCTb>;ckmVO<0}?U0G}hr->`Y}gm!MjrbNL=z%lj2M4ZTrI6mz;Ln_MPjsR5Fr0=fq*0KJ$*XWrp&Q8 zSv^45PISI&fhcyvt^XBQi5`I>{yo6e%GJ4LBIPBg^y|`e^|`MJa_9!|fMqf?(tRbb zHN`fRV0hKqwO2`)6#CKEaLxN}r7Bg@(Ao-R=NohF32aYJ49#v_{wPx$B8Z7)UE0lh zLMiPbJ(ZV>i(Yl1n?3r<_8{@7K>t&4Eoy4N`2>=?OPy^(s}ZwI=~DD}6%}TfFnbv- zrO8WIL!DYrnwS4)tBb>56s?-Sy%RA2#S2S&p0aUn%J6X&$~P!ex-=)hD%uXNvxb3F zdNClIK)g)1hMvoAKLonJ>$Nl!ujlgQYT(wb^HL{Mwj~!h7?_otJLTryo0PN(IU7gL zKSspGRS+S3={h)|2E)kaaNh3Sxzm4Kmp7k|G+#e4Rs8*{-t_$_AYP&?w0DBQI)GS6 z4@DP(?mWSNE%DljWqza6BvqVMU9*82M7x2>G!e$#(NcGCijQcZc6qYB7;&2%4uiPBi`X)A{q}sj8(S%FZ5wU<}iKfKhZZCp?;^ z2MTpa96ffm@h=P`FUc9fG!X^Y*l@OV0OtQ|jB{^{dnzkQqkg#7A1X<^z~wEIQM^@f z!J%pq_~Y$qSL)1X*cL%oc0XhHMCh#}!~U+xm}EPGz7&i{;^WS%8p8&skjI1XGxc$d zP3RvRv*WeYj4-&TpC>jxUz4jGabno3sVhz)|FR({XsYc+`hlR&cJt1Oxq6e0y%AHx z7)1Ca!{3d7t?)1^%{9fumQBj`^e(}*x|7k{`$=))_y@Zu6BT39{Ie_G`X1~MoT5!& zsRohETTEwcq4d9qTRm;!!=I1`3CJpXd%Jk6!+J}_Jiv$JKRi|Yr?qR_2Kux_*J|Y3 zR`Wf82M6@(mDpzX)b{6; z`MVi;>rRT|itX%=VFR5#ZjV8_Ojg`yd}d9GBa^0v;kAy3ycVm3oj20JM5tE1(c$E% zv2SX^R-Qm^A@(erK4@QBj9`c{bLvg|OaKOaibHEp<1+Ew8|b!y6(QTgO;zBiWDSbI z_0F!ZXB~3HMLQpxG6Dolk%gR;F?W9b`q~sq1>1QBW=-oD8CEcRP;DyeY5Ldez>Q_5 z`US6YJ_9oE-yit+yB4=fK6;H=zR*QX8nJKg){WT{H2-}rQ1a_^9t zz4q^STS_du5U{D>>EdF~c3m?+%dySugZ>Tb_?>Bg{IW)FXP^~`UaKn3c7tt^aDuae zS=}x@OgQYA%4^F{LL;O)JL2Y!*}#GoJAH3_@Wk$Tejv;-Hf0==Mn?@j5K`ax=XP)M z8Hnadz6N6}2pICXy|MvjGVjL-fd^KcPaBw8B{>P;u+x5#_!0?|7Srz@5*C0Crl>6b z<;%pIgb2dAq6^9)F3-=D-LxIhoa{+SVIJlOy^^^L|5}@A{L8pKkm0>6mJRrU@29sT z1ju#Z9!J<-hNakKQM&5ISsDV+fRFS)@qt}vk(*T zZ}(YSb z9bbY?e%r{(YT4os$^`syC$|WYetB=h=8br0UOA06vh%fgFZg zE=Zp__;#tRGX+LpX7>jw!a1}jIvG_OcGQ0F&(5vQ^m#Ed-Z8EN`K0{-;%7@yP^oMT zA{k1B9xL<8UF?=LiXADRL;DUt=QhE8Cq>{$V=NpU-oN~4K5A~&-0F!ts%PHXKPZe< zx3Yix>S-Ezh>|VUyx`W8c8=%cJ~V&aG4wY^&?$_JXHa%k|IfqUpOEu>P{#`8&oOE~ zoQ{cG(<=ESPs8tYd3QW@;y@0Yr3YKX8QLTfMKC5+0IPJ$^L^r3#t{E|$@w@}4QoIu zhj!RSFuRY;=GXW##{Zf!fT{)%!&sX#_dm2P2|ag#Z3na1Iz!*xaoDM}N=1v~ zaNA}vMdo(l@pGfsvlG0njrZd{FgrZ?W8$e05V{LfGJQU8IQ;e-nnd}@TqnULNEcg) zFnQ7X<{)<$+Z<4S>1ew*BM%LhPM@OmvSmGhog{E8qj$3fNAHFC=*>|FC(Z)e3Psq& zPtDL^ekz=AKYq%Za zlJsP;;jIIIobexXLRQswdFiBG)8a3I?bc{k%5f{=4pNy>OLEQ53yu?%O#jzmi^~`A zsiX|vQ{T`X0i(>p)48=fO^5AG$Gl5j3vS*n3L%@^_^=~m;ch-_Xox>=I<9f~)V{2{ zba6xLs3p&`-25!MoEz4!A0=C(E)Yu;i_T8_$S3)E=F%z|m*#R&k1p(M$h=naQpaO> zDUuzd!LN{5qgo05eaC_@s1P_F<0Un6o5W~sR%&ETEs2OfbdTYy0C=@H25&tLl`-YA z1e~?sr(>d38y|fmvw)SH>>ona!f;lXZ@s_&@9pLqSNBSf>pG4i)xx zaM*~+0I#Mi(Pz|skA<_3J*!@ z_kF{%^~aPHV$>_yp~H2ra*wZ(AGDeIpw9ufSffS|$L0!yGWrH}9)aI&=k=HPB6_YT zg=8|)3@Ly;LuKJBrdicGnF}UUwQk)-U9)T?zhCa0p=ZIYgQPU_MUA!iWBl)B_9?%8a-O7Q55c}?a^9mn*bQx04l-D+ErQAZ? z(>AbFTxI_i`RSLh{u24P&4K-Ox3%gr-bx!-aw4BHVjz{)Qt8@Yl|$cKWjtnMmJB962lyj>nMBz}GS8}kNiJRPzjb$YA<;B4WPoilq!GW&jh6lBLA4}L6)Zcy>NP=cC993uAI18yd(>$d)` zB!^a?`&Ftm5-toX%4;CHKFLWMOCLV>3Au;_-U!2_t0*a4E}pum+Fh}*C#Xw7FwU7P z0g5I;;X&iZJ$TwV25^Sjq{F_UQTx=OL_$apzRh_zvRVs+{SV(c=eK*-r|=PB%~i>s z%z2g>*7*UUJmk`e#2Z#IQ|`>v4gd1QY52k7MXIxdq$qu`SBGk!e_kJ&BPRN>51n#C z#dyjT>#^rAU;VsCPScg9bGG4w5?>SB90?&9hDINab$Wl5g>%^_lekpcAg5GOOxdhaQ=xIQ&OCe)SOVcnBt?gj{DVI)4w=p(Hg4B$Q%Q*D$(S4`LM>yIU-tfHvjz6?@iBzft@?{p-;99m z6>g>d05&$CGMy*J>;`pta!=8OY(sIe>WQ87G#1%O$sDRMNh*#KskD&knjq}mq=n#; zUtjwa$}a)Vtge-o#Y4t-g9}aD=B%_4s%?T^4Y+qmlC^$PSWqX}(VKAi_K8P%;8;d7 zw_#G(^mEhcTvu1$%c5FSPwfQ!>qULt&$83szkjYL5b!4%a6mc-yG@JxC;7^@U< z(g72t{8nrfD#_GgVP{~GPp_)DB2QoUpg8^X9(&#nBx@#w=&t7bNlaY zJlns?nw(n+Yq#!BeKtSsuZtI5h=TdWRTY=^9stFAWq9^~|2tru?UXMoH>y_opj$S0 zn!5B5vvq9rYT$M~6HxKNO3%vp9FBOp_ULy1ZgfM>AA4wU_*+jP2rHYbb?Xir;+nQz z{}dp?A@+r__GqwDe=n0d5n54KKSIAIwQdedi&vm6dR-WRd}-Lx&Ft)-r!7E!+Lz{&2#R<_Rg4fzkK!v z2LAYCq=@c)>Uf}s08eA2&8|Ojr2n08+N%Gcb$@LMI{e?!ro1lm2HFrHsuyo>8&%uP zKRK@Y?YMCgR1I-T?1LeDARQ1H=d;J_t&k7)!v2l7eH{XELJ_cBmtKdCdyzC>8D(YKUx#cnzqpCJ$UU z`S+hb#iys0snT(Ww69nm^+Rw+^B9G_2Q&;PK}0Kx>UHn3L6o987rh>%g0vbl)QQw{#&-}m19A4#^18j%WGrOI!>iy5C+B3^zZs>5z+f0Xg*jU zVG!|XIO*B`Ek{kw`6EAk{;t!DgSp1BZ~KmPDIzcEzjAHUy^-^pCjJ+S1SRK*D0e{* z($8b??w^dZ5@meP&8y#~hl%R}l#6x=8&yD8naY$Y9mY~v--j|f=HUO^3UG@|Hr~B?tlDf5qq0QiIIs?S)gqtw@NKUeg)f}xh*YiDs0mnNQ1_oZ6e4@5_;g;kv?t%0q>jT|Iz%R{Z`FPua!Nk>e&3D?O)9F z?nHnqpqKXO{{IciH6nOEt$5MR#C7|0CZON84qW>D$$O~ryeuwc4C3OJfdj@h`Es?| z42XBas9?%KsF9KUM2h!EOFeF!PEl|unFzqA_=h5il74kt9%m&} zAt(cF3&tL1I4&hNloU)C)N?4)`QD2CH zybtBa`**^5k(ycCD-Z+Z?`U9e6S&NiN$Rnr_SH+5pSA&e1tR~QIh0ga_=xd0+rriBPL70(Vg( z1yuNj{jHzJ=W$fJSHwVD97;`k|Dg3@R&r8OiNbh+h9Ni+plGcX=keO!-=^k8f(bhr zs>*LtXn^!&{J`i)POecpdY>~i*lT7;?s0cWnJj1&jBbwIDi5x=QHqpZYr=PdDHs__ zRSoAHUuR9fI@|bZMX^{n80hv!&TZ+c|0AKoXu?ozp-`*GTTYhuh(4Q>CU-?eDX=8!ZM=VIR4)p!_#|kp|NF$oO2+V_9HVoe&?n_7FuT zaXz&hKk!dv+o-W|4)l<5Q?%Lo=6h7}ZOS(M(HxGr$f3gJCA3NuLz#Fx1(CQ2nwYk7 zhO+e6l?av0OQ4`25FVS;48FlG2OnWRx9)jDqA`yD?=ATT?9)+UJFEa~eqy4vji zEUqk%^67K=;PXP-{8;q$9H*@Rz zRS4xGh?Zs49p%^5LIUor{x=r0axlp-TuJL3+mWI2TV$q1V~k~SpT_$KANrn*Tl>WW zq?y6BgS`p=NS?y{1sRr&o7!a9(LdW>`p96_!t0?bh4^cB|(6e*0TO_?e$>v9c_A1PV{olaPuOdk18kpY&jWvN{x@3ABL8h9ym^VMU=?} zS8!2!N&+8#?Vlw^`U{MQ3RU1ctYOh7W~VTPo_KMvQW#^4O0CbL^rlDPrMx@a&SzUP zMIc!>qJBhn?L_r(rhzhjad9Evpr3=apurBp*r9dF3oU+6DN0j0k9rIJ0?A$#9COw3 z4}r{u8j{ZIG_!0KQoP)^#3G7@UYg#$|#W{cx{F?qO0kZc z^?r>r7=p5sCT#0AZ8)WlQ-rdT<_+K**ym}b@d!1MK9kNZr6F466AuWLMNTT#q;a69 z9@$PKz91R7Q18O*4u5~y zLl&StQkK)lW>x@KsQ?)StqUIJ+;1gF(82jEP>Ne|fs4vUP&|;9n31)#N@qv#p!{oN z*2(nz0I@zo*d9%+@@4#!@%*3iD)V0DVl(s&hG<1%UmwDFnR%jHLkE7u z?ig=-nK*l*9Z?}jRxac9XtQ;gN=;rG3-Gk+t4>;S%+~rIZ~C~sXgZZF}B5K;_ARiev8|8t`+=S0MYLj&%P<`C_Z?W2foA40B@l2F_y zrWPZY_E-Y}xQOQ6Vtc-Rzq2#Q>NMU+bQJ4FmufhAi@gGYPRSGZ#lH;Q;X87V->oc> zAO0ZpB^nj`;J`@shMm#R<3wnxP3d1q$NI32pP9=~?E|E~MAOF5)AMNhjkAWA=84;% z$GsBv%K{{>6rG(Te1q9|`X{;o8r;Q!x>m_3+K;!uNCO~#BPC&LyOo3YPcqK&qIKmG z365Cw6M<7^OusWfeMxfb5=mW*yOhJ}L}}PV>N*q_(UiTyW_d~Yy+i;|JQz|1<7=EJ z1(G}wdpZKLEvxDOd)m9^rtJEcVxmy|hvU&Vj?v!n>L zP@mE30Ec!0O5yyqzTlb6x@d~0&=Qu87G=D^#W|P%#zl=OU;4C3uGftd--gPGoh5)Cz zS(wjCqk5Q#q?wm=dlld-xM#*5ZD2bOecaqIzXeJ`WRX7wIT&sQQ!tkH1Hf@As0+FY z#t)5taV)lwbbN9$B_HftyC2PI*qHh7r^TBtyU z9VYL7c-&u_bH*XHu%~jjUCopiOADsVd)ScS(;`9S;j#?rqIAU6pNbp*I!bSAY|yi@X6?t#z!&iXuTlsa*vbK?))h25iL#! z3WCyRq^ubD)ii@QT&QkJd!@RkX2PUWwG*+U`5GAu52)nePOC&%DBlATr@djuy)hE< zNqK$mmwtKLIdupFUi0wY5z~&PQ%mO-1*UtKy0#nRFala)EJJs9Gk*xR)pQX2K_o?z zP>_0ak}0lgR!DBH@o@gWz7|wsQI%d(u(c!%%xq1yAKTMeaGcuB4{5quad4JGgBmm>4fI-*HA#(9q=8wN)_nDU*vN-$pA z{*GFmZ9FvwpXAYp1`9hV0`J#B54>c+k${tR9+ToaSC0u~^ zb;#;D>ePk0N+Xx5X7|soN-M0xol4d#E+rO87cb-i82=I!4 z4Yh=!+{Dj8MH?DrC{bvtua~_l^L5@WMi~Fr;CKVPOWuU%>_m4izZD%x-@<8zvWW{= zqjdR9M<|r&zZn5@6;`wYW9he)&YH%inOCn&W0!DA@ztIGxCI_y?@06gJpyd3>`pCa z#)P0u^-VNM15@rJ=`Em)DtU`aTe!Yrijpe`s^wAc!s0PX(LGP7Z)Aqx1B&X+e^qDD ztiocrnAN{;xN6n7Ee~9O3fUMkYHNMxY4-{?@BG!nzf7CPfBpIA`mIB^>>BWSgSA&G zyh^NXk$Zaj)T&k=l&+EWs)yT~u|u0>o;!X!e0A8o7Z37df|jPu%J}1#G!A*fB{*RB zGc&1)S@0ecKXfnI9Cr^zH$*2;p`V-3jM!H2x4>apnaW)=8gGQyyq} zyH;5J`cN3kA_7vH@*qwnwa^F-e(aXd`uo4##8wyS6oNGB(mg9JX=@e=OHP*8hA}2Q zQQb*14P$V!A!MkYoM*0nLLO=^5*;CI?}b!13Nh91G7%8-L*aPQ=nqZqxjdEqRpxUSq0yT=J0VyhHCl%{u3omjP3|(F(U%0bIN54 zl&AhJ8C_>5bR|?DP&t%$(~7u&{6HrSDtPr&xw`~g_^~*96TM^#&5aQ1WvBK<4!&N_ zq_F1xxq8^3zA}5D7v$SG4!=gAs_ZE0dIrObnr>yiHQu`lk~D04fwX1(nlru zMa{)HQm8uCPP9vc$mQOvS2S_`4bC4wl5Cw06=Ai~9 z>}U$-T{;e!3g2%e4T)V%C@xl>m8{mH-0;Lpu~_$&+#daFmnOQR$#J} z>8)3auMa?HAI$@rc2Nn0uB8NzemkaCe zfhesw?c8k{c*vPs9ZDzvz-cBeGs<)hKJ+8bM?ZCwAt)nz6k+N;_I+gD((ApL*(t6u z$#-vYy=kc6JyJ#ertou>7Et5#JQRu^%*aYQARa6okd9OqW+BX&VsE$aE!CAHldiWE zPe9W7C1GC6H9kHDM%?(j?d!hL3|u656QYL#0g2io$utN_yxZP!q&%w9;Q0&5m0zMN=nfINZ>nVM$FyRgrCJG&6cvk23J!o!}nHe-4T8+8DX!Wl?{ z=R{p|Y}C-c4NCQ!gtwZPbKsZ;=j6D zOo70bM71v=+d-`{f&1~z<31)AXV^U8@Hi@^IT7l8W&GJ*Dw}Ld-C?8^VElManP zkc=t0qFa(^?KF;==ESp|M`oC71qMs3QbUOTY8yt~#JErPab?7t$LBah@XaSrON55V zXSp{7zP!pwzCx!tWL1;=#RnZ#X44pBAR$@cQz@q1TX%;usSL^(UX<mosqwLoAHVKz>b6@q2GfPpQ5+0t3h3Y@9qV6=dz|18 zDx%>R)Bv6przJc$TZGpusD}{;443RL@!D>#vGJUszY+^#jA}Dl*!11S}7Q) zC}cvF$}cW@6Pj&lM{4B4dx+dgYV;pW*_Q(KV*0{o%|&QOG*3zcm=Px-7nbg0P_#47d*)48tY1bvDwWwn zlx#JjtH+*_JuxFw10(lxFEv$92Ta5+@5h4$zYH-ht@}jP#o2J<(4(?FtI=sau}vML z!!!7k^kZ5YlNWYvvX`i=pWq5 zzy1u>4T@UXgcceODsD&#A>I9>pasuY0u)k)>d@BtJMT{t)@B+X%TUEJ(%l}vxd#F5 zdG|NCQA9JkcX8Rl0H6f@=EbhJmFpQX7KnQIE|I;KFNK50{qevzS(fk!a zCQ~tkGDZk>Z6dQlEc|7(Jedr*`X&`wm2fE-DIvxFD?;PPZsQ?b7oyq;M)tGs6!RgT z)3b8r$^b-Hz=>oeWH#Eiz3TO4q(VY*_z*Z#bVv42c4LU#59&IRW2pKAJJ&!r-qLb& z()V9#O01e;QFPMON9I9#YS2ZCiBz5O}WZy;najoszPclTq&LcaRyD^u;nV#E-vPpX&csN0S6qODPZW!A-@G}Kg$ z(c?#3uHhfxR!!CS|P}xmttQ@y3(^gE#*QCC|;fUqJ+)Fp3+`bh^Rd-<^ zj1wL>>;gnV-L@rtU(Oq@nK*ZvLCBuSoEilrN*mtQBlbhYqlXfGEQAOKAa@+Za@B|@ z0oNvQN>xn}JYy6kGli5Fz?%!WO0vB7lLDElMG3m^ z_^+1QgXjV`NVAtu^8iGoM%)72?{zi8v^23ZF^9rUo+E|^T&!PO31!%)R2jHHJ5jds z1lENk+0q0S+HE{W3M*(!BUOPKKQwR4fb|TK(c(j-q-dcbjWy}xBarSB`c;_^IYZic z9Ha-$+Y08<>KmmA=>6G}B+^ddQyr$wdgld%{oWoRaXn)V4EjT6F3Eb+(U zF=4Zf_<7lH&1F=d*7RGoL8rGX{v{a$PKR_Gu&BQn$&O38%)2S5VK}s-hD!2&V;M_m zQ_1k`L!n4@7>J}sI~{=wb|**cW5o zZ%uus3`m<0CRBZJS2f}39vh4KUj@?RAkH_bXsUEaND3y9emBzO*Vcx(F^IEWAuu}v zdjuY482N&Vk0eQbL%#~Nq#A>!O(Um}pH)_bzcP{dsL}A_2l-w7L~cNbVh^A^uwR=s+oKV<3O<7RIX8|J9-aHFVU=c-(IeB?v%xE zs0^gjr~y<`yd_f$N%38-4de*#(M$^H6`;J9SO)V&vW4TDj5oXg^l5XWnx#(*a|jk( zhE`ruM7je0zyQ((=>mKKEZ8!|qvu#?#2ZO1aS_&FFPtuUQPB!g42HdrlcE>@XW6p7 zyo-vkB9UWoHam1}^f6pjp<`g4B)>h`X_4peLF{9v`SF>gG=r#FZjCUw`}?cCRj8tdet;otl?hu0}dv(j`nZR zuda~XiSRd=^fSSxQoo1XC^CUfgq7;~J%Bc=?7gwQ$z<=9{)(%vxR3;Z-2VNOjkLqy zs@A2*29~s(?0rIAFwSibXX_WBDcla2SSpBgzGJXs*kkoFSx2L}5U z;rBgA$SFyqZZgiM(z=G|1wM@}j^WlQu0hl>!e=tOKU*xHvtP4YCrjxX@4Yp_^) z@X(?DZ2v3(XVU0?IK_duIHu1#iQbcRwo3q=tayV#6#vwvEOmNKlu{o%5s!^Y?&e*P zIEvJpoj{^Eay3dX%``5op+-+2Z8cDfmO<~uK;Bb>Y$lLB7=7xd12Q?p$BkT19Tzp4 z%tNBDGMu1BeX>d!S}K1Ku}{Nawa3WWNZGr2% zGj@N*{tzIA@Db|%e)nywN{{JimS-Zms44&^V@_8_-AZ+hVGsjP&zmJhgP}1CPGmb9 zA8A@WzA=qECH42eSKo?rD_x_cr8?mUe5GJ?mBTo!N(#Vwq)~=F{>#fX1fF~wVYGS_ z_yQ$jw|n>`spItJ3`8F6QvA*dK>=wJN6mdP4yV+IB*U4*OVx9nO>?Jc3K@$EvK#OX zy>VAEJy!@m6#?P@V{>jn_Zbm!`sG#{^{fc&yPh*Xc;+IaROJ3!sO-dOZqP*cs%8W3 zK1f|6f>yx!l>s4BrLR;U=4qxOg}i9FjPPD6I%*E0FW&;aPg0LQWV8M0mbb@2Uf@nA~(DoQ;>|QR@Fyh~WixJ$M6H-mW zdr%l@7P!i2PUDvLA$H&!k}0A}lYufx%L-;!82KY7dE%YlmKi5VZ0{7-a%`d+uP5F* zen`Rb{e<;ry+$wp6l%MAX+<{1QMxk_f#RnIT@f{Dh*0(q27^B>!8j+KeyuHE7pb*9PO7Kg{`YBto8$pwd zEiQ^y#K4)drYCLg^LzcVJ<@d1v!dS~g0$~z7wC1yVhr6qLIo+rByMUR1iN4lC4dVl zDeA~=8Mq}JSo+A6bQ>P47le>xju6xG)W58(gg4)~YuDW|2hwQ>QR36UA;57v=i`K_ zSlp2sxP{q|dh4_Am6sfmBpEr(o*>!MTVxVl zGYq_DdoY=gNO+_8b!aA^_i9EUq!twnzBBZ6DFW)*`4hq<(RlHB`}qzD#1o+TgRS*# z(B%@PN;x#eyD(xk>NIJ4r^cEeg4!$Fr&c+Fs>UuZX{|k8Qw7Suv!b3xQScegTZfzc zZpUk!HfIc-QP)vioigsN-`fQpBUK6|j7g9r*Dgi@sgH0 zU-Fzfco1}JGpqpaDz09`SLI7Jh8S0tO|7d^@V*C2`cx0w|F156tnsLgn=tfj) zIM9BYyJIh1MzRR6!h%_vC(5~(j46}?wlIv-W$y>61>Kr3g03(LJtNMCKCcb}PI!0D ziw_KTy=>oA1l`MgDxqWsg|VCW@YrY=59Mk%XmDNBy|5Oy5KV)HQl zPK16tR`4ZIIvi0A8mcwfXz46$ZMgTuJ-axcVX!CRS^aG0#2GpUz-9}5d~Gh>g3DH{ z2(9Rq{^mH-AaUOHJkr(Ig^`Z-%zT^_Tg`fa<=A%5!KyotJ&3*Ocmll%n)N%}-xLFE zFf%Rg$IQ$H{?`I^vRQnsi|%5=bxT%`B&?wtp^16{FHecp_l6~n1K$4 zMvpNp{bn4LTsBs#~sVR+=&=o8$Nb8od^%@990C*l`<8s-^`XrgH)?a3Rg&v*T;&*?q;5vSzf!-#Tn7cIwiAxXwbzuhx zMiO6F6UUn#68m8Nd&#CZ1vT|vHJZ5m5Q*d6kl8QtY?sqkW`lJ4IuG237L^*Ratp;-Sbyym7Dck@aN6&nBmMeqWU zX*w&h#mv3e?{(Zqj+lT~B7AV7HO%~89>Apy`)%)N3~H-d9RBEtnX91Ra&IeZ4x}Pn zmdkb>CW(HwB;&2VKwcSQ5@gkuR%X_g{<3aY0&6m2b;OyDM%`h6_G-o9875$#Ot0!* zpG=UEv4`m%f*76f^R*I&pFU(xZF&f6!tD_>NZ`Q?v*&UoGkZAZMLBhfroszG7aCGuz#3^0(wY~S(HkY7mTbz^HSIRnD^nyLu#2FWlf+%%4=d62B_P$V|Gg?KS9$iYGv~rM=K9CVb9H zJ0!7Gf*y@orl5SW!L|1~{@iV<`}S!mr08x_L*UtFH35 zC}tSCl*9kfWz`y39DYk-J&*e1L(-uX&xq6GIB$)ZFQguS+b$1Iar<$-q0G*a-YLa2 z?HT^XnRz+XTr@64b;P1~FZvPS^wtCi@^TF^1L?4)*@kFv0K2JbLt$Z$J;fcz9U81U zG5iS2`vC;;r{6r-rqm9dsFYV9z1fDH_EZ|jxGCx$1J=fpzDt*!0!!oUh*D2T*RwS_ znjx%SPw$FYph@P04)z1sDGf#ZqM>8v&a1x!3u7mC6_!ljrEjx~L!XbCa(&OFhzaOQ9uH&RiQ80b>lf`x49{bK8^Kkvfjgaei7gdLb?Ly^t zO2(-zGbRy1OCWaPMK|i86d*0r-yWB%Wb=(51?DGSDiyv*^fgzB+HNV`AdS^fiJYV( z^y*pd_-EYe_7#5|--#xULEzU_8R*RQgC5Z`+OAfLi8!s7vm4X`Lb;M{<1irqN%xb6 z-6|eI$y)=kp5JcUc5&f^yrhFaTPn}?^CAjpFeTx`vn@5dl7IEo<|xG$r4xg~^0%*x ztunbdg@m&KfsFbUs33_gLeaUNM<)e*$^3~^@fqq7?%&-?``Y&P7)Ip+7@f1iJ~Aj@#{ql+D1 z{7fAA)Q0r910DB*xo6kuf;uS0`PUJi8fx6A|F8d%f)Pi;nl>pi=Xt^ToB)PQI`6sk zOGl05>1a17I-=WE0Rozp&219>9sl`%J*kEWt7V$tK{U~jnUMjuw*8A2_IPys`0@Ek z(X(5OvGbJ7*-M7Ycx3?o_=Q`DJxL6ZZBNU6K;vUqFCgu(_ zZH>hS6=qd6QU@2;VoTB=pP3yrU;2xsAUkt6#mo*h*pkT6)#Qv8hbnd`@p=-dS$o>F zZ{HAfaK^N0JwwOeEy5f$q`E-qJEZo^Y11yDdZu1!$r<;LT~vrDP&mxEMCN=eC#8a9 zX^(dBWekQ!rsC5Dwow08sZ~w>xRox`PRD-COyL6u4yYqK{30IBuKRfm*BeOH3ghh; zr-!Zu^u%_w8on%EI!nNcf8Y|a2T(}et-oyiw1)|i)e;7uB(u{y(Q>9{?(Bd`5oB{V z#4wQs>w26nWPLdJ22IB7Jd>K>-yA_|w?H@zL|-W1<5kZ8^;?_q!NbjE4%4i~)IiQui;xO5s=zxtIV7sHMVP3mB4Tg!tN zPS_WDfeXJ@k3;;ELPD+!Z|=s`n%LEp>^e7aFJbdmhd(yLsTXuFK-$j|&ILZdW_Yh* zfLK@fbZK>a#d>oZ1c|eykI+Hy{VIpOaUXd7@0c&Q+~y4AG15vD*LTIGe7iTHOfb>L z#M;p}CM2pjE8h(cDaUS(j2~zi0Yx|J%6-^>ZD6;5h2kE667T?>!M(afV93}+u@m_5 zY5h3|40OhWVNPPo$yf~Bv8IGVt$$(>hBCUexz%4sRER?`{&8PLSG>m{<5UmBkedHA zhBgfeI#qg6`b6_I+*AFM-Bz`-NaIJxO+3rWzPta)E6_jktz*eFcC4syH!vP@|EVQU z>rAIfV^JgN65V7PE7*Hy7R`nMxF!cl|V=1AQ&DnvGm)I;; z>lf4JNB{d-zs0Kp<7boco!Zb=k7D7Z&QK&c@<>ryd{DXiKl56DuVermZ8`6?=!WR)emYKBNAbMc3#FB}d9~D%ew; zgPZE^f%gYyvwt&f)~v#`ndAVCd2?cd-D5@-h`Azqhaf3Xk7Ro8!Vr;ZF!Tk)M2XI! zF=SP~e0hjvVCRj_zQn*}p6OitZYIp6hjiX3lb`>uNBEhrNIh7X4we3RtBUo(r^##y zb3>edmLI?=sxoIBbl9cv$by9nS=in2t}n=`9NDobW`j$=At8e1gFJum7Y#ZIFaH_twcNqQ*+)7ghK%^WSgvGX zD!91(M*}%iI#d};0Ir>+Qh9_<()3evod^Il08-kp$_Ce+8b_6w0N;WolnsRjs;)zp z?g-XE^s{y8!BzaqIOYa9@So4ss>l1fX(ck9wS(p0B6fldRgdv;93 zFRrKY>YIpQ_JPhd8hT`pW}~Zzg#JTkvebcSGEmrhbLi@6SVWCBgI_!F>GudO3qJXOmGdFli>Y>vKz2-KR+>g4_cp?E<{7YKz z=BI;+s~veHKgvV9Ixj3p!0Y1I2RS=msefQT!6k(6&Q1MNyq>)6KzuL#Qx0A5tuo`v zWfyiz*w!vW?a4}FV65=uz7cZ2)vvkPC z(aACtYd#l=$0bmfS$EG@Vn&|_{nd}sqhBq&$SC?Ssyx5)z>PWUBtgo_X>1Y2Ib>$x z*%Oq2RBq$38-c4r2vrAl6pBL8NjW5@QmvtutY7Brr?M-WZN=U0^nBJaa4+KCy<|Jq z@nX}Gb}DwIa3mNb-1T%pf@csQy`M7?`25Z;)L1b)!EY!^yz6#-%?0!z-bG1RT}&nTtbI5r7K^9aWxjDVxqMcA&Q3yOlsOGu&%-i0cBuL5e} z)=`jFx1#uAdEloH`Awk@97}6LK6rS&3p~qDAfRWkioG8^^!1mxG7rbf6@oXpYdGQ)8voYstG!9WN zA}T!zOq<0;3*eD=dVT{3?`Ns{p3*YPoq4gvh5h^((EYXjx>NN%h9*wp+P}l5Y_qfn zB0ljni=?f&eF15Kl|5m^hBa&YCU*K=fhhhmebyvS4a!fxRA7vQeUjZ?IZF0VwJW|t z6X11cDThH0@@p`3m>(7P9SnUB?zsJmq96!DVMGG6>o6+f{|k-v>d>4%d0_7}R&zgX z^bx18I>iWhK=czRb>XWjD;wf$Ta|@_ zzTr5EU7k2v*+6j)=tYbbZX6uDe2746aB)DruCxN3$ht+jr?(&RS}=*KXi)|G`N2;$c>7PP@&qj zYu9~q5(TY6_>uudZz;>9$r)3jZ3K+1^p|ipEPh@eLLTXG`NP##(dIvk}h<8I79$nYD`=g#7r&O639TyzJ*DzP`A#r4%4xGR?j7^ck+SZP9 zBZY)29VSQ?kI<-jjkKB{Rk+p+yTFlXXn>k@#d$0w%y@M=QDEIG#tp{ z*S+xX;s^(6;7Ko!({FR5Jt8~%#G5tAWzKv*qUUFT`Y@pCZX6eVs#HJC{J@o@VqLw` z{)TsSYi4fXnGj;+Kghn19zDwZX;DCdN58R$zVX|bV{_)uzcBjHt$g?sFR$gK|9vt3 zxyqctbu?PUL7)A+7?(KW^gJe4r=H9S zz!!ABis8@NXxXiMcSMLa5zIQ)YsUC$ea4T^iXUp*)Z`1YlbW|Ftk$DPd~`$ z?K5*{Kdwv=EsZRq6~AEa!(IqA^EkjxxlvSoo(=ZAeVMEoGtHRYw@b1*uafQMi~!X1 z?l6c+&`b^+UsT{B;qGL&b$^2efY0=PbCQ9(6UgI4G>Ab2?5*q5 zQL)YHFRGF|2VZMt4|QPC!-%;Da{Z z%ASiEnzUwK?djcr=(eQr?b&((T^;|pH+oKN`09^e-3ZFxKmS1As6*rb4|8uGS9AXU zaUWx53^Vu|X0bCub}AuD88axAlr0ovi4Y$_5sZfaL1XyM_3 z$NfJa*D=|(-erQhP!I`{7SD!DY(_l$D`j%Wkk&4BJQBt|fBH0y>$;PeL|g%g+C&ID z+q7weR`1fBn|(YzZ(Yi`hjDg#J9=DRz6!eF>d&u?OSyGy$JlRYoa9DT=j6M9MZQTd zc@|bK*nq~o#!YY7U{j|?GYRZnuAZYN=$HnEw(`MSz{BGc7T70H)YE6UQ_N0@iK3d%l|5Z!aT|7DXTH;_mG1yfC8+)7h0cBLp@IE~)3j za~}s;4|~tO(wTo+GXCObplq>d-lEz&e_`?U9z=NDnfi8icF8ZvCf4`uOGNfXJ@@aZ z9(DFJ2oGBO~eDRRU11EMoF9oU{oVc1oN(XhHTGLWU(E%ka*S^~q|dH( z>jo`+xU_1kca^d-O3w_`>_>r|0IYl6t2?|*ecwy1rO0A%Y-6<#E3wqb?v6)`r7BMlACn|jdmYLjgTKTZq0-hW9wkK3acz1yQ& zjFc=;?r7ELD~Ho-&1#p&hEl)JAZJk55mdV7T*Zw9Z{qHV(}qXfM2CvV5?nHisEF=& zkO5X>EC2-Ux={jtiSag)pva+(r+D$%q9^KcfCR*~3dS%y*9oCOIpkq-W+HrWN@TSF3j@Wz@vi#&&88Xl-QCrCP! zDq@f@(Mj;C;jQtI))61TkE&j(XCh^=8}rxCqGw+k5b*_d$fd^z2Ol4t214hNQhUf{ z$hJ?r6Tq+fkQ$AuY25c@Zttw*qF$RN2Rv*ldxCXb*; zS8kR%meSpfk8kI{NGyF>ZNezmeO#u>vMra`9#QoGUY3L`?F9SL0}2hDZ6Q8?bH&oP zESNVh#Dcqimyts~STa$ODbchVpUiVY3=OfLp7MIn-?t8*uW^{j*TNClKKXGKYes~5 zVi}-7y_humqpUGD?`JUR9=$ok?}{Z)*JrIxG2axw_=D{VC6+=DWxfbb0JoZmY!M7U zaD~rw6jsT>Yhub;wQPxynJauCY5GvTkGRo{{7H~pESaZorIYb6?)voEG4K~L8XhFD zWlBHg*I^!pC;Nmo>F+&y|C3o1H6Z0J&sk2ODHIU!x!r6P<{8vv1@X(PC*f(lYMvFcAAI@F#_FK1y3l4GyL8D2_esJ07r z^6cAp=%!s4N78HBVPXF6-P{b4QQpz{>MnE*#4-w&I_=ei)B7syp=N2}h=%AIj=J8N zE_;x26QP?M9Yc4#diP?p|Lz2dZ_&>F-qT9EtGf!{BISs<@ee#sAu!G3(9XMf^X6gJ zA3e1SseNlg0oPbNYIt)3LV+bMARLkUJ5#?rHc#r+%77J-qh2#GVCA%XSOHD7<~^ec zRw`8m!_O+t>ZfzoL$6=@5SHJIMS-~;RMJiD7Wcq$|HmYPy~3@A2tFLV?;I;2QXP5z z&#xXjT-&|8Z-GXW9>F*bV5!er+!Qi28CCLIPN=a7Td274j`$-oeDL}dB?oJ=>QCvn zMpIBYPjxQ_hN`k!v@%k0BMTN%PYk*#(1ETAFqb2DEGJ)+9IvEbD3P6BJca%zu~ZmO z6;$hmA**qv0@ad&2N?-Usz4N;`wK$f%2i94Dksug!h;CVASr?L)=e54+&ODI?}NRF zLsj<^$Jpo8vNf_c@WC;{(l405khOZc?Dgx{HrJocqg&!0O!QQB^@Cqms#>@2*k{e9 zohh9lw9TPs4c&JQJ}01$ovA>_?I~mNVbEpgCAlxd3wZZ>RzupJiH}xMYyAcqu6hbb z+E%Vw@|O#QF;W;2Gl07~<>B6Jl`*8CkUJ7cu1!r;@sz$$m#^WUvha{d+G}95G+pqJ#rusmT5)+TWHZ~c@no$g2f`_L%cclrV@Qjvdq* zVl~r&*0EUM#@>Cmo&8PmzlAppuZtPdgX421b%-A%(sC&*}q zScSVMlDxl&8L_BJt8efRKwj+*{oM7OXzReLiuZ?{2u($aH2?spO+k$t>yHAnxE!G<9aVxT+8l<}VzmF{!b1>r2s>67jA3 z_ctT%t-_z@*1ZtF)$oA9NeIk8NQpHU zs+}aJ?ariCHPhh@6DB{4mcl<0W_$PT8=A8^9PD%iV-R=%0l?H|b@)~Il!zuyGCs8V z?z@6|Q&c}uiH)>MogQCTE#dIx8M7FyLQ}Zd6mP>+BZY(Vb=A6^G=mr4qfFLBx-JMO z<)`tkn|}TE*M?`?fvMEEy4cDsA^O$vNYR? z9i4g8^N^OJsRFONIrt=()4~x*EBWyozy-%(#(%YT)ZARAk!;GynZUGOaqiMa_Jw*2 zD}F2ITU^TEYA$tVpbIEe{_vlFsx3y4vs%mfv>}^zS-?;3!gF_RyZU$90S>0glhc(+ z!k@$GfJy-=wUMLv zH1vJ`l9B!3!2>3$9<#Zehg`fRjhr~wHbJoO32UtF1Ua~0#uIA>j_ zw}D!Q1?#NngY|oXE{dMf)2=SWc1P?sP3Z_iI{N+Ib#d737gsi_3WjXg@+jCH((aTN zQ0f@N;Ebe#4gFEKttiA?^h#Chb!qtJ$?LW=FI~EHm&+8p$ZzDxv@OSn z^)1z)K?7;$!rstSMEV5-^whxtC<5q(NqIwN&+ZgGB4!x$OzfoBofVC#A*}<{mB@(i zF6Mzp^<9y61&=x1(JA6##M7Y~SU&CUhxbomJ{6|HWyB<1##hlHMmQp{G(2hu5`@JAd zTrw81wt{1TzzTE{+<0b=2W<|$eL1v_e@?gJyPt1p-*R86spa=P&tCAZ-h!(Y{kk|g zp-UW0uU)#$PhHZm>|>jZ{BnrRfop@UqFQ%?0we(|71Ll~K?sc{Qj|ZyhJ?4<8Wz@L zt4sP;0*>|hw%PgH(qp=wJaVL?8cnq zd}AyJga9=i$KTkI=Fh>iQmUg|Q?`0Lt?SrZuSu#xEXc{)^}73&Ezjdl_(P>AeWX2z zV!}?WXE$i80jylc$B*r2AO{hBv2Ij&{+LRCwm@lV>$Uza#V2yiF-!?Q_wF}ckxU^m zO{#Je9`qq^R=*02nlaHM_;4rksIcdE$`@nbcLwI?*^Ho5D=su0Ds1p?oFJRauTU$* zNQt;~?p%1j|I>oA-bwW$i7Jr~-a67f(y+!1sIdG_**Gy^ECf!Kl2XTzvi!dyC__q( zb+X>hZEDf&%3v!T=9AHj0%=)s)PS>iuh#YI<&0j`>kK!cORyxIp*LX~r8ppMyYwx@ zJNtkk=|5iwoOFj%L;ZV|5=`}mpevN~az|X8JM9CKJwtWC{@jS%J*?03%tOVeGw6`9 zCdYbip$LRM?ad3FRbMRsSE(ZsCyOm8MO&vgCyL!f-dJ;Ix9J+s0J~!+oLi-p>W^ne z&yuIRP_B`x2s#Gj2{ehel?VdJ!BPvnOQS#DL8oaPay-=TP3ss_fcHo_wemZ1_P>|5 zX!F5;qJnaO(V$YUq&_O9;kg$qe0~9~r!Oejkd0}~0@-eGwdMohDD;K+p$&;T( z5I6`i5;_ALezRJI;BCO{3wnON<5Wj=V6zpRo*4?YWm37bC>gmZB7oX6Y1Yw6D9vzZ zHv#gX@zk+Sh$Dg5#nbI**A92CquK&BgD}7zl7+bLR0AxjGeI**MlOP+$pDym?0N`3N)o1O`pfd*8v6E1#N zPB$2jWiGXkKuO}{tgado77%>QHU~K?Scz*6W7>4uNt#{#Hk>X?NU3Bff_5|9V^3fK z(V%r7ND)k(wVPL?)JM01iDkx!so>qacT1%WJ0X*Z?V17uPx^#RY6&>Hg4SYjn<>;0 zPjmI!n?K7dn~!P=`PK#zsKp(gp!aMA0G)xl^hF z5UAh3`SGtR08gbs1KW&-nS;2CcCH7hE4NZ*oz+bTSYadiu;L@H?CAzsfeQ(=eJcSn=W?bJG1mQs4|-7g6Fu1H1%G&({eH`IbTSp z>FVBbonTv7+qwRz8Oq=;F_*7V+VH!wC)N1fM=2N)bvDZw99>ks7rih-%js9%MAFdl zsA!{&xA$&11!=tT5b%{aLbd4F>4TuwJ=4uI#<5WOXj=+_@eFUaHbu&HwJO$ugg}6E zmY&4gj#%v;K*kbG@sFFC1B+-mhnWid_`KZ7>eLels9o;k4xoqYZYsWclg5%Gg=eW! zrf3?(K3Ubw!bwA$;l`Kf7;=F$MOw@Z9!=XlI1S){HnCaxnWj z!Eg-x{B^ZzE&@$hNGv<*!jrPdkdQu7HLbj)W^1WL5T`f1JRW7l_t?g5hmFq<=Di)E z_>e?V389Mq0QDy&y8(*U|aItD7B5mM@Mg>%<(SOS0%pZkC>8+?s zrBm~6z+Y4UJRFkdcHWgl!-0o6*eYCDw{jAsf;5~-`VthK+byp0CTF>$ny%NW)*uoT z^b8Jw!J#Jn{8s9dQo@XpknqzCEC{Z-nPI_7Z;%&f*w&}{PV`SQudcgqlVR&hNGj#TNZU<(qX=}WaD6rc ztltYy$_t4jqOLe#PPt|ZhYr!!NB;a%TJKZfZ;BYRsNLJn5DPTqOrDZ&AKJPDts7K| zq*S2BpAnHus<~WPr7Yabs|O*X)IUg8(uot&H_w|pH`u}yZ9cqlqb6^p!-yLTKUgMw z@%610*>3MBq@l;O-Gj`x+g_6B)JvpWBf2ny+eZahMU+twT=1GFE{7bANy2g1&$`T+GLetxOHC}=VZP&d;C{kC{Y!-@0z9F zQ>2WlJaYcRpA+5Iepgb!1E>g9Vu&-qRI!WlCpwKqTWTj#mYOa~QlgTmCXgaO?aOxr zPMOd3)BH>c-16mN(Iwj;A4kvy5hs-`QJ6;&O*(qQ-sI`Cva%YkzD`*Ff~%E7J3m$Y zRHU=1{)rNqksv90R#f*=n=9yskX32@)dN#|!7sVC`rK;x``2dt?WelC`o&Bw)<5Kp z0mu2CGRQft4?fHJ!QU+_O1Q#kGyw11;ascopw!Q0PabJJ53`;u!IBl%&7shzzJqY* z=#24vZ?FZ5BiK~bL%SiP6AZb%u45WS7whR&t!gEDS+k_;bbcXG68K6y33ruL!#r<& zycC470!^%}hKM_AxyQ%PU--*s`=6(E(eb1ss*xj*Q@HA8y}T!XAIzAFkc0`K6#~JO zTd^@Qt{0cA$Mp^F9mqu%9f+iHD{Fszq;1UP83YT#>oYSmZOWJ22St#So2~^%NQ%Ah zc7J+`?)mdDXFRGes&4PCg1cb4yBYO_GIP|zOp>x(?hp&9DX1#I3W`KaiXs1&a~JMH zZ!CuM^ec#OD)dNiwR+0kdSWN=_8Z;S%p?uzLBEjNX{b#?>q%bykmZ;8zioGFWh)QO zP5KviA#2S5Mc^dG(u*oTk>L9X_GQ|-%J!7`7|@y4{C@U?@&PmU`LN6#RjQ>21+nEP z0jrM`kM#e++>#{9K(SLBv587>IRl*Q&n{fP+-1u1DMOyW=6!?Lt2r6%2%>7q)}`D0 z*d-0v39U@R26g}7@HnKm=L;<%Pnc)P?0xuadx_ImH%f$b|lu#O}C6;Rf zUCr;VZBC+nm%a7uO((LJ;8{B2M1swtT_m82;bU{+K+>Qw1yCTq>@3u0L^J-l)A{vJ zp|Qz5d-n9L7Bj7@lT)yTkVF`DsjBo4LF(?YJ^Txxui`lfyK+UUpz*5Rl*F8`vck&K z4jhpmcFT3kioYlex0-Y$KWyy6{MJ|P;$;Ft7E7~(v*$=!9fuCe7PM6FA~_HSDQ4D! z=ZjX{bPbR9hk5jFDbE42YzpX2S`pfwk zyX_ivb)&@bWC}4(GKKTaiVucG55kA8>PE`=UXH#}t!J^wm$^{aO(l9221Vh+F&Nk5 zQP&~An?$;R0g(+Ih87{Z|6L-UMy#der67}?T71cnceDv`f5YsBbLTqkegLo%yk0^Q z$*+i@qdb=qvvAr@Ub@r=Q0L_KJVxw<-5R;*4>ok${-+|?YJ_Xs#h>}adwP3&OQlOn zHn5ANw!$j8dWBf|PRAnqz66#}1Y&cqyT5YHZQim&wKS~{ifX77)nD8j%C1i~EXd?f z-v3zLqHD>67sh|f{oNs&=-BDoIvP34j=hCrE>J@B9xkK02D!o4AKD`vMD;|15M*s# zuX2(8--$atDNesM3LHmh>0ufd=;%%nthS@1+~wf1Pgj-RtBN-8 z0W`up%JyLP5>TD-_b+rK9q$^kGXL?74g%cC|AQIa`-n;YE7m* zue^f4yS+!Y+wGr!U{O`LU>{M$vZQ0xXixw?X*1D-8bQmB!tCedF6Ps}(9 z9)(uxaw)6f6j4rC-%ia%8M7YSb7_~XuM+;{s_R?BEl?%*CfDv9-{@Ay4+TxGCmdbg z`Uj*))>zXusf-eO?Wq@)vkY0)%{^XFZP)@Ca{geaa4G1aIDV(|Bxjy=L{5bu;+|mtYdaD|h^7Lz0xVL4IGEboa9V;caR7IX#as7<6-SuC z`uD{Ae1VIVChgW{;m((698$eO4H9CQLg7(FsNhY7$ey!~!tz-|ETghHc15*Xx{{PR zifzTG?Ok<LcDrH=iZUmO#shNFDm5Kogz=vL zyv`!*uA>RRifCc4`)RBTSN=Q7OJ}Khh@~NbyMyzW^9~OTFo|Io+|oC^*S@TqeC)QGV_XNY$QY`+N%%+EM}#!ziXT>VM;Bl5`b#X}|TSaH)O)BVtX zIpA>JRGDpo?)(kqFDLLqvDOdr`XfeiEthiZf4vTl#ocfS8A6-L)L$KI6yWnD#67hBvgFbm1g zP)S!9q19Z`^O>j9KfhxzjCDRS_%WF1G4`0mqyS$qPOA+msrvNii!n6cmv3oI{L>7O zF|-;*KrZYL6NTXR@H#&UHcA@Q5-A*=4h+@^-d{X{-CLr~Fm8vfkRWC&d&RP9z_d-l zZ}#5<^?>`fkxO6S%F3N$d2_qZjr;v}hNYT14hx>Qf-?pmTarpxe0J}+p`4YktD64& zsi;-=xHjqVf`)xi;&E^DT4>M+9HhcD={be<^6mybEp@RzvqnmI>xo;XCXOy)A1tvg zb%<#*%{|~=Qss`^rrj#pQT2dGb@fhX{c_MyNuz(trd*MOVO)`4_eDe-mfFdW&8(Sgzhaqc_`7by9jz9vgW1aC2F7=>) zPNcE-JJw5@V;b)bDQX)83kW$5?S}dV6uhtLKYv*YA4NPl44bL#m+p_|)B)EG(tA5B zY|S}UUy`V01iLou`3FfPRt-DOF8mZD5cV))N^tRgaLqrJj8ws48!IhrPHTPo>4;g5 zQ>o)XJWVWWIqDJ|cLJo@EDj$D$sOugYg)-iIviX|AB4C%fBp$(Kll1LL)O;~*69wt z|8bbBD+DU8MnZIvqsW)ve|g+3WD?JbZzio2uo`Zeo6&sRYIOCzuzd|AJqV=@pi_aj zUwVIFYnzHDsOmE23RVrAGKXU3#3f^!>|eg}K10aUgSI8qh$xXxe{T|;FG6ThAWFC> z=?#6a!5nd{(NzHfP(Wk8RAQ-;xh>eV%pJMc^WUu&6Bh!CiPY87P|DIA>df*aHA~mU z1};!c1M3hwt_8`B;+QDt!>_FYVjjRf`)FB_8gb^f-hKPt1KjQ|dMWK1svyBsUW%_z zP1T3gSO6DO))As>pTrplR*{u)8ZDEG7>XvJ!R({A-;GE}LJ5@q)Wm(Kx9KAsRlp{2 zlq(_fn$f>l(7ny`OAAz zEzKJzEH^{Q#0r+?KSB%zYwvc9pukl%Hu4;yzSwa~iy{yHD3neQXnYc69|lCmF7MJiC7f3*XAcaGP5OhHpMQ~=|<37 zC4%Vt-)~B&K)ru9gan0zqs#H2_o$2L?#1#W48_K^&?O7RpQUuL+p>8l^{=8@i;B z0k$hNmq;3&ls8R{J%qS0HZtxE(}z5blxODhtIDFpgtPBpSLf5+dkgKQs?O9~-dXGCBAD+Gj% zTxsFv&6(}RmnW?*vI1-9JzPr|E6NGMoFoRo%}nfHLz06O5Oj8MM#CVu#Ko;m{@mf( z5Ma|U*LBc06#N($%fpE95A12`yacc#$m36w%D<%0rh@#G8^3&|oC=%I6*Ag#L-_B0dxZuOS6%(DfKc7!a*VEV8Nu5FrN+9N>0jKH7ZST4(hAkApj0iU3s2 zHIhv_xpX1_^9c^l&hr(*0Q6C~`-MC3=411$`Eh>RXQ!-%{?95~ed~;OZ&eIXnIv_G zK0PZKhmiw3sZI#kRTSZV;MpdPtixexW@m~h3Z5dXxs+D9MOs#}QlQ(^g1s2)1{^G) zgIt+7ZUZGiZin$y?_~4P^q9l!GF8P`5F<~#8w^WuKKxGOOccd%&0WC%P2hmV9y z!x*E$x??G4Busg+v0xbnw?~~-x&b_&B0)@=UZLSSv+HQcn+5o+`wBNexGhapTZ!-H zi%)K-QQq_z{R6$|;5|=XBFRTR#x8SvXjl{ru>LkN|8IMdGnB&6K^L}i<;a4I8CVPW zNqxG+b_^L=OI=1MDsF+fO+NM^n=R@KTy@R7RAm_u_hNxB%=->*5MbaM)O>oX{{sO9 z*11NfBF_=70vJVLu;8w6DY{`Lkoy_+{wacJfg|ztg_WZs0J*;phvHZ>^FYwyOA&-f z3-krjjaI^qNA7r3kyH)w1x0zl^r`f~HB*>0wu1TD3@VMwm-vya5ipIEun@!njo=g9 zHsI;h@(*Ptl@t}}2%Y(3MS4vf%6v`A$|gLAEFa71RytrhRmN`#_wT?|NTWaneBBVj zqPAnJxrH^1?M&%QGYLS5j?-AF#ERs-z(`c6R4*i{c=6)O%(-!AIqg9s{*QC89Ht}% zHTYv^&-MVBRX0`U3I`=dDI5}_9sEDbL z(*1hOuzoxv!aDGH)6q+B-dL0}?caj#m73sR-I$=+abXpeAw)^%2~AT#jGsqs*wC3< zB^oYVKnWkB?&8+4q3w!fA2azNgHwX4q8i>VNnBO$NEi#mRc52m2%oE?h{SqRVo~Pj zIgD}4G6TpB6@IGLc8UfZN)MbWnshe0cOT4z4n$0Pb7wUg3D|rmw6Knu2B)gCGWnDD zKW@$s54`fmPr8bHPWVqrvITUgBLO;Tnf{nwNS>VqQgGVDg=VCl!;V2_%e}MTXd`w}~k^$(v{RYj9WDsT_NTe-H zLuM{$l09oXLU#`MhnQ=iK4Hg&k+v&x%n%mKWF4nwgN|&#NI~_|FniXE_n43!Q*IK1 zXi}Od9cCj2xPS9iVdsh=X*Cy@< z*V6Foyv+9t$bSuJ!Gmb-2lAb+jH=LA^Q;9En>tG2LlCGa1!%uHK79C)-A$_IZ*D;E z|ALdDE-aC6PK&9U#N^^`-4>h1w~q{ObwtFI?XNY{UIB9SUhLhV6rR#Z-Xg7gRMgXp{mH zJ-TsByy7%1L=|$hsU*fXc)&^i?5kqN=>yGA!i25(NZP^A0VReqbe0{xi*B{G=_Fj4 zfTNV`g-HkHYC;dtNE^MzJoDFR`Kypg=G+q>y5W^&WU)TWPs{p_n#bf^iN;`c&Vlrb zDyHLb{%bROfU!8>W`f$F_yS>us*irzra%C0MP9#;QdSNF;MQXIV4-l&dvy4?fFkeQ0H@G*8befgWlVqx$g2s=5 zrUUu3s6e+x&@ezExSo-7Kyquxp|S3P|1FR_c(@8{<6qz1#Dca+RG(CVqq zaCPu9AZ8>>WiUit;G8RCrbI8Io0TGWu6;YPOBW&o_f#*)Q_b*i7D6(4XlhxXtZHT^J(&8A1^-gF1L7#snHd zKQUkb$O?}U*XY@ENww+EyJ;(^GD~~ZbQ41T$*g+7lTigsa1%$hDQ{5wWAjsBS^MF^ zzelNRgEcvE>Qr}3G#&MItcG}K1$a<+uB$-nDyseutRrnq*gA|EnDK9m)zXJgY{ zJ*L%Wy#9fVi&1BMgi{cuI(N#F=YX5GNsw%-*JZs+%cAe1k(F!|6GiRix>0eT|@$+-A&j5u2$rqMa)Cg%6aqVNjifVRpZ<_2TRTl@8Rve%48>7ezGFD4l*zJ zfI9+(h@MOno<21=+00A!i4R}M)Bf{wVfOF8e=F*M9)+_i){|bsw8;aGQYI7iRu}N$ zmA&u^ZCTr>_C?%=!CqsQ)oDfw0yuS&0k5)nG})M*=fWC5unp|Bcrvu}{J{*xx#i5a zCn?_p(hFF&vw}Ge6gBo*R6ysH)ygJy70I9C*&V1@%2XJr=EoIq`oi3Y8}m=lYb(=; z2N8fNLYLG@VE$Np<*5lD_cpJ2(%7EZYHB$(758#ZS>C8_IO7;n1$074*5TicRBwF{8 zA*xc5kfQy~&+cf)p@*7e8{Mr|B3jjtLNpPM_48rdC2>7LiP9^|1(>@agqZHp|f zy&r(89jZ{^d@Y+^0>$*igBk@lnVDx0d!^1_W zgE**`AZCxH`*Ipzf2v&ueG_9}yDV-TyZpFGkv$qUkErYl1TA`YSB-@UYlb6Z; zs>r8IbVk5SPF@UQ*!enKbXW|Izv!$(m)W0o9c+xVe3fQvhyzUHTXSN8*IyLQ=s4g>6O^(acV7#7W?)KaoYF-?WYgA0k zJ$4)B`x%iPIYBF2L2fK}N_>H<)PPWxeV_8zoVtoHXB6G+ff{q9af7r9OWjD-C4A)< zUw*mV^eH{k($X^4bSNPf8x~MGI~OvkdeLXOPkk4ChUXm5dr&Ae3tQu=LPIN<&P>#| zbm$R;Xab}h$Upn^Q;{PxKbB^e%{@2C1Q4o-+NjOvzfCI~+L`P?tR<#LsYwZH4FfOE z;u$i#7KzWBN*Pqq3N~yVzT`Mm8;&!np{DK`oaPhWi2X^2*Ey60H_%R^QrUl=RJ^RoI6-qY z*_sPs0;J~DU3I4J>S=2^rkahQ2bwXbpIO{uXgMDsSXlsdOIZW4KmlN-4N~w(ze8Bk z?D43mC`hC(^OK52epn=-^H ziY#xLb?>2B91i4GSOTh>Mlm;gL_M46AQce6*E2APUb9>2Gk(LMzDIf1(ux#|Ydi8&f z9stbRHaa_d{MU$;sV-QwZt$GF2=`=gjF;;Wjv+WE7){FMz%SwKR0ZoIUg2~5lN~k! z2$^&N)Rd|*sG5F5tP=(Z>Pf7G`Lj}PfffD zur4)BYf1r8rW38`rlU0DEDNSgM5QY#@$%N31fyrs6B$|w)e;q))t(E==S%OVQYsF+ z={)83;(UYvmcj2+v$@p=B@CT|i&!$J2sjUxc6NQh5POk1D#cP31xemok<#PQB713Y z9kCBUL@z>nkP5^756z>nFD*QL(^bk=z)kcPWBS84}fR&P~doBZB~;i71R5~>pC0Dks-RsQaGW*mN-$$zc4n<7d>rbim zXgUC|1uwXZ8t!msgqWr@??O1|HI}#sc{`7e9cQ5`t}Hk+>HoTq;3S`T$=b3gYY)D; zUAgVc3xlRS*m+*`G#<+TGEfGUPOrDSJGF2BBk*!)NJuS5(eb%>a0vqF>FE+I7!sS- zIaqqdtb))qMXfNS7QCBbY#mD?K`T~jXgv?%?!joTNanFPGQrEu%Y6+pq;VqZ&*h9s5jpV5UPNO|`kJvMgBoCh z_V|pvx4?m{_|30Y_k|`Rnlq2R8pVshM=#=0#Y>w%^oD=qetRMzpEPLRya$x`_xYP; z#Ui)ZKSuk_RlyCWV^Jl7=CZ75dyH>$tcpF8BhNuB&{(`7K0bE|&+$X<*ogJ{m4}|+ z)ay}p`v*9Y}>`>(+gFk^Bjsxo}SNZNE*y+*n8inCDY* zS$)AG9Wd^Yya66tx?g^Lu{^P*a&{c8<^iV?y*&&53YnHM_@^R^JMj{iTkR?M!dQT2 zO&F$JB3z7(b_yr66|>aW*H>b`a<4!wj={oum8KHvux-7#3oKt-{%HEFO;L(2h%zM` zZDnm879Ji8g;}GK>O+LZ8=eJepkjEEVt~Yn_$Mwy*HwtWI8rQsn(8uOMNWTws6+x? zott82H5mQ4It1A9B>c8 znG{gelSJ)1+T+o$#DnjqJ!nIf2j}Dg9W;Z8f-UYsOPofyI1gR>^QC@c^a)))1Mx%Yg$dc#dX_N)1z7M01BC=Io2A?tvxD&FeG`a%?93V zhIrhlno(4>6g8!O_oh6LD{YRF0_+9XspmHDM)Y)GJ6-5FOXcc{5tcruOl?qp0;~+Q z=p<0VBWUn9ih&}*=Co;PB#;{c8C9{HS|8t|dy&tJWJJaT$R3dm#nW$HMN@MKA+o-0 z>2^Y`G|8wK<29}YRJ#wh;lWy~ zsZV5dpm!<$;Zz-Rir}+`sp`*6rtbpGOae*LVur!dakWFe;3Z3bocKP^*h<=T*8wvF%rikyolU_}ZkKrIiWGUQpCV&pi@)Tlnb)J=Dji86b+V zlyy^g_C+{As_;2BUn5|<8QaabJ0c#j1H44BoKuq`2Bqa(e3_Kg-(w)CBCo^Oe0;;9 zB=Ss(fK5eC#v7n%V9(6v#cN`#wgiGj;&L70GN5~w00mM7gnE->Q=$BdKx}TrABFgH}Ho(ba;S!MFSjY7ByUJ zHc#qC8;4vc{2P4^$rE*-mFf{Lf&wty-5`6gAWu7@WJXQfpnp-FM9OWz4iTH zJ7P2XTN_wfcFuo&)INYN<#5;Hk6T~C!)wXBr#@(|7G2HBQVF`_;WC*qiTfndu8B3& z>QyGBcR$^CPqNrj^jzNAy6>TC!Ybh5bI8$5g zeZ$t$(lQ8ikE9aWx`{&u4<>OpT&UXEtT$<8bamjT9#r;*5r(v+J7r7X8fV&QwLoGYHoq^bP~xBG5LLn^f`! zT|uxE+M`W_-O{$^q(B)Lh~XAKC~$pwFfX4MCA2^C^;xk z(9p4f@KMS{>64kTN5NQXdqfB4WpZgtQnJX3H;tIpS{RUOYcznVT|=o7qr1=0h0NJ( zL9eO+6^U1J`q_}iRKHARg2bY^Ms32z5Ngq4Co@LgWc5iq^%#_xh4;*Hm@|Qx3CCUU zn{U2lG3=(j-cA-I>Wp3KiM_?F<%v~Q#=x*j9)Jxw4RgN|fXN$y0!nKbtd7nA749-y z6!8NTSCGWQ->W3c?LpRahICK)z3y&q1MJFLb`u85B@(BTY*n46x)QUZqRtqL6&J5T zGYAMn+*O5vU!3mwn!-vnK%Mr4h-kOcXp&V0|CtkuctwYLPt%(}R+a(jTX_QiRymC1 za{1}vRlI?~(K(cW99G!V$;G7y@ukbqLp7u>)v+Td!%rmwb|Z@4Y8Q5BI}S;Dnb;zw z$wH*4lbc?=Y!S=rP>)91t*ncBGWQs(-4XAl=>r(@&QL!&o3YbOsj=GB}e<7rM zGb`)QHYM_p61u5fhh$9!JdVL#4GxKSilQ*o9m*epQ#r{ULXK$wElL zq25cRkR{v8D7OrX5~hx6;0s}qKY zA`*+`_R0F^Y<9oNPJ$nMkcsKqD{@stgyDOg-q5z)ATzF&A=hKVgk7cr1G@d(I$oD% z$PjUg(tV9yN-H4)2GE;)4_IC!BAn|UYJY_hBT!QnsWNlGh9W4{hvNY~=ZMzSHt+RK z)9vRU>wha0d%fxNz2^C+VFsaFX-o<6l9S>{;xE}7RS+mP5HWt_!K8qIbhR81>Sdq5%e2!{(~3<^y7i7L9&2Sx6CWyT5!4}JrOJj#}w8nE1#fJ%c z`=r7qjgjJ8JxC@6gmtPm`OkpQos0mbmf&Baw8-P*lZAbs9y^Rs(Ty6W@;q535M%*K zZ1bKiMba|kb^IhSvAqBm%%h?Lu%Z}3@&+uKBsd~Ixzf#cg^LU|W>=&YK>JM`#OQdp znc@|bZZpeW?JSxYpcpthzRGEGJc9%bz?)n47M^q<@C$IF z<>a|;YU)SN3D)UNb@-(FCi#V}UcC)*4{l9o-U-8ty9o7%a+0Q4lh{(abDF}tvM3)E2LyXDk|~1>T||7eTHP3)GvLEA2FovsE_7nn=)k{MfDnrN zjV;o)P;~>68npkD*%OG5g<%PEgY&>>wh?wEqdL>^4KHsqUP$NmH3+9EO&r+XaKJ>ll@;_b&pqeh~ z0w987MIh))oUJuL+r!B`71rn)Fn;N1K=4(94mR5Z@RQz)=A48PC|)aeN(ZX4Eqb*5 zNb_{Ign+@WR4WGzGEHNJN_~%G#<%yED6?w8YTZ`7Q`W`N@~|}Sa1Fh9Nwr&g2fiDh z?sz1^KG0hw3w3h?zYXU=${DHBKR4q4fU3se2W0B9M4SW-=vz>wl~}9;2Yk8@xg%Ri zbYf(rV+In#jJ4OEgc%BMi5(zMgr9CU5JM``F4)Lg6~!oUhz4V<_n+f5u^%Tznh?aB zcXyumPA@cCme$tk0o7`lhNuCWL_yh+%JclBA0}cxl_zL~tG)y~?p*ha6%?{HtD=%! zzy|dM01d!FGK5YkvbYXAP{f!ki^VJuHD_Xh7L#ba4-K9iy4`dLNr}pWs!Gt_ND&Ow ze?}^U%YK{mWFQIvQ-Nw4e=+E>ToNZu^+++-fc~VBAZyWdVG%6Easpw|yEm0y<1#gb zwk=v6s&Xrc5QnQ=)G-W)qz@4>sZaDRqZ4B6uc4$0y2NZ`>-rA7{@pmnPl)77=@LZD zUF?m?S>@yhFO;6QWqn@3K=k)UqRI)Np%*Z?CB~xUQ8d7zt0s3Obh3?@^?5_er=o00 zYJ|$k$>VeO&GoOM4t4sjHM^N+qRRoN{lZqQ+Db*ooq01E%Be6EkS7dD*DRifG%zzr z+dq|uCQwop5<~-q7R4uMUY()5yZ*RqN$61>mJuJ5S<2lQ>Y zUG^@?^Ks+V+`wr^d2k+Zt0DiRB5|(K3%D^Idh2t5x#5&4K_%Uw7f_PyesJ34^vVAI zc%yWHuFd#{;c#P2;O2+9zLe5>L6`PH{FN|y`RO<5Wtd8!^nzp02QJZAz8$?)(i*UB zPZ1w`UucO9>>=G?ZV`cDx(cvtY<=QL*tpFiJ}H4kswe0lS4yT)uaui{-@8`J_uMzg8Gmiw!lYh`a!2T){L5?Fwj|0216^HvvW7j zocV!R-vAsKItq|y!0sykSlsZq#DCuKH;VK!ZE&Kc{r?=04G3mT0@MkYm_rM}{xP|N zjrq69Zdqh5+=G&_X2*#x*tV?+V9i;dUl-^EN^8GjtmC#D<{VUm|40HEh>vH~28>&j ze9)m1)m4{GOHX&nKIHlxfiH*#?xJKZ|IlGTHfM<3B@F@}gj9Bg>swC(2`ZncOV*@K zjl1LjQ5AsYA|)<^WjTy;Jpx{Qzh+7cuY_WsMR6QQ4Z?jg;G56h+>cLD8N`?PTjWb- zNG;rlZIP&fm?hw)Eh1{=L~ul zfDfId{tc4nj-4jf9uFldeU4>EvFwqg7N!GhExnb)4s<%^h{OgpKL^uheeTJ&UP<&B z?fsyPA<1v>|EY}|CZ;lno~@-z3v&iJ^v3XJeQb;_B570Axoz=pjn${IFUKFP~_mDqq& zdZnsK-Cf1`nI$DxKFvX5#24v9pb&CdW zYC-D8o%npGj0d0;n-QO{t7-HRhXrxL1GTn2N&mthF)W;s>BJ2z`WnVSrEmaqu*9M`v-QDr zm@pyji5#0;S8k6~)?_?58q#njf!Oy15$|ElITSSjoPG4=S66qBUQMK-+!Mk39vXZR zQoe;#F~goeKCfsFa2RrD%P0{Gi6$YvW6^GbIT=^z#xeqw^kU{av4AghZ(wmo%U%e> z9snBNVdZo*h!FDAUU5#(N=jK(0#1-{M>Hlzdo&`!DLrTZgHmShCDR^>I}eI6=P=vP zL9{mVCUW z^neeMr$e)wMGQvLqsLHNMleQG40#HJ0_iZ)YSUEY_RQ&km_ z4*{Ku-KYq>>yUDQv22pt7B2vv9YvK@Y{t0RENjxX&V!4%m&YZDhJA@Clo$|J2)B{x zLt~h;W*yOMxX|b22sBdvP+=s!mC=j(hIECbfyQAzcKGnEiET%-hzgCUfl3G=C@83P zKbCUF_iIDUM)b0_o!p5>ME;cUk;s}F?!n$u)lXzb0-3On-Hwdor%q+e|8T*(Hvez| z(1@V(w_|3~qXEoK;T!`l462^sA+jbeLnmxK{0BxX6LeJRNNq-Np+GPSv2PA<^CY1rI%)v; zFT9q}EwEXy510h{P0?~+lAoAdxHQR4)0`nsg$#1g@rT(v{{B#ao7Q zUcUvt*6AJkT|6{4st zOwYAy3wRJI@TbqIeS~%1Cea;%7-iChKhT#TN}!2>X2wns!(9QZy{7dBBo)Xlv<;o z-S@_zaNinH9;ZoY+38|cu2JK7jw)S!d_SlnriPC#iiR6P$QNH=F*Hj~!XC1(G;HYo zO`AP*skrp2G>CvIzFU>Z@4&lyfdg{$8o&^`5CA>>wO^5J!f^jE*IKy<#mcS#PNCR( zvXkc!7ilRv8?ed?yx5K0Nm(wFDbECyaa+stc6^t=J7Q|tw1kGQ?ejc)FZu4n&mOxi z&n#BqqejWaH+Ux18dj~&_+ouCd%u za;Dp|5+9^2k6ioB_N0f87Nyl~P(EN{lL?Q`kMNoOF|HElca>`g9qE8M)SxTq4X<^jLdZ~QBi0TaelJ~HsF|t;@*=X)p*;L=o5wNW9$4E{qw^J zGknlIug`YwxBvH$7?(t@SyK~uE4fMFo0C_@o^0d^kWr>(rjZ`=Hq>=TZ``!0OI_Q& zCmZi;c>UeOiO&coeSqPcL=@k0j;`B~yLaz8FSm&eC}nt@SNOzzmXo3lA*`<0YV2U> z+ba=UX^GxJtJ}*4#ihHp4m&;KbBZM?wwFhSG?|{$W$&G}^PI9BHcbG=8_HwiMm*xU zU7?Hs&ueS;xcBku%)uUH1p$m%J@!)NB9*in_~)+-9Xk-4$-aF?-&;SD8Jycm$~ z(uI$0!92c<^XIc)CFW1hsbI*wZv8KRZdfA{mvBPt+az|*US3|*GFL)=yanv(3b^Bx zJ$)D{oGTw5k8Qn8jM*rn_!mDAf>pE`=2neD>wW69QR588z#;#7pEH*l_TAMGkDmfo zXg&F9*2w)&j>K-+lHBZQm5_)Dmtq61^`SP=Pe&sc>5S7Xgoj;;`bSQ;eza11$PAGv zkD?#BT=gN!RnAzRFzhbwxm`uuk-eGkzL?_c($jrwk))^8Surx0X-hi;N_7tad2whuMn0{T zAt}FNdURnyDp|S(6&ZU0tKnK3oLj3KKbIeS*>Crdn1~5Zz(?b*P0qf>0cjKw{+0Kn znu&QYwv9VcFDBIW$GPLLj5!*`-2`K6904)8e2aerupo=@R^xmqg(c+P?Ui_fGI89| zYE!ee|54`T;!f4VNwsaKrsU_Qjeyd3eTKs*!M4k`$h<=xjkJPk!dfdXQ zLDJOC-UsZU%^V+p-fJlv(j4O037n4g$P3%61e|Ff8yo9+b11<+^j*68>?I#_?UKFIaQL`%Sz$ zelm4286S(||%S-g4pFGfO%Y7%Zk4I`cT~UI z*03N3o%o9wt#OEYFLCAO{aE?m2BY4!B#S?4@2w^Mrassi-h4#Qb}wSP-ukuH!eapJ zXAq(f=bzK&J z+O7fGzu($k81NG>W;FXgj}?BqJY8Ge3T^ZBSgkkcuP&>#u;n-By(W5rctikiv}4wX zx87)C|G=YFCl1C5Kg!FyG-6xwQ2W0xwxKOK|H-|E-uqilyfUWKLYmy`Ol^{M$#ev% zNnb%GLN_dNT%n$S^*+<5Pe(s@6(gA2QBxm8+2O%7PIyYUPF2I`F&Rwd!4op#zHOtS zs{p7hql|s))Bdl2Pn>=m#_!WDqo+5}d}t8yjDDj(dTjuT^=Bg0Vu&T7tY~nv!rNyv z7s_2W7ygt^2r)RmBd=|(#}iQ(wk=({^qbU0GeWITx4hZEo&BW8nP^hgZgG9uxN(Dv zj~Iq|&_(c`_T+E?(F6CodqOimCynYc{hYCzc7IUylRwO^x&#RvYnY#-u05LZJ5hHG zmjKMl=it+@ZCR}1j924OtaHMHugW-g&bF8B_E}{hyLRu@D|^@FQ4P2&7KN3#$kcag z8c#ZJV`Gy=5O$hdvz)rSg(D%)(&fvS-{%8eB0kxy`hgF#W=^`axY`|W_Upec(y-t@ z<=<2EYP|`xu(NkZ|N)IG%2) z8!MfA*?QDJpE2wMxnT1BW!$OV0i_;S?-!8zeJjlLGe$42Pq^f@-r#m@nQhd+0tWK7 z;K>sKL;KiqJSZI{bJ$jKOt#}fyW9Nh>I`#NaL2U4pH;M4&u13{wKbVtE_pAU*oI++ zN2$U@?!}OFmlnwjTR-e=_fhZSMB~Rvew~II9KJ7fc9DBv{nUk;;MAi>kDLkvN`Hn1 zH6i$Ep21rSw4`vKzNdm@2iQE|C1YFD>LpdW1RXp5%$cAP)h0#iSsR)?tHv;1oAxaE z`J?NJ0j0Lj0>E41x9egZaC04@FABYbaVf|);yz{}S5P(fTksElL8o0eC5ydt~@%ILk2Zy(vI>Ge! z%sw$5a*a&FMebUcm@~)72A9%5+foEcfX8uvmg|(oxojBmeJy&~dLf6j5`=hB{V?U+Ch_grZDTXKbqaG`SB^#20VAe$ zH9gPSC}Qb?pAG$x4szt1CmHWfuS#XSor)cdsoo5I;vz51srXEem%*McE;l}})E81o=|jaL4QAWlo6kT`V)4d}8+r1d zP+xB57>9z?7me5KV5H}>tCjPL8E+Q`0S@IJeRjxqdgDErGy75e@{AkIm{0svtDEz! z$0GHeO*GxOU#CI$032|Ak;c1e#r}D>?$OXRd|2Qn5$((Kd@aV}{BQl1?s{@*V(zVs zjEpRP`84=*mzdNKY55GNkL5PHaXM68vf1!LvSZ}03WsDIb3R+1R5tMSAGwcb+c%78 zZ+($F8)6}8G%mi)I$N^?>O89KtaEFq_c;cG(S6mpnDInmXQo4ZU-AT<6O;QX^|+jP zIizTwD7@*cTNbXejYvq%acGaOS$y_eKQ6^Z|36A!_(A=~_?v8D(vVTw7SFK69UaUT)!u5cG}D zBbzns^P44xy1{>+lOq$r$~eLaKP&X{b%?_1xPnv>f9g4d2G9xY9=#hZP@8la*s|mOSP*L4azxb z6PS9HraOeZPo_J^g#dy39T8%%LP3jwhiqKlhev zWy7}*XrD8`4lK9dEgZ?n^KXrl=YaV3j3A}?xE{{{b0@G^?H&(${rJX+7p!9Up*QB| z+kS}JRI&#FgrXdCZv+SR=GAh3BQyxEj-&j(B5{09E7$-b79u>GPV3>f^&LGTezDHG z{PjVeL2}%fQ{O`AK-@e)(&C`9? zC;p;uOvJT!6ML5*@$AyhRMRs~s5J?f!?iBrtACw1>VMw29k>{MZKLM|Pz!Z@%3cg5 zv<*y2K7;Vsl;GbkmCH?t)(YnPeR2>1sIb$9|2WE6z;ZSJt(x}qcv?M|+YvkdraqY0 zWc5@6-gJ`Ok-IMXB=vq##_c<`ZulZTe?r$q@>$le`rOh+vU4TT| znlXhY$#N}W?8V`mwpdZH<+FXDp>_=x`o&IDKFS7+IxS!Wtf*k8*QTYsCcNvyeo&F_ zr9Ck#@gIJE?|c(>pixBXU^tqInxR8Q2}f|YJ{4SegNe9q!{FQTo$zhBJ&0E^kU z@$S{j(UrS+w0&i-)n&yW|5*Kezj4X$Ut-#Y>|I~-u`Z(QKm_^kTSQ|hx4iB6n4bJ> zPLoTA4jo+Ey~)xgZ+(Az2Stp0bNZu&zu3>GDZ4d}Oq;8xvZx5hYhU0?0KLf(g?H}| zr1ki#6D~(ZM@K7?tWsz$=jIS+)wUD-B|9VU=! zr?pDzbBUw>EP3<{YdRUG$T`~#GLIpJK$5S9QZvGEpGyOt3??Q&jXaf`S>ATIdygJN z*x3|PHv-BLzO)E(pcY)tEf)Ddvlo-31HQ$9vv#sf>-#~3`lhh0ur%Y)jk?w`hFm^F zLU<`(`8zgcHNiWs#W=SRM04l{dN~yU@)@n7e8#ymv;bpA7vU(1qzgOq-rkELo(;G4 z>!Yj&fUp);1f%U1RG82lrZLJ}sbtP)14>^dnYlPJJ~<6QNC#nu*>N5v0F_mKN=U4) zO(Ky(;v`5bo@)O0R!@I<92h~lJE>{AB*Km*LWcFl%gnx*4VhwsU4HoeXELc0%UX^Z z$Q85l&rH&Hn|)^Qx@3D6$`Fa-dypZPg|Z)nwTyxKu5f&9VJx!U^=rF;VNjA@s5zBX z>Xi(RK-!8{gF9=VCl$KF&E=Io(us7dAgkZMNEef{d*Z9a*N62KPYU0x_fVjK+ZaSw z@Q|Denb-H^M6ZZ@u_9yqPZZF`k}qfKCNGwB-RxmqFOJ&xAPz7)3$1p5z}%~FRFte| zajY8hxw(pK-v8WNiG^$@%L9>ybiK1c=g+FOY1wz=j_-NgOoIDj@BeP^Y~aoQj+hoJ z5(MzfCp1y+bza%B8PHTe{pIR~;m!8bIcnAB1^Gbz-L2>t2>frAX)Mm&GUOLZ0~p7r zLFZWwAbOh=xPHV3Uu*)wA)+3?5HQ^o+yUyAHG7h}Y}BZWV~$oGcf9seVV})sol2=|UTF&bRKWNdt9`&i3dP#vAMyaA@Y0377p^J@`$w$MC4I zqt-aS50SqKY+O69u?Zrmvf&c|z?3)$LfJtb%G6_9_m?&M^vRHRhXWiA**&v^^&~o3 z#_~0{=m74D(a5P{$eiB2IRIDazuGazvQK3`cV8LXMfMeRds{%6-}=;D>HqDFSNpEt zqD8;6(h7d{dS%7??|=IItejR^A51Y%yN~*8A*H)5gtF|6$Hi;;8+sqzTezZM-ef$h zskQ+oi`QAhci17QEHPfB^hFGvj^o$$>}Hn~Xm2qGx039%=Q?+->v!banPo zRaI#iKh!joI#$!Dlb*fkc`qILI}``8~Jto~gfDt+?l$z2E)r=Y5`MzZ*ZZc!bo8 zZw|ovFs|z<)BCIv-?}zy4D^Ty&u} z2>y6uQkUZ1h^uyv_2oc?*VEr7>Q* zud)Dlh{Mh`k>%r-iCE(F>b*kf$oo^#u8O%LVl>QtR43{t(XKY-m}$=PW2Dgo zB9bBC0yIBJY1=sPe|Rmdg0pRB**wH@@8xbu)8|L`ivIg}kW0(8pBBSy6_17_oV>Z@Y;H8%++)-QH<{<~N5N$1phWIA zvQSqK=kxcnZE44z%a@lRsZ*UBb6#2QULoE^TLyk<7iikJI3 zxNTc);8yz>2!j?{WtQaxyz*SIvtn{Kn#+-sW@_=&hgbLcq^{I0C8B$q{}S1xqQi;0 zv@XA+!Aq~Y#Pq5WrxAJW3(I~!p@n;N-p&&fo63V`!=FjmEUbB~Pov~A26!Szu?E=x z>Rv%n7{!;0q}71=Vfmc9^@~5+GG|#6YeT(Wgqs!vx8s}Vmua5g30ObLZIX!Z25c(1 z`oqQ|$eNU&zpW=e&aH(VZ@|#(;K8zvd}`7G;-A(I2tWuCAuqj*@d$4}7jr#V+X2hO z+32%)CL~k)_g}eFZ1oQ**UrT_pr3_nN}Oxnok4BNwWU7TBBs;lF6&PZepA5t$BQ-T zCN0J2{VmP*@+YUZWCqgNqX&O`<4#~3BG=<{<<}$<4i9X<;aw$yQTlMh}i_X9Y0Jy1yUTB(AW|Sj?#g6q}Kb4 zJuN*#?(i@^o$@po^bvQ*!}$B-tgl9Zq)R!1cOM{%XI04?eD}y%<4BkONJXy1@n@s@ zGxK14a=5N8PIUYL&W??3KlP@d&qS~BnV)Rg;-Gu`Iu=AMev@Y2hBHtFm;P@|a;M^E z5ic*H*+F*_E;q@j4XyqNZDM%`yu6)Zj@7)GK+mVo;f_m?E`wRt3D8~ZA_kTWo{zY6 z7U1WAtIFa2O+|9938wY<7vY+`6Ybpr7vZuS4j}&^+i-NdW;{U z&)njg$DHgvndm9mc!YCJ9-JQM9uK+t6jXXHa{NA`#L>|VvlS}5rq zQvOaa?c@66nR#w{aZ0Vyri5;`$O_<6RFG$IHFZs%W6_1g&BJvN zrIa1oc{ZSIW>D9Y-*JCLB1ZF7k}M^0AR$E1_F#y6)fk-@5QszBq$LY(q9D~vhIx(8 zh?R~F=-olF1(#74O6+N{-*!X4@{rMlfNmsR`}z+Xq>{EC%I~DrZt+m-qqpRZ{M*CI zE$9~^fb}_)b5nSDq-T5g9>2YQXT`!jtyZAxgIvjw+&!@2{ZjIYE%V<~a2&lT>bP&h z{(F?*)5|^57MdPiHXmZef@44hN-&q%t0Ep@op1}%(|*FydRO@9d+M)uUP2|%=kuy03PxtD#!g|XbAg4GTYV*cbl+C`|TBO^;OMTo+tKHC=;5d3_W_Zw0)Hvs{x;@$O zl-c$VvgB6D?3xa&tP?&t@L+P!W@Og)$Bc<++1HDoyI?UCtp3*$f&A-t2xrmKZ!5 zvFLH$^n5BF=`Lw5-r23sgTrxAK9O;rFP86u;=Rj^ZJX63uaLND!boy(OX`0&D#&1- zhUyo_nSTKr{1^%-M?jggD&@OA_50!fTbaKs`}6%;;7Xc6u@2=c$haXm-r^}T!s0v( zZ&PM|V2SWJ3`Z60TMI#;s!pP%i_1>mtD?-2a@JojnA%p9|003=OHI(#h(GNvGz#tn zjfdxObCs~BYZCQjzJ8VcgETaQrZq3DsL+4}FS;6gvNV}RJFRwo`q7?jy3Q3hF5>Dv zlhl|13_r^(P9ilFDL5fA4{}vTbWplwD-5B)Rx`pJ0qWA+p%{A2IhvO`Y5`6LgoJ;N2 zcl1^|=S;cBG=*t&=d|FzRAQ3gt?CvxL$7Xo>*njF5(Mag-WB~89$zUB7-#%qRVeBP zD*^|Luu8=)VBxX?XtAk~6i9|EB-YDEQ$(@2y#GT% zu2*GWMJihr4*MPXyL@_Tx72m%gmRx10`&xGhr{7jdBSyav+L=Kd zS%3A&&IE#E6&<8r``UzyBj2I7#~>IbyRZ$o(?*64BvKL)gw<}v)*6*g=CojpD``0} zOo9MR|0tJ{Z_I_yE+gwO0_7p5>CSd}v_`!&T^_AXk=q7J#$#0{o_fR6f|fGGT6ad) zHrcF%b{U-eaNo;ml!)i45$fzo)ut1Sv@335((tartK*_+*5|!^fW9kMQmXW8C3tYe z8n9_u(wQKU4N5rh75?-FE5$FZM5z}bX<3$e?-*je)!1rkkymgZ{-C-9NjF(S|GeXrT`my@tV~CS{o00n;+Xf>`auNgYHdAgDIl__6kodS9tz%1?04yr<|1e2lgwTOOs z!4&iiSRg$3q6JK-t7>m4i^Mj=dB{t`sF2e6emqkDE&+v_K`gyKl%ge$Bf0Nm^x2c- zXN;TXb^)YKEGDm3=8hWn-q@BW08p85Oct6YND0aul=t%Iv!yC?v}`?q{HM;(k6Q9v zEralGe;BYE5=8Qx*2i$_7S6L*a=h{6HQ9nC$hag0X0VywY;gRmNg%!e`dxj>Jklnb z2B0G!Ryv5RS%pdQ(govZt8W{jS-%hbe!;boxa?KymTB1+`-gI?VbUglfr)EpK)FN%~dh~{fJ2bp`Bb;(^@r*>b=vw0=Kvc gJO2MUbK9 literal 0 HcmV?d00001 diff --git a/cranelift/peepmatic/examples/simple.peepmatic b/cranelift/peepmatic/examples/simple.peepmatic new file mode 100644 index 0000000000..a90e0c1a69 --- /dev/null +++ b/cranelift/peepmatic/examples/simple.peepmatic @@ -0,0 +1,11 @@ +(=> (bor $x (bor $x $y)) + (bor $x $y)) + +(=> (bor $y (bor $x $y)) + (bor $x $y)) + +(=> (bor (bor $x $y) $x) + (bor $x $y)) + +(=> (bor (bor $x $y) $y) + (bor $x $y)) diff --git a/cranelift/peepmatic/src/ast.rs b/cranelift/peepmatic/src/ast.rs new file mode 100644 index 0000000000..daac871553 --- /dev/null +++ b/cranelift/peepmatic/src/ast.rs @@ -0,0 +1,508 @@ +//! Abstract syntax tree type definitions. +//! +//! This file makes fairly heavy use of macros, which are defined in the +//! `peepmatic_macro` crate that lives at `crates/macro`. Notably, the following +//! traits are all derived via `derive(Ast)`: +//! +//! * `Span` -- access the `wast::Span` where an AST node was parsed from. For +//! `struct`s, there must be a `span: wast::Span` field, because the macro +//! always generates an implementation that returns `self.span` for +//! `struct`s. For `enum`s, every variant must have a single, unnamed field +//! which implements the `Span` trait. The macro will generate code to return +//! the span of whatever variant it is. +//! +//! * `ChildNodes` -- get each of the child AST nodes that a given node +//! references. Some fields in an AST type aren't actually considered an AST +//! node (like spans) and these are ignored via the `#[peepmatic(skip_child)]` +//! attribute. Some fields contain multiple AST nodes (like vectors of +//! operands) and these are flattened with `#[peepmatic(flatten)]`. +//! +//! * `From<&'a Self> for DynAstRef<'a>` -- convert a particular AST type into +//! `DynAstRef`, which is an `enum` of all the different kinds of AST nodes. + +use peepmatic_macro::Ast; +use peepmatic_runtime::{ + operator::{Operator, UnquoteOperator}, + r#type::{BitWidth, Type}, +}; +use std::cell::Cell; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; +use wast::Id; + +/// A reference to any AST node. +#[derive(Debug, Clone, Copy)] +pub enum DynAstRef<'a> { + /// A reference to an `Optimizations`. + Optimizations(&'a Optimizations<'a>), + + /// A reference to an `Optimization`. + Optimization(&'a Optimization<'a>), + + /// A reference to an `Lhs`. + Lhs(&'a Lhs<'a>), + + /// A reference to an `Rhs`. + Rhs(&'a Rhs<'a>), + + /// A reference to a `Pattern`. + Pattern(&'a Pattern<'a>), + + /// A reference to a `Precondition`. + Precondition(&'a Precondition<'a>), + + /// A reference to a `ConstraintOperand`. + ConstraintOperand(&'a ConstraintOperand<'a>), + + /// A reference to a `ValueLiteral`. + ValueLiteral(&'a ValueLiteral<'a>), + + /// A reference to a `Constant`. + Constant(&'a Constant<'a>), + + /// A reference to a `PatternOperation`. + PatternOperation(&'a Operation<'a, Pattern<'a>>), + + /// A reference to a `Variable`. + Variable(&'a Variable<'a>), + + /// A reference to an `Integer`. + Integer(&'a Integer<'a>), + + /// A reference to a `Boolean`. + Boolean(&'a Boolean<'a>), + + /// A reference to a `ConditionCode`. + ConditionCode(&'a ConditionCode<'a>), + + /// A reference to an `Unquote`. + Unquote(&'a Unquote<'a>), + + /// A reference to an `RhsOperation`. + RhsOperation(&'a Operation<'a, Rhs<'a>>), +} + +impl<'a, 'b> ChildNodes<'a, 'b> for DynAstRef<'a> { + fn child_nodes(&'b self, sink: &mut impl Extend>) { + match self { + Self::Optimizations(x) => x.child_nodes(sink), + Self::Optimization(x) => x.child_nodes(sink), + Self::Lhs(x) => x.child_nodes(sink), + Self::Rhs(x) => x.child_nodes(sink), + Self::Pattern(x) => x.child_nodes(sink), + Self::Precondition(x) => x.child_nodes(sink), + Self::ConstraintOperand(x) => x.child_nodes(sink), + Self::ValueLiteral(x) => x.child_nodes(sink), + Self::Constant(x) => x.child_nodes(sink), + Self::PatternOperation(x) => x.child_nodes(sink), + Self::Variable(x) => x.child_nodes(sink), + Self::Integer(x) => x.child_nodes(sink), + Self::Boolean(x) => x.child_nodes(sink), + Self::ConditionCode(x) => x.child_nodes(sink), + Self::Unquote(x) => x.child_nodes(sink), + Self::RhsOperation(x) => x.child_nodes(sink), + } + } +} + +/// A trait implemented by all AST nodes. +/// +/// All AST nodes can: +/// +/// * Enumerate their children via `ChildNodes`. +/// +/// * Give you the `wast::Span` where they were defined. +/// +/// * Be converted into a `DynAstRef`. +/// +/// This trait is blanked implemented for everything that does those three +/// things, and in practice those three thrings are all implemented by the +/// `derive(Ast)` macro. +pub trait Ast<'a>: 'a + ChildNodes<'a, 'a> + Span +where + DynAstRef<'a>: From<&'a Self>, +{ +} + +impl<'a, T> Ast<'a> for T +where + T: 'a + ?Sized + ChildNodes<'a, 'a> + Span, + DynAstRef<'a>: From<&'a Self>, +{ +} + +/// Enumerate the child AST nodes of a given node. +pub trait ChildNodes<'a, 'b> { + /// Get each of this AST node's children, in order. + fn child_nodes(&'b self, sink: &mut impl Extend>); +} + +/// A trait for getting the span where an AST node was defined. +pub trait Span { + /// Get the span where this AST node was defined. + fn span(&self) -> wast::Span; +} + +/// A set of optimizations. +/// +/// This is the root AST node. +#[derive(Debug, Ast)] +pub struct Optimizations<'a> { + /// Where these `Optimizations` were defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// The optimizations. + #[peepmatic(flatten)] + pub optimizations: Vec>, +} + +/// A complete optimization: a left-hand side to match against and a right-hand +/// side replacement. +#[derive(Debug, Ast)] +pub struct Optimization<'a> { + /// Where this `Optimization` was defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// The left-hand side that matches when this optimization applies. + pub lhs: Lhs<'a>, + + /// The new sequence of instructions to replace an old sequence that matches + /// the left-hand side with. + pub rhs: Rhs<'a>, +} + +/// A left-hand side describes what is required for a particular optimization to +/// apply. +/// +/// A left-hand side has two parts: a structural pattern for describing +/// candidate instruction sequences, and zero or more preconditions that add +/// additional constraints upon instruction sequences matched by the pattern. +#[derive(Debug, Ast)] +pub struct Lhs<'a> { + /// Where this `Lhs` was defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// A pattern that describes sequences of instructions to match. + pub pattern: Pattern<'a>, + + /// Additional constraints that a match must satisfy in addition to + /// structually matching the pattern, e.g. some constant must be a power of + /// two. + #[peepmatic(flatten)] + pub preconditions: Vec>, +} + +/// A structural pattern, potentially with wildcard variables for matching whole +/// subtrees. +#[derive(Debug, Ast)] +pub enum Pattern<'a> { + /// A specific value. These are written as `1234` or `0x1234` or `true` or + /// `false`. + ValueLiteral(ValueLiteral<'a>), + + /// A constant that matches any constant value. This subsumes value + /// patterns. These are upper-case identifiers like `$C`. + Constant(Constant<'a>), + + /// An operation pattern with zero or more operand patterns. These are + /// s-expressions like `(iadd $x $y)`. + Operation(Operation<'a, Pattern<'a>>), + + /// A variable that matches any kind of subexpression. This subsumes all + /// other patterns. These are lower-case identifiers like `$x`. + Variable(Variable<'a>), +} + +/// An integer or boolean value literal. +#[derive(Debug, Ast)] +pub enum ValueLiteral<'a> { + /// An integer value. + Integer(Integer<'a>), + + /// A boolean value: `true` or `false`. + Boolean(Boolean<'a>), + + /// A condition code: `eq`, `ne`, etc... + ConditionCode(ConditionCode<'a>), +} + +/// An integer literal. +#[derive(Debug, PartialEq, Eq, Ast)] +pub struct Integer<'a> { + /// Where this `Integer` was defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// The integer value. + /// + /// Note that although Cranelift allows 128 bits wide values, the widest + /// supported constants as immediates are 64 bits. + #[peepmatic(skip_child)] + pub value: i64, + + /// The bit width of this integer. + /// + /// This is either a fixed bit width, or polymorphic over the width of the + /// optimization. + /// + /// This field is initialized from `None` to `Some` by the type checking + /// pass in `src/verify.rs`. + #[peepmatic(skip_child)] + pub bit_width: Cell>, + + #[allow(missing_docs)] + #[peepmatic(skip_child)] + pub marker: PhantomData<&'a ()>, +} + +impl Hash for Integer<'_> { + fn hash(&self, state: &mut H) + where + H: Hasher, + { + let Integer { + span, + value, + bit_width, + marker: _, + } = self; + span.hash(state); + value.hash(state); + let bit_width = bit_width.get(); + bit_width.hash(state); + } +} + +/// A boolean literal. +#[derive(Debug, PartialEq, Eq, Ast)] +pub struct Boolean<'a> { + /// Where this `Boolean` was defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// The boolean value. + #[peepmatic(skip_child)] + pub value: bool, + + /// The bit width of this boolean. + /// + /// This is either a fixed bit width, or polymorphic over the width of the + /// optimization. + /// + /// This field is initialized from `None` to `Some` by the type checking + /// pass in `src/verify.rs`. + #[peepmatic(skip_child)] + pub bit_width: Cell>, + + #[allow(missing_docs)] + #[peepmatic(skip_child)] + pub marker: PhantomData<&'a ()>, +} + +impl Hash for Boolean<'_> { + fn hash(&self, state: &mut H) + where + H: Hasher, + { + let Boolean { + span, + value, + bit_width, + marker: _, + } = self; + span.hash(state); + value.hash(state); + let bit_width = bit_width.get(); + bit_width.hash(state); + } +} + +/// A condition code. +#[derive(Debug, Ast)] +pub struct ConditionCode<'a> { + /// Where this `ConditionCode` was defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// The actual condition code. + #[peepmatic(skip_child)] + pub cc: peepmatic_runtime::cc::ConditionCode, + + #[allow(missing_docs)] + #[peepmatic(skip_child)] + pub marker: PhantomData<&'a ()>, +} + +/// A symbolic constant. +/// +/// These are identifiers containing uppercase letters: `$C`, `$MY-CONST`, +/// `$CONSTANT1`. +#[derive(Debug, Ast)] +pub struct Constant<'a> { + /// Where this `Constant` was defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// This constant's identifier. + #[peepmatic(skip_child)] + pub id: Id<'a>, +} + +/// A variable that matches any subtree. +/// +/// Duplicate uses of the same variable constrain each occurrence's match to +/// being the same as each other occurrence as well, e.g. `(iadd $x $x)` matches +/// `(iadd 5 5)` but not `(iadd 1 2)`. +#[derive(Debug, Ast)] +pub struct Variable<'a> { + /// Where this `Variable` was defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// This variable's identifier. + #[peepmatic(skip_child)] + pub id: Id<'a>, +} + +/// An operation with an operator, and operands of type `T`. +#[derive(Debug, Ast)] +#[peepmatic(no_into_dyn_node)] +pub struct Operation<'a, T> +where + T: 'a + Ast<'a>, + DynAstRef<'a>: From<&'a T>, +{ + /// The span where this operation was written. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// The operator for this operation, e.g. `imul` or `iadd`. + #[peepmatic(skip_child)] + pub operator: Operator, + + /// An optional ascribed or inferred type for the operator. + #[peepmatic(skip_child)] + pub r#type: Cell>, + + /// This operation's operands. + /// + /// When `Operation` is used in a pattern, these are the sub-patterns for + /// the operands. When `Operation is used in a right-hand side replacement, + /// these are the sub-replacements for the operands. + #[peepmatic(flatten)] + pub operands: Vec, + + #[allow(missing_docs)] + #[peepmatic(skip_child)] + pub marker: PhantomData<&'a ()>, +} + +impl<'a> From<&'a Operation<'a, Pattern<'a>>> for DynAstRef<'a> { + #[inline] + fn from(o: &'a Operation<'a, Pattern<'a>>) -> DynAstRef<'a> { + DynAstRef::PatternOperation(o) + } +} + +impl<'a> From<&'a Operation<'a, Rhs<'a>>> for DynAstRef<'a> { + #[inline] + fn from(o: &'a Operation<'a, Rhs<'a>>) -> DynAstRef<'a> { + DynAstRef::RhsOperation(o) + } +} + +/// A precondition adds additional constraints to a pattern, such as "$C must be +/// a power of two". +#[derive(Debug, Ast)] +pub struct Precondition<'a> { + /// Where this `Precondition` was defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// The constraint operator. + #[peepmatic(skip_child)] + pub constraint: Constraint, + + /// The operands of the constraint. + #[peepmatic(flatten)] + pub operands: Vec>, +} + +/// Contraint operators. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub enum Constraint { + /// Is the operand a power of two? + IsPowerOfTwo, + + /// Check the bit width of a value. + BitWidth, + + /// Does the argument fit within our target architecture's native word size? + FitsInNativeWord, +} + +/// An operand of a precondition's constraint. +#[derive(Debug, Ast)] +pub enum ConstraintOperand<'a> { + /// A value literal operand. + ValueLiteral(ValueLiteral<'a>), + + /// A constant operand. + Constant(Constant<'a>), + + /// A variable operand. + Variable(Variable<'a>), +} + +/// The right-hand side of an optimization that contains the instructions to +/// replace any matched left-hand side with. +#[derive(Debug, Ast)] +pub enum Rhs<'a> { + /// A value literal right-hand side. + ValueLiteral(ValueLiteral<'a>), + + /// A constant right-hand side (the constant must have been matched and + /// bound in the left-hand side's pattern). + Constant(Constant<'a>), + + /// A variable right-hand side (the variable must have been matched and + /// bound in the left-hand side's pattern). + Variable(Variable<'a>), + + /// An unquote expression that is evaluated while replacing the left-hand + /// side with the right-hand side. The result of the evaluation is used in + /// the replacement. + Unquote(Unquote<'a>), + + /// A compound right-hand side consisting of an operation and subsequent + /// right-hand side operands. + Operation(Operation<'a, Rhs<'a>>), +} + +/// An unquote operation. +/// +/// Rather than replaciong a left-hand side, these are evaluated and then the +/// result of the evaluation replaces the left-hand side. This allows for +/// compile-time computation while replacing a matched left-hand side with a +/// right-hand side. +/// +/// For example, given the unqouted right-hand side `$(log2 $C)`, we replace any +/// instructions that match its left-hand side with the compile-time result of +/// `log2($C)` (the left-hand side must match and bind the constant `$C`). +#[derive(Debug, Ast)] +pub struct Unquote<'a> { + /// Where this `Unquote` was defined. + #[peepmatic(skip_child)] + pub span: wast::Span, + + /// The operator for this unquote operation. + #[peepmatic(skip_child)] + pub operator: UnquoteOperator, + + /// The operands for this unquote operation. + #[peepmatic(flatten)] + pub operands: Vec>, +} diff --git a/cranelift/peepmatic/src/automatize.rs b/cranelift/peepmatic/src/automatize.rs new file mode 100644 index 0000000000..3310bef118 --- /dev/null +++ b/cranelift/peepmatic/src/automatize.rs @@ -0,0 +1,31 @@ +//! Compile a set of linear optimizations into an automaton. + +use peepmatic_automata::{Automaton, Builder}; +use peepmatic_runtime::linear; + +/// Construct an automaton from a set of linear optimizations. +pub fn automatize( + opts: &linear::Optimizations, +) -> Automaton, linear::MatchOp, Vec> { + debug_assert!(crate::linear_passes::is_sorted_lexicographically(opts)); + + let mut builder = Builder::, linear::MatchOp, Vec>::new(); + + for opt in &opts.optimizations { + let mut insertion = builder.insert(); + for inc in &opt.increments { + // Ensure that this state's associated data is this increment's + // match operation. + if let Some(op) = insertion.get_state_data() { + assert_eq!(*op, inc.operation); + } else { + insertion.set_state_data(inc.operation); + } + + insertion.next(inc.expected, inc.actions.clone()); + } + insertion.finish(); + } + + builder.finish() +} diff --git a/cranelift/peepmatic/src/dot_fmt.rs b/cranelift/peepmatic/src/dot_fmt.rs new file mode 100644 index 0000000000..a2c75de02c --- /dev/null +++ b/cranelift/peepmatic/src/dot_fmt.rs @@ -0,0 +1,142 @@ +//! Formatting a peephole optimizer's automata for GraphViz Dot. +//! +//! See also `crates/automata/src/dot.rs`. + +use peepmatic_automata::dot::DotFmt; +use peepmatic_runtime::{ + cc::ConditionCode, + integer_interner::{IntegerId, IntegerInterner}, + linear, + operator::Operator, + paths::{PathId, PathInterner}, +}; +use std::convert::TryFrom; +use std::io::{self, Write}; + +#[derive(Debug)] +pub(crate) struct PeepholeDotFmt<'a>(pub(crate) &'a PathInterner, pub(crate) &'a IntegerInterner); + +impl DotFmt, linear::MatchOp, Vec> for PeepholeDotFmt<'_> { + fn fmt_transition( + &self, + w: &mut impl Write, + from: Option<&linear::MatchOp>, + input: &Option, + _to: Option<&linear::MatchOp>, + ) -> io::Result<()> { + let from = from.expect("we should have match op for every state"); + if let Some(x) = input { + match from { + linear::MatchOp::Opcode { .. } => { + let opcode = + Operator::try_from(*x).expect("we shouldn't generate non-opcode edges"); + write!(w, "{}", opcode) + } + linear::MatchOp::ConditionCode { .. } => { + let cc = + ConditionCode::try_from(*x).expect("we shouldn't generate non-CC edges"); + write!(w, "{}", cc) + } + linear::MatchOp::IntegerValue { .. } => { + let x = self.1.lookup(IntegerId(*x)); + write!(w, "{}", x) + } + _ => write!(w, "{}", x), + } + } else { + write!(w, "(else)") + } + } + + fn fmt_state(&self, w: &mut impl Write, op: &linear::MatchOp) -> io::Result<()> { + use linear::MatchOp::*; + + write!(w, r#""#)?; + + let p = p(self.0); + match op { + Opcode { path } => write!(w, "opcode @ {}", p(path))?, + IsConst { path } => write!(w, "is-const? @ {}", p(path))?, + IsPowerOfTwo { path } => write!(w, "is-power-of-two? @ {}", p(path))?, + BitWidth { path } => write!(w, "bit-width @ {}", p(path))?, + FitsInNativeWord { path } => write!(w, "fits-in-native-word @ {}", p(path))?, + Eq { path_a, path_b } => write!(w, "{} == {}", p(path_a), p(path_b))?, + IntegerValue { path } => write!(w, "integer-value @ {}", p(path))?, + BooleanValue { path } => write!(w, "boolean-value @ {}", p(path))?, + ConditionCode { path } => write!(w, "condition-code @ {}", p(path))?, + Nop => write!(w, "nop")?, + } + + writeln!(w, "") + } + + fn fmt_output(&self, w: &mut impl Write, actions: &Vec) -> io::Result<()> { + use linear::Action::*; + + if actions.is_empty() { + return writeln!(w, "(no output)"); + } + + write!(w, r#""#)?; + + let p = p(self.0); + + for a in actions { + match a { + GetLhs { path } => write!(w, "get-lhs @ {}
", p(path))?, + UnaryUnquote { operator, operand } => { + write!(w, "eval {} $rhs{}
", operator, operand.0)? + } + BinaryUnquote { operator, operands } => write!( + w, + "eval {} $rhs{}, $rhs{}
", + operator, operands[0].0, operands[1].0, + )?, + MakeIntegerConst { + value, + bit_width: _, + } => write!(w, "make {}
", self.1.lookup(*value))?, + MakeBooleanConst { + value, + bit_width: _, + } => write!(w, "make {}
", value)?, + MakeConditionCode { cc } => write!(w, "{}
", cc)?, + MakeUnaryInst { + operand, + operator, + r#type: _, + } => write!(w, "make {} $rhs{}
", operator, operand.0,)?, + MakeBinaryInst { + operator, + operands, + r#type: _, + } => write!( + w, + "make {} $rhs{}, $rhs{}
", + operator, operands[0].0, operands[1].0, + )?, + MakeTernaryInst { + operator, + operands, + r#type: _, + } => write!( + w, + "make {} $rhs{}, $rhs{}, $rhs{}
", + operator, operands[0].0, operands[1].0, operands[2].0, + )?, + } + } + + writeln!(w, "") + } +} + +fn p<'a>(paths: &'a PathInterner) -> impl Fn(&PathId) -> String + 'a { + move |path: &PathId| { + let mut s = vec![]; + for b in paths.lookup(*path).0 { + s.push(b.to_string()); + } + s.join(".") + } +} diff --git a/cranelift/peepmatic/src/lib.rs b/cranelift/peepmatic/src/lib.rs new file mode 100755 index 0000000000..0cf4147db8 --- /dev/null +++ b/cranelift/peepmatic/src/lib.rs @@ -0,0 +1,165 @@ +/*! + +`peepmatic` is a DSL and compiler for generating peephole optimizers. + +The user writes a set of optimizations in the DSL, and then `peepmatic` compiles +the set of optimizations into an efficient peephole optimizer. + + */ + +#![deny(missing_docs)] +#![deny(missing_debug_implementations)] + +mod ast; +mod automatize; +mod dot_fmt; +mod linear_passes; +mod linearize; +mod parser; +mod traversals; +mod verify; +pub use self::{ + ast::*, automatize::*, linear_passes::*, linearize::*, parser::*, traversals::*, verify::*, +}; + +use peepmatic_runtime::PeepholeOptimizations; +use std::fs; +use std::path::Path; + +/// Compile the given DSL file into a compact peephole optimizations automaton! +/// +/// ## Example +/// +/// ```no_run +/// # fn main() -> anyhow::Result<()> { +/// use std::path::Path; +/// +/// let peep_opts = peepmatic::compile_file(Path::new( +/// "path/to/optimizations.peepmatic" +/// ))?; +/// +/// // Use the peephole optimizations or serialize them into bytes here... +/// # Ok(()) +/// # } +/// ``` +/// +/// ## Visualizing the Peephole Optimizer's Automaton +/// +/// To visualize (or debug) the peephole optimizer's automaton, set the +/// `PEEPMATIC_DOT` environment variable to a file path. A [GraphViz +/// Dot]((https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf)) file showing the +/// peephole optimizer's automaton will be written to that file path. +pub fn compile_file(filename: &Path) -> anyhow::Result { + let source = fs::read_to_string(filename)?; + compile_str(&source, filename) +} + +/// Compile the given DSL source text down into a compact peephole optimizations +/// automaton. +/// +/// This is like [compile_file][crate::compile_file] but you bring your own file +/// I/O. +/// +/// The `filename` parameter is used to provide better error messages. +/// +/// ## Example +/// +/// ```no_run +/// # fn main() -> anyhow::Result<()> { +/// use std::path::Path; +/// +/// let peep_opts = peepmatic::compile_str( +/// " +/// (=> (iadd $x 0) $x) +/// (=> (imul $x 0) 0) +/// (=> (imul $x 1) $x) +/// ", +/// Path::new("my-optimizations"), +/// )?; +/// +/// // Use the peephole optimizations or serialize them into bytes here... +/// # Ok(()) +/// # } +/// ``` +/// +/// ## Visualizing the Peephole Optimizer's Automaton +/// +/// To visualize (or debug) the peephole optimizer's automaton, set the +/// `PEEPMATIC_DOT` environment variable to a file path. A [GraphViz +/// Dot]((https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf)) file showing the +/// peephole optimizer's automaton will be written to that file path. +pub fn compile_str(source: &str, filename: &Path) -> anyhow::Result { + let buf = wast::parser::ParseBuffer::new(source).map_err(|mut e| { + e.set_path(filename); + e.set_text(source); + e + })?; + + let opts = wast::parser::parse::(&buf).map_err(|mut e| { + e.set_path(filename); + e.set_text(source); + e + })?; + + verify(&opts).map_err(|mut e| { + e.set_path(filename); + e.set_text(source); + e + })?; + + let mut opts = crate::linearize(&opts); + sort_least_to_most_general(&mut opts); + remove_unnecessary_nops(&mut opts); + match_in_same_order(&mut opts); + sort_lexicographically(&mut opts); + + let automata = automatize(&opts); + let paths = opts.paths; + let integers = opts.integers; + + if let Ok(path) = std::env::var("PEEPMATIC_DOT") { + let f = dot_fmt::PeepholeDotFmt(&paths, &integers); + if let Err(e) = automata.write_dot_file(&f, &path) { + panic!( + "failed to write GraphViz Dot file to PEEPMATIC_DOT={}; error: {}", + path, e + ); + } + } + + Ok(PeepholeOptimizations { + paths, + integers, + automata, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_compiles(path: &str) { + match compile_file(Path::new(path)) { + Ok(_) => return, + Err(e) => { + eprintln!("error: {}", e); + panic!("error: {}", e); + } + } + } + + #[test] + fn compile_redundant_bor() { + assert_compiles("examples/redundant-bor.peepmatic"); + } + + #[test] + fn mul_by_pow2() { + assert_compiles("examples/mul-by-pow2.peepmatic"); + } + + #[test] + fn compile_preopt() { + assert_compiles("examples/preopt.peepmatic"); + } +} diff --git a/cranelift/peepmatic/src/linear_passes.rs b/cranelift/peepmatic/src/linear_passes.rs new file mode 100644 index 0000000000..cac1ca3d19 --- /dev/null +++ b/cranelift/peepmatic/src/linear_passes.rs @@ -0,0 +1,444 @@ +//! Passes over the linear IR. + +use peepmatic_runtime::{ + linear, + paths::{PathId, PathInterner}, +}; +use std::cmp::Ordering; + +/// Sort a set of optimizations from least to most general. +/// +/// This helps us ensure that we always match the least-general (aka +/// most-specific) optimization that we can for a particular instruction +/// sequence. +/// +/// For example, if we have both of these optimizations: +/// +/// ```lisp +/// (=> (imul $C $x) +/// (imul_imm $C $x)) +/// +/// (=> (when (imul $C $x)) +/// (is-power-of-two $C)) +/// (ishl $x $C)) +/// ``` +/// +/// and we are matching `(imul 4 (..))`, then we want to apply the second +/// optimization, because it is more specific than the first. +pub fn sort_least_to_most_general(opts: &mut linear::Optimizations) { + let linear::Optimizations { + ref mut optimizations, + ref paths, + .. + } = opts; + + // NB: we *cannot* use an unstable sort here, because we want deterministic + // compilation of optimizations to automata. + optimizations.sort_by(|a, b| compare_optimization_generality(paths, a, b)); + debug_assert!(is_sorted_by_generality(opts)); +} + +/// Sort the linear optimizations lexicographically. +/// +/// This sort order is required for automata construction. +pub fn sort_lexicographically(opts: &mut linear::Optimizations) { + let linear::Optimizations { + ref mut optimizations, + ref paths, + .. + } = opts; + + // NB: we *cannot* use an unstable sort here, same as above. + optimizations + .sort_by(|a, b| compare_optimizations(paths, a, b, |a_len, b_len| a_len.cmp(&b_len))); +} + +fn compare_optimizations( + paths: &PathInterner, + a: &linear::Optimization, + b: &linear::Optimization, + compare_lengths: impl Fn(usize, usize) -> Ordering, +) -> Ordering { + for (a, b) in a.increments.iter().zip(b.increments.iter()) { + let c = compare_match_op_generality(paths, a.operation, b.operation); + if c != Ordering::Equal { + return c; + } + + let c = a.expected.cmp(&b.expected).reverse(); + if c != Ordering::Equal { + return c; + } + } + + compare_lengths(a.increments.len(), b.increments.len()) +} + +fn compare_optimization_generality( + paths: &PathInterner, + a: &linear::Optimization, + b: &linear::Optimization, +) -> Ordering { + compare_optimizations(paths, a, b, |a_len, b_len| { + // If they shared equivalent prefixes, then compare lengths and invert the + // result because longer patterns are less general than shorter patterns. + a_len.cmp(&b_len).reverse() + }) +} + +fn compare_match_op_generality( + paths: &PathInterner, + a: linear::MatchOp, + b: linear::MatchOp, +) -> Ordering { + use linear::MatchOp::*; + match (a, b) { + (Opcode { path: a }, Opcode { path: b }) => compare_paths(paths, a, b), + (Opcode { .. }, _) => Ordering::Less, + (_, Opcode { .. }) => Ordering::Greater, + + (IntegerValue { path: a }, IntegerValue { path: b }) => compare_paths(paths, a, b), + (IntegerValue { .. }, _) => Ordering::Less, + (_, IntegerValue { .. }) => Ordering::Greater, + + (BooleanValue { path: a }, BooleanValue { path: b }) => compare_paths(paths, a, b), + (BooleanValue { .. }, _) => Ordering::Less, + (_, BooleanValue { .. }) => Ordering::Greater, + + (ConditionCode { path: a }, ConditionCode { path: b }) => compare_paths(paths, a, b), + (ConditionCode { .. }, _) => Ordering::Less, + (_, ConditionCode { .. }) => Ordering::Greater, + + (IsConst { path: a }, IsConst { path: b }) => compare_paths(paths, a, b), + (IsConst { .. }, _) => Ordering::Less, + (_, IsConst { .. }) => Ordering::Greater, + + ( + Eq { + path_a: pa1, + path_b: pb1, + }, + Eq { + path_a: pa2, + path_b: pb2, + }, + ) => compare_paths(paths, pa1, pa2).then(compare_paths(paths, pb1, pb2)), + (Eq { .. }, _) => Ordering::Less, + (_, Eq { .. }) => Ordering::Greater, + + (IsPowerOfTwo { path: a }, IsPowerOfTwo { path: b }) => compare_paths(paths, a, b), + (IsPowerOfTwo { .. }, _) => Ordering::Less, + (_, IsPowerOfTwo { .. }) => Ordering::Greater, + + (BitWidth { path: a }, BitWidth { path: b }) => compare_paths(paths, a, b), + (BitWidth { .. }, _) => Ordering::Less, + (_, BitWidth { .. }) => Ordering::Greater, + + (FitsInNativeWord { path: a }, FitsInNativeWord { path: b }) => compare_paths(paths, a, b), + (FitsInNativeWord { .. }, _) => Ordering::Less, + (_, FitsInNativeWord { .. }) => Ordering::Greater, + + (Nop, Nop) => Ordering::Equal, + } +} + +fn compare_paths(paths: &PathInterner, a: PathId, b: PathId) -> Ordering { + if a == b { + Ordering::Equal + } else { + let a = paths.lookup(a); + let b = paths.lookup(b); + a.0.cmp(&b.0) + } +} + +/// Are the given optimizations sorted from least to most general? +pub(crate) fn is_sorted_by_generality(opts: &linear::Optimizations) -> bool { + opts.optimizations + .windows(2) + .all(|w| compare_optimization_generality(&opts.paths, &w[0], &w[1]) <= Ordering::Equal) +} + +/// Are the given optimizations sorted lexicographically? +pub(crate) fn is_sorted_lexicographically(opts: &linear::Optimizations) -> bool { + opts.optimizations.windows(2).all(|w| { + compare_optimizations(&opts.paths, &w[0], &w[1], |a_len, b_len| a_len.cmp(&b_len)) + <= Ordering::Equal + }) +} + +/// Ensure that we emit match operations in a consistent order. +/// +/// There are many linear optimizations, each of which have their own sequence +/// of match operations that need to be tested. But when interpreting the +/// automata against some instructions, we only perform a single sequence of +/// match operations, and at any given moment, we only want one match operation +/// to interpret next. This means that two optimizations that are next to each +/// other in the sorting must have their shared prefixes diverge on an +/// **expected result edge**, not on which match operation to preform next. And +/// if they have zero shared prefix, then we need to create one, that +/// immediately divereges on the expected result. +/// +/// For example, consider these two patterns that don't have any shared prefix: +/// +/// ```lisp +/// (=> (iadd $x $y) ...) +/// (=> $C ...) +/// ``` +/// +/// These produce the following linear match operations and expected results: +/// +/// ```text +/// opcode @ 0 --iadd--> +/// is-const? @ 0 --true--> +/// ``` +/// +/// In order to ensure that we only have one match operation to interpret at any +/// given time when evaluating the automata, this pass transforms the second +/// optimization so that it shares a prefix match operation, but diverges on the +/// expected result: +/// +/// ```text +/// opcode @ 0 --iadd--> +/// opcode @ 0 --(else)--> is-const? @ 0 --true--> +/// ``` +pub fn match_in_same_order(opts: &mut linear::Optimizations) { + assert!(!opts.optimizations.is_empty()); + + let mut prefix = vec![]; + + for opt in &mut opts.optimizations { + assert!(!opt.increments.is_empty()); + + let mut old_increments = opt.increments.iter().peekable(); + let mut new_increments = vec![]; + + for (last_op, last_expected) in &prefix { + match old_increments.peek() { + None => { + break; + } + Some(inc) if *last_op == inc.operation => { + let inc = old_increments.next().unwrap(); + new_increments.push(inc.clone()); + if inc.expected != *last_expected { + break; + } + } + Some(_) => { + new_increments.push(linear::Increment { + operation: *last_op, + expected: None, + actions: vec![], + }); + if last_expected.is_some() { + break; + } + } + } + } + + new_increments.extend(old_increments.cloned()); + assert!(new_increments.len() >= opt.increments.len()); + opt.increments = new_increments; + + prefix.clear(); + prefix.extend( + opt.increments + .iter() + .map(|inc| (inc.operation, inc.expected)), + ); + } + + // Should still be sorted after this pass. + debug_assert!(is_sorted_by_generality(&opts)); +} + +/// 99.99% of nops are unnecessary; remove them. +/// +/// They're only needed for when a LHS pattern is just a variable, and that's +/// it. However, it is easier to have basically unused nop matching operations +/// for the DSL's edge-cases than it is to try and statically eliminate their +/// existence completely. So we just emit nop match operations for all variable +/// patterns, and then in this post-processing pass, we fuse them and their +/// actions with their preceding increment. +pub fn remove_unnecessary_nops(opts: &mut linear::Optimizations) { + for opt in &mut opts.optimizations { + if opt.increments.len() < 2 { + debug_assert!(!opt.increments.is_empty()); + continue; + } + + for i in (1..opt.increments.len()).rev() { + if let linear::MatchOp::Nop = opt.increments[i].operation { + let nop = opt.increments.remove(i); + opt.increments[i - 1].actions.extend(nop.actions); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::*; + use linear::MatchOp::*; + use peepmatic_runtime::{operator::Operator, paths::*}; + + macro_rules! sorts_to { + ($test_name:ident, $source:expr, $make_expected:expr) => { + #[test] + fn $test_name() { + let buf = wast::parser::ParseBuffer::new($source).expect("should lex OK"); + + let opts = match wast::parser::parse::(&buf) { + Ok(opts) => opts, + Err(mut e) => { + e.set_path(std::path::Path::new(stringify!($test_name))); + e.set_text($source); + eprintln!("{}", e); + panic!("should parse OK") + } + }; + + if let Err(mut e) = crate::verify(&opts) { + e.set_path(std::path::Path::new(stringify!($test_name))); + e.set_text($source); + eprintln!("{}", e); + panic!("should verify OK") + } + + let mut opts = crate::linearize(&opts); + sort_least_to_most_general(&mut opts); + + let linear::Optimizations { + mut paths, + mut integers, + optimizations, + } = opts; + + let actual: Vec> = optimizations + .iter() + .map(|o| { + o.increments + .iter() + .map(|i| (i.operation, i.expected)) + .collect() + }) + .collect(); + + let mut p = |p: &[u8]| paths.intern(Path::new(&p)); + let mut i = |i: u64| Some(integers.intern(i).into()); + let expected = $make_expected(&mut p, &mut i); + + assert_eq!(expected, actual); + } + }; + } + + sorts_to!( + test_sort_least_to_most_general, + " +(=> $x 0) +(=> (iadd $x $y) 0) +(=> (iadd $x $x) 0) +(=> (iadd $x $C) 0) +(=> (when (iadd $x $C) (is-power-of-two $C)) 0) +(=> (when (iadd $x $C) (bit-width $x 32)) 0) +(=> (iadd $x 42) 0) +(=> (iadd $x (iadd $y $z)) 0) +", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> Option| vec![ + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + (Nop, None), + (Opcode { path: p(&[0, 1]) }, Some(Operator::Iadd as _)), + (Nop, None), + (Nop, None), + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + (Nop, None), + (IntegerValue { path: p(&[0, 1]) }, i(42)) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + (Nop, None), + (IsConst { path: p(&[0, 1]) }, Some(1)), + (IsPowerOfTwo { path: p(&[0, 1]) }, Some(1)) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + (Nop, None), + (IsConst { path: p(&[0, 1]) }, Some(1)), + (BitWidth { path: p(&[0, 0]) }, Some(32)) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + (Nop, None), + (IsConst { path: p(&[0, 1]) }, Some(1)) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + (Nop, None), + ( + Eq { + path_a: p(&[0, 1]), + path_b: p(&[0, 0]), + }, + Some(1) + ) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + (Nop, None), + (Nop, None), + ], + vec![(Nop, None)] + ] + ); + + sorts_to!( + expected_edges_are_sorted, + " +(=> (iadd 0 $x) $x) +(=> (iadd $x 0) $x) +(=> (imul 1 $x) $x) +(=> (imul $x 1) $x) +(=> (imul 2 $x) (ishl $x 1)) +(=> (imul $x 2) (ishl $x 1)) +", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> Option| vec![ + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Imul as _)), + (IntegerValue { path: p(&[0, 0]) }, i(2)), + (Nop, None) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Imul as _)), + (IntegerValue { path: p(&[0, 0]) }, i(1)), + (Nop, None) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Imul as _)), + (Nop, None), + (IntegerValue { path: p(&[0, 1]) }, i(2)) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Imul as _)), + (Nop, None), + (IntegerValue { path: p(&[0, 1]) }, i(1)) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + (IntegerValue { path: p(&[0, 0]) }, i(0)), + (Nop, None) + ], + vec![ + (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + (Nop, None), + (IntegerValue { path: p(&[0, 1]) }, i(0)) + ] + ] + ); +} diff --git a/cranelift/peepmatic/src/linearize.rs b/cranelift/peepmatic/src/linearize.rs new file mode 100644 index 0000000000..75205c93b9 --- /dev/null +++ b/cranelift/peepmatic/src/linearize.rs @@ -0,0 +1,831 @@ +//! Convert an AST into its linear equivalent. +//! +//! Convert each optimization's left-hand side into a linear series of match +//! operations. This makes it easy to create an automaton, because automatas +//! typically deal with a linear sequence of inputs. The optimization's +//! right-hand side is built incrementally inside actions that are taken on +//! transitions between match operations. +//! +//! See `crates/runtime/src/linear.rs` for the linear datatype definitions. +//! +//! ## Example +//! +//! As an example, if we linearize this optimization: +//! +//! ```lisp +//! (=> (when (imul $x $C) +//! (is-power-of-two $C)) +//! (ishl $x $(log2 C))) +//! ``` +//! +//! Then we should get the following linear chain of "increments": +//! +//! ```ignore +//! [ +//! // ( Match Operation, Expected Value, Actions ) +//! ( Opcode@0, imul, [$x = GetLhs@0.0, $C = GetLhs@0.1, ...] ), +//! ( IsConst(C), true, [] ), +//! ( IsPowerOfTwo(C), true, [] ), +//! ] +//! ``` +//! +//! Each increment will essentially become a state and a transition out of that +//! state in the final automata, along with the actions to perform when taking +//! that transition. The actions record the scope of matches from the left-hand +//! side and also incrementally build the right-hand side's instructions. (Note +//! that we've elided the actions that build up the optimization's right-hand +//! side in this example.) +//! +//! ## General Principles +//! +//! Here are the general principles that linearization should adhere to: +//! +//! * Actions should be pushed as early in the optimization's increment chain as +//! they can be. This means the tail has fewer side effects, and is therefore +//! more likely to be share-able with other optimizations in the automata that +//! we build. +//! +//! * RHS actions cannot reference matches from the LHS until they've been +//! defined. And finally, an RHS operation's operands must be defined before +//! the RHS operation itself. In general, definitions must come before uses! +//! +//! * Shorter increment chains are better! This means fewer tests when matching +//! left-hand sides, and a more-compact, more-cache-friendly automata, and +//! ultimately, a faster automata. +//! +//! * An increment's match operation should be a switch rather than a predicate +//! that returns a boolean. For example, we switch on an instruction's opcode, +//! rather than ask whether this operation is an `imul`. This allows for more +//! prefix sharing in the automata, which (again) makes it more compact and +//! more cache friendly. +//! +//! ## Implementation Overview +//! +//! We emit match operations for a left-hand side's pattern structure, followed +//! by match operations for its preconditions on that structure. This ensures +//! that anything bound in the pattern is defined before it is used in +//! precondition. +//! +//! Within matching the pattern structure, we emit matching operations in a +//! pre-order traversal of the pattern. This ensures that we've already matched +//! an operation before we consider its operands, and therefore we already know +//! the operands exist. See `PatternPreOrder` for details. +//! +//! As we define the match operations for a pattern, we remember the path where +//! each LHS id first occurred. These will later be reused when building the RHS +//! actions. See `LhsIdToPath` for details. +//! +//! After we've generated the match operations and expected result of those +//! match operations, then we generate the right-hand side actions. The +//! right-hand side is built up a post-order traversal, so that operands are +//! defined before they are used. See `RhsPostOrder` and `RhsBuilder` for +//! details. +//! +//! Finally, see `linearize_optimization` for the the main AST optimization into +//! linear optimization translation function. + +use crate::ast::*; +use crate::traversals::Dfs; +use peepmatic_runtime::{ + integer_interner::IntegerInterner, + linear, + paths::{Path, PathId, PathInterner}, +}; +use std::collections::BTreeMap; +use wast::Id; + +/// Translate the given AST optimizations into linear optimizations. +pub fn linearize(opts: &Optimizations) -> linear::Optimizations { + let mut optimizations = vec![]; + let mut paths = PathInterner::new(); + let mut integers = IntegerInterner::new(); + for opt in &opts.optimizations { + let lin_opt = linearize_optimization(&mut paths, &mut integers, opt); + optimizations.push(lin_opt); + } + linear::Optimizations { + optimizations, + paths, + integers, + } +} + +/// Translate an AST optimization into a linear optimization! +fn linearize_optimization( + paths: &mut PathInterner, + integers: &mut IntegerInterner, + opt: &Optimization, +) -> linear::Optimization { + let mut increments: Vec = vec![]; + + let mut lhs_id_to_path = LhsIdToPath::new(); + + // We do a pre-order traversal of the LHS because we don't know whether a + // child actually exists to match on until we've matched its parent, and we + // don't want to emit matching operations on things that might not exist! + let mut patterns = PatternPreOrder::new(&opt.lhs.pattern); + while let Some((path, pattern)) = patterns.next(paths) { + // Create the matching parts of an `Increment` for this part of the + // pattern, without any actions yet. + let (operation, expected) = pattern.to_linear_match_op(integers, &lhs_id_to_path, path); + increments.push(linear::Increment { + operation, + expected, + actions: vec![], + }); + + lhs_id_to_path.remember_path_to_pattern_ids(pattern, path); + + // Some operations require type ascriptions for us to infer the correct + // bit width of their results: `ireduce`, `sextend`, `uextend`, etc. + // When there is such a type ascription in the pattern, insert another + // increment that checks the instruction-being-matched's bit width. + if let Pattern::Operation(Operation { r#type, .. }) = pattern { + if let Some(w) = r#type.get().and_then(|ty| ty.bit_width.fixed_width()) { + increments.push(linear::Increment { + operation: linear::MatchOp::BitWidth { path }, + expected: Some(w as u32), + actions: vec![], + }); + } + } + } + + // Now that we've added all the increments for the LHS pattern, add the + // increments for its preconditions. + for pre in &opt.lhs.preconditions { + increments.push(pre.to_linear_increment(&lhs_id_to_path)); + } + + assert!(!increments.is_empty()); + + // Finally, generate the RHS-building actions and attach them to the first increment. + let mut rhs_builder = RhsBuilder::new(&opt.rhs); + rhs_builder.add_rhs_build_actions(integers, &lhs_id_to_path, &mut increments[0].actions); + + linear::Optimization { increments } +} + +/// A post-order, depth-first traversal of right-hand sides. +/// +/// Does not maintain any extra state about the traversal, such as where in the +/// tree each yielded node comes from. +struct RhsPostOrder<'a> { + dfs: Dfs<'a>, +} + +impl<'a> RhsPostOrder<'a> { + fn new(rhs: &'a Rhs<'a>) -> Self { + Self { dfs: Dfs::new(rhs) } + } +} + +impl<'a> Iterator for RhsPostOrder<'a> { + type Item = &'a Rhs<'a>; + + fn next(&mut self) -> Option<&'a Rhs<'a>> { + use crate::traversals::TraversalEvent as TE; + loop { + match self.dfs.next()? { + (TE::Exit, DynAstRef::Rhs(rhs)) => return Some(rhs), + _ => continue, + } + } + } +} + +/// A pre-order, depth-first traversal of left-hand side patterns. +/// +/// Keeps track of the path to each pattern, and yields it along side the +/// pattern AST node. +struct PatternPreOrder<'a> { + last_child: Option, + path: Vec, + dfs: Dfs<'a>, +} + +impl<'a> PatternPreOrder<'a> { + fn new(pattern: &'a Pattern<'a>) -> Self { + Self { + last_child: None, + path: vec![], + dfs: Dfs::new(pattern), + } + } + + fn next(&mut self, paths: &mut PathInterner) -> Option<(PathId, &'a Pattern<'a>)> { + use crate::traversals::TraversalEvent as TE; + loop { + match self.dfs.next()? { + (TE::Enter, DynAstRef::Pattern(pattern)) => { + let last_child = self.last_child.take(); + self.path.push(match last_child { + None => 0, + Some(c) => { + assert!( + c < std::u8::MAX, + "operators must have less than or equal u8::MAX arity" + ); + c + 1 + } + }); + let path = paths.intern(Path(&self.path)); + return Some((path, pattern)); + } + (TE::Exit, DynAstRef::Pattern(_)) => { + self.last_child = Some( + self.path + .pop() + .expect("should always have a non-empty path during traversal"), + ); + } + _ => {} + } + } + } +} + +/// A map from left-hand side identifiers to the path in the left-hand side +/// where they first occurred. +struct LhsIdToPath<'a> { + id_to_path: BTreeMap<&'a str, PathId>, +} + +impl<'a> LhsIdToPath<'a> { + /// Construct a new, empty `LhsIdToPath`. + fn new() -> Self { + Self { + id_to_path: Default::default(), + } + } + + /// Have we already seen the given identifier? + fn get_first_occurrence(&self, id: &Id) -> Option { + self.id_to_path.get(id.name()).copied() + } + + /// Get the path within the left-hand side pattern where we first saw the + /// given AST id. + /// + /// ## Panics + /// + /// Panics if the given AST id has not already been canonicalized. + fn unwrap_first_occurrence(&self, id: &Id) -> PathId { + self.id_to_path[id.name()] + } + + /// Remember the path to any LHS ids used in the given pattern. + fn remember_path_to_pattern_ids(&mut self, pattern: &'a Pattern<'a>, path: PathId) { + match pattern { + // If this is the first time we've seen an identifier defined on the + // left-hand side, remember it. + Pattern::Variable(Variable { id, .. }) | Pattern::Constant(Constant { id, .. }) => { + self.id_to_path.entry(id.name()).or_insert(path); + } + _ => {} + } + } +} + +/// An `RhsBuilder` emits the actions for building the right-hand side +/// instructions. +struct RhsBuilder<'a> { + // We do a post order traversal of the RHS because an RHS instruction cannot + // be created until after all of its operands are created. + rhs_post_order: RhsPostOrder<'a>, + + // A map from a right-hand side's span to its `linear::RhsId`. This is used + // by RHS-construction actions to reference operands. In practice the + // `RhsId` is roughly equivalent to its index in the post-order traversal of + // the RHS. + rhs_span_to_id: BTreeMap, +} + +impl<'a> RhsBuilder<'a> { + /// Create a new builder for the given right-hand side. + fn new(rhs: &'a Rhs<'a>) -> Self { + let rhs_post_order = RhsPostOrder::new(rhs); + let rhs_span_to_id = Default::default(); + Self { + rhs_post_order, + rhs_span_to_id, + } + } + + /// Get the `linear::RhsId` for the given right-hand side. + /// + /// ## Panics + /// + /// Panics if we haven't already emitted the action for building this RHS's + /// instruction. + fn get_rhs_id(&self, rhs: &Rhs) -> linear::RhsId { + self.rhs_span_to_id[&rhs.span()] + } + + /// Create actions for building up this right-hand side of an optimization. + /// + /// Because we are walking the right-hand side with a post-order traversal, + /// we know that we already created an instruction's operands that are + /// defined in the right-hand side, before we get to the parent instruction. + fn add_rhs_build_actions( + &mut self, + integers: &mut IntegerInterner, + lhs_id_to_path: &LhsIdToPath, + actions: &mut Vec, + ) { + while let Some(rhs) = self.rhs_post_order.next() { + actions.push(self.rhs_to_linear_action(integers, lhs_id_to_path, rhs)); + let id = linear::RhsId(self.rhs_span_to_id.len() as u32); + self.rhs_span_to_id.insert(rhs.span(), id); + } + } + + fn rhs_to_linear_action( + &self, + integers: &mut IntegerInterner, + lhs_id_to_path: &LhsIdToPath, + rhs: &Rhs, + ) -> linear::Action { + match rhs { + Rhs::ValueLiteral(ValueLiteral::Integer(i)) => linear::Action::MakeIntegerConst { + value: integers.intern(i.value as u64), + bit_width: i + .bit_width + .get() + .expect("should be initialized after type checking"), + }, + Rhs::ValueLiteral(ValueLiteral::Boolean(b)) => linear::Action::MakeBooleanConst { + value: b.value, + bit_width: b + .bit_width + .get() + .expect("should be initialized after type checking"), + }, + Rhs::ValueLiteral(ValueLiteral::ConditionCode(ConditionCode { cc, .. })) => { + linear::Action::MakeConditionCode { cc: *cc } + } + Rhs::Variable(Variable { id, .. }) | Rhs::Constant(Constant { id, .. }) => { + let path = lhs_id_to_path.unwrap_first_occurrence(id); + linear::Action::GetLhs { path } + } + Rhs::Unquote(unq) => match unq.operands.len() { + 1 => linear::Action::UnaryUnquote { + operator: unq.operator, + operand: self.get_rhs_id(&unq.operands[0]), + }, + 2 => linear::Action::BinaryUnquote { + operator: unq.operator, + operands: [ + self.get_rhs_id(&unq.operands[0]), + self.get_rhs_id(&unq.operands[1]), + ], + }, + n => unreachable!("no unquote operators of arity {}", n), + }, + Rhs::Operation(op) => match op.operands.len() { + 1 => linear::Action::MakeUnaryInst { + operator: op.operator, + r#type: op + .r#type + .get() + .expect("should be initialized after type checking"), + operand: self.get_rhs_id(&op.operands[0]), + }, + 2 => linear::Action::MakeBinaryInst { + operator: op.operator, + r#type: op + .r#type + .get() + .expect("should be initialized after type checking"), + operands: [ + self.get_rhs_id(&op.operands[0]), + self.get_rhs_id(&op.operands[1]), + ], + }, + 3 => linear::Action::MakeTernaryInst { + operator: op.operator, + r#type: op + .r#type + .get() + .expect("should be initialized after type checking"), + operands: [ + self.get_rhs_id(&op.operands[0]), + self.get_rhs_id(&op.operands[1]), + self.get_rhs_id(&op.operands[2]), + ], + }, + n => unreachable!("no instructions of arity {}", n), + }, + } + } +} + +impl Precondition<'_> { + /// Convert this precondition into a `linear::Increment`. + fn to_linear_increment(&self, lhs_id_to_path: &LhsIdToPath) -> linear::Increment { + match self.constraint { + Constraint::IsPowerOfTwo => { + let id = match &self.operands[0] { + ConstraintOperand::Constant(Constant { id, .. }) => id, + _ => unreachable!("checked in verification"), + }; + let path = lhs_id_to_path.unwrap_first_occurrence(&id); + linear::Increment { + operation: linear::MatchOp::IsPowerOfTwo { path }, + expected: Some(1), + actions: vec![], + } + } + Constraint::BitWidth => { + let id = match &self.operands[0] { + ConstraintOperand::Constant(Constant { id, .. }) + | ConstraintOperand::Variable(Variable { id, .. }) => id, + _ => unreachable!("checked in verification"), + }; + let path = lhs_id_to_path.unwrap_first_occurrence(&id); + + let width = match &self.operands[1] { + ConstraintOperand::ValueLiteral(ValueLiteral::Integer(Integer { + value, + .. + })) => *value, + _ => unreachable!("checked in verification"), + }; + debug_assert!(width <= 128); + debug_assert!((width as u8).is_power_of_two()); + + linear::Increment { + operation: linear::MatchOp::BitWidth { path }, + expected: Some(width as u32), + actions: vec![], + } + } + Constraint::FitsInNativeWord => { + let id = match &self.operands[0] { + ConstraintOperand::Constant(Constant { id, .. }) + | ConstraintOperand::Variable(Variable { id, .. }) => id, + _ => unreachable!("checked in verification"), + }; + let path = lhs_id_to_path.unwrap_first_occurrence(&id); + linear::Increment { + operation: linear::MatchOp::FitsInNativeWord { path }, + expected: Some(1), + actions: vec![], + } + } + } + } +} + +impl Pattern<'_> { + /// Convert this pattern into its linear match operation and the expected + /// result of that operation. + /// + /// NB: these mappings to expected values need to stay sync'd with the + /// runtime! + fn to_linear_match_op( + &self, + integers: &mut IntegerInterner, + lhs_id_to_path: &LhsIdToPath, + path: PathId, + ) -> (linear::MatchOp, Option) { + match self { + Pattern::ValueLiteral(ValueLiteral::Integer(Integer { value, .. })) => ( + linear::MatchOp::IntegerValue { path }, + Some(integers.intern(*value as u64).into()), + ), + Pattern::ValueLiteral(ValueLiteral::Boolean(Boolean { value, .. })) => { + (linear::MatchOp::BooleanValue { path }, Some(*value as u32)) + } + Pattern::ValueLiteral(ValueLiteral::ConditionCode(ConditionCode { cc, .. })) => { + (linear::MatchOp::ConditionCode { path }, Some(*cc as u32)) + } + Pattern::Constant(Constant { id, .. }) => { + if let Some(path_b) = lhs_id_to_path.get_first_occurrence(id) { + debug_assert!(path != path_b); + ( + linear::MatchOp::Eq { + path_a: path, + path_b, + }, + Some(1), + ) + } else { + (linear::MatchOp::IsConst { path }, Some(1)) + } + } + Pattern::Variable(Variable { id, .. }) => { + if let Some(path_b) = lhs_id_to_path.get_first_occurrence(id) { + debug_assert!(path != path_b); + ( + linear::MatchOp::Eq { + path_a: path, + path_b, + }, + Some(1), + ) + } else { + (linear::MatchOp::Nop, None) + } + } + Pattern::Operation(op) => (linear::MatchOp::Opcode { path }, Some(op.operator as u32)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use peepmatic_runtime::{ + integer_interner::IntegerId, + linear::{Action::*, MatchOp::*}, + operator::Operator, + r#type::{BitWidth, Kind, Type}, + }; + + macro_rules! linearizes_to { + ($name:ident, $source:expr, $make_expected:expr $(,)* ) => { + #[test] + fn $name() { + let buf = wast::parser::ParseBuffer::new($source).expect("should lex OK"); + + let opts = match wast::parser::parse::(&buf) { + Ok(opts) => opts, + Err(mut e) => { + e.set_path(std::path::Path::new(stringify!($name))); + e.set_text($source); + eprintln!("{}", e); + panic!("should parse OK") + } + }; + + assert_eq!( + opts.optimizations.len(), + 1, + "`linearizes_to!` only supports a single optimization; split the big test into \ + multiple small tests" + ); + + if let Err(mut e) = crate::verify(&opts) { + e.set_path(std::path::Path::new(stringify!($name))); + e.set_text($source); + eprintln!("{}", e); + panic!("should verify OK") + } + + let mut paths = PathInterner::new(); + let mut p = |p: &[u8]| paths.intern(Path::new(&p)); + + let mut integers = IntegerInterner::new(); + let mut i = |i: u64| integers.intern(i); + + #[allow(unused_variables)] + let expected = $make_expected(&mut p, &mut i); + dbg!(&expected); + + let actual = linearize_optimization(&mut paths, &mut integers, &opts.optimizations[0]); + dbg!(&actual); + + assert_eq!(expected, actual); + } + }; + } + + linearizes_to!( + mul_by_pow2_into_shift, + " +(=> (when (imul $x $C) + (is-power-of-two $C)) + (ishl $x $C)) + ", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { + linear::Optimization { + increments: vec![ + linear::Increment { + operation: Opcode { path: p(&[0]) }, + expected: Some(Operator::Imul as _), + actions: vec![ + GetLhs { path: p(&[0, 0]) }, + GetLhs { path: p(&[0, 1]) }, + MakeBinaryInst { + operator: Operator::Ishl, + r#type: Type { + kind: Kind::Int, + bit_width: BitWidth::Polymorphic, + }, + operands: [linear::RhsId(0), linear::RhsId(1)], + }, + ], + }, + linear::Increment { + operation: Nop, + expected: None, + actions: vec![], + }, + linear::Increment { + operation: IsConst { path: p(&[0, 1]) }, + expected: Some(1), + actions: vec![], + }, + linear::Increment { + operation: IsPowerOfTwo { path: p(&[0, 1]) }, + expected: Some(1), + actions: vec![], + }, + ], + } + }, + ); + + linearizes_to!( + variable_pattern_id_optimization, + "(=> $x $x)", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { + linear::Optimization { + increments: vec![linear::Increment { + operation: Nop, + expected: None, + actions: vec![GetLhs { path: p(&[0]) }], + }], + } + }, + ); + + linearizes_to!( + constant_pattern_id_optimization, + "(=> $C $C)", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { + linear::Optimization { + increments: vec![linear::Increment { + operation: IsConst { path: p(&[0]) }, + expected: Some(1), + actions: vec![GetLhs { path: p(&[0]) }], + }], + } + }, + ); + + linearizes_to!( + boolean_literal_id_optimization, + "(=> true true)", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { + linear::Optimization { + increments: vec![linear::Increment { + operation: BooleanValue { path: p(&[0]) }, + expected: Some(1), + actions: vec![MakeBooleanConst { + value: true, + bit_width: BitWidth::Polymorphic, + }], + }], + } + }, + ); + + linearizes_to!( + number_literal_id_optimization, + "(=> 5 5)", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { + linear::Optimization { + increments: vec![linear::Increment { + operation: IntegerValue { path: p(&[0]) }, + expected: Some(i(5).into()), + actions: vec![MakeIntegerConst { + value: i(5), + bit_width: BitWidth::Polymorphic, + }], + }], + } + }, + ); + + linearizes_to!( + operation_id_optimization, + "(=> (iconst $C) (iconst $C))", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { + linear::Optimization { + increments: vec![ + linear::Increment { + operation: Opcode { path: p(&[0]) }, + expected: Some(Operator::Iconst as _), + actions: vec![ + GetLhs { path: p(&[0, 0]) }, + MakeUnaryInst { + operator: Operator::Iconst, + r#type: Type { + kind: Kind::Int, + bit_width: BitWidth::Polymorphic, + }, + operand: linear::RhsId(0), + }, + ], + }, + linear::Increment { + operation: IsConst { path: p(&[0, 0]) }, + expected: Some(1), + actions: vec![], + }, + ], + } + }, + ); + + linearizes_to!( + redundant_bor, + "(=> (bor $x (bor $x $y)) (bor $x $y))", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { + linear::Optimization { + increments: vec![ + linear::Increment { + operation: Opcode { path: p(&[0]) }, + expected: Some(Operator::Bor as _), + actions: vec![ + GetLhs { path: p(&[0, 0]) }, + GetLhs { + path: p(&[0, 1, 1]), + }, + MakeBinaryInst { + operator: Operator::Bor, + r#type: Type { + kind: Kind::Int, + bit_width: BitWidth::Polymorphic, + }, + operands: [linear::RhsId(0), linear::RhsId(1)], + }, + ], + }, + linear::Increment { + operation: Nop, + expected: None, + actions: vec![], + }, + linear::Increment { + operation: Opcode { path: p(&[0, 1]) }, + expected: Some(Operator::Bor as _), + actions: vec![], + }, + linear::Increment { + operation: Eq { + path_a: p(&[0, 1, 0]), + path_b: p(&[0, 0]), + }, + expected: Some(1), + actions: vec![], + }, + linear::Increment { + operation: Nop, + expected: None, + actions: vec![], + }, + ], + } + }, + ); + + linearizes_to!( + large_integers, + // u64::MAX + "(=> 18446744073709551615 0)", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { + linear::Optimization { + increments: vec![linear::Increment { + operation: IntegerValue { path: p(&[0]) }, + expected: Some(i(std::u64::MAX).into()), + actions: vec![MakeIntegerConst { + value: i(0), + bit_width: BitWidth::Polymorphic, + }], + }], + } + } + ); + + linearizes_to!( + ireduce_with_type_ascription, + "(=> (ireduce{i32} $x) 0)", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { + linear::Optimization { + increments: vec![ + linear::Increment { + operation: Opcode { path: p(&[0]) }, + expected: Some(Operator::Ireduce as _), + actions: vec![MakeIntegerConst { + value: i(0), + bit_width: BitWidth::ThirtyTwo, + }], + }, + linear::Increment { + operation: linear::MatchOp::BitWidth { path: p(&[0]) }, + expected: Some(32), + actions: vec![], + }, + linear::Increment { + operation: Nop, + expected: None, + actions: vec![], + }, + ], + } + } + ); +} diff --git a/cranelift/peepmatic/src/parser.rs b/cranelift/peepmatic/src/parser.rs new file mode 100644 index 0000000000..19ad49017c --- /dev/null +++ b/cranelift/peepmatic/src/parser.rs @@ -0,0 +1,932 @@ +/*! + +This module implements parsing the DSL text format. It implements the +`wast::Parse` trait for all of our AST types. + +The grammar for the DSL is given below: + +```ebnf + ::= * + + ::= '(' '=>' ')' + + ::= + | '(' 'when' * ')' + + ::= + | + | > + | + + ::= + | + + ::= 'true' | 'false' + +> ::= '(' [] * ')' + + ::= '(' * ')' + + ::= + | + | + + ::= + | + | + | + | > + + ::= '$' '(' * ')' + + ::= + | +``` + + */ + +use crate::ast::*; +use peepmatic_runtime::r#type::Type; +use std::cell::Cell; +use std::marker::PhantomData; +use wast::{ + parser::{Cursor, Parse, Parser, Peek, Result as ParseResult}, + Id, LParen, +}; + +mod tok { + use wast::{custom_keyword, custom_reserved}; + + custom_keyword!(bit_width = "bit-width"); + custom_reserved!(dollar = "$"); + custom_keyword!(r#false = "false"); + custom_keyword!(fits_in_native_word = "fits-in-native-word"); + custom_keyword!(is_power_of_two = "is-power-of-two"); + custom_reserved!(left_curly = "{"); + custom_keyword!(log2); + custom_keyword!(neg); + custom_reserved!(replace = "=>"); + custom_reserved!(right_curly = "}"); + custom_keyword!(r#true = "true"); + custom_keyword!(when); + + custom_keyword!(eq); + custom_keyword!(ne); + custom_keyword!(slt); + custom_keyword!(ult); + custom_keyword!(sge); + custom_keyword!(uge); + custom_keyword!(sgt); + custom_keyword!(ugt); + custom_keyword!(sle); + custom_keyword!(ule); + custom_keyword!(of); + custom_keyword!(nof); +} + +impl<'a> Parse<'a> for Optimizations<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + let mut optimizations = vec![]; + while !p.is_empty() { + optimizations.push(p.parse()?); + } + Ok(Optimizations { + span, + optimizations, + }) + } +} + +impl<'a> Parse<'a> for Optimization<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + p.parens(|p| { + p.parse::()?; + let lhs = p.parse()?; + let rhs = p.parse()?; + Ok(Optimization { span, lhs, rhs }) + }) + } +} + +impl<'a> Parse<'a> for Lhs<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + let mut preconditions = vec![]; + if p.peek::() && p.peek2::() { + p.parens(|p| { + p.parse::()?; + let pattern = p.parse()?; + while p.peek::() { + preconditions.push(p.parse()?); + } + Ok(Lhs { + span, + pattern, + preconditions, + }) + }) + } else { + let span = p.cur_span(); + let pattern = p.parse()?; + Ok(Lhs { + span, + pattern, + preconditions, + }) + } + } +} + +impl<'a> Parse<'a> for Pattern<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + if p.peek::() { + return Ok(Pattern::ValueLiteral(p.parse()?)); + } + if p.peek::() { + return Ok(Pattern::Constant(p.parse()?)); + } + if p.peek::>() { + return Ok(Pattern::Operation(p.parse()?)); + } + if p.peek::() { + return Ok(Pattern::Variable(p.parse()?)); + } + Err(p.error("expected a left-hand side pattern")) + } +} + +impl<'a> Peek for Pattern<'a> { + fn peek(c: Cursor) -> bool { + ValueLiteral::peek(c) + || Constant::peek(c) + || Variable::peek(c) + || Operation::::peek(c) + } + + fn display() -> &'static str { + "left-hand side pattern" + } +} + +impl<'a> Parse<'a> for ValueLiteral<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + if let Ok(b) = p.parse::() { + return Ok(ValueLiteral::Boolean(b)); + } + if let Ok(i) = p.parse::() { + return Ok(ValueLiteral::Integer(i)); + } + if let Ok(cc) = p.parse::() { + return Ok(ValueLiteral::ConditionCode(cc)); + } + Err(p.error("expected an integer or boolean or condition code literal")) + } +} + +impl<'a> Peek for ValueLiteral<'a> { + fn peek(c: Cursor) -> bool { + c.integer().is_some() || Boolean::peek(c) || ConditionCode::peek(c) + } + + fn display() -> &'static str { + "value literal" + } +} + +impl<'a> Parse<'a> for Integer<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + p.step(|c| { + if let Some((i, rest)) = c.integer() { + let (s, base) = i.val(); + let val = i64::from_str_radix(s, base) + .or_else(|_| u128::from_str_radix(s, base).map(|i| i as i64)); + return match val { + Ok(value) => Ok(( + Integer { + span, + value, + bit_width: Default::default(), + marker: PhantomData, + }, + rest, + )), + Err(_) => Err(c.error("invalid integer: out of range")), + }; + } + Err(c.error("expected an integer")) + }) + } +} + +impl<'a> Parse<'a> for Boolean<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + if p.parse::().is_ok() { + return Ok(Boolean { + span, + value: true, + bit_width: Default::default(), + marker: PhantomData, + }); + } + if p.parse::().is_ok() { + return Ok(Boolean { + span, + value: false, + bit_width: Default::default(), + marker: PhantomData, + }); + } + Err(p.error("expected `true` or `false`")) + } +} + +impl<'a> Peek for Boolean<'a> { + fn peek(c: Cursor) -> bool { + ::peek(c) || ::peek(c) + } + + fn display() -> &'static str { + "boolean `true` or `false`" + } +} + +impl<'a> Parse<'a> for ConditionCode<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + + macro_rules! parse_cc { + ( $( $token:ident => $cc:ident, )* ) => { + $( + if p.peek::() { + p.parse::()?; + return Ok(Self { + span, + cc: peepmatic_runtime::cc::ConditionCode::$cc, + marker: PhantomData, + }); + } + )* + } + } + + parse_cc! { + eq => Eq, + ne => Ne, + slt => Slt, + ult => Ult, + sge => Sge, + uge => Uge, + sgt => Sgt, + ugt => Ugt, + sle => Sle, + ule => Ule, + of => Of, + nof => Nof, + } + + Err(p.error("expected a condition code")) + } +} + +impl<'a> Peek for ConditionCode<'a> { + fn peek(c: Cursor) -> bool { + macro_rules! peek_cc { + ( $( $token:ident, )* ) => { + false $( || ::peek(c) )* + } + } + + peek_cc! { + eq, + ne, + slt, + ult, + sge, + uge, + sgt, + ugt, + sle, + ule, + of, + nof, + } + } + + fn display() -> &'static str { + "condition code" + } +} + +impl<'a> Parse<'a> for Constant<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + let id = Id::parse(p)?; + if id + .name() + .chars() + .all(|c| !c.is_alphabetic() || c.is_uppercase()) + { + Ok(Constant { span, id }) + } else { + let upper = id + .name() + .chars() + .flat_map(|c| c.to_uppercase()) + .collect::(); + Err(p.error(format!( + "symbolic constants must start with an upper-case letter like ${}", + upper + ))) + } + } +} + +impl<'a> Peek for Constant<'a> { + fn peek(c: Cursor) -> bool { + if let Some((id, _rest)) = c.id() { + id.chars().all(|c| !c.is_alphabetic() || c.is_uppercase()) + } else { + false + } + } + + fn display() -> &'static str { + "symbolic constant" + } +} + +impl<'a> Parse<'a> for Variable<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + let id = Id::parse(p)?; + if id + .name() + .chars() + .all(|c| !c.is_alphabetic() || c.is_lowercase()) + { + Ok(Variable { span, id }) + } else { + let lower = id + .name() + .chars() + .flat_map(|c| c.to_lowercase()) + .collect::(); + Err(p.error(format!( + "variables must start with an lower-case letter like ${}", + lower + ))) + } + } +} + +impl<'a> Peek for Variable<'a> { + fn peek(c: Cursor) -> bool { + if let Some((id, _rest)) = c.id() { + id.chars().all(|c| !c.is_alphabetic() || c.is_lowercase()) + } else { + false + } + } + + fn display() -> &'static str { + "variable" + } +} + +impl<'a, T> Parse<'a> for Operation<'a, T> +where + T: 'a + Ast<'a> + Peek + Parse<'a>, + DynAstRef<'a>: From<&'a T>, +{ + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + p.parens(|p| { + let operator = p.parse()?; + + let r#type = Cell::new(if p.peek::() { + p.parse::()?; + let ty = p.parse::()?; + p.parse::()?; + Some(ty) + } else { + None + }); + + let mut operands = vec![]; + while p.peek::() { + operands.push(p.parse()?); + } + Ok(Operation { + span, + operator, + r#type, + operands, + marker: PhantomData, + }) + }) + } +} + +impl<'a, T> Peek for Operation<'a, T> +where + T: 'a + Ast<'a>, + DynAstRef<'a>: From<&'a T>, +{ + fn peek(c: Cursor) -> bool { + wast::LParen::peek(c) + } + + fn display() -> &'static str { + "operation" + } +} + +impl<'a> Parse<'a> for Precondition<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + p.parens(|p| { + let constraint = p.parse()?; + let mut operands = vec![]; + while p.peek::() { + operands.push(p.parse()?); + } + Ok(Precondition { + span, + constraint, + operands, + }) + }) + } +} + +impl<'a> Parse<'a> for Constraint { + fn parse(p: Parser<'a>) -> ParseResult { + if p.peek::() { + p.parse::()?; + return Ok(Constraint::IsPowerOfTwo); + } + if p.peek::() { + p.parse::()?; + return Ok(Constraint::BitWidth); + } + if p.peek::() { + p.parse::()?; + return Ok(Constraint::FitsInNativeWord); + } + Err(p.error("expected a precondition constraint")) + } +} + +impl<'a> Parse<'a> for ConstraintOperand<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + if p.peek::() { + return Ok(ConstraintOperand::ValueLiteral(p.parse()?)); + } + if p.peek::() { + return Ok(ConstraintOperand::Constant(p.parse()?)); + } + if p.peek::() { + return Ok(ConstraintOperand::Variable(p.parse()?)); + } + Err(p.error("expected an operand for precondition constraint")) + } +} + +impl<'a> Peek for ConstraintOperand<'a> { + fn peek(c: Cursor) -> bool { + ValueLiteral::peek(c) || Constant::peek(c) || Variable::peek(c) + } + + fn display() -> &'static str { + "operand for a precondition constraint" + } +} + +impl<'a> Parse<'a> for Rhs<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + if p.peek::() { + return Ok(Rhs::ValueLiteral(p.parse()?)); + } + if p.peek::() { + return Ok(Rhs::Constant(p.parse()?)); + } + if p.peek::() { + return Ok(Rhs::Variable(p.parse()?)); + } + if p.peek::() { + return Ok(Rhs::Unquote(p.parse()?)); + } + if p.peek::>() { + return Ok(Rhs::Operation(p.parse()?)); + } + Err(p.error("expected a right-hand side replacement")) + } +} + +impl<'a> Peek for Rhs<'a> { + fn peek(c: Cursor) -> bool { + ValueLiteral::peek(c) + || Constant::peek(c) + || Variable::peek(c) + || Unquote::peek(c) + || Operation::::peek(c) + } + + fn display() -> &'static str { + "right-hand side replacement" + } +} + +impl<'a> Parse<'a> for Unquote<'a> { + fn parse(p: Parser<'a>) -> ParseResult { + let span = p.cur_span(); + p.parse::()?; + p.parens(|p| { + let operator = p.parse()?; + let mut operands = vec![]; + while p.peek::() { + operands.push(p.parse()?); + } + Ok(Unquote { + span, + operator, + operands, + }) + }) + } +} + +impl<'a> Peek for Unquote<'a> { + fn peek(c: Cursor) -> bool { + tok::dollar::peek(c) + } + + fn display() -> &'static str { + "unquote expression" + } +} + +#[cfg(test)] +mod test { + use super::*; + use peepmatic_runtime::operator::Operator; + + macro_rules! test_parse { + ( + $( + $name:ident < $ast:ty > { + $( ok { $( $ok:expr , )* } )* + $( err { $( $err:expr , )* } )* + } + )* + ) => { + $( + #[test] + #[allow(non_snake_case)] + fn $name() { + $( + $({ + let input = $ok; + let buf = wast::parser::ParseBuffer::new(input).unwrap_or_else(|e| { + panic!("should lex OK, got error:\n\n{}\n\nInput:\n\n{}", e, input) + }); + if let Err(e) = wast::parser::parse::<$ast>(&buf) { + panic!( + "expected to parse OK, got error:\n\n{}\n\nInput:\n\n{}", + e, input + ); + } + })* + )* + + $( + $({ + let input = $err; + let buf = wast::parser::ParseBuffer::new(input).unwrap_or_else(|e| { + panic!("should lex OK, got error:\n\n{}\n\nInput:\n\n{}", e, input) + }); + if let Ok(ast) = wast::parser::parse::<$ast>(&buf) { + panic!( + "expected a parse error, got:\n\n{:?}\n\nInput:\n\n{}", + ast, input + ); + } + })* + )* + } + )* + } + } + + test_parse! { + parse_boolean { + ok { + "true", + "false", + } + err { + "", + "t", + "tr", + "tru", + "truezzz", + "f", + "fa", + "fal", + "fals", + "falsezzz", + } + } + parse_cc { + ok { + "eq", + "ne", + "slt", + "ult", + "sge", + "uge", + "sgt", + "ugt", + "sle", + "ule", + "of", + "nof", + } + err { + "", + "neq", + } + } + parse_constant { + ok { + "$C", + "$C1", + "$C2", + "$X", + "$Y", + "$SOME-CONSTANT", + "$SOME_CONSTANT", + } + err { + "", + "zzz", + "$", + "$variable", + "$Some-Constant", + "$Some_Constant", + "$Some_constant", + } + } + parse_constraint { + ok { + "is-power-of-two", + "bit-width", + "fits-in-native-word", + } + err { + "", + "iadd", + "imul", + } + } + parse_constraint_operand { + ok { + "1234", + "true", + "$CONSTANT", + "$variable", + } + err { + "", + "is-power-of-two", + "(is-power-of-two $C)", + "(iadd 1 2)", + } + } + parse_integer { + ok { + "0", + "1", + "12", + "123", + "1234", + "12345", + "123456", + "1234567", + "12345678", + "123456789", + "1234567890", + "0x0", + "0x1", + "0x12", + "0x123", + "0x1234", + "0x12345", + "0x123456", + "0x1234567", + "0x12345678", + "0x123456789", + "0x123456789a", + "0x123456789ab", + "0x123456789abc", + "0x123456789abcd", + "0x123456789abcde", + "0x123456789abcdef", + "0xffff_ffff_ffff_ffff", + } + err { + "", + "abcdef", + "01234567890abcdef", + "0xgggg", + "0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + } + } + parse_lhs { + ok { + "(when (imul $C1 $C2) (is-power-of-two $C1) (is-power-of-two $C2))", + "(when (imul $x $C) (is-power-of-two $C))", + "(imul $x $y)", + "(imul $x)", + "(imul)", + "$C", + "$x", + } + err { + "", + "()", + "abc", + } + } + parse_operation_pattern> { + ok { + "(iadd)", + "(iadd 1)", + "(iadd 1 2)", + "(iadd $x $C)", + "(iadd{i32} $x $y)", + "(icmp eq $x $y)", + } + err { + "", + "()", + "$var", + "$CONST", + "(ishl $x $(log2 $C))", + } + } + parse_operation_rhs> { + ok { + "(iadd)", + "(iadd 1)", + "(iadd 1 2)", + "(ishl $x $(log2 $C))", + } + err { + "", + "()", + "$var", + "$CONST", + } + } + parse_operator { + ok { + "bor", + "iadd", + "iadd_imm", + "iconst", + "imul", + "imul_imm", + "ishl", + "sdiv", + "sdiv_imm", + "sshr", + } + err { + "", + "iadd.i32", + "iadd{i32}", + } + } + parse_optimization { + ok { + "(=> (when (iadd $x $C) (is-power-of-two $C) (is-power-of-two $C)) (iadd $C $x))", + "(=> (when (iadd $x $C)) (iadd $C $x))", + "(=> (iadd $x $C) (iadd $C $x))", + } + err { + "", + "()", + "(=>)", + "(=> () ())", + } + } + parse_optimizations { + ok { + "", + r#" + ;; Canonicalize `a + (b + c)` into `(a + b) + c`. + (=> (iadd $a (iadd $b $c)) + (iadd (iadd $a $b) $c)) + + ;; Combine a `const` and an `iadd` into a `iadd_imm`. + (=> (iadd (iconst $C) $x) + (iadd_imm $C $x)) + + ;; When `C` is a power of two, replace `x * C` with `x << log2(C)`. + (=> (when (imul $x $C) + (is-power-of-two $C)) + (ishl $x $(log2 $C))) + "#, + } + } + parse_pattern { + ok { + "1234", + "$C", + "$x", + "(iadd $x $y)", + } + err { + "", + "()", + "abc", + } + } + parse_precondition { + ok { + "(is-power-of-two)", + "(is-power-of-two $C)", + "(is-power-of-two $C1 $C2)", + } + err { + "", + "1234", + "()", + "$var", + "$CONST", + } + } + parse_rhs { + ok { + "5", + "$C", + "$x", + "$(log2 $C)", + "(iadd $x 1)", + } + err { + "", + "()", + } + } + parse_unquote { + ok { + "$(log2)", + "$(log2 $C)", + "$(log2 $C1 1)", + "$(neg)", + "$(neg $C)", + "$(neg $C 1)", + } + err { + "", + "(log2 $C)", + "$()", + } + } + parse_value_literal { + ok { + "12345", + "true", + } + err { + "", + "'c'", + "\"hello\"", + "12.34", + } + } + parse_variable { + ok { + "$v", + "$v1", + "$v2", + "$x", + "$y", + "$some-var", + "$another_var", + } + err { + "zzz", + "$", + "$CONSTANT", + "$fooBar", + } + } + } +} diff --git a/cranelift/peepmatic/src/traversals.rs b/cranelift/peepmatic/src/traversals.rs new file mode 100644 index 0000000000..5eb4101c37 --- /dev/null +++ b/cranelift/peepmatic/src/traversals.rs @@ -0,0 +1,278 @@ +//! Traversals over the AST. + +use crate::ast::*; + +/// A low-level DFS traversal event: either entering or exiting the traversal of +/// an AST node. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TraversalEvent { + /// Entering traversal of an AST node. + /// + /// Processing an AST node upon this event corresponds to a pre-order + /// DFS traversal. + Enter, + + /// Exiting traversal of an AST node. + /// + /// Processing an AST node upon this event corresponds to a post-order DFS + /// traversal. + Exit, +} + +/// A depth-first traversal of an AST. +/// +/// This is a fairly low-level traversal type, and is intended to be used as a +/// building block for making specific pre-order or post-order traversals for +/// whatever problem is at hand. +/// +/// This implementation is not recursive, and exposes an `Iterator` interface +/// that yields pairs of `(TraversalEvent, DynAstRef)` items. +/// +/// The traversal can walk a whole set of `Optimization`s or just a subtree of +/// the AST, because the `new` constructor takes anything that can convert into +/// a `DynAstRef`. +#[derive(Debug, Clone)] +pub struct Dfs<'a> { + stack: Vec<(TraversalEvent, DynAstRef<'a>)>, +} + +impl<'a> Dfs<'a> { + /// Construct a new `Dfs` traversal starting at the given `start` AST node. + pub fn new(start: impl Into>) -> Self { + let start = start.into(); + Dfs { + stack: vec![ + (TraversalEvent::Exit, start), + (TraversalEvent::Enter, start), + ], + } + } + + /// Peek at the next traversal event and AST node pair, if any. + pub fn peek(&self) -> Option<(TraversalEvent, DynAstRef<'a>)> { + self.stack.last().cloned() + } +} + +impl<'a> Iterator for Dfs<'a> { + type Item = (TraversalEvent, DynAstRef<'a>); + + fn next(&mut self) -> Option<(TraversalEvent, DynAstRef<'a>)> { + let (event, node) = self.stack.pop()?; + if let TraversalEvent::Enter = event { + let mut enqueue_children = EnqueueChildren(self); + node.child_nodes(&mut enqueue_children) + } + return Some((event, node)); + + struct EnqueueChildren<'a, 'b>(&'b mut Dfs<'a>) + where + 'a: 'b; + + impl<'a, 'b> Extend> for EnqueueChildren<'a, 'b> + where + 'a: 'b, + { + fn extend>>(&mut self, iter: T) { + let iter = iter.into_iter(); + + let (min, max) = iter.size_hint(); + self.0.stack.reserve(max.unwrap_or(min) * 2); + + let start = self.0.stack.len(); + + for node in iter { + self.0.stack.push((TraversalEvent::Enter, node)); + self.0.stack.push((TraversalEvent::Exit, node)); + } + + // Reverse to make it so that we visit children in order + // (e.g. operands are visited in order). + self.0.stack[start..].reverse(); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use DynAstRef::*; + + #[test] + fn test_dfs_traversal() { + let input = " +(=> (when (imul $x $C) + (is-power-of-two $C)) + (ishl $x $(log2 $C))) +"; + let buf = wast::parser::ParseBuffer::new(input).expect("input should lex OK"); + let ast = match wast::parser::parse::(&buf) { + Ok(ast) => ast, + Err(e) => panic!("expected to parse OK, got error:\n\n{}", e), + }; + + let mut dfs = Dfs::new(&ast); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Optimizations(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Optimization(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Lhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Pattern(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, PatternOperation(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Pattern(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Variable(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Variable(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Pattern(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Pattern(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Constant(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Constant(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Pattern(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, PatternOperation(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Pattern(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Precondition(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, ConstraintOperand(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Constant(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Constant(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, ConstraintOperand(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Precondition(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Lhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Rhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, RhsOperation(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Rhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Variable(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Variable(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Rhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Rhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Unquote(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Rhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Enter, Constant(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Constant(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Rhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Unquote(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Rhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, RhsOperation(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Rhs(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Optimization(..))) + )); + assert!(matches!( + dbg!(dfs.next()), + Some((TraversalEvent::Exit, Optimizations(..))) + )); + assert!(dfs.next().is_none()); + } +} diff --git a/cranelift/peepmatic/src/verify.rs b/cranelift/peepmatic/src/verify.rs new file mode 100644 index 0000000000..03865458ca --- /dev/null +++ b/cranelift/peepmatic/src/verify.rs @@ -0,0 +1,1433 @@ +//! Verification and type checking of optimizations. +//! +//! For type checking, we compile the AST's type constraints down into Z3 +//! variables and assertions. If Z3 finds the assertions satisfiable, then we're +//! done! If it finds them unsatisfiable, we use the `get_unsat_core` method to +//! get the minimal subset of assertions that are in conflict, and report a +//! best-effort type error message with them. These messages aren't perfect, but +//! they're Good Enough when embedded in the source text via our tracking of +//! `wast::Span`s. +//! +//! Verifying that there aren't any counter-examples (inputs for which the LHS +//! and RHS produce different results) for a particular optimization is not +//! implemented yet. + +use crate::ast::{Span as _, *}; +use crate::traversals::{Dfs, TraversalEvent}; +use peepmatic_runtime::{ + operator::{Operator, TypingContext as TypingContextTrait}, + r#type::{BitWidth, Kind, Type}, +}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::convert::{TryFrom, TryInto}; +use std::fmt; +use std::hash::Hash; +use std::iter; +use std::mem; +use std::ops::{Deref, DerefMut}; +use std::path::Path; +use wast::{Error as WastError, Id, Span}; +use z3::ast::Ast; + +/// A verification or type checking error. +#[derive(Debug)] +pub struct VerifyError { + errors: Vec, +} + +impl fmt::Display for VerifyError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for e in &self.errors { + writeln!(f, "{}\n", e)?; + } + Ok(()) + } +} + +impl std::error::Error for VerifyError {} + +impl From for VerifyError { + fn from(e: WastError) -> Self { + VerifyError { + errors: vec![e.into()], + } + } +} + +impl From for VerifyError { + fn from(e: anyhow::Error) -> Self { + VerifyError { errors: vec![e] } + } +} + +impl VerifyError { + /// To provide a more useful error this function can be used to extract + /// relevant textual information about this error into the error itself. + /// + /// The `contents` here should be the full text of the original file being + /// parsed, and this will extract a sub-slice as necessary to render in the + /// `Display` implementation later on. + pub fn set_text(&mut self, contents: &str) { + for e in &mut self.errors { + if let Some(e) = e.downcast_mut::() { + e.set_text(contents); + } + } + } + + /// To provide a more useful error this function can be used to set + /// the file name that this error is associated with. + /// + /// The `path` here will be stored in this error and later rendered in the + /// `Display` implementation. + pub fn set_path(&mut self, path: &Path) { + for e in &mut self.errors { + if let Some(e) = e.downcast_mut::() { + e.set_path(path); + } + } + } +} + +/// Either `Ok(T)` or `Err(VerifyError)`. +pub type VerifyResult = Result; + +/// Verify and type check a set of optimizations. +pub fn verify(opts: &Optimizations) -> VerifyResult<()> { + if opts.optimizations.is_empty() { + return Err(anyhow::anyhow!("no optimizations").into()); + } + + verify_unique_left_hand_sides(opts)?; + + let z3 = &z3::Context::new(&z3::Config::new()); + for opt in &opts.optimizations { + verify_optimization(z3, opt)?; + } + Ok(()) +} + +/// Check that every LHS in the given optimizations is unique. +/// +/// If there were duplicates, then it would be nondeterministic which one we +/// applied and would make automata construction more difficult. It is better to +/// check for duplicates and reject them if found. +fn verify_unique_left_hand_sides(opts: &Optimizations) -> VerifyResult<()> { + let mut lefts = HashMap::new(); + for opt in &opts.optimizations { + let canon_lhs = canonicalized_lhs_key(&opt.lhs); + let existing = lefts.insert(canon_lhs, opt.lhs.span()); + if let Some(span) = existing { + return Err(VerifyError { + errors: vec![ + anyhow::anyhow!("error: two optimizations cannot have the same left-hand side"), + WastError::new(span, "note: first use of this left-hand side".into()).into(), + WastError::new( + opt.lhs.span(), + "note: second use of this left-hand side".into(), + ) + .into(), + ], + }); + } + } + Ok(()) +} + +/// When checking for duplicate left-hand sides, we need to consider patterns +/// that are duplicates up to renaming identifiers. For example, these LHSes +/// should be considered duplicates of each other: +/// +/// ```lisp +/// (=> (iadd $x $y) ...) +/// (=> (iadd $a $b) ...) +/// ``` +/// +/// This function creates an opaque, canonicalized hash key for left-hand sides +/// that sees through identifier renaming. +fn canonicalized_lhs_key(lhs: &Lhs) -> impl Hash + Eq { + let mut var_to_canon = HashMap::new(); + let mut const_to_canon = HashMap::new(); + let mut canonicalized = vec![]; + + for (event, ast) in Dfs::new(lhs) { + if event != TraversalEvent::Enter { + continue; + } + use CanonicalBit::*; + canonicalized.push(match ast { + DynAstRef::Lhs(_) => Other("Lhs"), + DynAstRef::Pattern(_) => Other("Pattern"), + DynAstRef::ValueLiteral(_) => Other("ValueLiteral"), + DynAstRef::Integer(i) => Integer(i.value), + DynAstRef::Boolean(b) => Boolean(b.value), + DynAstRef::ConditionCode(cc) => ConditionCode(cc.cc), + DynAstRef::PatternOperation(o) => Operation(o.operator, o.r#type.get()), + DynAstRef::Precondition(p) => Precondition(p.constraint), + DynAstRef::ConstraintOperand(_) => Other("ConstraintOperand"), + DynAstRef::Variable(Variable { id, .. }) => { + let new_id = var_to_canon.len() as u32; + let canon_id = var_to_canon.entry(id).or_insert(new_id); + Var(*canon_id) + } + DynAstRef::Constant(Constant { id, .. }) => { + let new_id = const_to_canon.len() as u32; + let canon_id = const_to_canon.entry(id).or_insert(new_id); + Const(*canon_id) + } + other => unreachable!("unreachable ast node: {:?}", other), + }); + } + + return canonicalized; + + #[derive(Hash, PartialEq, Eq)] + enum CanonicalBit { + Var(u32), + Const(u32), + Integer(i64), + Boolean(bool), + ConditionCode(peepmatic_runtime::cc::ConditionCode), + Operation(Operator, Option), + Precondition(Constraint), + Other(&'static str), + } +} + +pub(crate) struct TypingContext<'a> { + z3: &'a z3::Context, + type_kind_sort: z3::DatatypeSort<'a>, + solver: z3::Solver<'a>, + + // The type of the root of the optimization. Initialized when collecting + // type constraints. + root_ty: Option>, + + // See the comments above `enter_operation_scope`. + operation_scope: HashMap<&'static str, TypeVar<'a>>, + + // A map from identifiers to the type variable describing its type. + id_to_type_var: HashMap, TypeVar<'a>>, + + // A list of type constraints, the span of the AST node where the constraint + // originates from, and an optional message to be displayed if the + // constraint is not satisfied. + constraints: Vec<(z3::ast::Bool<'a>, Span, Option>)>, + + // Keep track of AST nodes that need to have their types assigned to + // them. For these AST nodes, we know what bit width to use when + // interpreting peephole optimization actions. + boolean_literals: Vec<(&'a Boolean<'a>, TypeVar<'a>)>, + integer_literals: Vec<(&'a Integer<'a>, TypeVar<'a>)>, + rhs_operations: Vec<(&'a Operation<'a, Rhs<'a>>, TypeVar<'a>)>, +} + +impl<'a> TypingContext<'a> { + fn new(z3: &'a z3::Context) -> Self { + let type_kind_sort = z3::DatatypeBuilder::new(z3) + .variant("int", &[]) + .variant("bool", &[]) + .variant("cpu_flags", &[]) + .variant("cc", &[]) + .variant("void", &[]) + .finish("TypeKind"); + TypingContext { + z3, + solver: z3::Solver::new(z3), + root_ty: None, + operation_scope: Default::default(), + id_to_type_var: Default::default(), + type_kind_sort, + constraints: vec![], + boolean_literals: Default::default(), + integer_literals: Default::default(), + rhs_operations: Default::default(), + } + } + + fn init_root_type(&mut self, span: Span, root_ty: TypeVar<'a>) { + assert!(self.root_ty.is_none()); + + // Make sure the root is a valid kind, i.e. not a condition code. + let is_int = self.is_int(&root_ty); + let is_bool = self.is_bool(&root_ty); + let is_void = self.is_void(&root_ty); + let is_cpu_flags = self.is_cpu_flags(&root_ty); + self.constraints.push(( + is_int.or(&[&is_bool, &is_void, &is_cpu_flags]), + span, + Some( + "the root of an optimization must be an integer, a boolean, void, or CPU flags" + .into(), + ), + )); + + self.root_ty = Some(root_ty); + } + + fn new_type_var(&self) -> TypeVar<'a> { + let kind = + z3::ast::Datatype::fresh_const(self.z3, "type-var-kind", &self.type_kind_sort.sort); + let width = z3::ast::BV::fresh_const(self.z3, "type-var-width", 8); + TypeVar { kind, width } + } + + fn get_or_create_type_var_for_id(&mut self, id: Id<'a>) -> TypeVar<'a> { + if let Some(ty) = self.id_to_type_var.get(&id) { + ty.clone() + } else { + // Note: can't use the entry API because we reborrow `self` here. + let ty = self.new_type_var(); + self.id_to_type_var.insert(id, ty.clone()); + ty + } + } + + fn get_type_var_for_id(&mut self, id: Id<'a>) -> VerifyResult> { + if let Some(ty) = self.id_to_type_var.get(&id) { + Ok(ty.clone()) + } else { + Err(WastError::new(id.span(), format!("unknown identifier: ${}", id.name())).into()) + } + } + + // The `#[peepmatic]` macro for operations allows defining operations' types + // like `(iNN, iNN) -> iNN` where `iNN` all refer to the same integer type + // variable that must have the same bit width. But other operations might + // *also* have that type signature but be instantiated at a different bit + // width. We don't want to mix up which `iNN` variables are and aren't the + // same. We use this method to track scopes within which all uses of `iNN` + // and similar refer to the same type variables. + fn enter_operation_scope<'b>( + &'b mut self, + ) -> impl DerefMut> + Drop + 'b { + assert!(self.operation_scope.is_empty()); + return Scope(self); + + struct Scope<'a, 'b>(&'b mut TypingContext<'a>) + where + 'a: 'b; + + impl<'a, 'b> Deref for Scope<'a, 'b> + where + 'a: 'b, + { + type Target = TypingContext<'a>; + fn deref(&self) -> &TypingContext<'a> { + self.0 + } + } + + impl<'a, 'b> DerefMut for Scope<'a, 'b> + where + 'a: 'b, + { + fn deref_mut(&mut self) -> &mut TypingContext<'a> { + self.0 + } + } + + impl Drop for Scope<'_, '_> { + fn drop(&mut self) { + self.0.operation_scope.clear(); + } + } + } + + fn remember_boolean_literal(&mut self, b: &'a Boolean<'a>, ty: TypeVar<'a>) { + self.assert_is_bool(b.span, &ty); + self.boolean_literals.push((b, ty)); + } + + fn remember_integer_literal(&mut self, i: &'a Integer<'a>, ty: TypeVar<'a>) { + self.assert_is_integer(i.span, &ty); + self.integer_literals.push((i, ty)); + } + + fn remember_rhs_operation(&mut self, op: &'a Operation<'a, Rhs<'a>>, ty: TypeVar<'a>) { + self.rhs_operations.push((op, ty)); + } + + fn is_int(&self, ty: &TypeVar<'a>) -> z3::ast::Bool<'a> { + self.type_kind_sort.variants[0] + .tester + .apply(&[&ty.kind.clone().into()]) + .as_bool() + .unwrap() + } + + fn is_bool(&self, ty: &TypeVar<'a>) -> z3::ast::Bool<'a> { + self.type_kind_sort.variants[1] + .tester + .apply(&[&ty.kind.clone().into()]) + .as_bool() + .unwrap() + } + + fn is_cpu_flags(&self, ty: &TypeVar<'a>) -> z3::ast::Bool<'a> { + self.type_kind_sort.variants[2] + .tester + .apply(&[&ty.kind.clone().into()]) + .as_bool() + .unwrap() + } + + fn is_condition_code(&self, ty: &TypeVar<'a>) -> z3::ast::Bool<'a> { + self.type_kind_sort.variants[3] + .tester + .apply(&[&ty.kind.clone().into()]) + .as_bool() + .unwrap() + } + + fn is_void(&self, ty: &TypeVar<'a>) -> z3::ast::Bool<'a> { + self.type_kind_sort.variants[4] + .tester + .apply(&[&ty.kind.clone().into()]) + .as_bool() + .unwrap() + } + + fn assert_is_integer(&mut self, span: Span, ty: &TypeVar<'a>) { + self.constraints.push(( + self.is_int(ty), + span, + Some("type error: expected integer".into()), + )); + } + + fn assert_is_bool(&mut self, span: Span, ty: &TypeVar<'a>) { + self.constraints.push(( + self.is_bool(ty), + span, + Some("type error: expected bool".into()), + )); + } + + fn assert_is_cpu_flags(&mut self, span: Span, ty: &TypeVar<'a>) { + self.constraints.push(( + self.is_cpu_flags(ty), + span, + Some("type error: expected CPU flags".into()), + )); + } + + fn assert_is_cc(&mut self, span: Span, ty: &TypeVar<'a>) { + self.constraints.push(( + self.is_condition_code(ty), + span, + Some("type error: expected condition code".into()), + )); + } + + fn assert_is_void(&mut self, span: Span, ty: &TypeVar<'a>) { + self.constraints.push(( + self.is_void(ty), + span, + Some("type error: expected void".into()), + )); + } + + fn assert_bit_width(&mut self, span: Span, ty: &TypeVar<'a>, width: u8) { + debug_assert!(width == 0 || width.is_power_of_two()); + let width_var = z3::ast::BV::from_i64(self.z3, width as i64, 8); + let is_width = width_var._eq(&ty.width); + self.constraints.push(( + is_width, + span, + Some(format!("type error: expected bit width = {}", width).into()), + )); + } + + fn assert_bit_width_lt(&mut self, span: Span, a: &TypeVar<'a>, b: &TypeVar<'a>) { + self.constraints.push(( + a.width.bvult(&b.width), + span, + Some("type error: expected narrower bit width".into()), + )); + } + + fn assert_bit_width_gt(&mut self, span: Span, a: &TypeVar<'a>, b: &TypeVar<'a>) { + self.constraints.push(( + a.width.bvugt(&b.width), + span, + Some("type error: expected wider bit width".into()), + )); + } + + fn assert_type_eq( + &mut self, + span: Span, + lhs: &TypeVar<'a>, + rhs: &TypeVar<'a>, + msg: Option>, + ) { + self.constraints + .push((lhs.kind._eq(&rhs.kind), span, msg.clone())); + self.constraints + .push((lhs.width._eq(&rhs.width), span, msg)); + } + + fn type_check(&self, span: Span) -> VerifyResult<()> { + let trackers = iter::repeat_with(|| z3::ast::Bool::fresh_const(self.z3, "type-constraint")) + .take(self.constraints.len()) + .collect::>(); + + let mut tracker_to_diagnostics = HashMap::with_capacity(self.constraints.len()); + + for (constraint_data, tracker) in self.constraints.iter().zip(trackers) { + let (constraint, span, msg) = constraint_data; + self.solver.assert_and_track(constraint, &tracker); + tracker_to_diagnostics.insert(tracker, (*span, msg.clone())); + } + + match self.solver.check() { + z3::SatResult::Sat => Ok(()), + z3::SatResult::Unsat => { + let core = self.solver.get_unsat_core(); + if core.is_empty() { + return Err(WastError::new( + span, + "z3 determined the type constraints for this optimization are \ + unsatisfiable, meaning there is a type error, but z3 did not give us any \ + additional information" + .into(), + ) + .into()); + } + + let mut errors = core + .iter() + .map(|tracker| { + let (span, msg) = &tracker_to_diagnostics[tracker]; + ( + *span, + WastError::new( + *span, + msg.clone().unwrap_or("type error".into()).into(), + ) + .into(), + ) + }) + .collect::>(); + errors.sort_by_key(|(span, _)| *span); + let errors = errors.into_iter().map(|(_, e)| e).collect(); + + Err(VerifyError { errors }) + } + z3::SatResult::Unknown => Err(anyhow::anyhow!( + "z3 returned 'unknown' when evaluating type constraints: {}", + self.solver + .get_reason_unknown() + .unwrap_or_else(|| "".into()) + ) + .into()), + } + } + + fn assign_types(&mut self) -> VerifyResult<()> { + for (int, ty) in mem::replace(&mut self.integer_literals, vec![]) { + let width = self.ty_var_to_width(&ty)?; + int.bit_width.set(Some(width)); + } + + for (b, ty) in mem::replace(&mut self.boolean_literals, vec![]) { + let width = self.ty_var_to_width(&ty)?; + b.bit_width.set(Some(width)); + } + + for (op, ty) in mem::replace(&mut self.rhs_operations, vec![]) { + let kind = self.op_ty_var_to_kind(&ty); + let bit_width = match kind { + Kind::CpuFlags | Kind::Void => BitWidth::One, + Kind::Int | Kind::Bool => self.ty_var_to_width(&ty)?, + }; + debug_assert!(op.r#type.get().is_none()); + op.r#type.set(Some(Type { kind, bit_width })); + } + + Ok(()) + } + + fn ty_var_to_width(&self, ty_var: &TypeVar<'a>) -> VerifyResult { + // Doing solver push/pops apparently clears out the model, so we have to + // re-check each time to ensure that it exists, and Z3 doesn't helpfully + // abort the process for us. This should be fast, since the solver + // remembers inferences from earlier checks. + assert_eq!(self.solver.check(), z3::SatResult::Sat); + + // Check if there is more than one satisfying assignment to + // `ty_var`'s width variable. If so, then it must be polymorphic. If + // not, then it must have a fixed value. + let model = self.solver.get_model(); + let width_var = model.eval(&ty_var.width).unwrap(); + let bit_width: u8 = width_var.as_u64().unwrap().try_into().unwrap(); + + self.solver.push(); + self.solver.assert(&ty_var.width._eq(&width_var).not()); + let is_polymorphic = match self.solver.check() { + z3::SatResult::Sat => true, + z3::SatResult::Unsat => false, + z3::SatResult::Unknown => panic!("Z3 cannot determine bit width of type"), + }; + self.solver.pop(1); + + if is_polymorphic { + // If something is polymorphic over bit widths, it must be + // polymorphic over the same bit width as the whole + // optimization. + // + // TODO: We should have a better model for bit-width + // polymorphism. The current setup works for all the use cases we + // currently care about, and is relatively easy to implement when + // matching and constructing the RHS, but is a bit ad-hoc. Maybe + // allow each LHS variable a polymorphic bit width, augment the AST + // with that info, and later emit match ops as necessary to express + // their relative constraints? *hand waves* + self.solver.push(); + self.solver + .assert(&ty_var.width._eq(&self.root_ty.as_ref().unwrap().width)); + match self.solver.check() { + z3::SatResult::Sat => {} + z3::SatResult::Unsat => { + return Err(anyhow::anyhow!( + "AST node is bit width polymorphic, but not over the optimization's root \ + width" + ) + .into()) + } + z3::SatResult::Unknown => panic!("Z3 cannot determine bit width of type"), + }; + self.solver.pop(1); + + Ok(BitWidth::Polymorphic) + } else { + Ok(BitWidth::try_from(bit_width).unwrap()) + } + } + + fn op_ty_var_to_kind(&self, ty_var: &TypeVar<'a>) -> Kind { + for (predicate, kind) in [ + (Self::is_int as fn(_, _) -> _, Kind::Int), + (Self::is_bool, Kind::Bool), + (Self::is_cpu_flags, Kind::CpuFlags), + (Self::is_void, Kind::Void), + ] + .iter() + { + self.solver.push(); + self.solver.assert(&predicate(self, ty_var)); + match self.solver.check() { + z3::SatResult::Sat => { + self.solver.pop(1); + return *kind; + } + z3::SatResult::Unsat => { + self.solver.pop(1); + continue; + } + z3::SatResult::Unknown => panic!("Z3 cannot determine the type's kind"), + } + } + + // This would only happen if given a `TypeVar` whose kind was a + // condition code, but we only use this function for RHS operations, + // which cannot be condition codes. + panic!("cannot convert type variable's kind to `peepmatic_runtime::type::Kind`") + } +} + +impl<'a> TypingContextTrait<'a> for TypingContext<'a> { + type TypeVariable = TypeVar<'a>; + + fn cc(&mut self, span: Span) -> TypeVar<'a> { + let ty = self.new_type_var(); + self.assert_is_cc(span, &ty); + ty + } + + fn bNN(&mut self, span: Span) -> TypeVar<'a> { + if let Some(ty) = self.operation_scope.get("bNN") { + return ty.clone(); + } + + let ty = self.new_type_var(); + self.assert_is_bool(span, &ty); + self.operation_scope.insert("bNN", ty.clone()); + ty + } + + fn iNN(&mut self, span: Span) -> TypeVar<'a> { + if let Some(ty) = self.operation_scope.get("iNN") { + return ty.clone(); + } + + let ty = self.new_type_var(); + self.assert_is_integer(span, &ty); + self.operation_scope.insert("iNN", ty.clone()); + ty + } + + fn iMM(&mut self, span: Span) -> TypeVar<'a> { + if let Some(ty) = self.operation_scope.get("iMM") { + return ty.clone(); + } + + let ty = self.new_type_var(); + self.assert_is_integer(span, &ty); + self.operation_scope.insert("iMM", ty.clone()); + ty + } + + fn cpu_flags(&mut self, span: Span) -> TypeVar<'a> { + if let Some(ty) = self.operation_scope.get("cpu_flags") { + return ty.clone(); + } + + let ty = self.new_type_var(); + self.assert_is_cpu_flags(span, &ty); + self.assert_bit_width(span, &ty, 1); + self.operation_scope.insert("cpu_flags", ty.clone()); + ty + } + + fn b1(&mut self, span: Span) -> TypeVar<'a> { + let b1 = self.new_type_var(); + self.assert_is_bool(span, &b1); + self.assert_bit_width(span, &b1, 1); + b1 + } + + fn void(&mut self, span: Span) -> TypeVar<'a> { + let void = self.new_type_var(); + self.assert_is_void(span, &void); + self.assert_bit_width(span, &void, 0); + void + } + + fn bool_or_int(&mut self, span: Span) -> TypeVar<'a> { + let ty = self.new_type_var(); + let is_int = self.type_kind_sort.variants[0] + .tester + .apply(&[&ty.kind.clone().into()]) + .as_bool() + .unwrap(); + let is_bool = self.type_kind_sort.variants[1] + .tester + .apply(&[&ty.kind.clone().into()]) + .as_bool() + .unwrap(); + self.constraints.push(( + is_int.or(&[&is_bool]), + span, + Some("type error: must be either an int or a bool type".into()), + )); + ty + } + + fn any_t(&mut self, _span: Span) -> TypeVar<'a> { + if let Some(ty) = self.operation_scope.get("any_t") { + return ty.clone(); + } + + let ty = self.new_type_var(); + self.operation_scope.insert("any_t", ty.clone()); + ty + } +} + +#[derive(Clone)] +pub(crate) struct TypeVar<'a> { + kind: z3::ast::Datatype<'a>, + width: z3::ast::BV<'a>, +} + +fn verify_optimization(z3: &z3::Context, opt: &Optimization) -> VerifyResult<()> { + let mut context = TypingContext::new(z3); + collect_type_constraints(&mut context, opt)?; + context.type_check(opt.span)?; + context.assign_types()?; + + // TODO: add another pass here to check for counter-examples to this + // optimization, i.e. inputs where the LHS and RHS are not equivalent. + + Ok(()) +} + +fn collect_type_constraints<'a>( + context: &mut TypingContext<'a>, + opt: &'a Optimization<'a>, +) -> VerifyResult<()> { + use crate::traversals::TraversalEvent as TE; + + let lhs_ty = context.new_type_var(); + context.init_root_type(opt.lhs.span, lhs_ty.clone()); + + let rhs_ty = context.new_type_var(); + context.assert_type_eq( + opt.span, + &lhs_ty, + &rhs_ty, + Some("type error: the left-hand side and right-hand side must have the same type".into()), + ); + + // A stack of type variables that we are constraining as we traverse the + // AST. Operations push new type variables for their operands' expected + // types, and exiting a `Pattern` in the traversal pops them off. + let mut expected_types = vec![lhs_ty]; + + // Build up the type constraints for the left-hand side. + for (event, node) in Dfs::new(&opt.lhs) { + match (event, node) { + (TE::Enter, DynAstRef::Pattern(Pattern::Constant(Constant { id, span }))) + | (TE::Enter, DynAstRef::Pattern(Pattern::Variable(Variable { id, span }))) => { + let id = context.get_or_create_type_var_for_id(*id); + context.assert_type_eq(*span, expected_types.last().unwrap(), &id, None); + } + (TE::Enter, DynAstRef::Pattern(Pattern::ValueLiteral(ValueLiteral::Integer(i)))) => { + let ty = expected_types.last().unwrap(); + context.remember_integer_literal(i, ty.clone()); + } + (TE::Enter, DynAstRef::Pattern(Pattern::ValueLiteral(ValueLiteral::Boolean(b)))) => { + let ty = expected_types.last().unwrap(); + context.remember_boolean_literal(b, ty.clone()); + } + ( + TE::Enter, + DynAstRef::Pattern(Pattern::ValueLiteral(ValueLiteral::ConditionCode(cc))), + ) => { + let ty = expected_types.last().unwrap(); + context.assert_is_cc(cc.span, ty); + } + (TE::Enter, DynAstRef::PatternOperation(op)) => { + let result_ty; + let mut operand_types = vec![]; + { + let mut scope = context.enter_operation_scope(); + result_ty = op.operator.result_type(&mut *scope, op.span); + op.operator + .immediate_types(&mut *scope, op.span, &mut operand_types); + op.operator + .param_types(&mut *scope, op.span, &mut operand_types); + } + + if op.operands.len() != operand_types.len() { + return Err(WastError::new( + op.span, + format!( + "Expected {} operands but found {}", + operand_types.len(), + op.operands.len() + ), + ) + .into()); + } + + for imm in op + .operands + .iter() + .take(op.operator.immediates_arity() as usize) + { + match imm { + Pattern::ValueLiteral(_) | + Pattern::Constant(_) | + Pattern::Variable(_) => continue, + Pattern::Operation(op) => return Err(WastError::new( + op.span, + "operations are invalid immediates; must be a value literal, constant, \ + or variable".into() + ).into()), + } + } + + match op.operator { + Operator::Ireduce | Operator::Uextend | Operator::Sextend => { + if op.r#type.get().is_none() { + return Err(WastError::new( + op.span, + "`ireduce`, `sextend`, and `uextend` require an ascribed type, \ + like `(sextend{i64} ...)`" + .into(), + ) + .into()); + } + } + _ => {} + } + + match op.operator { + Operator::Uextend | Operator::Sextend => { + context.assert_bit_width_gt(op.span, &result_ty, &operand_types[0]); + } + Operator::Ireduce => { + context.assert_bit_width_lt(op.span, &result_ty, &operand_types[0]); + } + _ => {} + } + + if let Some(ty) = op.r#type.get() { + match ty.kind { + Kind::Bool => context.assert_is_bool(op.span, &result_ty), + Kind::Int => context.assert_is_integer(op.span, &result_ty), + Kind::Void => context.assert_is_void(op.span, &result_ty), + Kind::CpuFlags => { + unreachable!("no syntax for ascribing CPU flags types right now") + } + } + if let Some(w) = ty.bit_width.fixed_width() { + context.assert_bit_width(op.span, &result_ty, w); + } + } + + context.assert_type_eq(op.span, expected_types.last().unwrap(), &result_ty, None); + + operand_types.reverse(); + expected_types.extend(operand_types); + } + (TE::Exit, DynAstRef::Pattern(..)) => { + expected_types.pop().unwrap(); + } + (TE::Enter, DynAstRef::Precondition(pre)) => { + type_constrain_precondition(context, pre)?; + } + _ => continue, + } + } + + // We should have exited exactly as many patterns as we entered: one for the + // root pattern and the initial `lhs_ty`, and then the rest for the operands + // of pattern operations. + assert!(expected_types.is_empty()); + + // Collect the type constraints for the right-hand side. + expected_types.push(rhs_ty); + for (event, node) in Dfs::new(&opt.rhs) { + match (event, node) { + (TE::Enter, DynAstRef::Rhs(Rhs::ValueLiteral(ValueLiteral::Integer(i)))) => { + let ty = expected_types.last().unwrap(); + context.remember_integer_literal(i, ty.clone()); + } + (TE::Enter, DynAstRef::Rhs(Rhs::ValueLiteral(ValueLiteral::Boolean(b)))) => { + let ty = expected_types.last().unwrap(); + context.remember_boolean_literal(b, ty.clone()); + } + (TE::Enter, DynAstRef::Rhs(Rhs::ValueLiteral(ValueLiteral::ConditionCode(cc)))) => { + let ty = expected_types.last().unwrap(); + context.assert_is_cc(cc.span, ty); + } + (TE::Enter, DynAstRef::Rhs(Rhs::Constant(Constant { span, id }))) + | (TE::Enter, DynAstRef::Rhs(Rhs::Variable(Variable { span, id }))) => { + let id_ty = context.get_type_var_for_id(*id)?; + context.assert_type_eq(*span, expected_types.last().unwrap(), &id_ty, None); + } + (TE::Enter, DynAstRef::RhsOperation(op)) => { + let result_ty; + let mut operand_types = vec![]; + { + let mut scope = context.enter_operation_scope(); + result_ty = op.operator.result_type(&mut *scope, op.span); + op.operator + .immediate_types(&mut *scope, op.span, &mut operand_types); + op.operator + .param_types(&mut *scope, op.span, &mut operand_types); + } + + if op.operands.len() != operand_types.len() { + return Err(WastError::new( + op.span, + format!( + "Expected {} operands but found {}", + operand_types.len(), + op.operands.len() + ), + ) + .into()); + } + + for imm in op + .operands + .iter() + .take(op.operator.immediates_arity() as usize) + { + match imm { + Rhs::ValueLiteral(_) + | Rhs::Constant(_) + | Rhs::Variable(_) + | Rhs::Unquote(_) => continue, + Rhs::Operation(op) => return Err(WastError::new( + op.span, + "operations are invalid immediates; must be a value literal, unquote, \ + constant, or variable" + .into(), + ) + .into()), + } + } + + match op.operator { + Operator::Ireduce | Operator::Uextend | Operator::Sextend => { + if op.r#type.get().is_none() { + return Err(WastError::new( + op.span, + "`ireduce`, `sextend`, and `uextend` require an ascribed type, \ + like `(sextend{i64} ...)`" + .into(), + ) + .into()); + } + } + _ => {} + } + + match op.operator { + Operator::Uextend | Operator::Sextend => { + context.assert_bit_width_gt(op.span, &result_ty, &operand_types[0]); + } + Operator::Ireduce => { + context.assert_bit_width_lt(op.span, &result_ty, &operand_types[0]); + } + _ => {} + } + + if let Some(ty) = op.r#type.get() { + match ty.kind { + Kind::Bool => context.assert_is_bool(op.span, &result_ty), + Kind::Int => context.assert_is_integer(op.span, &result_ty), + Kind::Void => context.assert_is_void(op.span, &result_ty), + Kind::CpuFlags => { + unreachable!("no syntax for ascribing CPU flags types right now") + } + } + if let Some(w) = ty.bit_width.fixed_width() { + context.assert_bit_width(op.span, &result_ty, w); + } + } + + context.assert_type_eq(op.span, expected_types.last().unwrap(), &result_ty, None); + if op.r#type.get().is_none() { + context.remember_rhs_operation(op, result_ty); + } + + operand_types.reverse(); + expected_types.extend(operand_types); + } + (TE::Enter, DynAstRef::Unquote(unq)) => { + let result_ty; + let mut operand_types = vec![]; + { + let mut scope = context.enter_operation_scope(); + result_ty = unq.operator.result_type(&mut *scope, unq.span); + unq.operator + .immediate_types(&mut *scope, unq.span, &mut operand_types); + unq.operator + .param_types(&mut *scope, unq.span, &mut operand_types); + } + + if unq.operands.len() != operand_types.len() { + return Err(WastError::new( + unq.span, + format!( + "Expected {} unquote operands but found {}", + operand_types.len(), + unq.operands.len() + ), + ) + .into()); + } + + for operand in &unq.operands { + match operand { + Rhs::ValueLiteral(_) | Rhs::Constant(_) => continue, + Rhs::Variable(_) | Rhs::Unquote(_) | Rhs::Operation(_) => { + return Err(WastError::new( + operand.span(), + "unquote operands must be value literals or constants".into(), + ) + .into()); + } + } + } + + context.assert_type_eq(unq.span, expected_types.last().unwrap(), &result_ty, None); + + operand_types.reverse(); + expected_types.extend(operand_types); + } + (TE::Exit, DynAstRef::Rhs(..)) => { + expected_types.pop().unwrap(); + } + _ => continue, + } + } + + // Again, we should have popped off all the expected types when exiting + // `Rhs` nodes in the traversal. + assert!(expected_types.is_empty()); + + Ok(()) +} + +fn type_constrain_precondition<'a>( + context: &mut TypingContext<'a>, + pre: &Precondition<'a>, +) -> VerifyResult<()> { + match pre.constraint { + Constraint::BitWidth => { + if pre.operands.len() != 2 { + return Err(WastError::new( + pre.span, + format!( + "the `bit-width` precondition requires exactly 2 operands, found \ + {} operands", + pre.operands.len(), + ), + ) + .into()); + } + + let id = match pre.operands[0] { + ConstraintOperand::ValueLiteral(_) => { + return Err(anyhow::anyhow!( + "the `bit-width` precondition requires a constant or variable as \ + its first operand" + ) + .into()) + } + ConstraintOperand::Constant(Constant { id, .. }) + | ConstraintOperand::Variable(Variable { id, .. }) => id, + }; + + let width = match pre.operands[1] { + ConstraintOperand::ValueLiteral(ValueLiteral::Integer(Integer { + value, .. + })) if value == 1 + || value == 8 + || value == 16 + || value == 32 + || value == 64 + || value == 128 => + { + value as u8 + } + ref op => return Err(WastError::new( + op.span(), + "the `bit-width` precondition requires a bit width of 1, 8, 16, 32, 64, or \ + 128" + .into(), + ) + .into()), + }; + + let ty = context.get_type_var_for_id(id)?; + context.assert_bit_width(pre.span, &ty, width); + Ok(()) + } + Constraint::IsPowerOfTwo => { + if pre.operands.len() != 1 { + return Err(WastError::new( + pre.span, + format!( + "the `is-power-of-two` precondition requires exactly 1 operand, found \ + {} operands", + pre.operands.len(), + ), + ) + .into()); + } + match &pre.operands[0] { + ConstraintOperand::Constant(Constant { id, .. }) => { + let ty = context.get_type_var_for_id(*id)?; + context.assert_is_integer(pre.span(), &ty); + Ok(()) + } + op => Err(WastError::new( + op.span(), + "`is-power-of-two` operands must be constant bindings".into(), + ) + .into()), + } + } + Constraint::FitsInNativeWord => { + if pre.operands.len() != 1 { + return Err(WastError::new( + pre.span, + format!( + "the `fits-in-native-word` precondition requires exactly 1 operand, found \ + {} operands", + pre.operands.len(), + ), + ) + .into()); + } + + match pre.operands[0] { + ConstraintOperand::ValueLiteral(_) => { + return Err(anyhow::anyhow!( + "the `fits-in-native-word` precondition requires a constant or variable as \ + its first operand" + ) + .into()) + } + ConstraintOperand::Constant(Constant { id, .. }) + | ConstraintOperand::Variable(Variable { id, .. }) => { + context.get_type_var_for_id(id)?; + Ok(()) + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! verify_ok { + ($name:ident, $src:expr) => { + #[test] + fn $name() { + let buf = wast::parser::ParseBuffer::new($src).expect("should lex OK"); + let opts = match wast::parser::parse::(&buf) { + Ok(opts) => opts, + Err(mut e) => { + e.set_path(Path::new(stringify!($name))); + e.set_text($src); + eprintln!("{}", e); + panic!("should parse OK") + } + }; + match verify(&opts) { + Ok(_) => return, + Err(mut e) => { + e.set_path(Path::new(stringify!($name))); + e.set_text($src); + eprintln!("{}", e); + panic!("should verify OK") + } + } + } + }; + } + + macro_rules! verify_err { + ($name:ident, $src:expr) => { + #[test] + fn $name() { + let buf = wast::parser::ParseBuffer::new($src).expect("should lex OK"); + let opts = match wast::parser::parse::(&buf) { + Ok(opts) => opts, + Err(mut e) => { + e.set_path(Path::new(stringify!($name))); + e.set_text($src); + eprintln!("{}", e); + panic!("should parse OK") + } + }; + match verify(&opts) { + Ok(_) => panic!("expected a verification error, but it verified OK"), + Err(mut e) => { + e.set_path(Path::new(stringify!($name))); + e.set_text($src); + eprintln!("{}", e); + return; + } + } + } + }; + } + + verify_ok!(bool_0, "(=> true true)"); + verify_ok!(bool_1, "(=> false false)"); + verify_ok!(bool_2, "(=> true false)"); + verify_ok!(bool_3, "(=> false true)"); + + verify_err!(bool_is_not_int_0, "(=> true 42)"); + verify_err!(bool_is_not_int_1, "(=> 42 true)"); + + verify_ok!( + bit_width_0, + " +(=> (when (iadd $x $y) + (bit-width $x 32) + (bit-width $y 32)) + (iadd $x $y)) +" + ); + verify_err!( + bit_width_1, + " +(=> (when (iadd $x $y) + (bit-width $x 32) + (bit-width $y 64)) + (iadd $x $y)) +" + ); + verify_err!( + bit_width_2, + " +(=> (when (iconst $C) + (bit-width $C)) + 5) +" + ); + verify_err!( + bit_width_3, + " +(=> (when (iconst $C) + (bit-width 32 32)) + 5) +" + ); + verify_err!( + bit_width_4, + " +(=> (when (iconst $C) + (bit-width $C $C)) + 5) +" + ); + verify_err!( + bit_width_5, + " +(=> (when (iconst $C) + (bit-width $C2 32)) + 5) +" + ); + verify_err!( + bit_width_6, + " +(=> (when (iconst $C) + (bit-width $C2 33)) + 5) +" + ); + + verify_ok!( + is_power_of_two_0, + " +(=> (when (imul $x $C) + (is-power-of-two $C)) + (ishl $x $(log2 $C))) +" + ); + verify_err!( + is_power_of_two_1, + " +(=> (when (imul $x $C) + (is-power-of-two)) + 5) +" + ); + verify_err!( + is_power_of_two_2, + " +(=> (when (imul $x $C) + (is-power-of-two $C $C)) + 5) +" + ); + + verify_ok!(pattern_ops_0, "(=> (iadd $x $C) 5)"); + verify_err!(pattern_ops_1, "(=> (iadd $x) 5)"); + verify_err!(pattern_ops_2, "(=> (iadd $x $y $z) 5)"); + + verify_ok!(unquote_0, "(=> $C $(log2 $C))"); + verify_err!(unquote_1, "(=> (iadd $C $D) $(log2 $C $D))"); + verify_err!(unquote_2, "(=> $x $(log2))"); + verify_ok!(unquote_3, "(=> $C $(neg $C))"); + verify_err!(unquote_4, "(=> $x $(neg))"); + verify_err!(unquote_5, "(=> (iadd $x $y) $(neg $x $y))"); + verify_err!(unquote_6, "(=> $x $(neg $x))"); + + verify_ok!(rhs_0, "(=> $x (iadd $x (iconst 0)))"); + verify_err!(rhs_1, "(=> $x (iadd $x))"); + verify_err!(rhs_2, "(=> $x (iadd $x 0 0))"); + + verify_err!(no_optimizations, ""); + + verify_err!( + duplicate_left_hand_sides, + " +(=> (iadd $x $y) 0) +(=> (iadd $x $y) 1) +" + ); + verify_err!( + canonically_duplicate_left_hand_sides_0, + " +(=> (iadd $x $y) 0) +(=> (iadd $y $x) 1) +" + ); + verify_err!( + canonically_duplicate_left_hand_sides_1, + " +(=> (iadd $X $Y) 0) +(=> (iadd $Y $X) 1) +" + ); + verify_err!( + canonically_duplicate_left_hand_sides_2, + " +(=> (iadd $x $x) 0) +(=> (iadd $y $y) 1) +" + ); + + verify_ok!( + canonically_different_left_hand_sides_0, + " +(=> (iadd $x $C) 0) +(=> (iadd $C $x) 1) +" + ); + verify_ok!( + canonically_different_left_hand_sides_1, + " +(=> (iadd $x $x) 0) +(=> (iadd $x $y) 1) +" + ); + + verify_ok!( + fits_in_native_word_0, + "(=> (when (iadd $x $y) (fits-in-native-word $x)) 0)" + ); + verify_err!( + fits_in_native_word_1, + "(=> (when (iadd $x $y) (fits-in-native-word)) 0)" + ); + verify_err!( + fits_in_native_word_2, + "(=> (when (iadd $x $y) (fits-in-native-word $x $y)) 0)" + ); + verify_err!( + fits_in_native_word_3, + "(=> (when (iadd $x $y) (fits-in-native-word true)) 0)" + ); + + verify_err!(reduce_extend_0, "(=> (sextend (ireduce -1)) 0)"); + verify_err!(reduce_extend_1, "(=> (uextend (ireduce -1)) 0)"); + verify_ok!(reduce_extend_2, "(=> (sextend{i64} (ireduce{i32} -1)) 0)"); + verify_ok!(reduce_extend_3, "(=> (uextend{i64} (ireduce{i32} -1)) 0)"); + verify_err!(reduce_extend_4, "(=> (sextend{i64} (ireduce{i64} -1)) 0)"); + verify_err!(reduce_extend_5, "(=> (uextend{i64} (ireduce{i64} -1)) 0)"); + verify_err!(reduce_extend_6, "(=> (sextend{i32} (ireduce{i64} -1)) 0)"); + verify_err!(reduce_extend_7, "(=> (uextend{i32} (ireduce{i64} -1)) 0)"); + + verify_err!( + using_an_operation_as_an_immediate_in_lhs, + "(=> (iadd_imm (imul $x $y) $z) 0)" + ); + verify_err!( + using_an_operation_as_an_immediate_in_rhs, + "(=> (iadd (imul $x $y) $z) (iadd_imm (imul $x $y) $z))" + ); + + verify_err!( + using_a_condition_code_as_the_root_of_an_optimization, + "(=> eq eq)" + ); +} From 2828da1f5603e226345c82ec540d01d518d1572f Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 1 May 2020 15:45:01 -0700 Subject: [PATCH 05/22] peepmatic: Introduce the `peepmatic-test` crate This crate provides testing utilities for `peepmatic`, and a test-only instruction set we can use to check that various optimizations do or don't apply. --- cranelift/peepmatic/crates/test/Cargo.toml | 14 + cranelift/peepmatic/crates/test/src/lib.rs | 531 ++++++++++++++++++ .../peepmatic/crates/test/tests/tests.rs | 393 +++++++++++++ 3 files changed, 938 insertions(+) create mode 100644 cranelift/peepmatic/crates/test/Cargo.toml create mode 100644 cranelift/peepmatic/crates/test/src/lib.rs create mode 100644 cranelift/peepmatic/crates/test/tests/tests.rs diff --git a/cranelift/peepmatic/crates/test/Cargo.toml b/cranelift/peepmatic/crates/test/Cargo.toml new file mode 100644 index 0000000000..bca85f8bf8 --- /dev/null +++ b/cranelift/peepmatic/crates/test/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "peepmatic-test" +version = "0.1.0" +authors = ["Nick Fitzgerald "] +edition = "2018" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +env_logger = "0.7.1" +log = "0.4.8" +peepmatic = { path = "../.." } +peepmatic-runtime = { path = "../runtime" } diff --git a/cranelift/peepmatic/crates/test/src/lib.rs b/cranelift/peepmatic/crates/test/src/lib.rs new file mode 100644 index 0000000000..c43ce89957 --- /dev/null +++ b/cranelift/peepmatic/crates/test/src/lib.rs @@ -0,0 +1,531 @@ +//! Testing utilities and a testing-only instruction set for `peepmatic`. + +#![deny(missing_debug_implementations)] + +use peepmatic_runtime::{ + cc::ConditionCode, + instruction_set::InstructionSet, + operator::Operator, + part::{Constant, Part}, + paths::Path, + r#type::{BitWidth, Kind, Type}, +}; +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::convert::TryFrom; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct Instruction(pub usize); + +#[derive(Debug)] +pub struct InstructionData { + pub operator: Operator, + pub r#type: Type, + pub immediates: Vec, + pub arguments: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Immediate { + Constant(Constant), + ConditionCode(ConditionCode), +} + +impl Immediate { + fn unwrap_constant(&self) -> Constant { + match *self { + Immediate::Constant(c) => c, + _ => panic!("not a constant"), + } + } +} + +impl From for Immediate { + fn from(c: Constant) -> Immediate { + Immediate::Constant(c) + } +} + +impl From for Immediate { + fn from(cc: ConditionCode) -> Immediate { + Immediate::ConditionCode(cc) + } +} + +impl From for Part { + fn from(imm: Immediate) -> Part { + match imm { + Immediate::Constant(c) => Part::Constant(c), + Immediate::ConditionCode(cc) => Part::ConditionCode(cc), + } + } +} + +impl TryFrom> for Immediate { + type Error = &'static str; + + fn try_from(part: Part) -> Result { + match part { + Part::Constant(c) => Ok(Immediate::Constant(c)), + Part::ConditionCode(c) => Ok(Immediate::ConditionCode(c)), + Part::Instruction(_) => Err("instruction parts cannot be converted into immediates"), + } + } +} + +#[derive(Debug, Default)] +pub struct Program { + instr_counter: usize, + instruction_data: BTreeMap, + replacements: RefCell>, +} + +impl Program { + /// Are `a` and `b` structurally equivalent, even if they use different + /// `Instruction`s for various arguments? + pub fn structurally_eq(&mut self, a: Instruction, b: Instruction) -> bool { + macro_rules! ensure_eq { + ($a:expr, $b:expr) => {{ + let a = &$a; + let b = &$b; + if a != b { + log::debug!( + "{} != {} ({:?} != {:?})", + stringify!($a), + stringify!($b), + a, + b + ); + return false; + } + }}; + } + + let a = self.resolve(a); + let b = self.resolve(b); + if a == b { + return true; + } + + let a = self.data(a); + let b = self.data(b); + log::debug!("structurally_eq({:?}, {:?})", a, b); + + ensure_eq!(a.operator, b.operator); + ensure_eq!(a.r#type, b.r#type); + ensure_eq!(a.immediates, b.immediates); + ensure_eq!(a.arguments.len(), b.arguments.len()); + a.arguments + .clone() + .into_iter() + .zip(b.arguments.clone().into_iter()) + .all(|(a, b)| self.structurally_eq(a, b)) + } + + pub fn instructions(&self) -> impl Iterator { + self.instruction_data.iter().map(|(k, v)| (*k, v)) + } + + pub fn replace_instruction(&mut self, old: Instruction, new: Instruction) { + log::debug!("replacing {:?} with {:?}", old, new); + + let old = self.resolve(old); + let new = self.resolve(new); + if old == new { + return; + } + + let mut replacements = self.replacements.borrow_mut(); + let existing_replacement = replacements.insert(old, new); + assert!(existing_replacement.is_none()); + + let old_data = self.instruction_data.remove(&old); + assert!(old_data.is_some()); + } + + pub fn resolve(&self, inst: Instruction) -> Instruction { + let mut replacements = self.replacements.borrow_mut(); + let mut replacements_followed = 0; + let mut resolved = inst; + while let Some(i) = replacements.get(&resolved).cloned() { + log::trace!("resolving replaced instruction: {:?} -> {:?}", resolved, i); + replacements_followed += 1; + assert!( + replacements_followed <= replacements.len(), + "cyclic replacements" + ); + + resolved = i; + continue; + } + + if inst != resolved { + let old_replacement = replacements.insert(inst, resolved); + assert!(old_replacement.is_some()); + } + + resolved + } + + pub fn data(&self, inst: Instruction) -> &InstructionData { + let inst = self.resolve(inst); + &self.instruction_data[&inst] + } + + pub fn new_instruction( + &mut self, + operator: Operator, + r#type: Type, + immediates: Vec, + arguments: Vec, + ) -> Instruction { + assert_eq!( + operator.immediates_arity() as usize, + immediates.len(), + "wrong number of immediates for {:?}: expected {}, found {}", + operator, + operator.immediates_arity(), + immediates.len(), + ); + assert_eq!( + operator.params_arity() as usize, + arguments.len(), + "wrong number of arguments for {:?}: expected {}, found {}", + operator, + operator.params_arity(), + arguments.len(), + ); + + assert!(!r#type.bit_width.is_polymorphic()); + assert!(immediates.iter().all(|imm| match imm { + Immediate::Constant(Constant::Bool(_, w)) + | Immediate::Constant(Constant::Int(_, w)) => !w.is_polymorphic(), + Immediate::ConditionCode(_) => true, + })); + + let inst = Instruction(self.instr_counter); + self.instr_counter += 1; + + let data = InstructionData { + operator, + r#type, + immediates, + arguments, + }; + + log::trace!("new instruction: {:?} = {:?}", inst, data); + self.instruction_data.insert(inst, data); + inst + } + + pub fn r#const(&mut self, c: Constant, root_bit_width: BitWidth) -> Instruction { + assert!(!root_bit_width.is_polymorphic()); + match c { + Constant::Bool(_, bit_width) => self.new_instruction( + Operator::Bconst, + if bit_width.is_polymorphic() { + Type { + kind: Kind::Bool, + bit_width: root_bit_width, + } + } else { + Type { + kind: Kind::Bool, + bit_width, + } + }, + vec![c.into()], + vec![], + ), + Constant::Int(_, bit_width) => self.new_instruction( + Operator::Iconst, + if bit_width.is_polymorphic() { + Type { + kind: Kind::Int, + bit_width: root_bit_width, + } + } else { + Type { + kind: Kind::Int, + bit_width, + } + }, + vec![c.into()], + vec![], + ), + } + } + + fn instruction_to_constant(&mut self, inst: Instruction) -> Option { + match self.data(inst) { + InstructionData { + operator: Operator::Iconst, + immediates, + .. + } => Some(immediates[0].unwrap_constant()), + InstructionData { + operator: Operator::Bconst, + immediates, + .. + } => Some(immediates[0].unwrap_constant()), + _ => None, + } + } + + fn part_to_immediate(&mut self, part: Part) -> Result { + match part { + Part::Instruction(i) => self + .instruction_to_constant(i) + .map(|c| c.into()) + .ok_or("non-constant instructions cannot be converted into immediates"), + Part::Constant(c) => Ok(c.into()), + Part::ConditionCode(cc) => Ok(Immediate::ConditionCode(cc)), + } + } + + fn part_to_instruction( + &mut self, + root: Instruction, + part: Part, + ) -> Result { + match part { + Part::Instruction(inst) => { + let inst = self.resolve(inst); + Ok(inst) + } + Part::Constant(c) => { + let root_width = self.data(root).r#type.bit_width; + Ok(self.r#const(c, root_width)) + } + Part::ConditionCode(_) => Err("condition codes cannot be converted into instructions"), + } + } +} + +#[derive(Debug)] +pub struct TestIsa { + pub native_word_size_in_bits: u8, +} + +impl<'a> InstructionSet<'a> for TestIsa { + type Context = Program; + + type Instruction = Instruction; + + fn replace_instruction( + &self, + program: &mut Program, + old: Instruction, + new: Part, + ) -> Instruction { + log::debug!("replace_instruction({:?}, {:?})", old, new); + let new = program.part_to_instruction(old, new).unwrap(); + program.replace_instruction(old, new); + new + } + + fn get_part_at_path( + &self, + program: &mut Program, + root: Instruction, + path: Path, + ) -> Option> { + log::debug!("get_part_at_path({:?})", path); + + assert!(!path.0.is_empty()); + assert_eq!(path.0[0], 0); + + let mut part = Part::Instruction(root); + for p in &path.0[1..] { + if let Part::Instruction(inst) = part { + let data = program.data(inst); + let p = *p as usize; + + if p < data.immediates.len() { + part = data.immediates[p].into(); + continue; + } + + if let Some(inst) = data.arguments.get(p - data.immediates.len()).copied() { + part = Part::Instruction(inst); + continue; + } + } + + return None; + } + + Some(part) + } + + fn operator(&self, program: &mut Program, instr: Instruction) -> Option { + log::debug!("operator({:?})", instr); + let data = program.data(instr); + Some(data.operator) + } + + fn make_inst_1( + &self, + program: &mut Program, + root: Instruction, + operator: Operator, + r#type: Type, + a: Part, + ) -> Instruction { + log::debug!( + "make_inst_1(\n\toperator = {:?},\n\ttype = {},\n\ta = {:?},\n)", + operator, + r#type, + a, + ); + + let (imms, args) = match operator.immediates_arity() { + 0 => { + assert_eq!(operator.params_arity(), 1); + (vec![], vec![program.part_to_instruction(root, a).unwrap()]) + } + 1 => { + assert_eq!(operator.params_arity(), 0); + (vec![program.part_to_immediate(a).unwrap()], vec![]) + } + _ => unreachable!(), + }; + program.new_instruction(operator, r#type, imms, args) + } + + fn make_inst_2( + &self, + program: &mut Program, + root: Instruction, + operator: Operator, + r#type: Type, + a: Part, + b: Part, + ) -> Instruction { + log::debug!( + "make_inst_2(\n\toperator = {:?},\n\ttype = {},\n\ta = {:?},\n\tb = {:?},\n)", + operator, + r#type, + a, + b, + ); + + let (imms, args) = match operator.immediates_arity() { + 0 => { + assert_eq!(operator.params_arity(), 2); + ( + vec![], + vec![ + program.part_to_instruction(root, a).unwrap(), + program.part_to_instruction(root, b).unwrap(), + ], + ) + } + 1 => { + assert_eq!(operator.params_arity(), 1); + ( + vec![program.part_to_immediate(a).unwrap()], + vec![program.part_to_instruction(root, b).unwrap()], + ) + } + 2 => { + assert_eq!(operator.params_arity(), 0); + ( + vec![ + program.part_to_immediate(a).unwrap(), + program.part_to_immediate(b).unwrap(), + ], + vec![], + ) + } + _ => unreachable!(), + }; + program.new_instruction(operator, r#type, imms, args) + } + + fn make_inst_3( + &self, + program: &mut Program, + root: Instruction, + operator: Operator, + r#type: Type, + a: Part, + b: Part, + c: Part, + ) -> Instruction { + log::debug!( + "make_inst_3(\n\toperator = {:?},\n\ttype = {},\n\ta = {:?},\n\tb = {:?},\n\tc = {:?},\n)", + operator, + r#type, + a, + b, + c, + ); + let (imms, args) = match operator.immediates_arity() { + 0 => { + assert_eq!(operator.params_arity(), 3); + ( + vec![], + vec![ + program.part_to_instruction(root, a).unwrap(), + program.part_to_instruction(root, b).unwrap(), + program.part_to_instruction(root, c).unwrap(), + ], + ) + } + 1 => { + assert_eq!(operator.params_arity(), 2); + ( + vec![program.part_to_immediate(a).unwrap()], + vec![ + program.part_to_instruction(root, b).unwrap(), + program.part_to_instruction(root, c).unwrap(), + ], + ) + } + 2 => { + assert_eq!(operator.params_arity(), 1); + ( + vec![ + program.part_to_immediate(a).unwrap(), + program.part_to_immediate(b).unwrap(), + ], + vec![program.part_to_instruction(root, c).unwrap()], + ) + } + 3 => { + assert_eq!(operator.params_arity(), 0); + ( + vec![ + program.part_to_immediate(a).unwrap(), + program.part_to_immediate(b).unwrap(), + program.part_to_immediate(c).unwrap(), + ], + vec![], + ) + } + _ => unreachable!(), + }; + program.new_instruction(operator, r#type, imms, args) + } + + fn instruction_to_constant( + &self, + program: &mut Program, + inst: Instruction, + ) -> Option { + log::debug!("instruction_to_constant({:?})", inst); + program.instruction_to_constant(inst) + } + + fn instruction_result_bit_width(&self, program: &mut Program, inst: Instruction) -> u8 { + log::debug!("instruction_result_bit_width({:?})", inst); + let ty = program.data(inst).r#type; + ty.bit_width.fixed_width().unwrap() + } + + fn native_word_size_in_bits(&self, _program: &mut Program) -> u8 { + log::debug!("native_word_size_in_bits"); + self.native_word_size_in_bits + } +} diff --git a/cranelift/peepmatic/crates/test/tests/tests.rs b/cranelift/peepmatic/crates/test/tests/tests.rs new file mode 100644 index 0000000000..989424adb8 --- /dev/null +++ b/cranelift/peepmatic/crates/test/tests/tests.rs @@ -0,0 +1,393 @@ +use peepmatic_runtime::{ + cc::ConditionCode, + operator::Operator, + part::Constant, + r#type::{BitWidth, Type}, +}; +use peepmatic_test::*; + +const TEST_ISA: TestIsa = TestIsa { + native_word_size_in_bits: 32, +}; + +macro_rules! optimizer { + ($opts:ident, $source:expr) => {{ + let _ = env_logger::try_init(); + $opts = peepmatic::compile_str($source, std::path::Path::new("peepmatic-test")).unwrap(); + $opts.optimizer(TEST_ISA) + }}; +} + +#[test] +fn opcode() { + let opts; + let mut optimizer = optimizer!(opts, "(=> (iadd $x 0) $x)"); + + let mut program = Program::default(); + let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let zero = program.r#const(Constant::Int(0, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let add = program.new_instruction(Operator::Iadd, Type::i32(), vec![], vec![five, zero]); + + let new = optimizer.apply_one(&mut program, add); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, five)); + + let add = program.new_instruction(Operator::Iadd, Type::i32(), vec![], vec![five, five]); + let replacement = optimizer.apply_one(&mut program, add); + assert!(replacement.is_none()); +} + +#[test] +fn constant() { + let opts; + let mut optimizer = optimizer!(opts, "(=> (iadd $C $x) (iadd_imm $C $x))"); + + let mut program = Program::default(); + let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let zero = program.r#const(Constant::Int(0, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let add = program.new_instruction(Operator::Iadd, Type::i32(), vec![], vec![five, zero]); + + let expected = program.new_instruction( + Operator::IaddImm, + Type::i32(), + vec![Constant::Int(5, BitWidth::ThirtyTwo).into()], + vec![zero], + ); + + let new = optimizer.apply_one(&mut program, add); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, expected)); + + let mul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, zero]); + let add = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![mul, five]); + let replacement = optimizer.apply_one(&mut program, add); + assert!(replacement.is_none()); +} + +#[test] +fn boolean() { + let opts; + let mut optimizer = optimizer!(opts, "(=> (bint true) 1)"); + + let mut program = Program::default(); + let t = program.r#const(Constant::Bool(true, BitWidth::One), BitWidth::One); + let bint = program.new_instruction(Operator::Bint, Type::i1(), vec![], vec![t]); + let one = program.r#const(Constant::Int(1, BitWidth::One), BitWidth::ThirtyTwo); + + let new = optimizer.apply_one(&mut program, bint); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, one)); + + let f = program.r#const(Constant::Bool(false, BitWidth::One), BitWidth::One); + let bint = program.new_instruction(Operator::Bint, Type::i1(), vec![], vec![f]); + let replacement = optimizer.apply_one(&mut program, bint); + assert!(replacement.is_none()); +} + +#[test] +fn condition_codes() { + let opts; + let mut optimizer = optimizer!(opts, "(=> (icmp eq $x $x) true)"); + + let mut program = Program::default(); + let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::One); + let icmp_eq = program.new_instruction( + Operator::Icmp, + Type::b1(), + vec![ConditionCode::Eq.into()], + vec![five, five], + ); + let t = program.r#const(Constant::Bool(true, BitWidth::One), BitWidth::One); + + let new = optimizer.apply_one(&mut program, icmp_eq); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, t)); + + let icmp_ne = program.new_instruction( + Operator::Icmp, + Type::b1(), + vec![ConditionCode::Ne.into()], + vec![five, five], + ); + let replacement = optimizer.apply_one(&mut program, icmp_ne); + assert!(replacement.is_none()); +} + +#[test] +fn is_power_of_two() { + let opts; + let mut optimizer = optimizer!( + opts, + " +(=> (when (imul $x $C) + (is-power-of-two $C)) + (ishl $x $(log2 $C))) +" + ); + + let mut program = Program::default(); + let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let two = program.r#const(Constant::Int(2, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, two]); + + let one = program.r#const(Constant::Int(1, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let ishl = program.new_instruction(Operator::Ishl, Type::i32(), vec![], vec![five, one]); + + let new = optimizer.apply_one(&mut program, imul); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, ishl)); + + let three = program.r#const(Constant::Int(3, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, three]); + + let replacement = optimizer.apply_one(&mut program, imul); + assert!(replacement.is_none()); +} + +#[test] +fn bit_width() { + let opts; + let mut optimizer = optimizer!( + opts, + " +(=> (when (imul $C $x) + (bit-width $C 32)) + (imul_imm $C $x)) +" + ); + + let mut program = Program::default(); + let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let two = program.r#const(Constant::Int(2, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, two]); + + let imul_imm = program.new_instruction( + Operator::ImulImm, + Type::i32(), + vec![Constant::Int(5, BitWidth::ThirtyTwo).into()], + vec![two], + ); + + let new = optimizer.apply_one(&mut program, imul); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, imul_imm)); + + let five = program.r#const(Constant::Int(5, BitWidth::SixtyFour), BitWidth::SixtyFour); + let two = program.r#const(Constant::Int(2, BitWidth::SixtyFour), BitWidth::SixtyFour); + let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, two]); + + let replacement = optimizer.apply_one(&mut program, imul); + assert!(replacement.is_none()); +} + +#[test] +fn fits_in_native_word() { + let opts; + let mut optimizer = optimizer!( + opts, + " +(=> (when (imul $C $x) + (fits-in-native-word $C)) + (imul_imm $C $x)) +" + ); + + let mut program = Program::default(); + let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let two = program.r#const(Constant::Int(2, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, two]); + + let imul_imm = program.new_instruction( + Operator::ImulImm, + Type::i32(), + vec![Constant::Int(5, BitWidth::ThirtyTwo).into()], + vec![two], + ); + + let new = optimizer.apply_one(&mut program, imul); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, imul_imm)); + + let five = program.r#const(Constant::Int(5, BitWidth::SixtyFour), BitWidth::SixtyFour); + let two = program.r#const(Constant::Int(2, BitWidth::SixtyFour), BitWidth::SixtyFour); + let imul = program.new_instruction(Operator::Imul, Type::i64(), vec![], vec![five, two]); + + let replacement = optimizer.apply_one(&mut program, imul); + assert!(replacement.is_none()); +} + +#[test] +fn unquote_neg() { + let opts; + let mut optimizer = optimizer!( + opts, + " +(=> (isub $x $C) + (iadd_imm $(neg $C) $x)) +" + ); + + let mut program = Program::default(); + let five = program.r#const(Constant::Int(5, BitWidth::SixtyFour), BitWidth::SixtyFour); + let two = program.r#const(Constant::Int(2, BitWidth::SixtyFour), BitWidth::SixtyFour); + let isub = program.new_instruction(Operator::Isub, Type::i64(), vec![], vec![five, two]); + + let iadd_imm = program.new_instruction( + Operator::IaddImm, + Type::i64(), + vec![Constant::Int(-2 as _, BitWidth::SixtyFour).into()], + vec![five], + ); + + let new = optimizer.apply_one(&mut program, isub); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, iadd_imm)); +} + +#[test] +fn subsumption() { + let opts; + let mut optimizer = optimizer!( + opts, + " +;; NB: the following optimizations are ordered from least to most general, so +;; the first applicable optimization should be the one that is applied. + +(=> (iadd (iadd (iadd $w $x) $y) $z) + (iadd (iadd $w $x) (iadd $y $z))) + +(=> (iadd $C1 $C2) + $(iadd $C1 $C2)) + +(=> (iadd $C $x) + (iadd_imm $C $x)) + +(=> (iadd $x $x) + (ishl_imm 1 $x)) +" + ); + + let mut program = Program::default(); + + let w = program.r#const(Constant::Int(11, BitWidth::SixtyFour), BitWidth::SixtyFour); + let x = program.r#const(Constant::Int(22, BitWidth::SixtyFour), BitWidth::SixtyFour); + let y = program.r#const(Constant::Int(33, BitWidth::SixtyFour), BitWidth::SixtyFour); + let z = program.r#const(Constant::Int(44, BitWidth::SixtyFour), BitWidth::SixtyFour); + + log::debug!("(iadd (iadd (iadd w x) y) z) => (iadd (iadd w x) (iadd y z))"); + + let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![w, x]); + let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![iadd, y]); + let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![iadd, z]); + let expected_lhs = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![w, x]); + let expected_rhs = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![y, z]); + let expected = program.new_instruction( + Operator::Iadd, + Type::i64(), + vec![], + vec![expected_lhs, expected_rhs], + ); + + let new = optimizer.apply_one(&mut program, iadd); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, expected)); + + log::debug!("(iadd w x) => y"); + + let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![w, x]); + let new = optimizer.apply_one(&mut program, iadd); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, y)); + + log::debug!("(iadd x (iadd y z)) => (iadd_imm x (iadd y z))"); + + let iadd_y_z = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![y, z]); + let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![x, iadd_y_z]); + let iadd_imm = program.new_instruction( + Operator::IaddImm, + Type::i64(), + vec![Constant::Int(22, BitWidth::SixtyFour).into()], + vec![iadd_y_z], + ); + let new = optimizer.apply_one(&mut program, iadd); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, iadd_imm)); + + log::debug!("(iadd (imul_imm x 1) (imul_imm x 1)) => (ishl_imm 1 (imul_imm x 1))"); + + let imul_imm = program.new_instruction( + Operator::ImulImm, + Type::i64(), + vec![Constant::Int(1, BitWidth::SixtyFour).into()], + vec![x], + ); + let iadd = program.new_instruction( + Operator::Iadd, + Type::i64(), + vec![], + vec![imul_imm, imul_imm], + ); + let ishl_imm = program.new_instruction( + Operator::IshlImm, + Type::i64(), + vec![Constant::Int(1, BitWidth::SixtyFour).into()], + vec![imul_imm], + ); + let new = optimizer.apply_one(&mut program, iadd); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, ishl_imm)); + + log::debug!("(iadd (imul w x) (imul y z)) does not match any optimization."); + + let imul_w_x = program.new_instruction(Operator::Imul, Type::i64(), vec![], vec![w, x]); + let imul_y_z = program.new_instruction(Operator::Imul, Type::i64(), vec![], vec![y, z]); + let iadd = program.new_instruction( + Operator::Iadd, + Type::i64(), + vec![], + vec![imul_w_x, imul_y_z], + ); + + let replacement = optimizer.apply_one(&mut program, iadd); + assert!(replacement.is_none()); +} + +#[test] +fn polymorphic_bit_widths() { + let opts; + let mut optimizer = optimizer!(opts, "(=> (iadd $C $x) (iadd_imm $C $x))"); + + let mut program = Program::default(); + + // Applies to 32 bit adds. + + let x = program.r#const(Constant::Int(42, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let y = program.r#const(Constant::Int(420, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo); + let iadd = program.new_instruction(Operator::Iadd, Type::i32(), vec![], vec![x, y]); + let iadd_imm = program.new_instruction( + Operator::IaddImm, + Type::i32(), + vec![Constant::Int(42, BitWidth::ThirtyTwo).into()], + vec![y], + ); + + let new = optimizer.apply_one(&mut program, iadd); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, iadd_imm)); + + // Applies to 16 bit adds. + + let x = program.r#const(Constant::Int(42, BitWidth::Sixteen), BitWidth::Sixteen); + let y = program.r#const(Constant::Int(420, BitWidth::Sixteen), BitWidth::Sixteen); + let iadd = program.new_instruction(Operator::Iadd, Type::i16(), vec![], vec![x, y]); + let iadd_imm = program.new_instruction( + Operator::IaddImm, + Type::i16(), + vec![Constant::Int(42, BitWidth::Sixteen).into()], + vec![y], + ); + + let new = optimizer.apply_one(&mut program, iadd); + let new = new.expect("optimization should have applied"); + assert!(program.structurally_eq(new, iadd_imm)); +} From 1a7670f964afe763c285110d544ccdadd083136f Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 1 May 2020 15:47:47 -0700 Subject: [PATCH 06/22] peepmatic: Introduce the `peepmatic-fuzzing` crate This crate contains oracles, generators, and fuzz targets for use with fuzzing engines (e.g. libFuzzer). This doesn't contain the actual `libfuzzer_sys::fuzz_target!` definitions (those are in the `peepmatic-fuzz` crate) but does those definitions are one liners calling out to functions defined in this crate. --- cranelift/peepmatic/crates/fuzzing/Cargo.toml | 22 ++ .../peepmatic/crates/fuzzing/src/automata.rs | 187 +++++++++ .../peepmatic/crates/fuzzing/src/compile.rs | 71 ++++ .../peepmatic/crates/fuzzing/src/interp.rs | 374 ++++++++++++++++++ cranelift/peepmatic/crates/fuzzing/src/lib.rs | 119 ++++++ .../peepmatic/crates/fuzzing/src/parser.rs | 29 ++ 6 files changed, 802 insertions(+) create mode 100644 cranelift/peepmatic/crates/fuzzing/Cargo.toml create mode 100644 cranelift/peepmatic/crates/fuzzing/src/automata.rs create mode 100644 cranelift/peepmatic/crates/fuzzing/src/compile.rs create mode 100644 cranelift/peepmatic/crates/fuzzing/src/interp.rs create mode 100644 cranelift/peepmatic/crates/fuzzing/src/lib.rs create mode 100644 cranelift/peepmatic/crates/fuzzing/src/parser.rs diff --git a/cranelift/peepmatic/crates/fuzzing/Cargo.toml b/cranelift/peepmatic/crates/fuzzing/Cargo.toml new file mode 100644 index 0000000000..5e36dcc6af --- /dev/null +++ b/cranelift/peepmatic/crates/fuzzing/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "peepmatic-fuzzing" +version = "0.1.0" +authors = ["Nick Fitzgerald "] +edition = "2018" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +arbitrary = { version = "0.4.1", features = ["derive"] } +bincode = "1.2.1" +env_logger = "0.7.1" +fst = "0.4.1" +log = "0.4.8" +peepmatic = { path = "../.." } +peepmatic-automata = { path = "../automata", features = ["serde"] } +peepmatic-runtime = { path = "../runtime", features = ["construct"] } +peepmatic-test = { path = "../test" } +rand = { version = "0.7.3", features = ["small_rng"] } +serde = "1.0.106" +wast = "13.0.0" diff --git a/cranelift/peepmatic/crates/fuzzing/src/automata.rs b/cranelift/peepmatic/crates/fuzzing/src/automata.rs new file mode 100644 index 0000000000..c399da037f --- /dev/null +++ b/cranelift/peepmatic/crates/fuzzing/src/automata.rs @@ -0,0 +1,187 @@ +//! Helpers for fuzzing the `peepmatic-automata` crate. + +use peepmatic_automata::{Automaton, Builder, Output}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; + +fn serde_roundtrip( + automata: Automaton, +) -> Automaton +where + TAlphabet: Serialize + for<'de> Deserialize<'de> + Clone + Eq + Hash + Ord, + TState: Serialize + for<'de> Deserialize<'de> + Clone + Eq + Hash, + TOutput: Serialize + for<'de> Deserialize<'de> + Output, +{ + let encoded: Vec = bincode::serialize(&automata).expect("should serialize OK"); + bincode::deserialize(&encoded).expect("should deserialize OK") +} + +/// Construct an automaton from the the given input-output pairs, and assert +/// that: +/// +/// * Putting in each of the input strings should result in the expected output +/// string. +/// +/// * Putting in an input string that is not one of the given inputs from our +/// input-output pairs should never yield an output value. +pub fn simple_automata(input_output_pairs: Vec)>>) { + let _ = env_logger::try_init(); + + let full_input = |pair: &[(u8, Vec)]| { + let mut full_input = vec![]; + for (input, _) in pair { + full_input.push(*input); + } + full_input + }; + + let mut inputs = HashSet::new(); + + let mut input_output_pairs: Vec<_> = input_output_pairs + .into_iter() + .filter(|pair| { + !pair.is_empty() && { + // Make sure we don't have duplicate inputs. + let is_new = inputs.insert(full_input(pair)); + is_new + } + }) + .collect(); + + input_output_pairs.sort_by(|a, b| full_input(a).cmp(&full_input(b))); + + if input_output_pairs.is_empty() { + return; + } + + // A map from one of our concatenated input strings to its concatenated + // output. + let mut expected = HashMap::with_capacity(input_output_pairs.len()); + + let mut builder = Builder::>::new(); + for pair in &input_output_pairs { + let mut full_input = vec![]; + let mut full_output = vec![]; + + let mut ins = builder.insert(); + for (input, output) in pair.iter().cloned() { + full_input.push(input); + full_output.extend(output.iter().copied()); + + ins.next(input, output); + } + + let old = expected.insert(full_input, full_output); + assert!(old.is_none()); + + ins.finish(); + } + + let automata = builder.finish(); + let automata = serde_roundtrip(automata); + + // Assert that each of our input strings yields the expected output. + for (input, expected_output) in &expected { + log::debug!("Testing input: {:?}", input); + let actual_output = automata.get(input); + assert!(actual_output.is_some()); + assert_eq!(actual_output.as_ref().unwrap(), expected_output); + } + + // Test that mutations of our input strings (that aren't themselves other + // input strings!) do not yeild any output. + for input in expected.keys() { + for i in 0..input.len() { + let mut mutated = input.clone(); + mutated[i] = mutated[i].wrapping_add(1); + log::debug!("Testing mutated input: {:?}", mutated); + if !expected.contains_key(&mutated) { + assert!(automata.get(&mutated).is_none()); + } + } + } +} + +/// Do differential testing against the `fst` crate, which is another +/// implementation of the algorithm we use for finite-state transducer +/// construction in `peepmatic-automata`. +pub fn fst_differential(map: HashMap, u64>) { + let _ = env_logger::try_init(); + + let mut inputs: Vec<_> = map.keys().filter(|i| !i.is_empty()).cloned().collect(); + inputs.sort(); + inputs.dedup(); + if inputs.is_empty() { + return; + } + + let mut fst = fst::MapBuilder::memory(); + let mut builder = Builder::::new(); + + for inp in &inputs { + fst.insert(inp, map[inp]).unwrap(); + + let mut ins = builder.insert(); + for (i, ch) in inp.iter().enumerate() { + ins.next(*ch, if i == 0 { map[inp] } else { 0 }); + } + ins.finish(); + } + + let fst = fst.into_map(); + let automata = builder.finish(); + let automata = serde_roundtrip(automata); + + for inp in inputs { + // Check we have the same result as `fst` for inputs we know are in the + // automata. + log::debug!("Testing input {:?}", inp); + let expected = fst.get(&inp).expect("`fst` should have entry for `inp`"); + let actual = automata + .get(&inp) + .expect("automata should have entry for `inp`"); + assert_eq!(expected, actual); + + // Check that we have the same result as `fst` for inputs that may or + // may not be in the automata. + for i in 0..inp.len() { + let mut mutated = inp.clone(); + mutated[i] = mutated[i].wrapping_add(1); + log::debug!("Testing mutated input {:?}", mutated); + let expected = fst.get(&mutated); + let actual = automata.get(&mutated); + assert_eq!(expected, actual); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn check_simple_automata() { + crate::check(simple_automata); + } + + #[test] + fn check_fst_differential() { + crate::check(fst_differential); + } + + #[test] + fn regression_test_0() { + simple_automata(vec![vec![(0, vec![0]), (0, vec![1])], vec![(0, vec![2])]]); + } + + #[test] + fn regression_test_1() { + fst_differential(vec![(vec![1, 3], 5), (vec![1, 2], 4)].into_iter().collect()); + } + + #[test] + fn regression_test_2() { + simple_automata(vec![vec![(0, vec![11]), (0, vec![])], vec![(0, vec![11])]]); + } +} diff --git a/cranelift/peepmatic/crates/fuzzing/src/compile.rs b/cranelift/peepmatic/crates/fuzzing/src/compile.rs new file mode 100644 index 0000000000..8df8a685c5 --- /dev/null +++ b/cranelift/peepmatic/crates/fuzzing/src/compile.rs @@ -0,0 +1,71 @@ +//! Fuzz testing utilities related to AST pattern matching. + +use peepmatic_runtime::PeepholeOptimizations; +use std::path::Path; +use std::str; + +/// Attempt to interpret the given bytes as UTF-8 and then compile them as if +/// they were source text of our DSL. +pub fn compile(data: &[u8]) { + let source = match str::from_utf8(data) { + Err(_) => return, + Ok(s) => s, + }; + + let opt = match peepmatic::compile_str(source, Path::new("fuzz")) { + Err(_) => return, + Ok(o) => o, + }; + + // Should be able to serialize and deserialize the peephole optimizer. + let opt_bytes = bincode::serialize(&opt).expect("should serialize peephole optimizations OK"); + let _: PeepholeOptimizations = + bincode::deserialize(&opt_bytes).expect("should deserialize peephole optimizations OK"); + + // Compiling the same source text again should be deterministic. + let opt2 = peepmatic::compile_str(source, Path::new("fuzz")) + .expect("should be able to compile source text again, if it compiled OK the first time"); + let opt2_bytes = + bincode::serialize(&opt2).expect("should serialize second peephole optimizations OK"); + assert_eq!(opt_bytes, opt2_bytes); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn check_compile() { + crate::check(|s: String| compile(s.as_bytes())); + } + + #[test] + fn regression_0() { + compile( + b" + (=> (bor (bor $x $y) $y) $x) + (=> (bor (bor $x $z) $y) $x) + ", + ); + } + + #[test] + fn regression_1() { + compile( + b" + (=> (bor (bor $x $y) 0) $x) + (=> (bor $x 0) $x) + (=> (bor $y $x) $x) + ", + ); + } + + #[test] + fn regression_2() { + compile( + b" + (=> (sshr $x 11111111110) $x) + ", + ); + } +} diff --git a/cranelift/peepmatic/crates/fuzzing/src/interp.rs b/cranelift/peepmatic/crates/fuzzing/src/interp.rs new file mode 100644 index 0000000000..0ebdeebe28 --- /dev/null +++ b/cranelift/peepmatic/crates/fuzzing/src/interp.rs @@ -0,0 +1,374 @@ +//! Interpreting compiled peephole optimizations against test instruction sequences. + +use peepmatic::{ + Constraint, Dfs, DynAstRef, Optimizations, Pattern, Span, TraversalEvent, ValueLiteral, + Variable, +}; +use peepmatic_runtime::{ + cc::ConditionCode, + operator::TypingContext as TypingContextTrait, + part::Constant, + r#type::BitWidth, + r#type::{Kind, Type}, +}; +use peepmatic_test::{Program, TestIsa}; +use std::collections::{BTreeMap, HashMap}; +use std::path::Path; +use std::str; + +/// Compile the given source text, and if it is a valid set of optimizations, +/// then interpret the optimizations against test instruction sequences created +/// to reflect the optimizations. +pub fn interp(data: &[u8]) { + let _ = env_logger::try_init(); + + let source = match str::from_utf8(data) { + Err(_) => return, + Ok(s) => s, + }; + + let peep_opts = match peepmatic::compile_str(source, Path::new("fuzz")) { + Err(_) => return, + Ok(o) => o, + }; + let mut optimizer = peep_opts.optimizer(TestIsa { + native_word_size_in_bits: 32, + }); + + // Okay, we know it compiles and verifies alright, so (re)parse the AST. + let buf = wast::parser::ParseBuffer::new(&source).unwrap(); + let ast = wast::parser::parse::(&buf).unwrap(); + + // And we need access to the assigned types, so re-verify it as well. + peepmatic::verify(&ast).unwrap(); + + // Walk over each optimization and create an instruction sequence that + // matches the optimization. + let mut program = Program::default(); + for opt in &ast.optimizations { + // The instruction sequence we generate must match an optimization (not + // necessarily *this* optimization, if there is another that is more + // specific but also matches) unless there is an `bit-width` + // precondition or an implicit `bit-width` precondition via a type + // ascription. When those things exist, we might have constructed + // instructions with the wrong bit widths to match. + let mut allow_no_match = false; + + // The last instruction we generated. After we've generated the full + // instruction sequence, this will be its root. + let mut last_inst = None; + + // Remember the instructions associated with variables and constants, so + // that when they appear multiple times, we reuse the same instruction. + let mut id_to_inst = HashMap::new(); + + // Map from a pattern's span to the instruction we generated for + // it. This allows parent operations to get the instructions for their + // children. + let mut span_to_inst = BTreeMap::new(); + + for (te, lhs) in Dfs::new(&opt.lhs) { + // NB: We use a post-order traversal because we want arguments to be + // generated before they are used. + if te != TraversalEvent::Exit { + continue; + } + + match lhs { + DynAstRef::Precondition(p) => { + allow_no_match |= p.constraint == Constraint::BitWidth; + } + + DynAstRef::Pattern(Pattern::Operation(op)) => { + allow_no_match |= op.r#type.get().is_some(); + + let num_imms = op.operator.immediates_arity() as usize; + + // Generate this operation's immediates. + let mut imm_tys = vec![]; + op.operator + .immediate_types(&mut TypingContext, op.span(), &mut imm_tys); + let imms: Vec<_> = op + .operands + .iter() + .take(num_imms) + .zip(imm_tys) + .map(|(pat, ty)| match pat { + Pattern::ValueLiteral(ValueLiteral::Integer(i)) => { + Constant::Int(i.value as _, BitWidth::ThirtyTwo).into() + } + Pattern::ValueLiteral(ValueLiteral::Boolean(b)) => { + Constant::Bool(b.value, BitWidth::One).into() + } + Pattern::ValueLiteral(ValueLiteral::ConditionCode(cc)) => cc.cc.into(), + Pattern::Constant(_) | Pattern::Variable(_) => match ty { + TypeOrConditionCode::ConditionCode => ConditionCode::Eq.into(), + TypeOrConditionCode::Type(ty) => match ty.kind { + Kind::Int => Constant::Int(1, ty.bit_width).into(), + Kind::Bool => Constant::Bool(false, ty.bit_width).into(), + Kind::Void | Kind::CpuFlags => { + unreachable!("void and cpu flags cannot be immediates") + } + }, + }, + Pattern::Operation(_) => { + unreachable!("operations not allowed as immediates") + } + }) + .collect(); + + // Generate (or collect already-generated) instructions for + // this operation's arguments. + let mut arg_tys = vec![]; + op.operator + .param_types(&mut TypingContext, op.span(), &mut arg_tys); + let args: Vec<_> = op + .operands + .iter() + .skip(num_imms) + .zip(arg_tys) + .map(|(pat, ty)| match pat { + Pattern::Operation(op) => span_to_inst[&op.span()], + Pattern::ValueLiteral(ValueLiteral::Integer(i)) => program.r#const( + Constant::Int(i.value as _, BitWidth::ThirtyTwo), + BitWidth::ThirtyTwo, + ), + Pattern::ValueLiteral(ValueLiteral::Boolean(b)) => program.r#const( + Constant::Bool(b.value, BitWidth::One), + BitWidth::ThirtyTwo, + ), + Pattern::ValueLiteral(ValueLiteral::ConditionCode(_)) => { + unreachable!("condition codes cannot be arguments") + } + Pattern::Constant(peepmatic::Constant { id, .. }) + | Pattern::Variable(Variable { id, .. }) => match ty { + TypeOrConditionCode::Type(ty) => { + *id_to_inst.entry(id).or_insert_with(|| match ty.kind { + Kind::Int => program.r#const( + Constant::Int(1, ty.bit_width), + BitWidth::ThirtyTwo, + ), + Kind::Bool => program.r#const( + Constant::Bool(false, ty.bit_width), + BitWidth::ThirtyTwo, + ), + Kind::CpuFlags => { + unreachable!("cpu flags cannot be an argument") + } + Kind::Void => unreachable!("void cannot be an argument"), + }) + } + TypeOrConditionCode::ConditionCode => { + unreachable!("condition codes cannot be arguments") + } + }, + }) + .collect(); + + let ty = match op.operator.result_type(&mut TypingContext, op.span()) { + TypeOrConditionCode::Type(ty) => ty, + TypeOrConditionCode::ConditionCode => { + unreachable!("condition codes cannot be operation results") + } + }; + let inst = program.new_instruction(op.operator, ty, imms, args); + last_inst = Some(inst); + let old_inst = span_to_inst.insert(op.span(), inst); + assert!(old_inst.is_none()); + } + _ => continue, + } + } + + // Run the optimizer on our newly generated instruction sequence. + if let Some(inst) = last_inst { + let replacement = optimizer.apply_one(&mut program, inst); + assert!( + replacement.is_some() || allow_no_match, + "an optimization should match the generated instruction sequence" + ); + } + } + + // Finally, just try and run the optimizer on every instruction we + // generated, just to potentially shake out some more bugs. + let instructions: Vec<_> = program.instructions().map(|(k, _)| k).collect(); + for inst in instructions { + let _ = optimizer.apply_one(&mut program, inst); + } +} + +enum TypeOrConditionCode { + Type(Type), + ConditionCode, +} + +struct TypingContext; + +impl<'a> TypingContextTrait<'a> for TypingContext { + type TypeVariable = TypeOrConditionCode; + + fn cc(&mut self, _: wast::Span) -> Self::TypeVariable { + TypeOrConditionCode::ConditionCode + } + + fn bNN(&mut self, _: wast::Span) -> Self::TypeVariable { + TypeOrConditionCode::Type(Type::b1()) + } + + fn iNN(&mut self, _: wast::Span) -> Self::TypeVariable { + TypeOrConditionCode::Type(Type::i32()) + } + + fn iMM(&mut self, _: wast::Span) -> Self::TypeVariable { + TypeOrConditionCode::Type(Type::i32()) + } + + fn cpu_flags(&mut self, _: wast::Span) -> Self::TypeVariable { + TypeOrConditionCode::Type(Type::cpu_flags()) + } + + fn b1(&mut self, _: wast::Span) -> Self::TypeVariable { + TypeOrConditionCode::Type(Type::b1()) + } + + fn void(&mut self, _: wast::Span) -> Self::TypeVariable { + TypeOrConditionCode::Type(Type::void()) + } + + fn bool_or_int(&mut self, _: wast::Span) -> Self::TypeVariable { + TypeOrConditionCode::Type(Type::b1()) + } + + fn any_t(&mut self, _: wast::Span) -> Self::TypeVariable { + TypeOrConditionCode::Type(Type::i32()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn check_interp() { + crate::check(|s: Vec| interp(String::from_utf8_lossy(&s).as_bytes())); + } + + #[test] + fn regression_0() { + interp(b"(=> (imul $x $x) $x)"); + } + + #[test] + fn regression_1() { + interp(b"(=> (when (imul $x $C) (is-power-of-two $C)) $x)"); + } + + #[test] + fn regression_2() { + interp( + b" + (=> (bor (bor $x $y) $x) (bor $x $y)) + (=> (bor (bor $x $C) 5) $x) + ", + ); + } + + #[test] + fn regression_3() { + interp( + b" + (=> (bor $y (bor $x 9)) $x) + (=> (bor (bor $x $y) $x) $x) + ", + ); + } + + #[test] + fn regression_4() { + interp( + b" + (=> (bor $C 33) 0) + (=> (bor $x 22) 1) + (=> (bor $x 11) 2) + ", + ); + } + + #[test] + fn regression_5() { + interp( + b" + (=> (bor $y (bor $x $y)) (bor $x $y)) + (=> (bor (bor $x $y) $z) $x) + (=> (bor (bor $x $y) $y) $x) + ", + ); + } + + #[test] + fn regression_6() { + interp(b"(=> (imul $x $f) of)"); + } + + #[test] + fn regression_7() { + interp( + b" + (=> (when (sdiv $x $C) + (fits-in-native-word $y)) + (sdiv $C $x)) + ", + ); + } + + #[test] + fn regression_8() { + interp( + b" + (=> (adjust_sp_down $C) (adjust_sp_down_imm $C)) + ", + ); + } + + #[test] + fn regression_9() { + interp( + b" + (=> (when $x) $x) + (=> (trapnz $x) (trapnz $x)) + ", + ); + } + + #[test] + fn regression_10() { + interp(b"(=> (sshr{i1} $x 0) $x)"); + } + + #[test] + fn regression_11() { + interp( + b" + (=> (when (ushr_imm $x (ishl 4 3)) + (bit-width $x 64)) + (sextend{i64} (ireduce{i32} $x))) + ", + ); + } + + #[test] + fn regression_12() { + interp(b"(=> (band $C1 (band_imm $C1 1)) 1)"); + } + + #[test] + fn regression_13() { + interp(b"(=> (brz (icmp eq 0 $x)) (brz (ireduce{i32} $x)))"); + } + + #[test] + fn regression_14() { + interp(b"(=> (brz (icmp $E 0 $x)) (brz $x))"); + } +} diff --git a/cranelift/peepmatic/crates/fuzzing/src/lib.rs b/cranelift/peepmatic/crates/fuzzing/src/lib.rs new file mode 100644 index 0000000000..82661d8b7d --- /dev/null +++ b/cranelift/peepmatic/crates/fuzzing/src/lib.rs @@ -0,0 +1,119 @@ +//! Utilities for fuzzing. +//! +//! The actual fuzz targets are defined in `peepmatic/fuzz/*`. This crate just +//! has oracles and generators for fuzzing. + +#![deny(missing_debug_implementations)] +#![deny(missing_docs)] + +use arbitrary::{Arbitrary, Unstructured}; +use rand::prelude::*; +use std::fmt::Debug; +use std::panic; +use std::time; + +pub mod automata; +pub mod compile; +pub mod interp; +pub mod parser; + +/// A quickcheck-style runner for fuzz targets. +/// +/// This is *not* intended to replace a long-running, coverage-guided fuzzing +/// engine like libFuzzer! This is only for defining quick, purely random tests +/// for use with `cargo test` and CI. +pub fn check(mut f: impl FnMut(A)) +where + A: Clone + Debug + Arbitrary, +{ + let seed = rand::thread_rng().gen(); + let mut rng = rand::rngs::SmallRng::seed_from_u64(seed); + + const INITIAL_LENGTH: usize = 16; + const MAX_LENGTH: usize = 4096; + + let mut buf: Vec = (0..INITIAL_LENGTH).map(|_| rng.gen()).collect(); + let mut num_checked = 0; + + let time_budget = time::Duration::from_secs(2); + let then = time::Instant::now(); + + let (failing_input, panic_info) = loop { + if num_checked > 0 && time::Instant::now().duration_since(then) > time_budget { + eprintln!("Checked {} random inputs.", num_checked); + return; + } + + match ::arbitrary_take_rest(Unstructured::new(&buf)) { + Ok(input) => { + num_checked += 1; + eprintln!("Checking input: {:#?}", input); + if let Err(p) = panic::catch_unwind(panic::AssertUnwindSafe(|| f(input.clone()))) { + break (input, p); + } + } + Err(e @ arbitrary::Error::NotEnoughData) => { + eprintln!("warning: {}", e); + if *buf.last().unwrap() == 0 { + if buf.len() < MAX_LENGTH { + let new_size = std::cmp::min(buf.len() * 2, MAX_LENGTH); + eprintln!("Growing buffer size to {}", new_size); + let delta = new_size - buf.len(); + buf.reserve(delta); + for _ in 0..delta { + buf.push(rng.gen()); + } + continue; + } else { + // Regenerate `buf` in the loop below and see if that + // fixes things... + eprintln!("Regenerating buffer data."); + } + } else { + // Shrink values in the end of `buf`, which is where + // `Arbitrary` pulls container lengths from. Then try again. + eprintln!("Shrinking buffer's tail values."); + let i = (buf.len() as f64).sqrt() as usize; + for j in i..buf.len() { + buf[j] /= 2; + } + continue; + } + } + Err(e) => { + eprintln!("warning: {}", e); + // Usually this happens because `A` requires a sequence utf-8 + // bytes but its given sequence wasn't valid utf-8. Just skip + // this iteration and try again after we've updated `buf` below. + } + }; + + // Double the size of the buffer every so often, so we don't only + // explore small inputs. + if num_checked == buf.len() { + buf.resize(std::cmp::min(buf.len() * 2, MAX_LENGTH), 0); + } + + for i in 0..buf.len() { + buf[i] = rng.gen(); + } + }; + + // Shrink the failing input. + let mut smallest_failing_input = failing_input; + let mut panic_info = panic_info; + 'shrinking: loop { + eprintln!("Smallest failing input: {:#?}", smallest_failing_input); + for input in smallest_failing_input.shrink() { + if let Err(p) = panic::catch_unwind(panic::AssertUnwindSafe(|| f(input.clone()))) { + smallest_failing_input = input; + panic_info = p; + continue 'shrinking; + } + } + break; + } + + // Resume the panic for the smallest input. + panic::resume_unwind(panic_info); +} diff --git a/cranelift/peepmatic/crates/fuzzing/src/parser.rs b/cranelift/peepmatic/crates/fuzzing/src/parser.rs new file mode 100644 index 0000000000..a9972ff7ff --- /dev/null +++ b/cranelift/peepmatic/crates/fuzzing/src/parser.rs @@ -0,0 +1,29 @@ +//! Utilities for fuzzing our DSL's parser. + +use peepmatic::Optimizations; +use std::str; + +/// Attempt to parse the given string as if it were a snippet of our DSL. +pub fn parse(data: &[u8]) { + let source = match str::from_utf8(data) { + Ok(s) => s, + Err(_) => return, + }; + + let buf = match wast::parser::ParseBuffer::new(&source) { + Ok(buf) => buf, + Err(_) => return, + }; + + let _ = wast::parser::parse::(&buf); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn check_parse() { + crate::check(|s: String| parse(s.as_bytes())); + } +} From 4b16a4ad85588124022dcaa11a93be93e6edf17b Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 1 May 2020 16:17:43 -0700 Subject: [PATCH 07/22] peepmatic: Define fuzz targets for various parts of `peepmatic` --- fuzz/Cargo.toml | 31 +++++++++++++++++++ fuzz/fuzz_targets/peepmatic_compile.rs | 8 +++++ .../peepmatic_fst_differential.rs | 8 +++++ fuzz/fuzz_targets/peepmatic_interp.rs | 8 +++++ fuzz/fuzz_targets/peepmatic_parser.rs | 8 +++++ .../fuzz_targets/peepmatic_simple_automata.rs | 7 +++++ 6 files changed, 70 insertions(+) create mode 100644 fuzz/fuzz_targets/peepmatic_compile.rs create mode 100644 fuzz/fuzz_targets/peepmatic_fst_differential.rs create mode 100644 fuzz/fuzz_targets/peepmatic_interp.rs create mode 100644 fuzz/fuzz_targets/peepmatic_parser.rs create mode 100644 fuzz/fuzz_targets/peepmatic_simple_automata.rs diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 38fa092d58..f6ed67ac8e 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -14,6 +14,7 @@ cranelift-reader = { path = "../cranelift/reader" } cranelift-wasm = { path = "../cranelift/wasm" } libfuzzer-sys = "0.3.2" target-lexicon = "0.10" +peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing" } wasmtime = { path = "../crates/wasmtime" } wasmtime-fuzzing = { path = "../crates/fuzzing" } @@ -56,5 +57,35 @@ path = "fuzz_targets/spectests.rs" test = false doc = false +[[bin]] +name = "peepmatic_simple_automata" +path = "fuzz_targets/peepmatic_simple_automata.rs" +test = false +doc = false + +[[bin]] +name = "peepmatic_fst_differential" +path = "fuzz_targets/peepmatic_fst_differential.rs" +test = false +doc = false + +[[bin]] +name = "peepmatic_parser" +path = "fuzz_targets/peepmatic_parser.rs" +test = false +doc = false + +[[bin]] +name = "peepmatic_compile" +path = "fuzz_targets/peepmatic_compile.rs" +test = false +doc = false + +[[bin]] +name = "peepmatic_interp" +path = "fuzz_targets/peepmatic_interp.rs" +test = false +doc = false + [features] binaryen = ['wasmtime-fuzzing/binaryen'] diff --git a/fuzz/fuzz_targets/peepmatic_compile.rs b/fuzz/fuzz_targets/peepmatic_compile.rs new file mode 100644 index 0000000000..623d747280 --- /dev/null +++ b/fuzz/fuzz_targets/peepmatic_compile.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use peepmatic_fuzzing::compile::compile; + +fuzz_target!(|data: &[u8]| { + compile(data); +}); diff --git a/fuzz/fuzz_targets/peepmatic_fst_differential.rs b/fuzz/fuzz_targets/peepmatic_fst_differential.rs new file mode 100644 index 0000000000..00549112db --- /dev/null +++ b/fuzz/fuzz_targets/peepmatic_fst_differential.rs @@ -0,0 +1,8 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use peepmatic_fuzzing::automata::fst_differential; +use std::collections::HashMap; + +fuzz_target!(|map: HashMap, u64>| { + fst_differential(map); +}); diff --git a/fuzz/fuzz_targets/peepmatic_interp.rs b/fuzz/fuzz_targets/peepmatic_interp.rs new file mode 100644 index 0000000000..dd3c1732ab --- /dev/null +++ b/fuzz/fuzz_targets/peepmatic_interp.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use peepmatic_fuzzing::interp::interp; + +fuzz_target!(|data: &[u8]| { + interp(data); +}); diff --git a/fuzz/fuzz_targets/peepmatic_parser.rs b/fuzz/fuzz_targets/peepmatic_parser.rs new file mode 100644 index 0000000000..5664dad41b --- /dev/null +++ b/fuzz/fuzz_targets/peepmatic_parser.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use peepmatic_fuzzing::parser::parse; + +fuzz_target!(|data: &[u8]| { + parse(data); +}); diff --git a/fuzz/fuzz_targets/peepmatic_simple_automata.rs b/fuzz/fuzz_targets/peepmatic_simple_automata.rs new file mode 100644 index 0000000000..ecf2fd7850 --- /dev/null +++ b/fuzz/fuzz_targets/peepmatic_simple_automata.rs @@ -0,0 +1,7 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use peepmatic_fuzzing::automata::simple_automata; + +fuzz_target!(|input_output_pairs: Vec)>>| { + simple_automata(input_output_pairs); +}); From 18663fede9decb3290c48857f56e0021f4fc9dda Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 1 May 2020 16:41:41 -0700 Subject: [PATCH 08/22] ci: Exercise the peepmatic fuzz targets in CI --- .github/workflows/main.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e99a2c6b3c..402ea814db 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -164,6 +164,41 @@ jobs: | xargs cargo fuzz run differential --release --debug-assertions --features binaryen env: RUST_BACKTRACE: 1 + - run: | + find fuzz/corpus/peepmatic_compile -type f \ + | shuf \ + | head -n 10000 \ + | xargs cargo fuzz run peepmatic_compile --release --debug-assertions --features binaryen + env: + RUST_BACKTRACE: 1 + - run: | + find fuzz/corpus/peepmatic_fst_differential -type f \ + | shuf \ + | head -n 10000 \ + | xargs cargo fuzz run peepmatic_fst_differential --release --debug-assertions --features binaryen + env: + RUST_BACKTRACE: 1 + - run: | + find fuzz/corpus/peepmatic_interp -type f \ + | shuf \ + | head -n 5000 \ + | xargs cargo fuzz run peepmatic_interp --release --debug-assertions --features binaryen + env: + RUST_BACKTRACE: 1 + - run: | + find fuzz/corpus/peepmatic_parser -type f \ + | shuf \ + | head -n 10000 \ + | xargs cargo fuzz run peepmatic_parser --release --debug-assertions --features binaryen + env: + RUST_BACKTRACE: 1 + - run: | + find fuzz/corpus/peepmatic_simple_automata -type f \ + | shuf \ + | head -n 1000 \ + | xargs cargo fuzz run peepmatic_simple_automata --release --debug-assertions --features binaryen + env: + RUST_BACKTRACE: 1 # Perform all tests (debug mode) for `wasmtime`. This runs stable/beta/nightly # channels of Rust as well as macOS/Linux/Windows. From 090d1c2d322be7be6c3c5c2a31feabd22b988864 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 28 Apr 2020 16:43:32 -0700 Subject: [PATCH 09/22] cranelift: Port most of `simple_preopt.rs` over to the `peepmatic` DSL This ports all of the identity, no-op, simplification, and canonicalization related optimizations over from being hand-coded to the `peepmatic` DSL. This does not handle the branch-to-branch optimizations or most of the divide-by-constant optimizations. --- .gitignore | 1 + Cargo.lock | 294 ++++-- cranelift/codegen/Cargo.toml | 6 + cranelift/codegen/build.rs | 18 + cranelift/codegen/src/ir/dfg.rs | 25 +- cranelift/codegen/src/ir/function.rs | 24 + cranelift/codegen/src/ir/instructions.rs | 28 +- cranelift/codegen/src/lib.rs | 1 + cranelift/codegen/src/peepmatic.rs | 847 ++++++++++++++++++ .../examples => codegen/src}/preopt.peepmatic | 4 +- cranelift/codegen/src/preopt.serialized | Bin 0 -> 5716 bytes cranelift/codegen/src/simple_preopt.rs | 397 +------- .../filetests/isa/x86/isub_imm-i8.clif | 8 +- .../simple_preopt/div_by_const_indirect.clif | 50 +- ...ing_instructions_and_cfg_predecessors.clif | 22 + .../filetests/simple_preopt/simplify32.clif | 1 - .../filetests/simple_preopt/simplify64.clif | 31 + ...plify_instruction_into_alias_of_value.clif | 17 + cranelift/filetests/src/test_simple_preopt.rs | 1 + cranelift/peepmatic/src/lib.rs | 2 +- 20 files changed, 1289 insertions(+), 488 deletions(-) create mode 100644 cranelift/codegen/src/peepmatic.rs rename cranelift/{peepmatic/examples => codegen/src}/preopt.peepmatic (97%) create mode 100644 cranelift/codegen/src/preopt.serialized create mode 100644 cranelift/filetests/filetests/simple_preopt/replace_branching_instructions_and_cfg_predecessors.clif create mode 100644 cranelift/filetests/filetests/simple_preopt/simplify_instruction_into_alias_of_value.clif diff --git a/.gitignore b/.gitignore index 4c37c4f83d..f890733b7d 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ docs/book rusty-tags.* tags target +.z3-trace diff --git a/Cargo.lock b/Cargo.lock index c143251096..99373b1fae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,9 +11,12 @@ dependencies = [ [[package]] name = "ahash" -version = "0.3.3" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b909d1c126f78ace756fc337133356c499eebeefcce930fa5fb018823f2b2d" +checksum = "0989268a37e128d4d7a8028f1c60099430113fdbc70419010601ce51a228e4fe" +dependencies = [ + "const-random", +] [[package]] name = "aho-corasick" @@ -41,9 +44,9 @@ checksum = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff" [[package]] name = "arbitrary" -version = "0.4.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5eb01a9ab8a3369f2f7632b9461c34f5920bd454774bab5b9fc6744f21d6143" +checksum = "75153c95fdedd7db9732dfbfc3702324a1627eec91ba56e37cd0ac78314ab2ed" dependencies = [ "derive_arbitrary", ] @@ -97,9 +100,9 @@ dependencies = [ [[package]] name = "backtrace-sys" -version = "0.1.36" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78848718ee1255a2485d1309ad9cdecfc2e7d0362dd11c6829364c6b35ae1bc7" +checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" dependencies = [ "cc", "libc", @@ -201,6 +204,12 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "bumpalo" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187" + [[package]] name = "byte-tools" version = "0.3.1" @@ -233,9 +242,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.52" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d87b23d6a92cd03af510a5ade527033f6aa6fa92161e2d5863a907d4c5e31d" +checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" dependencies = [ "jobserver", ] @@ -261,6 +270,18 @@ dependencies = [ "vec_map", ] +[[package]] +name = "clicolors-control" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90082ee5dcdd64dc4e9e0d37fbf3ee325419e39c0092191e0393df65518f741e" +dependencies = [ + "atty", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -281,19 +302,18 @@ dependencies = [ [[package]] name = "console" -version = "0.11.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea0f3e2e8d7dba335e913b97f9e1992c86c4399d54f8be1d31c8727d0652064" +checksum = "6728a28023f207181b193262711102bfbaf47cc9d13bc71d0736607ef8efe88c" dependencies = [ + "clicolors-control", "encode_unicode", "lazy_static", "libc", "regex", - "terminal_size", "termios", "unicode-width", "winapi", - "winapi-util", ] [[package]] @@ -357,8 +377,10 @@ dependencies = [ "cranelift-codegen-shared", "cranelift-entity", "gimli", - "hashbrown 0.7.2", + "hashbrown 0.7.1", "log", + "peepmatic", + "peepmatic-runtime", "regalloc", "serde", "smallvec", @@ -424,7 +446,7 @@ name = "cranelift-frontend" version = "0.63.0" dependencies = [ "cranelift-codegen", - "hashbrown 0.7.2", + "hashbrown 0.7.1", "log", "smallvec", "target-lexicon", @@ -438,7 +460,7 @@ dependencies = [ "cranelift-entity", "cranelift-frontend", "cranelift-reader", - "hashbrown 0.7.2", + "hashbrown 0.7.1", "log", "pretty_env_logger", "thiserror", @@ -566,12 +588,12 @@ dependencies = [ "cranelift-codegen", "cranelift-entity", "cranelift-frontend", - "hashbrown 0.7.2", + "hashbrown 0.7.1", "log", "serde", "target-lexicon", "thiserror", - "wasmparser", + "wasmparser 0.52.2", "wat", ] @@ -642,13 +664,13 @@ dependencies = [ [[package]] name = "derive_arbitrary" -version = "0.4.4" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cee758ebd1c79a9c6fb95f242dcc30bdbf555c28369ae908d21fdaf81537496" +checksum = "caedd6a71b6d00bdc458ec8ffbfd12689c1ee7ffa69ad9933310aaf2f08f18d8" dependencies = [ "proc-macro2", - "quote", "syn", + "synstructure", ] [[package]] @@ -877,6 +899,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +[[package]] +name = "fst" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f9cac32c1741cdf6b66be7dcf0d9c7f25ccf12f8aa84c16cfa31f9f14513b3" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -952,11 +980,11 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.7.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" +checksum = "479e9d9a1a3f8c489868a935b557ab5710e3e223836da2ecd52901d88935cb56" dependencies = [ - "ahash 0.3.3", + "ahash 0.3.2", "autocfg 1.0.0", ] @@ -971,9 +999,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.12" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61565ff7aaace3525556587bd2dc31d4a07071957be715e63ce7b1eccf51a8f4" +checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e" dependencies = [ "libc", ] @@ -1100,7 +1128,7 @@ dependencies = [ "staticvec", "thiserror", "typemap", - "wasmparser", + "wasmparser 0.52.2", "wat", ] @@ -1170,9 +1198,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" dependencies = [ "hermit-abi", "libc", @@ -1226,6 +1254,76 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "peepmatic" +version = "0.1.0" +dependencies = [ + "anyhow", + "peepmatic-automata", + "peepmatic-macro", + "peepmatic-runtime", + "wast 13.0.0", + "z3", +] + +[[package]] +name = "peepmatic-automata" +version = "0.1.0" +dependencies = [ + "serde", +] + +[[package]] +name = "peepmatic-fuzzing" +version = "0.1.0" +dependencies = [ + "arbitrary", + "bincode", + "env_logger 0.7.1", + "fst", + "log", + "peepmatic", + "peepmatic-automata", + "peepmatic-runtime", + "peepmatic-test", + "rand 0.7.3", + "serde", + "wast 13.0.0", +] + +[[package]] +name = "peepmatic-macro" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "peepmatic-runtime" +version = "0.1.0" +dependencies = [ + "bincode", + "bumpalo", + "log", + "peepmatic-automata", + "peepmatic-macro", + "serde", + "thiserror", + "wast 13.0.0", +] + +[[package]] +name = "peepmatic-test" +version = "0.1.0" +dependencies = [ + "env_logger 0.7.1", + "log", + "peepmatic", + "peepmatic-runtime", +] + [[package]] name = "plain" version = "0.2.3" @@ -1291,9 +1389,9 @@ dependencies = [ [[package]] name = "proptest" -version = "0.9.6" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c477819b845fe023d33583ebf10c9f62518c8d79a0960ba5c36d6ac8a55a5b" +checksum = "bf6147d103a7c9d7598f4105cf049b15c99e2ecd93179bf024f0fd349be5ada4" dependencies = [ "bit-set", "bitflags", @@ -1350,7 +1448,7 @@ dependencies = [ "rand_isaac", "rand_jitter", "rand_os", - "rand_pcg", + "rand_pcg 0.1.2", "rand_xorshift", "winapi", ] @@ -1366,6 +1464,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc 0.2.0", + "rand_pcg 0.2.1", ] [[package]] @@ -1474,6 +1573,15 @@ dependencies = [ "rand_core 0.4.2", ] +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rand_xorshift" version = "0.1.1" @@ -1557,9 +1665,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.7" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3" dependencies = [ "aho-corasick", "memchr", @@ -1648,9 +1756,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" +checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" [[package]] name = "same-file" @@ -1724,9 +1832,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.52" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7894c8ed05b7a3a279aeb79025fdec1d3158080b75b98a08faf2806bb799edd" +checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9" dependencies = [ "itoa", "ryu", @@ -1747,9 +1855,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.4.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" +checksum = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" [[package]] name = "stable_deref_trait" @@ -1780,9 +1888,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "structopt" -version = "0.3.14" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef" +checksum = "ff6da2e8d107dfd7b74df5ef4d205c6aebee0706c647f6bc6a2d5789905c00fb" dependencies = [ "clap", "lazy_static", @@ -1791,9 +1899,9 @@ dependencies = [ [[package]] name = "structopt-derive" -version = "0.4.7" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a" +checksum = "a489c87c08fbaf12e386665109dd13470dcc9c4583ea3e10dd2b4523e5ebd9ac" dependencies = [ "heck", "proc-macro-error", @@ -1804,9 +1912,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.18" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "410a7488c0a728c7ceb4ad59b9567eb4053d02e8cc7f5c0e0eeeb39518369213" +checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" dependencies = [ "proc-macro2", "quote", @@ -1824,6 +1932,18 @@ dependencies = [ "syn", ] +[[package]] +name = "synstructure" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + [[package]] name = "target-lexicon" version = "0.10.0" @@ -1863,16 +1983,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "terminal_size" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8038f95fc7a6f351163f4b964af631bd26c9e828f7db085f2a84aca56f70d13b" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "termios" version = "0.3.2" @@ -1909,18 +2019,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.16" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d12a1dae4add0f0d568eebc7bf142f145ba1aa2544cafb195c76f0f409091b60" +checksum = "54b3d3d2ff68104100ab257bb6bb0cb26c901abe4bd4ba15961f3bf867924012" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.16" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f34e0c1caaa462fd840ec6b768946ea1e7842620d94fe29d5b847138f521269" +checksum = "ca972988113b7715266f91250ddb98070d033c62a011fa0fcc57434a649310dd" dependencies = [ "proc-macro2", "quote", @@ -1962,9 +2072,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.12.0" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" [[package]] name = "unicode-segmentation" @@ -2051,6 +2161,12 @@ dependencies = [ "yanix", ] +[[package]] +name = "wasmparser" +version = "0.51.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb1956b19469d1c5e63e459d29e7b5aa0f558d9f16fcef09736f8a265e6c10a" + [[package]] name = "wasmparser" version = "0.52.2" @@ -2059,12 +2175,12 @@ checksum = "733954023c0b39602439e60a65126fd31b003196d3a1e8e4531b055165a79b31" [[package]] name = "wasmprinter" -version = "0.2.4" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21c3ac16b1f882bf1e1ce007e4f194559d2e3332013367863ddfbc828d1f044f" +checksum = "8bd423d45b95fcee11775472bfdce66c63c45ada23c1b338e0a63d623a6c475b" dependencies = [ "anyhow", - "wasmparser", + "wasmparser 0.51.4", ] [[package]] @@ -2080,7 +2196,7 @@ dependencies = [ "rustc-demangle", "target-lexicon", "tempfile", - "wasmparser", + "wasmparser 0.52.2", "wasmtime-environ", "wasmtime-jit", "wasmtime-profiling", @@ -2150,7 +2266,7 @@ dependencies = [ "more-asserts", "target-lexicon", "thiserror", - "wasmparser", + "wasmparser 0.52.2", "wasmtime-environ", ] @@ -2181,7 +2297,7 @@ dependencies = [ "tempfile", "thiserror", "toml", - "wasmparser", + "wasmparser 0.52.2", "winapi", "zstd", ] @@ -2194,6 +2310,7 @@ dependencies = [ "cranelift-reader", "cranelift-wasm", "libfuzzer-sys", + "peepmatic-fuzzing", "target-lexicon", "wasmtime", "wasmtime-fuzzing", @@ -2209,7 +2326,7 @@ dependencies = [ "env_logger 0.7.1", "log", "rayon", - "wasmparser", + "wasmparser 0.52.2", "wasmprinter", "wasmtime", "wasmtime-wast", @@ -2233,7 +2350,7 @@ dependencies = [ "region", "target-lexicon", "thiserror", - "wasmparser", + "wasmparser 0.52.2", "wasmtime-debug", "wasmtime-environ", "wasmtime-profiling", @@ -2336,6 +2453,15 @@ dependencies = [ "leb128", ] +[[package]] +name = "wast" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b20abd8b4a26f7e0d4dd5e357e90a3d555ec190e94472c9b2b27c5b9777f9ae" +dependencies = [ + "leb128", +] + [[package]] name = "wast" version = "15.0.0" @@ -2347,11 +2473,11 @@ dependencies = [ [[package]] name = "wat" -version = "1.0.16" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "526d28df6c047d9f9a92d4925b98afd8d8d95b1b3aa4f13eb1306f17d1da56c4" +checksum = "51a615830ee3e7200b505c441fec09aac2f114deae69df52f215cb828ba112c4" dependencies = [ - "wast 15.0.0", + "wast 13.0.0", ] [[package]] @@ -2424,9 +2550,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e" dependencies = [ "winapi", ] @@ -2470,6 +2596,26 @@ dependencies = [ "log", ] +[[package]] +name = "z3" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afd636f549e919f8058567000156e19efbb4728ddb3be226dcc332d0ff622ab5" +dependencies = [ + "lazy_static", + "log", + "z3-sys", +] + +[[package]] +name = "z3-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2487974559d1494e8e8925df85362d6109ae5555092510508a5b1077346c2833" +dependencies = [ + "cmake", +] + [[package]] name = "zstd" version = "0.5.1+zstd.1.4.4" diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 692c85bdd4..4737904eb3 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -24,6 +24,7 @@ gimli = { version = "0.20.0", default-features = false, features = ["write"], op smallvec = { version = "1.0.0" } thiserror = "1.0.4" byteorder = { version = "1.3.2", default-features = false } +peepmatic-runtime = { path = "../peepmatic/crates/runtime" } regalloc = "0.0.23" # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary @@ -32,6 +33,7 @@ regalloc = "0.0.23" [build-dependencies] cranelift-codegen-meta = { path = "meta", version = "0.63.0" } +peepmatic = { path = "../peepmatic", optional = true } [features] default = ["std", "unwind"] @@ -72,5 +74,9 @@ all-arch = [ # For dependent crates that want to serialize some parts of cranelift enable-serde = ["serde"] +# Recompile our optimizations that are written in the peepmatic DSL into a +# compact finite-state transducer automaton. +rebuild-peephole-optimizers = ["peepmatic"] + [badges] maintenance = { status = "experimental" } diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index bb14364050..b7352f37c3 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -71,4 +71,22 @@ fn main() { ); println!("cargo:warning=Generated files are in {}", out_dir); } + + #[cfg(feature = "rebuild-peephole-optimizers")] + rebuild_peephole_optimizers(); +} + +#[cfg(feature = "rebuild-peephole-optimizers")] +fn rebuild_peephole_optimizers() { + use std::path::Path; + + let source_path = Path::new("src").join("preopt.peepmatic"); + println!("cargo:rerun-if-changed={}", source_path.display()); + + let preopt = + peepmatic::compile_file(&source_path).expect("failed to compile `src/preopt.peepmatic`"); + + preopt + .serialize_to_file(&Path::new("src").join("preopt.serialized")) + .expect("failed to serialize peephole optimizer to `src/preopt.serialized`"); } diff --git a/cranelift/codegen/src/ir/dfg.rs b/cranelift/codegen/src/ir/dfg.rs index 787f519de9..58d101aace 100644 --- a/cranelift/codegen/src/ir/dfg.rs +++ b/cranelift/codegen/src/ir/dfg.rs @@ -234,11 +234,7 @@ impl DataFlowGraph { /// Get the type of a value. pub fn value_type(&self, v: Value) -> Type { - match self.values[v] { - ValueData::Inst { ty, .. } - | ValueData::Param { ty, .. } - | ValueData::Alias { ty, .. } => ty, - } + self.values[v].ty() } /// Get the definition of a value. @@ -383,9 +379,14 @@ pub enum ValueDef { impl ValueDef { /// Unwrap the instruction where the value was defined, or panic. pub fn unwrap_inst(&self) -> Inst { + self.inst().expect("Value is not an instruction result") + } + + /// Get the instruction where the value was defined, if any. + pub fn inst(&self) -> Option { match *self { - Self::Result(inst, _) => inst, - _ => panic!("Value is not an instruction result"), + Self::Result(inst, _) => Some(inst), + _ => None, } } @@ -428,6 +429,16 @@ enum ValueData { Alias { ty: Type, original: Value }, } +impl ValueData { + fn ty(&self) -> Type { + match *self { + ValueData::Inst { ty, .. } + | ValueData::Param { ty, .. } + | ValueData::Alias { ty, .. } => ty, + } + } +} + /// Instructions. /// impl DataFlowGraph { diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 892af400af..f425eb669f 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -308,6 +308,30 @@ impl Function { // function, assume it is not a leaf. self.dfg.signatures.is_empty() } + + /// Replace the `dst` instruction's data with the `src` instruction's data + /// and then remove `src`. + /// + /// `src` and its result values should not be used at all, as any uses would + /// be left dangling after calling this method. + /// + /// `src` and `dst` must have the same number of resulting values, and + /// `src`'s i^th value must have the same type as `dst`'s i^th value. + pub fn transplant_inst(&mut self, dst: Inst, src: Inst) { + debug_assert_eq!( + self.dfg.inst_results(dst).len(), + self.dfg.inst_results(src).len() + ); + debug_assert!(self + .dfg + .inst_results(dst) + .iter() + .zip(self.dfg.inst_results(src)) + .all(|(a, b)| self.dfg.value_type(*a) == self.dfg.value_type(*b))); + + self.dfg[dst] = self.dfg[src].clone(); + self.layout.remove_inst(src); + } } /// Additional annotations for function display. diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs index deb79130d4..1ada09fd71 100644 --- a/cranelift/codegen/src/ir/instructions.rs +++ b/cranelift/codegen/src/ir/instructions.rs @@ -11,9 +11,7 @@ use core::fmt::{self, Display, Formatter}; use core::ops::{Deref, DerefMut}; use core::str::FromStr; -use crate::ir; -use crate::ir::types; -use crate::ir::{Block, FuncRef, JumpTable, SigRef, Type, Value}; +use crate::ir::{self, trapcode::TrapCode, types, Block, FuncRef, JumpTable, SigRef, Type, Value}; use crate::isa; use crate::bitset::BitSet; @@ -257,6 +255,30 @@ impl InstructionData { } } + /// If this is a trapping instruction, get its trap code. Otherwise, return + /// `None`. + pub fn trap_code(&self) -> Option { + match *self { + Self::CondTrap { code, .. } + | Self::FloatCondTrap { code, .. } + | Self::IntCondTrap { code, .. } + | Self::Trap { code, .. } => Some(code), + _ => None, + } + } + + /// If this is a trapping instruction, get an exclusive reference to its + /// trap code. Otherwise, return `None`. + pub fn trap_code_mut(&mut self) -> Option<&mut TrapCode> { + match self { + Self::CondTrap { code, .. } + | Self::FloatCondTrap { code, .. } + | Self::IntCondTrap { code, .. } + | Self::Trap { code, .. } => Some(code), + _ => None, + } + } + /// Return information about a call instruction. /// /// Any instruction that can call another function reveals its call signature here. diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index efe0ea00c6..05c5583f5b 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -101,6 +101,7 @@ mod licm; mod nan_canonicalization; mod num_uses; mod partition_slice; +mod peepmatic; mod postopt; mod predicates; mod redundant_reload_remover; diff --git a/cranelift/codegen/src/peepmatic.rs b/cranelift/codegen/src/peepmatic.rs new file mode 100644 index 0000000000..f78f5ccd3e --- /dev/null +++ b/cranelift/codegen/src/peepmatic.rs @@ -0,0 +1,847 @@ +//! Glue for working with `peepmatic`-generated peephole optimizers. + +use crate::cursor::{Cursor, FuncCursor}; +use crate::ir::{ + dfg::DataFlowGraph, + entities::{Inst, Value}, + immediates::{Imm64, Uimm64}, + instructions::{InstructionData, Opcode}, + types, InstBuilder, +}; +use crate::isa::TargetIsa; +use cranelift_codegen_shared::condcodes::IntCC; +use peepmatic_runtime::{ + cc::ConditionCode, + instruction_set::InstructionSet, + operator::Operator, + part::{Constant, Part}, + paths::Path, + r#type::{BitWidth, Kind, Type}, + PeepholeOptimizations, PeepholeOptimizer, +}; +use std::boxed::Box; +use std::convert::{TryFrom, TryInto}; +use std::ptr; +use std::sync::atomic::{AtomicPtr, Ordering}; + +/// Get the `preopt.peepmatic` peephole optimizer. +pub(crate) fn preopt<'a, 'b>( + isa: &'b dyn TargetIsa, +) -> PeepholeOptimizer<'static, 'a, &'b dyn TargetIsa> { + static SERIALIZED: &[u8] = include_bytes!("preopt.serialized"); + + // Once initialized, this must never be re-assigned. The initialized value + // is semantically "static data" and is intentionally leaked for the whole + // program's lifetime. + static DESERIALIZED: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + + // If `DESERIALIZED` has already been initialized, then just use it. + let ptr = DESERIALIZED.load(Ordering::SeqCst); + if let Some(peep_opts) = unsafe { ptr.as_ref() } { + return peep_opts.optimizer(isa); + } + + // Otherwise, if `DESERIALIZED` hasn't been initialized, then we need to + // deserialize the peephole optimizations and initialize it. However, + // another thread could be doing the same thing concurrently, so there is a + // race to see who initializes `DESERIALIZED` first, and we need to be + // prepared to both win or lose that race. + let peep_opts = PeepholeOptimizations::deserialize(SERIALIZED) + .expect("should always be able to deserialize `preopt.serialized`"); + let peep_opts = Box::into_raw(Box::new(peep_opts)); + + // Only update `DESERIALIZE` if it is still null, attempting to perform the + // one-time transition from null -> non-null. + if DESERIALIZED + .compare_and_swap(ptr::null_mut(), peep_opts, Ordering::SeqCst) + .is_null() + { + // We won the race to initialize `DESERIALIZED`. + debug_assert_eq!(DESERIALIZED.load(Ordering::SeqCst), peep_opts); + let peep_opts = unsafe { &*peep_opts }; + return peep_opts.optimizer(isa); + } + + // We lost the race to initialize `DESERIALIZED`. Drop our no-longer-needed + // instance of `peep_opts` and get the pointer to the instance that won the + // race. + let _ = unsafe { Box::from_raw(peep_opts) }; + let peep_opts = DESERIALIZED.load(Ordering::SeqCst); + let peep_opts = unsafe { peep_opts.as_ref().unwrap() }; + peep_opts.optimizer(isa) +} + +/// Either a `Value` or an `Inst`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ValueOrInst { + Value(Value), + Inst(Inst), +} + +impl ValueOrInst { + /// Get the underlying `Value` if any. + pub fn value(&self) -> Option { + match *self { + Self::Value(v) => Some(v), + Self::Inst(_) => None, + } + } + + /// Get the underlying `Inst` if any. + pub fn inst(&self) -> Option { + match *self { + Self::Inst(i) => Some(i), + Self::Value(_) => None, + } + } + + /// Unwrap the underlying `Value`, panicking if it is not a `Value. + pub fn unwrap_value(&self) -> Value { + self.value().unwrap() + } + + /// Unwrap the underlying `Inst`, panicking if it is not a `Inst. + pub fn unwrap_inst(&self) -> Inst { + self.inst().unwrap() + } + + /// Is this a `Value`? + pub fn is_value(&self) -> bool { + self.value().is_some() + } + + /// Is this an `Inst`? + pub fn is_inst(&self) -> bool { + self.inst().is_some() + } + + fn resolve_inst(&self, dfg: &DataFlowGraph) -> Option { + match *self { + ValueOrInst::Inst(i) => Some(i), + ValueOrInst::Value(v) => dfg.value_def(v).inst(), + } + } + + fn result_bit_width(&self, dfg: &DataFlowGraph) -> u8 { + match *self { + ValueOrInst::Value(v) => dfg.value_type(v).bits().try_into().unwrap(), + ValueOrInst::Inst(inst) => { + let result = dfg.first_result(inst); + dfg.value_type(result).bits().try_into().unwrap() + } + } + } + + fn to_constant(&self, pos: &mut FuncCursor) -> Option { + let inst = self.resolve_inst(&pos.func.dfg)?; + match pos.func.dfg[inst] { + InstructionData::UnaryImm { + opcode: Opcode::Iconst, + imm, + } => { + let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap(); + let x: i64 = imm.into(); + Some(Constant::Int(x as u64, width)) + } + InstructionData::UnaryBool { + opcode: Opcode::Bconst, + imm, + } => { + let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap(); + Some(Constant::Bool(imm, width)) + } + _ => None, + } + } +} + +impl From for ValueOrInst { + fn from(v: Value) -> ValueOrInst { + ValueOrInst::Value(v) + } +} + +impl From for ValueOrInst { + fn from(i: Inst) -> ValueOrInst { + ValueOrInst::Inst(i) + } +} + +/// Get the fixed bit width of `bit_width`, or if it is polymorphic, the bit +/// width of `root`. +fn bit_width(dfg: &DataFlowGraph, bit_width: BitWidth, root: Inst) -> u8 { + bit_width.fixed_width().unwrap_or_else(|| { + let tyvar = dfg.ctrl_typevar(root); + let ty = dfg.compute_result_type(root, 0, tyvar).unwrap(); + u8::try_from(ty.bits()).unwrap() + }) +} + +/// Convert the constant `c` into an instruction. +fn const_to_value<'a>(builder: impl InstBuilder<'a>, c: Constant, root: Inst) -> Value { + match c { + Constant::Bool(b, width) => { + let width = bit_width(builder.data_flow_graph(), width, root); + let ty = match width { + 1 => types::B1, + 8 => types::B8, + 16 => types::B16, + 32 => types::B32, + 64 => types::B64, + 128 => types::B128, + _ => unreachable!(), + }; + builder.bconst(ty, b) + } + Constant::Int(x, width) => { + let width = bit_width(builder.data_flow_graph(), width, root); + let ty = match width { + 1 | 8 => types::I8, + 16 => types::I16, + 32 => types::I32, + 64 => types::I64, + 128 => types::I128, + _ => unreachable!(), + }; + builder.iconst(ty, x as i64) + } + } +} + +fn part_to_inst(pos: &mut FuncCursor, root: Inst, part: Part) -> Option { + match part { + Part::Instruction(ValueOrInst::Inst(inst)) => Some(inst), + Part::Instruction(ValueOrInst::Value(v)) => { + let inst = pos.func.dfg.value_def(v).inst()?; + if pos.func.dfg.inst_results(inst).len() == 1 { + Some(inst) + } else { + None + } + } + Part::Constant(c) => { + let v = const_to_value(pos.ins(), c, root); + let inst = pos.func.dfg.value_def(v).unwrap_inst(); + Some(inst) + } + Part::ConditionCode(_) => None, + } +} + +fn part_to_value(pos: &mut FuncCursor, root: Inst, part: Part) -> Option { + match part { + Part::Instruction(ValueOrInst::Inst(inst)) => { + pos.func.dfg.inst_results(inst).first().copied() + } + Part::Instruction(ValueOrInst::Value(v)) => Some(v), + Part::Constant(c) => Some(const_to_value(pos.ins(), c, root)), + Part::ConditionCode(_) => None, + } +} + +impl Opcode { + fn to_peepmatic_operator(&self) -> Option { + macro_rules! convert { + ( $( $op:ident $(,)* )* ) => { + match self { + $( Self::$op => Some(Operator::$op), )* + _ => None, + } + } + } + + convert!( + AdjustSpDown, + AdjustSpDownImm, + Band, + BandImm, + Bconst, + Bint, + Bor, + BorImm, + Brnz, + Brz, + Bxor, + BxorImm, + Iadd, + IaddImm, + Icmp, + IcmpImm, + Iconst, + Ifcmp, + IfcmpImm, + Imul, + ImulImm, + Ireduce, + IrsubImm, + Ishl, + IshlImm, + Isub, + Rotl, + RotlImm, + Rotr, + RotrImm, + Sdiv, + SdivImm, + Select, + Sextend, + Srem, + SremImm, + Sshr, + SshrImm, + Trapnz, + Trapz, + Udiv, + UdivImm, + Uextend, + Urem, + UremImm, + Ushr, + UshrImm, + ) + } +} + +impl TryFrom for Imm64 { + type Error = &'static str; + + fn try_from(c: Constant) -> Result { + match c { + Constant::Int(x, _) => Ok(Imm64::from(x as i64)), + Constant::Bool(..) => Err("cannot create Imm64 from Constant::Bool"), + } + } +} + +impl Into for Imm64 { + #[inline] + fn into(self) -> Constant { + let x: i64 = self.into(); + Constant::Int(x as _, BitWidth::SixtyFour) + } +} + +impl Into> for Imm64 { + #[inline] + fn into(self) -> Part { + let c: Constant = self.into(); + c.into() + } +} + +fn part_to_imm64(pos: &mut FuncCursor, part: Part) -> Imm64 { + return match part { + Part::Instruction(x) => match x.to_constant(pos).unwrap_or_else(|| cannot_convert()) { + Constant::Int(x, _) => (x as i64).into(), + Constant::Bool(..) => cannot_convert(), + }, + Part::Constant(Constant::Int(x, _)) => (x as i64).into(), + Part::ConditionCode(_) | Part::Constant(Constant::Bool(..)) => cannot_convert(), + }; + + #[inline(never)] + #[cold] + fn cannot_convert() -> ! { + panic!("cannot convert part into `Imm64`") + } +} + +impl Into for Uimm64 { + #[inline] + fn into(self) -> Constant { + let x: u64 = self.into(); + Constant::Int(x, BitWidth::SixtyFour) + } +} + +impl Into> for Uimm64 { + #[inline] + fn into(self) -> Part { + let c: Constant = self.into(); + c.into() + } +} + +fn peepmatic_to_intcc(cc: ConditionCode) -> IntCC { + match cc { + ConditionCode::Eq => IntCC::Equal, + ConditionCode::Ne => IntCC::NotEqual, + ConditionCode::Slt => IntCC::SignedLessThan, + ConditionCode::Sle => IntCC::SignedGreaterThanOrEqual, + ConditionCode::Sgt => IntCC::SignedGreaterThan, + ConditionCode::Sge => IntCC::SignedLessThanOrEqual, + ConditionCode::Ult => IntCC::UnsignedLessThan, + ConditionCode::Uge => IntCC::UnsignedGreaterThanOrEqual, + ConditionCode::Ugt => IntCC::UnsignedGreaterThan, + ConditionCode::Ule => IntCC::UnsignedLessThanOrEqual, + ConditionCode::Of => IntCC::Overflow, + ConditionCode::Nof => IntCC::NotOverflow, + } +} + +fn intcc_to_peepmatic(cc: IntCC) -> ConditionCode { + match cc { + IntCC::Equal => ConditionCode::Eq, + IntCC::NotEqual => ConditionCode::Ne, + IntCC::SignedLessThan => ConditionCode::Slt, + IntCC::SignedGreaterThanOrEqual => ConditionCode::Sle, + IntCC::SignedGreaterThan => ConditionCode::Sgt, + IntCC::SignedLessThanOrEqual => ConditionCode::Sge, + IntCC::UnsignedLessThan => ConditionCode::Ult, + IntCC::UnsignedGreaterThanOrEqual => ConditionCode::Uge, + IntCC::UnsignedGreaterThan => ConditionCode::Ugt, + IntCC::UnsignedLessThanOrEqual => ConditionCode::Ule, + IntCC::Overflow => ConditionCode::Of, + IntCC::NotOverflow => ConditionCode::Nof, + } +} + +fn get_immediate(dfg: &DataFlowGraph, inst: Inst, i: usize) -> Part { + return match dfg[inst] { + InstructionData::BinaryImm { imm, .. } if i == 0 => imm.into(), + InstructionData::BranchIcmp { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(), + InstructionData::BranchInt { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(), + InstructionData::IntCompare { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(), + InstructionData::IntCompareImm { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(), + InstructionData::IntCompareImm { imm, .. } if i == 1 => imm.into(), + InstructionData::IntCond { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(), + InstructionData::IntCondTrap { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(), + InstructionData::IntSelect { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(), + InstructionData::UnaryBool { imm, .. } if i == 0 => { + Constant::Bool(imm, BitWidth::Polymorphic).into() + } + InstructionData::UnaryImm { imm, .. } if i == 0 => imm.into(), + ref otherwise => unsupported(otherwise), + }; + + #[inline(never)] + #[cold] + fn unsupported(data: &InstructionData) -> ! { + panic!("unsupported instruction data: {:?}", data) + } +} + +fn get_argument(dfg: &DataFlowGraph, inst: Inst, i: usize) -> Option { + dfg.inst_args(inst).get(i).copied() +} + +fn peepmatic_ty_to_ir_ty(ty: Type, dfg: &DataFlowGraph, root: Inst) -> types::Type { + match (ty.kind, bit_width(dfg, ty.bit_width, root)) { + (Kind::Int, 1) | (Kind::Int, 8) => types::I8, + (Kind::Int, 16) => types::I16, + (Kind::Int, 32) => types::I32, + (Kind::Int, 64) => types::I64, + (Kind::Int, 128) => types::I128, + (Kind::Bool, 1) => types::B1, + (Kind::Bool, 8) => types::I8, + (Kind::Bool, 16) => types::I16, + (Kind::Bool, 32) => types::I32, + (Kind::Bool, 64) => types::I64, + (Kind::Bool, 128) => types::I128, + _ => unreachable!(), + } +} + +impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { + type Context = FuncCursor<'b>; + + type Instruction = ValueOrInst; + + fn replace_instruction( + &self, + pos: &mut FuncCursor<'b>, + old: ValueOrInst, + new: Part, + ) -> ValueOrInst { + log::trace!("replace {:?} with {:?}", old, new); + let old_inst = old.unwrap_inst(); + + // Try to convert `new` to an instruction, because we prefer replacing + // an old instruction with a new one wholesale. However, if the + // replacement cannot be converted to an instruction (e.g. the + // right-hand side is a block/function parameter value) then we change + // the old instruction's result to an alias of the new value. + match part_to_inst(pos, old_inst, new) { + Some(new_inst) => { + pos.func.transplant_inst(old_inst, new_inst); + debug_assert_eq!(pos.current_inst(), Some(old_inst)); + old_inst.into() + } + None => { + let new_value = part_to_value(pos, old_inst, new).unwrap(); + + let old_results = pos.func.dfg.detach_results(old_inst); + let old_results = old_results.as_slice(&pos.func.dfg.value_lists); + assert_eq!(old_results.len(), 1); + let old_value = old_results[0]; + + pos.func.dfg.change_to_alias(old_value, new_value); + pos.func.dfg.replace(old_inst).nop(); + + new_value.into() + } + } + } + + fn get_part_at_path( + &self, + pos: &mut FuncCursor<'b>, + root: ValueOrInst, + path: Path, + ) -> Option> { + // The root is path [0]. + debug_assert!(!path.0.is_empty()); + debug_assert_eq!(path.0[0], 0); + + let mut part = Part::Instruction(root); + for p in path.0[1..].iter().copied() { + let inst = part.as_instruction()?.resolve_inst(&pos.func.dfg)?; + let operator = pos.func.dfg[inst].opcode().to_peepmatic_operator()?; + + if p < operator.immediates_arity() { + part = get_immediate(&pos.func.dfg, inst, p as usize); + continue; + } + + let arg = p - operator.immediates_arity(); + let arg = arg as usize; + let value = get_argument(&pos.func.dfg, inst, arg)?; + part = Part::Instruction(value.into()); + } + + log::trace!("get_part_at_path({:?}) = {:?}", path, part); + Some(part) + } + + fn operator(&self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst) -> Option { + let inst = value_or_inst.resolve_inst(&pos.func.dfg)?; + pos.func.dfg[inst].opcode().to_peepmatic_operator() + } + + fn make_inst_1( + &self, + pos: &mut FuncCursor<'b>, + root: ValueOrInst, + operator: Operator, + r#type: Type, + a: Part, + ) -> ValueOrInst { + log::trace!("make_inst_1: {:?}({:?})", operator, a); + + let root = root.unwrap_inst(); + match operator { + Operator::AdjustSpDown => { + let a = part_to_value(pos, root, a).unwrap(); + pos.ins().adjust_sp_down(a).into() + } + Operator::AdjustSpDownImm => { + let c = a.unwrap_constant(); + let imm = Imm64::try_from(c).unwrap(); + pos.ins().adjust_sp_down_imm(imm).into() + } + Operator::Bconst => { + let c = a.unwrap_constant(); + const_to_value(pos.ins(), c, root).into() + } + Operator::Bint => { + let a = part_to_value(pos, root, a).unwrap(); + let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); + pos.ins().bint(ty, a).into() + } + Operator::Brnz => { + let a = part_to_value(pos, root, a).unwrap(); + + // NB: branching instructions must be the root of an + // optimization's right-hand side, so we get the destination + // block and arguments from the left-hand side's root. Peepmatic + // doesn't currently represent labels or varargs. + let block = pos.func.dfg[root].branch_destination().unwrap(); + let args = pos.func.dfg.inst_args(root)[1..].to_vec(); + + pos.ins().brnz(a, block, &args).into() + } + Operator::Brz => { + let a = part_to_value(pos, root, a).unwrap(); + + // See the comment in the `Operator::Brnz` match argm. + let block = pos.func.dfg[root].branch_destination().unwrap(); + let args = pos.func.dfg.inst_args(root)[1..].to_vec(); + + pos.ins().brz(a, block, &args).into() + } + Operator::Iconst => { + let a = a.unwrap_constant(); + const_to_value(pos.ins(), a, root).into() + } + Operator::Ireduce => { + let a = part_to_value(pos, root, a).unwrap(); + let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); + pos.ins().ireduce(ty, a).into() + } + Operator::Sextend => { + let a = part_to_value(pos, root, a).unwrap(); + let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); + pos.ins().sextend(ty, a).into() + } + Operator::Trapnz => { + let a = part_to_value(pos, root, a).unwrap(); + + // NB: similar to branching instructions (see comment in the + // `Operator::Brnz` match arm) trapping instructions must be the + // root of an optimization's right-hand side, and we get the + // trap code from the root of the left-hand side. Peepmatic + // doesn't currently represent trap codes. + let code = pos.func.dfg[root].trap_code().unwrap(); + + pos.ins().trapnz(a, code).into() + } + Operator::Trapz => { + let a = part_to_value(pos, root, a).unwrap(); + // See comment in the `Operator::Trapnz` match arm. + let code = pos.func.dfg[root].trap_code().unwrap(); + pos.ins().trapz(a, code).into() + } + Operator::Uextend => { + let a = part_to_value(pos, root, a).unwrap(); + let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); + pos.ins().uextend(ty, a).into() + } + _ => unreachable!(), + } + } + + fn make_inst_2( + &self, + pos: &mut FuncCursor<'b>, + root: ValueOrInst, + operator: Operator, + _: Type, + a: Part, + b: Part, + ) -> ValueOrInst { + log::trace!("make_inst_2: {:?}({:?}, {:?})", operator, a, b); + + let root = root.unwrap_inst(); + match operator { + Operator::Band => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().band(a, b).into() + } + Operator::BandImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().band_imm(b, a).into() + } + Operator::Bor => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().bor(a, b).into() + } + Operator::BorImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().bor_imm(b, a).into() + } + Operator::Bxor => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().bxor(a, b).into() + } + Operator::BxorImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().bxor_imm(b, a).into() + } + Operator::Iadd => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().iadd(a, b).into() + } + Operator::IaddImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().iadd_imm(b, a).into() + } + Operator::Ifcmp => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().ifcmp(a, b).into() + } + Operator::IfcmpImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().ifcmp_imm(b, a).into() + } + Operator::Imul => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().imul(a, b).into() + } + Operator::ImulImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().imul_imm(b, a).into() + } + Operator::IrsubImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().irsub_imm(b, a).into() + } + Operator::Ishl => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().ishl(a, b).into() + } + Operator::IshlImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().ishl_imm(b, a).into() + } + Operator::Isub => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().isub(a, b).into() + } + Operator::Rotl => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().rotl(a, b).into() + } + Operator::RotlImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().rotl_imm(b, a).into() + } + Operator::Rotr => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().rotr(a, b).into() + } + Operator::RotrImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().rotr_imm(b, a).into() + } + Operator::Sdiv => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().sdiv(a, b).into() + } + Operator::SdivImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().sdiv_imm(b, a).into() + } + Operator::Srem => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().srem(a, b).into() + } + Operator::SremImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().srem_imm(b, a).into() + } + Operator::Sshr => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().sshr(a, b).into() + } + Operator::SshrImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().sshr_imm(b, a).into() + } + Operator::Udiv => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().udiv(a, b).into() + } + Operator::UdivImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().udiv_imm(b, a).into() + } + Operator::Urem => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().urem(a, b).into() + } + Operator::UremImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().urem_imm(b, a).into() + } + Operator::Ushr => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().ushr(a, b).into() + } + Operator::UshrImm => { + let a = part_to_imm64(pos, a); + let b = part_to_value(pos, root, b).unwrap(); + pos.ins().ushr_imm(b, a).into() + } + _ => unreachable!(), + } + } + + fn make_inst_3( + &self, + pos: &mut FuncCursor<'b>, + root: ValueOrInst, + operator: Operator, + _: Type, + a: Part, + b: Part, + c: Part, + ) -> ValueOrInst { + log::trace!("make_inst_3: {:?}({:?}, {:?}, {:?})", operator, a, b, c); + + let root = root.unwrap_inst(); + match operator { + Operator::Icmp => { + let cond = a.unwrap_condition_code(); + let cond = peepmatic_to_intcc(cond); + let b = part_to_value(pos, root, b).unwrap(); + let c = part_to_value(pos, root, c).unwrap(); + pos.ins().icmp(cond, b, c).into() + } + Operator::IcmpImm => { + let cond = a.unwrap_condition_code(); + let cond = peepmatic_to_intcc(cond); + let imm = part_to_imm64(pos, b); + let c = part_to_value(pos, root, c).unwrap(); + pos.ins().icmp_imm(cond, c, imm).into() + } + Operator::Select => { + let a = part_to_value(pos, root, a).unwrap(); + let b = part_to_value(pos, root, b).unwrap(); + let c = part_to_value(pos, root, c).unwrap(); + pos.ins().select(a, b, c).into() + } + _ => unreachable!(), + } + } + + fn instruction_to_constant( + &self, + pos: &mut FuncCursor<'b>, + value_or_inst: ValueOrInst, + ) -> Option { + value_or_inst.to_constant(pos) + } + + fn instruction_result_bit_width( + &self, + pos: &mut FuncCursor<'b>, + value_or_inst: ValueOrInst, + ) -> u8 { + value_or_inst.result_bit_width(&pos.func.dfg) + } + + fn native_word_size_in_bits(&self, _pos: &mut FuncCursor<'b>) -> u8 { + self.pointer_bits() + } +} diff --git a/cranelift/peepmatic/examples/preopt.peepmatic b/cranelift/codegen/src/preopt.peepmatic similarity index 97% rename from cranelift/peepmatic/examples/preopt.peepmatic rename to cranelift/codegen/src/preopt.peepmatic index 22523f0a01..604460dd57 100644 --- a/cranelift/peepmatic/examples/preopt.peepmatic +++ b/cranelift/codegen/src/preopt.peepmatic @@ -178,8 +178,8 @@ ;; Division and remainder by constants. ;; -;; Note that this section is incomplete, and a bunch of related optimizations -;; are still hand-coded in `simple_preopt.rs`. +;; TODO: this section is incomplete, and a bunch of related optimizations are +;; still hand-coded in `simple_preopt.rs`. ;; (Division by one is handled above.) diff --git a/cranelift/codegen/src/preopt.serialized b/cranelift/codegen/src/preopt.serialized new file mode 100644 index 0000000000000000000000000000000000000000..eadd89e8cdb6e344e1a74b08edabf462c328f1f5 GIT binary patch literal 5716 zcmb`L=XTpL428{frhD(k`(7i1Tii^2$X2=t}2x_u6D!> zTnX9E32Dn}U`w#8LApxPon5)5Q4tx>HaS378h^^hJB>eSkua6+>}_rx@#okH0XQ$cJBc|1E(pn#>F7khZ&!&msv0SnS?0_Jm2nqce*% zT;($o^BD{|Yf_nz1fw+#UU*14#B!YQ5^BMayxrXyn{>UF*bs|;C_GpNuB+ITECF0% z6YIhlicYB0`uL46f0{r8o{*=ji3?n8io3?}j6^ajzDh}L3>1$9WDZ~|g8<;M zVlW3ly5l7TfaxHA8&tO5Rc%*XO_~FWB##F?4|OBC#NHx1vJ?@6i!4PLQ?Bu4r?Au_m1(F_+`On9x0`=uz8q4`dc})b?}$R)5s?bb#Zurvn_PJssdU?dbqk z0;tqsxJ$6nY&<)sYj)2)rFO`#7f@K^_>JbaDDT;ox0b?Ufa5K@GNG5vMrxs;-5(^k zP{2G)iU|G4gn2W_g9Va{7AiTmk#qL|hMjB%#)K#%TCa{7 z)%=#%ancb4qrwYmYgJyjO}#~)FKs?o6&Y&9s!BdrVDo>fl2J*NsEfL5(h&qBrCw)t zk2RrrlPvV|&|-MM`H!?`ku?&fy@QF$l? zVUbC${Cwr)$M<-gpXPqF-r8>ZeP;l}8|9Nt=GrZw@iJ1o@#+zgyYh6IoS2}D-_ufH ztrcrr^>%0u2{K@ddU4Vb1S3B-Y%ST>YUFovcKT-bxso-rcuFc+0GPqnl4YZfvVn}} z;*w=FoEMjD(h&rsjgP~#Dz6IuO^E!kwrkwXo(3U)*d_8O@m(_;{b%2oEMOCS&H{Gr zhb-Xy_AU!}4{`Jt`5XC_W;Pby-_NsvFPWBvyl^aiVwb-9F63sULa}7h!`7E)BLt7G z(A~WbNAJcqLJq`tdd;j+;W+bHGaCzk==7lVr;q#`njeu|Ohg`^v%`H!Y%ENAA+iI$ z9)hhk8G>_H4>6z30+F-XBqBBGA?CUT;@Y_Dc5Rel&xE(_$`4+>Or0PafCs;}D-&Q| z%Y9>dXcTUU9h!=d`f$$*kp24UOD+Mn}0vlEyMr- literal 0 HcmV?d00001 diff --git a/cranelift/codegen/src/simple_preopt.rs b/cranelift/codegen/src/simple_preopt.rs index 5090246cf1..a03c8219d2 100644 --- a/cranelift/codegen/src/simple_preopt.rs +++ b/cranelift/codegen/src/simple_preopt.rs @@ -10,13 +10,12 @@ use crate::divconst_magic_numbers::{MS32, MS64, MU32, MU64}; use crate::flowgraph::ControlFlowGraph; use crate::ir::{ condcodes::{CondCode, IntCC}, - dfg::ValueDef, - immediates, - instructions::{Opcode, ValueList}, - types::{I16, I32, I64, I8}, + instructions::Opcode, + types::{I32, I64}, Block, DataFlowGraph, Function, Inst, InstBuilder, InstructionData, Type, Value, }; use crate::isa::TargetIsa; +use crate::peepmatic::ValueOrInst; use crate::timing; #[inline] @@ -183,12 +182,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso // U32 div by 1: identity // U32 rem by 1: zero - DivRemByConstInfo::DivU32(n1, 1) | DivRemByConstInfo::RemU32(n1, 1) => { - if is_rem { - pos.func.dfg.replace(inst).iconst(I32, 0); - } else { - replace_single_result_with_alias(&mut pos.func.dfg, inst, n1); - } + DivRemByConstInfo::DivU32(_, 1) | DivRemByConstInfo::RemU32(_, 1) => { + unreachable!("unsigned division and remainder by one is handled in `preopt.peepmatic`"); } // U32 div, rem by a power-of-2 @@ -203,7 +198,10 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso let mask = (1u64 << k) - 1; pos.func.dfg.replace(inst).band_imm(n1, mask as i64); } else { - pos.func.dfg.replace(inst).ushr_imm(n1, k as i64); + unreachable!( + "unsigned division by a power of two is handled in \ + `preopt.peepmatic`" + ); } } @@ -253,12 +251,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso // U64 div by 1: identity // U64 rem by 1: zero - DivRemByConstInfo::DivU64(n1, 1) | DivRemByConstInfo::RemU64(n1, 1) => { - if is_rem { - pos.func.dfg.replace(inst).iconst(I64, 0); - } else { - replace_single_result_with_alias(&mut pos.func.dfg, inst, n1); - } + DivRemByConstInfo::DivU64(_, 1) | DivRemByConstInfo::RemU64(_, 1) => { + unreachable!("unsigned division and remainder by one is handled in `preopt.peepmatic`"); } // U64 div, rem by a power-of-2 @@ -273,7 +267,9 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso let mask = (1u64 << k) - 1; pos.func.dfg.replace(inst).band_imm(n1, mask as i64); } else { - pos.func.dfg.replace(inst).ushr_imm(n1, k as i64); + unreachable!( + "unsigned division by a power of two is handled in `preopt.peepmatic`" + ); } } @@ -326,12 +322,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso // S32 div by 1: identity // S32 rem by 1: zero - DivRemByConstInfo::DivS32(n1, 1) | DivRemByConstInfo::RemS32(n1, 1) => { - if is_rem { - pos.func.dfg.replace(inst).iconst(I32, 0); - } else { - replace_single_result_with_alias(&mut pos.func.dfg, inst, n1); - } + DivRemByConstInfo::DivS32(_, 1) | DivRemByConstInfo::RemS32(_, 1) => { + unreachable!("signed division and remainder by one is handled in `preopt.peepmatic`"); } DivRemByConstInfo::DivS32(n1, d) | DivRemByConstInfo::RemS32(n1, d) => { @@ -401,12 +393,8 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso // S64 div by 1: identity // S64 rem by 1: zero - DivRemByConstInfo::DivS64(n1, 1) | DivRemByConstInfo::RemS64(n1, 1) => { - if is_rem { - pos.func.dfg.replace(inst).iconst(I64, 0); - } else { - replace_single_result_with_alias(&mut pos.func.dfg, inst, n1); - } + DivRemByConstInfo::DivS64(_, 1) | DivRemByConstInfo::RemS64(_, 1) => { + unreachable!("division and remaineder by one are handled in `preopt.peepmatic`"); } DivRemByConstInfo::DivS64(n1, d) | DivRemByConstInfo::RemS64(n1, d) => { @@ -468,340 +456,6 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso } } -#[inline] -fn resolve_imm64_value(dfg: &DataFlowGraph, value: Value) -> Option { - if let ValueDef::Result(candidate_inst, _) = dfg.value_def(value) { - if let InstructionData::UnaryImm { - opcode: Opcode::Iconst, - imm, - } = dfg[candidate_inst] - { - return Some(imm); - } - } - None -} - -/// Try to transform [(x << N) >> N] into a (un)signed-extending move. -/// Returns true if the final instruction has been converted to such a move. -fn try_fold_extended_move( - pos: &mut FuncCursor, - inst: Inst, - opcode: Opcode, - arg: Value, - imm: immediates::Imm64, -) -> bool { - if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) { - if let InstructionData::BinaryImm { - opcode: Opcode::IshlImm, - arg: prev_arg, - imm: prev_imm, - } = &pos.func.dfg[arg_inst] - { - if imm != *prev_imm { - return false; - } - - let dest_ty = pos.func.dfg.ctrl_typevar(inst); - if dest_ty != pos.func.dfg.ctrl_typevar(arg_inst) || !dest_ty.is_int() { - return false; - } - - let imm_bits: i64 = imm.into(); - let ireduce_ty = match (dest_ty.lane_bits() as i64).wrapping_sub(imm_bits) { - 8 => I8, - 16 => I16, - 32 => I32, - _ => return false, - }; - let ireduce_ty = ireduce_ty.by(dest_ty.lane_count()).unwrap(); - - // This becomes a no-op, since ireduce_ty has a smaller lane width than - // the argument type (also the destination type). - let arg = *prev_arg; - let narrower_arg = pos.ins().ireduce(ireduce_ty, arg); - - if opcode == Opcode::UshrImm { - pos.func.dfg.replace(inst).uextend(dest_ty, narrower_arg); - } else { - pos.func.dfg.replace(inst).sextend(dest_ty, narrower_arg); - } - return true; - } - } - false -} - -/// Apply basic simplifications. -/// -/// This folds constants with arithmetic to form `_imm` instructions, and other minor -/// simplifications. -/// -/// Doesn't apply some simplifications if the native word width (in bytes) is smaller than the -/// controlling type's width of the instruction. This would result in an illegal instruction that -/// would likely be expanded back into an instruction on smaller types with the same initial -/// opcode, creating unnecessary churn. -fn simplify(pos: &mut FuncCursor, inst: Inst, native_word_width: u32) { - match pos.func.dfg[inst] { - InstructionData::Binary { opcode, args } => { - if let Some(mut imm) = resolve_imm64_value(&pos.func.dfg, args[1]) { - let new_opcode = match opcode { - Opcode::Iadd => Opcode::IaddImm, - Opcode::Imul => Opcode::ImulImm, - Opcode::Sdiv => Opcode::SdivImm, - Opcode::Udiv => Opcode::UdivImm, - Opcode::Srem => Opcode::SremImm, - Opcode::Urem => Opcode::UremImm, - Opcode::Band => Opcode::BandImm, - Opcode::Bor => Opcode::BorImm, - Opcode::Bxor => Opcode::BxorImm, - Opcode::Rotl => Opcode::RotlImm, - Opcode::Rotr => Opcode::RotrImm, - Opcode::Ishl => Opcode::IshlImm, - Opcode::Ushr => Opcode::UshrImm, - Opcode::Sshr => Opcode::SshrImm, - Opcode::Isub => { - imm = imm.wrapping_neg(); - Opcode::IaddImm - } - Opcode::Ifcmp => Opcode::IfcmpImm, - _ => return, - }; - let ty = pos.func.dfg.ctrl_typevar(inst); - if ty.bytes() <= native_word_width { - pos.func - .dfg - .replace(inst) - .BinaryImm(new_opcode, ty, imm, args[0]); - - // Repeat for BinaryImm simplification. - simplify(pos, inst, native_word_width); - } - } else if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[0]) { - let new_opcode = match opcode { - Opcode::Iadd => Opcode::IaddImm, - Opcode::Imul => Opcode::ImulImm, - Opcode::Band => Opcode::BandImm, - Opcode::Bor => Opcode::BorImm, - Opcode::Bxor => Opcode::BxorImm, - Opcode::Isub => Opcode::IrsubImm, - _ => return, - }; - let ty = pos.func.dfg.ctrl_typevar(inst); - if ty.bytes() <= native_word_width { - pos.func - .dfg - .replace(inst) - .BinaryImm(new_opcode, ty, imm, args[1]); - } - } - } - - InstructionData::Unary { opcode, arg } => { - if let Opcode::AdjustSpDown = opcode { - if let Some(imm) = resolve_imm64_value(&pos.func.dfg, arg) { - // Note this works for both positive and negative immediate values. - pos.func.dfg.replace(inst).adjust_sp_down_imm(imm); - } - } - } - - InstructionData::BinaryImm { opcode, arg, imm } => { - let ty = pos.func.dfg.ctrl_typevar(inst); - - let mut arg = arg; - let mut imm = imm; - match opcode { - Opcode::IaddImm - | Opcode::ImulImm - | Opcode::BorImm - | Opcode::BandImm - | Opcode::BxorImm => { - // Fold binary_op(C2, binary_op(C1, x)) into binary_op(binary_op(C1, C2), x) - if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) { - if let InstructionData::BinaryImm { - opcode: prev_opcode, - arg: prev_arg, - imm: prev_imm, - } = &pos.func.dfg[arg_inst] - { - if opcode == *prev_opcode && ty == pos.func.dfg.ctrl_typevar(arg_inst) { - let lhs: i64 = imm.into(); - let rhs: i64 = (*prev_imm).into(); - let new_imm = match opcode { - Opcode::BorImm => lhs | rhs, - Opcode::BandImm => lhs & rhs, - Opcode::BxorImm => lhs ^ rhs, - Opcode::IaddImm => lhs.wrapping_add(rhs), - Opcode::ImulImm => lhs.wrapping_mul(rhs), - _ => panic!("can't happen"), - }; - let new_imm = immediates::Imm64::from(new_imm); - let new_arg = *prev_arg; - pos.func - .dfg - .replace(inst) - .BinaryImm(opcode, ty, new_imm, new_arg); - imm = new_imm; - arg = new_arg; - } - } - } - } - - Opcode::UshrImm | Opcode::SshrImm => { - if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width - && try_fold_extended_move(pos, inst, opcode, arg, imm) - { - return; - } - } - - _ => {} - }; - - // Replace operations that are no-ops. - match (opcode, imm.into()) { - (Opcode::IaddImm, 0) - | (Opcode::ImulImm, 1) - | (Opcode::SdivImm, 1) - | (Opcode::UdivImm, 1) - | (Opcode::BorImm, 0) - | (Opcode::BandImm, -1) - | (Opcode::BxorImm, 0) - | (Opcode::RotlImm, 0) - | (Opcode::RotrImm, 0) - | (Opcode::IshlImm, 0) - | (Opcode::UshrImm, 0) - | (Opcode::SshrImm, 0) => { - // Alias the result value with the original argument. - replace_single_result_with_alias(&mut pos.func.dfg, inst, arg); - } - (Opcode::ImulImm, 0) | (Opcode::BandImm, 0) => { - // Replace by zero. - pos.func.dfg.replace(inst).iconst(ty, 0); - } - (Opcode::BorImm, -1) => { - // Replace by minus one. - pos.func.dfg.replace(inst).iconst(ty, -1); - } - _ => {} - } - } - - InstructionData::IntCompare { opcode, cond, args } => { - debug_assert_eq!(opcode, Opcode::Icmp); - if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[1]) { - if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width { - pos.func.dfg.replace(inst).icmp_imm(cond, args[0], imm); - } - } - } - - InstructionData::CondTrap { .. } - | InstructionData::Branch { .. } - | InstructionData::Ternary { - opcode: Opcode::Select, - .. - } => { - // Fold away a redundant `bint`. - let condition_def = { - let args = pos.func.dfg.inst_args(inst); - pos.func.dfg.value_def(args[0]) - }; - if let ValueDef::Result(def_inst, _) = condition_def { - if let InstructionData::Unary { - opcode: Opcode::Bint, - arg: bool_val, - } = pos.func.dfg[def_inst] - { - let args = pos.func.dfg.inst_args_mut(inst); - args[0] = bool_val; - } - } - } - - _ => {} - } -} - -struct BranchOptInfo { - br_inst: Inst, - cmp_arg: Value, - args: ValueList, - new_opcode: Opcode, -} - -/// Fold comparisons into branch operations when possible. -/// -/// This matches against operations which compare against zero, then use the -/// result in a `brz` or `brnz` branch. It folds those two operations into a -/// single `brz` or `brnz`. -fn branch_opt(pos: &mut FuncCursor, inst: Inst) { - let mut info = if let InstructionData::Branch { - opcode: br_opcode, - args: ref br_args, - .. - } = pos.func.dfg[inst] - { - let first_arg = { - let args = pos.func.dfg.inst_args(inst); - args[0] - }; - - let icmp_inst = if let ValueDef::Result(icmp_inst, _) = pos.func.dfg.value_def(first_arg) { - icmp_inst - } else { - return; - }; - - if let InstructionData::IntCompareImm { - opcode: Opcode::IcmpImm, - arg: cmp_arg, - cond: cmp_cond, - imm: cmp_imm, - } = pos.func.dfg[icmp_inst] - { - let cmp_imm: i64 = cmp_imm.into(); - if cmp_imm != 0 { - return; - } - - // icmp_imm returns non-zero when the comparison is true. So, if - // we're branching on zero, we need to invert the condition. - let cond = match br_opcode { - Opcode::Brz => cmp_cond.inverse(), - Opcode::Brnz => cmp_cond, - _ => return, - }; - - let new_opcode = match cond { - IntCC::Equal => Opcode::Brz, - IntCC::NotEqual => Opcode::Brnz, - _ => return, - }; - - BranchOptInfo { - br_inst: inst, - cmp_arg, - args: br_args.clone(), - new_opcode, - } - } else { - return; - } - } else { - return; - }; - - info.args.as_mut_slice(&mut pos.func.dfg.value_lists)[0] = info.cmp_arg; - if let InstructionData::Branch { ref mut opcode, .. } = pos.func.dfg[info.br_inst] { - *opcode = info.new_opcode; - } else { - panic!(); - } -} - enum BranchOrderKind { BrzToBrnz(Value), BrnzToBrz(Value), @@ -945,14 +599,20 @@ fn branch_order(pos: &mut FuncCursor, cfg: &mut ControlFlowGraph, block: Block, } /// The main pre-opt pass. -pub fn do_preopt(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &dyn TargetIsa) { +pub fn do_preopt<'func, 'isa>( + func: &'func mut Function, + cfg: &mut ControlFlowGraph, + isa: &'isa dyn TargetIsa, +) { let _tt = timing::preopt(); + let mut pos = FuncCursor::new(func); - let native_word_width = isa.pointer_bytes(); + let mut preopt = crate::peepmatic::preopt(isa); + while let Some(block) = pos.next_block() { while let Some(inst) = pos.next_inst() { - // Apply basic simplifications. - simplify(&mut pos, inst, native_word_width as u32); + preopt.apply_all(&mut pos, ValueOrInst::Inst(inst)); + let inst = pos.current_inst().unwrap(); // Try to transform divide-by-constant into simpler operations. if let Some(divrem_info) = get_div_info(inst, &pos.func.dfg) { @@ -960,7 +620,6 @@ pub fn do_preopt(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &dyn Targ continue; } - branch_opt(&mut pos, inst); branch_order(&mut pos, cfg, block, inst); } } diff --git a/cranelift/filetests/filetests/isa/x86/isub_imm-i8.clif b/cranelift/filetests/filetests/isa/x86/isub_imm-i8.clif index 018ac95fbc..e59226c7de 100644 --- a/cranelift/filetests/filetests/isa/x86/isub_imm-i8.clif +++ b/cranelift/filetests/filetests/isa/x86/isub_imm-i8.clif @@ -6,9 +6,9 @@ function u0:0(i8) -> i8 fast { block0(v0: i8): v1 = iconst.i8 0 v2 = isub v1, v0 - ; check: v3 = uextend.i32 v0 - ; nextln: v5 = iconst.i32 0 - ; nextln = isub v5, v3 - ; nextln = ireduce.i8 v4 + ; check: v4 = uextend.i32 v0 + ; nextln: v6 = iconst.i32 0 + ; nextln: v5 = isub v6, v4 + ; nextln: v2 = ireduce.i8 v5 return v2 } diff --git a/cranelift/filetests/filetests/simple_preopt/div_by_const_indirect.clif b/cranelift/filetests/filetests/simple_preopt/div_by_const_indirect.clif index 101e4eb201..4a4b7a80b6 100644 --- a/cranelift/filetests/filetests/simple_preopt/div_by_const_indirect.clif +++ b/cranelift/filetests/filetests/simple_preopt/div_by_const_indirect.clif @@ -7,14 +7,13 @@ function %indir_udiv32(i32) -> i32 { block0(v0: i32): v1 = iconst.i32 7 v2 = udiv v0, v1 - ; check: iconst.i32 7 - ; check: iconst.i32 0x2492_4925 - ; check: umulhi v0, v3 - ; check: isub v0, v4 - ; check: ushr_imm v5, 1 - ; check: iadd v6, v4 - ; check: v8 = ushr_imm v7, 2 - ; check: v2 -> v8 + ; check: v4 = iconst.i32 0x2492_4925 + ; nextln: v5 = umulhi v0, v4 + ; nextln: v6 = isub v0, v5 + ; nextln: v7 = ushr_imm v6, 1 + ; nextln: v8 = iadd v7, v5 + ; nextln: v9 = ushr_imm v8, 2 + ; nextln: v2 -> v9 return v2 } @@ -22,13 +21,12 @@ function %indir_sdiv32(i32) -> i32 { block0(v0: i32): v1 = iconst.i32 -17 v2 = sdiv v0, v1 - ; check: iconst.i32 -17 - ; check: iconst.i32 0xffff_ffff_8787_8787 - ; check: smulhi v0, v3 - ; check: sshr_imm v4, 3 - ; check: ushr_imm v5, 31 - ; check: v7 = iadd v5, v6 - ; check: v2 -> v7 + ; check: v4 = iconst.i32 0xffff_ffff_8787_8787 + ; nextln: v5 = smulhi v0, v4 + ; nextln: v6 = sshr_imm v5, 3 + ; nextln: v7 = ushr_imm v6, 31 + ; nextln: v8 = iadd v6, v7 + ; nextln: v2 -> v8 return v2 } @@ -36,11 +34,10 @@ function %indir_udiv64(i64) -> i64 { block0(v0: i64): v1 = iconst.i64 1337 v2 = udiv v0, v1 - ; check: iconst.i64 1337 - ; check: iconst.i64 0xc411_9d95_2866_a139 - ; check: umulhi v0, v3 - ; check: v5 = ushr_imm v4, 10 - ; check: v2 -> v5 + ; check: v4 = iconst.i64 0xc411_9d95_2866_a139 + ; nextln: v5 = umulhi v0, v4 + ; nextln: v6 = ushr_imm v5, 10 + ; nextln: v2 -> v6 return v2 } @@ -48,12 +45,11 @@ function %indir_sdiv64(i64) -> i64 { block0(v0: i64): v1 = iconst.i64 -90210 v2 = sdiv v0, v1 - ; check: iconst.i64 0xffff_ffff_fffe_9f9e - ; check: iconst.i64 0xd181_4ee8_939c_b8bb - ; check: smulhi v0, v3 - ; check: sshr_imm v4, 14 - ; check: ushr_imm v5, 63 - ; check: v7 = iadd v5, v6 - ; check: v2 -> v7 + ; check: v4 = iconst.i64 0xd181_4ee8_939c_b8bb + ; nextln: v5 = smulhi v0, v4 + ; nextln: v6 = sshr_imm v5, 14 + ; nextln: v7 = ushr_imm v6, 63 + ; nextln: v8 = iadd v6, v7 + ; nextln: v2 -> v8 return v2 } diff --git a/cranelift/filetests/filetests/simple_preopt/replace_branching_instructions_and_cfg_predecessors.clif b/cranelift/filetests/filetests/simple_preopt/replace_branching_instructions_and_cfg_predecessors.clif new file mode 100644 index 0000000000..702896c22d --- /dev/null +++ b/cranelift/filetests/filetests/simple_preopt/replace_branching_instructions_and_cfg_predecessors.clif @@ -0,0 +1,22 @@ +test simple_preopt +target x86_64 + +function u0:2(i64 , i64) { + gv1 = load.i64 notrap aligned gv0 + heap0 = static gv1 + block0(v0: i64, v1: i64): + v16 = iconst.i32 6 + v17 = heap_addr.i64 heap0, v16, 1 + v18 = load.i32 v17 + v19 = iconst.i32 4 + v20 = icmp ne v18, v19 + v21 = bint.i32 v20 + brnz v21, block2 + jump block4 + block4: + jump block1 + block2: + jump block1 + block1: + return +} diff --git a/cranelift/filetests/filetests/simple_preopt/simplify32.clif b/cranelift/filetests/filetests/simple_preopt/simplify32.clif index 2582fd69aa..cf238fb5ed 100644 --- a/cranelift/filetests/filetests/simple_preopt/simplify32.clif +++ b/cranelift/filetests/filetests/simple_preopt/simplify32.clif @@ -58,4 +58,3 @@ block0(v0: i64): ; nextln: v2 = iadd v0, v1 ; nextln: return v2 ; nextln: } - diff --git a/cranelift/filetests/filetests/simple_preopt/simplify64.clif b/cranelift/filetests/filetests/simple_preopt/simplify64.clif index 4ceabdc335..6489c3bd1e 100644 --- a/cranelift/filetests/filetests/simple_preopt/simplify64.clif +++ b/cranelift/filetests/filetests/simple_preopt/simplify64.clif @@ -44,6 +44,37 @@ block0(v0: i32): ; nextln: return v3 ; nextln: } +function %ifcmp_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 2 + v2 = ifcmp v0, v1 + brif eq v2, block1 + jump block2 + +block1: + v3 = iconst.i32 1 + return v3 + +block2: + v4 = iconst.i32 2 + return v4 +} +; sameln: function %ifcmp_imm +; nextln: block0(v0: i32): +; nextln: v1 = iconst.i32 2 +; nextln: v2 = ifcmp_imm v0, 2 +; nextln: brif eq v2, block1 +; nextln: jump block2 +; nextln: +; nextln: block1: +; nextln: v3 = iconst.i32 1 +; nextln: return v3 +; nextln: +; nextln: block2: +; nextln: v4 = iconst.i32 2 +; nextln: return v4 +; nextln: } + function %brz_bint(i32) { block0(v0: i32): v3 = icmp_imm slt v0, 0 diff --git a/cranelift/filetests/filetests/simple_preopt/simplify_instruction_into_alias_of_value.clif b/cranelift/filetests/filetests/simple_preopt/simplify_instruction_into_alias_of_value.clif new file mode 100644 index 0000000000..076d7b17b5 --- /dev/null +++ b/cranelift/filetests/filetests/simple_preopt/simplify_instruction_into_alias_of_value.clif @@ -0,0 +1,17 @@ +test simple_preopt +target x86_64 + +;; The `isub` is a no-op, but we can't replace the whole `isub` instruction with +;; its `v2` operand's instruction because `v2` is one of many results. Instead, +;; we need to make an alias `v3 -> v2`. + +function %replace_inst_with_alias() -> i32 { +block0: + v0 = iconst.i32 0 + v1, v2 = x86_smulx v0, v0 + v3 = isub v2, v0 + ; check: v0 = iconst.i32 0 + ; nextln: v1, v2 = x86_smulx v0, v0 + ; nextln: v3 -> v2 + return v3 +} diff --git a/cranelift/filetests/src/test_simple_preopt.rs b/cranelift/filetests/src/test_simple_preopt.rs index 286a86ba23..1463d1c69a 100644 --- a/cranelift/filetests/src/test_simple_preopt.rs +++ b/cranelift/filetests/src/test_simple_preopt.rs @@ -38,6 +38,7 @@ impl SubTest for TestSimplePreopt { .preopt(isa) .map_err(|e| pretty_error(&comp_ctx.func, context.isa, Into::into(e)))?; let text = &comp_ctx.func.display(isa).to_string(); + log::debug!("After simple_preopt:\n{}", text); run_filecheck(&text, context) } } diff --git a/cranelift/peepmatic/src/lib.rs b/cranelift/peepmatic/src/lib.rs index 0cf4147db8..407ba3670d 100755 --- a/cranelift/peepmatic/src/lib.rs +++ b/cranelift/peepmatic/src/lib.rs @@ -160,6 +160,6 @@ mod tests { #[test] fn compile_preopt() { - assert_compiles("examples/preopt.peepmatic"); + assert_compiles("../codegen/src/preopt.peepmatic"); } } From c2ec1523bcc924e6eabf3bd212bf57e1654fd27e Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 1 May 2020 17:00:26 -0700 Subject: [PATCH 10/22] ci: Test rebuilding the peephole optimizers in CI --- .github/workflows/main.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 402ea814db..1edd76aeeb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -200,6 +200,20 @@ jobs: env: RUST_BACKTRACE: 1 + rebuild_peephole_optimizers: + name: Rebuild Peephole Optimizers + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: true + - name: Build + run: | + cd cranelift && cargo build --features 'cranelift-codegen/rebuild-peephole-optimizers' + - name: Test + run: | + cd cranelift && cargo test --features 'cranelift-codegen/rebuild-peephole-optimizers' + # Perform all tests (debug mode) for `wasmtime`. This runs stable/beta/nightly # channels of Rust as well as macOS/Linux/Windows. test: From 9a1f8038b7ca387316bbcae86f87c9e77ef246f9 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Mon, 4 May 2020 15:49:01 -0700 Subject: [PATCH 11/22] peepmatic: Do not transplant instructions whose results are potentially used elsewhere --- cranelift/codegen/src/peepmatic.rs | 169 +++++++++++------- ...order_instructions_when_transplanting.clif | 22 +++ 2 files changed, 125 insertions(+), 66 deletions(-) create mode 100644 cranelift/filetests/filetests/simple_preopt/do_not_reorder_instructions_when_transplanting.clif diff --git a/cranelift/codegen/src/peepmatic.rs b/cranelift/codegen/src/peepmatic.rs index f78f5ccd3e..24dd0981eb 100644 --- a/cranelift/codegen/src/peepmatic.rs +++ b/cranelift/codegen/src/peepmatic.rs @@ -208,26 +208,6 @@ fn const_to_value<'a>(builder: impl InstBuilder<'a>, c: Constant, root: Inst) -> } } -fn part_to_inst(pos: &mut FuncCursor, root: Inst, part: Part) -> Option { - match part { - Part::Instruction(ValueOrInst::Inst(inst)) => Some(inst), - Part::Instruction(ValueOrInst::Value(v)) => { - let inst = pos.func.dfg.value_def(v).inst()?; - if pos.func.dfg.inst_results(inst).len() == 1 { - Some(inst) - } else { - None - } - } - Part::Constant(c) => { - let v = const_to_value(pos.ins(), c, root); - let inst = pos.func.dfg.value_def(v).unwrap_inst(); - Some(inst) - } - Part::ConditionCode(_) => None, - } -} - fn part_to_value(pos: &mut FuncCursor, root: Inst, part: Part) -> Option { match part { Part::Instruction(ValueOrInst::Inst(inst)) => { @@ -454,14 +434,30 @@ impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { new: Part, ) -> ValueOrInst { log::trace!("replace {:?} with {:?}", old, new); - let old_inst = old.unwrap_inst(); + let old_inst = old.resolve_inst(&pos.func.dfg).unwrap(); // Try to convert `new` to an instruction, because we prefer replacing // an old instruction with a new one wholesale. However, if the // replacement cannot be converted to an instruction (e.g. the // right-hand side is a block/function parameter value) then we change // the old instruction's result to an alias of the new value. - match part_to_inst(pos, old_inst, new) { + let new_inst = match new { + Part::Instruction(ValueOrInst::Inst(inst)) => Some(inst), + Part::Instruction(ValueOrInst::Value(_)) => { + // Do not try and follow the value definition. If we transplant + // this value's instruction, and there are other uses of this + // value, then we could mess up ordering between instructions. + None + } + Part::Constant(c) => { + let v = const_to_value(pos.ins(), c, old_inst); + let inst = pos.func.dfg.value_def(v).unwrap_inst(); + Some(inst) + } + Part::ConditionCode(_) => None, + }; + + match new_inst { Some(new_inst) => { pos.func.transplant_inst(old_inst, new_inst); debug_assert_eq!(pos.current_inst(), Some(old_inst)); @@ -528,7 +524,7 @@ impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { ) -> ValueOrInst { log::trace!("make_inst_1: {:?}({:?})", operator, a); - let root = root.unwrap_inst(); + let root = root.resolve_inst(&pos.func.dfg).unwrap(); match operator { Operator::AdjustSpDown => { let a = part_to_value(pos, root, a).unwrap(); @@ -541,12 +537,14 @@ impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { } Operator::Bconst => { let c = a.unwrap_constant(); - const_to_value(pos.ins(), c, root).into() + let val = const_to_value(pos.ins(), c, root); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Bint => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); - pos.ins().bint(ty, a).into() + let val = pos.ins().bint(ty, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Brnz => { let a = part_to_value(pos, root, a).unwrap(); @@ -571,17 +569,20 @@ impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { } Operator::Iconst => { let a = a.unwrap_constant(); - const_to_value(pos.ins(), a, root).into() + let val = const_to_value(pos.ins(), a, root); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Ireduce => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); - pos.ins().ireduce(ty, a).into() + let val = pos.ins().ireduce(ty, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Sextend => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); - pos.ins().sextend(ty, a).into() + let val = pos.ins().sextend(ty, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Trapnz => { let a = part_to_value(pos, root, a).unwrap(); @@ -604,7 +605,8 @@ impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { Operator::Uextend => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); - pos.ins().uextend(ty, a).into() + let val = pos.ins().uextend(ty, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } _ => unreachable!(), } @@ -621,167 +623,199 @@ impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { ) -> ValueOrInst { log::trace!("make_inst_2: {:?}({:?}, {:?})", operator, a, b); - let root = root.unwrap_inst(); + let root = root.resolve_inst(&pos.func.dfg).unwrap(); match operator { Operator::Band => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().band(a, b).into() + let val = pos.ins().band(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::BandImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().band_imm(b, a).into() + let val = pos.ins().band_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Bor => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().bor(a, b).into() + let val = pos.ins().bor(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::BorImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().bor_imm(b, a).into() + let val = pos.ins().bor_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Bxor => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().bxor(a, b).into() + let val = pos.ins().bxor(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::BxorImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().bxor_imm(b, a).into() + let val = pos.ins().bxor_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Iadd => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().iadd(a, b).into() + let val = pos.ins().iadd(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::IaddImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().iadd_imm(b, a).into() + let val = pos.ins().iadd_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Ifcmp => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().ifcmp(a, b).into() + let val = pos.ins().ifcmp(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::IfcmpImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().ifcmp_imm(b, a).into() + let val = pos.ins().ifcmp_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Imul => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().imul(a, b).into() + let val = pos.ins().imul(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::ImulImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().imul_imm(b, a).into() + let val = pos.ins().imul_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::IrsubImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().irsub_imm(b, a).into() + let val = pos.ins().irsub_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Ishl => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().ishl(a, b).into() + let val = pos.ins().ishl(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::IshlImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().ishl_imm(b, a).into() + let val = pos.ins().ishl_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Isub => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().isub(a, b).into() + let val = pos.ins().isub(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Rotl => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().rotl(a, b).into() + let val = pos.ins().rotl(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::RotlImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().rotl_imm(b, a).into() + let val = pos.ins().rotl_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Rotr => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().rotr(a, b).into() + let val = pos.ins().rotr(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::RotrImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().rotr_imm(b, a).into() + let val = pos.ins().rotr_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Sdiv => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().sdiv(a, b).into() + let val = pos.ins().sdiv(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::SdivImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().sdiv_imm(b, a).into() + let val = pos.ins().sdiv_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Srem => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().srem(a, b).into() + let val = pos.ins().srem(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::SremImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().srem_imm(b, a).into() + let val = pos.ins().srem_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Sshr => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().sshr(a, b).into() + let val = pos.ins().sshr(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::SshrImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().sshr_imm(b, a).into() + let val = pos.ins().sshr_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Udiv => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().udiv(a, b).into() + let val = pos.ins().udiv(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::UdivImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().udiv_imm(b, a).into() + let val = pos.ins().udiv_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Urem => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().urem(a, b).into() + let val = pos.ins().urem(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::UremImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().urem_imm(b, a).into() + let val = pos.ins().urem_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Ushr => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().ushr(a, b).into() + let val = pos.ins().ushr(a, b); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::UshrImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); - pos.ins().ushr_imm(b, a).into() + let val = pos.ins().ushr_imm(b, a); + pos.func.dfg.value_def(val).unwrap_inst().into() } _ => unreachable!(), } @@ -799,27 +833,30 @@ impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { ) -> ValueOrInst { log::trace!("make_inst_3: {:?}({:?}, {:?}, {:?})", operator, a, b, c); - let root = root.unwrap_inst(); + let root = root.resolve_inst(&pos.func.dfg).unwrap(); match operator { Operator::Icmp => { let cond = a.unwrap_condition_code(); let cond = peepmatic_to_intcc(cond); let b = part_to_value(pos, root, b).unwrap(); let c = part_to_value(pos, root, c).unwrap(); - pos.ins().icmp(cond, b, c).into() + let val = pos.ins().icmp(cond, b, c); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::IcmpImm => { let cond = a.unwrap_condition_code(); let cond = peepmatic_to_intcc(cond); let imm = part_to_imm64(pos, b); let c = part_to_value(pos, root, c).unwrap(); - pos.ins().icmp_imm(cond, c, imm).into() + let val = pos.ins().icmp_imm(cond, c, imm); + pos.func.dfg.value_def(val).unwrap_inst().into() } Operator::Select => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let c = part_to_value(pos, root, c).unwrap(); - pos.ins().select(a, b, c).into() + let val = pos.ins().select(a, b, c); + pos.func.dfg.value_def(val).unwrap_inst().into() } _ => unreachable!(), } diff --git a/cranelift/filetests/filetests/simple_preopt/do_not_reorder_instructions_when_transplanting.clif b/cranelift/filetests/filetests/simple_preopt/do_not_reorder_instructions_when_transplanting.clif new file mode 100644 index 0000000000..a1e9f47f5a --- /dev/null +++ b/cranelift/filetests/filetests/simple_preopt/do_not_reorder_instructions_when_transplanting.clif @@ -0,0 +1,22 @@ +test simple_preopt +target x86_64 + +;; Test that although v5 can be replaced with v1, we don't transplant `load.i32 +;; v0` on top of `iadd v3, v4`, because that would move the load past other uses +;; of its result. + +function %foo(i64) -> i32 { +block0(v0: i64): + v1 = load.i32 v0 + v2 = iconst.i32 16 + v3 = iadd_imm v1, -16 + v4 = iconst.i32 16 + v5 = iadd v3, v4 + ; check: v1 = load.i32 v0 + ; nextln: v5 -> v1 + ; nextln: v2 = iconst.i32 16 + ; nextln: v3 = iadd_imm v1, -16 + ; nextln: v4 = iconst.i32 16 + ; nextln: nop + return v5 +} From 469104c4d32cdc9b1fde068ac933cd946d8c3356 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 7 May 2020 12:15:50 -0700 Subject: [PATCH 12/22] peepmatic: Make the results of match operations a smaller and more cache friendly --- cranelift/codegen/src/peepmatic.rs | 5 +- cranelift/codegen/src/preopt.serialized | Bin 5716 -> 6088 bytes cranelift/peepmatic/crates/runtime/src/cc.rs | 2 + .../crates/runtime/src/instruction_set.rs | 11 +- .../crates/runtime/src/integer_interner.rs | 17 +- .../peepmatic/crates/runtime/src/linear.rs | 47 ++- .../peepmatic/crates/runtime/src/operator.rs | 3 + .../crates/runtime/src/optimizations.rs | 4 +- .../peepmatic/crates/runtime/src/optimizer.rs | 156 +++++--- cranelift/peepmatic/crates/test/src/lib.rs | 8 +- cranelift/peepmatic/src/automatize.rs | 4 +- cranelift/peepmatic/src/dot_fmt.rs | 15 +- cranelift/peepmatic/src/linear_passes.rs | 365 +++++++++++++++--- cranelift/peepmatic/src/linearize.rs | 92 +++-- 14 files changed, 580 insertions(+), 149 deletions(-) diff --git a/cranelift/codegen/src/peepmatic.rs b/cranelift/codegen/src/peepmatic.rs index 24dd0981eb..ba9aeb38d5 100644 --- a/cranelift/codegen/src/peepmatic.rs +++ b/cranelift/codegen/src/peepmatic.rs @@ -422,7 +422,10 @@ fn peepmatic_ty_to_ir_ty(ty: Type, dfg: &DataFlowGraph, root: Inst) -> types::Ty } } -impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { +// NB: the unsafe contract we must uphold here is that our implementation of +// `instruction_result_bit_width` must always return a valid, non-zero bit +// width. +unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { type Context = FuncCursor<'b>; type Instruction = ValueOrInst; diff --git a/cranelift/codegen/src/preopt.serialized b/cranelift/codegen/src/preopt.serialized index eadd89e8cdb6e344e1a74b08edabf462c328f1f5..035de61c6289a0a3123e2f18e4104627db5cb361 100644 GIT binary patch literal 6088 zcmchb=W-h{41{$~b)4RNo8le!^F-O+L>v0b`@4B0}0lu7Wc~33b4cx=VOrLg+?q;zKU>z`%o5 z;&BzrnoR&V@tM|zGc@glQ5+wC<$u>%6Tu>OtIN*c3mXqh6~-NuG*#q9xB$>VflSJ_ziB^hG&IBL0UHhLkFpJw#x zJ(yQwux&-u?!ChIR@Dgu|5q^}{4d5hE(W=ULB0ze4DfRuj=`E4$E}0uhstpImYZcgrONX?8t{PTK&~J6|4UGJ_73`{dJ6hnEKS~@47fJAnvwS zpN@*x__1l9UW9wAPr}K4@^_$<_Q^4*njDN?pVq~J0jbteTHZ6S%|~|$&zU&A_pzpy zA9t&I6YO*+WnISCJ16goZ(?|@ks250Dqh@U4vCG$WtP}Zbj8-4RTu8C%qcAa^w2zJf4mlXTKCX)c1M=sI{EUxE z;PbDF`=HDFA00%l?}b}@@SMIa#KQ4xKPMe$?^Duo2fR-@em}iUy1kI~+eRk7171p~ zGv6D}laB9HUo&QYD?W{(cMEjBMr3+I&G=%Iomu_j+4Pxs#* z{0r90p?}I0s$Be@W=O`a+Bw$`P#7Wia^)Cdb5_^o;=Qql>+hD7BwXGP*WuEu9&YZZ zaOV+wBZlN)r{3%bezm0fj=;7hxBs}hyR&V)^ P?_uNPBy8?8|NrYRqEs!S literal 5716 zcmb`L=XTpL428{frhD(k`(7i1Tii^2$X2=t}2x_u6D!> zTnX9E32Dn}U`w#8LApxPon5)5Q4tx>HaS378h^^hJB>eSkua6+>}_rx@#okH0XQ$cJBc|1E(pn#>F7khZ&!&msv0SnS?0_Jm2nqce*% zT;($o^BD{|Yf_nz1fw+#UU*14#B!YQ5^BMayxrXyn{>UF*bs|;C_GpNuB+ITECF0% z6YIhlicYB0`uL46f0{r8o{*=ji3?n8io3?}j6^ajzDh}L3>1$9WDZ~|g8<;M zVlW3ly5l7TfaxHA8&tO5Rc%*XO_~FWB##F?4|OBC#NHx1vJ?@6i!4PLQ?Bu4r?Au_m1(F_+`On9x0`=uz8q4`dc})b?}$R)5s?bb#Zurvn_PJssdU?dbqk z0;tqsxJ$6nY&<)sYj)2)rFO`#7f@K^_>JbaDDT;ox0b?Ufa5K@GNG5vMrxs;-5(^k zP{2G)iU|G4gn2W_g9Va{7AiTmk#qL|hMjB%#)K#%TCa{7 z)%=#%ancb4qrwYmYgJyjO}#~)FKs?o6&Y&9s!BdrVDo>fl2J*NsEfL5(h&qBrCw)t zk2RrrlPvV|&|-MM`H!?`ku?&fy@QF$l? zVUbC${Cwr)$M<-gpXPqF-r8>ZeP;l}8|9Nt=GrZw@iJ1o@#+zgyYh6IoS2}D-_ufH ztrcrr^>%0u2{K@ddU4Vb1S3B-Y%ST>YUFovcKT-bxso-rcuFc+0GPqnl4YZfvVn}} z;*w=FoEMjD(h&rsjgP~#Dz6IuO^E!kwrkwXo(3U)*d_8O@m(_;{b%2oEMOCS&H{Gr zhb-Xy_AU!}4{`Jt`5XC_W;Pby-_NsvFPWBvyl^aiVwb-9F63sULa}7h!`7E)BLt7G z(A~WbNAJcqLJq`tdd;j+;W+bHGaCzk==7lVr;q#`njeu|Ohg`^v%`H!Y%ENAA+iI$ z9)hhk8G>_H4>6z30+F-XBqBBGA?CUT;@Y_Dc5Rel&xE(_$`4+>Or0PafCs;}D-&Q| z%Y9>dXcTUU9h!=d`f$$*kp24UOD+Mn}0vlEyMr- diff --git a/cranelift/peepmatic/crates/runtime/src/cc.rs b/cranelift/peepmatic/crates/runtime/src/cc.rs index 69f44db2ae..d6cb489eab 100644 --- a/cranelift/peepmatic/crates/runtime/src/cc.rs +++ b/cranelift/peepmatic/crates/runtime/src/cc.rs @@ -12,6 +12,8 @@ use std::fmt; #[repr(u32)] pub enum ConditionCode { /// Equal. + // NB: We convert `ConditionCode` into `NonZeroU32`s with unchecked + // conversions; memory safety relies on no variant being zero. Eq = 1, /// Not equal. diff --git a/cranelift/peepmatic/crates/runtime/src/instruction_set.rs b/cranelift/peepmatic/crates/runtime/src/instruction_set.rs index 1a501d4ce4..4d9a0752e5 100644 --- a/cranelift/peepmatic/crates/runtime/src/instruction_set.rs +++ b/cranelift/peepmatic/crates/runtime/src/instruction_set.rs @@ -21,7 +21,11 @@ use std::fmt::Debug; /// new `MachInst` and vcode backend easier, since all that needs to be done is /// "just" implementing this trait. (And probably add/modify some /// `peepmatic_runtime::operation::Operation`s as well). -pub trait InstructionSet<'a> { +/// +/// ## Safety +/// +/// See doc comment for `instruction_result_bit_width`. +pub unsafe trait InstructionSet<'a> { /// Mutable context passed into all trait methods. Can be whatever you want! /// /// In practice, this is a `FuncCursor` for `cranelift-codegen`'s trait @@ -124,7 +128,10 @@ pub trait InstructionSet<'a> { /// Get the bit width of the given instruction's result. /// - /// Must be one of 1, 8, 16, 32, 64, or 128. + /// ## Safety + /// + /// There is code that makes memory-safety assumptions that the result is + /// always one of 1, 8, 16, 32, 64, or 128. Implementors must uphold this. fn instruction_result_bit_width( &self, context: &mut Self::Context, diff --git a/cranelift/peepmatic/crates/runtime/src/integer_interner.rs b/cranelift/peepmatic/crates/runtime/src/integer_interner.rs index 93dd2e7183..c2f0830bf0 100644 --- a/cranelift/peepmatic/crates/runtime/src/integer_interner.rs +++ b/cranelift/peepmatic/crates/runtime/src/integer_interner.rs @@ -7,11 +7,11 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use std::convert::TryInto; +use std::num::NonZeroU32; /// An identifier for an interned integer. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct IntegerId(#[doc(hidden)] pub u32); +pub struct IntegerId(#[doc(hidden)] pub NonZeroU32); /// An interner for integer values. #[derive(Debug, Default, Serialize, Deserialize)] @@ -40,7 +40,8 @@ impl IntegerInterner { return *id; } - let id = IntegerId(self.values.len().try_into().unwrap()); + assert!((self.values.len() as u64) < (std::u32::MAX as u64)); + let id = IntegerId(unsafe { NonZeroU32::new_unchecked(self.values.len() as u32 + 1) }); self.values.push(value); self.map.insert(value, id); @@ -59,13 +60,21 @@ impl IntegerInterner { /// Lookup a previously interned integer by id. #[inline] pub fn lookup(&self, id: IntegerId) -> u64 { - self.values[id.0 as usize] + let index = id.0.get() as usize - 1; + self.values[index] } } impl From for u32 { #[inline] fn from(id: IntegerId) -> u32 { + id.0.get() + } +} + +impl From for NonZeroU32 { + #[inline] + fn from(id: IntegerId) -> NonZeroU32 { id.0 } } diff --git a/cranelift/peepmatic/crates/runtime/src/linear.rs b/cranelift/peepmatic/crates/runtime/src/linear.rs index 006f18e0e5..fe98edb897 100644 --- a/cranelift/peepmatic/crates/runtime/src/linear.rs +++ b/cranelift/peepmatic/crates/runtime/src/linear.rs @@ -11,6 +11,7 @@ use crate::operator::{Operator, UnquoteOperator}; use crate::paths::{PathId, PathInterner}; use crate::r#type::{BitWidth, Type}; use serde::{Deserialize, Serialize}; +use std::num::NonZeroU32; /// A set of linear optimizations. #[derive(Debug)] @@ -32,6 +33,26 @@ pub struct Optimization { pub increments: Vec, } +/// Match any value. +/// +/// This can be used to create fallback, wildcard-style transitions between +/// states. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct Else; + +/// The result of evaluating a `MatchOp`. +/// +/// This is either a specific non-zero `u32`, or a fallback that matches +/// everything. +pub type MatchResult = Result; + +/// Convert a boolean to a `MatchResult`. +#[inline] +pub fn bool_to_match_result(b: bool) -> MatchResult { + let b = b as u32; + unsafe { Ok(NonZeroU32::new_unchecked(b + 1)) } +} + /// An increment is a matching operation, the expected result from that /// operation to continue to the next increment, and the actions to take to /// build up the LHS scope and RHS instructions given that we got the expected @@ -44,9 +65,9 @@ pub struct Increment { pub operation: MatchOp, /// The expected result of our matching operation, that enables us to - /// continue to the next increment. `None` is used for wildcard-style "else" - /// transitions. - pub expected: Option, + /// continue to the next increment, or `Else` for "don't care" + /// wildcard-style matching. + pub expected: MatchResult, /// Actions to perform, given that the operation resulted in the expected /// value. @@ -217,3 +238,23 @@ pub enum Action { operands: [RhsId; 3], }, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn match_result_is_4_bytes_in_size() { + assert_eq!(std::mem::size_of::(), 4); + } + + #[test] + fn match_op_is_12_bytes_in_size() { + assert_eq!(std::mem::size_of::(), 12); + } + + #[test] + fn action_is_20_bytes_in_size() { + assert_eq!(std::mem::size_of::(), 20); + } +} diff --git a/cranelift/peepmatic/crates/runtime/src/operator.rs b/cranelift/peepmatic/crates/runtime/src/operator.rs index 35e17a89f7..c8f06b8514 100644 --- a/cranelift/peepmatic/crates/runtime/src/operator.rs +++ b/cranelift/peepmatic/crates/runtime/src/operator.rs @@ -20,6 +20,9 @@ use serde::{Deserialize, Serialize}; pub enum Operator { /// `adjust_sp_down` #[peepmatic(params(iNN), result(void))] + // NB: We convert `Operator`s into `NonZeroU32`s with unchecked casts; + // memory safety relies on `Operator` starting at `1` and no variant ever + // being zero. AdjustSpDown = 1, /// `adjust_sp_down_imm` diff --git a/cranelift/peepmatic/crates/runtime/src/optimizations.rs b/cranelift/peepmatic/crates/runtime/src/optimizations.rs index 29dfa7f388..4ed66a8a12 100644 --- a/cranelift/peepmatic/crates/runtime/src/optimizations.rs +++ b/cranelift/peepmatic/crates/runtime/src/optimizations.rs @@ -3,7 +3,7 @@ use crate::error::Result; use crate::instruction_set::InstructionSet; use crate::integer_interner::IntegerInterner; -use crate::linear::{Action, MatchOp}; +use crate::linear::{Action, MatchOp, MatchResult}; use crate::optimizer::PeepholeOptimizer; use crate::paths::PathInterner; use peepmatic_automata::Automaton; @@ -29,7 +29,7 @@ pub struct PeepholeOptimizations { /// The underlying automata for matching optimizations' left-hand sides, and /// building up the corresponding right-hand side. - pub automata: Automaton, MatchOp, Vec>, + pub automata: Automaton>, } impl PeepholeOptimizations { diff --git a/cranelift/peepmatic/crates/runtime/src/optimizer.rs b/cranelift/peepmatic/crates/runtime/src/optimizer.rs index 535bb56219..fc5916f247 100644 --- a/cranelift/peepmatic/crates/runtime/src/optimizer.rs +++ b/cranelift/peepmatic/crates/runtime/src/optimizer.rs @@ -1,7 +1,7 @@ //! An optimizer for a set of peephole optimizations. use crate::instruction_set::InstructionSet; -use crate::linear::{Action, MatchOp}; +use crate::linear::{bool_to_match_result, Action, Else, MatchOp, MatchResult}; use crate::operator::UnquoteOperator; use crate::optimizations::PeepholeOptimizations; use crate::part::{Constant, Part}; @@ -10,6 +10,7 @@ use peepmatic_automata::State; use std::convert::TryFrom; use std::fmt::{self, Debug}; use std::mem; +use std::num::NonZeroU32; /// A peephole optimizer instance that can apply a set of peephole /// optimizations to instructions. @@ -275,43 +276,72 @@ where context: &mut I::Context, root: I::Instruction, match_op: MatchOp, - ) -> Option { + ) -> MatchResult { use crate::linear::MatchOp::*; log::trace!("Evaluating match operation: {:?}", match_op); - let result = match match_op { + let result: MatchResult = (|| match match_op { Opcode { path } => { let path = self.peep_opt.paths.lookup(path); - let part = self.instr_set.get_part_at_path(context, root, path)?; - let inst = part.as_instruction()?; - self.instr_set.operator(context, inst).map(|op| op as u32) + let part = self + .instr_set + .get_part_at_path(context, root, path) + .ok_or(Else)?; + let inst = part.as_instruction().ok_or(Else)?; + let op = self.instr_set.operator(context, inst).ok_or(Else)?; + let op = op as u32; + debug_assert!( + op != 0, + "`Operator` doesn't have any variant represented + with zero" + ); + Ok(unsafe { NonZeroU32::new_unchecked(op as u32) }) } IsConst { path } => { let path = self.peep_opt.paths.lookup(path); - let part = self.instr_set.get_part_at_path(context, root, path)?; + let part = self + .instr_set + .get_part_at_path(context, root, path) + .ok_or(Else)?; let is_const = match part { Part::Instruction(i) => { self.instr_set.instruction_to_constant(context, i).is_some() } Part::ConditionCode(_) | Part::Constant(_) => true, }; - Some(is_const as u32) + bool_to_match_result(is_const) } IsPowerOfTwo { path } => { let path = self.peep_opt.paths.lookup(path); - let part = self.instr_set.get_part_at_path(context, root, path)?; + let part = self + .instr_set + .get_part_at_path(context, root, path) + .ok_or(Else)?; match part { - Part::Constant(c) => Some(c.as_int().unwrap().is_power_of_two() as u32), - Part::Instruction(i) => { - let c = self.instr_set.instruction_to_constant(context, i)?; - Some(c.as_int().unwrap().is_power_of_two() as u32) + Part::Constant(c) => { + let is_pow2 = c.as_int().unwrap().is_power_of_two(); + bool_to_match_result(is_pow2) } - Part::ConditionCode(_) => panic!("IsPowerOfTwo on a condition code"), + Part::Instruction(i) => { + let c = self + .instr_set + .instruction_to_constant(context, i) + .ok_or(Else)?; + let is_pow2 = c.as_int().unwrap().is_power_of_two(); + bool_to_match_result(is_pow2) + } + Part::ConditionCode(_) => unreachable!( + "IsPowerOfTwo on a condition + code" + ), } } BitWidth { path } => { let path = self.peep_opt.paths.lookup(path); - let part = self.instr_set.get_part_at_path(context, root, path)?; + let part = self + .instr_set + .get_part_at_path(context, root, path) + .ok_or(Else)?; let bit_width = match part { Part::Instruction(i) => self.instr_set.instruction_result_bit_width(context, i), Part::Constant(Constant::Int(_, w)) | Part::Constant(Constant::Bool(_, w)) => { @@ -321,14 +351,22 @@ where } Part::ConditionCode(_) => panic!("BitWidth on condition code"), }; - Some(bit_width as u32) + debug_assert!( + bit_width != 0, + "`InstructionSet` implementors must uphold the contract that \ + `instruction_result_bit_width` returns one of 1, 8, 16, 32, 64, or 128" + ); + Ok(unsafe { NonZeroU32::new_unchecked(bit_width as u32) }) } FitsInNativeWord { path } => { let native_word_size = self.instr_set.native_word_size_in_bits(context); debug_assert!(native_word_size.is_power_of_two()); let path = self.peep_opt.paths.lookup(path); - let part = self.instr_set.get_part_at_path(context, root, path)?; + let part = self + .instr_set + .get_part_at_path(context, root, path) + .ok_or(Else)?; let fits = match part { Part::Instruction(i) => { let size = self.instr_set.instruction_result_bit_width(context, i); @@ -341,13 +379,19 @@ where } Part::ConditionCode(_) => panic!("FitsInNativeWord on condition code"), }; - Some(fits as u32) + bool_to_match_result(fits) } Eq { path_a, path_b } => { let path_a = self.peep_opt.paths.lookup(path_a); - let part_a = self.instr_set.get_part_at_path(context, root, path_a)?; + let part_a = self + .instr_set + .get_part_at_path(context, root, path_a) + .ok_or(Else)?; let path_b = self.peep_opt.paths.lookup(path_b); - let part_b = self.instr_set.get_part_at_path(context, root, path_b)?; + let part_b = self + .instr_set + .get_part_at_path(context, root, path_b) + .ok_or(Else)?; let eq = match (part_a, part_b) { (Part::Instruction(inst), Part::Constant(c1)) | (Part::Constant(c1), Part::Instruction(inst)) => { @@ -358,43 +402,67 @@ where } (a, b) => a == b, }; - Some(eq as _) + bool_to_match_result(eq) } IntegerValue { path } => { let path = self.peep_opt.paths.lookup(path); - let part = self.instr_set.get_part_at_path(context, root, path)?; + let part = self + .instr_set + .get_part_at_path(context, root, path) + .ok_or(Else)?; match part { Part::Constant(c) => { - let x = c.as_int()?; - self.peep_opt.integers.already_interned(x).map(|id| id.0) + let x = c.as_int().ok_or(Else)?; + let id = self.peep_opt.integers.already_interned(x).ok_or(Else)?; + Ok(id.0) } Part::Instruction(i) => { - let c = self.instr_set.instruction_to_constant(context, i)?; - let x = c.as_int()?; - self.peep_opt.integers.already_interned(x).map(|id| id.0) + let c = self + .instr_set + .instruction_to_constant(context, i) + .ok_or(Else)?; + let x = c.as_int().ok_or(Else)?; + let id = self.peep_opt.integers.already_interned(x).ok_or(Else)?; + Ok(id.0) } - Part::ConditionCode(_) => panic!("IntegerValue on condition code"), + Part::ConditionCode(_) => unreachable!("IntegerValue on condition code"), } } BooleanValue { path } => { let path = self.peep_opt.paths.lookup(path); - let part = self.instr_set.get_part_at_path(context, root, path)?; + let part = self + .instr_set + .get_part_at_path(context, root, path) + .ok_or(Else)?; match part { - Part::Constant(c) => c.as_bool().map(|b| b as u32), - Part::Instruction(i) => { - let c = self.instr_set.instruction_to_constant(context, i)?; - c.as_bool().map(|b| b as u32) + Part::Constant(c) => { + let b = c.as_bool().ok_or(Else)?; + bool_to_match_result(b) } - Part::ConditionCode(_) => panic!("IntegerValue on condition code"), + Part::Instruction(i) => { + let c = self + .instr_set + .instruction_to_constant(context, i) + .ok_or(Else)?; + let b = c.as_bool().ok_or(Else)?; + bool_to_match_result(b) + } + Part::ConditionCode(_) => unreachable!("IntegerValue on condition code"), } } ConditionCode { path } => { let path = self.peep_opt.paths.lookup(path); - let part = self.instr_set.get_part_at_path(context, root, path)?; - part.as_condition_code().map(|cc| cc as u32) + let part = self + .instr_set + .get_part_at_path(context, root, path) + .ok_or(Else)?; + let cc = part.as_condition_code().ok_or(Else)?; + let cc = cc as u32; + debug_assert!(cc != 0); + Ok(unsafe { NonZeroU32::new_unchecked(cc) }) } - MatchOp::Nop => None, - }; + MatchOp::Nop => Err(Else), + })(); log::trace!("Evaluated match operation: {:?} = {:?}", match_op, result); result } @@ -437,12 +505,12 @@ where r#final = Some((query.current_state(), self.actions.len())); } - // Anything following a `None` transition doesn't care about the + // Anything following a `Else` transition doesn't care about the // result of this match operation, so if we partially follow the - // current non-`None` path, but don't ultimately find a matching + // current non-`Else` path, but don't ultimately find a matching // optimization, we want to be able to backtrack to this state and - // then try taking the `None` transition. - if query.has_transition_on(&None) { + // then try taking the `Else` transition. + if query.has_transition_on(&Err(Else)) { self.backtracking_states .push((query.current_state(), self.actions.len())); } @@ -462,8 +530,8 @@ where query.go_to_state(state); self.actions.truncate(actions_len); query - .next(&None) - .expect("backtracking states always have `None` transitions") + .next(&Err(Else)) + .expect("backtracking states always have `Else` transitions") } else { break; }; diff --git a/cranelift/peepmatic/crates/test/src/lib.rs b/cranelift/peepmatic/crates/test/src/lib.rs index c43ce89957..f74e9b7c53 100644 --- a/cranelift/peepmatic/crates/test/src/lib.rs +++ b/cranelift/peepmatic/crates/test/src/lib.rs @@ -307,7 +307,9 @@ pub struct TestIsa { pub native_word_size_in_bits: u8, } -impl<'a> InstructionSet<'a> for TestIsa { +// Unsafe because we must ensure that `instruction_result_bit_width` never +// returns zero. +unsafe impl<'a> InstructionSet<'a> for TestIsa { type Context = Program; type Instruction = Instruction; @@ -521,7 +523,9 @@ impl<'a> InstructionSet<'a> for TestIsa { fn instruction_result_bit_width(&self, program: &mut Program, inst: Instruction) -> u8 { log::debug!("instruction_result_bit_width({:?})", inst); let ty = program.data(inst).r#type; - ty.bit_width.fixed_width().unwrap() + let width = ty.bit_width.fixed_width().unwrap(); + assert!(width != 0); + width } fn native_word_size_in_bits(&self, _program: &mut Program) -> u8 { diff --git a/cranelift/peepmatic/src/automatize.rs b/cranelift/peepmatic/src/automatize.rs index 3310bef118..5b96339f9f 100644 --- a/cranelift/peepmatic/src/automatize.rs +++ b/cranelift/peepmatic/src/automatize.rs @@ -6,10 +6,10 @@ use peepmatic_runtime::linear; /// Construct an automaton from a set of linear optimizations. pub fn automatize( opts: &linear::Optimizations, -) -> Automaton, linear::MatchOp, Vec> { +) -> Automaton> { debug_assert!(crate::linear_passes::is_sorted_lexicographically(opts)); - let mut builder = Builder::, linear::MatchOp, Vec>::new(); + let mut builder = Builder::>::new(); for opt in &opts.optimizations { let mut insertion = builder.insert(); diff --git a/cranelift/peepmatic/src/dot_fmt.rs b/cranelift/peepmatic/src/dot_fmt.rs index a2c75de02c..959fdd96d9 100644 --- a/cranelift/peepmatic/src/dot_fmt.rs +++ b/cranelift/peepmatic/src/dot_fmt.rs @@ -12,36 +12,37 @@ use peepmatic_runtime::{ }; use std::convert::TryFrom; use std::io::{self, Write}; +use std::num::NonZeroU32; #[derive(Debug)] pub(crate) struct PeepholeDotFmt<'a>(pub(crate) &'a PathInterner, pub(crate) &'a IntegerInterner); -impl DotFmt, linear::MatchOp, Vec> for PeepholeDotFmt<'_> { +impl DotFmt> for PeepholeDotFmt<'_> { fn fmt_transition( &self, w: &mut impl Write, from: Option<&linear::MatchOp>, - input: &Option, + input: &linear::MatchResult, _to: Option<&linear::MatchOp>, ) -> io::Result<()> { let from = from.expect("we should have match op for every state"); - if let Some(x) = input { + if let Some(x) = input.ok().map(|x| x.get()) { match from { linear::MatchOp::Opcode { .. } => { let opcode = - Operator::try_from(*x).expect("we shouldn't generate non-opcode edges"); + Operator::try_from(x).expect("we shouldn't generate non-opcode edges"); write!(w, "{}", opcode) } linear::MatchOp::ConditionCode { .. } => { let cc = - ConditionCode::try_from(*x).expect("we shouldn't generate non-CC edges"); + ConditionCode::try_from(x).expect("we shouldn't generate non-CC edges"); write!(w, "{}", cc) } linear::MatchOp::IntegerValue { .. } => { - let x = self.1.lookup(IntegerId(*x)); + let x = self.1.lookup(IntegerId(NonZeroU32::new(x).unwrap())); write!(w, "{}", x) } - _ => write!(w, "{}", x), + _ => write!(w, "Ok({})", x), } } else { write!(w, "(else)") diff --git a/cranelift/peepmatic/src/linear_passes.rs b/cranelift/peepmatic/src/linear_passes.rs index cac1ca3d19..9e87ae0036 100644 --- a/cranelift/peepmatic/src/linear_passes.rs +++ b/cranelift/peepmatic/src/linear_passes.rs @@ -65,7 +65,12 @@ fn compare_optimizations( return c; } - let c = a.expected.cmp(&b.expected).reverse(); + let c = match (a.expected, b.expected) { + (Ok(a), Ok(b)) => a.cmp(&b).reverse(), + (Err(_), Ok(_)) => Ordering::Greater, + (Ok(_), Err(_)) => Ordering::Less, + (Err(linear::Else), Err(linear::Else)) => Ordering::Equal, + }; if c != Ordering::Equal { return c; } @@ -228,10 +233,10 @@ pub fn match_in_same_order(opts: &mut linear::Optimizations) { Some(_) => { new_increments.push(linear::Increment { operation: *last_op, - expected: None, + expected: Err(linear::Else), actions: vec![], }); - if last_expected.is_some() { + if last_expected.is_ok() { break; } } @@ -282,12 +287,99 @@ pub fn remove_unnecessary_nops(opts: &mut linear::Optimizations) { mod tests { use super::*; use crate::ast::*; - use linear::MatchOp::*; - use peepmatic_runtime::{operator::Operator, paths::*}; + use peepmatic_runtime::{ + linear::{bool_to_match_result, Else, MatchOp::*, MatchResult}, + operator::Operator, + paths::*, + }; + use std::num::NonZeroU32; + + #[test] + fn ok_non_zero_less_than_err_else() { + assert!(Ok(NonZeroU32::new(1).unwrap()) < Err(Else)); + } macro_rules! sorts_to { ($test_name:ident, $source:expr, $make_expected:expr) => { #[test] + #[allow(unused_variables)] + fn $test_name() { + let buf = wast::parser::ParseBuffer::new($source).expect("should lex OK"); + + let opts = match wast::parser::parse::(&buf) { + Ok(opts) => opts, + Err(mut e) => { + e.set_path(std::path::Path::new(stringify!($test_name))); + e.set_text($source); + eprintln!("{}", e); + panic!("should parse OK") + } + }; + + if let Err(mut e) = crate::verify(&opts) { + e.set_path(std::path::Path::new(stringify!($test_name))); + e.set_text($source); + eprintln!("{}", e); + panic!("should verify OK") + } + + let mut opts = crate::linearize(&opts); + + let before = opts + .optimizations + .iter() + .map(|o| { + o.increments + .iter() + .map(|i| format!("{:?} == {:?}", i.operation, i.expected)) + .collect::>() + }) + .collect::>(); + eprintln!("before = {:#?}", before); + + sort_least_to_most_general(&mut opts); + + let after = opts + .optimizations + .iter() + .map(|o| { + o.increments + .iter() + .map(|i| format!("{:?} == {:?}", i.operation, i.expected)) + .collect::>() + }) + .collect::>(); + eprintln!("after = {:#?}", before); + + let linear::Optimizations { + mut paths, + mut integers, + optimizations, + } = opts; + + let actual: Vec> = optimizations + .iter() + .map(|o| { + o.increments + .iter() + .map(|i| (i.operation, i.expected)) + .collect() + }) + .collect(); + + let mut p = |p: &[u8]| paths.intern(Path::new(&p)); + let mut i = |i: u64| Ok(integers.intern(i).into()); + let expected = $make_expected(&mut p, &mut i); + + assert_eq!(expected, actual); + } + }; + } + + macro_rules! match_in_same_order { + ($test_name:ident, $source:expr, $make_expected:expr) => { + #[test] + #[allow(unused_variables)] fn $test_name() { let buf = wast::parser::ParseBuffer::new($source).expect("should lex OK"); @@ -311,6 +403,32 @@ mod tests { let mut opts = crate::linearize(&opts); sort_least_to_most_general(&mut opts); + let before = opts + .optimizations + .iter() + .map(|o| { + o.increments + .iter() + .map(|i| format!("{:?} == {:?}", i.operation, i.expected)) + .collect::>() + }) + .collect::>(); + eprintln!("before = {:#?}", before); + + match_in_same_order(&mut opts); + + let after = opts + .optimizations + .iter() + .map(|o| { + o.increments + .iter() + .map(|i| format!("{:?} == {:?}", i.operation, i.expected)) + .collect::>() + }) + .collect::>(); + eprintln!("after = {:#?}", before); + let linear::Optimizations { mut paths, mut integers, @@ -328,7 +446,7 @@ mod tests { .collect(); let mut p = |p: &[u8]| paths.intern(Path::new(&p)); - let mut i = |i: u64| Some(integers.intern(i).into()); + let mut i = |i: u64| Ok(integers.intern(i).into()); let expected = $make_expected(&mut p, &mut i); assert_eq!(expected, actual); @@ -348,53 +466,83 @@ mod tests { (=> (iadd $x 42) 0) (=> (iadd $x (iadd $y $z)) 0) ", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> Option| vec![ + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![ vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), - (Nop, None), - (Opcode { path: p(&[0, 1]) }, Some(Operator::Iadd as _)), - (Nop, None), - (Nop, None), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), + (Nop, Err(Else)), + ( + Opcode { path: p(&[0, 1]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), + (Nop, Err(Else)), + (Nop, Err(Else)), ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), - (Nop, None), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), + (Nop, Err(Else)), (IntegerValue { path: p(&[0, 1]) }, i(42)) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), - (Nop, None), - (IsConst { path: p(&[0, 1]) }, Some(1)), - (IsPowerOfTwo { path: p(&[0, 1]) }, Some(1)) + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), + (Nop, Err(Else)), + (IsConst { path: p(&[0, 1]) }, bool_to_match_result(true)), + ( + IsPowerOfTwo { path: p(&[0, 1]) }, + bool_to_match_result(true) + ) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), - (Nop, None), - (IsConst { path: p(&[0, 1]) }, Some(1)), - (BitWidth { path: p(&[0, 0]) }, Some(32)) + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), + (Nop, Err(Else)), + (IsConst { path: p(&[0, 1]) }, bool_to_match_result(true)), + ( + BitWidth { path: p(&[0, 0]) }, + Ok(NonZeroU32::new(32).unwrap()) + ) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), - (Nop, None), - (IsConst { path: p(&[0, 1]) }, Some(1)) + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), + (Nop, Err(Else)), + (IsConst { path: p(&[0, 1]) }, bool_to_match_result(true)) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), - (Nop, None), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), + (Nop, Err(Else)), ( Eq { path_a: p(&[0, 1]), path_b: p(&[0, 0]), }, - Some(1) + bool_to_match_result(true) ) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), - (Nop, None), - (Nop, None), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), + (Nop, Err(Else)), + (Nop, Err(Else)), ], - vec![(Nop, None)] + vec![(Nop, Err(Else))] ] ); @@ -408,37 +556,164 @@ mod tests { (=> (imul 2 $x) (ishl $x 1)) (=> (imul $x 2) (ishl $x 1)) ", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> Option| vec![ + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![ vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Imul as _)), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Imul as _).unwrap()) + ), (IntegerValue { path: p(&[0, 0]) }, i(2)), - (Nop, None) + (Nop, Err(Else)) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Imul as _)), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Imul as _).unwrap()) + ), (IntegerValue { path: p(&[0, 0]) }, i(1)), - (Nop, None) + (Nop, Err(Else)) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Imul as _)), - (Nop, None), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Imul as _).unwrap()) + ), + (Nop, Err(Else)), (IntegerValue { path: p(&[0, 1]) }, i(2)) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Imul as _)), - (Nop, None), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Imul as _).unwrap()) + ), + (Nop, Err(Else)), (IntegerValue { path: p(&[0, 1]) }, i(1)) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), (IntegerValue { path: p(&[0, 0]) }, i(0)), - (Nop, None) + (Nop, Err(Else)) ], vec![ - (Opcode { path: p(&[0]) }, Some(Operator::Iadd as _)), - (Nop, None), + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Iadd as _).unwrap()) + ), + (Nop, Err(Else)), (IntegerValue { path: p(&[0, 1]) }, i(0)) ] ] ); + + sorts_to!( + sort_redundant_bor, + " + (=> (bor (bor $x $y) $x) + (bor $x $y)) + + (=> (bor (bor $x $y) $y) + (bor $x $y)) + ", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![ + vec![ + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Bor as _).unwrap()) + ), + ( + Opcode { path: p(&[0, 0]) }, + Ok(NonZeroU32::new(Operator::Bor as _).unwrap()) + ), + (Nop, Err(Else)), + (Nop, Err(Else)), + ( + Eq { + path_a: p(&[0, 1]), + path_b: p(&[0, 0, 0]), + }, + bool_to_match_result(true) + ), + ], + vec![ + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Bor as _).unwrap()) + ), + ( + Opcode { path: p(&[0, 0]) }, + Ok(NonZeroU32::new(Operator::Bor as _).unwrap()) + ), + (Nop, Err(Else)), + (Nop, Err(Else)), + ( + Eq { + path_a: p(&[0, 1]), + path_b: p(&[0, 0, 1]), + }, + bool_to_match_result(true) + ), + ], + ] + ); + + match_in_same_order!( + match_in_same_order_redundant_bor, + " + (=> (bor (bor $x $y) $x) + (bor $x $y)) + + (=> (bor (bor $x $y) $y) + (bor $x $y)) + ", + |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![ + vec![ + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Bor as _).unwrap()) + ), + ( + Opcode { path: p(&[0, 0]) }, + Ok(NonZeroU32::new(Operator::Bor as _).unwrap()) + ), + (Nop, Err(Else)), + (Nop, Err(Else)), + ( + Eq { + path_a: p(&[0, 1]), + path_b: p(&[0, 0, 0]), + }, + bool_to_match_result(true) + ), + ], + vec![ + ( + Opcode { path: p(&[0]) }, + Ok(NonZeroU32::new(Operator::Bor as _).unwrap()) + ), + ( + Opcode { path: p(&[0, 0]) }, + Ok(NonZeroU32::new(Operator::Bor as _).unwrap()) + ), + (Nop, Err(Else)), + (Nop, Err(Else)), + ( + Eq { + path_a: p(&[0, 1]), + path_b: p(&[0, 0, 0]), + }, + Err(Else), + ), + ( + Eq { + path_a: p(&[0, 1]), + path_b: p(&[0, 0, 1]), + }, + bool_to_match_result(true) + ), + ], + ] + ); } diff --git a/cranelift/peepmatic/src/linearize.rs b/cranelift/peepmatic/src/linearize.rs index 75205c93b9..5a47abcfcf 100644 --- a/cranelift/peepmatic/src/linearize.rs +++ b/cranelift/peepmatic/src/linearize.rs @@ -92,6 +92,7 @@ use peepmatic_runtime::{ paths::{Path, PathId, PathInterner}, }; use std::collections::BTreeMap; +use std::num::NonZeroU32; use wast::Id; /// Translate the given AST optimizations into linear optimizations. @@ -142,9 +143,12 @@ fn linearize_optimization( // increment that checks the instruction-being-matched's bit width. if let Pattern::Operation(Operation { r#type, .. }) = pattern { if let Some(w) = r#type.get().and_then(|ty| ty.bit_width.fixed_width()) { + debug_assert!(w != 0, "All fixed-width bit widths are non-zero"); + let expected = Ok(unsafe { NonZeroU32::new_unchecked(w as u32) }); + increments.push(linear::Increment { operation: linear::MatchOp::BitWidth { path }, - expected: Some(w as u32), + expected, actions: vec![], }); } @@ -432,7 +436,7 @@ impl Precondition<'_> { let path = lhs_id_to_path.unwrap_first_occurrence(&id); linear::Increment { operation: linear::MatchOp::IsPowerOfTwo { path }, - expected: Some(1), + expected: linear::bool_to_match_result(true), actions: vec![], } } @@ -451,12 +455,14 @@ impl Precondition<'_> { })) => *value, _ => unreachable!("checked in verification"), }; - debug_assert!(width <= 128); - debug_assert!((width as u8).is_power_of_two()); + + assert!(0 < width && width <= 128); + assert!((width as u8).is_power_of_two()); + let expected = Ok(unsafe { NonZeroU32::new_unchecked(width as u32) }); linear::Increment { operation: linear::MatchOp::BitWidth { path }, - expected: Some(width as u32), + expected, actions: vec![], } } @@ -469,7 +475,7 @@ impl Precondition<'_> { let path = lhs_id_to_path.unwrap_first_occurrence(&id); linear::Increment { operation: linear::MatchOp::FitsInNativeWord { path }, - expected: Some(1), + expected: linear::bool_to_match_result(true), actions: vec![], } } @@ -488,17 +494,21 @@ impl Pattern<'_> { integers: &mut IntegerInterner, lhs_id_to_path: &LhsIdToPath, path: PathId, - ) -> (linear::MatchOp, Option) { + ) -> (linear::MatchOp, linear::MatchResult) { match self { Pattern::ValueLiteral(ValueLiteral::Integer(Integer { value, .. })) => ( linear::MatchOp::IntegerValue { path }, - Some(integers.intern(*value as u64).into()), + Ok(integers.intern(*value as u64).into()), + ), + Pattern::ValueLiteral(ValueLiteral::Boolean(Boolean { value, .. })) => ( + linear::MatchOp::BooleanValue { path }, + linear::bool_to_match_result(*value), ), - Pattern::ValueLiteral(ValueLiteral::Boolean(Boolean { value, .. })) => { - (linear::MatchOp::BooleanValue { path }, Some(*value as u32)) - } Pattern::ValueLiteral(ValueLiteral::ConditionCode(ConditionCode { cc, .. })) => { - (linear::MatchOp::ConditionCode { path }, Some(*cc as u32)) + let cc = *cc as u32; + debug_assert!(cc != 0, "no `ConditionCode` variants are zero"); + let expected = Ok(unsafe { NonZeroU32::new_unchecked(cc) }); + (linear::MatchOp::ConditionCode { path }, expected) } Pattern::Constant(Constant { id, .. }) => { if let Some(path_b) = lhs_id_to_path.get_first_occurrence(id) { @@ -508,10 +518,13 @@ impl Pattern<'_> { path_a: path, path_b, }, - Some(1), + linear::bool_to_match_result(true), ) } else { - (linear::MatchOp::IsConst { path }, Some(1)) + ( + linear::MatchOp::IsConst { path }, + linear::bool_to_match_result(true), + ) } } Pattern::Variable(Variable { id, .. }) => { @@ -522,13 +535,18 @@ impl Pattern<'_> { path_a: path, path_b, }, - Some(1), + linear::bool_to_match_result(true), ) } else { - (linear::MatchOp::Nop, None) + (linear::MatchOp::Nop, Err(linear::Else)) } } - Pattern::Operation(op) => (linear::MatchOp::Opcode { path }, Some(op.operator as u32)), + Pattern::Operation(op) => { + let op = op.operator as u32; + debug_assert!(op != 0, "no `Operator` variants are zero"); + let expected = Ok(unsafe { NonZeroU32::new_unchecked(op) }); + (linear::MatchOp::Opcode { path }, expected) + } } } } @@ -538,7 +556,7 @@ mod tests { use super::*; use peepmatic_runtime::{ integer_interner::IntegerId, - linear::{Action::*, MatchOp::*}, + linear::{bool_to_match_result, Action::*, Else, MatchOp::*}, operator::Operator, r#type::{BitWidth, Kind, Type}, }; @@ -603,7 +621,7 @@ mod tests { increments: vec![ linear::Increment { operation: Opcode { path: p(&[0]) }, - expected: Some(Operator::Imul as _), + expected: Ok(NonZeroU32::new(Operator::Imul as _).unwrap()), actions: vec![ GetLhs { path: p(&[0, 0]) }, GetLhs { path: p(&[0, 1]) }, @@ -619,17 +637,17 @@ mod tests { }, linear::Increment { operation: Nop, - expected: None, + expected: Err(Else), actions: vec![], }, linear::Increment { operation: IsConst { path: p(&[0, 1]) }, - expected: Some(1), + expected: bool_to_match_result(true), actions: vec![], }, linear::Increment { operation: IsPowerOfTwo { path: p(&[0, 1]) }, - expected: Some(1), + expected: bool_to_match_result(true), actions: vec![], }, ], @@ -644,7 +662,7 @@ mod tests { linear::Optimization { increments: vec![linear::Increment { operation: Nop, - expected: None, + expected: Err(Else), actions: vec![GetLhs { path: p(&[0]) }], }], } @@ -658,7 +676,7 @@ mod tests { linear::Optimization { increments: vec![linear::Increment { operation: IsConst { path: p(&[0]) }, - expected: Some(1), + expected: bool_to_match_result(true), actions: vec![GetLhs { path: p(&[0]) }], }], } @@ -672,7 +690,7 @@ mod tests { linear::Optimization { increments: vec![linear::Increment { operation: BooleanValue { path: p(&[0]) }, - expected: Some(1), + expected: bool_to_match_result(true), actions: vec![MakeBooleanConst { value: true, bit_width: BitWidth::Polymorphic, @@ -689,7 +707,7 @@ mod tests { linear::Optimization { increments: vec![linear::Increment { operation: IntegerValue { path: p(&[0]) }, - expected: Some(i(5).into()), + expected: Ok(i(5).into()), actions: vec![MakeIntegerConst { value: i(5), bit_width: BitWidth::Polymorphic, @@ -707,7 +725,7 @@ mod tests { increments: vec![ linear::Increment { operation: Opcode { path: p(&[0]) }, - expected: Some(Operator::Iconst as _), + expected: Ok(NonZeroU32::new(Operator::Iconst as _).unwrap()), actions: vec![ GetLhs { path: p(&[0, 0]) }, MakeUnaryInst { @@ -722,7 +740,7 @@ mod tests { }, linear::Increment { operation: IsConst { path: p(&[0, 0]) }, - expected: Some(1), + expected: bool_to_match_result(true), actions: vec![], }, ], @@ -738,7 +756,7 @@ mod tests { increments: vec![ linear::Increment { operation: Opcode { path: p(&[0]) }, - expected: Some(Operator::Bor as _), + expected: Ok(NonZeroU32::new(Operator::Bor as _).unwrap()), actions: vec![ GetLhs { path: p(&[0, 0]) }, GetLhs { @@ -756,12 +774,12 @@ mod tests { }, linear::Increment { operation: Nop, - expected: None, + expected: Err(Else), actions: vec![], }, linear::Increment { operation: Opcode { path: p(&[0, 1]) }, - expected: Some(Operator::Bor as _), + expected: Ok(NonZeroU32::new(Operator::Bor as _).unwrap()), actions: vec![], }, linear::Increment { @@ -769,12 +787,12 @@ mod tests { path_a: p(&[0, 1, 0]), path_b: p(&[0, 0]), }, - expected: Some(1), + expected: bool_to_match_result(true), actions: vec![], }, linear::Increment { operation: Nop, - expected: None, + expected: Err(Else), actions: vec![], }, ], @@ -790,7 +808,7 @@ mod tests { linear::Optimization { increments: vec![linear::Increment { operation: IntegerValue { path: p(&[0]) }, - expected: Some(i(std::u64::MAX).into()), + expected: Ok(i(std::u64::MAX).into()), actions: vec![MakeIntegerConst { value: i(0), bit_width: BitWidth::Polymorphic, @@ -808,7 +826,7 @@ mod tests { increments: vec![ linear::Increment { operation: Opcode { path: p(&[0]) }, - expected: Some(Operator::Ireduce as _), + expected: Ok(NonZeroU32::new(Operator::Ireduce as _).unwrap()), actions: vec![MakeIntegerConst { value: i(0), bit_width: BitWidth::ThirtyTwo, @@ -816,12 +834,12 @@ mod tests { }, linear::Increment { operation: linear::MatchOp::BitWidth { path: p(&[0]) }, - expected: Some(32), + expected: Ok(NonZeroU32::new(32).unwrap()), actions: vec![], }, linear::Increment { operation: Nop, - expected: None, + expected: Err(Else), actions: vec![], }, ], From 210b0363206dfe4cb58edb7879d4ab53fc6c3ea9 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 7 May 2020 13:56:02 -0700 Subject: [PATCH 13/22] peepmatic: Represent various id types with `u16` These ids end up in the automaton, so making them smaller should give us better data cache locality and also smaller serialized sizes. --- cranelift/codegen/src/preopt.serialized | Bin 6088 -> 5536 bytes .../crates/runtime/src/integer_interner.rs | 17 +++++------------ .../peepmatic/crates/runtime/src/linear.rs | 18 +++++++++++------- .../peepmatic/crates/runtime/src/optimizer.rs | 4 ++-- .../peepmatic/crates/runtime/src/paths.rs | 4 ++-- cranelift/peepmatic/src/dot_fmt.rs | 8 +++++--- cranelift/peepmatic/src/linearize.rs | 3 ++- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/cranelift/codegen/src/preopt.serialized b/cranelift/codegen/src/preopt.serialized index 035de61c6289a0a3123e2f18e4104627db5cb361..f016c6d32d260ec758239ee2c0bfdb630735d80e 100644 GIT binary patch literal 5536 zcmcJT*OJ>X3`HrIE$O`%Z+4SS@BJzN|5FK$z-xlCMVFayh9(H$f&d6ol8=+OE{|y< zqEOwEHr9yT~G8JwL+{v+EdF+zbo1~-6o26@+I!O02HBXC79jA|(I!Zq?b(sF< zHchK5xRNxqiY#j_*^>1|wvB)CF|+#Nm;L@0qYrVOx&vd(Tz%4>k*N)UQ58$t%;2^H zj^%}n$?n+T9{0ZF^xA-DWdL3@0Ti!mc%A{6x@jA1qScGdtl5&`#9`{#f&;f=@OsvQ zw`5`UurDC2^$t6k{^U_Zlu{_S3wLho0ZLho0ZLho0ZLho0Zpu3zohDZH7MrZ@s#{q6Zj`-c!AgVCEKO;2uk0Q|Znf14=N9mY(A60!Cr`7XtQ!XQxb+Ah=V-)VEXKI1G-g<5xY(P*6HInjhm*Ghx1;`f1;0y zUn3fxhnGf;<-sk)Jj8d2UGosJ+vH)LzDHKqhQ}g_%ih>jvHA(uxzZaJv5jkSyT$zJ zmZ*8PWxb#AUiCJtZ=}7~MI@fJ*9Oz}kT{~vZ9G)$c*t{988Xs5tY_^y)3+ALcn`>i zM&$0Lv?O8fJUE}Mk5!=~{MS!oe6H_Wg%L{zu-m%L*6AChZ``x3)A%*(F(`g$kEM*r z#4DPGLk^jJvp$w36IpYNXDP0@YnCDwtJrO2X`O!Wkp09eh4)+n8?npxd(`RjSAnwP zF?n8o1S}*w@O{8z$G!}BRQ~4ykMsIrz~d|7-GIkEq_=sAxQAb9l1J|C7Xu#8tr#z2 zzJi`v_op-*?VD%O|4%`S$US=AbZP8CYafuIDU^|k=kWp}G~QO8hB+kvtF&zEe;0@O z3VaLFedZGPlqcdtXDol2w^pe2-6rR2{R+HMwXDTAE_ujX(|T)Uqh+nmmix9a%uVa7 z(VrOiuJyRbenWe9ru~=x!Zb?L9-kVkd7ke~LQp>;kq`7~b$oehcpSmqOJe5mxJDm3M%<+_LSwvo% LTjWlO|E>H3!dxw) literal 6088 zcmchb=W-h{41{$~b)4RNo8le!^F-O+L>v0b`@4B0}0lu7Wc~33b4cx=VOrLg+?q;zKU>z`%o5 z;&BzrnoR&V@tM|zGc@glQ5+wC<$u>%6Tu>OtIN*c3mXqh6~-NuG*#q9xB$>VflSJ_ziB^hG&IBL0UHhLkFpJw#x zJ(yQwux&-u?!ChIR@Dgu|5q^}{4d5hE(W=ULB0ze4DfRuj=`E4$E}0uhstpImYZcgrONX?8t{PTK&~J6|4UGJ_73`{dJ6hnEKS~@47fJAnvwS zpN@*x__1l9UW9wAPr}K4@^_$<_Q^4*njDN?pVq~J0jbteTHZ6S%|~|$&zU&A_pzpy zA9t&I6YO*+WnISCJ16goZ(?|@ks250Dqh@U4vCG$WtP}Zbj8-4RTu8C%qcAa^w2zJf4mlXTKCX)c1M=sI{EUxE z;PbDF`=HDFA00%l?}b}@@SMIa#KQ4xKPMe$?^Duo2fR-@em}iUy1kI~+eRk7171p~ zGv6D}laB9HUo&QYD?W{(cMEjBMr3+I&G=%Iomu_j+4Pxs#* z{0r90p?}I0s$Be@W=O`a+Bw$`P#7Wia^)Cdb5_^o;=Qql>+hD7BwXGP*WuEu9&YZZ zaOV+wBZlN)r{3%bezm0fj=;7hxBs}hyR&V)^ P?_uNPBy8?8|NrYRqEs!S diff --git a/cranelift/peepmatic/crates/runtime/src/integer_interner.rs b/cranelift/peepmatic/crates/runtime/src/integer_interner.rs index c2f0830bf0..996100a66d 100644 --- a/cranelift/peepmatic/crates/runtime/src/integer_interner.rs +++ b/cranelift/peepmatic/crates/runtime/src/integer_interner.rs @@ -7,11 +7,11 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use std::num::NonZeroU32; +use std::num::{NonZeroU16, NonZeroU32}; /// An identifier for an interned integer. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct IntegerId(#[doc(hidden)] pub NonZeroU32); +pub struct IntegerId(#[doc(hidden)] pub NonZeroU16); /// An interner for integer values. #[derive(Debug, Default, Serialize, Deserialize)] @@ -40,8 +40,8 @@ impl IntegerInterner { return *id; } - assert!((self.values.len() as u64) < (std::u32::MAX as u64)); - let id = IntegerId(unsafe { NonZeroU32::new_unchecked(self.values.len() as u32 + 1) }); + assert!((self.values.len() as u64) < (std::u16::MAX as u64)); + let id = IntegerId(unsafe { NonZeroU16::new_unchecked(self.values.len() as u16 + 1) }); self.values.push(value); self.map.insert(value, id); @@ -65,16 +65,9 @@ impl IntegerInterner { } } -impl From for u32 { - #[inline] - fn from(id: IntegerId) -> u32 { - id.0.get() - } -} - impl From for NonZeroU32 { #[inline] fn from(id: IntegerId) -> NonZeroU32 { - id.0 + id.0.into() } } diff --git a/cranelift/peepmatic/crates/runtime/src/linear.rs b/cranelift/peepmatic/crates/runtime/src/linear.rs index fe98edb897..a51159f708 100644 --- a/cranelift/peepmatic/crates/runtime/src/linear.rs +++ b/cranelift/peepmatic/crates/runtime/src/linear.rs @@ -149,11 +149,11 @@ pub enum MatchOp { /// A canonicalized identifier for a left-hand side value that was bound in a /// pattern. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -pub struct LhsId(pub u32); +pub struct LhsId(pub u16); /// A canonicalized identifier for a right-hand side value. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct RhsId(pub u32); +pub struct RhsId(pub u16); /// An action to perform when transitioning between states in the automata. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -243,18 +243,22 @@ pub enum Action { mod tests { use super::*; + // These types all end up in the automaton, so we should take care that they + // are small and don't fill up the data cache (or take up too much + // serialized size). + #[test] - fn match_result_is_4_bytes_in_size() { + fn match_result_size() { assert_eq!(std::mem::size_of::(), 4); } #[test] - fn match_op_is_12_bytes_in_size() { - assert_eq!(std::mem::size_of::(), 12); + fn match_op_size() { + assert_eq!(std::mem::size_of::(), 6); } #[test] - fn action_is_20_bytes_in_size() { - assert_eq!(std::mem::size_of::(), 20); + fn action_size() { + assert_eq!(std::mem::size_of::(), 16); } } diff --git a/cranelift/peepmatic/crates/runtime/src/optimizer.rs b/cranelift/peepmatic/crates/runtime/src/optimizer.rs index fc5916f247..2cbf823a4b 100644 --- a/cranelift/peepmatic/crates/runtime/src/optimizer.rs +++ b/cranelift/peepmatic/crates/runtime/src/optimizer.rs @@ -414,7 +414,7 @@ where Part::Constant(c) => { let x = c.as_int().ok_or(Else)?; let id = self.peep_opt.integers.already_interned(x).ok_or(Else)?; - Ok(id.0) + Ok(id.into()) } Part::Instruction(i) => { let c = self @@ -423,7 +423,7 @@ where .ok_or(Else)?; let x = c.as_int().ok_or(Else)?; let id = self.peep_opt.integers.already_interned(x).ok_or(Else)?; - Ok(id.0) + Ok(id.into()) } Part::ConditionCode(_) => unreachable!("IntegerValue on condition code"), } diff --git a/cranelift/peepmatic/crates/runtime/src/paths.rs b/cranelift/peepmatic/crates/runtime/src/paths.rs index a30aa81886..d826ba76bb 100644 --- a/cranelift/peepmatic/crates/runtime/src/paths.rs +++ b/cranelift/peepmatic/crates/runtime/src/paths.rs @@ -46,7 +46,7 @@ impl Path<'_> { /// An identifier for an interned path. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct PathId(u32); +pub struct PathId(u16); /// An interner and de-duplicator for `Path`s. /// @@ -93,7 +93,7 @@ impl PathInterner { #[inline(never)] fn intern_new<'a>(&mut self, path: Path<'a>) -> PathId { - let id: u32 = self + let id: u16 = self .paths .len() .try_into() diff --git a/cranelift/peepmatic/src/dot_fmt.rs b/cranelift/peepmatic/src/dot_fmt.rs index 959fdd96d9..1d854d6561 100644 --- a/cranelift/peepmatic/src/dot_fmt.rs +++ b/cranelift/peepmatic/src/dot_fmt.rs @@ -10,9 +10,9 @@ use peepmatic_runtime::{ operator::Operator, paths::{PathId, PathInterner}, }; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use std::io::{self, Write}; -use std::num::NonZeroU32; +use std::num::NonZeroU16; #[derive(Debug)] pub(crate) struct PeepholeDotFmt<'a>(pub(crate) &'a PathInterner, pub(crate) &'a IntegerInterner); @@ -39,7 +39,9 @@ impl DotFmt> for Peeph write!(w, "{}", cc) } linear::MatchOp::IntegerValue { .. } => { - let x = self.1.lookup(IntegerId(NonZeroU32::new(x).unwrap())); + let x = self + .1 + .lookup(IntegerId(NonZeroU16::new(x.try_into().unwrap()).unwrap())); write!(w, "{}", x) } _ => write!(w, "Ok({})", x), diff --git a/cranelift/peepmatic/src/linearize.rs b/cranelift/peepmatic/src/linearize.rs index 5a47abcfcf..eb03853fa2 100644 --- a/cranelift/peepmatic/src/linearize.rs +++ b/cranelift/peepmatic/src/linearize.rs @@ -92,6 +92,7 @@ use peepmatic_runtime::{ paths::{Path, PathId, PathInterner}, }; use std::collections::BTreeMap; +use std::convert::TryInto; use std::num::NonZeroU32; use wast::Id; @@ -339,7 +340,7 @@ impl<'a> RhsBuilder<'a> { ) { while let Some(rhs) = self.rhs_post_order.next() { actions.push(self.rhs_to_linear_action(integers, lhs_id_to_path, rhs)); - let id = linear::RhsId(self.rhs_span_to_id.len() as u32); + let id = linear::RhsId(self.rhs_span_to_id.len().try_into().unwrap()); self.rhs_span_to_id.insert(rhs.span(), id); } } From eb2dab0aa4eefb42f863928f1c56196ac265ca12 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 7 May 2020 14:14:21 -0700 Subject: [PATCH 14/22] peepmatic: Save RHS actions as a boxed slice, not vec A boxed slice is only two words, while a vec is three words. This should cut down on the memory size of our automata and improve cache usage. --- .../crates/automata/src/output_impls.rs | 37 +++++++++++++++++++ .../crates/runtime/src/optimizations.rs | 2 +- cranelift/peepmatic/src/automatize.rs | 6 +-- cranelift/peepmatic/src/dot_fmt.rs | 6 +-- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/cranelift/peepmatic/crates/automata/src/output_impls.rs b/cranelift/peepmatic/crates/automata/src/output_impls.rs index 6701f7d867..308c466d5b 100644 --- a/cranelift/peepmatic/crates/automata/src/output_impls.rs +++ b/cranelift/peepmatic/crates/automata/src/output_impls.rs @@ -57,6 +57,43 @@ where } } +impl Output for Box<[T]> +where + T: Clone + Eq + Hash, +{ + fn empty() -> Self { + vec![].into_boxed_slice() + } + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn prefix(a: &Self, b: &Self) -> Self { + a.iter() + .cloned() + .zip(b.iter().cloned()) + .take_while(|(a, b)| a == b) + .map(|(a, _)| a) + .collect() + } + + fn difference(a: &Self, b: &Self) -> Self { + let i = a + .iter() + .zip(b.iter()) + .position(|(a, b)| a != b) + .unwrap_or(cmp::min(a.len(), b.len())); + a[i..].to_vec().into_boxed_slice() + } + + fn concat(a: &Self, b: &Self) -> Self { + let mut c = a.clone().to_vec(); + c.extend(b.iter().cloned()); + c.into_boxed_slice() + } +} + #[cfg(test)] mod tests { use crate::Output; diff --git a/cranelift/peepmatic/crates/runtime/src/optimizations.rs b/cranelift/peepmatic/crates/runtime/src/optimizations.rs index 4ed66a8a12..ff838ff720 100644 --- a/cranelift/peepmatic/crates/runtime/src/optimizations.rs +++ b/cranelift/peepmatic/crates/runtime/src/optimizations.rs @@ -29,7 +29,7 @@ pub struct PeepholeOptimizations { /// The underlying automata for matching optimizations' left-hand sides, and /// building up the corresponding right-hand side. - pub automata: Automaton>, + pub automata: Automaton>, } impl PeepholeOptimizations { diff --git a/cranelift/peepmatic/src/automatize.rs b/cranelift/peepmatic/src/automatize.rs index 5b96339f9f..de053c5574 100644 --- a/cranelift/peepmatic/src/automatize.rs +++ b/cranelift/peepmatic/src/automatize.rs @@ -6,10 +6,10 @@ use peepmatic_runtime::linear; /// Construct an automaton from a set of linear optimizations. pub fn automatize( opts: &linear::Optimizations, -) -> Automaton> { +) -> Automaton> { debug_assert!(crate::linear_passes::is_sorted_lexicographically(opts)); - let mut builder = Builder::>::new(); + let mut builder = Builder::>::new(); for opt in &opts.optimizations { let mut insertion = builder.insert(); @@ -22,7 +22,7 @@ pub fn automatize( insertion.set_state_data(inc.operation); } - insertion.next(inc.expected, inc.actions.clone()); + insertion.next(inc.expected, inc.actions.clone().into_boxed_slice()); } insertion.finish(); } diff --git a/cranelift/peepmatic/src/dot_fmt.rs b/cranelift/peepmatic/src/dot_fmt.rs index 1d854d6561..6939577147 100644 --- a/cranelift/peepmatic/src/dot_fmt.rs +++ b/cranelift/peepmatic/src/dot_fmt.rs @@ -17,7 +17,7 @@ use std::num::NonZeroU16; #[derive(Debug)] pub(crate) struct PeepholeDotFmt<'a>(pub(crate) &'a PathInterner, pub(crate) &'a IntegerInterner); -impl DotFmt> for PeepholeDotFmt<'_> { +impl DotFmt> for PeepholeDotFmt<'_> { fn fmt_transition( &self, w: &mut impl Write, @@ -73,7 +73,7 @@ impl DotFmt> for Peeph writeln!(w, "") } - fn fmt_output(&self, w: &mut impl Write, actions: &Vec) -> io::Result<()> { + fn fmt_output(&self, w: &mut impl Write, actions: &Box<[linear::Action]>) -> io::Result<()> { use linear::Action::*; if actions.is_empty() { @@ -84,7 +84,7 @@ impl DotFmt> for Peeph let p = p(self.0); - for a in actions { + for a in actions.iter() { match a { GetLhs { path } => write!(w, "get-lhs @ {}
", p(path))?, UnaryUnquote { operator, operand } => { From 6e135b3aea242c24d7acdaf573b0d660164703a8 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 7 May 2020 17:13:03 -0700 Subject: [PATCH 15/22] peepmatic: Fix a failed assertion due to extra iterations after fixed point After replacing an instruction with an alias to an earlier value, trying to further optimize that value is unnecessary, since we've already processed it, and also was triggering an assertion. --- cranelift/codegen/src/simple_preopt.rs | 22 +++++++++++++++++-- ...zations_after_replacing_with_an_alias.clif | 14 ++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 cranelift/filetests/filetests/peepmatic/do_not_keep_applying_optimizations_after_replacing_with_an_alias.clif diff --git a/cranelift/codegen/src/simple_preopt.rs b/cranelift/codegen/src/simple_preopt.rs index a03c8219d2..a77c8cb19a 100644 --- a/cranelift/codegen/src/simple_preopt.rs +++ b/cranelift/codegen/src/simple_preopt.rs @@ -611,8 +611,26 @@ pub fn do_preopt<'func, 'isa>( while let Some(block) = pos.next_block() { while let Some(inst) = pos.next_inst() { - preopt.apply_all(&mut pos, ValueOrInst::Inst(inst)); - let inst = pos.current_inst().unwrap(); + // After we apply one optimization, that might make another + // optimization applicable. Keep running the peephole optimizer + // until either: + // + // * No optimization applied, and therefore it doesn't make sense to + // try again, because no optimization will apply again. + // + // * Or when we replaced an instruction with an alias to an existing + // value, because we already ran the peephole optimizer over the + // aliased value's instruction in an early part of the traversal + // over the function. + while let Some(ValueOrInst::Inst(new_inst)) = + preopt.apply_one(&mut pos, ValueOrInst::Inst(inst)) + { + // We transplanted a new instruction into the current + // instruction, so the "new" instruction is actually the same + // one, just with different data. + debug_assert_eq!(new_inst, inst); + } + debug_assert_eq!(pos.current_inst(), Some(inst)); // Try to transform divide-by-constant into simpler operations. if let Some(divrem_info) = get_div_info(inst, &pos.func.dfg) { diff --git a/cranelift/filetests/filetests/peepmatic/do_not_keep_applying_optimizations_after_replacing_with_an_alias.clif b/cranelift/filetests/filetests/peepmatic/do_not_keep_applying_optimizations_after_replacing_with_an_alias.clif new file mode 100644 index 0000000000..ceefd5bd1c --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/do_not_keep_applying_optimizations_after_replacing_with_an_alias.clif @@ -0,0 +1,14 @@ +test simple_preopt +target x86_64 + +;; This file used to trigger assertions where we would keep trying to +;; unnecessarily apply optimizations after replacing an instruction with an +;; alias of another value that we had already optimized. + +function %foo() { +block0: + v0 = iconst.i32 3 + v1 = srem_imm v0, 2 + v2 = sdiv_imm v1, 1 + trap unreachable +} From 52c6ece5f399bd6934e3bd6b3867dee795148511 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 8 May 2020 12:15:23 -0700 Subject: [PATCH 16/22] peepmatic: Make peepmatic optional to enable Rather than outright replacing parts of our existing peephole optimizations passes, this makes peepmatic an optional cargo feature that can be enabled. This allows us to take a conservative approach with enabling peepmatic everywhere, while also allowing us to get it in-tree and make it easier to collaborate on improving it quickly. --- cranelift/Cargo.toml | 1 + cranelift/codegen/Cargo.toml | 7 +- cranelift/codegen/src/lib.rs | 4 +- cranelift/codegen/src/simple_preopt.rs | 476 ++++++++++++++++-- cranelift/filetests/Cargo.toml | 3 + .../filetests/isa/x86/isub_imm-i8.clif | 8 +- .../filetests/filetests/peepmatic/branch.clif | 81 +++ .../peepmatic/div_by_const_indirect.clif | 55 ++ .../div_by_const_non_power_of_2.clif | 266 ++++++++++ .../peepmatic/div_by_const_power_of_2.clif | 292 +++++++++++ ...zations_after_replacing_with_an_alias.clif | 2 +- ...order_instructions_when_transplanting.clif | 22 + .../fold-extended-move-wraparound.clif | 14 + .../rem_by_const_non_power_of_2.clif | 285 +++++++++++ .../peepmatic/rem_by_const_power_of_2.clif | 291 +++++++++++ ...ing_instructions_and_cfg_predecessors.clif | 22 + .../filetests/peepmatic/simplify32.clif | 60 +++ .../filetests/peepmatic/simplify64.clif | 326 ++++++++++++ ...plify_instruction_into_alias_of_value.clif | 17 + .../simple_preopt/div_by_const_indirect.clif | 50 +- .../filetests/simple_preopt/simplify32.clif | 1 + .../filetests/simple_preopt/simplify64.clif | 31 -- cranelift/filetests/src/lib.rs | 2 + cranelift/filetests/src/test_peepmatic.rs | 56 +++ cranelift/filetests/src/test_simple_preopt.rs | 12 +- 25 files changed, 2284 insertions(+), 100 deletions(-) create mode 100644 cranelift/filetests/filetests/peepmatic/branch.clif create mode 100644 cranelift/filetests/filetests/peepmatic/div_by_const_indirect.clif create mode 100644 cranelift/filetests/filetests/peepmatic/div_by_const_non_power_of_2.clif create mode 100644 cranelift/filetests/filetests/peepmatic/div_by_const_power_of_2.clif create mode 100644 cranelift/filetests/filetests/peepmatic/do_not_reorder_instructions_when_transplanting.clif create mode 100644 cranelift/filetests/filetests/peepmatic/fold-extended-move-wraparound.clif create mode 100644 cranelift/filetests/filetests/peepmatic/rem_by_const_non_power_of_2.clif create mode 100644 cranelift/filetests/filetests/peepmatic/rem_by_const_power_of_2.clif create mode 100644 cranelift/filetests/filetests/peepmatic/replace_branching_instructions_and_cfg_predecessors.clif create mode 100644 cranelift/filetests/filetests/peepmatic/simplify32.clif create mode 100644 cranelift/filetests/filetests/peepmatic/simplify64.clif create mode 100644 cranelift/filetests/filetests/peepmatic/simplify_instruction_into_alias_of_value.clif create mode 100644 cranelift/filetests/src/test_peepmatic.rs diff --git a/cranelift/Cargo.toml b/cranelift/Cargo.toml index f2e4ea9607..335a69c808 100644 --- a/cranelift/Cargo.toml +++ b/cranelift/Cargo.toml @@ -47,4 +47,5 @@ walkdir = "2.2" [features] default = ["disas", "wasm", "cranelift-codegen/all-arch"] disas = ["capstone"] +enable-peepmatic = ["cranelift-codegen/enable-peepmatic", "cranelift-filetests/enable-peepmatic"] wasm = ["wat", "cranelift-wasm"] diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 4737904eb3..0bc1c32006 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -24,7 +24,7 @@ gimli = { version = "0.20.0", default-features = false, features = ["write"], op smallvec = { version = "1.0.0" } thiserror = "1.0.4" byteorder = { version = "1.3.2", default-features = false } -peepmatic-runtime = { path = "../peepmatic/crates/runtime" } +peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true } regalloc = "0.0.23" # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary @@ -74,9 +74,12 @@ all-arch = [ # For dependent crates that want to serialize some parts of cranelift enable-serde = ["serde"] -# Recompile our optimizations that are written in the peepmatic DSL into a +# Recompile our optimizations that are written in the `peepmatic` DSL into a # compact finite-state transducer automaton. rebuild-peephole-optimizers = ["peepmatic"] +# Enable the use of `peepmatic`-generated peephole optimizers. +enable-peepmatic = ["peepmatic-runtime"] + [badges] maintenance = { status = "experimental" } diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 05c5583f5b..3483219fea 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -101,7 +101,6 @@ mod licm; mod nan_canonicalization; mod num_uses; mod partition_slice; -mod peepmatic; mod postopt; mod predicates; mod redundant_reload_remover; @@ -116,6 +115,9 @@ mod topo_order; mod unreachable_code; mod value_label; +#[cfg(feature = "enable-peepmatic")] +mod peepmatic; + pub use crate::result::{CodegenError, CodegenResult}; /// Version number of this crate. diff --git a/cranelift/codegen/src/simple_preopt.rs b/cranelift/codegen/src/simple_preopt.rs index a77c8cb19a..7413b01e90 100644 --- a/cranelift/codegen/src/simple_preopt.rs +++ b/cranelift/codegen/src/simple_preopt.rs @@ -15,7 +15,6 @@ use crate::ir::{ Block, DataFlowGraph, Function, Inst, InstBuilder, InstructionData, Type, Value, }; use crate::isa::TargetIsa; -use crate::peepmatic::ValueOrInst; use crate::timing; #[inline] @@ -182,8 +181,12 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso // U32 div by 1: identity // U32 rem by 1: zero - DivRemByConstInfo::DivU32(_, 1) | DivRemByConstInfo::RemU32(_, 1) => { - unreachable!("unsigned division and remainder by one is handled in `preopt.peepmatic`"); + DivRemByConstInfo::DivU32(n1, 1) | DivRemByConstInfo::RemU32(n1, 1) => { + if is_rem { + pos.func.dfg.replace(inst).iconst(I32, 0); + } else { + replace_single_result_with_alias(&mut pos.func.dfg, inst, n1); + } } // U32 div, rem by a power-of-2 @@ -198,10 +201,7 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso let mask = (1u64 << k) - 1; pos.func.dfg.replace(inst).band_imm(n1, mask as i64); } else { - unreachable!( - "unsigned division by a power of two is handled in \ - `preopt.peepmatic`" - ); + pos.func.dfg.replace(inst).ushr_imm(n1, k as i64); } } @@ -251,8 +251,12 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso // U64 div by 1: identity // U64 rem by 1: zero - DivRemByConstInfo::DivU64(_, 1) | DivRemByConstInfo::RemU64(_, 1) => { - unreachable!("unsigned division and remainder by one is handled in `preopt.peepmatic`"); + DivRemByConstInfo::DivU64(n1, 1) | DivRemByConstInfo::RemU64(n1, 1) => { + if is_rem { + pos.func.dfg.replace(inst).iconst(I64, 0); + } else { + replace_single_result_with_alias(&mut pos.func.dfg, inst, n1); + } } // U64 div, rem by a power-of-2 @@ -267,9 +271,7 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso let mask = (1u64 << k) - 1; pos.func.dfg.replace(inst).band_imm(n1, mask as i64); } else { - unreachable!( - "unsigned division by a power of two is handled in `preopt.peepmatic`" - ); + pos.func.dfg.replace(inst).ushr_imm(n1, k as i64); } } @@ -322,8 +324,12 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso // S32 div by 1: identity // S32 rem by 1: zero - DivRemByConstInfo::DivS32(_, 1) | DivRemByConstInfo::RemS32(_, 1) => { - unreachable!("signed division and remainder by one is handled in `preopt.peepmatic`"); + DivRemByConstInfo::DivS32(n1, 1) | DivRemByConstInfo::RemS32(n1, 1) => { + if is_rem { + pos.func.dfg.replace(inst).iconst(I32, 0); + } else { + replace_single_result_with_alias(&mut pos.func.dfg, inst, n1); + } } DivRemByConstInfo::DivS32(n1, d) | DivRemByConstInfo::RemS32(n1, d) => { @@ -393,8 +399,12 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso // S64 div by 1: identity // S64 rem by 1: zero - DivRemByConstInfo::DivS64(_, 1) | DivRemByConstInfo::RemS64(_, 1) => { - unreachable!("division and remaineder by one are handled in `preopt.peepmatic`"); + DivRemByConstInfo::DivS64(n1, 1) | DivRemByConstInfo::RemS64(n1, 1) => { + if is_rem { + pos.func.dfg.replace(inst).iconst(I64, 0); + } else { + replace_single_result_with_alias(&mut pos.func.dfg, inst, n1); + } } DivRemByConstInfo::DivS64(n1, d) | DivRemByConstInfo::RemS64(n1, d) => { @@ -598,6 +608,416 @@ fn branch_order(pos: &mut FuncCursor, cfg: &mut ControlFlowGraph, block: Block, cfg.recompute_block(pos.func, block); } +#[cfg(feature = "enable-peepmatic")] +mod simplify { + use super::*; + use crate::peepmatic::ValueOrInst; + + pub type PeepholeOptimizer<'a, 'b> = + peepmatic_runtime::optimizer::PeepholeOptimizer<'static, 'a, &'b dyn TargetIsa>; + + pub fn peephole_optimizer<'a, 'b>(isa: &'b dyn TargetIsa) -> PeepholeOptimizer<'a, 'b> { + crate::peepmatic::preopt(isa) + } + + pub fn apply_all<'a, 'b>( + optimizer: &mut PeepholeOptimizer<'a, 'b>, + pos: &mut FuncCursor<'a>, + inst: Inst, + _native_word_width: u32, + ) { + // After we apply one optimization, that might make another + // optimization applicable. Keep running the peephole optimizer + // until either: + // + // * No optimization applied, and therefore it doesn't make sense to + // try again, because no optimization will apply again. + // + // * Or when we replaced an instruction with an alias to an existing + // value, because we already ran the peephole optimizer over the + // aliased value's instruction in an early part of the traversal + // over the function. + while let Some(ValueOrInst::Inst(new_inst)) = + optimizer.apply_one(pos, ValueOrInst::Inst(inst)) + { + // We transplanted a new instruction into the current + // instruction, so the "new" instruction is actually the same + // one, just with different data. + debug_assert_eq!(new_inst, inst); + } + debug_assert_eq!(pos.current_inst(), Some(inst)); + } +} + +#[cfg(not(feature = "enable-peepmatic"))] +mod simplify { + use super::*; + use crate::ir::{ + dfg::ValueDef, + immediates, + instructions::{Opcode, ValueList}, + types::{I16, I32, I8}, + }; + use std::marker::PhantomData; + + pub struct PeepholeOptimizer<'a, 'b> { + phantom: PhantomData<(&'a (), &'b ())>, + } + + pub fn peephole_optimizer<'a, 'b>(_: &dyn TargetIsa) -> PeepholeOptimizer<'a, 'b> { + PeepholeOptimizer { + phantom: PhantomData, + } + } + + pub fn apply_all<'a, 'b>( + _optimizer: &mut PeepholeOptimizer<'a, 'b>, + pos: &mut FuncCursor<'a>, + inst: Inst, + native_word_width: u32, + ) { + simplify(pos, inst, native_word_width); + branch_opt(pos, inst); + } + + #[inline] + fn resolve_imm64_value(dfg: &DataFlowGraph, value: Value) -> Option { + if let ValueDef::Result(candidate_inst, _) = dfg.value_def(value) { + if let InstructionData::UnaryImm { + opcode: Opcode::Iconst, + imm, + } = dfg[candidate_inst] + { + return Some(imm); + } + } + None + } + + /// Try to transform [(x << N) >> N] into a (un)signed-extending move. + /// Returns true if the final instruction has been converted to such a move. + fn try_fold_extended_move( + pos: &mut FuncCursor, + inst: Inst, + opcode: Opcode, + arg: Value, + imm: immediates::Imm64, + ) -> bool { + if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) { + if let InstructionData::BinaryImm { + opcode: Opcode::IshlImm, + arg: prev_arg, + imm: prev_imm, + } = &pos.func.dfg[arg_inst] + { + if imm != *prev_imm { + return false; + } + + let dest_ty = pos.func.dfg.ctrl_typevar(inst); + if dest_ty != pos.func.dfg.ctrl_typevar(arg_inst) || !dest_ty.is_int() { + return false; + } + + let imm_bits: i64 = imm.into(); + let ireduce_ty = match (dest_ty.lane_bits() as i64).wrapping_sub(imm_bits) { + 8 => I8, + 16 => I16, + 32 => I32, + _ => return false, + }; + let ireduce_ty = ireduce_ty.by(dest_ty.lane_count()).unwrap(); + + // This becomes a no-op, since ireduce_ty has a smaller lane width than + // the argument type (also the destination type). + let arg = *prev_arg; + let narrower_arg = pos.ins().ireduce(ireduce_ty, arg); + + if opcode == Opcode::UshrImm { + pos.func.dfg.replace(inst).uextend(dest_ty, narrower_arg); + } else { + pos.func.dfg.replace(inst).sextend(dest_ty, narrower_arg); + } + return true; + } + } + false + } + + /// Apply basic simplifications. + /// + /// This folds constants with arithmetic to form `_imm` instructions, and other minor + /// simplifications. + /// + /// Doesn't apply some simplifications if the native word width (in bytes) is smaller than the + /// controlling type's width of the instruction. This would result in an illegal instruction that + /// would likely be expanded back into an instruction on smaller types with the same initial + /// opcode, creating unnecessary churn. + fn simplify(pos: &mut FuncCursor, inst: Inst, native_word_width: u32) { + match pos.func.dfg[inst] { + InstructionData::Binary { opcode, args } => { + if let Some(mut imm) = resolve_imm64_value(&pos.func.dfg, args[1]) { + let new_opcode = match opcode { + Opcode::Iadd => Opcode::IaddImm, + Opcode::Imul => Opcode::ImulImm, + Opcode::Sdiv => Opcode::SdivImm, + Opcode::Udiv => Opcode::UdivImm, + Opcode::Srem => Opcode::SremImm, + Opcode::Urem => Opcode::UremImm, + Opcode::Band => Opcode::BandImm, + Opcode::Bor => Opcode::BorImm, + Opcode::Bxor => Opcode::BxorImm, + Opcode::Rotl => Opcode::RotlImm, + Opcode::Rotr => Opcode::RotrImm, + Opcode::Ishl => Opcode::IshlImm, + Opcode::Ushr => Opcode::UshrImm, + Opcode::Sshr => Opcode::SshrImm, + Opcode::Isub => { + imm = imm.wrapping_neg(); + Opcode::IaddImm + } + Opcode::Ifcmp => Opcode::IfcmpImm, + _ => return, + }; + let ty = pos.func.dfg.ctrl_typevar(inst); + if ty.bytes() <= native_word_width { + pos.func + .dfg + .replace(inst) + .BinaryImm(new_opcode, ty, imm, args[0]); + + // Repeat for BinaryImm simplification. + simplify(pos, inst, native_word_width); + } + } else if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[0]) { + let new_opcode = match opcode { + Opcode::Iadd => Opcode::IaddImm, + Opcode::Imul => Opcode::ImulImm, + Opcode::Band => Opcode::BandImm, + Opcode::Bor => Opcode::BorImm, + Opcode::Bxor => Opcode::BxorImm, + Opcode::Isub => Opcode::IrsubImm, + _ => return, + }; + let ty = pos.func.dfg.ctrl_typevar(inst); + if ty.bytes() <= native_word_width { + pos.func + .dfg + .replace(inst) + .BinaryImm(new_opcode, ty, imm, args[1]); + } + } + } + + InstructionData::Unary { opcode, arg } => { + if let Opcode::AdjustSpDown = opcode { + if let Some(imm) = resolve_imm64_value(&pos.func.dfg, arg) { + // Note this works for both positive and negative immediate values. + pos.func.dfg.replace(inst).adjust_sp_down_imm(imm); + } + } + } + + InstructionData::BinaryImm { opcode, arg, imm } => { + let ty = pos.func.dfg.ctrl_typevar(inst); + + let mut arg = arg; + let mut imm = imm; + match opcode { + Opcode::IaddImm + | Opcode::ImulImm + | Opcode::BorImm + | Opcode::BandImm + | Opcode::BxorImm => { + // Fold binary_op(C2, binary_op(C1, x)) into binary_op(binary_op(C1, C2), x) + if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) { + if let InstructionData::BinaryImm { + opcode: prev_opcode, + arg: prev_arg, + imm: prev_imm, + } = &pos.func.dfg[arg_inst] + { + if opcode == *prev_opcode + && ty == pos.func.dfg.ctrl_typevar(arg_inst) + { + let lhs: i64 = imm.into(); + let rhs: i64 = (*prev_imm).into(); + let new_imm = match opcode { + Opcode::BorImm => lhs | rhs, + Opcode::BandImm => lhs & rhs, + Opcode::BxorImm => lhs ^ rhs, + Opcode::IaddImm => lhs.wrapping_add(rhs), + Opcode::ImulImm => lhs.wrapping_mul(rhs), + _ => panic!("can't happen"), + }; + let new_imm = immediates::Imm64::from(new_imm); + let new_arg = *prev_arg; + pos.func + .dfg + .replace(inst) + .BinaryImm(opcode, ty, new_imm, new_arg); + imm = new_imm; + arg = new_arg; + } + } + } + } + + Opcode::UshrImm | Opcode::SshrImm => { + if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width + && try_fold_extended_move(pos, inst, opcode, arg, imm) + { + return; + } + } + + _ => {} + }; + + // Replace operations that are no-ops. + match (opcode, imm.into()) { + (Opcode::IaddImm, 0) + | (Opcode::ImulImm, 1) + | (Opcode::SdivImm, 1) + | (Opcode::UdivImm, 1) + | (Opcode::BorImm, 0) + | (Opcode::BandImm, -1) + | (Opcode::BxorImm, 0) + | (Opcode::RotlImm, 0) + | (Opcode::RotrImm, 0) + | (Opcode::IshlImm, 0) + | (Opcode::UshrImm, 0) + | (Opcode::SshrImm, 0) => { + // Alias the result value with the original argument. + replace_single_result_with_alias(&mut pos.func.dfg, inst, arg); + } + (Opcode::ImulImm, 0) | (Opcode::BandImm, 0) => { + // Replace by zero. + pos.func.dfg.replace(inst).iconst(ty, 0); + } + (Opcode::BorImm, -1) => { + // Replace by minus one. + pos.func.dfg.replace(inst).iconst(ty, -1); + } + _ => {} + } + } + + InstructionData::IntCompare { opcode, cond, args } => { + debug_assert_eq!(opcode, Opcode::Icmp); + if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[1]) { + if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width { + pos.func.dfg.replace(inst).icmp_imm(cond, args[0], imm); + } + } + } + + InstructionData::CondTrap { .. } + | InstructionData::Branch { .. } + | InstructionData::Ternary { + opcode: Opcode::Select, + .. + } => { + // Fold away a redundant `bint`. + let condition_def = { + let args = pos.func.dfg.inst_args(inst); + pos.func.dfg.value_def(args[0]) + }; + if let ValueDef::Result(def_inst, _) = condition_def { + if let InstructionData::Unary { + opcode: Opcode::Bint, + arg: bool_val, + } = pos.func.dfg[def_inst] + { + let args = pos.func.dfg.inst_args_mut(inst); + args[0] = bool_val; + } + } + } + + _ => {} + } + } + + struct BranchOptInfo { + br_inst: Inst, + cmp_arg: Value, + args: ValueList, + new_opcode: Opcode, + } + + /// Fold comparisons into branch operations when possible. + /// + /// This matches against operations which compare against zero, then use the + /// result in a `brz` or `brnz` branch. It folds those two operations into a + /// single `brz` or `brnz`. + fn branch_opt(pos: &mut FuncCursor, inst: Inst) { + let mut info = if let InstructionData::Branch { + opcode: br_opcode, + args: ref br_args, + .. + } = pos.func.dfg[inst] + { + let first_arg = { + let args = pos.func.dfg.inst_args(inst); + args[0] + }; + + let icmp_inst = + if let ValueDef::Result(icmp_inst, _) = pos.func.dfg.value_def(first_arg) { + icmp_inst + } else { + return; + }; + + if let InstructionData::IntCompareImm { + opcode: Opcode::IcmpImm, + arg: cmp_arg, + cond: cmp_cond, + imm: cmp_imm, + } = pos.func.dfg[icmp_inst] + { + let cmp_imm: i64 = cmp_imm.into(); + if cmp_imm != 0 { + return; + } + + // icmp_imm returns non-zero when the comparison is true. So, if + // we're branching on zero, we need to invert the condition. + let cond = match br_opcode { + Opcode::Brz => cmp_cond.inverse(), + Opcode::Brnz => cmp_cond, + _ => return, + }; + + let new_opcode = match cond { + IntCC::Equal => Opcode::Brz, + IntCC::NotEqual => Opcode::Brnz, + _ => return, + }; + + BranchOptInfo { + br_inst: inst, + cmp_arg, + args: br_args.clone(), + new_opcode, + } + } else { + return; + } + } else { + return; + }; + + info.args.as_mut_slice(&mut pos.func.dfg.value_lists)[0] = info.cmp_arg; + if let InstructionData::Branch { ref mut opcode, .. } = pos.func.dfg[info.br_inst] { + *opcode = info.new_opcode; + } else { + panic!(); + } + } +} + /// The main pre-opt pass. pub fn do_preopt<'func, 'isa>( func: &'func mut Function, @@ -607,30 +1027,12 @@ pub fn do_preopt<'func, 'isa>( let _tt = timing::preopt(); let mut pos = FuncCursor::new(func); - let mut preopt = crate::peepmatic::preopt(isa); + let native_word_width = isa.pointer_bytes() as u32; + let mut optimizer = simplify::peephole_optimizer(isa); while let Some(block) = pos.next_block() { while let Some(inst) = pos.next_inst() { - // After we apply one optimization, that might make another - // optimization applicable. Keep running the peephole optimizer - // until either: - // - // * No optimization applied, and therefore it doesn't make sense to - // try again, because no optimization will apply again. - // - // * Or when we replaced an instruction with an alias to an existing - // value, because we already ran the peephole optimizer over the - // aliased value's instruction in an early part of the traversal - // over the function. - while let Some(ValueOrInst::Inst(new_inst)) = - preopt.apply_one(&mut pos, ValueOrInst::Inst(inst)) - { - // We transplanted a new instruction into the current - // instruction, so the "new" instruction is actually the same - // one, just with different data. - debug_assert_eq!(new_inst, inst); - } - debug_assert_eq!(pos.current_inst(), Some(inst)); + simplify::apply_all(&mut optimizer, &mut pos, inst, native_word_width); // Try to transform divide-by-constant into simpler operations. if let Some(divrem_info) = get_div_info(inst, &pos.func.dfg) { diff --git a/cranelift/filetests/Cargo.toml b/cranelift/filetests/Cargo.toml index 481401cf8a..705c31fc61 100644 --- a/cranelift/filetests/Cargo.toml +++ b/cranelift/filetests/Cargo.toml @@ -26,3 +26,6 @@ num_cpus = "1.8.0" region = "2.1.2" target-lexicon = "0.10" thiserror = "1.0.15" + +[features] +enable-peepmatic = [] diff --git a/cranelift/filetests/filetests/isa/x86/isub_imm-i8.clif b/cranelift/filetests/filetests/isa/x86/isub_imm-i8.clif index e59226c7de..dcf6c77e9a 100644 --- a/cranelift/filetests/filetests/isa/x86/isub_imm-i8.clif +++ b/cranelift/filetests/filetests/isa/x86/isub_imm-i8.clif @@ -6,9 +6,9 @@ function u0:0(i8) -> i8 fast { block0(v0: i8): v1 = iconst.i8 0 v2 = isub v1, v0 - ; check: v4 = uextend.i32 v0 - ; nextln: v6 = iconst.i32 0 - ; nextln: v5 = isub v6, v4 - ; nextln: v2 = ireduce.i8 v5 + ; check: uextend.i32 + ; nextln: iconst.i32 + ; nextln: isub + ; nextln: ireduce.i8 return v2 } diff --git a/cranelift/filetests/filetests/peepmatic/branch.clif b/cranelift/filetests/filetests/peepmatic/branch.clif new file mode 100644 index 0000000000..0f68bbe9cb --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/branch.clif @@ -0,0 +1,81 @@ +test peepmatic +target x86_64 + +function %icmp_to_brz_fold(i32) -> i32 { +block0(v0: i32): + v1 = icmp_imm eq v0, 0 + brnz v1, block1 + jump block2 +block1: + v3 = iconst.i32 1 + return v3 +block2: + v4 = iconst.i32 2 + return v4 +} +; sameln: function %icmp_to_brz_fold +; nextln: block0(v0: i32): +; nextln: v1 = icmp_imm eq v0, 0 +; nextln: brnz v0, block2 +; nextln: jump block1 +; nextln: +; nextln: block1: +; nextln: v3 = iconst.i32 1 +; nextln: return v3 +; nextln: +; nextln: block2: +; nextln: v4 = iconst.i32 2 +; nextln: return v4 +; nextln: } + +function %icmp_to_brz_inverted_fold(i32) -> i32 { +block0(v0: i32): + v1 = icmp_imm ne v0, 0 + brz v1, block1 + jump block2 +block1: + v3 = iconst.i32 1 + return v3 +block2: + v4 = iconst.i32 2 + return v4 +} +; sameln: function %icmp_to_brz_inve +; nextln: block0(v0: i32): +; nextln: v1 = icmp_imm ne v0, 0 +; nextln: brnz v0, block2 +; nextln: jump block1 +; nextln: +; nextln: block1: +; nextln: v3 = iconst.i32 1 +; nextln: return v3 +; nextln: +; nextln: block2: +; nextln: v4 = iconst.i32 2 +; nextln: return v4 +; nextln: } + +function %br_icmp_inversion(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + br_icmp ugt v0, v1, block1 + jump block2 +block1: + v2 = iconst.i32 1 + return v2 +block2: + v3 = iconst.i32 2 + return v3 +} +; sameln: function %br_icmp_inversio +; nextln: block0(v0: i32, v1: i32): +; nextln: br_icmp ule v0, v1, block2 +; nextln: jump block1 +; nextln: +; nextln: block1: +; nextln: v2 = iconst.i32 1 +; nextln: return v2 +; nextln: +; nextln: block2: +; nextln: v3 = iconst.i32 2 +; nextln: return v3 +; nextln: } diff --git a/cranelift/filetests/filetests/peepmatic/div_by_const_indirect.clif b/cranelift/filetests/filetests/peepmatic/div_by_const_indirect.clif new file mode 100644 index 0000000000..ba65b2418c --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/div_by_const_indirect.clif @@ -0,0 +1,55 @@ +test peepmatic +target x86_64 baseline + +; Cases where the denominator is created by an iconst + +function %indir_udiv32(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 7 + v2 = udiv v0, v1 + ; check: v4 = iconst.i32 0x2492_4925 + ; nextln: v5 = umulhi v0, v4 + ; nextln: v6 = isub v0, v5 + ; nextln: v7 = ushr_imm v6, 1 + ; nextln: v8 = iadd v7, v5 + ; nextln: v9 = ushr_imm v8, 2 + ; nextln: v2 -> v9 + return v2 +} + +function %indir_sdiv32(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 -17 + v2 = sdiv v0, v1 + ; check: v4 = iconst.i32 0xffff_ffff_8787_8787 + ; nextln: v5 = smulhi v0, v4 + ; nextln: v6 = sshr_imm v5, 3 + ; nextln: v7 = ushr_imm v6, 31 + ; nextln: v8 = iadd v6, v7 + ; nextln: v2 -> v8 + return v2 +} + +function %indir_udiv64(i64) -> i64 { +block0(v0: i64): + v1 = iconst.i64 1337 + v2 = udiv v0, v1 + ; check: v4 = iconst.i64 0xc411_9d95_2866_a139 + ; nextln: v5 = umulhi v0, v4 + ; nextln: v6 = ushr_imm v5, 10 + ; nextln: v2 -> v6 + return v2 +} + +function %indir_sdiv64(i64) -> i64 { +block0(v0: i64): + v1 = iconst.i64 -90210 + v2 = sdiv v0, v1 + ; check: v4 = iconst.i64 0xd181_4ee8_939c_b8bb + ; nextln: v5 = smulhi v0, v4 + ; nextln: v6 = sshr_imm v5, 14 + ; nextln: v7 = ushr_imm v6, 63 + ; nextln: v8 = iadd v6, v7 + ; nextln: v2 -> v8 + return v2 +} diff --git a/cranelift/filetests/filetests/peepmatic/div_by_const_non_power_of_2.clif b/cranelift/filetests/filetests/peepmatic/div_by_const_non_power_of_2.clif new file mode 100644 index 0000000000..0759f92ca9 --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/div_by_const_non_power_of_2.clif @@ -0,0 +1,266 @@ +test peepmatic +target i686 baseline + +; -------- U32 -------- + +; complex case (mul, sub, shift, add, shift) +function %t_udiv32_p7(i32) -> i32 { +block0(v0: i32): + v1 = udiv_imm v0, 7 + ; check: iconst.i32 0x2492_4925 + ; check: umulhi v0, v2 + ; check: isub v0, v3 + ; check: ushr_imm v4, 1 + ; check: iadd v5, v3 + ; check: v7 = ushr_imm v6, 2 + ; check: v1 -> v7 + return v1 +} + +; simple case (mul, shift) +function %t_udiv32_p125(i32) -> i32 { +block0(v0: i32): + v1 = udiv_imm v0, 125 + ; check: iconst.i32 0x1062_4dd3 + ; check: umulhi v0, v2 + ; check: v4 = ushr_imm v3, 3 + ; check: v1 -> v4 + return v1 +} + +; simple case w/ shift by zero (mul) +function %t_udiv32_p641(i32) -> i32 { +block0(v0: i32): + v1 = udiv_imm v0, 641 + ; check: iconst.i32 0x0066_3d81 + ; check: v3 = umulhi v0, v2 + ; check: v1 -> v3 + return v1 +} + + +; -------- S32 -------- + +; simple case w/ shift by zero (mul, add-sign-bit) +function %t_sdiv32_n6(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, -6 + ; check: iconst.i32 0xffff_ffff_d555_5555 + ; check: smulhi v0, v2 + ; check: ushr_imm v3, 31 + ; check: v5 = iadd v3, v4 + ; check: v1 -> v5 + return v1 +} + +; simple case (mul, shift, add-sign-bit) +function %t_sdiv32_n5(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, -5 + ; check: iconst.i32 0xffff_ffff_9999_9999 + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 1 + ; check: ushr_imm v4, 31 + ; check: v6 = iadd v4, v5 + ; check: v1 -> v6 + return v1 +} + +; case d < 0 && M > 0 (mul, sub, shift, add-sign-bit) +function %t_sdiv32_n3(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, -3 + ; check: iconst.i32 0x5555_5555 + ; check: smulhi v0, v2 + ; check: isub v3, v0 + ; check: sshr_imm v4, 1 + ; check: ushr_imm v5, 31 + ; check: v7 = iadd v5, v6 + ; check: v1 -> v7 + return v1 +} + +; simple case w/ shift by zero (mul, add-sign-bit) +function %t_sdiv32_p6(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, 6 + ; check: iconst.i32 0x2aaa_aaab + ; check: smulhi v0, v2 + ; check: ushr_imm v3, 31 + ; check: v5 = iadd v3, v4 + ; check: v1 -> v5 + return v1 +} + +; case d > 0 && M < 0 (mull, add, shift, add-sign-bit) +function %t_sdiv32_p7(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, 7 + ; check: iconst.i32 0xffff_ffff_9249_2493 + ; check: smulhi v0, v2 + ; check: iadd v3, v0 + ; check: sshr_imm v4, 2 + ; check: ushr_imm v5, 31 + ; check: v7 = iadd v5, v6 + ; check: v1 -> v7 + return v1 +} + +; simple case (mul, shift, add-sign-bit) +function %t_sdiv32_p625(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, 625 + ; check: iconst.i32 0x68db_8bad + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 8 + ; check: ushr_imm v4, 31 + ; check: v6 = iadd v4, v5 + ; check: v1 -> v6 + return v1 +} + + +; -------- U64 -------- + +; complex case (mul, sub, shift, add, shift) +function %t_udiv64_p7(i64) -> i64 { +block0(v0: i64): + v1 = udiv_imm v0, 7 + ; check: iconst.i64 0x2492_4924_9249_2493 + ; check: umulhi v0, v2 + ; check: isub v0, v3 + ; check: ushr_imm v4, 1 + ; check: iadd v5, v3 + ; check: v7 = ushr_imm v6, 2 + ; check: v1 -> v7 + return v1 +} + +; simple case (mul, shift) +function %t_udiv64_p9(i64) -> i64 { +block0(v0: i64): + v1 = udiv_imm v0, 9 + ; check: iconst.i64 0xe38e_38e3_8e38_e38f + ; check: umulhi v0, v2 + ; check: v4 = ushr_imm v3, 3 + ; check: v1 -> v4 + return v1 +} + +; complex case (mul, sub, shift, add, shift) +function %t_udiv64_p125(i64) -> i64 { +block0(v0: i64): + v1 = udiv_imm v0, 125 + ; check: iconst.i64 0x0624_dd2f_1a9f_be77 + ; check: umulhi v0, v2 + ; check: isub v0, v3 + ; check: ushr_imm v4, 1 + ; check: iadd v5, v3 + ; check: v7 = ushr_imm v6, 6 + ; check: v1 -> v7 + return v1 +} + +; simple case w/ shift by zero (mul) +function %t_udiv64_p274177(i64) -> i64 { +block0(v0: i64): + v1 = udiv_imm v0, 274177 + ; check: iconst.i64 0x3d30_f19c_d101 + ; check: v3 = umulhi v0, v2 + ; check: v1 -> v3 + return v1 +} + + +; -------- S64 -------- + +; simple case (mul, shift, add-sign-bit) +function %t_sdiv64_n625(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, -625 + ; check: iconst.i64 0xcb92_3a29_c779_a6b5 + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 7 + ; check: ushr_imm v4, 63 + ; check: v6 = iadd v4, v5 + ; check: v1 -> v6 + return v1 +} + +; simple case w/ zero shift (mul, add-sign-bit) +function %t_sdiv64_n6(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, -6 + ; check: iconst.i64 0xd555_5555_5555_5555 + ; check: smulhi v0, v2 + ; check: ushr_imm v3, 63 + ; check: v5 = iadd v3, v4 + ; check: v1 -> v5 + return v1 +} + +; simple case w/ zero shift (mul, add-sign-bit) +function %t_sdiv64_n5(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, -5 + ; check: iconst.i64 0x9999_9999_9999_9999 + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 1 + ; check: ushr_imm v4, 63 + ; check: v6 = iadd v4, v5 + ; check: v1 -> v6 + return v1 +} + +; case d < 0 && M > 0 (mul, sub, shift, add-sign-bit) +function %t_sdiv64_n3(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, -3 + ; check: iconst.i64 0x5555_5555_5555_5555 + ; check: smulhi v0, v2 + ; check: isub v3, v0 + ; check: sshr_imm v4, 1 + ; check: ushr_imm v5, 63 + ; check: v7 = iadd v5, v6 + ; check: v1 -> v7 + return v1 +} + +; simple case w/ zero shift (mul, add-sign-bit) +function %t_sdiv64_p6(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, 6 + ; check: iconst.i64 0x2aaa_aaaa_aaaa_aaab + ; check: smulhi v0, v2 + ; check: ushr_imm v3, 63 + ; check: v5 = iadd v3, v4 + ; check: v1 -> v5 + return v1 +} + +; case d > 0 && M < 0 (mul, add, shift, add-sign-bit) +function %t_sdiv64_p15(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, 15 + ; check: iconst.i64 0x8888_8888_8888_8889 + ; check: smulhi v0, v2 + ; check: iadd v3, v0 + ; check: sshr_imm v4, 3 + ; check: ushr_imm v5, 63 + ; check: v7 = iadd v5, v6 + ; check: v1 -> v7 + return v1 +} + +; simple case (mul, shift, add-sign-bit) +function %t_sdiv64_p625(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, 625 + ; check: iconst.i64 0x346d_c5d6_3886_594b + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 7 + ; check: ushr_imm v4, 63 + ; check: v6 = iadd v4, v5 + ; check: v1 -> v6 + return v1 +} diff --git a/cranelift/filetests/filetests/peepmatic/div_by_const_power_of_2.clif b/cranelift/filetests/filetests/peepmatic/div_by_const_power_of_2.clif new file mode 100644 index 0000000000..a2110a5a75 --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/div_by_const_power_of_2.clif @@ -0,0 +1,292 @@ +test peepmatic +target i686 baseline + +; -------- U32 -------- + +; ignored +function %t_udiv32_p0(i32) -> i32 { +block0(v0: i32): + v1 = udiv_imm v0, 0 + ; check: udiv_imm v0, 0 + return v1 +} + +; converted to a nop +function %t_udiv32_p1(i32) -> i32 { +block0(v0: i32): + v1 = udiv_imm v0, 1 + ; check: nop + return v1 +} + +; shift +function %t_udiv32_p2(i32) -> i32 { +block0(v0: i32): + v1 = udiv_imm v0, 2 + ; check: ushr_imm v0, 1 + return v1 +} + +; shift +function %t_udiv32_p2p31(i32) -> i32 { +block0(v0: i32): + v1 = udiv_imm v0, 0x8000_0000 + ; check: ushr_imm v0, 31 + return v1 +} + + +; -------- U64 -------- + +; ignored +function %t_udiv64_p0(i64) -> i64 { +block0(v0: i64): + v1 = udiv_imm v0, 0 + ; check: udiv_imm v0, 0 + return v1 +} + +; converted to a nop +function %t_udiv64_p1(i64) -> i64 { +block0(v0: i64): + v1 = udiv_imm v0, 1 + ; check: nop + return v1 +} + +; shift +function %t_udiv64_p2(i64) -> i64 { +block0(v0: i64): + v1 = udiv_imm v0, 2 + ; check: ushr_imm v0, 1 + return v1 +} + +; shift +function %t_udiv64_p2p63(i64) -> i64 { +block0(v0: i64): + v1 = udiv_imm v0, 0x8000_0000_0000_0000 + ; check: ushr_imm v0, 63 + return v1 +} + + +; -------- S32 -------- + +; ignored +function %t_sdiv32_p0(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, 0 + ; check: sdiv_imm v0, 0 + return v1 +} + +; converted to a nop +function %t_sdiv32_p1(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, 1 + ; check: nop + return v1 +} + +; ignored +function %t_sdiv32_n1(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, -1 + ; check: sdiv_imm v0, -1 + return v1 +} + +; shift +function %t_sdiv32_p2(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, 2 + ; check: ushr_imm v0, 31 + ; check: iadd v0, v2 + ; check: sshr_imm v3, 1 + ; check: v1 -> v4 + return v1 +} + +; shift +function %t_sdiv32_n2(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, -2 + ; check: ushr_imm v0, 31 + ; check: iadd v0, v2 + ; check: sshr_imm v3, 1 + ; check: irsub_imm v4, 0 + return v1 +} + +; shift +function %t_sdiv32_p4(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, 4 + ; check: v2 = sshr_imm v0, 1 + ; check: ushr_imm v2, 30 + ; check: iadd v0, v3 + ; check: v5 = sshr_imm v4, 2 + ; check: v1 -> v5 + + return v1 +} + +; shift +function %t_sdiv32_n4(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, -4 + ; check: sshr_imm v0, 1 + ; check: ushr_imm v2, 30 + ; check: iadd v0, v3 + ; check: sshr_imm v4, 2 + ; check: irsub_imm v5, 0 + return v1 +} + +; shift +function %t_sdiv32_p2p30(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, 0x4000_0000 + ; check: sshr_imm v0, 29 + ; check: ushr_imm v2, 2 + ; check: iadd v0, v3 + ; check: v5 = sshr_imm v4, 30 + ; check: v1 -> v5 + return v1 +} + +; shift +function %t_sdiv32_n2p30(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, -0x4000_0000 + ; check: sshr_imm v0, 29 + ; check: ushr_imm v2, 2 + ; check: iadd v0, v3 + ; check: sshr_imm v4, 30 + ; check: irsub_imm v5, 0 + return v1 +} + +; there's no positive version of this, since -(-0x8000_0000) isn't +; representable. +function %t_sdiv32_n2p31(i32) -> i32 { +block0(v0: i32): + v1 = sdiv_imm v0, -0x8000_0000 + ; check: sshr_imm v0, 30 + ; check: ushr_imm v2, 1 + ; check: iadd v0, v3 + ; check: sshr_imm v4, 31 + ; check: irsub_imm v5, 0 + return v1 +} + + +; -------- S64 -------- + +; ignored +function %t_sdiv64_p0(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, 0 + ; check: sdiv_imm v0, 0 + return v1 +} + +; converted to a nop +function %t_sdiv64_p1(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, 1 + ; check: nop + return v1 +} + +; ignored +function %t_sdiv64_n1(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, -1 + ; check: sdiv_imm v0, -1 + return v1 +} + +; shift +function %t_sdiv64_p2(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, 2 + ; check: ushr_imm v0, 63 + ; check: iadd v0, v2 + ; check: v4 = sshr_imm v3, 1 + ; check: v1 -> v4 + return v1 +} + +; shift +function %t_sdiv64_n2(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, -2 + ; check: ushr_imm v0, 63 + ; check: iadd v0, v2 + ; check: sshr_imm v3, 1 + ; check: irsub_imm v4, 0 + return v1 +} + +; shift +function %t_sdiv64_p4(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, 4 + ; check: sshr_imm v0, 1 + ; check: ushr_imm v2, 62 + ; check: iadd v0, v3 + ; check: v5 = sshr_imm v4, 2 + ; check: v1 -> v5 + return v1 +} + +; shift +function %t_sdiv64_n4(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, -4 + ; check: sshr_imm v0, 1 + ; check: ushr_imm v2, 62 + ; check: iadd v0, v3 + ; check: sshr_imm v4, 2 + ; check: irsub_imm v5, 0 + return v1 +} + +; shift +function %t_sdiv64_p2p62(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, 0x4000_0000_0000_0000 + ; check: sshr_imm v0, 61 + ; check: ushr_imm v2, 2 + ; check: iadd v0, v3 + ; check: v5 = sshr_imm v4, 62 + ; check: v1 -> v5 + return v1 +} + +; shift +function %t_sdiv64_n2p62(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, -0x4000_0000_0000_0000 + ; check: sshr_imm v0, 61 + ; check: ushr_imm v2, 2 + ; check: iadd v0, v3 + ; check: sshr_imm v4, 62 + ; check: irsub_imm v5, 0 + return v1 +} + +; there's no positive version of this, since -(-0x8000_0000_0000_0000) isn't +; representable. +function %t_sdiv64_n2p63(i64) -> i64 { +block0(v0: i64): + v1 = sdiv_imm v0, -0x8000_0000_0000_0000 + ; check: sshr_imm v0, 62 + ; check: ushr_imm v2, 1 + ; check: iadd v0, v3 + ; check: sshr_imm v4, 63 + ; check: irsub_imm v5, 0 + return v1 +} diff --git a/cranelift/filetests/filetests/peepmatic/do_not_keep_applying_optimizations_after_replacing_with_an_alias.clif b/cranelift/filetests/filetests/peepmatic/do_not_keep_applying_optimizations_after_replacing_with_an_alias.clif index ceefd5bd1c..cc24167267 100644 --- a/cranelift/filetests/filetests/peepmatic/do_not_keep_applying_optimizations_after_replacing_with_an_alias.clif +++ b/cranelift/filetests/filetests/peepmatic/do_not_keep_applying_optimizations_after_replacing_with_an_alias.clif @@ -1,4 +1,4 @@ -test simple_preopt +test peepmatic target x86_64 ;; This file used to trigger assertions where we would keep trying to diff --git a/cranelift/filetests/filetests/peepmatic/do_not_reorder_instructions_when_transplanting.clif b/cranelift/filetests/filetests/peepmatic/do_not_reorder_instructions_when_transplanting.clif new file mode 100644 index 0000000000..7fc95f0fdb --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/do_not_reorder_instructions_when_transplanting.clif @@ -0,0 +1,22 @@ +test peepmatic +target x86_64 + +;; Test that although v5 can be replaced with v1, we don't transplant `load.i32 +;; v0` on top of `iadd v3, v4`, because that would move the load past other uses +;; of its result. + +function %foo(i64) -> i32 { +block0(v0: i64): + v1 = load.i32 v0 + v2 = iconst.i32 16 + v3 = iadd_imm v1, -16 + v4 = iconst.i32 16 + v5 = iadd v3, v4 + ; check: v1 = load.i32 v0 + ; nextln: v5 -> v1 + ; nextln: v2 = iconst.i32 16 + ; nextln: v3 = iadd_imm v1, -16 + ; nextln: v4 = iconst.i32 16 + ; nextln: nop + return v5 +} diff --git a/cranelift/filetests/filetests/peepmatic/fold-extended-move-wraparound.clif b/cranelift/filetests/filetests/peepmatic/fold-extended-move-wraparound.clif new file mode 100644 index 0000000000..e48b91a4b1 --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/fold-extended-move-wraparound.clif @@ -0,0 +1,14 @@ +test peepmatic +target x86_64 + +function %wraparound(i64 vmctx) -> f32 system_v { + gv0 = vmctx + gv1 = iadd_imm.i64 gv0, 48 + +block35(v0: i64): + v88 = iconst.i64 0 + v89 = iconst.i64 0x8000_0000_0000_0000 + v90 = ishl_imm v88, 0x8000_0000_0000_0000 + v91 = sshr v90, v89; check: sshr_imm v90, 0x8000_0000_0000_0000 + trap user0 +} diff --git a/cranelift/filetests/filetests/peepmatic/rem_by_const_non_power_of_2.clif b/cranelift/filetests/filetests/peepmatic/rem_by_const_non_power_of_2.clif new file mode 100644 index 0000000000..7df5baf4e3 --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/rem_by_const_non_power_of_2.clif @@ -0,0 +1,285 @@ +test peepmatic +target i686 baseline + +; -------- U32 -------- + +; complex case (mul, sub, shift, add, shift) +function %t_urem32_p7(i32) -> i32 { +block0(v0: i32): + v1 = urem_imm v0, 7 + ; check: iconst.i32 0x2492_4925 + ; check: umulhi v0, v2 + ; check: isub v0, v3 + ; check: ushr_imm v4, 1 + ; check: iadd v5, v3 + ; check: ushr_imm v6, 2 + ; check: imul_imm v7, 7 + ; check: isub v0, v8 + return v1 +} + +; simple case (mul, shift) +function %t_urem32_p125(i32) -> i32 { +block0(v0: i32): + v1 = urem_imm v0, 125 + ; check: iconst.i32 0x1062_4dd3 + ; check: umulhi v0, v2 + ; check: ushr_imm v3, 3 + ; check: imul_imm v4, 125 + ; check: isub v0, v5 + return v1 +} + +; simple case w/ shift by zero (mul) +function %t_urem32_p641(i32) -> i32 { +block0(v0: i32): + v1 = urem_imm v0, 641 + ; check: iconst.i32 0x0066_3d81 + ; check: umulhi v0, v2 + ; check: imul_imm v3, 641 + ; check: isub v0, v4 + return v1 +} + + +; -------- S32 -------- + +; simple case w/ shift by zero (mul, add-sign-bit) +function %t_srem32_n6(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, -6 + ; check: iconst.i32 0xffff_ffff_d555_5555 + ; check: smulhi v0, v2 + ; check: ushr_imm v3, 31 + ; check: iadd v3, v4 + ; check: imul_imm v5, -6 + ; check: isub v0, v6 + return v1 +} + +; simple case (mul, shift, add-sign-bit) +function %t_srem32_n5(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, -5 + ; check: iconst.i32 0xffff_ffff_9999_9999 + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 1 + ; check: ushr_imm v4, 31 + ; check: iadd v4, v5 + ; check: imul_imm v6, -5 + ; check: isub v0, v7 + return v1 +} + +; case d < 0 && M > 0 (mul, sub, shift, add-sign-bit) +function %t_srem32_n3(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, -3 + ; check: iconst.i32 0x5555_5555 + ; check: smulhi v0, v2 + ; check: isub v3, v0 + ; check: sshr_imm v4, 1 + ; check: ushr_imm v5, 31 + ; check: iadd v5, v6 + ; check: imul_imm v7, -3 + ; check: isub v0, v8 + return v1 +} + +; simple case w/ shift by zero (mul, add-sign-bit) +function %t_srem32_p6(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, 6 + ; check: iconst.i32 0x2aaa_aaab + ; check: smulhi v0, v2 + ; check: ushr_imm v3, 31 + ; check: iadd v3, v4 + ; check: imul_imm v5, 6 + ; check: isub v0, v6 + return v1 +} + +; case d > 0 && M < 0 (mull, add, shift, add-sign-bit) +function %t_srem32_p7(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, 7 + ; check: iconst.i32 0xffff_ffff_9249_2493 + ; check: smulhi v0, v2 + ; check: iadd v3, v0 + ; check: sshr_imm v4, 2 + ; check: ushr_imm v5, 31 + ; check: iadd v5, v6 + ; check: imul_imm v7, 7 + ; check: isub v0, v8 + return v1 +} + +; simple case (mul, shift, add-sign-bit) +function %t_srem32_p625(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, 625 + ; check: iconst.i32 0x68db_8bad + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 8 + ; check: ushr_imm v4, 31 + ; check: iadd v4, v5 + ; check: imul_imm v6, 625 + ; check: isub v0, v7 + return v1 +} + + +; -------- U64 -------- + +; complex case (mul, sub, shift, add, shift) +function %t_urem64_p7(i64) -> i64 { +block0(v0: i64): + v1 = urem_imm v0, 7 + ; check: umulhi v0, v2 + ; check: isub v0, v3 + ; check: ushr_imm v4, 1 + ; check: iadd v5, v3 + ; check: ushr_imm v6, 2 + ; check: imul_imm v7, 7 + ; check: isub v0, v8 + return v1 +} + +; simple case (mul, shift) +function %t_urem64_p9(i64) -> i64 { +block0(v0: i64): + v1 = urem_imm v0, 9 + ; check: iconst.i64 0xe38e_38e3_8e38_e38f + ; check: umulhi v0, v2 + ; check: ushr_imm v3, 3 + ; check: imul_imm v4, 9 + ; check: isub v0, v5 + return v1 +} + +; complex case (mul, sub, shift, add, shift) +function %t_urem64_p125(i64) -> i64 { +block0(v0: i64): + v1 = urem_imm v0, 125 + ; check: iconst.i64 0x0624_dd2f_1a9f_be77 + ; check: umulhi v0, v2 + ; check: isub v0, v3 + ; check: ushr_imm v4, 1 + ; check: iadd v5, v3 + ; check: ushr_imm v6, 6 + ; check: imul_imm v7, 125 + ; check: isub v0, v8 + return v1 +} + +; simple case w/ shift by zero (mul) +function %t_urem64_p274177(i64) -> i64 { +block0(v0: i64): + v1 = urem_imm v0, 274177 + ; check: iconst.i64 0x3d30_f19c_d101 + ; check: umulhi v0, v2 + ; check: imul_imm v3, 0x0004_2f01 + ; check: isub v0, v4 + return v1 +} + + +; -------- S64 -------- + +; simple case (mul, shift, add-sign-bit) +function %t_srem64_n625(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, -625 + ; check: iconst.i64 0xcb92_3a29_c779_a6b5 + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 7 + ; check: ushr_imm v4, 63 + ; check: iadd v4, v5 + ; check: imul_imm v6, -625 + ; check: isub v0, v7 + return v1 +} + +; simple case w/ zero shift (mul, add-sign-bit) +function %t_srem64_n6(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, -6 + ; check: iconst.i64 0xd555_5555_5555_5555 + ; check: smulhi v0, v2 + ; check: ushr_imm v3, 63 + ; check: iadd v3, v4 + ; check: imul_imm v5, -6 + ; check: isub v0, v6 + return v1 +} + +; simple case w/ zero shift (mul, add-sign-bit) +function %t_srem64_n5(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, -5 + ; check: iconst.i64 0x9999_9999_9999_9999 + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 1 + ; check: ushr_imm v4, 63 + ; check: iadd v4, v5 + ; check: imul_imm v6, -5 + ; check: isub v0, v7 + return v1 +} + +; case d < 0 && M > 0 (mul, sub, shift, add-sign-bit) +function %t_srem64_n3(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, -3 + ; check: iconst.i64 0x5555_5555_5555_5555 + ; check: smulhi v0, v2 + ; check: isub v3, v0 + ; check: sshr_imm v4, 1 + ; check: ushr_imm v5, 63 + ; check: iadd v5, v6 + ; check: imul_imm v7, -3 + ; check: isub v0, v8 + return v1 +} + +; simple case w/ zero shift (mul, add-sign-bit) +function %t_srem64_p6(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, 6 + ; check: iconst.i64 0x2aaa_aaaa_aaaa_aaab + ; check: smulhi v0, v2 + ; check: ushr_imm v3, 63 + ; check: iadd v3, v4 + ; check: imul_imm v5, 6 + ; check: isub v0, v6 + return v1 +} + +; case d > 0 && M < 0 (mul, add, shift, add-sign-bit) +function %t_srem64_p15(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, 15 + ; check: iconst.i64 0x8888_8888_8888_8889 + ; check: smulhi v0, v2 + ; check: iadd v3, v0 + ; check: sshr_imm v4, 3 + ; check: ushr_imm v5, 63 + ; check: iadd v5, v6 + ; check: imul_imm v7, 15 + ; check: isub v0, v8 + return v1 +} + +; simple case (mul, shift, add-sign-bit) +function %t_srem64_p625(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, 625 + ; check: iconst.i64 0x346d_c5d6_3886_594b + ; check: smulhi v0, v2 + ; check: sshr_imm v3, 7 + ; check: ushr_imm v4, 63 + ; check: iadd v4, v5 + ; check: imul_imm v6, 625 + ; check: isub v0, v7 + return v1 +} diff --git a/cranelift/filetests/filetests/peepmatic/rem_by_const_power_of_2.clif b/cranelift/filetests/filetests/peepmatic/rem_by_const_power_of_2.clif new file mode 100644 index 0000000000..c795b73c19 --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/rem_by_const_power_of_2.clif @@ -0,0 +1,291 @@ +test peepmatic +target i686 baseline + +; -------- U32 -------- + +; ignored +function %t_urem32_p0(i32) -> i32 { +block0(v0: i32): + v1 = urem_imm v0, 0 + ; check: urem_imm v0, 0 + return v1 +} + +; converted to constant zero +function %t_urem32_p1(i32) -> i32 { +block0(v0: i32): + v1 = urem_imm v0, 1 + ; check: iconst.i32 0 + return v1 +} + +; shift +function %t_urem32_p2(i32) -> i32 { +block0(v0: i32): + v1 = urem_imm v0, 2 + ; check: band_imm v0, 1 + return v1 +} + +; shift +function %t_urem32_p2p31(i32) -> i32 { +block0(v0: i32): + v1 = urem_imm v0, 0x8000_0000 + ; check: band_imm v0, 0x7fff_ffff + return v1 +} + + +; -------- U64 -------- + +; ignored +function %t_urem64_p0(i64) -> i64 { +block0(v0: i64): + v1 = urem_imm v0, 0 + ; check: urem_imm v0, 0 + return v1 +} + +; converted to constant zero +function %t_urem64_p1(i64) -> i64 { +block0(v0: i64): + v1 = urem_imm v0, 1 + ; check: iconst.i64 0 + return v1 +} + +; shift +function %t_urem64_p2(i64) -> i64 { +block0(v0: i64): + v1 = urem_imm v0, 2 + ; check: band_imm v0, 1 + return v1 +} + +; shift +function %t_urem64_p2p63(i64) -> i64 { +block0(v0: i64): + v1 = urem_imm v0, 0x8000_0000_0000_0000 + ; check: band_imm v0, 0x7fff_ffff_ffff_ffff + return v1 +} + + +; -------- S32 -------- + +; ignored +function %t_srem32_n1(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, -1 + ; check: srem_imm v0, -1 + return v1 +} + +; ignored +function %t_srem32_p0(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, 0 + ; check: srem_imm v0, 0 + return v1 +} + +; converted to constant zero +function %t_srem32_p1(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, 1 + ; check: iconst.i32 0 + return v1 +} + +; shift +function %t_srem32_p2(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, 2 + ; check: ushr_imm v0, 31 + ; check: iadd v0, v2 + ; check: band_imm v3, -2 + ; check: isub v0, v4 + return v1 +} + +; shift +function %t_srem32_n2(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, -2 + ; check: ushr_imm v0, 31 + ; check: iadd v0, v2 + ; check: band_imm v3, -2 + ; check: isub v0, v4 + return v1 +} + +; shift +function %t_srem32_p4(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, 4 + ; check: sshr_imm v0, 1 + ; check: ushr_imm v2, 30 + ; check: iadd v0, v3 + ; check: band_imm v4, -4 + ; check: isub v0, v5 + return v1 +} + +; shift +function %t_srem32_n4(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, -4 + ; check: sshr_imm v0, 1 + ; check: ushr_imm v2, 30 + ; check: iadd v0, v3 + ; check: band_imm v4, -4 + ; check: isub v0, v5 + return v1 +} + +; shift +function %t_srem32_p2p30(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, 0x4000_0000 + ; check: sshr_imm v0, 29 + ; check: ushr_imm v2, 2 + ; check: iadd v0, v3 + ; check: band_imm v4, 0xffff_ffff_c000_0000 + ; check: isub v0, v5 + return v1 +} + +; shift +function %t_srem32_n2p30(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, -0x4000_0000 + ; check: sshr_imm v0, 29 + ; check: ushr_imm v2, 2 + ; check: iadd v0, v3 + ; check: band_imm v4, 0xffff_ffff_c000_0000 + ; check: isub v0, v5 + return v1 +} + +; there's no positive version of this, since -(-0x8000_0000) isn't +; representable. +function %t_srem32_n2p31(i32) -> i32 { +block0(v0: i32): + v1 = srem_imm v0, -0x8000_0000 + ; check: sshr_imm v0, 30 + ; check: ushr_imm v2, 1 + ; check: iadd v0, v3 + ; check: band_imm v4, 0xffff_ffff_8000_0000 + ; check: isub v0, v5 + return v1 +} + + +; -------- S64 -------- + +; ignored +function %t_srem64_n1(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, -1 + ; check: srem_imm v0, -1 + return v1 +} + +; ignored +function %t_srem64_p0(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, 0 + ; check: srem_imm v0, 0 + return v1 +} + +; converted to constant zero +function %t_srem64_p1(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, 1 + ; check: iconst.i64 0 + return v1 +} + +; shift +function %t_srem64_p2(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, 2 + ; check: ushr_imm v0, 63 + ; check: iadd v0, v2 + ; check: band_imm v3, -2 + ; check: isub v0, v4 + return v1 +} + +; shift +function %t_srem64_n2(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, -2 + ; check: ushr_imm v0, 63 + ; check: iadd v0, v2 + ; check: band_imm v3, -2 + ; check: isub v0, v4 + return v1 +} + +; shift +function %t_srem64_p4(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, 4 + ; check: sshr_imm v0, 1 + ; check: ushr_imm v2, 62 + ; check: iadd v0, v3 + ; check: band_imm v4, -4 + ; check: isub v0, v5 + return v1 +} + +; shift +function %t_srem64_n4(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, -4 + ; check: sshr_imm v0, 1 + ; check: ushr_imm v2, 62 + ; check: iadd v0, v3 + ; check: band_imm v4, -4 + ; check: isub v0, v5 + return v1 +} + +; shift +function %t_srem64_p2p62(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, 0x4000_0000_0000_0000 + ; check: sshr_imm v0, 61 + ; check: ushr_imm v2, 2 + ; check: iadd v0, v3 + ; check: band_imm v4, 0xc000_0000_0000_0000 + ; check: isub v0, v5 + return v1 +} + +; shift +function %t_srem64_n2p62(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, -0x4000_0000_0000_0000 + ; check: sshr_imm v0, 61 + ; check: ushr_imm v2, 2 + ; check: iadd v0, v3 + ; check: band_imm v4, 0xc000_0000_0000_0000 + ; check: isub v0, v5 + return v1 +} + +; there's no positive version of this, since -(-0x8000_0000_0000_0000) isn't +; representable. +function %t_srem64_n2p63(i64) -> i64 { +block0(v0: i64): + v1 = srem_imm v0, -0x8000_0000_0000_0000 + ; check: sshr_imm v0, 62 + ; check: ushr_imm v2, 1 + ; check: iadd v0, v3 + ; check: band_imm v4, 0x8000_0000_0000_0000 + ; check: isub v0, v5 + return v1 +} diff --git a/cranelift/filetests/filetests/peepmatic/replace_branching_instructions_and_cfg_predecessors.clif b/cranelift/filetests/filetests/peepmatic/replace_branching_instructions_and_cfg_predecessors.clif new file mode 100644 index 0000000000..17ca472b7e --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/replace_branching_instructions_and_cfg_predecessors.clif @@ -0,0 +1,22 @@ +test peepmatic +target x86_64 + +function u0:2(i64 , i64) { + gv1 = load.i64 notrap aligned gv0 + heap0 = static gv1 + block0(v0: i64, v1: i64): + v16 = iconst.i32 6 + v17 = heap_addr.i64 heap0, v16, 1 + v18 = load.i32 v17 + v19 = iconst.i32 4 + v20 = icmp ne v18, v19 + v21 = bint.i32 v20 + brnz v21, block2 + jump block4 + block4: + jump block1 + block2: + jump block1 + block1: + return +} diff --git a/cranelift/filetests/filetests/peepmatic/simplify32.clif b/cranelift/filetests/filetests/peepmatic/simplify32.clif new file mode 100644 index 0000000000..b1c6786a05 --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/simplify32.clif @@ -0,0 +1,60 @@ +test peepmatic +target i686 + +;; 32-bits platforms. + +function %iadd_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 2 + v2 = iadd v0, v1 + return v2 +} +; sameln: function %iadd_imm +; nextln: block0(v0: i32): +; nextln: v1 = iconst.i32 2 +; nextln: v2 = iadd_imm v0, 2 +; nextln: return v2 +; nextln: } + +function %isub_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 2 + v2 = isub v0, v1 + return v2 +} +; sameln: function %isub_imm +; nextln: block0(v0: i32): +; nextln: v1 = iconst.i32 2 +; nextln: v2 = iadd_imm v0, -2 +; nextln: return v2 +; nextln: } + +function %icmp_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 2 + v2 = icmp slt v0, v1 + v3 = bint.i32 v2 + return v3 +} +; sameln: function %icmp_imm +; nextln: block0(v0: i32): +; nextln: v1 = iconst.i32 2 +; nextln: v2 = icmp_imm slt v0, 2 +; nextln: v3 = bint.i32 v2 +; nextln: return v3 +; nextln: } + +;; Don't simplify operations that would get illegal because of lack of native +;; support. +function %iadd_imm(i64) -> i64 { +block0(v0: i64): + v1 = iconst.i64 2 + v2 = iadd v0, v1 + return v2 +} +; sameln: function %iadd_imm +; nextln: block0(v0: i64): +; nextln: v1 = iconst.i64 2 +; nextln: v2 = iadd v0, v1 +; nextln: return v2 +; nextln: } diff --git a/cranelift/filetests/filetests/peepmatic/simplify64.clif b/cranelift/filetests/filetests/peepmatic/simplify64.clif new file mode 100644 index 0000000000..93c289ccdd --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/simplify64.clif @@ -0,0 +1,326 @@ +test peepmatic +target x86_64 + +;; 64-bits platforms. + +function %iadd_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 2 + v2 = iadd v0, v1 + return v2 +} +; sameln: function %iadd_imm +; nextln: block0(v0: i32): +; nextln: v1 = iconst.i32 2 +; nextln: v2 = iadd_imm v0, 2 +; nextln: return v2 +; nextln: } + +function %isub_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 2 + v2 = isub v0, v1 + return v2 +} +; sameln: function %isub_imm +; nextln: block0(v0: i32): +; nextln: v1 = iconst.i32 2 +; nextln: v2 = iadd_imm v0, -2 +; nextln: return v2 +; nextln: } + +function %icmp_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 2 + v2 = icmp slt v0, v1 + v3 = bint.i32 v2 + return v3 +} +; sameln: function %icmp_imm +; nextln: block0(v0: i32): +; nextln: v1 = iconst.i32 2 +; nextln: v2 = icmp_imm slt v0, 2 +; nextln: v3 = bint.i32 v2 +; nextln: return v3 +; nextln: } + +function %ifcmp_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 2 + v2 = ifcmp v0, v1 + brif eq v2, block1 + jump block2 + +block1: + v3 = iconst.i32 1 + return v3 + +block2: + v4 = iconst.i32 2 + return v4 +} +; sameln: function %ifcmp_imm +; nextln: block0(v0: i32): +; nextln: v1 = iconst.i32 2 +; nextln: v2 = ifcmp_imm v0, 2 +; nextln: brif eq v2, block1 +; nextln: jump block2 +; nextln: +; nextln: block1: +; nextln: v3 = iconst.i32 1 +; nextln: return v3 +; nextln: +; nextln: block2: +; nextln: v4 = iconst.i32 2 +; nextln: return v4 +; nextln: } + +function %brz_bint(i32) { +block0(v0: i32): + v3 = icmp_imm slt v0, 0 + v1 = bint.i32 v3 + v2 = select v1, v1, v1 + trapz v1, user0 + brz v1, block1 + jump block2 + +block1: + return + +block2: + return +} +; sameln: function %brz_bint +; nextln: (v0: i32): +; nextln: v3 = icmp_imm slt v0, 0 +; nextln: v1 = bint.i32 v3 +; nextln: v2 = select v3, v1, v1 +; nextln: trapz v3, user0 +; nextln: brnz v3, block2 +; nextln: jump block1 + +function %irsub_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 2 + v2 = isub v1, v0 + return v2 +} +; sameln: function %irsub_imm +; nextln: block0(v0: i32): +; nextln: v1 = iconst.i32 2 +; nextln: v2 = irsub_imm v0, 2 +; nextln: return v2 +; nextln: } + +;; Sign-extensions. + +;; 8 -> 16 +function %uextend_8_16() -> i16 { +block0: + v0 = iconst.i16 37 + v1 = ishl_imm v0, 8 + v2 = ushr_imm v1, 8 + return v2 +} +; sameln: function %uextend_8_16 +; nextln: block0: +; nextln: v0 = iconst.i16 37 +; nextln: v1 = ishl_imm v0, 8 +; nextln: v3 = ireduce.i8 v0 +; nextln: v2 = uextend.i16 v3 +; nextln: return v2 +; nextln: } + +function %sextend_8_16() -> i16 { +block0: + v0 = iconst.i16 37 + v1 = ishl_imm v0, 8 + v2 = sshr_imm v1, 8 + return v2 +} +; sameln: function %sextend_8_16 +; nextln: block0: +; nextln: v0 = iconst.i16 37 +; nextln: v1 = ishl_imm v0, 8 +; nextln: v3 = ireduce.i8 v0 +; nextln: v2 = sextend.i16 v3 +; nextln: return v2 +; nextln: } + +;; 8 -> 32 +function %uextend_8_32() -> i32 { +block0: + v0 = iconst.i32 37 + v1 = ishl_imm v0, 24 + v2 = ushr_imm v1, 24 + return v2 +} +; sameln: function %uextend_8_32 +; nextln: block0: +; nextln: v0 = iconst.i32 37 +; nextln: v1 = ishl_imm v0, 24 +; nextln: v3 = ireduce.i8 v0 +; nextln: v2 = uextend.i32 v3 +; nextln: return v2 +; nextln: } + +function %sextend_8_32() -> i32 { +block0: + v0 = iconst.i32 37 + v1 = ishl_imm v0, 24 + v2 = sshr_imm v1, 24 + return v2 +} +; sameln: function %sextend_8_32 +; nextln: block0: +; nextln: v0 = iconst.i32 37 +; nextln: v1 = ishl_imm v0, 24 +; nextln: v3 = ireduce.i8 v0 +; nextln: v2 = sextend.i32 v3 +; nextln: return v2 +; nextln: } + +;; 16 -> 32 +function %uextend_16_32() -> i32 { +block0: + v0 = iconst.i32 37 + v1 = ishl_imm v0, 16 + v2 = ushr_imm v1, 16 + return v2 +} +; sameln: function %uextend_16_32 +; nextln: block0: +; nextln: v0 = iconst.i32 37 +; nextln: v1 = ishl_imm v0, 16 +; nextln: v3 = ireduce.i16 v0 +; nextln: v2 = uextend.i32 v3 +; nextln: return v2 +; nextln: } + +function %sextend_16_32() -> i32 { +block0: + v0 = iconst.i32 37 + v1 = ishl_imm v0, 16 + v2 = sshr_imm v1, 16 + return v2 +} +; sameln: function %sextend_16_32 +; nextln: block0: +; nextln: v0 = iconst.i32 37 +; nextln: v1 = ishl_imm v0, 16 +; nextln: v3 = ireduce.i16 v0 +; nextln: v2 = sextend.i32 v3 +; nextln: return v2 +; nextln: } + +;; 8 -> 64 +function %uextend_8_64() -> i64 { +block0: + v0 = iconst.i64 37 + v1 = ishl_imm v0, 56 + v2 = ushr_imm v1, 56 + return v2 +} +; sameln: function %uextend_8_64 +; nextln: block0: +; nextln: v0 = iconst.i64 37 +; nextln: v1 = ishl_imm v0, 56 +; nextln: v3 = ireduce.i8 v0 +; nextln: v2 = uextend.i64 v3 +; nextln: return v2 +; nextln: } + +function %sextend_8_64() -> i64 { +block0: + v0 = iconst.i64 37 + v1 = ishl_imm v0, 56 + v2 = sshr_imm v1, 56 + return v2 +} +; sameln: function %sextend_8_64 +; nextln: block0: +; nextln: v0 = iconst.i64 37 +; nextln: v1 = ishl_imm v0, 56 +; nextln: v3 = ireduce.i8 v0 +; nextln: v2 = sextend.i64 v3 +; nextln: return v2 +; nextln: } + +;; 16 -> 64 +function %uextend_16_64() -> i64 { +block0: + v0 = iconst.i64 37 + v1 = ishl_imm v0, 48 + v2 = ushr_imm v1, 48 + return v2 +} +; sameln: function %uextend_16_64 +; nextln: block0: +; nextln: v0 = iconst.i64 37 +; nextln: v1 = ishl_imm v0, 48 +; nextln: v3 = ireduce.i16 v0 +; nextln: v2 = uextend.i64 v3 +; nextln: return v2 +; nextln: } + +function %sextend_16_64() -> i64 { +block0: + v0 = iconst.i64 37 + v1 = ishl_imm v0, 48 + v2 = sshr_imm v1, 48 + return v2 +} +; sameln: function %sextend_16_64 +; nextln: block0: +; nextln: v0 = iconst.i64 37 +; nextln: v1 = ishl_imm v0, 48 +; nextln: v3 = ireduce.i16 v0 +; nextln: v2 = sextend.i64 v3 +; nextln: return v2 +; nextln: } + +;; 32 -> 64 +function %uextend_32_64() -> i64 { +block0: + v0 = iconst.i64 37 + v1 = ishl_imm v0, 32 + v2 = ushr_imm v1, 32 + return v2 +} +; sameln: function %uextend_32_64 +; nextln: block0: +; nextln: v0 = iconst.i64 37 +; nextln: v1 = ishl_imm v0, 32 +; nextln: v3 = ireduce.i32 v0 +; nextln: v2 = uextend.i64 v3 +; nextln: return v2 +; nextln: } + +function %sextend_32_64() -> i64 { +block0: + v0 = iconst.i64 37 + v1 = ishl_imm v0, 32 + v2 = sshr_imm v1, 32 + return v2 +} +; sameln: function %sextend_32_64 +; nextln: block0: +; nextln: v0 = iconst.i64 37 +; nextln: v1 = ishl_imm v0, 32 +; nextln: v3 = ireduce.i32 v0 +; nextln: v2 = sextend.i64 v3 +; nextln: return v2 +; nextln: } + +function %add_imm_fold(i32) -> i32 { +block0(v0: i32): + v1 = iadd_imm v0, 42 + v2 = iadd_imm v1, -42 + return v2 +} +; sameln: function %add_imm_fold(i32) +; nextln: block0(v0: i32): +; nextln: v2 -> v0 +; nextln: v1 = iadd_imm v0, 42 +; nextln: nop +; nextln: return v2 diff --git a/cranelift/filetests/filetests/peepmatic/simplify_instruction_into_alias_of_value.clif b/cranelift/filetests/filetests/peepmatic/simplify_instruction_into_alias_of_value.clif new file mode 100644 index 0000000000..3000369bb5 --- /dev/null +++ b/cranelift/filetests/filetests/peepmatic/simplify_instruction_into_alias_of_value.clif @@ -0,0 +1,17 @@ +test peepmatic +target x86_64 + +;; The `isub` is a no-op, but we can't replace the whole `isub` instruction with +;; its `v2` operand's instruction because `v2` is one of many results. Instead, +;; we need to make an alias `v3 -> v2`. + +function %replace_inst_with_alias() -> i32 { +block0: + v0 = iconst.i32 0 + v1, v2 = x86_smulx v0, v0 + v3 = isub v2, v0 + ; check: v0 = iconst.i32 0 + ; nextln: v1, v2 = x86_smulx v0, v0 + ; nextln: v3 -> v2 + return v3 +} diff --git a/cranelift/filetests/filetests/simple_preopt/div_by_const_indirect.clif b/cranelift/filetests/filetests/simple_preopt/div_by_const_indirect.clif index 4a4b7a80b6..101e4eb201 100644 --- a/cranelift/filetests/filetests/simple_preopt/div_by_const_indirect.clif +++ b/cranelift/filetests/filetests/simple_preopt/div_by_const_indirect.clif @@ -7,13 +7,14 @@ function %indir_udiv32(i32) -> i32 { block0(v0: i32): v1 = iconst.i32 7 v2 = udiv v0, v1 - ; check: v4 = iconst.i32 0x2492_4925 - ; nextln: v5 = umulhi v0, v4 - ; nextln: v6 = isub v0, v5 - ; nextln: v7 = ushr_imm v6, 1 - ; nextln: v8 = iadd v7, v5 - ; nextln: v9 = ushr_imm v8, 2 - ; nextln: v2 -> v9 + ; check: iconst.i32 7 + ; check: iconst.i32 0x2492_4925 + ; check: umulhi v0, v3 + ; check: isub v0, v4 + ; check: ushr_imm v5, 1 + ; check: iadd v6, v4 + ; check: v8 = ushr_imm v7, 2 + ; check: v2 -> v8 return v2 } @@ -21,12 +22,13 @@ function %indir_sdiv32(i32) -> i32 { block0(v0: i32): v1 = iconst.i32 -17 v2 = sdiv v0, v1 - ; check: v4 = iconst.i32 0xffff_ffff_8787_8787 - ; nextln: v5 = smulhi v0, v4 - ; nextln: v6 = sshr_imm v5, 3 - ; nextln: v7 = ushr_imm v6, 31 - ; nextln: v8 = iadd v6, v7 - ; nextln: v2 -> v8 + ; check: iconst.i32 -17 + ; check: iconst.i32 0xffff_ffff_8787_8787 + ; check: smulhi v0, v3 + ; check: sshr_imm v4, 3 + ; check: ushr_imm v5, 31 + ; check: v7 = iadd v5, v6 + ; check: v2 -> v7 return v2 } @@ -34,10 +36,11 @@ function %indir_udiv64(i64) -> i64 { block0(v0: i64): v1 = iconst.i64 1337 v2 = udiv v0, v1 - ; check: v4 = iconst.i64 0xc411_9d95_2866_a139 - ; nextln: v5 = umulhi v0, v4 - ; nextln: v6 = ushr_imm v5, 10 - ; nextln: v2 -> v6 + ; check: iconst.i64 1337 + ; check: iconst.i64 0xc411_9d95_2866_a139 + ; check: umulhi v0, v3 + ; check: v5 = ushr_imm v4, 10 + ; check: v2 -> v5 return v2 } @@ -45,11 +48,12 @@ function %indir_sdiv64(i64) -> i64 { block0(v0: i64): v1 = iconst.i64 -90210 v2 = sdiv v0, v1 - ; check: v4 = iconst.i64 0xd181_4ee8_939c_b8bb - ; nextln: v5 = smulhi v0, v4 - ; nextln: v6 = sshr_imm v5, 14 - ; nextln: v7 = ushr_imm v6, 63 - ; nextln: v8 = iadd v6, v7 - ; nextln: v2 -> v8 + ; check: iconst.i64 0xffff_ffff_fffe_9f9e + ; check: iconst.i64 0xd181_4ee8_939c_b8bb + ; check: smulhi v0, v3 + ; check: sshr_imm v4, 14 + ; check: ushr_imm v5, 63 + ; check: v7 = iadd v5, v6 + ; check: v2 -> v7 return v2 } diff --git a/cranelift/filetests/filetests/simple_preopt/simplify32.clif b/cranelift/filetests/filetests/simple_preopt/simplify32.clif index cf238fb5ed..2582fd69aa 100644 --- a/cranelift/filetests/filetests/simple_preopt/simplify32.clif +++ b/cranelift/filetests/filetests/simple_preopt/simplify32.clif @@ -58,3 +58,4 @@ block0(v0: i64): ; nextln: v2 = iadd v0, v1 ; nextln: return v2 ; nextln: } + diff --git a/cranelift/filetests/filetests/simple_preopt/simplify64.clif b/cranelift/filetests/filetests/simple_preopt/simplify64.clif index 6489c3bd1e..4ceabdc335 100644 --- a/cranelift/filetests/filetests/simple_preopt/simplify64.clif +++ b/cranelift/filetests/filetests/simple_preopt/simplify64.clif @@ -44,37 +44,6 @@ block0(v0: i32): ; nextln: return v3 ; nextln: } -function %ifcmp_imm(i32) -> i32 { -block0(v0: i32): - v1 = iconst.i32 2 - v2 = ifcmp v0, v1 - brif eq v2, block1 - jump block2 - -block1: - v3 = iconst.i32 1 - return v3 - -block2: - v4 = iconst.i32 2 - return v4 -} -; sameln: function %ifcmp_imm -; nextln: block0(v0: i32): -; nextln: v1 = iconst.i32 2 -; nextln: v2 = ifcmp_imm v0, 2 -; nextln: brif eq v2, block1 -; nextln: jump block2 -; nextln: -; nextln: block1: -; nextln: v3 = iconst.i32 1 -; nextln: return v3 -; nextln: -; nextln: block2: -; nextln: v4 = iconst.i32 2 -; nextln: return v4 -; nextln: } - function %brz_bint(i32) { block0(v0: i32): v3 = icmp_imm slt v0, 0 diff --git a/cranelift/filetests/src/lib.rs b/cranelift/filetests/src/lib.rs index bc1a6df1e2..5cf7331225 100644 --- a/cranelift/filetests/src/lib.rs +++ b/cranelift/filetests/src/lib.rs @@ -45,6 +45,7 @@ mod test_domtree; mod test_interpret; mod test_legalizer; mod test_licm; +mod test_peepmatic; mod test_postopt; mod test_preopt; mod test_print_cfg; @@ -128,6 +129,7 @@ fn new_subtest(parsed: &TestCommand) -> subtest::SubtestResult test_interpret::subtest(parsed), "legalizer" => test_legalizer::subtest(parsed), "licm" => test_licm::subtest(parsed), + "peepmatic" => test_peepmatic::subtest(parsed), "postopt" => test_postopt::subtest(parsed), "preopt" => test_preopt::subtest(parsed), "print-cfg" => test_print_cfg::subtest(parsed), diff --git a/cranelift/filetests/src/test_peepmatic.rs b/cranelift/filetests/src/test_peepmatic.rs new file mode 100644 index 0000000000..fc701c7046 --- /dev/null +++ b/cranelift/filetests/src/test_peepmatic.rs @@ -0,0 +1,56 @@ +//! Test command for `peepmatic`-generated peephole optimizers. + +use crate::subtest::{run_filecheck, Context, SubTest, SubtestResult}; +use cranelift_codegen; +use cranelift_codegen::ir::Function; +use cranelift_codegen::print_errors::pretty_error; +use cranelift_reader::TestCommand; +use std::borrow::Cow; + +struct TestPreopt; + +pub fn subtest(parsed: &TestCommand) -> SubtestResult> { + assert_eq!(parsed.command, "peepmatic"); + if parsed.options.is_empty() { + Ok(Box::new(TestPreopt)) + } else { + Err(format!("No options allowed on {}", parsed)) + } +} + +impl SubTest for TestPreopt { + fn name(&self) -> &'static str { + "peepmatic" + } + + fn is_mutating(&self) -> bool { + true + } + + fn needs_isa(&self) -> bool { + true + } + + fn run(&self, func: Cow, context: &Context) -> SubtestResult<()> { + let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned()); + let isa = context.isa.expect("preopt needs an ISA"); + + comp_ctx.compute_cfg(); + comp_ctx + .preopt(isa) + .map_err(|e| pretty_error(&comp_ctx.func, context.isa, Into::into(e)))?; + let text = &comp_ctx.func.display(isa).to_string(); + log::debug!("After peepmatic-based simple_preopt:\n{}", text); + + // Only actually run the filecheck if peepmatic is enabled, because it + // can generate slightly different code (alias a result vs replace an + // instruction) than the non-peepmatic versions of peephole + // optimizations. Note that the non-`peepmatic` results can be tested + // with the `test simple_preopt` subtest. + if cfg!(feature = "enable-peepmatic") { + run_filecheck(&text, context) + } else { + Ok(()) + } + } +} diff --git a/cranelift/filetests/src/test_simple_preopt.rs b/cranelift/filetests/src/test_simple_preopt.rs index 1463d1c69a..f6cdec391f 100644 --- a/cranelift/filetests/src/test_simple_preopt.rs +++ b/cranelift/filetests/src/test_simple_preopt.rs @@ -39,6 +39,16 @@ impl SubTest for TestSimplePreopt { .map_err(|e| pretty_error(&comp_ctx.func, context.isa, Into::into(e)))?; let text = &comp_ctx.func.display(isa).to_string(); log::debug!("After simple_preopt:\n{}", text); - run_filecheck(&text, context) + + // Only actually run the filecheck if peepmatic is *not* enabled, + // because it can generate slightly different code (alias a result vs + // replace an instruction) than the non-peepmatic versions of peephole + // optimizations. Note that the `peepmatic`-based results can be tested + // with the `test peepmatic` subtest. + if cfg!(feature = "enable-peepmatic") { + Ok(()) + } else { + run_filecheck(&text, context) + } } } From fd4f08e75f6dd68a629fd4ac0d6375b6a0930446 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 8 May 2020 12:24:53 -0700 Subject: [PATCH 17/22] peepmatic: rustfmt --- .../crates/automata/src/serde_impls.rs | 35 ++++++++++++++++--- cranelift/peepmatic/crates/macro/src/span.rs | 4 ++- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/cranelift/peepmatic/crates/automata/src/serde_impls.rs b/cranelift/peepmatic/crates/automata/src/serde_impls.rs index f7543626e5..8a5d79214f 100644 --- a/cranelift/peepmatic/crates/automata/src/serde_impls.rs +++ b/cranelift/peepmatic/crates/automata/src/serde_impls.rs @@ -155,27 +155,52 @@ where &self, )); } - None => return Err(de::Error::invalid_length(0, &"Automaton expects 5 elements")), + None => { + return Err(de::Error::invalid_length( + 0, + &"Automaton expects 5 elements", + )) + } } let final_states = match seq.next_element::>()? { Some(x) => x, - None => return Err(de::Error::invalid_length(1, &"Automaton expects 5 elements")), + None => { + return Err(de::Error::invalid_length( + 1, + &"Automaton expects 5 elements", + )) + } }; let start_state = match seq.next_element::()? { Some(x) => x, - None => return Err(de::Error::invalid_length(2, &"Automaton expects 5 elements")), + None => { + return Err(de::Error::invalid_length( + 2, + &"Automaton expects 5 elements", + )) + } }; let state_data = match seq.next_element::>>()? { Some(x) => x, - None => return Err(de::Error::invalid_length(3, &"Automaton expects 5 elements")), + None => { + return Err(de::Error::invalid_length( + 3, + &"Automaton expects 5 elements", + )) + } }; let transitions = match seq.next_element::>>()? { Some(x) => x, - None => return Err(de::Error::invalid_length(4, &"Automaton expects 5 elements")), + None => { + return Err(de::Error::invalid_length( + 4, + &"Automaton expects 5 elements", + )) + } }; let automata = Automaton { diff --git a/cranelift/peepmatic/crates/macro/src/span.rs b/cranelift/peepmatic/crates/macro/src/span.rs index f994920dff..bdc0a63655 100644 --- a/cranelift/peepmatic/crates/macro/src/span.rs +++ b/cranelift/peepmatic/crates/macro/src/span.rs @@ -23,7 +23,9 @@ pub fn derive_span(input: &DeriveInput) -> Result { } } } - syn::Data::Union(_) => panic!("derive(Ast) can only be used with structs and enums, not unions"), + syn::Data::Union(_) => { + panic!("derive(Ast) can only be used with structs and enums, not unions") + } }; let generics = add_span_trait_bounds(input.generics.clone()); From a9b280ca3a4b700bff6679bfaedfbf70116d30a4 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 8 May 2020 12:25:17 -0700 Subject: [PATCH 18/22] CI: Ensure that the built `peepmatic` peephole optimizers are up to date Beyond just ensuring that they can still be built, ensure that rebuilding them doesn't result in a different built artifact. --- .github/workflows/main.yml | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1edd76aeeb..8a0686df8d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -207,12 +207,16 @@ jobs: - uses: actions/checkout@v1 with: submodules: true - - name: Build + - name: Rebuild peepmatic peephole optimizers run: | - cd cranelift && cargo build --features 'cranelift-codegen/rebuild-peephole-optimizers' - - name: Test + cd cranelift/ + cargo build --features 'enable-peepmatic cranelift-codegen/rebuild-peephole-optimizers' + - name: Check that peephole optimizers are up to date + run: git diff --exit-code + - name: Test with peepmatic run: | - cd cranelift && cargo test --features 'cranelift-codegen/rebuild-peephole-optimizers' + cd cranelift/ + cargo test --features 'enable-peepmatic' # Perform all tests (debug mode) for `wasmtime`. This runs stable/beta/nightly # channels of Rust as well as macOS/Linux/Windows. @@ -383,11 +387,29 @@ jobs: # Build `wasmtime` and executables - run: $CENTOS cargo build --release --bin wasmtime shell: bash + # Build `libwasmtime.so` - run: $CENTOS cargo build --release --manifest-path crates/c-api/Cargo.toml shell: bash - # Test what we just built - - run: $CENTOS cargo test --features test-programs/test_programs --release --all --exclude lightbeam + + # Test what we just built. + # + # Ignore some optional dependencies that are used by features that aren't + # enabled by default, are tested in other, dedicated CI jobs, and which + # would increase build times here. + - run: | + $CENTOS cargo test \ + --features test-programs/test_programs \ + --release \ + --all \ + --exclude lightbeam \ + --exclude peepmatic \ + --exclude peepmatic-automata \ + --exclude peepmatic-fuzzing \ + --exclude peepmatic-macro \ + --exclude peepmatic-runtime \ + --exclude peepmatic-test \ + --exclude wasmtime-fuzz shell: bash env: RUST_BACKTRACE: 1 From 22a070ed4f2b893ee21f01552bd7affe380fecf1 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 8 May 2020 13:47:14 -0700 Subject: [PATCH 19/22] peepmatic: Apply some review suggestions from @bjorn3 --- cranelift/codegen/src/peepmatic.rs | 4 +- .../peepmatic/crates/runtime/src/linear.rs | 30 +- .../peepmatic/crates/runtime/src/paths.rs | 14 +- cranelift/peepmatic/src/linearize.rs | 352 ++++++++---------- 4 files changed, 181 insertions(+), 219 deletions(-) diff --git a/cranelift/codegen/src/peepmatic.rs b/cranelift/codegen/src/peepmatic.rs index ba9aeb38d5..1ea2031bfa 100644 --- a/cranelift/codegen/src/peepmatic.rs +++ b/cranelift/codegen/src/peepmatic.rs @@ -196,7 +196,7 @@ fn const_to_value<'a>(builder: impl InstBuilder<'a>, c: Constant, root: Inst) -> Constant::Int(x, width) => { let width = bit_width(builder.data_flow_graph(), width, root); let ty = match width { - 1 | 8 => types::I8, + 8 => types::I8, 16 => types::I16, 32 => types::I32, 64 => types::I64, @@ -407,7 +407,7 @@ fn get_argument(dfg: &DataFlowGraph, inst: Inst, i: usize) -> Option { fn peepmatic_ty_to_ir_ty(ty: Type, dfg: &DataFlowGraph, root: Inst) -> types::Type { match (ty.kind, bit_width(dfg, ty.bit_width, root)) { - (Kind::Int, 1) | (Kind::Int, 8) => types::I8, + (Kind::Int, 8) => types::I8, (Kind::Int, 16) => types::I16, (Kind::Int, 32) => types::I32, (Kind::Int, 64) => types::I64, diff --git a/cranelift/peepmatic/crates/runtime/src/linear.rs b/cranelift/peepmatic/crates/runtime/src/linear.rs index a51159f708..f156311740 100644 --- a/cranelift/peepmatic/crates/runtime/src/linear.rs +++ b/cranelift/peepmatic/crates/runtime/src/linear.rs @@ -53,6 +53,9 @@ pub fn bool_to_match_result(b: bool) -> MatchResult { unsafe { Ok(NonZeroU32::new_unchecked(b + 1)) } } +/// A partial match of an optimization's LHS and partial construction of its +/// RHS. +/// /// An increment is a matching operation, the expected result from that /// operation to continue to the next increment, and the actions to take to /// build up the LHS scope and RHS instructions given that we got the expected @@ -156,17 +159,18 @@ pub struct LhsId(pub u16); pub struct RhsId(pub u16); /// An action to perform when transitioning between states in the automata. +/// +/// When evaluating actions, the `i^th` action implicitly defines the +/// `RhsId(i)`. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum Action { - /// Implicitly define the n^th built up RHS instruction as something from - /// the left-hand side. + /// Reuse something from the left-hand side. GetLhs { /// The path to the instruction or value. path: PathId, }, - /// Implicitly define the n^th RHS instruction as the result of the - /// compile-time evaluation off this unquote operation. + /// Perform compile-time evaluation. UnaryUnquote { /// The unquote operator. operator: UnquoteOperator, @@ -174,8 +178,7 @@ pub enum Action { operand: RhsId, }, - /// Implicitly define the n^th RHS instruction as the result of the - /// compile-time evaluation off this unquote operation. + /// Perform compile-time evaluation. BinaryUnquote { /// The unquote operator. operator: UnquoteOperator, @@ -183,7 +186,7 @@ pub enum Action { operands: [RhsId; 2], }, - /// Implicitly define the n^th RHS as an integer constant. + /// Create an integer constant. MakeIntegerConst { /// The constant integer value. value: IntegerId, @@ -191,7 +194,7 @@ pub enum Action { bit_width: BitWidth, }, - /// Implicitly define the n^th RHS as a boolean constant. + /// Create a boolean constant. MakeBooleanConst { /// The constant boolean value. value: bool, @@ -199,14 +202,13 @@ pub enum Action { bit_width: BitWidth, }, - /// Implicitly defint the n^th RHS as a condition code. + /// Create a condition code. MakeConditionCode { /// The condition code. cc: ConditionCode, }, - /// Implicitly define the n^th RHS instruction by making a unary - /// instruction. + /// Make a unary instruction. MakeUnaryInst { /// The operand for this instruction. operand: RhsId, @@ -216,8 +218,7 @@ pub enum Action { operator: Operator, }, - /// Implicitly define the n^th RHS instruction by making a binary - /// instruction. + /// Make a binary instruction. MakeBinaryInst { /// The opcode for this instruction. operator: Operator, @@ -227,8 +228,7 @@ pub enum Action { operands: [RhsId; 2], }, - /// Implicitly define the n^th RHS instruction by making a ternary - /// instruction. + /// Make a ternary instruction. MakeTernaryInst { /// The opcode for this instruction. operator: Operator, diff --git a/cranelift/peepmatic/crates/runtime/src/paths.rs b/cranelift/peepmatic/crates/runtime/src/paths.rs index d826ba76bb..f824f19a74 100644 --- a/cranelift/peepmatic/crates/runtime/src/paths.rs +++ b/cranelift/peepmatic/crates/runtime/src/paths.rs @@ -54,17 +54,17 @@ pub struct PathId(u16); /// path mapping. #[derive(Debug, Default)] pub struct PathInterner { - // A map from a path (whose owned data is inside `arena`) to the canonical - // `PathId` we assigned it when interning it. + /// A map from a path (whose owned data is inside `arena`) to the canonical + /// `PathId` we assigned it when interning it. map: HashMap, - // A map from a `PathId` index to an unsafe, self-borrowed path pointing - // into `arena`. It is safe to given these out as safe `Path`s, as long as - // the lifetime is not longer than this `PathInterner`'s lifetime. + /// A map from a `PathId` index to an unsafe, self-borrowed path pointing + /// into `arena`. It is safe to given these out as safe `Path`s, as long as + /// the lifetime is not longer than this `PathInterner`'s lifetime. paths: Vec, - // Bump allocation arena for path data. The bump arena ensures that these - // allocations never move, and are therefore safe for self-references. + /// Bump allocation arena for path data. The bump arena ensures that these + /// allocations never move, and are therefore safe for self-references. arena: bumpalo::Bump, } diff --git a/cranelift/peepmatic/src/linearize.rs b/cranelift/peepmatic/src/linearize.rs index eb03853fa2..9f7c1fe602 100644 --- a/cranelift/peepmatic/src/linearize.rs +++ b/cranelift/peepmatic/src/linearize.rs @@ -599,13 +599,17 @@ mod tests { let mut i = |i: u64| integers.intern(i); #[allow(unused_variables)] - let expected = $make_expected(&mut p, &mut i); + let make_expected: fn( + &mut dyn FnMut(&[u8]) -> PathId, + &mut dyn FnMut(u64) -> IntegerId, + ) -> Vec = $make_expected; + let expected = make_expected(&mut p, &mut i); dbg!(&expected); let actual = linearize_optimization(&mut paths, &mut integers, &opts.optimizations[0]); - dbg!(&actual); + dbg!(&actual.increments); - assert_eq!(expected, actual); + assert_eq!(expected, actual.increments); } }; } @@ -617,234 +621,192 @@ mod tests { (is-power-of-two $C)) (ishl $x $C)) ", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { - linear::Optimization { - increments: vec![ - linear::Increment { - operation: Opcode { path: p(&[0]) }, - expected: Ok(NonZeroU32::new(Operator::Imul as _).unwrap()), - actions: vec![ - GetLhs { path: p(&[0, 0]) }, - GetLhs { path: p(&[0, 1]) }, - MakeBinaryInst { - operator: Operator::Ishl, - r#type: Type { - kind: Kind::Int, - bit_width: BitWidth::Polymorphic, - }, - operands: [linear::RhsId(0), linear::RhsId(1)], - }, - ], - }, - linear::Increment { - operation: Nop, - expected: Err(Else), - actions: vec![], - }, - linear::Increment { - operation: IsConst { path: p(&[0, 1]) }, - expected: bool_to_match_result(true), - actions: vec![], - }, - linear::Increment { - operation: IsPowerOfTwo { path: p(&[0, 1]) }, - expected: bool_to_match_result(true), - actions: vec![], + |p, i| vec![ + linear::Increment { + operation: Opcode { path: p(&[0]) }, + expected: Ok(NonZeroU32::new(Operator::Imul as _).unwrap()), + actions: vec![ + GetLhs { path: p(&[0, 0]) }, + GetLhs { path: p(&[0, 1]) }, + MakeBinaryInst { + operator: Operator::Ishl, + r#type: Type { + kind: Kind::Int, + bit_width: BitWidth::Polymorphic, + }, + operands: [linear::RhsId(0), linear::RhsId(1)], }, ], - } - }, + }, + linear::Increment { + operation: Nop, + expected: Err(Else), + actions: vec![], + }, + linear::Increment { + operation: IsConst { path: p(&[0, 1]) }, + expected: bool_to_match_result(true), + actions: vec![], + }, + linear::Increment { + operation: IsPowerOfTwo { path: p(&[0, 1]) }, + expected: bool_to_match_result(true), + actions: vec![], + }, + ], ); - linearizes_to!( - variable_pattern_id_optimization, - "(=> $x $x)", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { - linear::Optimization { - increments: vec![linear::Increment { - operation: Nop, - expected: Err(Else), - actions: vec![GetLhs { path: p(&[0]) }], - }], - } - }, - ); + linearizes_to!(variable_pattern_id_optimization, "(=> $x $x)", |p, i| vec![ + linear::Increment { + operation: Nop, + expected: Err(Else), + actions: vec![GetLhs { path: p(&[0]) }], + } + ]); - linearizes_to!( - constant_pattern_id_optimization, - "(=> $C $C)", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { - linear::Optimization { - increments: vec![linear::Increment { - operation: IsConst { path: p(&[0]) }, - expected: bool_to_match_result(true), - actions: vec![GetLhs { path: p(&[0]) }], - }], - } - }, - ); + linearizes_to!(constant_pattern_id_optimization, "(=> $C $C)", |p, i| vec![ + linear::Increment { + operation: IsConst { path: p(&[0]) }, + expected: bool_to_match_result(true), + actions: vec![GetLhs { path: p(&[0]) }], + } + ]); linearizes_to!( boolean_literal_id_optimization, "(=> true true)", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { - linear::Optimization { - increments: vec![linear::Increment { - operation: BooleanValue { path: p(&[0]) }, - expected: bool_to_match_result(true), - actions: vec![MakeBooleanConst { - value: true, - bit_width: BitWidth::Polymorphic, - }], - }], - } - }, + |p, i| vec![linear::Increment { + operation: BooleanValue { path: p(&[0]) }, + expected: bool_to_match_result(true), + actions: vec![MakeBooleanConst { + value: true, + bit_width: BitWidth::Polymorphic, + }], + }] ); - linearizes_to!( - number_literal_id_optimization, - "(=> 5 5)", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { - linear::Optimization { - increments: vec![linear::Increment { - operation: IntegerValue { path: p(&[0]) }, - expected: Ok(i(5).into()), - actions: vec![MakeIntegerConst { - value: i(5), - bit_width: BitWidth::Polymorphic, - }], - }], - } - }, - ); + linearizes_to!(number_literal_id_optimization, "(=> 5 5)", |p, i| vec![ + linear::Increment { + operation: IntegerValue { path: p(&[0]) }, + expected: Ok(i(5).into()), + actions: vec![MakeIntegerConst { + value: i(5), + bit_width: BitWidth::Polymorphic, + }], + } + ]); linearizes_to!( operation_id_optimization, "(=> (iconst $C) (iconst $C))", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { - linear::Optimization { - increments: vec![ - linear::Increment { - operation: Opcode { path: p(&[0]) }, - expected: Ok(NonZeroU32::new(Operator::Iconst as _).unwrap()), - actions: vec![ - GetLhs { path: p(&[0, 0]) }, - MakeUnaryInst { - operator: Operator::Iconst, - r#type: Type { - kind: Kind::Int, - bit_width: BitWidth::Polymorphic, - }, - operand: linear::RhsId(0), - }, - ], - }, - linear::Increment { - operation: IsConst { path: p(&[0, 0]) }, - expected: bool_to_match_result(true), - actions: vec![], + |p, i| vec![ + linear::Increment { + operation: Opcode { path: p(&[0]) }, + expected: Ok(NonZeroU32::new(Operator::Iconst as _).unwrap()), + actions: vec![ + GetLhs { path: p(&[0, 0]) }, + MakeUnaryInst { + operator: Operator::Iconst, + r#type: Type { + kind: Kind::Int, + bit_width: BitWidth::Polymorphic, + }, + operand: linear::RhsId(0), }, ], - } - }, + }, + linear::Increment { + operation: IsConst { path: p(&[0, 0]) }, + expected: bool_to_match_result(true), + actions: vec![], + }, + ] ); linearizes_to!( redundant_bor, "(=> (bor $x (bor $x $y)) (bor $x $y))", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { - linear::Optimization { - increments: vec![ - linear::Increment { - operation: Opcode { path: p(&[0]) }, - expected: Ok(NonZeroU32::new(Operator::Bor as _).unwrap()), - actions: vec![ - GetLhs { path: p(&[0, 0]) }, - GetLhs { - path: p(&[0, 1, 1]), - }, - MakeBinaryInst { - operator: Operator::Bor, - r#type: Type { - kind: Kind::Int, - bit_width: BitWidth::Polymorphic, - }, - operands: [linear::RhsId(0), linear::RhsId(1)], - }, - ], + |p, i| vec![ + linear::Increment { + operation: Opcode { path: p(&[0]) }, + expected: Ok(NonZeroU32::new(Operator::Bor as _).unwrap()), + actions: vec![ + GetLhs { path: p(&[0, 0]) }, + GetLhs { + path: p(&[0, 1, 1]), }, - linear::Increment { - operation: Nop, - expected: Err(Else), - actions: vec![], - }, - linear::Increment { - operation: Opcode { path: p(&[0, 1]) }, - expected: Ok(NonZeroU32::new(Operator::Bor as _).unwrap()), - actions: vec![], - }, - linear::Increment { - operation: Eq { - path_a: p(&[0, 1, 0]), - path_b: p(&[0, 0]), + MakeBinaryInst { + operator: Operator::Bor, + r#type: Type { + kind: Kind::Int, + bit_width: BitWidth::Polymorphic, }, - expected: bool_to_match_result(true), - actions: vec![], - }, - linear::Increment { - operation: Nop, - expected: Err(Else), - actions: vec![], + operands: [linear::RhsId(0), linear::RhsId(1)], }, ], - } - }, + }, + linear::Increment { + operation: Nop, + expected: Err(Else), + actions: vec![], + }, + linear::Increment { + operation: Opcode { path: p(&[0, 1]) }, + expected: Ok(NonZeroU32::new(Operator::Bor as _).unwrap()), + actions: vec![], + }, + linear::Increment { + operation: Eq { + path_a: p(&[0, 1, 0]), + path_b: p(&[0, 0]), + }, + expected: bool_to_match_result(true), + actions: vec![], + }, + linear::Increment { + operation: Nop, + expected: Err(Else), + actions: vec![], + }, + ] ); linearizes_to!( large_integers, // u64::MAX "(=> 18446744073709551615 0)", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { - linear::Optimization { - increments: vec![linear::Increment { - operation: IntegerValue { path: p(&[0]) }, - expected: Ok(i(std::u64::MAX).into()), - actions: vec![MakeIntegerConst { - value: i(0), - bit_width: BitWidth::Polymorphic, - }], - }], - } - } + |p, i| vec![linear::Increment { + operation: IntegerValue { path: p(&[0]) }, + expected: Ok(i(std::u64::MAX).into()), + actions: vec![MakeIntegerConst { + value: i(0), + bit_width: BitWidth::Polymorphic, + }], + }] ); linearizes_to!( ireduce_with_type_ascription, "(=> (ireduce{i32} $x) 0)", - |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> IntegerId| { - linear::Optimization { - increments: vec![ - linear::Increment { - operation: Opcode { path: p(&[0]) }, - expected: Ok(NonZeroU32::new(Operator::Ireduce as _).unwrap()), - actions: vec![MakeIntegerConst { - value: i(0), - bit_width: BitWidth::ThirtyTwo, - }], - }, - linear::Increment { - operation: linear::MatchOp::BitWidth { path: p(&[0]) }, - expected: Ok(NonZeroU32::new(32).unwrap()), - actions: vec![], - }, - linear::Increment { - operation: Nop, - expected: Err(Else), - actions: vec![], - }, - ], - } - } + |p, i| vec![ + linear::Increment { + operation: Opcode { path: p(&[0]) }, + expected: Ok(NonZeroU32::new(Operator::Ireduce as _).unwrap()), + actions: vec![MakeIntegerConst { + value: i(0), + bit_width: BitWidth::ThirtyTwo, + }], + }, + linear::Increment { + operation: linear::MatchOp::BitWidth { path: p(&[0]) }, + expected: Ok(NonZeroU32::new(32).unwrap()), + actions: vec![], + }, + linear::Increment { + operation: Nop, + expected: Err(Else), + actions: vec![], + }, + ] ); } From 8d7ed0fd139a8905efd94d891cb087d77222834d Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 8 May 2020 15:12:39 -0700 Subject: [PATCH 20/22] deps: Update `wast` to 15.0.0 This also updates `wat` in the lockfile so that the SIMD spec tests are passing again. --- Cargo.lock | 21 ++++++------------- cranelift/peepmatic/Cargo.toml | 2 +- cranelift/peepmatic/crates/fuzzing/Cargo.toml | 2 +- cranelift/peepmatic/crates/runtime/Cargo.toml | 2 +- 4 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99373b1fae..f9509a2572 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1262,7 +1262,7 @@ dependencies = [ "peepmatic-automata", "peepmatic-macro", "peepmatic-runtime", - "wast 13.0.0", + "wast 15.0.0", "z3", ] @@ -1288,7 +1288,7 @@ dependencies = [ "peepmatic-test", "rand 0.7.3", "serde", - "wast 13.0.0", + "wast 15.0.0", ] [[package]] @@ -1311,7 +1311,7 @@ dependencies = [ "peepmatic-macro", "serde", "thiserror", - "wast 13.0.0", + "wast 15.0.0", ] [[package]] @@ -2453,15 +2453,6 @@ dependencies = [ "leb128", ] -[[package]] -name = "wast" -version = "13.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b20abd8b4a26f7e0d4dd5e357e90a3d555ec190e94472c9b2b27c5b9777f9ae" -dependencies = [ - "leb128", -] - [[package]] name = "wast" version = "15.0.0" @@ -2473,11 +2464,11 @@ dependencies = [ [[package]] name = "wat" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51a615830ee3e7200b505c441fec09aac2f114deae69df52f215cb828ba112c4" +checksum = "526d28df6c047d9f9a92d4925b98afd8d8d95b1b3aa4f13eb1306f17d1da56c4" dependencies = [ - "wast 13.0.0", + "wast 15.0.0", ] [[package]] diff --git a/cranelift/peepmatic/Cargo.toml b/cranelift/peepmatic/Cargo.toml index 843b06a1a2..7a1dc1c535 100644 --- a/cranelift/peepmatic/Cargo.toml +++ b/cranelift/peepmatic/Cargo.toml @@ -11,5 +11,5 @@ anyhow = "1.0.27" peepmatic-automata = { version = "0.1.0", path = "crates/automata", features = ["dot"] } peepmatic-macro = { version = "0.1.0", path = "crates/macro" } peepmatic-runtime = { version = "0.1.0", path = "crates/runtime", features = ["construct"] } -wast = "13.0.0" +wast = "15.0.0" z3 = { version = "0.5.0", features = ["static-link-z3"] } diff --git a/cranelift/peepmatic/crates/fuzzing/Cargo.toml b/cranelift/peepmatic/crates/fuzzing/Cargo.toml index 5e36dcc6af..a3ccb91791 100644 --- a/cranelift/peepmatic/crates/fuzzing/Cargo.toml +++ b/cranelift/peepmatic/crates/fuzzing/Cargo.toml @@ -19,4 +19,4 @@ peepmatic-runtime = { path = "../runtime", features = ["construct"] } peepmatic-test = { path = "../test" } rand = { version = "0.7.3", features = ["small_rng"] } serde = "1.0.106" -wast = "13.0.0" +wast = "15.0.0" diff --git a/cranelift/peepmatic/crates/runtime/Cargo.toml b/cranelift/peepmatic/crates/runtime/Cargo.toml index 31c4655b25..0a5312ed86 100644 --- a/cranelift/peepmatic/crates/runtime/Cargo.toml +++ b/cranelift/peepmatic/crates/runtime/Cargo.toml @@ -14,7 +14,7 @@ peepmatic-automata = { version = "0.1.0", path = "../automata", features = ["ser peepmatic-macro = { version = "0.1.0", path = "../macro" } serde = { version = "1.0.105", features = ["derive"] } thiserror = "1.0.15" -wast = { version = "13.0.0", optional = true } +wast = { version = "15.0.0", optional = true } [features] From 923a73be7b6f9107cb0258fb53f731aaf44dd1f6 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 12 May 2020 12:22:40 -0700 Subject: [PATCH 21/22] deps: Bump `z3` to 0.5.1 This fixes Windows builds. --- Cargo.lock | 8 ++++---- cranelift/peepmatic/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9509a2572..0a9e38fd62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2589,9 +2589,9 @@ dependencies = [ [[package]] name = "z3" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afd636f549e919f8058567000156e19efbb4728ddb3be226dcc332d0ff622ab5" +checksum = "ded00cd90f8e3a7ea3155bddd72573f2b099ea201877542d924e47b58dd04e72" dependencies = [ "lazy_static", "log", @@ -2600,9 +2600,9 @@ dependencies = [ [[package]] name = "z3-sys" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2487974559d1494e8e8925df85362d6109ae5555092510508a5b1077346c2833" +checksum = "4002d8a1facb54d02dbfb86151281e5450618ab330936bc2f3acaab31eae11ae" dependencies = [ "cmake", ] diff --git a/cranelift/peepmatic/Cargo.toml b/cranelift/peepmatic/Cargo.toml index 7a1dc1c535..abe9676f0b 100644 --- a/cranelift/peepmatic/Cargo.toml +++ b/cranelift/peepmatic/Cargo.toml @@ -12,4 +12,4 @@ peepmatic-automata = { version = "0.1.0", path = "crates/automata", features = [ peepmatic-macro = { version = "0.1.0", path = "crates/macro" } peepmatic-runtime = { version = "0.1.0", path = "crates/runtime", features = ["construct"] } wast = "15.0.0" -z3 = { version = "0.5.0", features = ["static-link-z3"] } +z3 = { version = "0.5.1", features = ["static-link-z3"] } From c093dee79e1f2f035e7f1251d565a1cd2fd03ad0 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Thu, 14 May 2020 07:49:56 -0700 Subject: [PATCH 22/22] cranelift: Let lifetime elision elide lifetimes --- cranelift/codegen/src/simple_preopt.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cranelift/codegen/src/simple_preopt.rs b/cranelift/codegen/src/simple_preopt.rs index 7413b01e90..2f47c7d91b 100644 --- a/cranelift/codegen/src/simple_preopt.rs +++ b/cranelift/codegen/src/simple_preopt.rs @@ -1019,11 +1019,7 @@ mod simplify { } /// The main pre-opt pass. -pub fn do_preopt<'func, 'isa>( - func: &'func mut Function, - cfg: &mut ControlFlowGraph, - isa: &'isa dyn TargetIsa, -) { +pub fn do_preopt(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &dyn TargetIsa) { let _tt = timing::preopt(); let mut pos = FuncCursor::new(func);