Merge pull request #1647 from fitzgen/integrate-peepmatic

Introduce peepmatic: a peephole optimizations DSL and peephole optimizer compiler

.github/workflows/main.yml

@@ -164,6 +164,59 @@ jobs:
             | xargs cargo fuzz run differential --release --debug-assertions --features binaryen
       env:
         RUST_BACKTRACE: 1
+    - run: |
+        find fuzz/corpus/peepmatic_compile -type f \
+            | shuf \
+            | head -n 10000 \
+            | xargs cargo fuzz run peepmatic_compile --release --debug-assertions --features binaryen
+      env:
+        RUST_BACKTRACE: 1
+    - run: |
+        find fuzz/corpus/peepmatic_fst_differential -type f \
+            | shuf \
+            | head -n 10000 \
+            | xargs cargo fuzz run peepmatic_fst_differential --release --debug-assertions --features binaryen
+      env:
+        RUST_BACKTRACE: 1
+    - run: |
+        find fuzz/corpus/peepmatic_interp -type f \
+            | shuf \
+            | head -n 5000 \
+            | xargs cargo fuzz run peepmatic_interp --release --debug-assertions --features binaryen
+      env:
+        RUST_BACKTRACE: 1
+    - run: |
+        find fuzz/corpus/peepmatic_parser -type f \
+            | shuf \
+            | head -n 10000 \
+            | xargs cargo fuzz run peepmatic_parser --release --debug-assertions --features binaryen
+      env:
+        RUST_BACKTRACE: 1
+    - run: |
+        find fuzz/corpus/peepmatic_simple_automata -type f \
+            | shuf \
+            | head -n 1000 \
+            | xargs cargo fuzz run peepmatic_simple_automata --release --debug-assertions --features binaryen
+      env:
+        RUST_BACKTRACE: 1
+
+  rebuild_peephole_optimizers:
+    name: Rebuild Peephole Optimizers
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: true
+    - name: Rebuild peepmatic peephole optimizers
+      run: |
+        cd cranelift/
+        cargo build --features 'enable-peepmatic cranelift-codegen/rebuild-peephole-optimizers'
+    - name: Check that peephole optimizers are up to date
+      run: git diff --exit-code
+    - name: Test with peepmatic
+      run: |
+        cd cranelift/
+        cargo test --features 'enable-peepmatic'
 
   # Perform all tests (debug mode) for `wasmtime`. This runs stable/beta/nightly
   # channels of Rust as well as macOS/Linux/Windows.
@@ -334,11 +387,29 @@ jobs:
     # Build `wasmtime` and executables
     - run: $CENTOS cargo build --release --bin wasmtime
       shell: bash
+
     # Build `libwasmtime.so`
     - run: $CENTOS cargo build --release --manifest-path crates/c-api/Cargo.toml
       shell: bash
-    # Test what we just built
+
-    - run: $CENTOS cargo test --features test-programs/test_programs --release --all --exclude lightbeam
+    # Test what we just built.
+    #
+    # Ignore some optional dependencies that are used by features that aren't
+    # enabled by default, are tested in other, dedicated CI jobs, and which
+    # would increase build times here.
+    - run: |
+        $CENTOS cargo test \
+            --features test-programs/test_programs \
+            --release \
+            --all \
+            --exclude lightbeam \
+            --exclude peepmatic \
+            --exclude peepmatic-automata \
+            --exclude peepmatic-fuzzing \
+            --exclude peepmatic-macro \
+            --exclude peepmatic-runtime \
+            --exclude peepmatic-test \
+            --exclude wasmtime-fuzz
       shell: bash
       env:
         RUST_BACKTRACE: 1

.gitignore

@@ -14,3 +14,4 @@ docs/book
 rusty-tags.*
 tags
 target
+.z3-trace

Cargo.lock

@@ -11,9 +11,12 @@ dependencies = [
 
 [[package]]
 name = "ahash"
-version = "0.3.3"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35b909d1c126f78ace756fc337133356c499eebeefcce930fa5fb018823f2b2d"
+checksum = "0989268a37e128d4d7a8028f1c60099430113fdbc70419010601ce51a228e4fe"
+dependencies = [
+ "const-random",
+]
 
 [[package]]
 name = "aho-corasick"
@@ -41,9 +44,9 @@ checksum = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff"
 
 [[package]]
 name = "arbitrary"
-version = "0.4.4"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5eb01a9ab8a3369f2f7632b9461c34f5920bd454774bab5b9fc6744f21d6143"
+checksum = "75153c95fdedd7db9732dfbfc3702324a1627eec91ba56e37cd0ac78314ab2ed"
 dependencies = [
  "derive_arbitrary",
 ]
@@ -97,9 +100,9 @@ dependencies = [
 
 [[package]]
 name = "backtrace-sys"
-version = "0.1.36"
+version = "0.1.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78848718ee1255a2485d1309ad9cdecfc2e7d0362dd11c6829364c6b35ae1bc7"
+checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118"
 dependencies = [
  "cc",
  "libc",
@@ -201,6 +204,12 @@ dependencies = [
  "byte-tools",
 ]
 
+[[package]]
+name = "bumpalo"
+version = "3.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187"
+
 [[package]]
 name = "byte-tools"
 version = "0.3.1"
@@ -233,9 +242,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.52"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d87b23d6a92cd03af510a5ade527033f6aa6fa92161e2d5863a907d4c5e31d"
+checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd"
 dependencies = [
  "jobserver",
 ]
@@ -261,6 +270,18 @@ dependencies = [
  "vec_map",
 ]
 
+[[package]]
+name = "clicolors-control"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90082ee5dcdd64dc4e9e0d37fbf3ee325419e39c0092191e0393df65518f741e"
+dependencies = [
+ "atty",
+ "lazy_static",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "cloudabi"
 version = "0.0.3"
@@ -281,19 +302,18 @@ dependencies = [
 
 [[package]]
 name = "console"
-version = "0.11.2"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dea0f3e2e8d7dba335e913b97f9e1992c86c4399d54f8be1d31c8727d0652064"
+checksum = "6728a28023f207181b193262711102bfbaf47cc9d13bc71d0736607ef8efe88c"
 dependencies = [
+ "clicolors-control",
  "encode_unicode",
  "lazy_static",
  "libc",
  "regex",
- "terminal_size",
  "termios",
  "unicode-width",
  "winapi",
- "winapi-util",
 ]
 
 [[package]]
@@ -357,8 +377,10 @@ dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
  "gimli",
- "hashbrown 0.7.2",
+ "hashbrown 0.7.1",
  "log",
+ "peepmatic",
+ "peepmatic-runtime",
  "regalloc",
  "serde",
  "smallvec",
@@ -424,7 +446,7 @@ name = "cranelift-frontend"
 version = "0.63.0"
 dependencies = [
  "cranelift-codegen",
- "hashbrown 0.7.2",
+ "hashbrown 0.7.1",
  "log",
  "smallvec",
  "target-lexicon",
@@ -438,7 +460,7 @@ dependencies = [
  "cranelift-entity",
  "cranelift-frontend",
  "cranelift-reader",
- "hashbrown 0.7.2",
+ "hashbrown 0.7.1",
  "log",
  "pretty_env_logger",
  "thiserror",
@@ -566,12 +588,12 @@ dependencies = [
  "cranelift-codegen",
  "cranelift-entity",
  "cranelift-frontend",
- "hashbrown 0.7.2",
+ "hashbrown 0.7.1",
  "log",
  "serde",
  "target-lexicon",
  "thiserror",
- "wasmparser",
+ "wasmparser 0.52.2",
  "wat",
 ]
 
@@ -642,13 +664,13 @@ dependencies = [
 
 [[package]]
 name = "derive_arbitrary"
-version = "0.4.4"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cee758ebd1c79a9c6fb95f242dcc30bdbf555c28369ae908d21fdaf81537496"
+checksum = "caedd6a71b6d00bdc458ec8ffbfd12689c1ee7ffa69ad9933310aaf2f08f18d8"
 dependencies = [
  "proc-macro2",
- "quote",
  "syn",
+ "synstructure",
 ]
 
 [[package]]
@@ -877,6 +899,12 @@ version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 
+[[package]]
+name = "fst"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81f9cac32c1741cdf6b66be7dcf0d9c7f25ccf12f8aa84c16cfa31f9f14513b3"
+
 [[package]]
 name = "fuchsia-cprng"
 version = "0.1.1"
@@ -952,11 +980,11 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.7.2"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf"
+checksum = "479e9d9a1a3f8c489868a935b557ab5710e3e223836da2ecd52901d88935cb56"
 dependencies = [
- "ahash 0.3.3",
+ "ahash 0.3.2",
  "autocfg 1.0.0",
 ]
 
@@ -971,9 +999,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.12"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61565ff7aaace3525556587bd2dc31d4a07071957be715e63ce7b1eccf51a8f4"
+checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e"
 dependencies = [
  "libc",
 ]
@@ -1100,7 +1128,7 @@ dependencies = [
  "staticvec",
  "thiserror",
  "typemap",
- "wasmparser",
+ "wasmparser 0.52.2",
  "wat",
 ]
 
@@ -1170,9 +1198,9 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.13.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6"
 dependencies = [
  "hermit-abi",
  "libc",
@@ -1226,6 +1254,76 @@ dependencies = [
  "stable_deref_trait",
 ]
 
+[[package]]
+name = "peepmatic"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "peepmatic-automata",
+ "peepmatic-macro",
+ "peepmatic-runtime",
+ "wast 15.0.0",
+ "z3",
+]
+
+[[package]]
+name = "peepmatic-automata"
+version = "0.1.0"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "peepmatic-fuzzing"
+version = "0.1.0"
+dependencies = [
+ "arbitrary",
+ "bincode",
+ "env_logger 0.7.1",
+ "fst",
+ "log",
+ "peepmatic",
+ "peepmatic-automata",
+ "peepmatic-runtime",
+ "peepmatic-test",
+ "rand 0.7.3",
+ "serde",
+ "wast 15.0.0",
+]
+
+[[package]]
+name = "peepmatic-macro"
+version = "0.1.0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "peepmatic-runtime"
+version = "0.1.0"
+dependencies = [
+ "bincode",
+ "bumpalo",
+ "log",
+ "peepmatic-automata",
+ "peepmatic-macro",
+ "serde",
+ "thiserror",
+ "wast 15.0.0",
+]
+
+[[package]]
+name = "peepmatic-test"
+version = "0.1.0"
+dependencies = [
+ "env_logger 0.7.1",
+ "log",
+ "peepmatic",
+ "peepmatic-runtime",
+]
+
 [[package]]
 name = "plain"
 version = "0.2.3"
@@ -1291,9 +1389,9 @@ dependencies = [
 
 [[package]]
 name = "proptest"
-version = "0.9.6"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01c477819b845fe023d33583ebf10c9f62518c8d79a0960ba5c36d6ac8a55a5b"
+checksum = "bf6147d103a7c9d7598f4105cf049b15c99e2ecd93179bf024f0fd349be5ada4"
 dependencies = [
  "bit-set",
  "bitflags",
@@ -1350,7 +1448,7 @@ dependencies = [
  "rand_isaac",
  "rand_jitter",
  "rand_os",
- "rand_pcg",
+ "rand_pcg 0.1.2",
  "rand_xorshift",
  "winapi",
 ]
@@ -1366,6 +1464,7 @@ dependencies = [
  "rand_chacha 0.2.2",
  "rand_core 0.5.1",
  "rand_hc 0.2.0",
+ "rand_pcg 0.2.1",
 ]
 
 [[package]]
@@ -1474,6 +1573,15 @@ dependencies = [
  "rand_core 0.4.2",
 ]
 
+[[package]]
+name = "rand_pcg"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
+dependencies = [
+ "rand_core 0.5.1",
+]
+
 [[package]]
 name = "rand_xorshift"
 version = "0.1.1"
@@ -1557,9 +1665,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.3.7"
+version = "1.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692"
+checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1648,9 +1756,9 @@ dependencies = [
 
 [[package]]
 name = "ryu"
-version = "1.0.4"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1"
+checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76"
 
 [[package]]
 name = "same-file"
@@ -1724,9 +1832,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.52"
+version = "1.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7894c8ed05b7a3a279aeb79025fdec1d3158080b75b98a08faf2806bb799edd"
+checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9"
 dependencies = [
  "itoa",
  "ryu",
@@ -1747,9 +1855,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.4.0"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4"
+checksum = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a"
 
 [[package]]
 name = "stable_deref_trait"
@@ -1780,9 +1888,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "structopt"
-version = "0.3.14"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef"
+checksum = "ff6da2e8d107dfd7b74df5ef4d205c6aebee0706c647f6bc6a2d5789905c00fb"
 dependencies = [
  "clap",
  "lazy_static",
@@ -1791,9 +1899,9 @@ dependencies = [
 
 [[package]]
 name = "structopt-derive"
-version = "0.4.7"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a"
+checksum = "a489c87c08fbaf12e386665109dd13470dcc9c4583ea3e10dd2b4523e5ebd9ac"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -1804,9 +1912,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.18"
+version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "410a7488c0a728c7ceb4ad59b9567eb4053d02e8cc7f5c0e0eeeb39518369213"
+checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1824,6 +1932,18 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "unicode-xid",
+]
+
 [[package]]
 name = "target-lexicon"
 version = "0.10.0"
@@ -1863,16 +1983,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "terminal_size"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8038f95fc7a6f351163f4b964af631bd26c9e828f7db085f2a84aca56f70d13b"
-dependencies = [
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "termios"
 version = "0.3.2"
@@ -1909,18 +2019,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.16"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d12a1dae4add0f0d568eebc7bf142f145ba1aa2544cafb195c76f0f409091b60"
+checksum = "54b3d3d2ff68104100ab257bb6bb0cb26c901abe4bd4ba15961f3bf867924012"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.16"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f34e0c1caaa462fd840ec6b768946ea1e7842620d94fe29d5b847138f521269"
+checksum = "ca972988113b7715266f91250ddb98070d033c62a011fa0fcc57434a649310dd"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1962,9 +2072,9 @@ dependencies = [
 
 [[package]]
 name = "typenum"
-version = "1.12.0"
+version = "1.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
+checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9"
 
 [[package]]
 name = "unicode-segmentation"
@@ -2051,6 +2161,12 @@ dependencies = [
  "yanix",
 ]
 
+[[package]]
+name = "wasmparser"
+version = "0.51.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb1956b19469d1c5e63e459d29e7b5aa0f558d9f16fcef09736f8a265e6c10a"
+
 [[package]]
 name = "wasmparser"
 version = "0.52.2"
@@ -2059,12 +2175,12 @@ checksum = "733954023c0b39602439e60a65126fd31b003196d3a1e8e4531b055165a79b31"
 
 [[package]]
 name = "wasmprinter"
-version = "0.2.4"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21c3ac16b1f882bf1e1ce007e4f194559d2e3332013367863ddfbc828d1f044f"
+checksum = "8bd423d45b95fcee11775472bfdce66c63c45ada23c1b338e0a63d623a6c475b"
 dependencies = [
  "anyhow",
- "wasmparser",
+ "wasmparser 0.51.4",
 ]
 
 [[package]]
@@ -2080,7 +2196,7 @@ dependencies = [
  "rustc-demangle",
  "target-lexicon",
  "tempfile",
- "wasmparser",
+ "wasmparser 0.52.2",
  "wasmtime-environ",
  "wasmtime-jit",
  "wasmtime-profiling",
@@ -2150,7 +2266,7 @@ dependencies = [
  "more-asserts",
  "target-lexicon",
  "thiserror",
- "wasmparser",
+ "wasmparser 0.52.2",
  "wasmtime-environ",
 ]
 
@@ -2181,7 +2297,7 @@ dependencies = [
  "tempfile",
  "thiserror",
  "toml",
- "wasmparser",
+ "wasmparser 0.52.2",
  "winapi",
  "zstd",
 ]
@@ -2194,6 +2310,7 @@ dependencies = [
  "cranelift-reader",
  "cranelift-wasm",
  "libfuzzer-sys",
+ "peepmatic-fuzzing",
  "target-lexicon",
  "wasmtime",
  "wasmtime-fuzzing",
@@ -2209,7 +2326,7 @@ dependencies = [
  "env_logger 0.7.1",
  "log",
  "rayon",
- "wasmparser",
+ "wasmparser 0.52.2",
  "wasmprinter",
  "wasmtime",
  "wasmtime-wast",
@@ -2233,7 +2350,7 @@ dependencies = [
  "region",
  "target-lexicon",
  "thiserror",
- "wasmparser",
+ "wasmparser 0.52.2",
  "wasmtime-debug",
  "wasmtime-environ",
  "wasmtime-profiling",
@@ -2424,9 +2541,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.5"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e"
 dependencies = [
  "winapi",
 ]
@@ -2470,6 +2587,26 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "z3"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ded00cd90f8e3a7ea3155bddd72573f2b099ea201877542d924e47b58dd04e72"
+dependencies = [
+ "lazy_static",
+ "log",
+ "z3-sys",
+]
+
+[[package]]
+name = "z3-sys"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4002d8a1facb54d02dbfb86151281e5450618ab330936bc2f3acaab31eae11ae"
+dependencies = [
+ "cmake",
+]
+
 [[package]]
 name = "zstd"
 version = "0.5.1+zstd.1.4.4"

cranelift/Cargo.toml

@@ -47,4 +47,5 @@ walkdir = "2.2"
 [features]
 default = ["disas", "wasm", "cranelift-codegen/all-arch"]
 disas = ["capstone"]
+enable-peepmatic = ["cranelift-codegen/enable-peepmatic", "cranelift-filetests/enable-peepmatic"]
 wasm = ["wat", "cranelift-wasm"]

cranelift/codegen/Cargo.toml

@@ -24,6 +24,7 @@ gimli = { version = "0.20.0", default-features = false, features = ["write"], op
 smallvec = { version = "1.0.0" }
 thiserror = "1.0.4"
 byteorder = { version = "1.3.2", default-features = false }
+peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true }
 regalloc = "0.0.23"
 # It is a goal of the cranelift-codegen crate to have minimal external dependencies.
 # Please don't add any unless they are essential to the task of creating binary
@@ -32,6 +33,7 @@ regalloc = "0.0.23"
 
 [build-dependencies]
 cranelift-codegen-meta = { path = "meta", version = "0.63.0" }
+peepmatic = { path = "../peepmatic", optional = true }
 
 [features]
 default = ["std", "unwind"]
@@ -72,5 +74,12 @@ all-arch = [
 # For dependent crates that want to serialize some parts of cranelift
 enable-serde = ["serde"]
 
+# Recompile our optimizations that are written in the `peepmatic` DSL into a
+# compact finite-state transducer automaton.
+rebuild-peephole-optimizers = ["peepmatic"]
+
+# Enable the use of `peepmatic`-generated peephole optimizers.
+enable-peepmatic = ["peepmatic-runtime"]
+
 [badges]
 maintenance = { status = "experimental" }

cranelift/codegen/build.rs

@@ -71,4 +71,22 @@ fn main() {
         );
         println!("cargo:warning=Generated files are in {}", out_dir);
     }
+
+    #[cfg(feature = "rebuild-peephole-optimizers")]
+    rebuild_peephole_optimizers();
+}
+
+#[cfg(feature = "rebuild-peephole-optimizers")]
+fn rebuild_peephole_optimizers() {
+    use std::path::Path;
+
+    let source_path = Path::new("src").join("preopt.peepmatic");
+    println!("cargo:rerun-if-changed={}", source_path.display());
+
+    let preopt =
+        peepmatic::compile_file(&source_path).expect("failed to compile `src/preopt.peepmatic`");
+
+    preopt
+        .serialize_to_file(&Path::new("src").join("preopt.serialized"))
+        .expect("failed to serialize peephole optimizer to `src/preopt.serialized`");
 }

cranelift/codegen/src/ir/dfg.rs

@@ -234,11 +234,7 @@ impl DataFlowGraph {
 
     /// Get the type of a value.
     pub fn value_type(&self, v: Value) -> Type {
-        match self.values[v] {
+        self.values[v].ty()
-            ValueData::Inst { ty, .. }
-            | ValueData::Param { ty, .. }
-            | ValueData::Alias { ty, .. } => ty,
-        }
     }
 
     /// Get the definition of a value.
@@ -383,9 +379,14 @@ pub enum ValueDef {
 impl ValueDef {
     /// Unwrap the instruction where the value was defined, or panic.
     pub fn unwrap_inst(&self) -> Inst {
+        self.inst().expect("Value is not an instruction result")
+    }
+
+    /// Get the instruction where the value was defined, if any.
+    pub fn inst(&self) -> Option<Inst> {
         match *self {
-            Self::Result(inst, _) => inst,
+            Self::Result(inst, _) => Some(inst),
-            _ => panic!("Value is not an instruction result"),
+            _ => None,
         }
     }
 
@@ -428,6 +429,16 @@ enum ValueData {
     Alias { ty: Type, original: Value },
 }
 
+impl ValueData {
+    fn ty(&self) -> Type {
+        match *self {
+            ValueData::Inst { ty, .. }
+            | ValueData::Param { ty, .. }
+            | ValueData::Alias { ty, .. } => ty,
+        }
+    }
+}
+
 /// Instructions.
 ///
 impl DataFlowGraph {

cranelift/codegen/src/ir/function.rs

@@ -308,6 +308,30 @@ impl Function {
         // function, assume it is not a leaf.
         self.dfg.signatures.is_empty()
     }
+
+    /// Replace the `dst` instruction's data with the `src` instruction's data
+    /// and then remove `src`.
+    ///
+    /// `src` and its result values should not be used at all, as any uses would
+    /// be left dangling after calling this method.
+    ///
+    /// `src` and `dst` must have the same number of resulting values, and
+    /// `src`'s i^th value must have the same type as `dst`'s i^th value.
+    pub fn transplant_inst(&mut self, dst: Inst, src: Inst) {
+        debug_assert_eq!(
+            self.dfg.inst_results(dst).len(),
+            self.dfg.inst_results(src).len()
+        );
+        debug_assert!(self
+            .dfg
+            .inst_results(dst)
+            .iter()
+            .zip(self.dfg.inst_results(src))
+            .all(|(a, b)| self.dfg.value_type(*a) == self.dfg.value_type(*b)));
+
+        self.dfg[dst] = self.dfg[src].clone();
+        self.layout.remove_inst(src);
+    }
 }
 
 /// Additional annotations for function display.

cranelift/codegen/src/ir/instructions.rs

@@ -11,9 +11,7 @@ use core::fmt::{self, Display, Formatter};
 use core::ops::{Deref, DerefMut};
 use core::str::FromStr;
 
-use crate::ir;
+use crate::ir::{self, trapcode::TrapCode, types, Block, FuncRef, JumpTable, SigRef, Type, Value};
-use crate::ir::types;
-use crate::ir::{Block, FuncRef, JumpTable, SigRef, Type, Value};
 use crate::isa;
 
 use crate::bitset::BitSet;
@@ -257,6 +255,30 @@ impl InstructionData {
         }
     }
 
+    /// If this is a trapping instruction, get its trap code. Otherwise, return
+    /// `None`.
+    pub fn trap_code(&self) -> Option<TrapCode> {
+        match *self {
+            Self::CondTrap { code, .. }
+            | Self::FloatCondTrap { code, .. }
+            | Self::IntCondTrap { code, .. }
+            | Self::Trap { code, .. } => Some(code),
+            _ => None,
+        }
+    }
+
+    /// If this is a trapping instruction, get an exclusive reference to its
+    /// trap code. Otherwise, return `None`.
+    pub fn trap_code_mut(&mut self) -> Option<&mut TrapCode> {
+        match self {
+            Self::CondTrap { code, .. }
+            | Self::FloatCondTrap { code, .. }
+            | Self::IntCondTrap { code, .. }
+            | Self::Trap { code, .. } => Some(code),
+            _ => None,
+        }
+    }
+
     /// Return information about a call instruction.
     ///
     /// Any instruction that can call another function reveals its call signature here.

cranelift/codegen/src/lib.rs

@@ -115,6 +115,9 @@ mod topo_order;
 mod unreachable_code;
 mod value_label;
 
+#[cfg(feature = "enable-peepmatic")]
+mod peepmatic;
+
 pub use crate::result::{CodegenError, CodegenResult};
 
 /// Version number of this crate.

cranelift/codegen/src/peepmatic.rs

@@ -0,0 +1,887 @@
+//! Glue for working with `peepmatic`-generated peephole optimizers.
+
+use crate::cursor::{Cursor, FuncCursor};
+use crate::ir::{
+    dfg::DataFlowGraph,
+    entities::{Inst, Value},
+    immediates::{Imm64, Uimm64},
+    instructions::{InstructionData, Opcode},
+    types, InstBuilder,
+};
+use crate::isa::TargetIsa;
+use cranelift_codegen_shared::condcodes::IntCC;
+use peepmatic_runtime::{
+    cc::ConditionCode,
+    instruction_set::InstructionSet,
+    operator::Operator,
+    part::{Constant, Part},
+    paths::Path,
+    r#type::{BitWidth, Kind, Type},
+    PeepholeOptimizations, PeepholeOptimizer,
+};
+use std::boxed::Box;
+use std::convert::{TryFrom, TryInto};
+use std::ptr;
+use std::sync::atomic::{AtomicPtr, Ordering};
+
+/// Get the `preopt.peepmatic` peephole optimizer.
+pub(crate) fn preopt<'a, 'b>(
+    isa: &'b dyn TargetIsa,
+) -> PeepholeOptimizer<'static, 'a, &'b dyn TargetIsa> {
+    static SERIALIZED: &[u8] = include_bytes!("preopt.serialized");
+
+    // Once initialized, this must never be re-assigned. The initialized value
+    // is semantically "static data" and is intentionally leaked for the whole
+    // program's lifetime.
+    static DESERIALIZED: AtomicPtr<PeepholeOptimizations> = AtomicPtr::new(ptr::null_mut());
+
+    // If `DESERIALIZED` has already been initialized, then just use it.
+    let ptr = DESERIALIZED.load(Ordering::SeqCst);
+    if let Some(peep_opts) = unsafe { ptr.as_ref() } {
+        return peep_opts.optimizer(isa);
+    }
+
+    // Otherwise, if `DESERIALIZED` hasn't been initialized, then we need to
+    // deserialize the peephole optimizations and initialize it. However,
+    // another thread could be doing the same thing concurrently, so there is a
+    // race to see who initializes `DESERIALIZED` first, and we need to be
+    // prepared to both win or lose that race.
+    let peep_opts = PeepholeOptimizations::deserialize(SERIALIZED)
+        .expect("should always be able to deserialize `preopt.serialized`");
+    let peep_opts = Box::into_raw(Box::new(peep_opts));
+
+    // Only update `DESERIALIZE` if it is still null, attempting to perform the
+    // one-time transition from null -> non-null.
+    if DESERIALIZED
+        .compare_and_swap(ptr::null_mut(), peep_opts, Ordering::SeqCst)
+        .is_null()
+    {
+        // We won the race to initialize `DESERIALIZED`.
+        debug_assert_eq!(DESERIALIZED.load(Ordering::SeqCst), peep_opts);
+        let peep_opts = unsafe { &*peep_opts };
+        return peep_opts.optimizer(isa);
+    }
+
+    // We lost the race to initialize `DESERIALIZED`. Drop our no-longer-needed
+    // instance of `peep_opts` and get the pointer to the instance that won the
+    // race.
+    let _ = unsafe { Box::from_raw(peep_opts) };
+    let peep_opts = DESERIALIZED.load(Ordering::SeqCst);
+    let peep_opts = unsafe { peep_opts.as_ref().unwrap() };
+    peep_opts.optimizer(isa)
+}
+
+/// Either a `Value` or an `Inst`.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ValueOrInst {
+    Value(Value),
+    Inst(Inst),
+}
+
+impl ValueOrInst {
+    /// Get the underlying `Value` if any.
+    pub fn value(&self) -> Option<Value> {
+        match *self {
+            Self::Value(v) => Some(v),
+            Self::Inst(_) => None,
+        }
+    }
+
+    /// Get the underlying `Inst` if any.
+    pub fn inst(&self) -> Option<Inst> {
+        match *self {
+            Self::Inst(i) => Some(i),
+            Self::Value(_) => None,
+        }
+    }
+
+    /// Unwrap the underlying `Value`, panicking if it is not a `Value.
+    pub fn unwrap_value(&self) -> Value {
+        self.value().unwrap()
+    }
+
+    /// Unwrap the underlying `Inst`, panicking if it is not a `Inst.
+    pub fn unwrap_inst(&self) -> Inst {
+        self.inst().unwrap()
+    }
+
+    /// Is this a `Value`?
+    pub fn is_value(&self) -> bool {
+        self.value().is_some()
+    }
+
+    /// Is this an `Inst`?
+    pub fn is_inst(&self) -> bool {
+        self.inst().is_some()
+    }
+
+    fn resolve_inst(&self, dfg: &DataFlowGraph) -> Option<Inst> {
+        match *self {
+            ValueOrInst::Inst(i) => Some(i),
+            ValueOrInst::Value(v) => dfg.value_def(v).inst(),
+        }
+    }
+
+    fn result_bit_width(&self, dfg: &DataFlowGraph) -> u8 {
+        match *self {
+            ValueOrInst::Value(v) => dfg.value_type(v).bits().try_into().unwrap(),
+            ValueOrInst::Inst(inst) => {
+                let result = dfg.first_result(inst);
+                dfg.value_type(result).bits().try_into().unwrap()
+            }
+        }
+    }
+
+    fn to_constant(&self, pos: &mut FuncCursor) -> Option<Constant> {
+        let inst = self.resolve_inst(&pos.func.dfg)?;
+        match pos.func.dfg[inst] {
+            InstructionData::UnaryImm {
+                opcode: Opcode::Iconst,
+                imm,
+            } => {
+                let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap();
+                let x: i64 = imm.into();
+                Some(Constant::Int(x as u64, width))
+            }
+            InstructionData::UnaryBool {
+                opcode: Opcode::Bconst,
+                imm,
+            } => {
+                let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap();
+                Some(Constant::Bool(imm, width))
+            }
+            _ => None,
+        }
+    }
+}
+
+impl From<Value> for ValueOrInst {
+    fn from(v: Value) -> ValueOrInst {
+        ValueOrInst::Value(v)
+    }
+}
+
+impl From<Inst> for ValueOrInst {
+    fn from(i: Inst) -> ValueOrInst {
+        ValueOrInst::Inst(i)
+    }
+}
+
+/// Get the fixed bit width of `bit_width`, or if it is polymorphic, the bit
+/// width of `root`.
+fn bit_width(dfg: &DataFlowGraph, bit_width: BitWidth, root: Inst) -> u8 {
+    bit_width.fixed_width().unwrap_or_else(|| {
+        let tyvar = dfg.ctrl_typevar(root);
+        let ty = dfg.compute_result_type(root, 0, tyvar).unwrap();
+        u8::try_from(ty.bits()).unwrap()
+    })
+}
+
+/// Convert the constant `c` into an instruction.
+fn const_to_value<'a>(builder: impl InstBuilder<'a>, c: Constant, root: Inst) -> Value {
+    match c {
+        Constant::Bool(b, width) => {
+            let width = bit_width(builder.data_flow_graph(), width, root);
+            let ty = match width {
+                1 => types::B1,
+                8 => types::B8,
+                16 => types::B16,
+                32 => types::B32,
+                64 => types::B64,
+                128 => types::B128,
+                _ => unreachable!(),
+            };
+            builder.bconst(ty, b)
+        }
+        Constant::Int(x, width) => {
+            let width = bit_width(builder.data_flow_graph(), width, root);
+            let ty = match width {
+                8 => types::I8,
+                16 => types::I16,
+                32 => types::I32,
+                64 => types::I64,
+                128 => types::I128,
+                _ => unreachable!(),
+            };
+            builder.iconst(ty, x as i64)
+        }
+    }
+}
+
+fn part_to_value(pos: &mut FuncCursor, root: Inst, part: Part<ValueOrInst>) -> Option<Value> {
+    match part {
+        Part::Instruction(ValueOrInst::Inst(inst)) => {
+            pos.func.dfg.inst_results(inst).first().copied()
+        }
+        Part::Instruction(ValueOrInst::Value(v)) => Some(v),
+        Part::Constant(c) => Some(const_to_value(pos.ins(), c, root)),
+        Part::ConditionCode(_) => None,
+    }
+}
+
+impl Opcode {
+    fn to_peepmatic_operator(&self) -> Option<Operator> {
+        macro_rules! convert {
+            ( $( $op:ident $(,)* )* ) => {
+                match self {
+                    $( Self::$op => Some(Operator::$op), )*
+                    _ => None,
+                }
+            }
+        }
+
+        convert!(
+            AdjustSpDown,
+            AdjustSpDownImm,
+            Band,
+            BandImm,
+            Bconst,
+            Bint,
+            Bor,
+            BorImm,
+            Brnz,
+            Brz,
+            Bxor,
+            BxorImm,
+            Iadd,
+            IaddImm,
+            Icmp,
+            IcmpImm,
+            Iconst,
+            Ifcmp,
+            IfcmpImm,
+            Imul,
+            ImulImm,
+            Ireduce,
+            IrsubImm,
+            Ishl,
+            IshlImm,
+            Isub,
+            Rotl,
+            RotlImm,
+            Rotr,
+            RotrImm,
+            Sdiv,
+            SdivImm,
+            Select,
+            Sextend,
+            Srem,
+            SremImm,
+            Sshr,
+            SshrImm,
+            Trapnz,
+            Trapz,
+            Udiv,
+            UdivImm,
+            Uextend,
+            Urem,
+            UremImm,
+            Ushr,
+            UshrImm,
+        )
+    }
+}
+
+impl TryFrom<Constant> for Imm64 {
+    type Error = &'static str;
+
+    fn try_from(c: Constant) -> Result<Self, Self::Error> {
+        match c {
+            Constant::Int(x, _) => Ok(Imm64::from(x as i64)),
+            Constant::Bool(..) => Err("cannot create Imm64 from Constant::Bool"),
+        }
+    }
+}
+
+impl Into<Constant> for Imm64 {
+    #[inline]
+    fn into(self) -> Constant {
+        let x: i64 = self.into();
+        Constant::Int(x as _, BitWidth::SixtyFour)
+    }
+}
+
+impl Into<Part<ValueOrInst>> for Imm64 {
+    #[inline]
+    fn into(self) -> Part<ValueOrInst> {
+        let c: Constant = self.into();
+        c.into()
+    }
+}
+
+fn part_to_imm64(pos: &mut FuncCursor, part: Part<ValueOrInst>) -> Imm64 {
+    return match part {
+        Part::Instruction(x) => match x.to_constant(pos).unwrap_or_else(|| cannot_convert()) {
+            Constant::Int(x, _) => (x as i64).into(),
+            Constant::Bool(..) => cannot_convert(),
+        },
+        Part::Constant(Constant::Int(x, _)) => (x as i64).into(),
+        Part::ConditionCode(_) | Part::Constant(Constant::Bool(..)) => cannot_convert(),
+    };
+
+    #[inline(never)]
+    #[cold]
+    fn cannot_convert() -> ! {
+        panic!("cannot convert part into `Imm64`")
+    }
+}
+
+impl Into<Constant> for Uimm64 {
+    #[inline]
+    fn into(self) -> Constant {
+        let x: u64 = self.into();
+        Constant::Int(x, BitWidth::SixtyFour)
+    }
+}
+
+impl Into<Part<ValueOrInst>> for Uimm64 {
+    #[inline]
+    fn into(self) -> Part<ValueOrInst> {
+        let c: Constant = self.into();
+        c.into()
+    }
+}
+
+fn peepmatic_to_intcc(cc: ConditionCode) -> IntCC {
+    match cc {
+        ConditionCode::Eq => IntCC::Equal,
+        ConditionCode::Ne => IntCC::NotEqual,
+        ConditionCode::Slt => IntCC::SignedLessThan,
+        ConditionCode::Sle => IntCC::SignedGreaterThanOrEqual,
+        ConditionCode::Sgt => IntCC::SignedGreaterThan,
+        ConditionCode::Sge => IntCC::SignedLessThanOrEqual,
+        ConditionCode::Ult => IntCC::UnsignedLessThan,
+        ConditionCode::Uge => IntCC::UnsignedGreaterThanOrEqual,
+        ConditionCode::Ugt => IntCC::UnsignedGreaterThan,
+        ConditionCode::Ule => IntCC::UnsignedLessThanOrEqual,
+        ConditionCode::Of => IntCC::Overflow,
+        ConditionCode::Nof => IntCC::NotOverflow,
+    }
+}
+
+fn intcc_to_peepmatic(cc: IntCC) -> ConditionCode {
+    match cc {
+        IntCC::Equal => ConditionCode::Eq,
+        IntCC::NotEqual => ConditionCode::Ne,
+        IntCC::SignedLessThan => ConditionCode::Slt,
+        IntCC::SignedGreaterThanOrEqual => ConditionCode::Sle,
+        IntCC::SignedGreaterThan => ConditionCode::Sgt,
+        IntCC::SignedLessThanOrEqual => ConditionCode::Sge,
+        IntCC::UnsignedLessThan => ConditionCode::Ult,
+        IntCC::UnsignedGreaterThanOrEqual => ConditionCode::Uge,
+        IntCC::UnsignedGreaterThan => ConditionCode::Ugt,
+        IntCC::UnsignedLessThanOrEqual => ConditionCode::Ule,
+        IntCC::Overflow => ConditionCode::Of,
+        IntCC::NotOverflow => ConditionCode::Nof,
+    }
+}
+
+fn get_immediate(dfg: &DataFlowGraph, inst: Inst, i: usize) -> Part<ValueOrInst> {
+    return match dfg[inst] {
+        InstructionData::BinaryImm { imm, .. } if i == 0 => imm.into(),
+        InstructionData::BranchIcmp { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::BranchInt { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntCompare { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntCompareImm { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntCompareImm { imm, .. } if i == 1 => imm.into(),
+        InstructionData::IntCond { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntCondTrap { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::IntSelect { cond, .. } if i == 0 => intcc_to_peepmatic(cond).into(),
+        InstructionData::UnaryBool { imm, .. } if i == 0 => {
+            Constant::Bool(imm, BitWidth::Polymorphic).into()
+        }
+        InstructionData::UnaryImm { imm, .. } if i == 0 => imm.into(),
+        ref otherwise => unsupported(otherwise),
+    };
+
+    #[inline(never)]
+    #[cold]
+    fn unsupported(data: &InstructionData) -> ! {
+        panic!("unsupported instruction data: {:?}", data)
+    }
+}
+
+fn get_argument(dfg: &DataFlowGraph, inst: Inst, i: usize) -> Option<Value> {
+    dfg.inst_args(inst).get(i).copied()
+}
+
+fn peepmatic_ty_to_ir_ty(ty: Type, dfg: &DataFlowGraph, root: Inst) -> types::Type {
+    match (ty.kind, bit_width(dfg, ty.bit_width, root)) {
+        (Kind::Int, 8) => types::I8,
+        (Kind::Int, 16) => types::I16,
+        (Kind::Int, 32) => types::I32,
+        (Kind::Int, 64) => types::I64,
+        (Kind::Int, 128) => types::I128,
+        (Kind::Bool, 1) => types::B1,
+        (Kind::Bool, 8) => types::I8,
+        (Kind::Bool, 16) => types::I16,
+        (Kind::Bool, 32) => types::I32,
+        (Kind::Bool, 64) => types::I64,
+        (Kind::Bool, 128) => types::I128,
+        _ => unreachable!(),
+    }
+}
+
+// NB: the unsafe contract we must uphold here is that our implementation of
+// `instruction_result_bit_width` must always return a valid, non-zero bit
+// width.
+unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
+    type Context = FuncCursor<'b>;
+
+    type Instruction = ValueOrInst;
+
+    fn replace_instruction(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        old: ValueOrInst,
+        new: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("replace {:?} with {:?}", old, new);
+        let old_inst = old.resolve_inst(&pos.func.dfg).unwrap();
+
+        // Try to convert `new` to an instruction, because we prefer replacing
+        // an old instruction with a new one wholesale. However, if the
+        // replacement cannot be converted to an instruction (e.g. the
+        // right-hand side is a block/function parameter value) then we change
+        // the old instruction's result to an alias of the new value.
+        let new_inst = match new {
+            Part::Instruction(ValueOrInst::Inst(inst)) => Some(inst),
+            Part::Instruction(ValueOrInst::Value(_)) => {
+                // Do not try and follow the value definition. If we transplant
+                // this value's instruction, and there are other uses of this
+                // value, then we could mess up ordering between instructions.
+                None
+            }
+            Part::Constant(c) => {
+                let v = const_to_value(pos.ins(), c, old_inst);
+                let inst = pos.func.dfg.value_def(v).unwrap_inst();
+                Some(inst)
+            }
+            Part::ConditionCode(_) => None,
+        };
+
+        match new_inst {
+            Some(new_inst) => {
+                pos.func.transplant_inst(old_inst, new_inst);
+                debug_assert_eq!(pos.current_inst(), Some(old_inst));
+                old_inst.into()
+            }
+            None => {
+                let new_value = part_to_value(pos, old_inst, new).unwrap();
+
+                let old_results = pos.func.dfg.detach_results(old_inst);
+                let old_results = old_results.as_slice(&pos.func.dfg.value_lists);
+                assert_eq!(old_results.len(), 1);
+                let old_value = old_results[0];
+
+                pos.func.dfg.change_to_alias(old_value, new_value);
+                pos.func.dfg.replace(old_inst).nop();
+
+                new_value.into()
+            }
+        }
+    }
+
+    fn get_part_at_path(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        path: Path,
+    ) -> Option<Part<ValueOrInst>> {
+        // The root is path [0].
+        debug_assert!(!path.0.is_empty());
+        debug_assert_eq!(path.0[0], 0);
+
+        let mut part = Part::Instruction(root);
+        for p in path.0[1..].iter().copied() {
+            let inst = part.as_instruction()?.resolve_inst(&pos.func.dfg)?;
+            let operator = pos.func.dfg[inst].opcode().to_peepmatic_operator()?;
+
+            if p < operator.immediates_arity() {
+                part = get_immediate(&pos.func.dfg, inst, p as usize);
+                continue;
+            }
+
+            let arg = p - operator.immediates_arity();
+            let arg = arg as usize;
+            let value = get_argument(&pos.func.dfg, inst, arg)?;
+            part = Part::Instruction(value.into());
+        }
+
+        log::trace!("get_part_at_path({:?}) = {:?}", path, part);
+        Some(part)
+    }
+
+    fn operator(&self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst) -> Option<Operator> {
+        let inst = value_or_inst.resolve_inst(&pos.func.dfg)?;
+        pos.func.dfg[inst].opcode().to_peepmatic_operator()
+    }
+
+    fn make_inst_1(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        operator: Operator,
+        r#type: Type,
+        a: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("make_inst_1: {:?}({:?})", operator, a);
+
+        let root = root.resolve_inst(&pos.func.dfg).unwrap();
+        match operator {
+            Operator::AdjustSpDown => {
+                let a = part_to_value(pos, root, a).unwrap();
+                pos.ins().adjust_sp_down(a).into()
+            }
+            Operator::AdjustSpDownImm => {
+                let c = a.unwrap_constant();
+                let imm = Imm64::try_from(c).unwrap();
+                pos.ins().adjust_sp_down_imm(imm).into()
+            }
+            Operator::Bconst => {
+                let c = a.unwrap_constant();
+                let val = const_to_value(pos.ins(), c, root);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Bint => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                let val = pos.ins().bint(ty, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Brnz => {
+                let a = part_to_value(pos, root, a).unwrap();
+
+                // NB: branching instructions must be the root of an
+                // optimization's right-hand side, so we get the destination
+                // block and arguments from the left-hand side's root. Peepmatic
+                // doesn't currently represent labels or varargs.
+                let block = pos.func.dfg[root].branch_destination().unwrap();
+                let args = pos.func.dfg.inst_args(root)[1..].to_vec();
+
+                pos.ins().brnz(a, block, &args).into()
+            }
+            Operator::Brz => {
+                let a = part_to_value(pos, root, a).unwrap();
+
+                // See the comment in the `Operator::Brnz` match argm.
+                let block = pos.func.dfg[root].branch_destination().unwrap();
+                let args = pos.func.dfg.inst_args(root)[1..].to_vec();
+
+                pos.ins().brz(a, block, &args).into()
+            }
+            Operator::Iconst => {
+                let a = a.unwrap_constant();
+                let val = const_to_value(pos.ins(), a, root);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Ireduce => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                let val = pos.ins().ireduce(ty, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Sextend => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                let val = pos.ins().sextend(ty, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Trapnz => {
+                let a = part_to_value(pos, root, a).unwrap();
+
+                // NB: similar to branching instructions (see comment in the
+                // `Operator::Brnz` match arm) trapping instructions must be the
+                // root of an optimization's right-hand side, and we get the
+                // trap code from the root of the left-hand side. Peepmatic
+                // doesn't currently represent trap codes.
+                let code = pos.func.dfg[root].trap_code().unwrap();
+
+                pos.ins().trapnz(a, code).into()
+            }
+            Operator::Trapz => {
+                let a = part_to_value(pos, root, a).unwrap();
+                // See comment in the `Operator::Trapnz` match arm.
+                let code = pos.func.dfg[root].trap_code().unwrap();
+                pos.ins().trapz(a, code).into()
+            }
+            Operator::Uextend => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                let val = pos.ins().uextend(ty, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn make_inst_2(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        operator: Operator,
+        _: Type,
+        a: Part<ValueOrInst>,
+        b: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("make_inst_2: {:?}({:?}, {:?})", operator, a, b);
+
+        let root = root.resolve_inst(&pos.func.dfg).unwrap();
+        match operator {
+            Operator::Band => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().band(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::BandImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().band_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Bor => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().bor(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::BorImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().bor_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Bxor => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().bxor(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::BxorImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().bxor_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Iadd => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().iadd(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::IaddImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().iadd_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Ifcmp => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ifcmp(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::IfcmpImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ifcmp_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Imul => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().imul(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::ImulImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().imul_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::IrsubImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().irsub_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Ishl => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ishl(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::IshlImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ishl_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Isub => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().isub(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Rotl => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().rotl(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::RotlImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().rotl_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Rotr => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().rotr(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::RotrImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().rotr_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Sdiv => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().sdiv(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::SdivImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().sdiv_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Srem => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().srem(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::SremImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().srem_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Sshr => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().sshr(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::SshrImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().sshr_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Udiv => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().udiv(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::UdivImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().udiv_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Urem => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().urem(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::UremImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().urem_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Ushr => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ushr(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::UshrImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ushr_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn make_inst_3(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        operator: Operator,
+        _: Type,
+        a: Part<ValueOrInst>,
+        b: Part<ValueOrInst>,
+        c: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("make_inst_3: {:?}({:?}, {:?}, {:?})", operator, a, b, c);
+
+        let root = root.resolve_inst(&pos.func.dfg).unwrap();
+        match operator {
+            Operator::Icmp => {
+                let cond = a.unwrap_condition_code();
+                let cond = peepmatic_to_intcc(cond);
+                let b = part_to_value(pos, root, b).unwrap();
+                let c = part_to_value(pos, root, c).unwrap();
+                let val = pos.ins().icmp(cond, b, c);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::IcmpImm => {
+                let cond = a.unwrap_condition_code();
+                let cond = peepmatic_to_intcc(cond);
+                let imm = part_to_imm64(pos, b);
+                let c = part_to_value(pos, root, c).unwrap();
+                let val = pos.ins().icmp_imm(cond, c, imm);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Operator::Select => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let c = part_to_value(pos, root, c).unwrap();
+                let val = pos.ins().select(a, b, c);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn instruction_to_constant(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        value_or_inst: ValueOrInst,
+    ) -> Option<Constant> {
+        value_or_inst.to_constant(pos)
+    }
+
+    fn instruction_result_bit_width(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        value_or_inst: ValueOrInst,
+    ) -> u8 {
+        value_or_inst.result_bit_width(&pos.func.dfg)
+    }
+
+    fn native_word_size_in_bits(&self, _pos: &mut FuncCursor<'b>) -> u8 {
+        self.pointer_bits()
+    }
+}

cranelift/codegen/src/preopt.peepmatic

@@ -0,0 +1,193 @@
+;; Apply basic simplifications.
+;;
+;; This folds constants with arithmetic to form `_imm` instructions, and other
+;; minor simplifications.
+;;
+;; Doesn't apply some simplifications if the native word width (in bytes) is
+;; smaller than the controlling type's width of the instruction. This would
+;; result in an illegal instruction that would likely be expanded back into an
+;; instruction on smaller types with the same initial opcode, creating
+;; unnecessary churn.
+
+;; Binary instructions whose second argument is constant.
+(=> (when (iadd $x $C)
+      (fits-in-native-word $C))
+    (iadd_imm $C $x))
+(=> (when (imul $x $C)
+      (fits-in-native-word $C))
+    (imul_imm $C $x))
+(=> (when (sdiv $x $C)
+      (fits-in-native-word $C))
+    (sdiv_imm $C $x))
+(=> (when (udiv $x $C)
+      (fits-in-native-word $C))
+    (udiv_imm $C $x))
+(=> (when (srem $x $C)
+      (fits-in-native-word $C))
+    (srem_imm $C $x))
+(=> (when (urem $x $C)
+      (fits-in-native-word $C))
+    (urem_imm $C $x))
+(=> (when (band $x $C)
+      (fits-in-native-word $C))
+    (band_imm $C $x))
+(=> (when (bor $x $C)
+      (fits-in-native-word $C))
+    (bor_imm $C $x))
+(=> (when (bxor $x $C)
+      (fits-in-native-word $C))
+    (bxor_imm $C $x))
+(=> (when (rotl $x $C)
+      (fits-in-native-word $C))
+    (rotl_imm $C $x))
+(=> (when (rotr $x $C)
+      (fits-in-native-word $C))
+    (rotr_imm $C $x))
+(=> (when (ishl $x $C)
+      (fits-in-native-word $C))
+    (ishl_imm $C $x))
+(=> (when (ushr $x $C)
+      (fits-in-native-word $C))
+    (ushr_imm $C $x))
+(=> (when (sshr $x $C)
+      (fits-in-native-word $C))
+    (sshr_imm $C $x))
+(=> (when (isub $x $C)
+      (fits-in-native-word $C))
+    (iadd_imm $(neg $C) $x))
+(=> (when (ifcmp $x $C)
+      (fits-in-native-word $C))
+    (ifcmp_imm $C $x))
+(=> (when (icmp $cond $x $C)
+      (fits-in-native-word $C))
+    (icmp_imm $cond $C $x))
+
+;; Binary instructions whose first operand is constant.
+(=> (when (iadd $C $x)
+      (fits-in-native-word $C))
+    (iadd_imm $C $x))
+(=> (when (imul $C $x)
+      (fits-in-native-word $C))
+    (imul_imm $C $x))
+(=> (when (band $C $x)
+      (fits-in-native-word $C))
+    (band_imm $C $x))
+(=> (when (bor $C $x)
+      (fits-in-native-word $C))
+    (bor_imm $C $x))
+(=> (when (bxor $C $x)
+      (fits-in-native-word $C))
+    (bxor_imm $C $x))
+(=> (when (isub $C $x)
+      (fits-in-native-word $C))
+    (irsub_imm $C $x))
+
+;; Unary instructions whose operand is constant.
+(=> (adjust_sp_down $C) (adjust_sp_down_imm $C))
+
+;; Fold `(binop_imm $C1 (binop_imm $C2 $x))` into `(binop_imm $(binop $C2 $C1) $x)`.
+(=> (iadd_imm $C1 (iadd_imm $C2 $x)) (iadd_imm $(iadd $C1 $C2) $x))
+(=> (imul_imm $C1 (imul_imm $C2 $x)) (imul_imm $(imul $C1 $C2) $x))
+(=> (bor_imm $C1 (bor_imm $C2 $x)) (bor_imm $(bor $C1 $C2) $x))
+(=> (band_imm $C1 (band_imm $C2 $x)) (band_imm $(band $C1 $C2) $x))
+(=> (bxor_imm $C1 (bxor_imm $C2 $x)) (bxor_imm $(bxor $C1 $C2) $x))
+
+;; Remove operations that are no-ops.
+(=> (iadd_imm 0 $x) $x)
+(=> (imul_imm 1 $x) $x)
+(=> (sdiv_imm 1 $x) $x)
+(=> (udiv_imm 1 $x) $x)
+(=> (bor_imm 0 $x) $x)
+(=> (band_imm -1 $x) $x)
+(=> (bxor_imm 0 $x) $x)
+(=> (rotl_imm 0 $x) $x)
+(=> (rotr_imm 0 $x) $x)
+(=> (ishl_imm 0 $x) $x)
+(=> (ushr_imm 0 $x) $x)
+(=> (sshr_imm 0 $x) $x)
+
+;; Replace with zero.
+(=> (imul_imm 0 $x) 0)
+(=> (band_imm 0 $x) 0)
+
+;; Replace with negative 1.
+(=> (bor_imm -1 $x) -1)
+
+;; Transform `[(x << N) >> N]` into a (un)signed-extending move.
+;;
+;; i16 -> i8 -> i16
+(=> (when (ushr_imm 8 (ishl_imm 8 $x))
+      (bit-width $x 16))
+    (uextend{i16} (ireduce{i8} $x)))
+(=> (when (sshr_imm 8 (ishl_imm 8 $x))
+      (bit-width $x 16))
+    (sextend{i16} (ireduce{i8} $x)))
+;; i32 -> i8 -> i32
+(=> (when (ushr_imm 24 (ishl_imm 24 $x))
+      (bit-width $x 32))
+    (uextend{i32} (ireduce{i8} $x)))
+(=> (when (sshr_imm 24 (ishl_imm 24 $x))
+      (bit-width $x 32))
+    (sextend{i32} (ireduce{i8} $x)))
+;; i32 -> i16 -> i32
+(=> (when (ushr_imm 16 (ishl_imm 16 $x))
+      (bit-width $x 32))
+    (uextend{i32} (ireduce{i16} $x)))
+(=> (when (sshr_imm 16 (ishl_imm 16 $x))
+      (bit-width $x 32))
+    (sextend{i32} (ireduce{i16} $x)))
+;; i64 -> i8 -> i64
+(=> (when (ushr_imm 56 (ishl_imm 56 $x))
+      (bit-width $x 64))
+    (uextend{i64} (ireduce{i8} $x)))
+(=> (when (sshr_imm 56 (ishl_imm 56 $x))
+      (bit-width $x 64))
+    (sextend{i64} (ireduce{i8} $x)))
+;; i64 -> i16 -> i64
+(=> (when (ushr_imm 48 (ishl_imm 48 $x))
+      (bit-width $x 64))
+    (uextend{i64} (ireduce{i16} $x)))
+(=> (when (sshr_imm 48 (ishl_imm 48 $x))
+      (bit-width $x 64))
+    (sextend{i64} (ireduce{i16} $x)))
+;; i64 -> i32 -> i64
+(=> (when (ushr_imm 32 (ishl_imm 32 $x))
+      (bit-width $x 64))
+    (uextend{i64} (ireduce{i32} $x)))
+(=> (when (sshr_imm 32 (ishl_imm 32 $x))
+      (bit-width $x 64))
+    (sextend{i64} (ireduce{i32} $x)))
+
+;; Fold away redundant `bint` instructions that accept both integer and boolean
+;; arguments.
+(=> (select (bint $x) $y $z) (select $x $y $z))
+(=> (brz (bint $x)) (brz $x))
+(=> (brnz (bint $x)) (brnz $x))
+(=> (trapz (bint $x)) (trapz $x))
+(=> (trapnz (bint $x)) (trapnz $x))
+
+;; Fold comparisons into branch operations when possible.
+;;
+;; This matches against operations which compare against zero, then use the
+;; result in a `brz` or `brnz` branch. It folds those two operations into a
+;; single `brz` or `brnz`.
+(=> (brnz (icmp_imm ne 0 $x)) (brnz $x))
+(=> (brz (icmp_imm ne 0 $x)) (brz $x))
+(=> (brnz (icmp_imm eq 0 $x)) (brz $x))
+(=> (brz (icmp_imm eq 0 $x)) (brnz $x))
+
+;; Division and remainder by constants.
+;;
+;; TODO: this section is incomplete, and a bunch of related optimizations are
+;; still hand-coded in `simple_preopt.rs`.
+
+;; (Division by one is handled above.)
+
+;; Remainder by one is zero.
+(=> (urem_imm 1 $x) 0)
+(=> (srem_imm 1 $x) 0)
+
+;; Division by a power of two -> shift right.
+(=> (when (udiv_imm $C $x)
+          (is-power-of-two $C))
+    (ushr_imm $(log2 $C) $x))

cranelift/codegen/src/simple_preopt.rs

@@ -10,10 +10,8 @@ use crate::divconst_magic_numbers::{MS32, MS64, MU32, MU64};
 use crate::flowgraph::ControlFlowGraph;
 use crate::ir::{
     condcodes::{CondCode, IntCC},
-    dfg::ValueDef,
+    instructions::Opcode,
-    immediates,
+    types::{I32, I64},
-    instructions::{Opcode, ValueList},
-    types::{I16, I32, I64, I8},
     Block, DataFlowGraph, Function, Inst, InstBuilder, InstructionData, Type, Value,
 };
 use crate::isa::TargetIsa;
@@ -468,340 +466,6 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
     }
 }
 
-#[inline]
-fn resolve_imm64_value(dfg: &DataFlowGraph, value: Value) -> Option<immediates::Imm64> {
-    if let ValueDef::Result(candidate_inst, _) = dfg.value_def(value) {
-        if let InstructionData::UnaryImm {
-            opcode: Opcode::Iconst,
-            imm,
-        } = dfg[candidate_inst]
-        {
-            return Some(imm);
-        }
-    }
-    None
-}
-
-/// Try to transform [(x << N) >> N] into a (un)signed-extending move.
-/// Returns true if the final instruction has been converted to such a move.
-fn try_fold_extended_move(
-    pos: &mut FuncCursor,
-    inst: Inst,
-    opcode: Opcode,
-    arg: Value,
-    imm: immediates::Imm64,
-) -> bool {
-    if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) {
-        if let InstructionData::BinaryImm {
-            opcode: Opcode::IshlImm,
-            arg: prev_arg,
-            imm: prev_imm,
-        } = &pos.func.dfg[arg_inst]
-        {
-            if imm != *prev_imm {
-                return false;
-            }
-
-            let dest_ty = pos.func.dfg.ctrl_typevar(inst);
-            if dest_ty != pos.func.dfg.ctrl_typevar(arg_inst) || !dest_ty.is_int() {
-                return false;
-            }
-
-            let imm_bits: i64 = imm.into();
-            let ireduce_ty = match (dest_ty.lane_bits() as i64).wrapping_sub(imm_bits) {
-                8 => I8,
-                16 => I16,
-                32 => I32,
-                _ => return false,
-            };
-            let ireduce_ty = ireduce_ty.by(dest_ty.lane_count()).unwrap();
-
-            // This becomes a no-op, since ireduce_ty has a smaller lane width than
-            // the argument type (also the destination type).
-            let arg = *prev_arg;
-            let narrower_arg = pos.ins().ireduce(ireduce_ty, arg);
-
-            if opcode == Opcode::UshrImm {
-                pos.func.dfg.replace(inst).uextend(dest_ty, narrower_arg);
-            } else {
-                pos.func.dfg.replace(inst).sextend(dest_ty, narrower_arg);
-            }
-            return true;
-        }
-    }
-    false
-}
-
-/// Apply basic simplifications.
-///
-/// This folds constants with arithmetic to form `_imm` instructions, and other minor
-/// simplifications.
-///
-/// Doesn't apply some simplifications if the native word width (in bytes) is smaller than the
-/// controlling type's width of the instruction. This would result in an illegal instruction that
-/// would likely be expanded back into an instruction on smaller types with the same initial
-/// opcode, creating unnecessary churn.
-fn simplify(pos: &mut FuncCursor, inst: Inst, native_word_width: u32) {
-    match pos.func.dfg[inst] {
-        InstructionData::Binary { opcode, args } => {
-            if let Some(mut imm) = resolve_imm64_value(&pos.func.dfg, args[1]) {
-                let new_opcode = match opcode {
-                    Opcode::Iadd => Opcode::IaddImm,
-                    Opcode::Imul => Opcode::ImulImm,
-                    Opcode::Sdiv => Opcode::SdivImm,
-                    Opcode::Udiv => Opcode::UdivImm,
-                    Opcode::Srem => Opcode::SremImm,
-                    Opcode::Urem => Opcode::UremImm,
-                    Opcode::Band => Opcode::BandImm,
-                    Opcode::Bor => Opcode::BorImm,
-                    Opcode::Bxor => Opcode::BxorImm,
-                    Opcode::Rotl => Opcode::RotlImm,
-                    Opcode::Rotr => Opcode::RotrImm,
-                    Opcode::Ishl => Opcode::IshlImm,
-                    Opcode::Ushr => Opcode::UshrImm,
-                    Opcode::Sshr => Opcode::SshrImm,
-                    Opcode::Isub => {
-                        imm = imm.wrapping_neg();
-                        Opcode::IaddImm
-                    }
-                    Opcode::Ifcmp => Opcode::IfcmpImm,
-                    _ => return,
-                };
-                let ty = pos.func.dfg.ctrl_typevar(inst);
-                if ty.bytes() <= native_word_width {
-                    pos.func
-                        .dfg
-                        .replace(inst)
-                        .BinaryImm(new_opcode, ty, imm, args[0]);
-
-                    // Repeat for BinaryImm simplification.
-                    simplify(pos, inst, native_word_width);
-                }
-            } else if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[0]) {
-                let new_opcode = match opcode {
-                    Opcode::Iadd => Opcode::IaddImm,
-                    Opcode::Imul => Opcode::ImulImm,
-                    Opcode::Band => Opcode::BandImm,
-                    Opcode::Bor => Opcode::BorImm,
-                    Opcode::Bxor => Opcode::BxorImm,
-                    Opcode::Isub => Opcode::IrsubImm,
-                    _ => return,
-                };
-                let ty = pos.func.dfg.ctrl_typevar(inst);
-                if ty.bytes() <= native_word_width {
-                    pos.func
-                        .dfg
-                        .replace(inst)
-                        .BinaryImm(new_opcode, ty, imm, args[1]);
-                }
-            }
-        }
-
-        InstructionData::Unary { opcode, arg } => {
-            if let Opcode::AdjustSpDown = opcode {
-                if let Some(imm) = resolve_imm64_value(&pos.func.dfg, arg) {
-                    // Note this works for both positive and negative immediate values.
-                    pos.func.dfg.replace(inst).adjust_sp_down_imm(imm);
-                }
-            }
-        }
-
-        InstructionData::BinaryImm { opcode, arg, imm } => {
-            let ty = pos.func.dfg.ctrl_typevar(inst);
-
-            let mut arg = arg;
-            let mut imm = imm;
-            match opcode {
-                Opcode::IaddImm
-                | Opcode::ImulImm
-                | Opcode::BorImm
-                | Opcode::BandImm
-                | Opcode::BxorImm => {
-                    // Fold binary_op(C2, binary_op(C1, x)) into binary_op(binary_op(C1, C2), x)
-                    if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) {
-                        if let InstructionData::BinaryImm {
-                            opcode: prev_opcode,
-                            arg: prev_arg,
-                            imm: prev_imm,
-                        } = &pos.func.dfg[arg_inst]
-                        {
-                            if opcode == *prev_opcode && ty == pos.func.dfg.ctrl_typevar(arg_inst) {
-                                let lhs: i64 = imm.into();
-                                let rhs: i64 = (*prev_imm).into();
-                                let new_imm = match opcode {
-                                    Opcode::BorImm => lhs | rhs,
-                                    Opcode::BandImm => lhs & rhs,
-                                    Opcode::BxorImm => lhs ^ rhs,
-                                    Opcode::IaddImm => lhs.wrapping_add(rhs),
-                                    Opcode::ImulImm => lhs.wrapping_mul(rhs),
-                                    _ => panic!("can't happen"),
-                                };
-                                let new_imm = immediates::Imm64::from(new_imm);
-                                let new_arg = *prev_arg;
-                                pos.func
-                                    .dfg
-                                    .replace(inst)
-                                    .BinaryImm(opcode, ty, new_imm, new_arg);
-                                imm = new_imm;
-                                arg = new_arg;
-                            }
-                        }
-                    }
-                }
-
-                Opcode::UshrImm | Opcode::SshrImm => {
-                    if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width
-                        && try_fold_extended_move(pos, inst, opcode, arg, imm)
-                    {
-                        return;
-                    }
-                }
-
-                _ => {}
-            };
-
-            // Replace operations that are no-ops.
-            match (opcode, imm.into()) {
-                (Opcode::IaddImm, 0)
-                | (Opcode::ImulImm, 1)
-                | (Opcode::SdivImm, 1)
-                | (Opcode::UdivImm, 1)
-                | (Opcode::BorImm, 0)
-                | (Opcode::BandImm, -1)
-                | (Opcode::BxorImm, 0)
-                | (Opcode::RotlImm, 0)
-                | (Opcode::RotrImm, 0)
-                | (Opcode::IshlImm, 0)
-                | (Opcode::UshrImm, 0)
-                | (Opcode::SshrImm, 0) => {
-                    // Alias the result value with the original argument.
-                    replace_single_result_with_alias(&mut pos.func.dfg, inst, arg);
-                }
-                (Opcode::ImulImm, 0) | (Opcode::BandImm, 0) => {
-                    // Replace by zero.
-                    pos.func.dfg.replace(inst).iconst(ty, 0);
-                }
-                (Opcode::BorImm, -1) => {
-                    // Replace by minus one.
-                    pos.func.dfg.replace(inst).iconst(ty, -1);
-                }
-                _ => {}
-            }
-        }
-
-        InstructionData::IntCompare { opcode, cond, args } => {
-            debug_assert_eq!(opcode, Opcode::Icmp);
-            if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[1]) {
-                if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width {
-                    pos.func.dfg.replace(inst).icmp_imm(cond, args[0], imm);
-                }
-            }
-        }
-
-        InstructionData::CondTrap { .. }
-        | InstructionData::Branch { .. }
-        | InstructionData::Ternary {
-            opcode: Opcode::Select,
-            ..
-        } => {
-            // Fold away a redundant `bint`.
-            let condition_def = {
-                let args = pos.func.dfg.inst_args(inst);
-                pos.func.dfg.value_def(args[0])
-            };
-            if let ValueDef::Result(def_inst, _) = condition_def {
-                if let InstructionData::Unary {
-                    opcode: Opcode::Bint,
-                    arg: bool_val,
-                } = pos.func.dfg[def_inst]
-                {
-                    let args = pos.func.dfg.inst_args_mut(inst);
-                    args[0] = bool_val;
-                }
-            }
-        }
-
-        _ => {}
-    }
-}
-
-struct BranchOptInfo {
-    br_inst: Inst,
-    cmp_arg: Value,
-    args: ValueList,
-    new_opcode: Opcode,
-}
-
-/// Fold comparisons into branch operations when possible.
-///
-/// This matches against operations which compare against zero, then use the
-/// result in a `brz` or `brnz` branch. It folds those two operations into a
-/// single `brz` or `brnz`.
-fn branch_opt(pos: &mut FuncCursor, inst: Inst) {
-    let mut info = if let InstructionData::Branch {
-        opcode: br_opcode,
-        args: ref br_args,
-        ..
-    } = pos.func.dfg[inst]
-    {
-        let first_arg = {
-            let args = pos.func.dfg.inst_args(inst);
-            args[0]
-        };
-
-        let icmp_inst = if let ValueDef::Result(icmp_inst, _) = pos.func.dfg.value_def(first_arg) {
-            icmp_inst
-        } else {
-            return;
-        };
-
-        if let InstructionData::IntCompareImm {
-            opcode: Opcode::IcmpImm,
-            arg: cmp_arg,
-            cond: cmp_cond,
-            imm: cmp_imm,
-        } = pos.func.dfg[icmp_inst]
-        {
-            let cmp_imm: i64 = cmp_imm.into();
-            if cmp_imm != 0 {
-                return;
-            }
-
-            // icmp_imm returns non-zero when the comparison is true. So, if
-            // we're branching on zero, we need to invert the condition.
-            let cond = match br_opcode {
-                Opcode::Brz => cmp_cond.inverse(),
-                Opcode::Brnz => cmp_cond,
-                _ => return,
-            };
-
-            let new_opcode = match cond {
-                IntCC::Equal => Opcode::Brz,
-                IntCC::NotEqual => Opcode::Brnz,
-                _ => return,
-            };
-
-            BranchOptInfo {
-                br_inst: inst,
-                cmp_arg,
-                args: br_args.clone(),
-                new_opcode,
-            }
-        } else {
-            return;
-        }
-    } else {
-        return;
-    };
-
-    info.args.as_mut_slice(&mut pos.func.dfg.value_lists)[0] = info.cmp_arg;
-    if let InstructionData::Branch { ref mut opcode, .. } = pos.func.dfg[info.br_inst] {
-        *opcode = info.new_opcode;
-    } else {
-        panic!();
-    }
-}
-
 enum BranchOrderKind {
     BrzToBrnz(Value),
     BrnzToBrz(Value),
@@ -944,15 +608,427 @@ fn branch_order(pos: &mut FuncCursor, cfg: &mut ControlFlowGraph, block: Block,
     cfg.recompute_block(pos.func, block);
 }
 
+#[cfg(feature = "enable-peepmatic")]
+mod simplify {
+    use super::*;
+    use crate::peepmatic::ValueOrInst;
+
+    pub type PeepholeOptimizer<'a, 'b> =
+        peepmatic_runtime::optimizer::PeepholeOptimizer<'static, 'a, &'b dyn TargetIsa>;
+
+    pub fn peephole_optimizer<'a, 'b>(isa: &'b dyn TargetIsa) -> PeepholeOptimizer<'a, 'b> {
+        crate::peepmatic::preopt(isa)
+    }
+
+    pub fn apply_all<'a, 'b>(
+        optimizer: &mut PeepholeOptimizer<'a, 'b>,
+        pos: &mut FuncCursor<'a>,
+        inst: Inst,
+        _native_word_width: u32,
+    ) {
+        // After we apply one optimization, that might make another
+        // optimization applicable. Keep running the peephole optimizer
+        // until either:
+        //
+        // * No optimization applied, and therefore it doesn't make sense to
+        //   try again, because no optimization will apply again.
+        //
+        // * Or when we replaced an instruction with an alias to an existing
+        //   value, because we already ran the peephole optimizer over the
+        //   aliased value's instruction in an early part of the traversal
+        //   over the function.
+        while let Some(ValueOrInst::Inst(new_inst)) =
+            optimizer.apply_one(pos, ValueOrInst::Inst(inst))
+        {
+            // We transplanted a new instruction into the current
+            // instruction, so the "new" instruction is actually the same
+            // one, just with different data.
+            debug_assert_eq!(new_inst, inst);
+        }
+        debug_assert_eq!(pos.current_inst(), Some(inst));
+    }
+}
+
+#[cfg(not(feature = "enable-peepmatic"))]
+mod simplify {
+    use super::*;
+    use crate::ir::{
+        dfg::ValueDef,
+        immediates,
+        instructions::{Opcode, ValueList},
+        types::{I16, I32, I8},
+    };
+    use std::marker::PhantomData;
+
+    pub struct PeepholeOptimizer<'a, 'b> {
+        phantom: PhantomData<(&'a (), &'b ())>,
+    }
+
+    pub fn peephole_optimizer<'a, 'b>(_: &dyn TargetIsa) -> PeepholeOptimizer<'a, 'b> {
+        PeepholeOptimizer {
+            phantom: PhantomData,
+        }
+    }
+
+    pub fn apply_all<'a, 'b>(
+        _optimizer: &mut PeepholeOptimizer<'a, 'b>,
+        pos: &mut FuncCursor<'a>,
+        inst: Inst,
+        native_word_width: u32,
+    ) {
+        simplify(pos, inst, native_word_width);
+        branch_opt(pos, inst);
+    }
+
+    #[inline]
+    fn resolve_imm64_value(dfg: &DataFlowGraph, value: Value) -> Option<immediates::Imm64> {
+        if let ValueDef::Result(candidate_inst, _) = dfg.value_def(value) {
+            if let InstructionData::UnaryImm {
+                opcode: Opcode::Iconst,
+                imm,
+            } = dfg[candidate_inst]
+            {
+                return Some(imm);
+            }
+        }
+        None
+    }
+
+    /// Try to transform [(x << N) >> N] into a (un)signed-extending move.
+    /// Returns true if the final instruction has been converted to such a move.
+    fn try_fold_extended_move(
+        pos: &mut FuncCursor,
+        inst: Inst,
+        opcode: Opcode,
+        arg: Value,
+        imm: immediates::Imm64,
+    ) -> bool {
+        if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) {
+            if let InstructionData::BinaryImm {
+                opcode: Opcode::IshlImm,
+                arg: prev_arg,
+                imm: prev_imm,
+            } = &pos.func.dfg[arg_inst]
+            {
+                if imm != *prev_imm {
+                    return false;
+                }
+
+                let dest_ty = pos.func.dfg.ctrl_typevar(inst);
+                if dest_ty != pos.func.dfg.ctrl_typevar(arg_inst) || !dest_ty.is_int() {
+                    return false;
+                }
+
+                let imm_bits: i64 = imm.into();
+                let ireduce_ty = match (dest_ty.lane_bits() as i64).wrapping_sub(imm_bits) {
+                    8 => I8,
+                    16 => I16,
+                    32 => I32,
+                    _ => return false,
+                };
+                let ireduce_ty = ireduce_ty.by(dest_ty.lane_count()).unwrap();
+
+                // This becomes a no-op, since ireduce_ty has a smaller lane width than
+                // the argument type (also the destination type).
+                let arg = *prev_arg;
+                let narrower_arg = pos.ins().ireduce(ireduce_ty, arg);
+
+                if opcode == Opcode::UshrImm {
+                    pos.func.dfg.replace(inst).uextend(dest_ty, narrower_arg);
+                } else {
+                    pos.func.dfg.replace(inst).sextend(dest_ty, narrower_arg);
+                }
+                return true;
+            }
+        }
+        false
+    }
+
+    /// Apply basic simplifications.
+    ///
+    /// This folds constants with arithmetic to form `_imm` instructions, and other minor
+    /// simplifications.
+    ///
+    /// Doesn't apply some simplifications if the native word width (in bytes) is smaller than the
+    /// controlling type's width of the instruction. This would result in an illegal instruction that
+    /// would likely be expanded back into an instruction on smaller types with the same initial
+    /// opcode, creating unnecessary churn.
+    fn simplify(pos: &mut FuncCursor, inst: Inst, native_word_width: u32) {
+        match pos.func.dfg[inst] {
+            InstructionData::Binary { opcode, args } => {
+                if let Some(mut imm) = resolve_imm64_value(&pos.func.dfg, args[1]) {
+                    let new_opcode = match opcode {
+                        Opcode::Iadd => Opcode::IaddImm,
+                        Opcode::Imul => Opcode::ImulImm,
+                        Opcode::Sdiv => Opcode::SdivImm,
+                        Opcode::Udiv => Opcode::UdivImm,
+                        Opcode::Srem => Opcode::SremImm,
+                        Opcode::Urem => Opcode::UremImm,
+                        Opcode::Band => Opcode::BandImm,
+                        Opcode::Bor => Opcode::BorImm,
+                        Opcode::Bxor => Opcode::BxorImm,
+                        Opcode::Rotl => Opcode::RotlImm,
+                        Opcode::Rotr => Opcode::RotrImm,
+                        Opcode::Ishl => Opcode::IshlImm,
+                        Opcode::Ushr => Opcode::UshrImm,
+                        Opcode::Sshr => Opcode::SshrImm,
+                        Opcode::Isub => {
+                            imm = imm.wrapping_neg();
+                            Opcode::IaddImm
+                        }
+                        Opcode::Ifcmp => Opcode::IfcmpImm,
+                        _ => return,
+                    };
+                    let ty = pos.func.dfg.ctrl_typevar(inst);
+                    if ty.bytes() <= native_word_width {
+                        pos.func
+                            .dfg
+                            .replace(inst)
+                            .BinaryImm(new_opcode, ty, imm, args[0]);
+
+                        // Repeat for BinaryImm simplification.
+                        simplify(pos, inst, native_word_width);
+                    }
+                } else if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[0]) {
+                    let new_opcode = match opcode {
+                        Opcode::Iadd => Opcode::IaddImm,
+                        Opcode::Imul => Opcode::ImulImm,
+                        Opcode::Band => Opcode::BandImm,
+                        Opcode::Bor => Opcode::BorImm,
+                        Opcode::Bxor => Opcode::BxorImm,
+                        Opcode::Isub => Opcode::IrsubImm,
+                        _ => return,
+                    };
+                    let ty = pos.func.dfg.ctrl_typevar(inst);
+                    if ty.bytes() <= native_word_width {
+                        pos.func
+                            .dfg
+                            .replace(inst)
+                            .BinaryImm(new_opcode, ty, imm, args[1]);
+                    }
+                }
+            }
+
+            InstructionData::Unary { opcode, arg } => {
+                if let Opcode::AdjustSpDown = opcode {
+                    if let Some(imm) = resolve_imm64_value(&pos.func.dfg, arg) {
+                        // Note this works for both positive and negative immediate values.
+                        pos.func.dfg.replace(inst).adjust_sp_down_imm(imm);
+                    }
+                }
+            }
+
+            InstructionData::BinaryImm { opcode, arg, imm } => {
+                let ty = pos.func.dfg.ctrl_typevar(inst);
+
+                let mut arg = arg;
+                let mut imm = imm;
+                match opcode {
+                    Opcode::IaddImm
+                    | Opcode::ImulImm
+                    | Opcode::BorImm
+                    | Opcode::BandImm
+                    | Opcode::BxorImm => {
+                        // Fold binary_op(C2, binary_op(C1, x)) into binary_op(binary_op(C1, C2), x)
+                        if let ValueDef::Result(arg_inst, _) = pos.func.dfg.value_def(arg) {
+                            if let InstructionData::BinaryImm {
+                                opcode: prev_opcode,
+                                arg: prev_arg,
+                                imm: prev_imm,
+                            } = &pos.func.dfg[arg_inst]
+                            {
+                                if opcode == *prev_opcode
+                                    && ty == pos.func.dfg.ctrl_typevar(arg_inst)
+                                {
+                                    let lhs: i64 = imm.into();
+                                    let rhs: i64 = (*prev_imm).into();
+                                    let new_imm = match opcode {
+                                        Opcode::BorImm => lhs | rhs,
+                                        Opcode::BandImm => lhs & rhs,
+                                        Opcode::BxorImm => lhs ^ rhs,
+                                        Opcode::IaddImm => lhs.wrapping_add(rhs),
+                                        Opcode::ImulImm => lhs.wrapping_mul(rhs),
+                                        _ => panic!("can't happen"),
+                                    };
+                                    let new_imm = immediates::Imm64::from(new_imm);
+                                    let new_arg = *prev_arg;
+                                    pos.func
+                                        .dfg
+                                        .replace(inst)
+                                        .BinaryImm(opcode, ty, new_imm, new_arg);
+                                    imm = new_imm;
+                                    arg = new_arg;
+                                }
+                            }
+                        }
+                    }
+
+                    Opcode::UshrImm | Opcode::SshrImm => {
+                        if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width
+                            && try_fold_extended_move(pos, inst, opcode, arg, imm)
+                        {
+                            return;
+                        }
+                    }
+
+                    _ => {}
+                };
+
+                // Replace operations that are no-ops.
+                match (opcode, imm.into()) {
+                    (Opcode::IaddImm, 0)
+                    | (Opcode::ImulImm, 1)
+                    | (Opcode::SdivImm, 1)
+                    | (Opcode::UdivImm, 1)
+                    | (Opcode::BorImm, 0)
+                    | (Opcode::BandImm, -1)
+                    | (Opcode::BxorImm, 0)
+                    | (Opcode::RotlImm, 0)
+                    | (Opcode::RotrImm, 0)
+                    | (Opcode::IshlImm, 0)
+                    | (Opcode::UshrImm, 0)
+                    | (Opcode::SshrImm, 0) => {
+                        // Alias the result value with the original argument.
+                        replace_single_result_with_alias(&mut pos.func.dfg, inst, arg);
+                    }
+                    (Opcode::ImulImm, 0) | (Opcode::BandImm, 0) => {
+                        // Replace by zero.
+                        pos.func.dfg.replace(inst).iconst(ty, 0);
+                    }
+                    (Opcode::BorImm, -1) => {
+                        // Replace by minus one.
+                        pos.func.dfg.replace(inst).iconst(ty, -1);
+                    }
+                    _ => {}
+                }
+            }
+
+            InstructionData::IntCompare { opcode, cond, args } => {
+                debug_assert_eq!(opcode, Opcode::Icmp);
+                if let Some(imm) = resolve_imm64_value(&pos.func.dfg, args[1]) {
+                    if pos.func.dfg.ctrl_typevar(inst).bytes() <= native_word_width {
+                        pos.func.dfg.replace(inst).icmp_imm(cond, args[0], imm);
+                    }
+                }
+            }
+
+            InstructionData::CondTrap { .. }
+            | InstructionData::Branch { .. }
+            | InstructionData::Ternary {
+                opcode: Opcode::Select,
+                ..
+            } => {
+                // Fold away a redundant `bint`.
+                let condition_def = {
+                    let args = pos.func.dfg.inst_args(inst);
+                    pos.func.dfg.value_def(args[0])
+                };
+                if let ValueDef::Result(def_inst, _) = condition_def {
+                    if let InstructionData::Unary {
+                        opcode: Opcode::Bint,
+                        arg: bool_val,
+                    } = pos.func.dfg[def_inst]
+                    {
+                        let args = pos.func.dfg.inst_args_mut(inst);
+                        args[0] = bool_val;
+                    }
+                }
+            }
+
+            _ => {}
+        }
+    }
+
+    struct BranchOptInfo {
+        br_inst: Inst,
+        cmp_arg: Value,
+        args: ValueList,
+        new_opcode: Opcode,
+    }
+
+    /// Fold comparisons into branch operations when possible.
+    ///
+    /// This matches against operations which compare against zero, then use the
+    /// result in a `brz` or `brnz` branch. It folds those two operations into a
+    /// single `brz` or `brnz`.
+    fn branch_opt(pos: &mut FuncCursor, inst: Inst) {
+        let mut info = if let InstructionData::Branch {
+            opcode: br_opcode,
+            args: ref br_args,
+            ..
+        } = pos.func.dfg[inst]
+        {
+            let first_arg = {
+                let args = pos.func.dfg.inst_args(inst);
+                args[0]
+            };
+
+            let icmp_inst =
+                if let ValueDef::Result(icmp_inst, _) = pos.func.dfg.value_def(first_arg) {
+                    icmp_inst
+                } else {
+                    return;
+                };
+
+            if let InstructionData::IntCompareImm {
+                opcode: Opcode::IcmpImm,
+                arg: cmp_arg,
+                cond: cmp_cond,
+                imm: cmp_imm,
+            } = pos.func.dfg[icmp_inst]
+            {
+                let cmp_imm: i64 = cmp_imm.into();
+                if cmp_imm != 0 {
+                    return;
+                }
+
+                // icmp_imm returns non-zero when the comparison is true. So, if
+                // we're branching on zero, we need to invert the condition.
+                let cond = match br_opcode {
+                    Opcode::Brz => cmp_cond.inverse(),
+                    Opcode::Brnz => cmp_cond,
+                    _ => return,
+                };
+
+                let new_opcode = match cond {
+                    IntCC::Equal => Opcode::Brz,
+                    IntCC::NotEqual => Opcode::Brnz,
+                    _ => return,
+                };
+
+                BranchOptInfo {
+                    br_inst: inst,
+                    cmp_arg,
+                    args: br_args.clone(),
+                    new_opcode,
+                }
+            } else {
+                return;
+            }
+        } else {
+            return;
+        };
+
+        info.args.as_mut_slice(&mut pos.func.dfg.value_lists)[0] = info.cmp_arg;
+        if let InstructionData::Branch { ref mut opcode, .. } = pos.func.dfg[info.br_inst] {
+            *opcode = info.new_opcode;
+        } else {
+            panic!();
+        }
+    }
+}
+
 /// The main pre-opt pass.
 pub fn do_preopt(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &dyn TargetIsa) {
     let _tt = timing::preopt();
+
     let mut pos = FuncCursor::new(func);
-    let native_word_width = isa.pointer_bytes();
+    let native_word_width = isa.pointer_bytes() as u32;
+    let mut optimizer = simplify::peephole_optimizer(isa);
+
     while let Some(block) = pos.next_block() {
         while let Some(inst) = pos.next_inst() {
-            // Apply basic simplifications.
+            simplify::apply_all(&mut optimizer, &mut pos, inst, native_word_width);
-            simplify(&mut pos, inst, native_word_width as u32);
 
             // Try to transform divide-by-constant into simpler operations.
             if let Some(divrem_info) = get_div_info(inst, &pos.func.dfg) {
@@ -960,7 +1036,6 @@ pub fn do_preopt(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &dyn Targ
                 continue;
             }
 
-            branch_opt(&mut pos, inst);
             branch_order(&mut pos, cfg, block, inst);
         }
     }

cranelift/filetests/Cargo.toml

@@ -26,3 +26,6 @@ num_cpus = "1.8.0"
 region = "2.1.2"
 target-lexicon = "0.10"
 thiserror = "1.0.15"
+
+[features]
+enable-peepmatic = []

cranelift/filetests/filetests/isa/x86/isub_imm-i8.clif

@@ -6,9 +6,9 @@ function u0:0(i8) -> i8 fast {
 block0(v0: i8):
     v1 = iconst.i8 0
     v2 = isub v1, v0
-    ; check: v3 = uextend.i32 v0
+    ; check:  uextend.i32
-    ; nextln: v5 = iconst.i32 0
+    ; nextln: iconst.i32
-    ; nextln = isub v5, v3
+    ; nextln: isub
-    ; nextln = ireduce.i8 v4
+    ; nextln: ireduce.i8
     return v2
 }

cranelift/filetests/filetests/peepmatic/branch.clif

@@ -0,0 +1,81 @@
+test peepmatic
+target x86_64
+
+function %icmp_to_brz_fold(i32) -> i32 {
+block0(v0: i32):
+    v1 = icmp_imm eq v0, 0
+    brnz v1, block1
+    jump block2
+block1:
+    v3 = iconst.i32 1
+    return v3
+block2:
+    v4 = iconst.i32 2
+    return v4
+}
+; sameln: function %icmp_to_brz_fold
+; nextln: block0(v0: i32):
+; nextln:     v1 = icmp_imm eq v0, 0
+; nextln:     brnz v0, block2
+; nextln:     jump block1
+; nextln: 
+; nextln: block1:
+; nextln:     v3 = iconst.i32 1
+; nextln:     return v3
+; nextln: 
+; nextln: block2:
+; nextln:     v4 = iconst.i32 2
+; nextln:     return v4
+; nextln: }
+
+function %icmp_to_brz_inverted_fold(i32) -> i32 {
+block0(v0: i32):
+    v1 = icmp_imm ne v0, 0
+    brz v1, block1
+    jump block2
+block1:
+    v3 = iconst.i32 1
+    return v3
+block2:
+    v4 = iconst.i32 2
+    return v4
+}
+; sameln: function %icmp_to_brz_inve
+; nextln: block0(v0: i32):
+; nextln:     v1 = icmp_imm ne v0, 0
+; nextln:     brnz v0, block2
+; nextln:     jump block1
+; nextln: 
+; nextln: block1:
+; nextln:     v3 = iconst.i32 1
+; nextln:     return v3
+; nextln: 
+; nextln: block2:
+; nextln:     v4 = iconst.i32 2
+; nextln:     return v4
+; nextln: }
+
+function %br_icmp_inversion(i32, i32) -> i32 {
+block0(v0: i32, v1: i32):
+    br_icmp ugt v0, v1, block1
+    jump block2
+block1:
+    v2 = iconst.i32 1
+    return v2
+block2:
+    v3 = iconst.i32 2
+    return v3
+}
+; sameln: function %br_icmp_inversio
+; nextln: block0(v0: i32, v1: i32):
+; nextln:     br_icmp ule v0, v1, block2
+; nextln:     jump block1
+; nextln: 
+; nextln: block1:
+; nextln:     v2 = iconst.i32 1
+; nextln:     return v2
+; nextln: 
+; nextln: block2:
+; nextln:     v3 = iconst.i32 2
+; nextln:     return v3
+; nextln: }

cranelift/filetests/filetests/peepmatic/div_by_const_indirect.clif

@@ -0,0 +1,55 @@
+test peepmatic
+target x86_64 baseline
+
+; Cases where the denominator is created by an iconst
+
+function %indir_udiv32(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 7
+    v2 = udiv v0, v1
+    ; check:  v4 = iconst.i32 0x2492_4925
+    ; nextln: v5 = umulhi v0, v4
+    ; nextln: v6 = isub v0, v5
+    ; nextln: v7 = ushr_imm v6, 1
+    ; nextln: v8 = iadd v7, v5
+    ; nextln: v9 = ushr_imm v8, 2
+    ; nextln: v2 -> v9
+    return v2
+}
+
+function %indir_sdiv32(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 -17
+    v2 = sdiv v0, v1
+    ; check:  v4 = iconst.i32 0xffff_ffff_8787_8787
+    ; nextln: v5 = smulhi v0, v4
+    ; nextln: v6 = sshr_imm v5, 3
+    ; nextln: v7 = ushr_imm v6, 31
+    ; nextln: v8 = iadd v6, v7
+    ; nextln: v2 -> v8
+    return v2
+}
+
+function %indir_udiv64(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 1337
+    v2 = udiv v0, v1
+    ; check:  v4 = iconst.i64 0xc411_9d95_2866_a139
+    ; nextln: v5 = umulhi v0, v4
+    ; nextln: v6 = ushr_imm v5, 10
+    ; nextln: v2 -> v6
+    return v2
+}
+
+function %indir_sdiv64(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 -90210
+    v2 = sdiv v0, v1
+    ; check:  v4 = iconst.i64 0xd181_4ee8_939c_b8bb
+    ; nextln: v5 = smulhi v0, v4
+    ; nextln: v6 = sshr_imm v5, 14
+    ; nextln: v7 = ushr_imm v6, 63
+    ; nextln: v8 = iadd v6, v7
+    ; nextln: v2 -> v8
+    return v2
+}

cranelift/filetests/filetests/peepmatic/div_by_const_non_power_of_2.clif

@@ -0,0 +1,266 @@
+test peepmatic
+target i686 baseline
+
+; -------- U32 --------
+
+; complex case (mul, sub, shift, add, shift)
+function %t_udiv32_p7(i32) -> i32 {
+block0(v0: i32):
+    v1 = udiv_imm v0, 7
+    ; check: iconst.i32 0x2492_4925
+    ; check: umulhi v0, v2
+    ; check: isub v0, v3
+    ; check: ushr_imm v4, 1
+    ; check: iadd v5, v3
+    ; check: v7 = ushr_imm v6, 2
+    ; check: v1 -> v7
+    return v1
+}
+
+; simple case (mul, shift)
+function %t_udiv32_p125(i32) -> i32 {
+block0(v0: i32):
+    v1 = udiv_imm v0, 125
+    ; check: iconst.i32 0x1062_4dd3
+    ; check: umulhi v0, v2
+    ; check: v4 = ushr_imm v3, 3
+    ; check: v1 -> v4
+    return v1
+}
+
+; simple case w/ shift by zero (mul)
+function %t_udiv32_p641(i32) -> i32 {
+block0(v0: i32):
+    v1 = udiv_imm v0, 641
+    ; check: iconst.i32 0x0066_3d81
+    ; check: v3 = umulhi v0, v2
+    ; check: v1 -> v3
+    return v1
+}
+
+
+; -------- S32 --------
+
+; simple case w/ shift by zero (mul, add-sign-bit)
+function %t_sdiv32_n6(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, -6
+    ; check: iconst.i32 0xffff_ffff_d555_5555
+    ; check: smulhi v0, v2
+    ; check: ushr_imm v3, 31
+    ; check: v5 = iadd v3, v4
+    ; check: v1 -> v5
+    return v1
+}
+
+; simple case (mul, shift, add-sign-bit)
+function %t_sdiv32_n5(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, -5
+    ; check: iconst.i32 0xffff_ffff_9999_9999
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 1
+    ; check: ushr_imm v4, 31
+    ; check: v6 = iadd v4, v5
+    ; check: v1 -> v6
+    return v1
+}
+
+; case d < 0 && M > 0 (mul, sub, shift, add-sign-bit)
+function %t_sdiv32_n3(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, -3
+    ; check: iconst.i32 0x5555_5555
+    ; check: smulhi v0, v2
+    ; check: isub v3, v0
+    ; check: sshr_imm v4, 1
+    ; check: ushr_imm v5, 31
+    ; check: v7 = iadd v5, v6
+    ; check: v1 -> v7
+    return v1
+}
+
+; simple case w/ shift by zero (mul, add-sign-bit)
+function %t_sdiv32_p6(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, 6
+    ; check: iconst.i32 0x2aaa_aaab
+    ; check: smulhi v0, v2
+    ; check: ushr_imm v3, 31
+    ; check: v5 = iadd v3, v4
+    ; check: v1 -> v5
+    return v1
+}
+
+; case d > 0 && M < 0 (mull, add, shift, add-sign-bit)
+function %t_sdiv32_p7(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, 7
+    ; check: iconst.i32 0xffff_ffff_9249_2493
+    ; check: smulhi v0, v2
+    ; check: iadd v3, v0
+    ; check: sshr_imm v4, 2
+    ; check: ushr_imm v5, 31
+    ; check: v7 = iadd v5, v6
+    ; check: v1 -> v7
+    return v1
+}
+
+; simple case (mul, shift, add-sign-bit)
+function %t_sdiv32_p625(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, 625
+    ; check: iconst.i32 0x68db_8bad
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 8
+    ; check: ushr_imm v4, 31
+    ; check: v6 = iadd v4, v5
+    ; check: v1 -> v6
+    return v1
+}
+
+
+; -------- U64 --------
+
+; complex case (mul, sub, shift, add, shift)
+function %t_udiv64_p7(i64) -> i64 {
+block0(v0: i64):
+    v1 = udiv_imm v0, 7
+    ; check: iconst.i64 0x2492_4924_9249_2493
+    ; check: umulhi v0, v2
+    ; check: isub v0, v3
+    ; check: ushr_imm v4, 1
+    ; check: iadd v5, v3
+    ; check: v7 = ushr_imm v6, 2
+    ; check: v1 -> v7
+    return v1
+}
+
+; simple case (mul, shift)
+function %t_udiv64_p9(i64) -> i64 {
+block0(v0: i64):
+    v1 = udiv_imm v0, 9
+    ; check: iconst.i64 0xe38e_38e3_8e38_e38f
+    ; check: umulhi v0, v2
+    ; check: v4 = ushr_imm v3, 3
+    ; check: v1 -> v4
+    return v1
+}
+
+; complex case (mul, sub, shift, add, shift)
+function %t_udiv64_p125(i64) -> i64 {
+block0(v0: i64):
+    v1 = udiv_imm v0, 125
+    ; check: iconst.i64 0x0624_dd2f_1a9f_be77
+    ; check: umulhi v0, v2
+    ; check: isub v0, v3
+    ; check: ushr_imm v4, 1
+    ; check: iadd v5, v3
+    ; check: v7 = ushr_imm v6, 6
+    ; check: v1 -> v7
+    return v1
+}
+
+; simple case w/ shift by zero (mul)
+function %t_udiv64_p274177(i64) -> i64 {
+block0(v0: i64):
+    v1 = udiv_imm v0, 274177
+    ; check: iconst.i64 0x3d30_f19c_d101
+    ; check: v3 = umulhi v0, v2
+    ; check: v1 -> v3
+    return v1
+}
+
+
+; -------- S64 --------
+
+; simple case (mul, shift, add-sign-bit)
+function %t_sdiv64_n625(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, -625
+    ; check: iconst.i64 0xcb92_3a29_c779_a6b5
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 7
+    ; check: ushr_imm v4, 63
+    ; check: v6 = iadd v4, v5
+    ; check: v1 -> v6
+    return v1
+}
+
+; simple case w/ zero shift (mul, add-sign-bit)
+function %t_sdiv64_n6(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, -6
+    ; check: iconst.i64 0xd555_5555_5555_5555
+    ; check: smulhi v0, v2
+    ; check: ushr_imm v3, 63
+    ; check: v5 = iadd v3, v4
+    ; check: v1 -> v5
+    return v1
+}
+
+; simple case w/ zero shift (mul, add-sign-bit)
+function %t_sdiv64_n5(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, -5
+    ; check: iconst.i64 0x9999_9999_9999_9999
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 1
+    ; check: ushr_imm v4, 63
+    ; check: v6 = iadd v4, v5
+    ; check: v1 -> v6
+    return v1
+}
+
+; case d < 0 && M > 0 (mul, sub, shift, add-sign-bit)
+function %t_sdiv64_n3(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, -3
+    ; check: iconst.i64 0x5555_5555_5555_5555
+    ; check: smulhi v0, v2
+    ; check: isub v3, v0
+    ; check: sshr_imm v4, 1
+    ; check: ushr_imm v5, 63
+    ; check: v7 = iadd v5, v6
+    ; check: v1 -> v7
+    return v1
+}
+
+; simple case w/ zero shift (mul, add-sign-bit)
+function %t_sdiv64_p6(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, 6
+    ; check: iconst.i64 0x2aaa_aaaa_aaaa_aaab
+    ; check: smulhi v0, v2
+    ; check: ushr_imm v3, 63
+    ; check: v5 = iadd v3, v4
+    ; check: v1 -> v5
+    return v1
+}
+
+; case d > 0 && M < 0 (mul, add, shift, add-sign-bit)
+function %t_sdiv64_p15(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, 15
+    ; check: iconst.i64 0x8888_8888_8888_8889
+    ; check: smulhi v0, v2
+    ; check: iadd v3, v0
+    ; check: sshr_imm v4, 3
+    ; check: ushr_imm v5, 63
+    ; check: v7 = iadd v5, v6
+    ; check: v1 -> v7
+    return v1
+}
+
+; simple case (mul, shift, add-sign-bit)
+function %t_sdiv64_p625(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, 625
+    ; check: iconst.i64 0x346d_c5d6_3886_594b
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 7
+    ; check: ushr_imm v4, 63
+    ; check: v6 = iadd v4, v5
+    ; check: v1 -> v6
+    return v1
+}

cranelift/filetests/filetests/peepmatic/div_by_const_power_of_2.clif

@@ -0,0 +1,292 @@
+test peepmatic
+target i686 baseline
+
+; -------- U32 --------
+
+; ignored
+function %t_udiv32_p0(i32) -> i32 {
+block0(v0: i32):
+    v1 = udiv_imm v0, 0
+    ; check: udiv_imm v0, 0
+    return v1
+}
+
+; converted to a nop
+function %t_udiv32_p1(i32) -> i32 {
+block0(v0: i32):
+    v1 = udiv_imm v0, 1
+    ; check: nop
+    return v1
+}
+
+; shift
+function %t_udiv32_p2(i32) -> i32 {
+block0(v0: i32):
+    v1 = udiv_imm v0, 2
+    ; check: ushr_imm v0, 1
+    return v1
+}
+
+; shift
+function %t_udiv32_p2p31(i32) -> i32 {
+block0(v0: i32):
+    v1 = udiv_imm v0, 0x8000_0000
+    ; check: ushr_imm v0, 31
+    return v1
+}
+
+
+; -------- U64 --------
+
+; ignored
+function %t_udiv64_p0(i64) -> i64 {
+block0(v0: i64):
+    v1 = udiv_imm v0, 0
+    ; check: udiv_imm v0, 0
+    return v1
+}
+
+; converted to a nop
+function %t_udiv64_p1(i64) -> i64 {
+block0(v0: i64):
+    v1 = udiv_imm v0, 1
+    ; check: nop
+    return v1
+}
+
+; shift
+function %t_udiv64_p2(i64) -> i64 {
+block0(v0: i64):
+    v1 = udiv_imm v0, 2
+    ; check: ushr_imm v0, 1
+    return v1
+}
+
+; shift
+function %t_udiv64_p2p63(i64) -> i64 {
+block0(v0: i64):
+    v1 = udiv_imm v0, 0x8000_0000_0000_0000
+    ; check: ushr_imm v0, 63
+    return v1
+}
+
+
+; -------- S32 --------
+
+; ignored
+function %t_sdiv32_p0(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, 0
+    ; check: sdiv_imm v0, 0
+    return v1
+}
+
+; converted to a nop
+function %t_sdiv32_p1(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, 1
+    ; check: nop
+    return v1
+}
+
+; ignored
+function %t_sdiv32_n1(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, -1
+    ; check: sdiv_imm v0, -1
+    return v1
+}
+
+; shift
+function %t_sdiv32_p2(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, 2
+    ; check: ushr_imm v0, 31
+    ; check: iadd v0, v2
+    ; check: sshr_imm v3, 1
+    ; check: v1 -> v4
+    return v1
+}
+
+; shift
+function %t_sdiv32_n2(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, -2
+    ; check: ushr_imm v0, 31
+    ; check: iadd v0, v2
+    ; check: sshr_imm v3, 1
+    ; check: irsub_imm v4, 0
+    return v1
+}
+
+; shift
+function %t_sdiv32_p4(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, 4
+    ; check: v2 = sshr_imm v0, 1
+    ; check: ushr_imm v2, 30
+    ; check: iadd v0, v3
+    ; check: v5 = sshr_imm v4, 2
+    ; check: v1 -> v5
+
+    return v1
+}
+
+; shift
+function %t_sdiv32_n4(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, -4
+    ; check: sshr_imm v0, 1
+    ; check: ushr_imm v2, 30
+    ; check: iadd v0, v3
+    ; check: sshr_imm v4, 2
+    ; check: irsub_imm v5, 0
+    return v1
+}
+
+; shift
+function %t_sdiv32_p2p30(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, 0x4000_0000
+    ; check: sshr_imm v0, 29
+    ; check: ushr_imm v2, 2
+    ; check: iadd v0, v3
+    ; check: v5 = sshr_imm v4, 30
+    ; check: v1 -> v5
+    return v1
+}
+
+; shift
+function %t_sdiv32_n2p30(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, -0x4000_0000
+    ; check: sshr_imm v0, 29
+    ; check: ushr_imm v2, 2
+    ; check: iadd v0, v3
+    ; check: sshr_imm v4, 30
+    ; check: irsub_imm v5, 0
+    return v1
+}
+
+; there's no positive version of this, since -(-0x8000_0000) isn't
+; representable.
+function %t_sdiv32_n2p31(i32) -> i32 {
+block0(v0: i32):
+    v1 = sdiv_imm v0, -0x8000_0000
+    ; check: sshr_imm v0, 30
+    ; check: ushr_imm v2, 1
+    ; check: iadd v0, v3
+    ; check: sshr_imm v4, 31
+    ; check: irsub_imm v5, 0
+    return v1
+}
+
+
+; -------- S64 --------
+
+; ignored
+function %t_sdiv64_p0(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, 0
+    ; check: sdiv_imm v0, 0
+    return v1
+}
+
+; converted to a nop
+function %t_sdiv64_p1(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, 1
+    ; check: nop
+    return v1
+}
+
+; ignored
+function %t_sdiv64_n1(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, -1
+    ; check: sdiv_imm v0, -1
+    return v1
+}
+
+; shift
+function %t_sdiv64_p2(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, 2
+    ; check: ushr_imm v0, 63
+    ; check: iadd v0, v2
+    ; check: v4 = sshr_imm v3, 1
+    ; check: v1 -> v4
+    return v1
+}
+
+; shift
+function %t_sdiv64_n2(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, -2
+    ; check: ushr_imm v0, 63
+    ; check: iadd v0, v2
+    ; check: sshr_imm v3, 1
+    ; check: irsub_imm v4, 0
+    return v1
+}
+
+; shift
+function %t_sdiv64_p4(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, 4
+    ; check: sshr_imm v0, 1
+    ; check: ushr_imm v2, 62
+    ; check: iadd v0, v3
+    ; check: v5 = sshr_imm v4, 2
+    ; check: v1 -> v5
+    return v1
+}
+
+; shift
+function %t_sdiv64_n4(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, -4
+    ; check: sshr_imm v0, 1
+    ; check: ushr_imm v2, 62
+    ; check: iadd v0, v3
+    ; check: sshr_imm v4, 2
+    ; check: irsub_imm v5, 0
+    return v1
+}
+
+; shift
+function %t_sdiv64_p2p62(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, 0x4000_0000_0000_0000
+    ; check: sshr_imm v0, 61
+    ; check: ushr_imm v2, 2
+    ; check: iadd v0, v3
+    ; check: v5 = sshr_imm v4, 62
+    ; check: v1 -> v5
+    return v1
+}
+
+; shift
+function %t_sdiv64_n2p62(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, -0x4000_0000_0000_0000
+    ; check: sshr_imm v0, 61
+    ; check: ushr_imm v2, 2
+    ; check: iadd v0, v3
+    ; check: sshr_imm v4, 62
+    ; check: irsub_imm v5, 0
+    return v1
+}
+
+; there's no positive version of this, since -(-0x8000_0000_0000_0000) isn't
+; representable.
+function %t_sdiv64_n2p63(i64) -> i64 {
+block0(v0: i64):
+    v1 = sdiv_imm v0, -0x8000_0000_0000_0000
+    ; check: sshr_imm v0, 62
+    ; check: ushr_imm v2, 1
+    ; check: iadd v0, v3
+    ; check: sshr_imm v4, 63
+    ; check: irsub_imm v5, 0
+    return v1
+}

cranelift/filetests/filetests/peepmatic/do_not_keep_applying_optimizations_after_replacing_with_an_alias.clif

@@ -0,0 +1,14 @@
+test peepmatic
+target x86_64
+
+;; This file used to trigger assertions where we would keep trying to
+;; unnecessarily apply optimizations after replacing an instruction with an
+;; alias of another value that we had already optimized.
+
+function %foo() {
+block0:
+    v0 = iconst.i32 3
+    v1 = srem_imm v0, 2
+    v2 = sdiv_imm v1, 1
+    trap unreachable
+}

cranelift/filetests/filetests/peepmatic/do_not_reorder_instructions_when_transplanting.clif

@@ -0,0 +1,22 @@
+test peepmatic
+target x86_64
+
+;; Test that although v5 can be replaced with v1, we don't transplant `load.i32
+;; v0` on top of `iadd v3, v4`, because that would move the load past other uses
+;; of its result.
+
+function %foo(i64) -> i32 {
+block0(v0: i64):
+    v1 = load.i32 v0
+    v2 = iconst.i32 16
+    v3 = iadd_imm v1, -16
+    v4 = iconst.i32 16
+    v5 = iadd v3, v4
+    ; check:  v1 = load.i32 v0
+    ; nextln: v5 -> v1
+    ; nextln: v2 = iconst.i32 16
+    ; nextln: v3 = iadd_imm v1, -16
+    ; nextln: v4 = iconst.i32 16
+    ; nextln: nop
+    return v5
+}

cranelift/filetests/filetests/peepmatic/fold-extended-move-wraparound.clif

@@ -0,0 +1,14 @@
+test peepmatic
+target x86_64
+
+function %wraparound(i64 vmctx) -> f32 system_v {
+    gv0 = vmctx
+    gv1 = iadd_imm.i64 gv0, 48
+
+block35(v0: i64):
+    v88 = iconst.i64 0
+    v89 = iconst.i64 0x8000_0000_0000_0000
+    v90 = ishl_imm v88, 0x8000_0000_0000_0000
+    v91 = sshr v90, v89; check: sshr_imm v90, 0x8000_0000_0000_0000
+    trap user0
+}

cranelift/filetests/filetests/peepmatic/rem_by_const_non_power_of_2.clif

@@ -0,0 +1,285 @@
+test peepmatic
+target i686 baseline
+
+; -------- U32 --------
+
+; complex case (mul, sub, shift, add, shift)
+function %t_urem32_p7(i32) -> i32 {
+block0(v0: i32):
+    v1 = urem_imm v0, 7
+    ; check: iconst.i32 0x2492_4925
+    ; check: umulhi v0, v2
+    ; check: isub v0, v3
+    ; check: ushr_imm v4, 1
+    ; check: iadd v5, v3
+    ; check: ushr_imm v6, 2
+    ; check: imul_imm v7, 7
+    ; check: isub v0, v8
+    return v1
+}
+
+; simple case (mul, shift)
+function %t_urem32_p125(i32) -> i32 {
+block0(v0: i32):
+    v1 = urem_imm v0, 125
+    ; check: iconst.i32 0x1062_4dd3
+    ; check: umulhi v0, v2
+    ; check: ushr_imm v3, 3
+    ; check: imul_imm v4, 125
+    ; check: isub v0, v5
+    return v1
+}
+
+; simple case w/ shift by zero (mul)
+function %t_urem32_p641(i32) -> i32 {
+block0(v0: i32):
+    v1 = urem_imm v0, 641
+    ; check: iconst.i32 0x0066_3d81
+    ; check: umulhi v0, v2
+    ; check: imul_imm v3, 641
+    ; check: isub v0, v4
+    return v1
+}
+
+
+; -------- S32 --------
+
+; simple case w/ shift by zero (mul, add-sign-bit)
+function %t_srem32_n6(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, -6
+    ; check: iconst.i32 0xffff_ffff_d555_5555
+    ; check: smulhi v0, v2
+    ; check: ushr_imm v3, 31
+    ; check: iadd v3, v4
+    ; check: imul_imm v5, -6
+    ; check: isub v0, v6
+    return v1
+}
+
+; simple case (mul, shift, add-sign-bit)
+function %t_srem32_n5(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, -5
+    ; check: iconst.i32 0xffff_ffff_9999_9999
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 1
+    ; check: ushr_imm v4, 31
+    ; check: iadd v4, v5
+    ; check: imul_imm v6, -5
+    ; check: isub v0, v7
+    return v1
+}
+
+; case d < 0 && M > 0 (mul, sub, shift, add-sign-bit)
+function %t_srem32_n3(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, -3
+    ; check: iconst.i32 0x5555_5555
+    ; check: smulhi v0, v2
+    ; check: isub v3, v0
+    ; check: sshr_imm v4, 1
+    ; check: ushr_imm v5, 31
+    ; check: iadd v5, v6
+    ; check: imul_imm v7, -3
+    ; check: isub v0, v8
+    return v1
+}
+
+; simple case w/ shift by zero (mul, add-sign-bit)
+function %t_srem32_p6(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, 6
+    ; check: iconst.i32 0x2aaa_aaab
+    ; check: smulhi v0, v2
+    ; check: ushr_imm v3, 31
+    ; check: iadd v3, v4
+    ; check: imul_imm v5, 6
+    ; check: isub v0, v6
+    return v1
+}
+
+; case d > 0 && M < 0 (mull, add, shift, add-sign-bit)
+function %t_srem32_p7(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, 7
+    ; check: iconst.i32 0xffff_ffff_9249_2493
+    ; check: smulhi v0, v2
+    ; check: iadd v3, v0
+    ; check: sshr_imm v4, 2
+    ; check: ushr_imm v5, 31
+    ; check: iadd v5, v6
+    ; check: imul_imm v7, 7
+    ; check: isub v0, v8
+    return v1
+}
+
+; simple case (mul, shift, add-sign-bit)
+function %t_srem32_p625(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, 625
+    ; check: iconst.i32 0x68db_8bad
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 8
+    ; check: ushr_imm v4, 31
+    ; check: iadd v4, v5
+    ; check: imul_imm v6, 625
+    ; check: isub v0, v7
+    return v1
+}
+
+
+; -------- U64 --------
+
+; complex case (mul, sub, shift, add, shift)
+function %t_urem64_p7(i64) -> i64 {
+block0(v0: i64):
+    v1 = urem_imm v0, 7
+    ; check: umulhi v0, v2
+    ; check: isub v0, v3
+    ; check: ushr_imm v4, 1
+    ; check: iadd v5, v3
+    ; check: ushr_imm v6, 2
+    ; check: imul_imm v7, 7
+    ; check: isub v0, v8
+    return v1
+}
+
+; simple case (mul, shift)
+function %t_urem64_p9(i64) -> i64 {
+block0(v0: i64):
+    v1 = urem_imm v0, 9
+    ; check: iconst.i64 0xe38e_38e3_8e38_e38f
+    ; check: umulhi v0, v2
+    ; check: ushr_imm v3, 3
+    ; check: imul_imm v4, 9
+    ; check: isub v0, v5
+    return v1
+}
+
+; complex case (mul, sub, shift, add, shift)
+function %t_urem64_p125(i64) -> i64 {
+block0(v0: i64):
+    v1 = urem_imm v0, 125
+    ; check: iconst.i64 0x0624_dd2f_1a9f_be77
+    ; check: umulhi v0, v2
+    ; check: isub v0, v3
+    ; check: ushr_imm v4, 1
+    ; check: iadd v5, v3
+    ; check: ushr_imm v6, 6
+    ; check: imul_imm v7, 125
+    ; check: isub v0, v8
+    return v1
+}
+
+; simple case w/ shift by zero (mul)
+function %t_urem64_p274177(i64) -> i64 {
+block0(v0: i64):
+    v1 = urem_imm v0, 274177
+    ; check: iconst.i64 0x3d30_f19c_d101
+    ; check: umulhi v0, v2
+    ; check: imul_imm v3, 0x0004_2f01
+    ; check: isub v0, v4
+    return v1
+}
+
+
+; -------- S64 --------
+
+; simple case (mul, shift, add-sign-bit)
+function %t_srem64_n625(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, -625
+    ; check: iconst.i64 0xcb92_3a29_c779_a6b5
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 7
+    ; check: ushr_imm v4, 63
+    ; check: iadd v4, v5
+    ; check: imul_imm v6, -625
+    ; check: isub v0, v7
+    return v1
+}
+
+; simple case w/ zero shift (mul, add-sign-bit)
+function %t_srem64_n6(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, -6
+    ; check: iconst.i64 0xd555_5555_5555_5555
+    ; check: smulhi v0, v2
+    ; check: ushr_imm v3, 63
+    ; check: iadd v3, v4
+    ; check: imul_imm v5, -6
+    ; check: isub v0, v6
+    return v1
+}
+
+; simple case w/ zero shift (mul, add-sign-bit)
+function %t_srem64_n5(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, -5
+    ; check: iconst.i64 0x9999_9999_9999_9999
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 1
+    ; check: ushr_imm v4, 63
+    ; check: iadd v4, v5
+    ; check: imul_imm v6, -5
+    ; check: isub v0, v7
+    return v1
+}
+
+; case d < 0 && M > 0 (mul, sub, shift, add-sign-bit)
+function %t_srem64_n3(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, -3
+    ; check: iconst.i64 0x5555_5555_5555_5555
+    ; check: smulhi v0, v2
+    ; check: isub v3, v0
+    ; check: sshr_imm v4, 1
+    ; check: ushr_imm v5, 63
+    ; check: iadd v5, v6
+    ; check: imul_imm v7, -3
+    ; check: isub v0, v8
+    return v1
+}
+
+; simple case w/ zero shift (mul, add-sign-bit)
+function %t_srem64_p6(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, 6
+    ; check: iconst.i64 0x2aaa_aaaa_aaaa_aaab
+    ; check: smulhi v0, v2
+    ; check: ushr_imm v3, 63
+    ; check: iadd v3, v4
+    ; check: imul_imm v5, 6
+    ; check: isub v0, v6
+    return v1
+}
+
+; case d > 0 && M < 0 (mul, add, shift, add-sign-bit)
+function %t_srem64_p15(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, 15
+    ; check: iconst.i64 0x8888_8888_8888_8889
+    ; check: smulhi v0, v2
+    ; check: iadd v3, v0
+    ; check: sshr_imm v4, 3
+    ; check: ushr_imm v5, 63
+    ; check: iadd v5, v6
+    ; check: imul_imm v7, 15
+    ; check: isub v0, v8
+    return v1
+}
+
+; simple case (mul, shift, add-sign-bit)
+function %t_srem64_p625(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, 625
+    ; check: iconst.i64 0x346d_c5d6_3886_594b
+    ; check: smulhi v0, v2
+    ; check: sshr_imm v3, 7
+    ; check: ushr_imm v4, 63
+    ; check: iadd v4, v5
+    ; check: imul_imm v6, 625
+    ; check: isub v0, v7
+    return v1
+}

cranelift/filetests/filetests/peepmatic/rem_by_const_power_of_2.clif

@@ -0,0 +1,291 @@
+test peepmatic
+target i686 baseline
+
+; -------- U32 --------
+
+; ignored
+function %t_urem32_p0(i32) -> i32 {
+block0(v0: i32):
+    v1 = urem_imm v0, 0
+    ; check: urem_imm v0, 0
+    return v1
+}
+
+; converted to constant zero
+function %t_urem32_p1(i32) -> i32 {
+block0(v0: i32):
+    v1 = urem_imm v0, 1
+    ; check: iconst.i32 0
+    return v1
+}
+
+; shift
+function %t_urem32_p2(i32) -> i32 {
+block0(v0: i32):
+    v1 = urem_imm v0, 2
+    ; check: band_imm v0, 1
+    return v1
+}
+
+; shift
+function %t_urem32_p2p31(i32) -> i32 {
+block0(v0: i32):
+    v1 = urem_imm v0, 0x8000_0000
+    ; check: band_imm v0, 0x7fff_ffff
+    return v1
+}
+
+
+; -------- U64 --------
+
+; ignored
+function %t_urem64_p0(i64) -> i64 {
+block0(v0: i64):
+    v1 = urem_imm v0, 0
+    ; check: urem_imm v0, 0
+    return v1
+}
+
+; converted to constant zero
+function %t_urem64_p1(i64) -> i64 {
+block0(v0: i64):
+    v1 = urem_imm v0, 1
+    ; check: iconst.i64 0
+    return v1
+}
+
+; shift
+function %t_urem64_p2(i64) -> i64 {
+block0(v0: i64):
+    v1 = urem_imm v0, 2
+    ; check: band_imm v0, 1
+   return v1
+}
+
+; shift
+function %t_urem64_p2p63(i64) -> i64 {
+block0(v0: i64):
+    v1 = urem_imm v0, 0x8000_0000_0000_0000
+    ; check: band_imm v0, 0x7fff_ffff_ffff_ffff
+    return v1
+}
+
+
+; -------- S32 --------
+
+; ignored
+function %t_srem32_n1(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, -1
+    ; check: srem_imm v0, -1
+    return v1
+}
+
+; ignored
+function %t_srem32_p0(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, 0
+    ; check: srem_imm v0, 0
+    return v1
+}
+
+; converted to constant zero
+function %t_srem32_p1(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, 1
+    ; check: iconst.i32 0
+    return v1
+}
+
+; shift
+function %t_srem32_p2(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, 2
+    ; check: ushr_imm v0, 31
+    ; check: iadd v0, v2
+    ; check: band_imm v3, -2
+    ; check: isub v0, v4
+    return v1
+}
+
+; shift
+function %t_srem32_n2(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, -2
+    ; check: ushr_imm v0, 31
+    ; check: iadd v0, v2
+    ; check: band_imm v3, -2
+    ; check: isub v0, v4
+    return v1
+}
+
+; shift
+function %t_srem32_p4(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, 4
+    ; check: sshr_imm v0, 1
+    ; check: ushr_imm v2, 30
+    ; check: iadd v0, v3
+    ; check: band_imm v4, -4
+    ; check: isub v0, v5
+    return v1
+}
+
+; shift
+function %t_srem32_n4(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, -4
+    ; check: sshr_imm v0, 1
+    ; check: ushr_imm v2, 30
+    ; check: iadd v0, v3
+    ; check: band_imm v4, -4
+    ; check: isub v0, v5
+    return v1
+}
+
+; shift
+function %t_srem32_p2p30(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, 0x4000_0000
+    ; check: sshr_imm v0, 29
+    ; check: ushr_imm v2, 2
+    ; check: iadd v0, v3
+    ; check: band_imm v4, 0xffff_ffff_c000_0000
+    ; check: isub v0, v5
+    return v1
+}
+
+; shift
+function %t_srem32_n2p30(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, -0x4000_0000
+    ; check: sshr_imm v0, 29
+    ; check: ushr_imm v2, 2
+    ; check: iadd v0, v3
+    ; check: band_imm v4, 0xffff_ffff_c000_0000
+    ; check: isub v0, v5
+    return v1
+}
+
+; there's no positive version of this, since -(-0x8000_0000) isn't
+; representable.
+function %t_srem32_n2p31(i32) -> i32 {
+block0(v0: i32):
+    v1 = srem_imm v0, -0x8000_0000
+    ; check: sshr_imm v0, 30
+    ; check: ushr_imm v2, 1
+    ; check: iadd v0, v3
+    ; check: band_imm v4, 0xffff_ffff_8000_0000
+    ; check: isub v0, v5
+    return v1
+}
+
+
+; -------- S64 --------
+
+; ignored
+function %t_srem64_n1(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, -1
+    ; check: srem_imm v0, -1
+    return v1
+}
+
+; ignored
+function %t_srem64_p0(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, 0
+    ; check: srem_imm v0, 0
+    return v1
+}
+
+; converted to constant zero
+function %t_srem64_p1(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, 1
+    ; check: iconst.i64 0
+    return v1
+}
+
+; shift
+function %t_srem64_p2(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, 2
+    ; check: ushr_imm v0, 63
+    ; check: iadd v0, v2
+    ; check: band_imm v3, -2
+    ; check: isub v0, v4
+    return v1
+}
+
+; shift
+function %t_srem64_n2(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, -2
+    ; check: ushr_imm v0, 63
+    ; check: iadd v0, v2
+    ; check: band_imm v3, -2
+    ; check: isub v0, v4
+    return v1
+}
+
+; shift
+function %t_srem64_p4(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, 4
+    ; check: sshr_imm v0, 1
+    ; check: ushr_imm v2, 62
+    ; check: iadd v0, v3
+    ; check: band_imm v4, -4
+    ; check: isub v0, v5
+    return v1
+}
+
+; shift
+function %t_srem64_n4(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, -4
+    ; check: sshr_imm v0, 1
+    ; check: ushr_imm v2, 62
+    ; check: iadd v0, v3
+    ; check: band_imm v4, -4
+    ; check: isub v0, v5
+    return v1
+}
+
+; shift
+function %t_srem64_p2p62(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, 0x4000_0000_0000_0000
+    ; check: sshr_imm v0, 61
+    ; check: ushr_imm v2, 2
+    ; check: iadd v0, v3
+    ; check: band_imm v4, 0xc000_0000_0000_0000
+    ; check: isub v0, v5
+    return v1
+}
+
+; shift
+function %t_srem64_n2p62(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, -0x4000_0000_0000_0000
+    ; check: sshr_imm v0, 61
+    ; check: ushr_imm v2, 2
+    ; check: iadd v0, v3
+    ; check: band_imm v4, 0xc000_0000_0000_0000
+    ; check: isub v0, v5
+    return v1
+}
+
+; there's no positive version of this, since -(-0x8000_0000_0000_0000) isn't
+; representable.
+function %t_srem64_n2p63(i64) -> i64 {
+block0(v0: i64):
+    v1 = srem_imm v0, -0x8000_0000_0000_0000
+    ; check: sshr_imm v0, 62
+    ; check: ushr_imm v2, 1
+    ; check: iadd v0, v3
+    ; check: band_imm v4, 0x8000_0000_0000_0000
+    ; check: isub v0, v5
+   return v1
+}

cranelift/filetests/filetests/peepmatic/replace_branching_instructions_and_cfg_predecessors.clif

@@ -0,0 +1,22 @@
+test peepmatic
+target x86_64
+
+function u0:2(i64 , i64) {
+    gv1 = load.i64 notrap aligned gv0
+    heap0 = static gv1
+    block0(v0: i64, v1: i64):
+        v16 = iconst.i32 6
+        v17 = heap_addr.i64 heap0, v16, 1
+        v18 = load.i32 v17
+        v19 = iconst.i32 4
+        v20 = icmp ne v18, v19
+        v21 = bint.i32 v20
+        brnz v21, block2
+        jump block4
+    block4:
+        jump block1
+    block2:
+        jump block1
+    block1:
+        return
+}

cranelift/filetests/filetests/peepmatic/simplify32.clif

@@ -0,0 +1,60 @@
+test peepmatic
+target i686
+
+;; 32-bits platforms.
+
+function %iadd_imm(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 2
+    v2 = iadd v0, v1
+    return v2
+}
+; sameln: function %iadd_imm
+; nextln: block0(v0: i32):
+; nextln:     v1 = iconst.i32 2
+; nextln:     v2 = iadd_imm v0, 2
+; nextln:     return v2
+; nextln: }
+
+function %isub_imm(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 2
+    v2 = isub v0, v1
+    return v2
+}
+; sameln: function %isub_imm
+; nextln: block0(v0: i32):
+; nextln:     v1 = iconst.i32 2
+; nextln:     v2 = iadd_imm v0, -2
+; nextln:     return v2
+; nextln: }
+
+function %icmp_imm(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 2
+    v2 = icmp slt v0, v1
+    v3 = bint.i32 v2
+    return v3
+}
+; sameln: function %icmp_imm
+; nextln: block0(v0: i32):
+; nextln:     v1 = iconst.i32 2
+; nextln:     v2 = icmp_imm slt v0, 2
+; nextln:     v3 = bint.i32 v2
+; nextln:     return v3
+; nextln: }
+
+;; Don't simplify operations that would get illegal because of lack of native
+;; support.
+function %iadd_imm(i64) -> i64 {
+block0(v0: i64):
+    v1 = iconst.i64 2
+    v2 = iadd v0, v1
+    return v2
+}
+; sameln: function %iadd_imm
+; nextln: block0(v0: i64):
+; nextln:     v1 = iconst.i64 2
+; nextln:     v2 = iadd v0, v1
+; nextln:     return v2
+; nextln: }

cranelift/filetests/filetests/peepmatic/simplify64.clif

@@ -0,0 +1,326 @@
+test peepmatic
+target x86_64
+
+;; 64-bits platforms.
+
+function %iadd_imm(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 2
+    v2 = iadd v0, v1
+    return v2
+}
+; sameln: function %iadd_imm
+; nextln: block0(v0: i32):
+; nextln:     v1 = iconst.i32 2
+; nextln:     v2 = iadd_imm v0, 2
+; nextln:     return v2
+; nextln: }
+
+function %isub_imm(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 2
+    v2 = isub v0, v1
+    return v2
+}
+; sameln: function %isub_imm
+; nextln: block0(v0: i32):
+; nextln:     v1 = iconst.i32 2
+; nextln:     v2 = iadd_imm v0, -2
+; nextln:     return v2
+; nextln: }
+
+function %icmp_imm(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 2
+    v2 = icmp slt v0, v1
+    v3 = bint.i32 v2
+    return v3
+}
+; sameln: function %icmp_imm
+; nextln: block0(v0: i32):
+; nextln:     v1 = iconst.i32 2
+; nextln:     v2 = icmp_imm slt v0, 2
+; nextln:     v3 = bint.i32 v2
+; nextln:     return v3
+; nextln: }
+
+function %ifcmp_imm(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 2
+    v2 = ifcmp v0, v1
+    brif eq v2, block1
+    jump block2
+
+block1:
+    v3 = iconst.i32 1
+    return v3
+
+block2:
+    v4 = iconst.i32 2
+    return v4
+}
+; sameln: function %ifcmp_imm
+; nextln: block0(v0: i32):
+; nextln:     v1 = iconst.i32 2
+; nextln:     v2 = ifcmp_imm v0, 2
+; nextln:     brif eq v2, block1
+; nextln:     jump block2
+; nextln: 
+; nextln: block1:
+; nextln:     v3 = iconst.i32 1
+; nextln:     return v3
+; nextln: 
+; nextln: block2:
+; nextln:     v4 = iconst.i32 2
+; nextln:     return v4
+; nextln: }
+
+function %brz_bint(i32) {
+block0(v0: i32):
+    v3 = icmp_imm slt v0, 0
+    v1 = bint.i32 v3
+    v2 = select v1, v1, v1
+    trapz v1, user0
+    brz v1, block1
+    jump block2
+
+block1:
+    return
+
+block2:
+    return
+}
+; sameln: function %brz_bint
+; nextln: (v0: i32):
+; nextln:    v3 = icmp_imm slt v0, 0
+; nextln:    v1 = bint.i32 v3
+; nextln:    v2 = select v3, v1, v1
+; nextln:    trapz v3, user0
+; nextln:    brnz v3, block2
+; nextln:    jump block1
+
+function %irsub_imm(i32) -> i32 {
+block0(v0: i32):
+    v1 = iconst.i32 2
+    v2 = isub v1, v0
+    return v2
+}
+; sameln: function %irsub_imm
+; nextln: block0(v0: i32):
+; nextln:     v1 = iconst.i32 2
+; nextln:     v2 = irsub_imm v0, 2
+; nextln:     return v2
+; nextln: }
+
+;; Sign-extensions.
+
+;; 8 -> 16
+function %uextend_8_16() -> i16 {
+block0:
+    v0 = iconst.i16 37
+    v1 = ishl_imm v0, 8
+    v2 = ushr_imm v1, 8
+    return v2
+}
+; sameln: function %uextend_8_16
+; nextln: block0:
+; nextln:     v0 = iconst.i16 37
+; nextln:     v1 = ishl_imm v0, 8
+; nextln:     v3 = ireduce.i8 v0
+; nextln:     v2 = uextend.i16 v3
+; nextln:     return v2
+; nextln: }
+
+function %sextend_8_16() -> i16 {
+block0:
+    v0 = iconst.i16 37
+    v1 = ishl_imm v0, 8
+    v2 = sshr_imm v1, 8
+    return v2
+}
+; sameln: function %sextend_8_16
+; nextln: block0:
+; nextln:     v0 = iconst.i16 37
+; nextln:     v1 = ishl_imm v0, 8
+; nextln:     v3 = ireduce.i8 v0
+; nextln:     v2 = sextend.i16 v3
+; nextln:     return v2
+; nextln: }
+
+;; 8 -> 32
+function %uextend_8_32() -> i32 {
+block0:
+    v0 = iconst.i32 37
+    v1 = ishl_imm v0, 24
+    v2 = ushr_imm v1, 24
+    return v2
+}
+; sameln: function %uextend_8_32
+; nextln: block0:
+; nextln:     v0 = iconst.i32 37
+; nextln:     v1 = ishl_imm v0, 24
+; nextln:     v3 = ireduce.i8 v0
+; nextln:     v2 = uextend.i32 v3
+; nextln:     return v2
+; nextln: }
+
+function %sextend_8_32() -> i32 {
+block0:
+    v0 = iconst.i32 37
+    v1 = ishl_imm v0, 24
+    v2 = sshr_imm v1, 24
+    return v2
+}
+; sameln: function %sextend_8_32
+; nextln: block0:
+; nextln:     v0 = iconst.i32 37
+; nextln:     v1 = ishl_imm v0, 24
+; nextln:     v3 = ireduce.i8 v0
+; nextln:     v2 = sextend.i32 v3
+; nextln:     return v2
+; nextln: }
+
+;; 16 -> 32
+function %uextend_16_32() -> i32 {
+block0:
+    v0 = iconst.i32 37
+    v1 = ishl_imm v0, 16
+    v2 = ushr_imm v1, 16
+    return v2
+}
+; sameln: function %uextend_16_32
+; nextln: block0:
+; nextln:     v0 = iconst.i32 37
+; nextln:     v1 = ishl_imm v0, 16
+; nextln:     v3 = ireduce.i16 v0
+; nextln:     v2 = uextend.i32 v3
+; nextln:     return v2
+; nextln: }
+
+function %sextend_16_32() -> i32 {
+block0:
+    v0 = iconst.i32 37
+    v1 = ishl_imm v0, 16
+    v2 = sshr_imm v1, 16
+    return v2
+}
+; sameln: function %sextend_16_32
+; nextln: block0:
+; nextln:     v0 = iconst.i32 37
+; nextln:     v1 = ishl_imm v0, 16
+; nextln:     v3 = ireduce.i16 v0
+; nextln:     v2 = sextend.i32 v3
+; nextln:     return v2
+; nextln: }
+
+;; 8 -> 64
+function %uextend_8_64() -> i64 {
+block0:
+    v0 = iconst.i64 37
+    v1 = ishl_imm v0, 56
+    v2 = ushr_imm v1, 56
+    return v2
+}
+; sameln: function %uextend_8_64
+; nextln: block0:
+; nextln:     v0 = iconst.i64 37
+; nextln:     v1 = ishl_imm v0, 56
+; nextln:     v3 = ireduce.i8 v0
+; nextln:     v2 = uextend.i64 v3
+; nextln:     return v2
+; nextln: }
+
+function %sextend_8_64() -> i64 {
+block0:
+    v0 = iconst.i64 37
+    v1 = ishl_imm v0, 56
+    v2 = sshr_imm v1, 56
+    return v2
+}
+; sameln: function %sextend_8_64
+; nextln: block0:
+; nextln:     v0 = iconst.i64 37
+; nextln:     v1 = ishl_imm v0, 56
+; nextln:     v3 = ireduce.i8 v0
+; nextln:     v2 = sextend.i64 v3
+; nextln:     return v2
+; nextln: }
+
+;; 16 -> 64
+function %uextend_16_64() -> i64 {
+block0:
+    v0 = iconst.i64 37
+    v1 = ishl_imm v0, 48
+    v2 = ushr_imm v1, 48
+    return v2
+}
+; sameln: function %uextend_16_64
+; nextln: block0:
+; nextln:     v0 = iconst.i64 37
+; nextln:     v1 = ishl_imm v0, 48
+; nextln:     v3 = ireduce.i16 v0
+; nextln:     v2 = uextend.i64 v3
+; nextln:     return v2
+; nextln: }
+
+function %sextend_16_64() -> i64 {
+block0:
+    v0 = iconst.i64 37
+    v1 = ishl_imm v0, 48
+    v2 = sshr_imm v1, 48
+    return v2
+}
+; sameln: function %sextend_16_64
+; nextln: block0:
+; nextln:     v0 = iconst.i64 37
+; nextln:     v1 = ishl_imm v0, 48
+; nextln:     v3 = ireduce.i16 v0
+; nextln:     v2 = sextend.i64 v3
+; nextln:     return v2
+; nextln: }
+
+;; 32 -> 64
+function %uextend_32_64() -> i64 {
+block0:
+    v0 = iconst.i64 37
+    v1 = ishl_imm v0, 32
+    v2 = ushr_imm v1, 32
+    return v2
+}
+; sameln: function %uextend_32_64
+; nextln: block0:
+; nextln:     v0 = iconst.i64 37
+; nextln:     v1 = ishl_imm v0, 32
+; nextln:     v3 = ireduce.i32 v0
+; nextln:     v2 = uextend.i64 v3
+; nextln:     return v2
+; nextln: }
+
+function %sextend_32_64() -> i64 {
+block0:
+    v0 = iconst.i64 37
+    v1 = ishl_imm v0, 32
+    v2 = sshr_imm v1, 32
+    return v2
+}
+; sameln: function %sextend_32_64
+; nextln: block0:
+; nextln:     v0 = iconst.i64 37
+; nextln:     v1 = ishl_imm v0, 32
+; nextln:     v3 = ireduce.i32 v0
+; nextln:     v2 = sextend.i64 v3
+; nextln:     return v2
+; nextln: }
+
+function %add_imm_fold(i32) -> i32 {
+block0(v0: i32):
+  v1 = iadd_imm v0, 42
+  v2 = iadd_imm v1, -42
+  return v2
+}
+; sameln: function %add_imm_fold(i32)
+; nextln: block0(v0: i32):
+; nextln:    v2 -> v0
+; nextln:    v1 = iadd_imm v0, 42
+; nextln:    nop
+; nextln:    return v2

cranelift/filetests/filetests/peepmatic/simplify_instruction_into_alias_of_value.clif

@@ -0,0 +1,17 @@
+test peepmatic
+target x86_64
+
+;; The `isub` is a no-op, but we can't replace the whole `isub` instruction with
+;; its `v2` operand's instruction because `v2` is one of many results. Instead,
+;; we need to make an alias `v3 -> v2`.
+
+function %replace_inst_with_alias() -> i32 {
+block0:
+    v0 = iconst.i32 0
+    v1, v2 = x86_smulx v0, v0
+    v3 = isub v2, v0
+    ; check:  v0 = iconst.i32 0
+    ; nextln: v1, v2 = x86_smulx v0, v0
+    ; nextln: v3 -> v2
+    return v3
+}

cranelift/filetests/filetests/simple_preopt/do_not_reorder_instructions_when_transplanting.clif

@@ -0,0 +1,22 @@
+test simple_preopt
+target x86_64
+
+;; Test that although v5 can be replaced with v1, we don't transplant `load.i32
+;; v0` on top of `iadd v3, v4`, because that would move the load past other uses
+;; of its result.
+
+function %foo(i64) -> i32 {
+block0(v0: i64):
+    v1 = load.i32 v0
+    v2 = iconst.i32 16
+    v3 = iadd_imm v1, -16
+    v4 = iconst.i32 16
+    v5 = iadd v3, v4
+    ; check:  v1 = load.i32 v0
+    ; nextln: v5 -> v1
+    ; nextln: v2 = iconst.i32 16
+    ; nextln: v3 = iadd_imm v1, -16
+    ; nextln: v4 = iconst.i32 16
+    ; nextln: nop
+    return v5
+}

cranelift/filetests/filetests/simple_preopt/replace_branching_instructions_and_cfg_predecessors.clif

@@ -0,0 +1,22 @@
+test simple_preopt
+target x86_64
+
+function u0:2(i64 , i64) {
+    gv1 = load.i64 notrap aligned gv0
+    heap0 = static gv1
+    block0(v0: i64, v1: i64):
+        v16 = iconst.i32 6
+        v17 = heap_addr.i64 heap0, v16, 1
+        v18 = load.i32 v17
+        v19 = iconst.i32 4
+        v20 = icmp ne v18, v19
+        v21 = bint.i32 v20
+        brnz v21, block2
+        jump block4
+    block4:
+        jump block1
+    block2:
+        jump block1
+    block1:
+        return
+}

cranelift/filetests/filetests/simple_preopt/simplify_instruction_into_alias_of_value.clif

@@ -0,0 +1,17 @@
+test simple_preopt
+target x86_64
+
+;; The `isub` is a no-op, but we can't replace the whole `isub` instruction with
+;; its `v2` operand's instruction because `v2` is one of many results. Instead,
+;; we need to make an alias `v3 -> v2`.
+
+function %replace_inst_with_alias() -> i32 {
+block0:
+    v0 = iconst.i32 0
+    v1, v2 = x86_smulx v0, v0
+    v3 = isub v2, v0
+    ; check:  v0 = iconst.i32 0
+    ; nextln: v1, v2 = x86_smulx v0, v0
+    ; nextln: v3 -> v2
+    return v3
+}

cranelift/filetests/src/lib.rs

@@ -45,6 +45,7 @@ mod test_domtree;
 mod test_interpret;
 mod test_legalizer;
 mod test_licm;
+mod test_peepmatic;
 mod test_postopt;
 mod test_preopt;
 mod test_print_cfg;
@@ -128,6 +129,7 @@ fn new_subtest(parsed: &TestCommand) -> subtest::SubtestResult<Box<dyn subtest::
         "interpret" => test_interpret::subtest(parsed),
         "legalizer" => test_legalizer::subtest(parsed),
         "licm" => test_licm::subtest(parsed),
+        "peepmatic" => test_peepmatic::subtest(parsed),
         "postopt" => test_postopt::subtest(parsed),
         "preopt" => test_preopt::subtest(parsed),
         "print-cfg" => test_print_cfg::subtest(parsed),

cranelift/filetests/src/test_peepmatic.rs

@@ -0,0 +1,56 @@
+//! Test command for `peepmatic`-generated peephole optimizers.
+
+use crate::subtest::{run_filecheck, Context, SubTest, SubtestResult};
+use cranelift_codegen;
+use cranelift_codegen::ir::Function;
+use cranelift_codegen::print_errors::pretty_error;
+use cranelift_reader::TestCommand;
+use std::borrow::Cow;
+
+struct TestPreopt;
+
+pub fn subtest(parsed: &TestCommand) -> SubtestResult<Box<dyn SubTest>> {
+    assert_eq!(parsed.command, "peepmatic");
+    if parsed.options.is_empty() {
+        Ok(Box::new(TestPreopt))
+    } else {
+        Err(format!("No options allowed on {}", parsed))
+    }
+}
+
+impl SubTest for TestPreopt {
+    fn name(&self) -> &'static str {
+        "peepmatic"
+    }
+
+    fn is_mutating(&self) -> bool {
+        true
+    }
+
+    fn needs_isa(&self) -> bool {
+        true
+    }
+
+    fn run(&self, func: Cow<Function>, context: &Context) -> SubtestResult<()> {
+        let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned());
+        let isa = context.isa.expect("preopt needs an ISA");
+
+        comp_ctx.compute_cfg();
+        comp_ctx
+            .preopt(isa)
+            .map_err(|e| pretty_error(&comp_ctx.func, context.isa, Into::into(e)))?;
+        let text = &comp_ctx.func.display(isa).to_string();
+        log::debug!("After peepmatic-based simple_preopt:\n{}", text);
+
+        // Only actually run the filecheck if peepmatic is enabled, because it
+        // can generate slightly different code (alias a result vs replace an
+        // instruction) than the non-peepmatic versions of peephole
+        // optimizations. Note that the non-`peepmatic` results can be tested
+        // with the `test simple_preopt` subtest.
+        if cfg!(feature = "enable-peepmatic") {
+            run_filecheck(&text, context)
+        } else {
+            Ok(())
+        }
+    }
+}

cranelift/filetests/src/test_simple_preopt.rs

@@ -38,6 +38,17 @@ impl SubTest for TestSimplePreopt {
             .preopt(isa)
             .map_err(|e| pretty_error(&comp_ctx.func, context.isa, Into::into(e)))?;
         let text = &comp_ctx.func.display(isa).to_string();
-        run_filecheck(&text, context)
+        log::debug!("After simple_preopt:\n{}", text);
+
+        // Only actually run the filecheck if peepmatic is *not* enabled,
+        // because it can generate slightly different code (alias a result vs
+        // replace an instruction) than the non-peepmatic versions of peephole
+        // optimizations. Note that the `peepmatic`-based results can be tested
+        // with the `test peepmatic` subtest.
+        if cfg!(feature = "enable-peepmatic") {
+            Ok(())
+        } else {
+            run_filecheck(&text, context)
+        }
     }
 }

cranelift/peepmatic/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "peepmatic"
+version = "0.1.0"
+authors = ["Nick Fitzgerald <fitzgen@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.27"
+peepmatic-automata = { version = "0.1.0", path = "crates/automata", features = ["dot"] }
+peepmatic-macro = { version = "0.1.0", path = "crates/macro" }
+peepmatic-runtime = { version = "0.1.0", path = "crates/runtime", features = ["construct"] }
+wast = "15.0.0"
+z3 = { version = "0.5.1", features = ["static-link-z3"] }

cranelift/peepmatic/README.md

@@ -0,0 +1,423 @@
+<div align="center">
+  <h1><code>peepmatic</code></h1>
+
+  <p>
+    <b>
+      <code>peepmatic</code> is a DSL and compiler for peephole optimizers for
+      <a href="https://github.com/bytecodealliance/wasmtime/tree/master/cranelift#readme">Cranelift</a>.
+    </b>
+  </p>
+
+  <img src="https://github.com/fitzgen/peepmatic/workflows/Rust/badge.svg"/>
+
+</div>
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
+- [About](#about)
+- [Example](#example)
+- [A DSL for Optimizations](#a-dsl-for-optimizations)
+  - [Variables](#variables)
+  - [Constants](#constants)
+  - [Nested Patterns](#nested-patterns)
+  - [Preconditions and Unquoting](#preconditions-and-unquoting)
+  - [Bit Widths](#bit-widths)
+- [Implementation](#implementation)
+  - [Parsing](#parsing)
+  - [Type Checking](#type-checking)
+  - [Linearization](#linearization)
+  - [Automatization](#automatization)
+- [References](#references)
+- [Acknowledgments](#acknowledgments)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## About
+
+Peepmatic is a DSL for peephole optimizations and compiler for generating
+peephole optimizers from them. The user writes a set of optimizations in the
+DSL, and then `peepmatic` compiles the set of optimizations into an efficient
+peephole optimizer:
+
+```
+DSL ----peepmatic----> Peephole Optimizer
+```
+
+The generated peephole optimizer has all of its optimizations' left-hand sides
+collapsed into a compact automata that makes matching candidate instruction
+sequences fast.
+
+The DSL's optimizations may be written by hand or discovered mechanically with a
+superoptimizer like [Souper][]. Eventually, `peepmatic` should have a verifier
+that ensures that the DSL's optimizations are sound, similar to what [Alive][]
+does for LLVM optimizations.
+
+Currently, `peepmatic` is targeting peephole optimizers that operate on
+Cranelift's clif intermediate representation. The intended next target is
+Cranelift's new backend's "vcode" intermediate representation. Supporting
+non-Cranelift targets is not a goal.
+
+[Cranelift]: https://github.com/bytecodealliance/wasmtime/tree/master/cranelift#readme
+[Souper]: https://github.com/google/souper
+[Alive]: https://github.com/AliveToolkit/alive2
+
+## Example
+
+This snippet of our DSL describes optimizations for removing redundant
+bitwise-or instructions that are no-ops:
+
+```lisp
+(=> (bor $x (bor $x $y))
+    (bor $x $y))
+
+(=> (bor $y (bor $x $y))
+    (bor $x $y))
+
+(=> (bor (bor $x $y) $x)
+    (bor $x $y))
+
+(=> (bor (bor $x $y) $y)
+    (bor $x $y))
+```
+
+When compiled into a peephole optimizer automaton, they look like this:
+
+![](examples/redundant-bor.png)
+
+## A DSL for Optimizations
+
+A single peephole optimization has two parts:
+
+1. A **left-hand side** that describes candidate instruction sequences that the
+   optimization applies to.
+2. A **right-hand side** that contains the new instruction sequence that
+   replaces old instruction sequences that the left-hand side matched.
+
+A left-hand side may bind sub-expressions to variables and the right-hand side
+may contain those bound variables to reuse the sub-expressions. The operations
+inside the left-hand and right-hand sides are a subset of clif operations.
+
+Let's take a look at an example:
+
+```lisp
+(=> (imul $x 2)
+    (ishl $x 1))
+```
+
+As you can see, the DSL uses S-expressions. (S-expressions are easy to parse and
+we also have a bunch of nice parsing infrastructure for S-expressions already
+for our [`wat`][wat] and [`wast`][wast] crates.)
+
+[wat]: https://crates.io/crates/wat
+[wast]: https://crates.io/crates/wast
+
+The left-hand side of this optimization is `(imul $x 2)`. It matches integer
+multiplication operations where a value is multiplied by the constant two. The
+value multiplied by two is bound to the variable `$x`.
+
+The right-hand side of this optimization is `(ishl $x 1)`. It reuses the `$x`
+variable that was bound in the left-hand side.
+
+This optimization replaces expressions of the form `x * 2` with `x << 1`. This
+is sound because multiplication by two is the same as shifting left by one for
+binary integers, and it is desirable because a shift-left instruction executes
+in fewer cycles than a multiplication.
+
+The general form of an optimization is:
+
+```lisp
+(=> <left-hand-side> <right-hand-side>)
+```
+
+### Variables
+
+Variables begin with a dollar sign and are followed by lowercase letters,
+numbers, hyphens, and underscores: `$x`, `$y`, `$my-var`, `$operand2`.
+
+Left-hand side patterns may contain variables that match any kind of
+sub-expression and give it a name so that it may be reused in the right-hand
+side.
+
+```lisp
+;; Replace `x + 0` with simply `x`.
+(=> (iadd $x 0)
+    $x)
+```
+
+Within a pattern, every occurrence of a variable with the same name must match
+the same value. That is `(iadd $x $x)` matches `(iadd 1 1)` but does not match
+`(iadd 1 2)`. This lets us write optimizations such as this:
+
+```lisp
+;; Xor'ing a value with itself is always zero.
+(=> (bxor $x $x)
+    (iconst 0))
+```
+
+### Constants
+
+We've already seen specific integer literals and wildcard variables in patterns,
+but we can also match any constant. These are written similar to variables, but
+use uppercase letters rather than lowercase: `$C`, `$MY-CONST`, and `$OPERAND2`.
+
+For example, we can use constant patterns to combine an `iconst` and `iadd` into
+a single `iadd_imm` instruction:
+
+```lisp
+(=> (iadd (iconst $C) $x)
+    (iadd_imm $C $x))
+```
+
+### Nested Patterns
+
+Patterns can also match nested operations with their own nesting:
+
+```lisp
+(=> (bor $x (bor $x $y))
+    (bor $x $y))
+```
+
+### Preconditions and Unquoting
+
+Let's reconsider our first example optimization:
+
+```lisp
+(=> (imul $x 2)
+    (ishl $x 1))
+```
+
+This optimization is a little too specific. Here is another version of this
+optimization that we'd like to support:
+
+```lisp
+(=> (imul $x 4)
+    (ishl $x 2))
+```
+
+We don't want to have to write out all instances of this general class of
+optimizations! That would be a lot of repetition and could also bloat the size
+of our generated peephole optimizer's matching automata.
+
+Instead, we can generalize this optimization by matching any multiplication by a
+power of two constant `C` and replacing it with a shift left of `log2(C)`.
+
+First, rather than match `2` directly, we want to match any constant variable `C`:
+
+```lisp
+(imul $x $C)
+```
+
+Note that variables begin with lowercase letters, while constants begin with
+uppercase letters. Both the constant pattern `$C` and variable pattern `$x` will
+match `5`, but only the variable pattern `$x` will match a whole sub-expression
+like `(iadd 1 2)`. The constant pattern `$C` only matches constant values.
+
+Next, we augment our left-hand side's pattern with a **precondition** that the
+constant `$C` must be a power of two. Preconditions are introduced by wrapping
+a pattern in the `when` form:
+
+```lisp
+;; Our new left-hand side, augmenting a pattern with a precondition!
+(when
+  ;; The pattern matching multiplication by a constant value.
+  (imul $x $C)
+
+  ;; The precondition that $C must be a power of two.
+  (is-power-of-two $C))
+ ```
+
+In the right-hand side, we use **unquoting** to perform compile-time evaluation
+of `log2($C)`. Unquoting is done with the `$(...)` form:
+
+```lisp
+;; Our new right-hand side, using unqouting to do compile-time evaluation of
+;; constants that were matched and bound in the left-hand side!
+(ishl $x $(log2 $C))
+```
+
+Finally, here is the general optimization putting our new left-hand and
+right-hand sides together:
+
+```lisp
+(=> (when (imul $x $C)
+          (is-power-of-two $C))
+    (ishl $x $(log2 $C)))
+```
+
+### Bit Widths
+
+Similar to how Cranelift's instructions are bit-width polymorphic, `peepmatic`
+optimizations are also bit-width polymorphic. Unless otherwise specified, a
+pattern will match expressions manipulating `i32`s just the same as expressions
+manipulating `i64`s, etc... An optimization that doesn't constrain its pattern's
+bit widths must be valid for all bit widths:
+
+* 1
+* 8
+* 16
+* 32
+* 64
+* 128
+
+To constrain an optimization to only match `i32`s, for example, you can use the
+`bit-width` precondition:
+
+```lisp
+(=> (when (iadd $x $y)
+          (bit-width $x 32)
+          (bit-width $y 32))
+    ...)
+```
+
+Alternatively, you can ascribe a type to an operation by putting the type inside
+curly brackets after the operator, like this:
+
+```lisp
+(=> (when (sextend{i64} (ireduce{i32} $x))
+          (bit-width $x 64))
+    (sshr (ishl $x 32) 32))
+```
+
+## Implementation
+
+Peepmatic has roughly four phases:
+
+1. Parsing
+2. Type Checking
+3. Linearization
+4. Automatization
+
+(I say "roughly" because there are a couple micro-passes that happen after
+linearization and before automatization. But those are the four main phases.)
+
+### Parsing
+
+Parsing transforms the DSL source text into an abstract syntax tree (AST).
+
+We use [the `wast` crate][wast]. It gives us nicely formatted errors with source
+context, as well as some other generally nice-to-have parsing infrastructure.
+
+Relevant source files:
+
+* `src/parser.rs`
+* `src/ast.rs`
+
+[wast]: https://crates.io/crates/wast
+
+### Type Checking
+
+Type checking operates on the AST. It checks that types and bit widths in the
+optimizations are all valid. For example, it ensures that the type and bit width
+of an optimization's left-hand side is the same as its right-hand side, because
+it doesn't make sense to replace an integer expression with a boolean
+expression.
+
+After type checking is complete, certain AST nodes are assigned a type and bit
+width, that are later used in linearization and when matching and applying
+optimizations.
+
+We walk the AST and gather type constraints. Every constraint is associated with
+a span in the source file. We hand these constraints off to Z3. In the case that
+there are type errors (i.e. Z3 returns `unsat`), we get the constraints that are
+in conflict with each othe via `z3::Solver::get_unsat_core` and report the type
+errors to the user, with the source context, thanks to the constraints'
+associated spans.
+
+Using Z3 not only makes implementing type checking easier than it otherwise
+would be, but makes it that much easier to extend type checking with searching
+for counterexample inputs in the future. That is, inputs for which the RHS is
+not equivalent to the LHS, implying that the optimization is unsound.
+
+Relevant source files:
+
+* `src/verify.rs`
+
+### Linearization
+
+Linearization takes the AST of optimizations and converts each optimization into
+a linear form. The goal is to make automaton construction easier in the
+automatization step, as well as simplifying the language to make matching and
+applying optimizations easier.
+
+Each optimization's left-hand side is converted into a sequence of
+
+* match operation,
+* path to the instruction/value/immediate to which the operation is applied, and
+* expected result of the operation.
+
+All match operations must have the expected result for the optimization to be
+applicable to an instruction sequence.
+
+Each optimization's right-hand side is converted into a sequence of build
+actions. These are commands that describe how to construct the right-hand side,
+given that the left-hand side has been matched.
+
+Relevant source files:
+
+* `src/linearize.rs`
+* `src/linear_passes.rs`
+* `crates/runtime/src/linear.rs`
+
+### Automatization
+
+Automatization takes a set of linear optimizations and combines them into a
+transducer automaton. This automaton is the final, compiled peephole
+optimizations. The goal is to de-duplicate as much as we can from all the linear
+optimizations, producing as compact and cache-friendly a representation as we
+can.
+
+Plain automata can tell you whether it matches an input string. It can be
+thought of as a compact representation of a set of strings. A transducer is a
+type of automaton that doesn't just match input strings, but can map them to
+output values. It can be thought of as a compact representation of a dictionary
+or map. By using transducers, we de-duplicate not only the prefix and suffix of
+the match operations, but also the right-hand side build actions.
+
+Each state in the emitted transducers is associated with a match operation and
+path. The transitions out of that state are over the result of the match
+operation. Each transition optionally accumulates some RHS build actions. By the
+time we reach a final state, the RHS build actions are complete and can be
+interpreted to apply the matched optimization.
+
+The relevant source files for constructing the transducer automaton are:
+
+* `src/automatize.rs`
+* `crates/automata/src/lib.rs`
+
+The relevant source files for the runtime that interprets the transducers and
+applies optimizations are:
+
+* `crates/runtime/src/optimizations.rs`
+* `crates/runtime/src/optimizer.rs`
+
+## References
+
+I found these resources helpful when designing `peepmatic`:
+
+* [Extending tree pattern matching for application to peephole
+  optimizations](https://pure.tue.nl/ws/portalfiles/portal/125543109/Thesis_JanekvOirschot.pdf)
+  by van Oirschot
+
+* [Interpreted Pattern Match Execution for
+  MLIR](https://drive.google.com/drive/folders/1hb_sXbdMbIz95X-aaa6Vf5wSYRwsJuve)
+  by Jeff Niu
+
+* [Direct Construction of Minimal Acyclic Subsequential
+  Transducers](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.24.3698&rep=rep1&type=pdf)
+  by Mihov and Maurel
+
+* [Index 1,600,000,000 Keys with Automata and
+  Rust](https://blog.burntsushi.net/transducers/) and [the `fst`
+  crate](https://crates.io/crates/fst) by Andrew Gallant
+
+## Acknowledgments
+
+Thanks to [Jubi Taneja], [Dan Gohman], [John Regehr], and [Nuno Lopes] for their
+input in design discussions and for sharing helpful resources!
+
+[Jubi Taneja]: https://www.cs.utah.edu/~jubi/
+[Dan Gohman]: https://github.com/sunfishcode
+[John Regehr]: https://www.cs.utah.edu/~regehr/
+[Nuno Lopes]: http://web.ist.utl.pt/nuno.lopes/

cranelift/peepmatic/crates/automata/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "peepmatic-automata"
+version = "0.1.0"
+authors = ["Nick Fitzgerald <fitzgen@gmail.com>"]
+edition = "2018"
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+serde = { version = "1.0.106", optional = true }
+
+[features]
+# Enable support for generating GraphViz Dot files that can be used to visually
+# render an automaton.
+#
+# https://en.wikipedia.org/wiki/DOT_%28graph_description_language%29
+dot = []

cranelift/peepmatic/crates/automata/src/dot.rs

@@ -0,0 +1,273 @@
+//! Helpers for generating [GraphViz
+//! Dot](https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf) files to visually
+//! render automata.
+//!
+//! **This module only exists when the `"dot"` cargo feature is enabled.**
+
+use crate::{Automaton, Output, State};
+use std::fmt::{Debug, Display};
+use std::fs;
+use std::hash::Hash;
+use std::io::{self, Write};
+use std::path::Path;
+
+/// Format the user-provided bits of an `Automaton` for Graphviz Dot output.
+///
+/// There are two provided implementations of `DotFmt`:
+///
+/// * [`DebugDotFmt`][crate::dot::DebugDotFmt] -- format each type parameter
+///   with its `std::fmt::Debug` implementation.
+///
+/// * [`DisplayDotFmt`][crate::dot::DisplayDotFmt] -- format each type parameter
+///   with its `std::fmt::Display` implementation.
+///
+/// You can also implement this trait yourself if your type parameters don't
+/// implement `Debug` or `Display`, or if you want to format them in some other
+/// way.
+pub trait DotFmt<TAlphabet, TState, TOutput> {
+    /// Format a transition edge: `from ---input---> to`.
+    ///
+    /// This will be inside an [HTML
+    /// label](https://www.graphviz.org/doc/info/shapes.html#html), so you may
+    /// use balanced HTML tags.
+    fn fmt_transition(
+        &self,
+        w: &mut impl Write,
+        from: Option<&TState>,
+        input: &TAlphabet,
+        to: Option<&TState>,
+    ) -> io::Result<()>;
+
+    /// Format the custom data associated with a state.
+    ///
+    /// This will be inside an [HTML
+    /// label](https://www.graphviz.org/doc/info/shapes.html#html), so you may
+    /// use balanced HTML tags.
+    fn fmt_state(&self, w: &mut impl Write, state: &TState) -> io::Result<()>;
+
+    /// Format a transition's output or the final output of a final state.
+    ///
+    /// This will be inside an [HTML
+    /// label](https://www.graphviz.org/doc/info/shapes.html#html), so you may
+    /// use balanced HTML tags.
+    fn fmt_output(&self, w: &mut impl Write, output: &TOutput) -> io::Result<()>;
+}
+
+impl<TAlphabet, TState, TOutput> Automaton<TAlphabet, TState, TOutput>
+where
+    TAlphabet: Clone + Eq + Hash + Ord,
+    TState: Clone + Eq + Hash,
+    TOutput: Output,
+{
+    /// Write this `Automaton` out as a [GraphViz
+    /// Dot](https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf) file at the
+    /// given path.
+    ///
+    /// The `formatter` parameter controls how `TAlphabet`, `TState`, and
+    /// `TOutput` are rendered. See the [`DotFmt`][crate::dot::DotFmt] trait for
+    /// details.
+    ///
+    /// **This method only exists when the `"dot"` cargo feature is enabled.**
+    pub fn write_dot_file(
+        &self,
+        formatter: &impl DotFmt<TAlphabet, TState, TOutput>,
+        path: impl AsRef<Path>,
+    ) -> io::Result<()> {
+        let mut file = fs::File::create(path)?;
+        self.write_dot(formatter, &mut file)?;
+        Ok(())
+    }
+
+    /// Write this `Automaton` out to the given write-able as a [GraphViz
+    /// Dot](https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf) file.
+    ///
+    /// The `formatter` parameter controls how `TAlphabet`, `TState`, and
+    /// `TOutput` are rendered. See the [`DotFmt`][crate::dot::DotFmt] trait for
+    /// details.
+    ///
+    /// **This method only exists when the `"dot"` cargo feature is enabled.**
+    pub fn write_dot(
+        &self,
+        formatter: &impl DotFmt<TAlphabet, TState, TOutput>,
+        w: &mut impl Write,
+    ) -> io::Result<()> {
+        writeln!(w, "digraph {{")?;
+        writeln!(w, "  rankdir = \"LR\";")?;
+        writeln!(w, "  nodesep = 2;")?;
+
+        // Fake state for the incoming arrow to the start state.
+        writeln!(w, "  \"\" [shape = none];")?;
+
+        // Each state, its associated custom data, and its final output.
+        for (i, state_data) in self.state_data.iter().enumerate() {
+            write!(
+                w,
+                r#"  state_{i} [shape = {shape}, label = <<table border="0"><tr><td cellpadding="5">{i}</td></tr><tr><td cellpadding="5">"#,
+                i = i,
+                shape = if self.final_states.contains_key(&State(i as u32)) {
+                    "doublecircle"
+                } else {
+                    "circle"
+                }
+            )?;
+            if let Some(state_data) = state_data {
+                formatter.fmt_state(w, state_data)?;
+            } else {
+                write!(w, "(no state data)")?;
+            }
+            write!(w, "</td></tr>")?;
+            if let Some(final_output) = self.final_states.get(&State(i as u32)) {
+                write!(w, r#"<tr><td cellpadding="5" align="left">"#)?;
+                formatter.fmt_output(w, final_output)?;
+                write!(w, "</td></tr>")?;
+            }
+            writeln!(w, "</table>>];")?;
+        }
+
+        // Fake transition to the start state.
+        writeln!(w, r#"  "" -> state_{};"#, self.start_state.0)?;
+
+        // Transitions between states and their outputs.
+        for (from, transitions) in self.transitions.iter().enumerate() {
+            for (input, (to, output)) in transitions {
+                write!(
+                    w,
+                    r#"  state_{from} -> state_{to} [label = <<table border="0"><tr><td cellpadding="5" align="left">Input:</td><td cellpadding="5" align="left">"#,
+                    from = from,
+                    to = to.0,
+                )?;
+                formatter.fmt_transition(
+                    w,
+                    self.state_data[from].as_ref(),
+                    input,
+                    self.state_data[to.0 as usize].as_ref(),
+                )?;
+                write!(
+                    w,
+                    r#"</td></tr><tr><td cellpadding="5" align="left">Output:</td><td cellpadding="5" align="left">"#,
+                )?;
+                formatter.fmt_output(w, output)?;
+                writeln!(w, "</td></tr></table>>];")?;
+            }
+        }
+
+        writeln!(w, "}}")?;
+        Ok(())
+    }
+}
+
+/// Format an `Automaton`'s `TAlphabet`, `TState`, and `TOutput` with their
+/// `std::fmt::Debug` implementations.
+#[derive(Debug)]
+pub struct DebugDotFmt;
+
+impl<TAlphabet, TState, TOutput> DotFmt<TAlphabet, TState, TOutput> for DebugDotFmt
+where
+    TAlphabet: Debug,
+    TState: Debug,
+    TOutput: Debug,
+{
+    fn fmt_transition(
+        &self,
+        w: &mut impl Write,
+        _from: Option<&TState>,
+        input: &TAlphabet,
+        _to: Option<&TState>,
+    ) -> io::Result<()> {
+        write!(w, r#"<font face="monospace">{:?}</font>"#, input)
+    }
+
+    fn fmt_state(&self, w: &mut impl Write, state: &TState) -> io::Result<()> {
+        write!(w, r#"<font face="monospace">{:?}</font>"#, state)
+    }
+
+    fn fmt_output(&self, w: &mut impl Write, output: &TOutput) -> io::Result<()> {
+        write!(w, r#"<font face="monospace">{:?}</font>"#, output)
+    }
+}
+
+/// Format an `Automaton`'s `TAlphabet`, `TState`, and `TOutput` with their
+/// `std::fmt::Display` implementations.
+#[derive(Debug)]
+pub struct DisplayDotFmt;
+
+impl<TAlphabet, TState, TOutput> DotFmt<TAlphabet, TState, TOutput> for DisplayDotFmt
+where
+    TAlphabet: Display,
+    TState: Display,
+    TOutput: Display,
+{
+    fn fmt_transition(
+        &self,
+        w: &mut impl Write,
+        _from: Option<&TState>,
+        input: &TAlphabet,
+        _to: Option<&TState>,
+    ) -> io::Result<()> {
+        write!(w, "{}", input)
+    }
+
+    fn fmt_state(&self, w: &mut impl Write, state: &TState) -> io::Result<()> {
+        write!(w, "{}", state)
+    }
+
+    fn fmt_output(&self, w: &mut impl Write, output: &TOutput) -> io::Result<()> {
+        write!(w, "{}", output)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::Builder;
+
+    #[test]
+    fn test_write_dot() {
+        let mut builder = Builder::<char, (), u64>::new();
+
+        // Insert "mon" -> 1
+        let mut insertion = builder.insert();
+        insertion.next('m', 1).next('o', 0).next('n', 0);
+        insertion.finish();
+
+        // Insert "sat" -> 6
+        let mut insertion = builder.insert();
+        insertion.next('s', 6).next('a', 0).next('t', 0);
+        insertion.finish();
+
+        // Insert "sun" -> 0
+        let mut insertion = builder.insert();
+        insertion.next('s', 0).next('u', 0).next('n', 0);
+        insertion.finish();
+
+        let automata = builder.finish();
+
+        let expected = r#"
+digraph {
+  rankdir = "LR";
+  nodesep = 2;
+  "" [shape = none];
+  state_0 [shape = doublecircle, label = <<table border="0"><tr><td cellpadding="5">0</td></tr><tr><td cellpadding="5">(no state data)</td></tr><tr><td cellpadding="5" align="left"><font face="monospace">0</font></td></tr></table>>];
+  state_1 [shape = circle, label = <<table border="0"><tr><td cellpadding="5">1</td></tr><tr><td cellpadding="5">(no state data)</td></tr></table>>];
+  state_2 [shape = circle, label = <<table border="0"><tr><td cellpadding="5">2</td></tr><tr><td cellpadding="5">(no state data)</td></tr></table>>];
+  state_3 [shape = circle, label = <<table border="0"><tr><td cellpadding="5">3</td></tr><tr><td cellpadding="5">(no state data)</td></tr></table>>];
+  state_4 [shape = circle, label = <<table border="0"><tr><td cellpadding="5">4</td></tr><tr><td cellpadding="5">(no state data)</td></tr></table>>];
+  state_5 [shape = circle, label = <<table border="0"><tr><td cellpadding="5">5</td></tr><tr><td cellpadding="5">(no state data)</td></tr></table>>];
+  "" -> state_5;
+  state_1 -> state_0 [label = <<table border="0"><tr><td cellpadding="5" align="left">Input:</td><td cellpadding="5" align="left"><font face="monospace">'n'</font></td></tr><tr><td cellpadding="5" align="left">Output:</td><td cellpadding="5" align="left"><font face="monospace">0</font></td></tr></table>>];
+  state_2 -> state_1 [label = <<table border="0"><tr><td cellpadding="5" align="left">Input:</td><td cellpadding="5" align="left"><font face="monospace">'o'</font></td></tr><tr><td cellpadding="5" align="left">Output:</td><td cellpadding="5" align="left"><font face="monospace">0</font></td></tr></table>>];
+  state_3 -> state_0 [label = <<table border="0"><tr><td cellpadding="5" align="left">Input:</td><td cellpadding="5" align="left"><font face="monospace">'t'</font></td></tr><tr><td cellpadding="5" align="left">Output:</td><td cellpadding="5" align="left"><font face="monospace">0</font></td></tr></table>>];
+  state_4 -> state_3 [label = <<table border="0"><tr><td cellpadding="5" align="left">Input:</td><td cellpadding="5" align="left"><font face="monospace">'a'</font></td></tr><tr><td cellpadding="5" align="left">Output:</td><td cellpadding="5" align="left"><font face="monospace">6</font></td></tr></table>>];
+  state_4 -> state_1 [label = <<table border="0"><tr><td cellpadding="5" align="left">Input:</td><td cellpadding="5" align="left"><font face="monospace">'u'</font></td></tr><tr><td cellpadding="5" align="left">Output:</td><td cellpadding="5" align="left"><font face="monospace">0</font></td></tr></table>>];
+  state_5 -> state_2 [label = <<table border="0"><tr><td cellpadding="5" align="left">Input:</td><td cellpadding="5" align="left"><font face="monospace">'m'</font></td></tr><tr><td cellpadding="5" align="left">Output:</td><td cellpadding="5" align="left"><font face="monospace">1</font></td></tr></table>>];
+  state_5 -> state_4 [label = <<table border="0"><tr><td cellpadding="5" align="left">Input:</td><td cellpadding="5" align="left"><font face="monospace">'s'</font></td></tr><tr><td cellpadding="5" align="left">Output:</td><td cellpadding="5" align="left"><font face="monospace">0</font></td></tr></table>>];
+}
+"#;
+
+        let mut buf = vec![];
+        automata.write_dot(&DebugDotFmt, &mut buf).unwrap();
+        let actual = String::from_utf8(buf).unwrap();
+        eprintln!("{}", actual);
+        assert_eq!(expected.trim(), actual.trim());
+    }
+}

cranelift/peepmatic/crates/automata/src/output_impls.rs

@@ -0,0 +1,130 @@
+use crate::Output;
+use std::cmp;
+use std::hash::Hash;
+
+impl Output for u64 {
+    fn empty() -> Self {
+        0
+    }
+
+    fn prefix(a: &Self, b: &Self) -> Self {
+        cmp::min(*a, *b)
+    }
+
+    fn difference(a: &Self, b: &Self) -> Self {
+        a - b
+    }
+
+    fn concat(a: &Self, b: &Self) -> Self {
+        a + b
+    }
+}
+
+impl<T> Output for Vec<T>
+where
+    T: Clone + Eq + Hash,
+{
+    fn empty() -> Self {
+        vec![]
+    }
+
+    fn is_empty(&self) -> bool {
+        self.is_empty()
+    }
+
+    fn prefix(a: &Self, b: &Self) -> Self {
+        a.iter()
+            .cloned()
+            .zip(b.iter().cloned())
+            .take_while(|(a, b)| a == b)
+            .map(|(a, _)| a)
+            .collect()
+    }
+
+    fn difference(a: &Self, b: &Self) -> Self {
+        let i = a
+            .iter()
+            .zip(b.iter())
+            .position(|(a, b)| a != b)
+            .unwrap_or(cmp::min(a.len(), b.len()));
+        a[i..].to_vec()
+    }
+
+    fn concat(a: &Self, b: &Self) -> Self {
+        let mut c = a.clone();
+        c.extend(b.iter().cloned());
+        c
+    }
+}
+
+impl<T> Output for Box<[T]>
+where
+    T: Clone + Eq + Hash,
+{
+    fn empty() -> Self {
+        vec![].into_boxed_slice()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    fn prefix(a: &Self, b: &Self) -> Self {
+        a.iter()
+            .cloned()
+            .zip(b.iter().cloned())
+            .take_while(|(a, b)| a == b)
+            .map(|(a, _)| a)
+            .collect()
+    }
+
+    fn difference(a: &Self, b: &Self) -> Self {
+        let i = a
+            .iter()
+            .zip(b.iter())
+            .position(|(a, b)| a != b)
+            .unwrap_or(cmp::min(a.len(), b.len()));
+        a[i..].to_vec().into_boxed_slice()
+    }
+
+    fn concat(a: &Self, b: &Self) -> Self {
+        let mut c = a.clone().to_vec();
+        c.extend(b.iter().cloned());
+        c.into_boxed_slice()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::Output;
+    use std::fmt::Debug;
+
+    // Assert the laws that `Output` requires for correctness. `a` and `b`
+    // should be two different instances of an `Output` type.
+    fn assert_laws<O>(a: O, b: O)
+    where
+        O: Clone + Debug + Output,
+    {
+        // Law 1
+        assert_eq!(O::concat(&O::empty(), &a), a.clone());
+
+        // Law 2
+        assert_eq!(O::prefix(&b, &a), O::prefix(&a, &b));
+
+        // Law 3
+        assert_eq!(O::prefix(&O::empty(), &a), O::empty());
+
+        // Law 4
+        assert_eq!(O::difference(&O::concat(&a, &b), &a), b);
+    }
+
+    #[test]
+    fn impl_for_u64() {
+        assert_laws(3, 5);
+    }
+
+    #[test]
+    fn impl_for_vec() {
+        assert_laws(vec![0, 1, 2, 3], vec![0, 2, 4, 6]);
+    }
+}

cranelift/peepmatic/crates/automata/src/serde_impls.rs

@@ -0,0 +1,220 @@
+//! `serde::Serialize` and `serde::Deserialize` implementations for `Automaton`.
+//!
+//! Rather than prefix each serialized field with which field it is, we always
+//! serialize fields in alphabetical order. Make sure to maintain this if you
+//! add or remove fields!
+//!
+//! Each time you add/remove a field, or change serialization in any other way,
+//! make sure to bump `SERIALIZATION_VERSION`.
+
+use crate::{Automaton, Output, State};
+use serde::{
+    de::{self, Deserializer, SeqAccess, Visitor},
+    ser::SerializeTupleStruct,
+    Deserialize, Serialize, Serializer,
+};
+use std::collections::BTreeMap;
+use std::fmt;
+use std::hash::Hash;
+use std::marker::PhantomData;
+
+const SERIALIZATION_VERSION: u32 = 1;
+
+impl Serialize for State {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_u32(self.0)
+    }
+}
+
+impl<'de> Deserialize<'de> for State {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Ok(State(deserializer.deserialize_u32(U32Visitor)?))
+    }
+}
+
+struct U32Visitor;
+
+impl<'de> Visitor<'de> for U32Visitor {
+    type Value = u32;
+
+    fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str("an integer between `0` and `2^32 - 1`")
+    }
+
+    fn visit_u8<E>(self, value: u8) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(u32::from(value))
+    }
+
+    fn visit_u32<E>(self, value: u32) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(value)
+    }
+
+    fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        use std::u32;
+        if value <= u64::from(u32::MAX) {
+            Ok(value as u32)
+        } else {
+            Err(E::custom(format!("u32 out of range: {}", value)))
+        }
+    }
+}
+
+impl<TAlphabet, TState, TOutput> Serialize for Automaton<TAlphabet, TState, TOutput>
+where
+    TAlphabet: Serialize + Clone + Eq + Hash + Ord,
+    TState: Serialize + Clone + Eq + Hash,
+    TOutput: Serialize + Output,
+{
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let Automaton {
+            final_states,
+            start_state,
+            state_data,
+            transitions,
+        } = self;
+
+        let mut s = serializer.serialize_tuple_struct("Automaton", 5)?;
+        s.serialize_field(&SERIALIZATION_VERSION)?;
+        s.serialize_field(final_states)?;
+        s.serialize_field(start_state)?;
+        s.serialize_field(state_data)?;
+        s.serialize_field(transitions)?;
+        s.end()
+    }
+}
+
+impl<'de, TAlphabet, TState, TOutput> Deserialize<'de> for Automaton<TAlphabet, TState, TOutput>
+where
+    TAlphabet: 'de + Deserialize<'de> + Clone + Eq + Hash + Ord,
+    TState: 'de + Deserialize<'de> + Clone + Eq + Hash,
+    TOutput: 'de + Deserialize<'de> + Output,
+{
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_tuple_struct(
+            "Automaton",
+            5,
+            AutomatonVisitor {
+                phantom: PhantomData,
+            },
+        )
+    }
+}
+
+struct AutomatonVisitor<'de, TAlphabet, TState, TOutput>
+where
+    TAlphabet: 'de + Deserialize<'de> + Clone + Eq + Hash + Ord,
+    TState: 'de + Deserialize<'de> + Clone + Eq + Hash,
+    TOutput: 'de + Deserialize<'de> + Output,
+{
+    phantom: PhantomData<&'de (TAlphabet, TState, TOutput)>,
+}
+
+impl<'de, TAlphabet, TState, TOutput> Visitor<'de>
+    for AutomatonVisitor<'de, TAlphabet, TState, TOutput>
+where
+    TAlphabet: 'de + Deserialize<'de> + Clone + Eq + Hash + Ord,
+    TState: 'de + Deserialize<'de> + Clone + Eq + Hash,
+    TOutput: 'de + Deserialize<'de> + Output,
+{
+    type Value = Automaton<TAlphabet, TState, TOutput>;
+
+    fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str("Automaton")
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: SeqAccess<'de>,
+    {
+        match seq.next_element::<u32>()? {
+            Some(v) if v == SERIALIZATION_VERSION => {}
+            Some(v) => {
+                return Err(de::Error::invalid_value(
+                    de::Unexpected::Unsigned(v as u64),
+                    &self,
+                ));
+            }
+            None => {
+                return Err(de::Error::invalid_length(
+                    0,
+                    &"Automaton expects 5 elements",
+                ))
+            }
+        }
+
+        let final_states = match seq.next_element::<BTreeMap<State, TOutput>>()? {
+            Some(x) => x,
+            None => {
+                return Err(de::Error::invalid_length(
+                    1,
+                    &"Automaton expects 5 elements",
+                ))
+            }
+        };
+
+        let start_state = match seq.next_element::<State>()? {
+            Some(x) => x,
+            None => {
+                return Err(de::Error::invalid_length(
+                    2,
+                    &"Automaton expects 5 elements",
+                ))
+            }
+        };
+
+        let state_data = match seq.next_element::<Vec<Option<TState>>>()? {
+            Some(x) => x,
+            None => {
+                return Err(de::Error::invalid_length(
+                    3,
+                    &"Automaton expects 5 elements",
+                ))
+            }
+        };
+
+        let transitions = match seq.next_element::<Vec<BTreeMap<TAlphabet, (State, TOutput)>>>()? {
+            Some(x) => x,
+            None => {
+                return Err(de::Error::invalid_length(
+                    4,
+                    &"Automaton expects 5 elements",
+                ))
+            }
+        };
+
+        let automata = Automaton {
+            final_states,
+            start_state,
+            state_data,
+            transitions,
+        };
+
+        // Ensure that the deserialized automata is well-formed.
+        automata
+            .check_representation()
+            .map_err(|msg| de::Error::custom(msg))?;
+
+        Ok(automata)
+    }
+}

cranelift/peepmatic/crates/fuzzing/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "peepmatic-fuzzing"
+version = "0.1.0"
+authors = ["Nick Fitzgerald <fitzgen@gmail.com>"]
+edition = "2018"
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+arbitrary = { version = "0.4.1", features = ["derive"] }
+bincode = "1.2.1"
+env_logger = "0.7.1"
+fst = "0.4.1"
+log = "0.4.8"
+peepmatic = { path = "../.." }
+peepmatic-automata = { path = "../automata", features = ["serde"] }
+peepmatic-runtime = { path = "../runtime", features = ["construct"] }
+peepmatic-test = { path = "../test" }
+rand = { version = "0.7.3", features = ["small_rng"] }
+serde = "1.0.106"
+wast = "15.0.0"

cranelift/peepmatic/crates/fuzzing/src/automata.rs

@@ -0,0 +1,187 @@
+//! Helpers for fuzzing the `peepmatic-automata` crate.
+
+use peepmatic_automata::{Automaton, Builder, Output};
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::hash::Hash;
+
+fn serde_roundtrip<TAlphabet, TState, TOutput>(
+    automata: Automaton<TAlphabet, TState, TOutput>,
+) -> Automaton<TAlphabet, TState, TOutput>
+where
+    TAlphabet: Serialize + for<'de> Deserialize<'de> + Clone + Eq + Hash + Ord,
+    TState: Serialize + for<'de> Deserialize<'de> + Clone + Eq + Hash,
+    TOutput: Serialize + for<'de> Deserialize<'de> + Output,
+{
+    let encoded: Vec<u8> = bincode::serialize(&automata).expect("should serialize OK");
+    bincode::deserialize(&encoded).expect("should deserialize OK")
+}
+
+/// Construct an automaton from the the given input-output pairs, and assert
+/// that:
+///
+/// * Putting in each of the input strings should result in the expected output
+///   string.
+///
+/// * Putting in an input string that is not one of the given inputs from our
+///   input-output pairs should never yield an output value.
+pub fn simple_automata(input_output_pairs: Vec<Vec<(u8, Vec<u8>)>>) {
+    let _ = env_logger::try_init();
+
+    let full_input = |pair: &[(u8, Vec<u8>)]| {
+        let mut full_input = vec![];
+        for (input, _) in pair {
+            full_input.push(*input);
+        }
+        full_input
+    };
+
+    let mut inputs = HashSet::new();
+
+    let mut input_output_pairs: Vec<_> = input_output_pairs
+        .into_iter()
+        .filter(|pair| {
+            !pair.is_empty() && {
+                // Make sure we don't have duplicate inputs.
+                let is_new = inputs.insert(full_input(pair));
+                is_new
+            }
+        })
+        .collect();
+
+    input_output_pairs.sort_by(|a, b| full_input(a).cmp(&full_input(b)));
+
+    if input_output_pairs.is_empty() {
+        return;
+    }
+
+    // A map from one of our concatenated input strings to its concatenated
+    // output.
+    let mut expected = HashMap::with_capacity(input_output_pairs.len());
+
+    let mut builder = Builder::<u8, (), Vec<u8>>::new();
+    for pair in &input_output_pairs {
+        let mut full_input = vec![];
+        let mut full_output = vec![];
+
+        let mut ins = builder.insert();
+        for (input, output) in pair.iter().cloned() {
+            full_input.push(input);
+            full_output.extend(output.iter().copied());
+
+            ins.next(input, output);
+        }
+
+        let old = expected.insert(full_input, full_output);
+        assert!(old.is_none());
+
+        ins.finish();
+    }
+
+    let automata = builder.finish();
+    let automata = serde_roundtrip(automata);
+
+    // Assert that each of our input strings yields the expected output.
+    for (input, expected_output) in &expected {
+        log::debug!("Testing input: {:?}", input);
+        let actual_output = automata.get(input);
+        assert!(actual_output.is_some());
+        assert_eq!(actual_output.as_ref().unwrap(), expected_output);
+    }
+
+    // Test that mutations of our input strings (that aren't themselves other
+    // input strings!) do not yeild any output.
+    for input in expected.keys() {
+        for i in 0..input.len() {
+            let mut mutated = input.clone();
+            mutated[i] = mutated[i].wrapping_add(1);
+            log::debug!("Testing mutated input: {:?}", mutated);
+            if !expected.contains_key(&mutated) {
+                assert!(automata.get(&mutated).is_none());
+            }
+        }
+    }
+}
+
+/// Do differential testing against the `fst` crate, which is another
+/// implementation of the algorithm we use for finite-state transducer
+/// construction in `peepmatic-automata`.
+pub fn fst_differential(map: HashMap<Vec<u8>, u64>) {
+    let _ = env_logger::try_init();
+
+    let mut inputs: Vec<_> = map.keys().filter(|i| !i.is_empty()).cloned().collect();
+    inputs.sort();
+    inputs.dedup();
+    if inputs.is_empty() {
+        return;
+    }
+
+    let mut fst = fst::MapBuilder::memory();
+    let mut builder = Builder::<u8, (), u64>::new();
+
+    for inp in &inputs {
+        fst.insert(inp, map[inp]).unwrap();
+
+        let mut ins = builder.insert();
+        for (i, ch) in inp.iter().enumerate() {
+            ins.next(*ch, if i == 0 { map[inp] } else { 0 });
+        }
+        ins.finish();
+    }
+
+    let fst = fst.into_map();
+    let automata = builder.finish();
+    let automata = serde_roundtrip(automata);
+
+    for inp in inputs {
+        // Check we have the same result as `fst` for inputs we know are in the
+        // automata.
+        log::debug!("Testing input {:?}", inp);
+        let expected = fst.get(&inp).expect("`fst` should have entry for `inp`");
+        let actual = automata
+            .get(&inp)
+            .expect("automata should have entry for `inp`");
+        assert_eq!(expected, actual);
+
+        // Check that we have the same result as `fst` for inputs that may or
+        // may not be in the automata.
+        for i in 0..inp.len() {
+            let mut mutated = inp.clone();
+            mutated[i] = mutated[i].wrapping_add(1);
+            log::debug!("Testing mutated input {:?}", mutated);
+            let expected = fst.get(&mutated);
+            let actual = automata.get(&mutated);
+            assert_eq!(expected, actual);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn check_simple_automata() {
+        crate::check(simple_automata);
+    }
+
+    #[test]
+    fn check_fst_differential() {
+        crate::check(fst_differential);
+    }
+
+    #[test]
+    fn regression_test_0() {
+        simple_automata(vec![vec![(0, vec![0]), (0, vec![1])], vec![(0, vec![2])]]);
+    }
+
+    #[test]
+    fn regression_test_1() {
+        fst_differential(vec![(vec![1, 3], 5), (vec![1, 2], 4)].into_iter().collect());
+    }
+
+    #[test]
+    fn regression_test_2() {
+        simple_automata(vec![vec![(0, vec![11]), (0, vec![])], vec![(0, vec![11])]]);
+    }
+}

cranelift/peepmatic/crates/fuzzing/src/compile.rs

@@ -0,0 +1,71 @@
+//! Fuzz testing utilities related to AST pattern matching.
+
+use peepmatic_runtime::PeepholeOptimizations;
+use std::path::Path;
+use std::str;
+
+/// Attempt to interpret the given bytes as UTF-8 and then compile them as if
+/// they were source text of our DSL.
+pub fn compile(data: &[u8]) {
+    let source = match str::from_utf8(data) {
+        Err(_) => return,
+        Ok(s) => s,
+    };
+
+    let opt = match peepmatic::compile_str(source, Path::new("fuzz")) {
+        Err(_) => return,
+        Ok(o) => o,
+    };
+
+    // Should be able to serialize and deserialize the peephole optimizer.
+    let opt_bytes = bincode::serialize(&opt).expect("should serialize peephole optimizations OK");
+    let _: PeepholeOptimizations =
+        bincode::deserialize(&opt_bytes).expect("should deserialize peephole optimizations OK");
+
+    // Compiling the same source text again should be deterministic.
+    let opt2 = peepmatic::compile_str(source, Path::new("fuzz"))
+        .expect("should be able to compile source text again, if it compiled OK the first time");
+    let opt2_bytes =
+        bincode::serialize(&opt2).expect("should serialize second peephole optimizations OK");
+    assert_eq!(opt_bytes, opt2_bytes);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn check_compile() {
+        crate::check(|s: String| compile(s.as_bytes()));
+    }
+
+    #[test]
+    fn regression_0() {
+        compile(
+            b"
+            (=> (bor (bor $x $y) $y) $x)
+            (=> (bor (bor $x $z) $y) $x)
+        ",
+        );
+    }
+
+    #[test]
+    fn regression_1() {
+        compile(
+            b"
+            (=> (bor (bor $x $y) 0) $x)
+            (=> (bor $x 0) $x)
+            (=> (bor $y $x) $x)
+        ",
+        );
+    }
+
+    #[test]
+    fn regression_2() {
+        compile(
+            b"
+            (=> (sshr $x 11111111110) $x)
+        ",
+        );
+    }
+}

cranelift/peepmatic/crates/fuzzing/src/interp.rs

@@ -0,0 +1,374 @@
+//! Interpreting compiled peephole optimizations against test instruction sequences.
+
+use peepmatic::{
+    Constraint, Dfs, DynAstRef, Optimizations, Pattern, Span, TraversalEvent, ValueLiteral,
+    Variable,
+};
+use peepmatic_runtime::{
+    cc::ConditionCode,
+    operator::TypingContext as TypingContextTrait,
+    part::Constant,
+    r#type::BitWidth,
+    r#type::{Kind, Type},
+};
+use peepmatic_test::{Program, TestIsa};
+use std::collections::{BTreeMap, HashMap};
+use std::path::Path;
+use std::str;
+
+/// Compile the given source text, and if it is a valid set of optimizations,
+/// then interpret the optimizations against test instruction sequences created
+/// to reflect the optimizations.
+pub fn interp(data: &[u8]) {
+    let _ = env_logger::try_init();
+
+    let source = match str::from_utf8(data) {
+        Err(_) => return,
+        Ok(s) => s,
+    };
+
+    let peep_opts = match peepmatic::compile_str(source, Path::new("fuzz")) {
+        Err(_) => return,
+        Ok(o) => o,
+    };
+    let mut optimizer = peep_opts.optimizer(TestIsa {
+        native_word_size_in_bits: 32,
+    });
+
+    // Okay, we know it compiles and verifies alright, so (re)parse the AST.
+    let buf = wast::parser::ParseBuffer::new(&source).unwrap();
+    let ast = wast::parser::parse::<Optimizations>(&buf).unwrap();
+
+    // And we need access to the assigned types, so re-verify it as well.
+    peepmatic::verify(&ast).unwrap();
+
+    // Walk over each optimization and create an instruction sequence that
+    // matches the optimization.
+    let mut program = Program::default();
+    for opt in &ast.optimizations {
+        // The instruction sequence we generate must match an optimization (not
+        // necessarily *this* optimization, if there is another that is more
+        // specific but also matches) unless there is an `bit-width`
+        // precondition or an implicit `bit-width` precondition via a type
+        // ascription. When those things exist, we might have constructed
+        // instructions with the wrong bit widths to match.
+        let mut allow_no_match = false;
+
+        // The last instruction we generated. After we've generated the full
+        // instruction sequence, this will be its root.
+        let mut last_inst = None;
+
+        // Remember the instructions associated with variables and constants, so
+        // that when they appear multiple times, we reuse the same instruction.
+        let mut id_to_inst = HashMap::new();
+
+        // Map from a pattern's span to the instruction we generated for
+        // it. This allows parent operations to get the instructions for their
+        // children.
+        let mut span_to_inst = BTreeMap::new();
+
+        for (te, lhs) in Dfs::new(&opt.lhs) {
+            // NB: We use a post-order traversal because we want arguments to be
+            // generated before they are used.
+            if te != TraversalEvent::Exit {
+                continue;
+            }
+
+            match lhs {
+                DynAstRef::Precondition(p) => {
+                    allow_no_match |= p.constraint == Constraint::BitWidth;
+                }
+
+                DynAstRef::Pattern(Pattern::Operation(op)) => {
+                    allow_no_match |= op.r#type.get().is_some();
+
+                    let num_imms = op.operator.immediates_arity() as usize;
+
+                    // Generate this operation's immediates.
+                    let mut imm_tys = vec![];
+                    op.operator
+                        .immediate_types(&mut TypingContext, op.span(), &mut imm_tys);
+                    let imms: Vec<_> = op
+                        .operands
+                        .iter()
+                        .take(num_imms)
+                        .zip(imm_tys)
+                        .map(|(pat, ty)| match pat {
+                            Pattern::ValueLiteral(ValueLiteral::Integer(i)) => {
+                                Constant::Int(i.value as _, BitWidth::ThirtyTwo).into()
+                            }
+                            Pattern::ValueLiteral(ValueLiteral::Boolean(b)) => {
+                                Constant::Bool(b.value, BitWidth::One).into()
+                            }
+                            Pattern::ValueLiteral(ValueLiteral::ConditionCode(cc)) => cc.cc.into(),
+                            Pattern::Constant(_) | Pattern::Variable(_) => match ty {
+                                TypeOrConditionCode::ConditionCode => ConditionCode::Eq.into(),
+                                TypeOrConditionCode::Type(ty) => match ty.kind {
+                                    Kind::Int => Constant::Int(1, ty.bit_width).into(),
+                                    Kind::Bool => Constant::Bool(false, ty.bit_width).into(),
+                                    Kind::Void | Kind::CpuFlags => {
+                                        unreachable!("void and cpu flags cannot be immediates")
+                                    }
+                                },
+                            },
+                            Pattern::Operation(_) => {
+                                unreachable!("operations not allowed as immediates")
+                            }
+                        })
+                        .collect();
+
+                    // Generate (or collect already-generated) instructions for
+                    // this operation's arguments.
+                    let mut arg_tys = vec![];
+                    op.operator
+                        .param_types(&mut TypingContext, op.span(), &mut arg_tys);
+                    let args: Vec<_> = op
+                        .operands
+                        .iter()
+                        .skip(num_imms)
+                        .zip(arg_tys)
+                        .map(|(pat, ty)| match pat {
+                            Pattern::Operation(op) => span_to_inst[&op.span()],
+                            Pattern::ValueLiteral(ValueLiteral::Integer(i)) => program.r#const(
+                                Constant::Int(i.value as _, BitWidth::ThirtyTwo),
+                                BitWidth::ThirtyTwo,
+                            ),
+                            Pattern::ValueLiteral(ValueLiteral::Boolean(b)) => program.r#const(
+                                Constant::Bool(b.value, BitWidth::One),
+                                BitWidth::ThirtyTwo,
+                            ),
+                            Pattern::ValueLiteral(ValueLiteral::ConditionCode(_)) => {
+                                unreachable!("condition codes cannot be arguments")
+                            }
+                            Pattern::Constant(peepmatic::Constant { id, .. })
+                            | Pattern::Variable(Variable { id, .. }) => match ty {
+                                TypeOrConditionCode::Type(ty) => {
+                                    *id_to_inst.entry(id).or_insert_with(|| match ty.kind {
+                                        Kind::Int => program.r#const(
+                                            Constant::Int(1, ty.bit_width),
+                                            BitWidth::ThirtyTwo,
+                                        ),
+                                        Kind::Bool => program.r#const(
+                                            Constant::Bool(false, ty.bit_width),
+                                            BitWidth::ThirtyTwo,
+                                        ),
+                                        Kind::CpuFlags => {
+                                            unreachable!("cpu flags cannot be an argument")
+                                        }
+                                        Kind::Void => unreachable!("void cannot be an argument"),
+                                    })
+                                }
+                                TypeOrConditionCode::ConditionCode => {
+                                    unreachable!("condition codes cannot be arguments")
+                                }
+                            },
+                        })
+                        .collect();
+
+                    let ty = match op.operator.result_type(&mut TypingContext, op.span()) {
+                        TypeOrConditionCode::Type(ty) => ty,
+                        TypeOrConditionCode::ConditionCode => {
+                            unreachable!("condition codes cannot be operation results")
+                        }
+                    };
+                    let inst = program.new_instruction(op.operator, ty, imms, args);
+                    last_inst = Some(inst);
+                    let old_inst = span_to_inst.insert(op.span(), inst);
+                    assert!(old_inst.is_none());
+                }
+                _ => continue,
+            }
+        }
+
+        // Run the optimizer on our newly generated instruction sequence.
+        if let Some(inst) = last_inst {
+            let replacement = optimizer.apply_one(&mut program, inst);
+            assert!(
+                replacement.is_some() || allow_no_match,
+                "an optimization should match the generated instruction sequence"
+            );
+        }
+    }
+
+    // Finally, just try and run the optimizer on every instruction we
+    // generated, just to potentially shake out some more bugs.
+    let instructions: Vec<_> = program.instructions().map(|(k, _)| k).collect();
+    for inst in instructions {
+        let _ = optimizer.apply_one(&mut program, inst);
+    }
+}
+
+enum TypeOrConditionCode {
+    Type(Type),
+    ConditionCode,
+}
+
+struct TypingContext;
+
+impl<'a> TypingContextTrait<'a> for TypingContext {
+    type TypeVariable = TypeOrConditionCode;
+
+    fn cc(&mut self, _: wast::Span) -> Self::TypeVariable {
+        TypeOrConditionCode::ConditionCode
+    }
+
+    fn bNN(&mut self, _: wast::Span) -> Self::TypeVariable {
+        TypeOrConditionCode::Type(Type::b1())
+    }
+
+    fn iNN(&mut self, _: wast::Span) -> Self::TypeVariable {
+        TypeOrConditionCode::Type(Type::i32())
+    }
+
+    fn iMM(&mut self, _: wast::Span) -> Self::TypeVariable {
+        TypeOrConditionCode::Type(Type::i32())
+    }
+
+    fn cpu_flags(&mut self, _: wast::Span) -> Self::TypeVariable {
+        TypeOrConditionCode::Type(Type::cpu_flags())
+    }
+
+    fn b1(&mut self, _: wast::Span) -> Self::TypeVariable {
+        TypeOrConditionCode::Type(Type::b1())
+    }
+
+    fn void(&mut self, _: wast::Span) -> Self::TypeVariable {
+        TypeOrConditionCode::Type(Type::void())
+    }
+
+    fn bool_or_int(&mut self, _: wast::Span) -> Self::TypeVariable {
+        TypeOrConditionCode::Type(Type::b1())
+    }
+
+    fn any_t(&mut self, _: wast::Span) -> Self::TypeVariable {
+        TypeOrConditionCode::Type(Type::i32())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn check_interp() {
+        crate::check(|s: Vec<u8>| interp(String::from_utf8_lossy(&s).as_bytes()));
+    }
+
+    #[test]
+    fn regression_0() {
+        interp(b"(=> (imul $x $x) $x)");
+    }
+
+    #[test]
+    fn regression_1() {
+        interp(b"(=> (when (imul $x $C) (is-power-of-two $C)) $x)");
+    }
+
+    #[test]
+    fn regression_2() {
+        interp(
+            b"
+            (=> (bor (bor $x $y) $x) (bor $x $y))
+            (=> (bor (bor $x $C) 5) $x)
+            ",
+        );
+    }
+
+    #[test]
+    fn regression_3() {
+        interp(
+            b"
+            (=> (bor $y (bor $x 9)) $x)
+            (=> (bor (bor $x $y) $x) $x)
+            ",
+        );
+    }
+
+    #[test]
+    fn regression_4() {
+        interp(
+            b"
+            (=> (bor $C 33) 0)
+            (=> (bor $x 22) 1)
+            (=> (bor $x 11) 2)
+            ",
+        );
+    }
+
+    #[test]
+    fn regression_5() {
+        interp(
+            b"
+            (=> (bor $y (bor $x $y)) (bor $x $y))
+            (=> (bor (bor $x $y) $z) $x)
+            (=> (bor (bor $x $y) $y) $x)
+            ",
+        );
+    }
+
+    #[test]
+    fn regression_6() {
+        interp(b"(=> (imul $x $f) of)");
+    }
+
+    #[test]
+    fn regression_7() {
+        interp(
+            b"
+            (=> (when (sdiv $x $C)
+                      (fits-in-native-word $y))
+                (sdiv $C $x))
+            ",
+        );
+    }
+
+    #[test]
+    fn regression_8() {
+        interp(
+            b"
+            (=> (adjust_sp_down $C) (adjust_sp_down_imm $C))
+            ",
+        );
+    }
+
+    #[test]
+    fn regression_9() {
+        interp(
+            b"
+            (=> (when $x) $x)
+            (=> (trapnz $x) (trapnz $x))
+            ",
+        );
+    }
+
+    #[test]
+    fn regression_10() {
+        interp(b"(=> (sshr{i1} $x 0) $x)");
+    }
+
+    #[test]
+    fn regression_11() {
+        interp(
+            b"
+            (=> (when (ushr_imm $x (ishl 4 3))
+                      (bit-width $x 64))
+                (sextend{i64} (ireduce{i32} $x)))
+            ",
+        );
+    }
+
+    #[test]
+    fn regression_12() {
+        interp(b"(=> (band $C1 (band_imm $C1 1)) 1)");
+    }
+
+    #[test]
+    fn regression_13() {
+        interp(b"(=> (brz (icmp eq 0 $x)) (brz (ireduce{i32} $x)))");
+    }
+
+    #[test]
+    fn regression_14() {
+        interp(b"(=> (brz (icmp $E 0 $x)) (brz $x))");
+    }
+}

cranelift/peepmatic/crates/fuzzing/src/lib.rs

@@ -0,0 +1,119 @@
+//! Utilities for fuzzing.
+//!
+//! The actual fuzz targets are defined in `peepmatic/fuzz/*`. This crate just
+//! has oracles and generators for fuzzing.
+
+#![deny(missing_debug_implementations)]
+#![deny(missing_docs)]
+
+use arbitrary::{Arbitrary, Unstructured};
+use rand::prelude::*;
+use std::fmt::Debug;
+use std::panic;
+use std::time;
+
+pub mod automata;
+pub mod compile;
+pub mod interp;
+pub mod parser;
+
+/// A quickcheck-style runner for fuzz targets.
+///
+/// This is *not* intended to replace a long-running, coverage-guided fuzzing
+/// engine like libFuzzer! This is only for defining quick, purely random tests
+/// for use with `cargo test` and CI.
+pub fn check<A>(mut f: impl FnMut(A))
+where
+    A: Clone + Debug + Arbitrary,
+{
+    let seed = rand::thread_rng().gen();
+    let mut rng = rand::rngs::SmallRng::seed_from_u64(seed);
+
+    const INITIAL_LENGTH: usize = 16;
+    const MAX_LENGTH: usize = 4096;
+
+    let mut buf: Vec<u8> = (0..INITIAL_LENGTH).map(|_| rng.gen()).collect();
+    let mut num_checked = 0;
+
+    let time_budget = time::Duration::from_secs(2);
+    let then = time::Instant::now();
+
+    let (failing_input, panic_info) = loop {
+        if num_checked > 0 && time::Instant::now().duration_since(then) > time_budget {
+            eprintln!("Checked {} random inputs.", num_checked);
+            return;
+        }
+
+        match <A as Arbitrary>::arbitrary_take_rest(Unstructured::new(&buf)) {
+            Ok(input) => {
+                num_checked += 1;
+                eprintln!("Checking input: {:#?}", input);
+                if let Err(p) = panic::catch_unwind(panic::AssertUnwindSafe(|| f(input.clone()))) {
+                    break (input, p);
+                }
+            }
+            Err(e @ arbitrary::Error::NotEnoughData) => {
+                eprintln!("warning: {}", e);
+                if *buf.last().unwrap() == 0 {
+                    if buf.len() < MAX_LENGTH {
+                        let new_size = std::cmp::min(buf.len() * 2, MAX_LENGTH);
+                        eprintln!("Growing buffer size to {}", new_size);
+                        let delta = new_size - buf.len();
+                        buf.reserve(delta);
+                        for _ in 0..delta {
+                            buf.push(rng.gen());
+                        }
+                        continue;
+                    } else {
+                        // Regenerate `buf` in the loop below and see if that
+                        // fixes things...
+                        eprintln!("Regenerating buffer data.");
+                    }
+                } else {
+                    // Shrink values in the end of `buf`, which is where
+                    // `Arbitrary` pulls container lengths from. Then try again.
+                    eprintln!("Shrinking buffer's tail values.");
+                    let i = (buf.len() as f64).sqrt() as usize;
+                    for j in i..buf.len() {
+                        buf[j] /= 2;
+                    }
+                    continue;
+                }
+            }
+            Err(e) => {
+                eprintln!("warning: {}", e);
+                // Usually this happens because `A` requires a sequence utf-8
+                // bytes but its given sequence wasn't valid utf-8. Just skip
+                // this iteration and try again after we've updated `buf` below.
+            }
+        };
+
+        // Double the size of the buffer every so often, so we don't only
+        // explore small inputs.
+        if num_checked == buf.len() {
+            buf.resize(std::cmp::min(buf.len() * 2, MAX_LENGTH), 0);
+        }
+
+        for i in 0..buf.len() {
+            buf[i] = rng.gen();
+        }
+    };
+
+    // Shrink the failing input.
+    let mut smallest_failing_input = failing_input;
+    let mut panic_info = panic_info;
+    'shrinking: loop {
+        eprintln!("Smallest failing input: {:#?}", smallest_failing_input);
+        for input in smallest_failing_input.shrink() {
+            if let Err(p) = panic::catch_unwind(panic::AssertUnwindSafe(|| f(input.clone()))) {
+                smallest_failing_input = input;
+                panic_info = p;
+                continue 'shrinking;
+            }
+        }
+        break;
+    }
+
+    // Resume the panic for the smallest input.
+    panic::resume_unwind(panic_info);
+}

cranelift/peepmatic/crates/fuzzing/src/parser.rs

@@ -0,0 +1,29 @@
+//! Utilities for fuzzing our DSL's parser.
+
+use peepmatic::Optimizations;
+use std::str;
+
+/// Attempt to parse the given string as if it were a snippet of our DSL.
+pub fn parse(data: &[u8]) {
+    let source = match str::from_utf8(data) {
+        Ok(s) => s,
+        Err(_) => return,
+    };
+
+    let buf = match wast::parser::ParseBuffer::new(&source) {
+        Ok(buf) => buf,
+        Err(_) => return,
+    };
+
+    let _ = wast::parser::parse::<Optimizations>(&buf);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn check_parse() {
+        crate::check(|s: String| parse(s.as_bytes()));
+    }
+}

cranelift/peepmatic/crates/macro/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "peepmatic-macro"
+version = "0.1.0"
+authors = ["Nick Fitzgerald <fitzgen@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+proc-macro2 = "1.0.9"
+quote = "1.0.3"
+syn = { version = "1.0.16", features = ['extra-traits'] }
+
+[lib]
+proc_macro = true

cranelift/peepmatic/crates/macro/src/child_nodes.rs

@@ -0,0 +1,110 @@
+use quote::quote;
+use syn::DeriveInput;
+use syn::{parse_quote, GenericParam, Generics, Result};
+
+pub fn derive_child_nodes(input: &DeriveInput) -> Result<impl quote::ToTokens> {
+    let children = get_child_nodes(&input.data)?;
+    let name = &input.ident;
+    let generics = add_trait_bounds(input.generics.clone());
+    let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
+
+    Ok(quote! {
+        impl #impl_generics ChildNodes<'a, 'a> for #name #ty_generics #where_clause {
+            fn child_nodes(&'a self, children: &mut impl Extend<DynAstRef<'a>>) {
+                #children
+            }
+        }
+    })
+}
+
+fn get_child_nodes(data: &syn::Data) -> Result<impl quote::ToTokens> {
+    match data {
+        syn::Data::Struct(s) => {
+            let mut fields = vec![];
+
+            match &s.fields {
+                syn::Fields::Named(n) => {
+                    for f in n.named.iter() {
+                        let opts = crate::PeepmaticOpts::from_attrs(&mut f.attrs.clone())?;
+
+                        if opts.skip_child {
+                            continue;
+                        }
+
+                        let field_name = f.ident.as_ref().unwrap();
+                        if opts.flatten {
+                            fields.push(quote! {
+                                self.#field_name.iter().map(DynAstRef::from)
+                            });
+                        } else {
+                            fields.push(quote! {
+                                std::iter::once(DynAstRef::from(&self.#field_name))
+                            });
+                        }
+                    }
+                }
+                syn::Fields::Unnamed(u) => {
+                    for (i, f) in u.unnamed.iter().enumerate() {
+                        let opts = crate::PeepmaticOpts::from_attrs(&mut f.attrs.clone())?;
+                        if opts.skip_child {
+                            continue;
+                        }
+                        if opts.flatten {
+                            return Err(syn::Error::new(
+                                u.paren_token.span,
+                                "#[peepmatic(flatten)] is only allowed with named fields",
+                            ));
+                        }
+                        fields.push(quote! {
+                            std::iter::once(DynAstRef::from(&self.#i))
+                        });
+                    }
+                }
+                syn::Fields::Unit => {}
+            }
+
+            Ok(match fields.as_slice() {
+                [] => quote! { let _ = children; },
+                [f, rest @ ..] => {
+                    let rest = rest.iter().map(|f| {
+                        quote! {
+                            .chain(#f)
+                        }
+                    });
+                    quote! {
+                        children.extend( #f #( #rest )* );
+                    }
+                }
+            })
+        }
+        syn::Data::Enum(e) => {
+            let mut match_arms = vec![];
+            for v in e.variants.iter() {
+                match v.fields {
+                    syn::Fields::Unnamed(ref u) if u.unnamed.len() == 1 => {
+                        let variant = &v.ident;
+                        match_arms.push(quote! {
+                            Self::#variant(x) => children.extend(Some(x.into())),
+                        });
+                    }
+                    _ => panic!("#[derive(ChildNodes)] only supports enums whose variants all ahve a single unnamed field")
+                }
+            }
+            Ok(quote! {
+                match self {
+                    #( #match_arms )*
+                }
+            })
+        }
+        syn::Data::Union(_) => panic!("#[derive(ChildNodes)] is not supported on unions"),
+    }
+}
+
+fn add_trait_bounds(mut generics: Generics) -> Generics {
+    for param in &mut generics.params {
+        if let GenericParam::Type(type_param) = param {
+            type_param.bounds.push(parse_quote!(ChildNodes<'a, 'a>));
+        }
+    }
+    generics
+}

cranelift/peepmatic/crates/macro/src/into_dyn_ast_ref.rs

@@ -0,0 +1,23 @@
+use quote::quote;
+use syn::DeriveInput;
+use syn::Result;
+
+pub fn derive_into_dyn_ast_ref(input: &DeriveInput) -> Result<impl quote::ToTokens> {
+    let ty = &input.ident;
+
+    let opts = crate::PeepmaticOpts::from_attrs(&mut input.attrs.clone())?;
+    if opts.no_into_dyn_node {
+        return Ok(quote! {});
+    }
+
+    let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
+
+    Ok(quote! {
+        impl #impl_generics From<&'a #ty #ty_generics> for DynAstRef<'a> #where_clause {
+            #[inline]
+            fn from(x: &'a #ty #ty_generics) -> Self {
+                Self::#ty(x)
+            }
+        }
+    })
+}

cranelift/peepmatic/crates/macro/src/lib.rs

@@ -0,0 +1,156 @@
+extern crate proc_macro;
+
+use crate::proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+use syn::parse::{Parse, ParseStream};
+use syn::punctuated::Punctuated;
+use syn::DeriveInput;
+use syn::Error;
+use syn::{parse_macro_input, Ident, Result};
+
+mod child_nodes;
+mod into_dyn_ast_ref;
+mod operator;
+mod span;
+
+#[proc_macro_derive(PeepmaticOperator, attributes(peepmatic))]
+pub fn operator(input: TokenStream) -> TokenStream {
+    operator::derive_operator(input)
+}
+
+#[proc_macro_derive(Ast, attributes(peepmatic))]
+pub fn derive_ast(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+
+    let span_impl = match span::derive_span(&input) {
+        Ok(s) => s,
+        Err(e) => return e.to_compile_error().into(),
+    };
+
+    let child_nodes_impl = match child_nodes::derive_child_nodes(&input) {
+        Ok(c) => c,
+        Err(e) => return e.to_compile_error().into(),
+    };
+
+    let into_dyn_ast_ref_impl = match into_dyn_ast_ref::derive_into_dyn_ast_ref(&input) {
+        Ok(n) => n,
+        Err(e) => return e.to_compile_error().into(),
+    };
+
+    let expanded = quote! {
+        #span_impl
+        #child_nodes_impl
+        #into_dyn_ast_ref_impl
+    };
+
+    TokenStream::from(expanded)
+}
+
+#[derive(Default)]
+pub(crate) struct PeepmaticOpts {
+    // `ChildNodes` options.
+    skip_child: bool,
+    flatten: bool,
+
+    // `From<&'a Self> for DynAstRef<'a>` options.
+    no_into_dyn_node: bool,
+
+    // Peepmatic operator options.
+    immediates_paren: syn::token::Paren,
+    immediates: Vec<syn::Ident>,
+    params_paren: syn::token::Paren,
+    params: Vec<syn::Ident>,
+    result: Option<syn::Ident>,
+}
+
+impl Parse for PeepmaticOpts {
+    fn parse(input: ParseStream) -> Result<Self> {
+        enum Attr {
+            Immediates(syn::token::Paren, Vec<syn::Ident>),
+            Params(syn::token::Paren, Vec<syn::Ident>),
+            Result(syn::Ident),
+            NoIntoDynNode,
+            SkipChild,
+            Flatten,
+        }
+
+        let attrs = Punctuated::<_, syn::token::Comma>::parse_terminated(input)?;
+        let mut ret = PeepmaticOpts::default();
+        for attr in attrs {
+            match attr {
+                Attr::Immediates(paren, imms) => {
+                    ret.immediates_paren = paren;
+                    ret.immediates = imms;
+                }
+                Attr::Params(paren, ps) => {
+                    ret.params_paren = paren;
+                    ret.params = ps;
+                }
+                Attr::Result(r) => ret.result = Some(r),
+                Attr::NoIntoDynNode => ret.no_into_dyn_node = true,
+                Attr::SkipChild => ret.skip_child = true,
+                Attr::Flatten => ret.flatten = true,
+            }
+        }
+
+        return Ok(ret);
+
+        impl Parse for Attr {
+            fn parse(input: ParseStream) -> Result<Self> {
+                let attr: Ident = input.parse()?;
+                if attr == "immediates" {
+                    let inner;
+                    let paren = syn::parenthesized!(inner in input);
+                    let imms = Punctuated::<_, syn::token::Comma>::parse_terminated(&inner)?;
+                    return Ok(Attr::Immediates(paren, imms.into_iter().collect()));
+                }
+                if attr == "params" {
+                    let inner;
+                    let paren = syn::parenthesized!(inner in input);
+                    let params = Punctuated::<_, syn::token::Comma>::parse_terminated(&inner)?;
+                    return Ok(Attr::Params(paren, params.into_iter().collect()));
+                }
+                if attr == "result" {
+                    let inner;
+                    syn::parenthesized!(inner in input);
+                    return Ok(Attr::Result(syn::Ident::parse(&inner)?));
+                }
+                if attr == "skip_child" {
+                    return Ok(Attr::SkipChild);
+                }
+                if attr == "no_into_dyn_node" {
+                    return Ok(Attr::NoIntoDynNode);
+                }
+                if attr == "flatten" {
+                    return Ok(Attr::Flatten);
+                }
+                return Err(Error::new(attr.span(), "unexpected attribute"));
+            }
+        }
+    }
+}
+
+fn peepmatic_attrs(attrs: &mut Vec<syn::Attribute>) -> TokenStream {
+    let mut ret = proc_macro2::TokenStream::new();
+    let ident = syn::Path::from(syn::Ident::new("peepmatic", Span::call_site()));
+    for i in (0..attrs.len()).rev() {
+        if attrs[i].path != ident {
+            continue;
+        }
+        let attr = attrs.remove(i);
+        let group = match attr.tokens.into_iter().next().unwrap() {
+            proc_macro2::TokenTree::Group(g) => g,
+            _ => panic!("#[peepmatic(...)] expected"),
+        };
+        ret.extend(group.stream());
+        ret.extend(quote! { , });
+    }
+    return ret.into();
+}
+
+impl PeepmaticOpts {
+    pub(crate) fn from_attrs(attrs: &mut Vec<syn::Attribute>) -> syn::Result<Self> {
+        syn::parse(peepmatic_attrs(attrs))
+    }
+}

cranelift/peepmatic/crates/macro/src/operator.rs

@@ -0,0 +1,325 @@
+//! Implementation of the `#[peepmatic]` macro for the `Operator` AST node.
+
+use crate::proc_macro::TokenStream;
+use crate::PeepmaticOpts;
+use proc_macro2::{Ident, Span};
+use quote::quote;
+use syn::DeriveInput;
+use syn::Error;
+use syn::{parse_macro_input, Result};
+
+pub fn derive_operator(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+
+    let variants = match get_enum_variants(&input) {
+        Ok(v) => v,
+        Err(e) => return e.to_compile_error().into(),
+    };
+
+    let arity = match create_arity(&variants) {
+        Ok(a) => a,
+        Err(e) => return e.to_compile_error().into(),
+    };
+
+    let num_operators = variants.len();
+    let type_methods = create_type_methods(&variants);
+    let parse_impl = create_parse_impl(&input.ident, &variants);
+    let display_impl = create_display_impl(&input.ident, &variants);
+    let try_from_u32_impl = create_try_from_u32_impl(&input.ident, &variants);
+    let ident = &input.ident;
+
+    let expanded = quote! {
+        impl #ident {
+            #arity
+            #type_methods
+
+            /// Get the total number of different operators.
+            pub const fn num_operators() -> usize {
+                #num_operators
+            }
+        }
+
+        #display_impl
+        #try_from_u32_impl
+        #parse_impl
+    };
+
+    // eprintln!("{}", expanded);
+    TokenStream::from(expanded)
+}
+
+fn get_enum_variants(input: &DeriveInput) -> Result<Vec<OperatorVariant>> {
+    let en = match &input.data {
+        syn::Data::Enum(en) => en,
+        syn::Data::Struct(_) => {
+            panic!("can only put #[peepmatic] on an enum; found it on a struct")
+        }
+        syn::Data::Union(_) => panic!("can only put #[peepmatic] on an enum; found it on a union"),
+    };
+    en.variants
+        .iter()
+        .cloned()
+        .map(|mut variant| {
+            Ok(OperatorVariant {
+                opts: PeepmaticOpts::from_attrs(&mut variant.attrs)?,
+                syn: variant,
+            })
+        })
+        .collect()
+}
+
+struct OperatorVariant {
+    syn: syn::Variant,
+    opts: PeepmaticOpts,
+}
+
+fn create_arity(variants: &[OperatorVariant]) -> Result<impl quote::ToTokens> {
+    let mut imm_arities = vec![];
+    let mut params_arities = vec![];
+
+    for v in variants {
+        let variant = &v.syn.ident;
+
+        let imm_arity = v.opts.immediates.len();
+        if imm_arity > std::u8::MAX as usize {
+            return Err(Error::new(
+                v.opts.immediates_paren.span,
+                "cannot have more than u8::MAX immediates",
+            ));
+        }
+        let imm_arity = imm_arity as u8;
+
+        imm_arities.push(quote! {
+            Self::#variant => #imm_arity,
+        });
+
+        let params_arity = v.opts.params.len();
+        if params_arity > std::u8::MAX as usize {
+            return Err(Error::new(
+                v.opts.params_paren.span,
+                "cannot have more than u8::MAX params",
+            ));
+        }
+        let params_arity = params_arity as u8;
+
+        params_arities.push(quote! {
+            Self::#variant => #params_arity,
+        });
+    }
+
+    Ok(quote! {
+        /// Get the number of immediates that this operator has.
+        pub fn immediates_arity(&self) -> u8 {
+            match *self {
+                #( #imm_arities )*
+            }
+        }
+
+        /// Get the number of parameters that this operator takes.
+        pub fn params_arity(&self) -> u8 {
+            match *self {
+                #( #params_arities )*
+            }
+        }
+    })
+}
+
+fn create_type_methods(variants: &[OperatorVariant]) -> impl quote::ToTokens {
+    let mut result_types = vec![];
+    let mut imm_types = vec![];
+    let mut param_types = vec![];
+
+    for v in variants {
+        let variant = &v.syn.ident;
+
+        let result_ty = v.opts.result.as_ref().unwrap_or_else(|| {
+            panic!(
+                "must define #[peepmatic(result(..))] on operator `{}`",
+                variant
+            )
+        });
+        result_types.push(quote! {
+            Self::#variant => {
+                context.#result_ty(span)
+            }
+        });
+
+        let imm_tys = match &v.opts.immediates[..] {
+            [] => quote! {},
+            [ty, rest @ ..] => {
+                let rest = rest.iter().map(|ty| {
+                    quote! { .chain(::std::iter::once(context.#ty(span))) }
+                });
+                quote! {
+                    types.extend(::std::iter::once(context.#ty(span))#( #rest )*);
+                }
+            }
+        };
+        imm_types.push(quote! {
+            Self::#variant => {
+                #imm_tys
+            }
+        });
+
+        let param_tys = match &v.opts.params[..] {
+            [] => quote! {},
+            [ty, rest @ ..] => {
+                let rest = rest.iter().map(|ty| {
+                    quote! { .chain(::std::iter::once(context.#ty(span))) }
+                });
+                quote! {
+                    types.extend(::std::iter::once(context.#ty(span))#( #rest )*);
+                }
+            }
+        };
+        param_types.push(quote! {
+            Self::#variant => {
+                #param_tys
+            }
+        });
+    }
+
+    quote! {
+        /// Get the result type of this operator.
+        #[cfg(feature = "construct")]
+        pub fn result_type<'a, C>(
+            &self,
+            context: &mut C,
+            span: wast::Span,
+        ) -> C::TypeVariable
+        where
+            C: 'a + TypingContext<'a>,
+        {
+            match *self {
+                #( #result_types )*
+            }
+        }
+
+        /// Get the immediate types of this operator.
+        #[cfg(feature = "construct")]
+        pub fn immediate_types<'a, C>(
+            &self,
+            context: &mut C,
+            span: wast::Span,
+            types: &mut impl Extend<C::TypeVariable>,
+        )
+        where
+            C: 'a + TypingContext<'a>,
+        {
+            match *self {
+                #( #imm_types )*
+            }
+        }
+
+        /// Get the parameter types of this operator.
+        #[cfg(feature = "construct")]
+        pub fn param_types<'a, C>(
+            &self,
+            context: &mut C,
+            span: wast::Span,
+            types: &mut impl Extend<C::TypeVariable>,
+        )
+        where
+            C: 'a + TypingContext<'a>,
+        {
+            match *self {
+                #( #param_types )*
+            }
+        }
+    }
+}
+
+fn snake_case(s: &str) -> String {
+    let mut t = String::with_capacity(s.len() + 1);
+    for (i, ch) in s.chars().enumerate() {
+        if i != 0 && ch.is_uppercase() {
+            t.push('_');
+        }
+        t.extend(ch.to_lowercase());
+    }
+    t
+}
+
+fn create_parse_impl(ident: &syn::Ident, variants: &[OperatorVariant]) -> impl quote::ToTokens {
+    let token_defs = variants.iter().map(|v| {
+        let tok = snake_case(&v.syn.ident.to_string());
+        let tok = Ident::new(&tok, Span::call_site());
+        quote! {
+            wast::custom_keyword!(#tok);
+        }
+    });
+
+    let parses = variants.iter().map(|v| {
+        let tok = snake_case(&v.syn.ident.to_string());
+        let tok = Ident::new(&tok, Span::call_site());
+        let ident = &v.syn.ident;
+        quote! {
+            if p.peek::<#tok>() {
+                p.parse::<#tok>()?;
+                return Ok(Self::#ident);
+            }
+        }
+    });
+
+    let expected = format!("expected {}", ident);
+
+    quote! {
+        #[cfg(feature = "construct")]
+        impl<'a> wast::parser::Parse<'a> for #ident {
+            fn parse(p: wast::parser::Parser<'a>) -> wast::parser::Result<Self> {
+                #( #token_defs )*
+
+                #( #parses )*
+
+                Err(p.error(#expected))
+            }
+        }
+    }
+}
+
+fn create_display_impl(ident: &syn::Ident, variants: &[OperatorVariant]) -> impl quote::ToTokens {
+    let displays = variants.iter().map(|v| {
+        let variant = &v.syn.ident;
+        let snake = snake_case(&v.syn.ident.to_string());
+        quote! {
+            Self::#variant => write!(f, #snake),
+        }
+    });
+
+    quote! {
+        impl std::fmt::Display for #ident {
+            fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+                match self {
+                    #( #displays )*
+                }
+            }
+        }
+    }
+}
+
+fn create_try_from_u32_impl(
+    ident: &syn::Ident,
+    variants: &[OperatorVariant],
+) -> impl quote::ToTokens {
+    let matches = variants.iter().map(|v| {
+        let variant = &v.syn.ident;
+        quote! {
+            x if Self::#variant as u32 == x => Ok(Self::#variant),
+        }
+    });
+
+    let error_msg = format!("value is not an `{}`", ident);
+
+    quote! {
+        impl std::convert::TryFrom<u32> for #ident {
+            type Error = &'static str;
+
+            fn try_from(value: u32) -> Result<Self, Self::Error> {
+                match value {
+                    #( #matches )*
+                    _ => Err(#error_msg)
+                }
+            }
+        }
+    }
+}

cranelift/peepmatic/crates/macro/src/span.rs

@@ -0,0 +1,52 @@
+use quote::quote;
+use syn::DeriveInput;
+use syn::{parse_quote, GenericParam, Generics, Result};
+
+pub fn derive_span(input: &DeriveInput) -> Result<impl quote::ToTokens> {
+    let ty = &input.ident;
+
+    let body = match &input.data {
+        syn::Data::Struct(_) => quote! { self.span },
+        syn::Data::Enum(e) => {
+            let variants = e.variants.iter().map(|v| match v.fields {
+                syn::Fields::Unnamed(ref fields) if fields.unnamed.len() == 1 => {
+                    let variant = &v.ident;
+                    quote! { #ty::#variant(x) => x.span(), }
+                }
+                _ => panic!(
+                    "derive(Ast) on enums only supports variants with a single, unnamed field"
+                ),
+            });
+            quote! {
+                match self {
+                    #( #variants )*
+                }
+            }
+        }
+        syn::Data::Union(_) => {
+            panic!("derive(Ast) can only be used with structs and enums, not unions")
+        }
+    };
+
+    let generics = add_span_trait_bounds(input.generics.clone());
+    let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
+
+    Ok(quote! {
+        impl #impl_generics Span for #ty #ty_generics #where_clause {
+            #[inline]
+            fn span(&self) -> wast::Span {
+                #body
+            }
+        }
+    })
+}
+
+// Add a bound `T: Span` to every type parameter `T`.
+fn add_span_trait_bounds(mut generics: Generics) -> Generics {
+    for param in &mut generics.params {
+        if let GenericParam::Type(ref mut type_param) = *param {
+            type_param.bounds.push(parse_quote!(Span));
+        }
+    }
+    generics
+}

cranelift/peepmatic/crates/runtime/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "peepmatic-runtime"
+version = "0.1.0"
+authors = ["Nick Fitzgerald <fitzgen@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bincode = "1.2.1"
+bumpalo = "3.2.0"
+log = "0.4.8"
+peepmatic-automata = { version = "0.1.0", path = "../automata", features = ["serde"] }
+peepmatic-macro = { version = "0.1.0", path = "../macro" }
+serde = { version = "1.0.105", features = ["derive"] }
+thiserror = "1.0.15"
+wast = { version = "15.0.0", optional = true }
+
+
+[features]
+# Enable support for a few extra methods that are required by the `peepmatic`
+# crate when constructing peephole optimizers, but are not needed when simply
+# using already-constructed peephole optimizers.
+construct = ["wast"]

cranelift/peepmatic/crates/runtime/src/cc.rs

@@ -0,0 +1,92 @@
+//! Condition codes.
+
+use serde::{Deserialize, Serialize};
+use std::convert::TryFrom;
+use std::fmt;
+
+/// A condition code.
+///
+/// This is a special kind of immediate for `icmp` instructions that dictate
+/// which parts of the comparison result we care about.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[repr(u32)]
+pub enum ConditionCode {
+    /// Equal.
+    // NB: We convert `ConditionCode` into `NonZeroU32`s with unchecked
+    // conversions; memory safety relies on no variant being zero.
+    Eq = 1,
+
+    /// Not equal.
+    Ne,
+
+    /// Signed less than.
+    Slt,
+
+    /// Unsigned less than.
+    Ult,
+
+    /// Signed greater than or equal.
+    Sge,
+
+    /// Unsigned greater than or equal.
+    Uge,
+
+    /// Signed greater than.
+    Sgt,
+
+    /// Unsigned greater than.
+    Ugt,
+
+    /// Signed less than or equal.
+    Sle,
+
+    /// Unsigned less than or equal.
+    Ule,
+
+    /// Overflow.
+    Of,
+
+    /// No overflow.
+    Nof,
+}
+
+impl TryFrom<u32> for ConditionCode {
+    type Error = &'static str;
+
+    fn try_from(x: u32) -> Result<Self, Self::Error> {
+        Ok(match x {
+            x if Self::Eq as u32 == x => Self::Eq,
+            x if Self::Ne as u32 == x => Self::Ne,
+            x if Self::Slt as u32 == x => Self::Slt,
+            x if Self::Ult as u32 == x => Self::Ult,
+            x if Self::Sge as u32 == x => Self::Sge,
+            x if Self::Uge as u32 == x => Self::Uge,
+            x if Self::Sgt as u32 == x => Self::Sgt,
+            x if Self::Ugt as u32 == x => Self::Ugt,
+            x if Self::Sle as u32 == x => Self::Sle,
+            x if Self::Ule as u32 == x => Self::Ule,
+            x if Self::Of as u32 == x => Self::Of,
+            x if Self::Nof as u32 == x => Self::Nof,
+            _ => return Err("not a valid condition code value"),
+        })
+    }
+}
+
+impl fmt::Display for ConditionCode {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::Eq => write!(f, "eq"),
+            Self::Ne => write!(f, "ne"),
+            Self::Slt => write!(f, "slt"),
+            Self::Ult => write!(f, "ult"),
+            Self::Sge => write!(f, "sge"),
+            Self::Uge => write!(f, "uge"),
+            Self::Sgt => write!(f, "sgt"),
+            Self::Ugt => write!(f, "ugt"),
+            Self::Sle => write!(f, "sle"),
+            Self::Ule => write!(f, "ule"),
+            Self::Of => write!(f, "of"),
+            Self::Nof => write!(f, "nof"),
+        }
+    }
+}

cranelift/peepmatic/crates/runtime/src/error.rs

@@ -0,0 +1,44 @@
+//! `Error` and `Result` types for this crate.
+
+use std::io;
+use thiserror::Error;
+
+/// A result type containing `Ok(T)` or `Err(peepmatic_runtime::Error)`.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Errors that `peepmatic_runtime` may generate.
+#[derive(Debug, Error)]
+#[error(transparent)]
+pub struct Error {
+    #[from]
+    inner: Box<ErrorInner>,
+}
+
+#[derive(Debug, Error)]
+enum ErrorInner {
+    #[error(transparent)]
+    Io(#[from] io::Error),
+
+    #[error(transparent)]
+    Bincode(#[from] bincode::Error),
+}
+
+impl From<io::Error> for Error {
+    fn from(e: io::Error) -> Error {
+        let e: ErrorInner = e.into();
+        e.into()
+    }
+}
+
+impl From<bincode::Error> for Error {
+    fn from(e: bincode::Error) -> Error {
+        let e: ErrorInner = e.into();
+        e.into()
+    }
+}
+
+impl From<ErrorInner> for Error {
+    fn from(e: ErrorInner) -> Error {
+        Box::new(e).into()
+    }
+}

cranelift/peepmatic/crates/runtime/src/instruction_set.rs

@@ -0,0 +1,143 @@
+//! Interfacing with actual instructions.
+
+use crate::operator::Operator;
+use crate::part::{Constant, Part};
+use crate::paths::Path;
+use crate::r#type::Type;
+use std::fmt::Debug;
+
+/// A trait for interfacing with actual instruction sequences.
+///
+/// This trait enables both:
+///
+/// * `peepmatic-runtime` to be used by `cranelift-codegen` without a circular
+///   dependency from `peepmatic-runtime` to `cranelift-codegen` to get access
+///   to Cranelift's IR types, and
+///
+/// * enables us to write local tests that exercise peephole optimizers on a
+///  simple, testing-only instruction set without pulling in all of Cranelift.
+///
+/// Finally, this should also make the task of adding support for Cranelift's
+/// new `MachInst` and vcode backend easier, since all that needs to be done is
+/// "just" implementing this trait. (And probably add/modify some
+/// `peepmatic_runtime::operation::Operation`s as well).
+///
+/// ## Safety
+///
+/// See doc comment for `instruction_result_bit_width`.
+pub unsafe trait InstructionSet<'a> {
+    /// Mutable context passed into all trait methods. Can be whatever you want!
+    ///
+    /// In practice, this is a `FuncCursor` for `cranelift-codegen`'s trait
+    /// implementation.
+    type Context;
+
+    /// An instruction (or identifier for an instruction).
+    type Instruction: Copy + Debug + Eq;
+
+    /// Replace the `old` instruction with `new`.
+    ///
+    /// `new` is either a `Part::Instruction` or a constant `Part::Boolean` or
+    /// `Part::Integer`. In the former case, it can directly replace `old`. In
+    /// the latter case, implementations of this trait should transparently
+    /// create an `iconst` or `bconst` instruction to wrap the given constant.
+    ///
+    /// `new` will never be `Part::ConditionCode`.
+    fn replace_instruction(
+        &self,
+        context: &mut Self::Context,
+        old: Self::Instruction,
+        new: Part<Self::Instruction>,
+    ) -> Self::Instruction;
+
+    /// Get the instruction, constant, or condition code at the given path.
+    ///
+    /// If there is no such entity at the given path (e.g. we run into a
+    /// function parameter and can't traverse the path any further) then `None`
+    /// should be returned.
+    fn get_part_at_path(
+        &self,
+        context: &mut Self::Context,
+        root: Self::Instruction,
+        path: Path,
+    ) -> Option<Part<Self::Instruction>>;
+
+    /// Get the given instruction's operator.
+    ///
+    /// If the instruction's opcode does not have an associated
+    /// `peepmatic_runtime::operator::Operator` variant (i.e. that instruction
+    /// isn't supported by `peepmatic` yet) then `None` should be returned.
+    fn operator(&self, context: &mut Self::Context, instr: Self::Instruction) -> Option<Operator>;
+
+    /// Make a unary instruction.
+    ///
+    /// If the type is not given, then it should be inferred.
+    fn make_inst_1(
+        &self,
+        context: &mut Self::Context,
+        root: Self::Instruction,
+        operator: Operator,
+        r#type: Type,
+        a: Part<Self::Instruction>,
+    ) -> Self::Instruction;
+
+    /// Make a binary instruction.
+    ///
+    /// Operands are given as immediates first and arguments following
+    /// them. Condition codes are treated as immediates. So if we are creating
+    /// an `iadd_imm` instruction, then `a` will be the constant integer
+    /// immediate and `b` will be the instruction whose result is the dynamic
+    /// argument.
+    fn make_inst_2(
+        &self,
+        context: &mut Self::Context,
+        root: Self::Instruction,
+        operator: Operator,
+        r#type: Type,
+        a: Part<Self::Instruction>,
+        b: Part<Self::Instruction>,
+    ) -> Self::Instruction;
+
+    /// Make a ternary instruction.
+    ///
+    /// Operands are given as immediates first and arguments following
+    /// them. Condition codes are treated as immediates. So if we are creating
+    /// an `icmp` instruction, then `a` will be the condition code, and `b` and
+    /// `c` will be instructions whose results are the dynamic arguments.
+    fn make_inst_3(
+        &self,
+        context: &mut Self::Context,
+        root: Self::Instruction,
+        operator: Operator,
+        r#type: Type,
+        a: Part<Self::Instruction>,
+        b: Part<Self::Instruction>,
+        c: Part<Self::Instruction>,
+    ) -> Self::Instruction;
+
+    /// Try to resolve the given instruction into a constant value.
+    ///
+    /// If we can tell that the instruction returns a constant value, then
+    /// return that constant value as either a `Part::Boolean` or
+    /// `Part::Integer`. Otherwise, return `None`.
+    fn instruction_to_constant(
+        &self,
+        context: &mut Self::Context,
+        inst: Self::Instruction,
+    ) -> Option<Constant>;
+
+    /// Get the bit width of the given instruction's result.
+    ///
+    /// ## Safety
+    ///
+    /// There is code that makes memory-safety assumptions that the result is
+    /// always one of 1, 8, 16, 32, 64, or 128. Implementors must uphold this.
+    fn instruction_result_bit_width(
+        &self,
+        context: &mut Self::Context,
+        inst: Self::Instruction,
+    ) -> u8;
+
+    /// Get the size of a native word in bits.
+    fn native_word_size_in_bits(&self, context: &mut Self::Context) -> u8;
+}

cranelift/peepmatic/crates/runtime/src/integer_interner.rs

@@ -0,0 +1,73 @@
+//! Interner for (potentially large) integer values.
+//!
+//! We support matching on integers that can be represented by `u64`, but only
+//! support automata results that fit in a `u32`. So we intern the (relatively
+//! few compared to the full range of `u64`) integers we are matching against
+//! here and then reference them by `IntegerId`.
+
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+use std::num::{NonZeroU16, NonZeroU32};
+
+/// An identifier for an interned integer.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct IntegerId(#[doc(hidden)] pub NonZeroU16);
+
+/// An interner for integer values.
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct IntegerInterner {
+    // Note: we use `BTreeMap`s for deterministic serialization.
+    map: BTreeMap<u64, IntegerId>,
+    values: Vec<u64>,
+}
+
+impl IntegerInterner {
+    /// Construct a new `IntegerInterner`.
+    #[inline]
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Intern a value into this `IntegerInterner`, returning its canonical
+    /// `IntegerId`.
+    #[inline]
+    pub fn intern(&mut self, value: impl Into<u64>) -> IntegerId {
+        debug_assert_eq!(self.map.len(), self.values.len());
+
+        let value = value.into();
+
+        if let Some(id) = self.map.get(&value) {
+            return *id;
+        }
+
+        assert!((self.values.len() as u64) < (std::u16::MAX as u64));
+        let id = IntegerId(unsafe { NonZeroU16::new_unchecked(self.values.len() as u16 + 1) });
+
+        self.values.push(value);
+        self.map.insert(value, id);
+        debug_assert_eq!(self.map.len(), self.values.len());
+
+        id
+    }
+
+    /// Get the id of an already-interned integer, or `None` if it has not been
+    /// interned.
+    pub fn already_interned(&self, value: impl Into<u64>) -> Option<IntegerId> {
+        let value = value.into();
+        self.map.get(&value).copied()
+    }
+
+    /// Lookup a previously interned integer by id.
+    #[inline]
+    pub fn lookup(&self, id: IntegerId) -> u64 {
+        let index = id.0.get() as usize - 1;
+        self.values[index]
+    }
+}
+
+impl From<IntegerId> for NonZeroU32 {
+    #[inline]
+    fn from(id: IntegerId) -> NonZeroU32 {
+        id.0.into()
+    }
+}

cranelift/peepmatic/crates/runtime/src/lib.rs

@@ -0,0 +1,34 @@
+//! Runtime support for `peepmatic`'s peephole optimizers.
+//!
+//! This crate contains everything required to use a `peepmatic`-generated
+//! peephole optimizer.
+//!
+//! ## Why is this a different crate from `peepmatic`?
+//!
+//! In short: build times and code size.
+//!
+//! If you are just using a peephole optimizer, you shouldn't need the functions
+//! to construct it from scratch from the DSL (and the implied code size and
+//! compilation time), let alone even build it at all. You should just
+//! deserialize an already-built peephole optimizer, and then use it.
+//!
+//! That's all that is contained here in this crate.
+
+#![deny(missing_docs)]
+#![deny(missing_debug_implementations)]
+
+pub mod cc;
+pub mod error;
+pub mod instruction_set;
+pub mod integer_interner;
+pub mod linear;
+pub mod operator;
+pub mod optimizations;
+pub mod optimizer;
+pub mod part;
+pub mod paths;
+pub mod r#type;
+
+pub use error::{Error, Result};
+pub use optimizations::PeepholeOptimizations;
+pub use optimizer::PeepholeOptimizer;

cranelift/peepmatic/crates/runtime/src/linear.rs

@@ -0,0 +1,264 @@
+//! A linear IR for optimizations.
+//!
+//! This IR is designed such that it should be easy to combine multiple linear
+//! optimizations into a single automata.
+//!
+//! See also `src/linearize.rs` for the AST to linear IR translation pass.
+
+use crate::cc::ConditionCode;
+use crate::integer_interner::{IntegerId, IntegerInterner};
+use crate::operator::{Operator, UnquoteOperator};
+use crate::paths::{PathId, PathInterner};
+use crate::r#type::{BitWidth, Type};
+use serde::{Deserialize, Serialize};
+use std::num::NonZeroU32;
+
+/// A set of linear optimizations.
+#[derive(Debug)]
+pub struct Optimizations {
+    /// The linear optimizations.
+    pub optimizations: Vec<Optimization>,
+
+    /// The de-duplicated paths referenced by these optimizations.
+    pub paths: PathInterner,
+
+    /// The integer literals referenced by these optimizations.
+    pub integers: IntegerInterner,
+}
+
+/// A linearized optimization.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Optimization {
+    /// The chain of increments for this optimization.
+    pub increments: Vec<Increment>,
+}
+
+/// Match any value.
+///
+/// This can be used to create fallback, wildcard-style transitions between
+/// states.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct Else;
+
+/// The result of evaluating a `MatchOp`.
+///
+/// This is either a specific non-zero `u32`, or a fallback that matches
+/// everything.
+pub type MatchResult = Result<NonZeroU32, Else>;
+
+/// Convert a boolean to a `MatchResult`.
+#[inline]
+pub fn bool_to_match_result(b: bool) -> MatchResult {
+    let b = b as u32;
+    unsafe { Ok(NonZeroU32::new_unchecked(b + 1)) }
+}
+
+/// A partial match of an optimization's LHS and partial construction of its
+/// RHS.
+///
+/// An increment is a matching operation, the expected result from that
+/// operation to continue to the next increment, and the actions to take to
+/// build up the LHS scope and RHS instructions given that we got the expected
+/// result from this increment's matching operation. Each increment will
+/// basically become a state and a transition edge out of that state in the
+/// final automata.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Increment {
+    /// The matching operation to perform.
+    pub operation: MatchOp,
+
+    /// The expected result of our matching operation, that enables us to
+    /// continue to the next increment, or `Else` for "don't care"
+    /// wildcard-style matching.
+    pub expected: MatchResult,
+
+    /// Actions to perform, given that the operation resulted in the expected
+    /// value.
+    pub actions: Vec<Action>,
+}
+
+/// A matching operation to be performed on some Cranelift instruction as part
+/// of determining whether an optimization is applicable.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)]
+pub enum MatchOp {
+    /// Switch on the opcode of an instruction.
+    Opcode {
+        /// The path to the instruction whose opcode we're switching on.
+        path: PathId,
+    },
+
+    /// Does an instruction have a constant value?
+    IsConst {
+        /// The path to the instruction (or immediate) that we're checking
+        /// whether it is constant or not.
+        path: PathId,
+    },
+
+    /// Is the constant value a power of two?
+    IsPowerOfTwo {
+        /// The path to the instruction (or immediate) that we are checking
+        /// whether it is a constant power of two or not.
+        path: PathId,
+    },
+
+    /// Switch on the bit width of a value.
+    BitWidth {
+        /// The path to the instruction (or immediate) whose result's bit width
+        /// we are checking.
+        path: PathId,
+    },
+
+    /// Does the value fit in our target architecture's native word size?
+    FitsInNativeWord {
+        /// The path to the instruction (or immediate) whose result we are
+        /// checking whether it fits in a native word or not.
+        path: PathId,
+    },
+
+    /// Are the instructions (or immediates) at the given paths the same?
+    Eq {
+        /// The path to the first instruction (or immediate).
+        path_a: PathId,
+        /// The path to the second instruction (or immediate).
+        path_b: PathId,
+    },
+
+    /// Switch on the constant integer value of an instruction.
+    IntegerValue {
+        /// The path to the instruction.
+        path: PathId,
+    },
+
+    /// Switch on the constant boolean value of an instruction.
+    BooleanValue {
+        /// The path to the instruction.
+        path: PathId,
+    },
+
+    /// Switch on a condition code.
+    ConditionCode {
+        /// The path to the condition code.
+        path: PathId,
+    },
+
+    /// No operation. Always evaluates to `None`.
+    ///
+    /// Exceedingly rare in real optimizations; nonetheless required to support
+    /// corner cases of the DSL, such as a LHS pattern that is nothing but a
+    /// variable pattern.
+    Nop,
+}
+
+/// A canonicalized identifier for a left-hand side value that was bound in a
+/// pattern.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
+pub struct LhsId(pub u16);
+
+/// A canonicalized identifier for a right-hand side value.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct RhsId(pub u16);
+
+/// An action to perform when transitioning between states in the automata.
+///
+/// When evaluating actions, the `i^th` action implicitly defines the
+/// `RhsId(i)`.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum Action {
+    /// Reuse something from the left-hand side.
+    GetLhs {
+        /// The path to the instruction or value.
+        path: PathId,
+    },
+
+    /// Perform compile-time evaluation.
+    UnaryUnquote {
+        /// The unquote operator.
+        operator: UnquoteOperator,
+        /// The constant operand to this unquote.
+        operand: RhsId,
+    },
+
+    /// Perform compile-time evaluation.
+    BinaryUnquote {
+        /// The unquote operator.
+        operator: UnquoteOperator,
+        /// The constant operands to this unquote.
+        operands: [RhsId; 2],
+    },
+
+    /// Create an integer constant.
+    MakeIntegerConst {
+        /// The constant integer value.
+        value: IntegerId,
+        /// The bit width of this constant.
+        bit_width: BitWidth,
+    },
+
+    /// Create a boolean constant.
+    MakeBooleanConst {
+        /// The constant boolean value.
+        value: bool,
+        /// The bit width of this constant.
+        bit_width: BitWidth,
+    },
+
+    /// Create a condition code.
+    MakeConditionCode {
+        /// The condition code.
+        cc: ConditionCode,
+    },
+
+    /// Make a unary instruction.
+    MakeUnaryInst {
+        /// The operand for this instruction.
+        operand: RhsId,
+        /// The type of this instruction's result.
+        r#type: Type,
+        /// The operator for this instruction.
+        operator: Operator,
+    },
+
+    /// Make a binary instruction.
+    MakeBinaryInst {
+        /// The opcode for this instruction.
+        operator: Operator,
+        /// The type of this instruction's result.
+        r#type: Type,
+        /// The operands for this instruction.
+        operands: [RhsId; 2],
+    },
+
+    /// Make a ternary instruction.
+    MakeTernaryInst {
+        /// The opcode for this instruction.
+        operator: Operator,
+        /// The type of this instruction's result.
+        r#type: Type,
+        /// The operands for this instruction.
+        operands: [RhsId; 3],
+    },
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // These types all end up in the automaton, so we should take care that they
+    // are small and don't fill up the data cache (or take up too much
+    // serialized size).
+
+    #[test]
+    fn match_result_size() {
+        assert_eq!(std::mem::size_of::<MatchResult>(), 4);
+    }
+
+    #[test]
+    fn match_op_size() {
+        assert_eq!(std::mem::size_of::<MatchOp>(), 6);
+    }
+
+    #[test]
+    fn action_size() {
+        assert_eq!(std::mem::size_of::<Action>(), 16);
+    }
+}

cranelift/peepmatic/crates/runtime/src/operator.rs

@@ -0,0 +1,298 @@
+//! Operator definitions.
+
+use peepmatic_macro::PeepmaticOperator;
+use serde::{Deserialize, Serialize};
+
+/// An operator.
+///
+/// These are a subset of Cranelift IR's operators.
+///
+/// ## Caveats for Branching and Trapping Operators
+///
+/// Branching operators are not fully modeled: we do not represent their label
+/// and jump arguments. It is up to the interpreter doing the instruction
+/// replacement to recognize when we are replacing one branch with another, and
+/// copy over the extra information.
+///
+/// Affected operations: `brz`, `brnz`, `trapz`, `trapnz`.
+#[derive(PeepmaticOperator, Clone, Copy, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)]
+#[repr(u32)]
+pub enum Operator {
+    /// `adjust_sp_down`
+    #[peepmatic(params(iNN), result(void))]
+    // NB: We convert `Operator`s into `NonZeroU32`s with unchecked casts;
+    // memory safety relies on `Operator` starting at `1` and no variant ever
+    // being zero.
+    AdjustSpDown = 1,
+
+    /// `adjust_sp_down_imm`
+    #[peepmatic(immediates(iNN), result(void))]
+    AdjustSpDownImm,
+
+    /// `band`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Band,
+
+    /// `band_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    BandImm,
+
+    /// `bconst`
+    #[peepmatic(immediates(b1), result(bNN))]
+    Bconst,
+
+    /// `bint`
+    #[peepmatic(params(bNN), result(iNN))]
+    Bint,
+
+    /// `bor`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Bor,
+
+    /// `bor_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    BorImm,
+
+    /// `brnz`
+    #[peepmatic(params(bool_or_int), result(void))]
+    Brnz,
+
+    /// `brz`
+    #[peepmatic(params(bool_or_int), result(void))]
+    Brz,
+
+    /// `bxor`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Bxor,
+
+    /// `bxor_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    BxorImm,
+
+    /// `iadd`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Iadd,
+
+    /// `iadd_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    IaddImm,
+
+    /// `icmp`
+    #[peepmatic(immediates(cc), params(iNN, iNN), result(b1))]
+    Icmp,
+
+    /// `icmp_imm`
+    #[peepmatic(immediates(cc, iNN), params(iNN), result(b1))]
+    IcmpImm,
+
+    /// `iconst`
+    #[peepmatic(immediates(iNN), result(iNN))]
+    Iconst,
+
+    /// `ifcmp`
+    #[peepmatic(params(iNN, iNN), result(cpu_flags))]
+    Ifcmp,
+
+    /// `ifcmp_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(cpu_flags))]
+    IfcmpImm,
+
+    /// `imul`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Imul,
+
+    /// `imul_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    ImulImm,
+
+    /// `ireduce`
+    #[peepmatic(params(iNN), result(iMM))]
+    Ireduce,
+
+    /// `irsub_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    IrsubImm,
+
+    /// `ishl`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Ishl,
+
+    /// `ishl_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    IshlImm,
+
+    /// `isub`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Isub,
+
+    /// `rotl`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Rotl,
+
+    /// `rotl_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    RotlImm,
+
+    /// `rotr`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Rotr,
+
+    /// `rotr_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    RotrImm,
+
+    /// `sdiv`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Sdiv,
+
+    /// `sdiv_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    SdivImm,
+
+    /// `select`
+    #[peepmatic(params(bool_or_int, any_t, any_t), result(any_t))]
+    Select,
+
+    /// `sextend`
+    #[peepmatic(params(iNN), result(iMM))]
+    Sextend,
+
+    /// `srem`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Srem,
+
+    /// `srem_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    SremImm,
+
+    /// `sshr`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Sshr,
+
+    /// `sshr_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    SshrImm,
+
+    /// `trapnz`
+    #[peepmatic(params(bool_or_int), result(void))]
+    Trapnz,
+
+    /// `trapz`
+    #[peepmatic(params(bool_or_int), result(void))]
+    Trapz,
+
+    /// `udiv`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Udiv,
+
+    /// `udiv_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    UdivImm,
+
+    /// `uextend`
+    #[peepmatic(params(iNN), result(iMM))]
+    Uextend,
+
+    /// `urem`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Urem,
+
+    /// `urem_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    UremImm,
+
+    /// `ushr`
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Ushr,
+
+    /// `ushr_imm`
+    #[peepmatic(immediates(iNN), params(iNN), result(iNN))]
+    UshrImm,
+}
+
+/// Compile-time unquote operators.
+///
+/// These are used in the right-hand side to perform compile-time evaluation of
+/// constants matched on the left-hand side.
+#[derive(PeepmaticOperator, Clone, Copy, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)]
+#[repr(u32)]
+pub enum UnquoteOperator {
+    /// Compile-time `band` of two constant values.
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Band,
+
+    /// Compile-time `bor` of two constant values.
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Bor,
+
+    /// Compile-time `bxor` of two constant values.
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Bxor,
+
+    /// Compile-time `iadd` of two constant values.
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Iadd,
+
+    /// Compile-time `imul` of two constant values.
+    #[peepmatic(params(iNN, iNN), result(iNN))]
+    Imul,
+
+    /// Take the base-2 log of a power of two integer.
+    #[peepmatic(params(iNN), result(iNN))]
+    Log2,
+
+    /// Wrapping negation of an integer.
+    #[peepmatic(params(iNN), result(iNN))]
+    Neg,
+}
+
+/// A trait to represent a typing context.
+///
+/// This is used by the macro-generated operator methods that create the type
+/// variables for their immediates, parameters, and results. This trait is
+/// implemented by the concrete typing context in `peepmatic/src/verify.rs`.
+#[cfg(feature = "construct")]
+pub trait TypingContext<'a> {
+    /// A type variable.
+    type TypeVariable;
+
+    /// Create a condition code type.
+    fn cc(&mut self, span: wast::Span) -> Self::TypeVariable;
+
+    /// Create a boolean type with a polymorphic bit width.
+    ///
+    /// Each use of `bNN` by the same operator refers to the same type variable.
+    #[allow(non_snake_case)]
+    fn bNN(&mut self, span: wast::Span) -> Self::TypeVariable;
+
+    /// Create an integer type with a polymorphic bit width.
+    ///
+    /// Each use of `iNN` by the same operator refers to the same type variable.
+    #[allow(non_snake_case)]
+    fn iNN(&mut self, span: wast::Span) -> Self::TypeVariable;
+
+    /// Create an integer type with a polymorphic bit width.
+    ///
+    /// Each use of `iMM` by the same operator refers to the same type variable.
+    #[allow(non_snake_case)]
+    fn iMM(&mut self, span: wast::Span) -> Self::TypeVariable;
+
+    /// Create the CPU flags type variable.
+    fn cpu_flags(&mut self, span: wast::Span) -> Self::TypeVariable;
+
+    /// Create a boolean type of size one bit.
+    fn b1(&mut self, span: wast::Span) -> Self::TypeVariable;
+
+    /// Create the void type, used as the result of operators that branch away,
+    /// or do not return anything.
+    fn void(&mut self, span: wast::Span) -> Self::TypeVariable;
+
+    /// Create a type variable that may be either a boolean or an integer.
+    fn bool_or_int(&mut self, span: wast::Span) -> Self::TypeVariable;
+
+    /// Create a type variable that can be any type T.
+    ///
+    /// Each use of `any_t` by the same operator refers to the same type
+    /// variable.
+    fn any_t(&mut self, span: wast::Span) -> Self::TypeVariable;
+}

cranelift/peepmatic/crates/runtime/src/optimizations.rs

@@ -0,0 +1,74 @@
+//! Compiled peephole optimizations.
+
+use crate::error::Result;
+use crate::instruction_set::InstructionSet;
+use crate::integer_interner::IntegerInterner;
+use crate::linear::{Action, MatchOp, MatchResult};
+use crate::optimizer::PeepholeOptimizer;
+use crate::paths::PathInterner;
+use peepmatic_automata::Automaton;
+use serde::{Deserialize, Serialize};
+
+#[cfg(feature = "construct")]
+use std::fs;
+#[cfg(feature = "construct")]
+use std::path::Path;
+
+/// A compiled set of peephole optimizations.
+///
+/// This is the compilation result of the `peepmatic` crate, after its taken a
+/// bunch of optimizations written in the DSL and lowered and combined them.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct PeepholeOptimizations {
+    /// The instruction paths referenced by the peephole optimizations.
+    pub paths: PathInterner,
+
+    /// Not all integers we're matching on fit in the `u32` that we use as the
+    /// result of match operations. So we intern them and refer to them by id.
+    pub integers: IntegerInterner,
+
+    /// The underlying automata for matching optimizations' left-hand sides, and
+    /// building up the corresponding right-hand side.
+    pub automata: Automaton<MatchResult, MatchOp, Box<[Action]>>,
+}
+
+impl PeepholeOptimizations {
+    /// Deserialize a `PeepholeOptimizations` from bytes.
+    pub fn deserialize(serialized: &[u8]) -> Result<Self> {
+        let peep_opt: Self = bincode::deserialize(serialized)?;
+        Ok(peep_opt)
+    }
+
+    /// Serialize these peephole optimizations out to the file at the given path.
+    ///
+    /// Requires that the `"construct"` cargo feature is enabled.
+    #[cfg(feature = "construct")]
+    pub fn serialize_to_file(&self, path: &Path) -> Result<()> {
+        let file = fs::File::create(path)?;
+        bincode::serialize_into(file, self)?;
+        Ok(())
+    }
+
+    /// Create a new peephole optimizer instance from this set of peephole
+    /// optimizations.
+    ///
+    /// The peephole optimizer instance can be used to apply these peephole
+    /// optimizations. When checking multiple instructions for whether they can
+    /// be optimized, it is more performant to reuse a single peephole optimizer
+    /// instance, rather than create a new one for each instruction. Reusing the
+    /// peephole optimizer instance allows the reuse of a few internal
+    /// allocations.
+    pub fn optimizer<'peep, 'ctx, I>(&'peep self, instr_set: I) -> PeepholeOptimizer<'peep, 'ctx, I>
+    where
+        I: InstructionSet<'ctx>,
+    {
+        PeepholeOptimizer {
+            peep_opt: self,
+            instr_set,
+            left_hand_sides: vec![],
+            right_hand_sides: vec![],
+            actions: vec![],
+            backtracking_states: vec![],
+        }
+    }
+}

cranelift/peepmatic/crates/runtime/src/optimizer.rs

@@ -0,0 +1,580 @@
+//! An optimizer for a set of peephole optimizations.
+
+use crate::instruction_set::InstructionSet;
+use crate::linear::{bool_to_match_result, Action, Else, MatchOp, MatchResult};
+use crate::operator::UnquoteOperator;
+use crate::optimizations::PeepholeOptimizations;
+use crate::part::{Constant, Part};
+use crate::r#type::{BitWidth, Type};
+use peepmatic_automata::State;
+use std::convert::TryFrom;
+use std::fmt::{self, Debug};
+use std::mem;
+use std::num::NonZeroU32;
+
+/// A peephole optimizer instance that can apply a set of peephole
+/// optimizations to instructions.
+///
+/// These are created from a set of peephole optimizations with the
+/// [`PeepholeOptimizer::instance`][crate::PeepholeOptimizer::instance] method.
+///
+/// Reusing an instance when applying peephole optimizations to different
+/// instruction sequences means that you reuse internal allocations that are
+/// used to match left-hand sides and build up right-hand sides.
+pub struct PeepholeOptimizer<'peep, 'ctx, I>
+where
+    I: InstructionSet<'ctx>,
+{
+    pub(crate) peep_opt: &'peep PeepholeOptimizations,
+    pub(crate) instr_set: I,
+    pub(crate) left_hand_sides: Vec<Part<I::Instruction>>,
+    pub(crate) right_hand_sides: Vec<Part<I::Instruction>>,
+    pub(crate) actions: Vec<Action>,
+    pub(crate) backtracking_states: Vec<(State, usize)>,
+}
+
+impl<'peep, 'ctx, I> Debug for PeepholeOptimizer<'peep, 'ctx, I>
+where
+    I: InstructionSet<'ctx>,
+{
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let PeepholeOptimizer {
+            peep_opt,
+            instr_set: _,
+            left_hand_sides,
+            right_hand_sides,
+            actions,
+            backtracking_states,
+        } = self;
+        f.debug_struct("PeepholeOptimizer")
+            .field("peep_opt", peep_opt)
+            .field("instr_set", &"_")
+            .field("left_hand_sides", left_hand_sides)
+            .field("right_hand_sides", right_hand_sides)
+            .field("actions", actions)
+            .field("backtracking_states", backtracking_states)
+            .finish()
+    }
+}
+
+impl<'peep, 'ctx, I> PeepholeOptimizer<'peep, 'ctx, I>
+where
+    I: InstructionSet<'ctx>,
+{
+    fn eval_unquote_1(&self, operator: UnquoteOperator, a: Constant) -> Constant {
+        use Constant::*;
+
+        macro_rules! map_int {
+            ( $c:expr , | $x:ident | $e:expr ) => {
+                match $c {
+                    Int($x, w) => Int($e, w),
+                    Bool(..) => panic!("not an integer"),
+                }
+            };
+        }
+
+        match operator {
+            UnquoteOperator::Log2 => map_int!(a, |x| x.trailing_zeros() as _),
+            UnquoteOperator::Neg => map_int!(a, |x| x.wrapping_neg()),
+            UnquoteOperator::Band
+            | UnquoteOperator::Bor
+            | UnquoteOperator::Bxor
+            | UnquoteOperator::Iadd
+            | UnquoteOperator::Imul => unreachable!("not a unary unquote operator: {:?}", operator),
+        }
+    }
+
+    fn eval_unquote_2(&self, operator: UnquoteOperator, a: Constant, b: Constant) -> Constant {
+        use Constant::*;
+
+        macro_rules! fold_ints {
+            ( $c1:expr , $c2:expr , | $x:ident , $y:ident | $e:expr ) => {
+                match ($c1, $c2) {
+                    (Int($x, w1), Int($y, w2)) if w1 == w2 => Int($e, w1),
+                    _ => panic!("not two integers of the same width"),
+                }
+            };
+        }
+
+        match operator {
+            UnquoteOperator::Band => fold_ints!(a, b, |x, y| x & y),
+            UnquoteOperator::Bor => fold_ints!(a, b, |x, y| x | y),
+            UnquoteOperator::Bxor => fold_ints!(a, b, |x, y| x ^ y),
+            UnquoteOperator::Iadd => fold_ints!(a, b, |x, y| x.wrapping_add(y)),
+            UnquoteOperator::Imul => fold_ints!(a, b, |x, y| x.wrapping_mul(y)),
+            UnquoteOperator::Log2 | UnquoteOperator::Neg => {
+                unreachable!("not a binary unquote operator: {:?}", operator)
+            }
+        }
+    }
+
+    fn eval_actions(&mut self, context: &mut I::Context, root: I::Instruction) {
+        let mut actions = mem::replace(&mut self.actions, vec![]);
+
+        for action in actions.drain(..) {
+            log::trace!("Evaluating action: {:?}", action);
+            match action {
+                Action::GetLhs { path } => {
+                    let path = self.peep_opt.paths.lookup(path);
+                    let lhs = self
+                        .instr_set
+                        .get_part_at_path(context, root, path)
+                        .expect("should always get part at path OK by the time it is bound");
+                    self.right_hand_sides.push(lhs);
+                }
+                Action::UnaryUnquote { operator, operand } => {
+                    let operand = self.right_hand_sides[operand.0 as usize];
+                    let operand = match operand {
+                        Part::Instruction(i) => self
+                            .instr_set
+                            .instruction_to_constant(context, i)
+                            .expect("cannot convert instruction to constant for unquote operand"),
+                        Part::Constant(c) => c,
+                        Part::ConditionCode(_) => {
+                            panic!("cannot use a condition code as an unquote operand")
+                        }
+                    };
+                    let result = self.eval_unquote_1(operator, operand);
+                    self.right_hand_sides.push(result.into());
+                }
+                Action::BinaryUnquote { operator, operands } => {
+                    let a = self.right_hand_sides[operands[0].0 as usize];
+                    let a = match a {
+                        Part::Instruction(i) => self
+                            .instr_set
+                            .instruction_to_constant(context, i)
+                            .expect("cannot convert instruction to constant for unquote operand"),
+                        Part::Constant(c) => c,
+                        Part::ConditionCode(_) => {
+                            panic!("cannot use a condition code as an unquote operand")
+                        }
+                    };
+
+                    let b = self.right_hand_sides[operands[1].0 as usize];
+                    let b = match b {
+                        Part::Instruction(i) => self
+                            .instr_set
+                            .instruction_to_constant(context, i)
+                            .expect("cannot convert instruction to constant for unquote operand"),
+                        Part::Constant(c) => c,
+                        Part::ConditionCode(_) => {
+                            panic!("cannot use a condition code as an unquote operand")
+                        }
+                    };
+
+                    let result = self.eval_unquote_2(operator, a, b);
+                    self.right_hand_sides.push(result.into());
+                }
+                Action::MakeIntegerConst {
+                    value,
+                    mut bit_width,
+                } => {
+                    let value = self.peep_opt.integers.lookup(value);
+                    if bit_width.is_polymorphic() {
+                        bit_width = BitWidth::try_from(
+                            self.instr_set.instruction_result_bit_width(context, root),
+                        )
+                        .unwrap();
+                    }
+                    self.right_hand_sides
+                        .push(Constant::Int(value, bit_width).into());
+                }
+                Action::MakeBooleanConst {
+                    value,
+                    mut bit_width,
+                } => {
+                    if bit_width.is_polymorphic() {
+                        bit_width = BitWidth::try_from(
+                            self.instr_set.instruction_result_bit_width(context, root),
+                        )
+                        .unwrap();
+                    }
+                    self.right_hand_sides
+                        .push(Constant::Bool(value, bit_width).into());
+                }
+                Action::MakeConditionCode { cc } => {
+                    self.right_hand_sides.push(Part::ConditionCode(cc));
+                }
+                Action::MakeUnaryInst {
+                    operator,
+                    r#type:
+                        Type {
+                            kind,
+                            mut bit_width,
+                        },
+                    operand,
+                } => {
+                    if bit_width.is_polymorphic() {
+                        bit_width = BitWidth::try_from(
+                            self.instr_set.instruction_result_bit_width(context, root),
+                        )
+                        .unwrap();
+                    }
+                    let ty = Type { kind, bit_width };
+                    let operand = self.right_hand_sides[operand.0 as usize];
+                    let inst = self
+                        .instr_set
+                        .make_inst_1(context, root, operator, ty, operand);
+                    self.right_hand_sides.push(Part::Instruction(inst));
+                }
+                Action::MakeBinaryInst {
+                    operator,
+                    r#type:
+                        Type {
+                            kind,
+                            mut bit_width,
+                        },
+                    operands,
+                } => {
+                    if bit_width.is_polymorphic() {
+                        bit_width = BitWidth::try_from(
+                            self.instr_set.instruction_result_bit_width(context, root),
+                        )
+                        .unwrap();
+                    }
+                    let ty = Type { kind, bit_width };
+                    let a = self.right_hand_sides[operands[0].0 as usize];
+                    let b = self.right_hand_sides[operands[1].0 as usize];
+                    let inst = self
+                        .instr_set
+                        .make_inst_2(context, root, operator, ty, a, b);
+                    self.right_hand_sides.push(Part::Instruction(inst));
+                }
+                Action::MakeTernaryInst {
+                    operator,
+                    r#type:
+                        Type {
+                            kind,
+                            mut bit_width,
+                        },
+                    operands,
+                } => {
+                    if bit_width.is_polymorphic() {
+                        bit_width = BitWidth::try_from(
+                            self.instr_set.instruction_result_bit_width(context, root),
+                        )
+                        .unwrap();
+                    }
+                    let ty = Type { kind, bit_width };
+                    let a = self.right_hand_sides[operands[0].0 as usize];
+                    let b = self.right_hand_sides[operands[1].0 as usize];
+                    let c = self.right_hand_sides[operands[2].0 as usize];
+                    let inst = self
+                        .instr_set
+                        .make_inst_3(context, root, operator, ty, a, b, c);
+                    self.right_hand_sides.push(Part::Instruction(inst));
+                }
+            }
+        }
+
+        // Reuse the heap elements allocation.
+        self.actions = actions;
+    }
+
+    fn eval_match_op(
+        &mut self,
+        context: &mut I::Context,
+        root: I::Instruction,
+        match_op: MatchOp,
+    ) -> MatchResult {
+        use crate::linear::MatchOp::*;
+
+        log::trace!("Evaluating match operation: {:?}", match_op);
+        let result: MatchResult = (|| match match_op {
+            Opcode { path } => {
+                let path = self.peep_opt.paths.lookup(path);
+                let part = self
+                    .instr_set
+                    .get_part_at_path(context, root, path)
+                    .ok_or(Else)?;
+                let inst = part.as_instruction().ok_or(Else)?;
+                let op = self.instr_set.operator(context, inst).ok_or(Else)?;
+                let op = op as u32;
+                debug_assert!(
+                    op != 0,
+                    "`Operator` doesn't have any variant represented
+        with zero"
+                );
+                Ok(unsafe { NonZeroU32::new_unchecked(op as u32) })
+            }
+            IsConst { path } => {
+                let path = self.peep_opt.paths.lookup(path);
+                let part = self
+                    .instr_set
+                    .get_part_at_path(context, root, path)
+                    .ok_or(Else)?;
+                let is_const = match part {
+                    Part::Instruction(i) => {
+                        self.instr_set.instruction_to_constant(context, i).is_some()
+                    }
+                    Part::ConditionCode(_) | Part::Constant(_) => true,
+                };
+                bool_to_match_result(is_const)
+            }
+            IsPowerOfTwo { path } => {
+                let path = self.peep_opt.paths.lookup(path);
+                let part = self
+                    .instr_set
+                    .get_part_at_path(context, root, path)
+                    .ok_or(Else)?;
+                match part {
+                    Part::Constant(c) => {
+                        let is_pow2 = c.as_int().unwrap().is_power_of_two();
+                        bool_to_match_result(is_pow2)
+                    }
+                    Part::Instruction(i) => {
+                        let c = self
+                            .instr_set
+                            .instruction_to_constant(context, i)
+                            .ok_or(Else)?;
+                        let is_pow2 = c.as_int().unwrap().is_power_of_two();
+                        bool_to_match_result(is_pow2)
+                    }
+                    Part::ConditionCode(_) => unreachable!(
+                        "IsPowerOfTwo on a condition
+        code"
+                    ),
+                }
+            }
+            BitWidth { path } => {
+                let path = self.peep_opt.paths.lookup(path);
+                let part = self
+                    .instr_set
+                    .get_part_at_path(context, root, path)
+                    .ok_or(Else)?;
+                let bit_width = match part {
+                    Part::Instruction(i) => self.instr_set.instruction_result_bit_width(context, i),
+                    Part::Constant(Constant::Int(_, w)) | Part::Constant(Constant::Bool(_, w)) => {
+                        w.fixed_width().unwrap_or_else(|| {
+                            self.instr_set.instruction_result_bit_width(context, root)
+                        })
+                    }
+                    Part::ConditionCode(_) => panic!("BitWidth on condition code"),
+                };
+                debug_assert!(
+                    bit_width != 0,
+                    "`InstructionSet` implementors must uphold the contract that \
+                     `instruction_result_bit_width` returns one of 1, 8, 16, 32, 64, or 128"
+                );
+                Ok(unsafe { NonZeroU32::new_unchecked(bit_width as u32) })
+            }
+            FitsInNativeWord { path } => {
+                let native_word_size = self.instr_set.native_word_size_in_bits(context);
+                debug_assert!(native_word_size.is_power_of_two());
+
+                let path = self.peep_opt.paths.lookup(path);
+                let part = self
+                    .instr_set
+                    .get_part_at_path(context, root, path)
+                    .ok_or(Else)?;
+                let fits = match part {
+                    Part::Instruction(i) => {
+                        let size = self.instr_set.instruction_result_bit_width(context, i);
+                        size <= native_word_size
+                    }
+                    Part::Constant(c) => {
+                        let root_width = self.instr_set.instruction_result_bit_width(context, root);
+                        let size = c.bit_width(root_width);
+                        size <= native_word_size
+                    }
+                    Part::ConditionCode(_) => panic!("FitsInNativeWord on condition code"),
+                };
+                bool_to_match_result(fits)
+            }
+            Eq { path_a, path_b } => {
+                let path_a = self.peep_opt.paths.lookup(path_a);
+                let part_a = self
+                    .instr_set
+                    .get_part_at_path(context, root, path_a)
+                    .ok_or(Else)?;
+                let path_b = self.peep_opt.paths.lookup(path_b);
+                let part_b = self
+                    .instr_set
+                    .get_part_at_path(context, root, path_b)
+                    .ok_or(Else)?;
+                let eq = match (part_a, part_b) {
+                    (Part::Instruction(inst), Part::Constant(c1))
+                    | (Part::Constant(c1), Part::Instruction(inst)) => {
+                        match self.instr_set.instruction_to_constant(context, inst) {
+                            Some(c2) => c1 == c2,
+                            None => false,
+                        }
+                    }
+                    (a, b) => a == b,
+                };
+                bool_to_match_result(eq)
+            }
+            IntegerValue { path } => {
+                let path = self.peep_opt.paths.lookup(path);
+                let part = self
+                    .instr_set
+                    .get_part_at_path(context, root, path)
+                    .ok_or(Else)?;
+                match part {
+                    Part::Constant(c) => {
+                        let x = c.as_int().ok_or(Else)?;
+                        let id = self.peep_opt.integers.already_interned(x).ok_or(Else)?;
+                        Ok(id.into())
+                    }
+                    Part::Instruction(i) => {
+                        let c = self
+                            .instr_set
+                            .instruction_to_constant(context, i)
+                            .ok_or(Else)?;
+                        let x = c.as_int().ok_or(Else)?;
+                        let id = self.peep_opt.integers.already_interned(x).ok_or(Else)?;
+                        Ok(id.into())
+                    }
+                    Part::ConditionCode(_) => unreachable!("IntegerValue on condition code"),
+                }
+            }
+            BooleanValue { path } => {
+                let path = self.peep_opt.paths.lookup(path);
+                let part = self
+                    .instr_set
+                    .get_part_at_path(context, root, path)
+                    .ok_or(Else)?;
+                match part {
+                    Part::Constant(c) => {
+                        let b = c.as_bool().ok_or(Else)?;
+                        bool_to_match_result(b)
+                    }
+                    Part::Instruction(i) => {
+                        let c = self
+                            .instr_set
+                            .instruction_to_constant(context, i)
+                            .ok_or(Else)?;
+                        let b = c.as_bool().ok_or(Else)?;
+                        bool_to_match_result(b)
+                    }
+                    Part::ConditionCode(_) => unreachable!("IntegerValue on condition code"),
+                }
+            }
+            ConditionCode { path } => {
+                let path = self.peep_opt.paths.lookup(path);
+                let part = self
+                    .instr_set
+                    .get_part_at_path(context, root, path)
+                    .ok_or(Else)?;
+                let cc = part.as_condition_code().ok_or(Else)?;
+                let cc = cc as u32;
+                debug_assert!(cc != 0);
+                Ok(unsafe { NonZeroU32::new_unchecked(cc) })
+            }
+            MatchOp::Nop => Err(Else),
+        })();
+        log::trace!("Evaluated match operation: {:?} = {:?}", match_op, result);
+        result
+    }
+
+    /// Attempt to apply a single peephole optimization to the given root
+    /// instruction.
+    ///
+    /// If an optimization is applied, then the `root` is replaced with the
+    /// optimization's right-hand side, and the root of the right-hand side is
+    /// returned as `Some`.
+    ///
+    /// If no optimization's left-hand side matches `root`, then `root` is left
+    /// untouched and `None` is returned.
+    pub fn apply_one(
+        &mut self,
+        context: &mut I::Context,
+        root: I::Instruction,
+    ) -> Option<I::Instruction> {
+        log::trace!("PeepholeOptimizer::apply_one");
+
+        self.backtracking_states.clear();
+        self.actions.clear();
+        self.left_hand_sides.clear();
+        self.right_hand_sides.clear();
+
+        let mut r#final = None;
+
+        let mut query = self.peep_opt.automata.query();
+        loop {
+            log::trace!("Current state: {:?}", query.current_state());
+
+            if query.is_in_final_state() {
+                // If we're in a final state (which means an optimization is
+                // applicable) then record that fact, but keep going. We don't
+                // want to stop yet, because we might discover another,
+                // more-specific optimization that is also applicable if we keep
+                // going. And we always want to apply the most specific
+                // optimization that matches.
+                log::trace!("Found a match at state {:?}", query.current_state());
+                r#final = Some((query.current_state(), self.actions.len()));
+            }
+
+            // Anything following a `Else` transition doesn't care about the
+            // result of this match operation, so if we partially follow the
+            // current non-`Else` path, but don't ultimately find a matching
+            // optimization, we want to be able to backtrack to this state and
+            // then try taking the `Else` transition.
+            if query.has_transition_on(&Err(Else)) {
+                self.backtracking_states
+                    .push((query.current_state(), self.actions.len()));
+            }
+
+            let match_op = match query.current_state_data() {
+                None => break,
+                Some(op) => op,
+            };
+
+            let input = self.eval_match_op(context, root, *match_op);
+
+            let actions = if let Some(actions) = query.next(&input) {
+                actions
+            } else if r#final.is_some() {
+                break;
+            } else if let Some((state, actions_len)) = self.backtracking_states.pop() {
+                query.go_to_state(state);
+                self.actions.truncate(actions_len);
+                query
+                    .next(&Err(Else))
+                    .expect("backtracking states always have `Else` transitions")
+            } else {
+                break;
+            };
+
+            self.actions.extend(actions.iter().copied());
+        }
+
+        // If `final` is none, then we didn't encounter any final states, so
+        // there are no applicable optimizations.
+        let (final_state, actions_len) = match r#final {
+            Some(f) => f,
+            None => {
+                log::trace!("No optimizations matched");
+                return None;
+            }
+        };
+
+        // Go to the last final state we saw, reset the LHS and RHS to how
+        // they were at the time we saw the final state, and process the
+        // final actions.
+        self.actions.truncate(actions_len);
+        query.go_to_state(final_state);
+        let final_actions = query.finish().expect("should be in a final state");
+        self.actions.extend(final_actions.iter().copied());
+        self.eval_actions(context, root);
+
+        // And finally, the root of the RHS for this optimization is the
+        // last entry in `self.right_hand_sides`, so replace the old root
+        // instruction with this one!
+        let result = self.right_hand_sides.pop().unwrap();
+        let new_root = self.instr_set.replace_instruction(context, root, result);
+        Some(new_root)
+    }
+
+    /// Keep applying peephole optimizations to the given instruction until none
+    /// can be applied anymore.
+    pub fn apply_all(&mut self, context: &mut I::Context, mut inst: I::Instruction) {
+        loop {
+            if let Some(new_inst) = self.apply_one(context, inst) {
+                inst = new_inst;
+            } else {
+                break;
+            }
+        }
+    }
+}

cranelift/peepmatic/crates/runtime/src/part.rs

@@ -0,0 +1,128 @@
+//! Parts of instructions.
+
+use crate::cc::ConditionCode;
+use crate::r#type::BitWidth;
+use std::fmt::Debug;
+
+/// A constant value.
+///
+/// Whether an integer is interpreted as signed or unsigned depends on the
+/// operations applied to it.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum Constant {
+    /// A boolean of the given width.
+    Bool(bool, BitWidth),
+
+    /// An integer constant of the given width,
+    Int(u64, BitWidth),
+}
+
+/// A part of an instruction, or a whole instruction itself.
+///
+/// These are the different values that can be matched in an optimization's
+/// left-hand side and then built up in its right-hand side.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum Part<I>
+where
+    I: Copy + Debug + Eq,
+{
+    /// An instruction (or result of an instruction).
+    Instruction(I),
+
+    /// A constant value.
+    Constant(Constant),
+
+    /// A condition code.
+    ConditionCode(ConditionCode),
+}
+
+impl<I> From<Constant> for Part<I>
+where
+    I: Copy + Debug + Eq,
+{
+    #[inline]
+    fn from(c: Constant) -> Part<I> {
+        Part::Constant(c)
+    }
+}
+
+impl<I> From<ConditionCode> for Part<I>
+where
+    I: Copy + Debug + Eq,
+{
+    #[inline]
+    fn from(c: ConditionCode) -> Part<I> {
+        Part::ConditionCode(c)
+    }
+}
+
+macro_rules! accessors {
+    ( $( $variant:ident , $result:ty , $getter:ident , $is:ident , $unwrap:ident ; )* ) => {
+        $(
+            #[inline]
+            #[allow(missing_docs)]
+            pub fn $getter(&self) -> Option<$result> {
+                match *self {
+                    Self::$variant(x, ..) => Some(x),
+                    _ => None
+                }
+            }
+
+            #[inline]
+            #[allow(missing_docs)]
+            pub fn $is(&self) -> bool {
+                self.$getter().is_some()
+            }
+
+            #[inline]
+            #[allow(missing_docs)]
+            pub fn $unwrap(&self) -> $result {
+                self.$getter().expect(concat!("failed to unwrap `", stringify!($variant), "`"))
+            }
+        )*
+    }
+}
+
+impl Constant {
+    /// If this is any kind of integer constant, get it as a 64-bit unsigned
+    /// integer.
+    pub fn as_int(&self) -> Option<u64> {
+        match *self {
+            Constant::Bool(..) => None,
+            Constant::Int(x, _) => Some(x),
+        }
+    }
+
+    /// If this is any kind of boolean constant, get its value.
+    pub fn as_bool(&self) -> Option<bool> {
+        match *self {
+            Constant::Bool(b, _) => Some(b),
+            Constant::Int(..) => None,
+        }
+    }
+
+    /// The number of bits required to represent this constant value's type.
+    pub fn bit_width(&self, root_width: u8) -> u8 {
+        match *self {
+            Constant::Bool(_, w) | Constant::Int(_, w) => {
+                if let Some(w) = w.fixed_width() {
+                    w
+                } else {
+                    debug_assert!(w.is_polymorphic());
+                    root_width
+                }
+            }
+        }
+    }
+}
+
+impl<I> Part<I>
+where
+    I: Copy + Debug + Eq,
+{
+    accessors! {
+        Instruction, I, as_instruction, is_instruction, unwrap_instruction;
+        Constant, Constant, as_constant, is_constant, unwrap_constant;
+        ConditionCode, ConditionCode, as_condition_code, is_condition_code, unwrap_condition_code;
+    }
+}

cranelift/peepmatic/crates/runtime/src/paths.rs

@@ -0,0 +1,242 @@
+//! Representing paths through the dataflow graph.
+//!
+//! Paths are relative from a *root* instruction, which is the instruction we
+//! are determining which, if any, optimizations apply.
+//!
+//! Paths are series of indices through each instruction's children as we
+//! traverse down the graph from the root. Children are immediates followed by
+//! arguments: `[imm0, imm1, ..., immN, arg0, arg1, ..., argN]`.
+//!
+//! ## Examples
+//!
+//! * `[0]` is the path to the root.
+//! * `[0, 0]` is the path to the root's first child.
+//! * `[0, 1]` is the path to the root's second child.
+//! * `[0, 1, 0]` is the path to the root's second child's first child.
+//!
+//! ## Interning
+//!
+//! To avoid extra allocations, de-duplicate paths, and reference them via a
+//! fixed-length value, we intern paths inside a `PathInterner` and then
+//! reference them via `PathId`.
+
+// TODO: Make `[]` the path to the root, and get rid of this redundant leading
+// zero that is currently in every single path.
+
+use serde::de::{Deserializer, SeqAccess, Visitor};
+use serde::ser::{SerializeSeq, Serializer};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::convert::TryInto;
+use std::fmt;
+use std::hash::{Hash, Hasher};
+use std::marker::PhantomData;
+
+/// A path through the data-flow graph from the root instruction.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Path<'a>(pub &'a [u8]);
+
+impl Path<'_> {
+    /// Construct a new path through the data-flow graph from the root
+    /// instruction.
+    pub fn new(path: &impl AsRef<[u8]>) -> Path {
+        Path(path.as_ref())
+    }
+}
+
+/// An identifier for an interned path.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct PathId(u16);
+
+/// An interner and de-duplicator for `Path`s.
+///
+/// Can be serialized and deserialized while maintaining the same id to interned
+/// path mapping.
+#[derive(Debug, Default)]
+pub struct PathInterner {
+    /// A map from a path (whose owned data is inside `arena`) to the canonical
+    /// `PathId` we assigned it when interning it.
+    map: HashMap<UnsafePath, PathId>,
+
+    /// A map from a `PathId` index to an unsafe, self-borrowed path pointing
+    /// into `arena`. It is safe to given these out as safe `Path`s, as long as
+    /// the lifetime is not longer than this `PathInterner`'s lifetime.
+    paths: Vec<UnsafePath>,
+
+    /// Bump allocation arena for path data. The bump arena ensures that these
+    /// allocations never move, and are therefore safe for self-references.
+    arena: bumpalo::Bump,
+}
+
+impl PathInterner {
+    /// Construct a new, empty `PathInterner`.
+    #[inline]
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Intern a path into this `PathInterner`, returning its canonical
+    /// `PathId`.
+    ///
+    /// If we've already interned this path before, then the existing id we
+    /// already assigned to it is returned. If we've never seen this path
+    /// before, then it is copied into this `PathInterner` and a new id is
+    /// assigned to it.
+    #[inline]
+    pub fn intern<'a>(&mut self, path: Path<'a>) -> PathId {
+        let unsafe_path = unsafe { UnsafePath::from_path(&path) };
+        if let Some(id) = self.map.get(&unsafe_path) {
+            return *id;
+        }
+        self.intern_new(path)
+    }
+
+    #[inline(never)]
+    fn intern_new<'a>(&mut self, path: Path<'a>) -> PathId {
+        let id: u16 = self
+            .paths
+            .len()
+            .try_into()
+            .expect("too many paths interned");
+        let id = PathId(id);
+
+        let our_path = self.arena.alloc_slice_copy(&path.0);
+        let unsafe_path = unsafe { UnsafePath::from_slice(&our_path) };
+
+        self.paths.push(unsafe_path.clone());
+        let old = self.map.insert(unsafe_path, id);
+
+        debug_assert!(old.is_none());
+        debug_assert_eq!(self.lookup(id), path);
+        debug_assert_eq!(self.intern(path), id);
+
+        id
+    }
+
+    /// Lookup a previously interned path by id.
+    #[inline]
+    pub fn lookup<'a>(&'a self, id: PathId) -> Path<'a> {
+        let unsafe_path = self
+            .paths
+            .get(id.0 as usize)
+            .unwrap_or_else(|| Self::lookup_failure());
+        unsafe { unsafe_path.as_path() }
+    }
+
+    #[inline(never)]
+    fn lookup_failure() -> ! {
+        panic!(
+            "no path for the given id; this can only happen when mixing `PathId`s with different \
+             `PathInterner`s"
+        )
+    }
+}
+
+impl Serialize for PathInterner {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut seq = serializer.serialize_seq(Some(self.paths.len()))?;
+        for p in &self.paths {
+            let p = unsafe { p.as_path() };
+            seq.serialize_element(&p)?;
+        }
+        seq.end()
+    }
+}
+
+impl<'de> Deserialize<'de> for PathInterner {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_seq(PathInternerVisitor {
+            marker: PhantomData,
+        })
+    }
+}
+
+struct PathInternerVisitor {
+    marker: PhantomData<fn() -> PathInterner>,
+}
+
+impl<'de> Visitor<'de> for PathInternerVisitor {
+    type Value = PathInterner;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        write!(formatter, "a `peepmatic_runtime::paths::PathInterner`")
+    }
+
+    fn visit_seq<M>(self, mut access: M) -> Result<Self::Value, M::Error>
+    where
+        M: SeqAccess<'de>,
+    {
+        const DEFAULT_CAPACITY: usize = 16;
+        let capacity = access.size_hint().unwrap_or(DEFAULT_CAPACITY);
+
+        let mut interner = PathInterner {
+            map: HashMap::with_capacity(capacity),
+            paths: Vec::with_capacity(capacity),
+            arena: bumpalo::Bump::new(),
+        };
+
+        while let Some(path) = access.next_element::<Path>()? {
+            interner.intern(path);
+        }
+
+        Ok(interner)
+    }
+}
+
+/// An unsafe, unchecked borrow of a path. Not for use outside of
+/// `PathInterner`!
+#[derive(Clone, Debug)]
+struct UnsafePath {
+    ptr: *const u8,
+    len: usize,
+}
+
+impl PartialEq for UnsafePath {
+    fn eq(&self, rhs: &UnsafePath) -> bool {
+        unsafe { self.as_slice() == rhs.as_slice() }
+    }
+}
+
+impl Eq for UnsafePath {}
+
+impl Hash for UnsafePath {
+    fn hash<H>(&self, hasher: &mut H)
+    where
+        H: Hasher,
+    {
+        unsafe { self.as_slice().hash(hasher) }
+    }
+}
+
+/// Safety: callers must ensure that the constructed values won't have unsafe
+/// usages of `PartialEq`, `Eq`, or `Hash`.
+impl UnsafePath {
+    unsafe fn from_path(p: &Path) -> Self {
+        Self::from_slice(&p.0)
+    }
+
+    unsafe fn from_slice(s: &[u8]) -> Self {
+        UnsafePath {
+            ptr: s.as_ptr(),
+            len: s.len(),
+        }
+    }
+}
+
+/// Safety: callers must ensure that `'a` does not outlive the lifetime of the
+/// underlying data.
+impl UnsafePath {
+    unsafe fn as_slice<'a>(&self) -> &'a [u8] {
+        std::slice::from_raw_parts(self.ptr, self.len)
+    }
+
+    unsafe fn as_path<'a>(&self) -> Path<'a> {
+        Path(self.as_slice())
+    }
+}

cranelift/peepmatic/crates/runtime/src/type.rs

@@ -0,0 +1,262 @@
+//! Types.
+
+use serde::{Deserialize, Serialize};
+use std::convert::TryFrom;
+use std::fmt;
+
+/// A bit width of a type.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[repr(u8)]
+pub enum BitWidth {
+    /// Polymorphic over bit width, with the same width as the root of the
+    /// optimization's LHS and RHS.
+    Polymorphic = 0,
+
+    /// A fixed bit width of 1.
+    One = 1,
+
+    /// A fixed bit width of 8.
+    Eight = 8,
+
+    /// A fixed bit width of 16.
+    Sixteen = 16,
+
+    /// A fixed bit width of 32.
+    ThirtyTwo = 32,
+
+    /// A fixed bit width of 64.
+    SixtyFour = 64,
+
+    /// A fixed bit width of 128.
+    OneTwentyEight = 128,
+}
+
+/// The kind of type we are looking at: either an integer kind or boolean kind.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum Kind {
+    /// Integer kind.
+    Int,
+
+    /// Boolean kind.
+    Bool,
+
+    /// CPU flags kind.
+    CpuFlags,
+
+    /// Void kind.
+    Void,
+}
+
+/// A type a value or the result of an operation.
+#[allow(missing_docs)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct Type {
+    /// This type's kind.
+    pub kind: Kind,
+
+    /// This type's bit width.
+    pub bit_width: BitWidth,
+}
+
+impl TryFrom<u8> for BitWidth {
+    type Error = &'static str;
+
+    #[inline]
+    fn try_from(x: u8) -> Result<Self, Self::Error> {
+        Ok(match x {
+            1 => Self::One,
+            8 => Self::Eight,
+            16 => Self::Sixteen,
+            32 => Self::ThirtyTwo,
+            64 => Self::SixtyFour,
+            128 => Self::OneTwentyEight,
+            _ => return Err("not a valid bit width"),
+        })
+    }
+}
+
+impl fmt::Display for Type {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self.kind {
+            Kind::CpuFlags => return write!(f, "cpu-flags"),
+            Kind::Void => return write!(f, "void"),
+            Kind::Int => write!(f, "i")?,
+            Kind::Bool => write!(f, "b")?,
+        }
+        match self.bit_width {
+            BitWidth::Polymorphic => write!(f, "NN"),
+            otherwise => write!(f, "{}", otherwise as u8),
+        }
+    }
+}
+
+impl BitWidth {
+    /// Is this a polymorphic bit width?
+    pub fn is_polymorphic(&self) -> bool {
+        matches!(self, BitWidth::Polymorphic)
+    }
+
+    /// Get this width in bits, unless this is a polymorphic bit width.
+    pub fn fixed_width(&self) -> Option<u8> {
+        if self.is_polymorphic() {
+            None
+        } else {
+            Some(*self as u8)
+        }
+    }
+}
+
+macro_rules! type_ctors {
+    ( $( $( #[$attr:meta] )* $name:ident ( $kind:ident , $width:ident ) ; )* ) => {
+        $(
+            $( #[$attr] )*
+            pub const fn $name() -> Self {
+                Type {
+                    kind: Kind::$kind,
+                    bit_width: BitWidth::$width,
+                }
+            }
+        )*
+    }
+}
+
+impl Type {
+    type_ctors! {
+        /// Get the `i1` type.
+        i1(Int, One);
+        /// Get the `i8` type.
+        i8(Int, Eight);
+        /// Get the `i16` type.
+        i16(Int, Sixteen);
+        /// Get the `i32` type.
+        i32(Int, ThirtyTwo);
+        /// Get the `i64` type.
+        i64(Int, SixtyFour);
+        /// Get the `i128` type.
+        i128(Int, OneTwentyEight);
+        /// Get the `b1` type.
+        b1(Bool, One);
+        /// Get the `b8` type.
+        b8(Bool, Eight);
+        /// Get the `b16` type.
+        b16(Bool, Sixteen);
+        /// Get the `b32` type.
+        b32(Bool, ThirtyTwo);
+        /// Get the `b64` type.
+        b64(Bool, SixtyFour);
+        /// Get the `b128` type.
+        b128(Bool, OneTwentyEight);
+        /// Get the CPU flags type.
+        cpu_flags(CpuFlags, One);
+        /// Get the void type.
+        void(Void, One);
+    }
+}
+
+#[cfg(feature = "construct")]
+mod tok {
+    use wast::custom_keyword;
+    custom_keyword!(b1);
+    custom_keyword!(b8);
+    custom_keyword!(b16);
+    custom_keyword!(b32);
+    custom_keyword!(b64);
+    custom_keyword!(b128);
+    custom_keyword!(i1);
+    custom_keyword!(i8);
+    custom_keyword!(i16);
+    custom_keyword!(i32);
+    custom_keyword!(i64);
+    custom_keyword!(i128);
+}
+
+#[cfg(feature = "construct")]
+impl<'a> wast::parser::Parse<'a> for Type {
+    fn parse(p: wast::parser::Parser<'a>) -> wast::parser::Result<Self> {
+        if p.peek::<tok::b1>() {
+            p.parse::<tok::b1>()?;
+            return Ok(Type {
+                kind: Kind::Bool,
+                bit_width: BitWidth::One,
+            });
+        }
+        if p.peek::<tok::b8>() {
+            p.parse::<tok::b8>()?;
+            return Ok(Type {
+                kind: Kind::Bool,
+                bit_width: BitWidth::Eight,
+            });
+        }
+        if p.peek::<tok::b16>() {
+            p.parse::<tok::b16>()?;
+            return Ok(Type {
+                kind: Kind::Bool,
+                bit_width: BitWidth::Sixteen,
+            });
+        }
+        if p.peek::<tok::b32>() {
+            p.parse::<tok::b32>()?;
+            return Ok(Type {
+                kind: Kind::Bool,
+                bit_width: BitWidth::ThirtyTwo,
+            });
+        }
+        if p.peek::<tok::b64>() {
+            p.parse::<tok::b64>()?;
+            return Ok(Type {
+                kind: Kind::Bool,
+                bit_width: BitWidth::SixtyFour,
+            });
+        }
+        if p.peek::<tok::b128>() {
+            p.parse::<tok::b128>()?;
+            return Ok(Type {
+                kind: Kind::Bool,
+                bit_width: BitWidth::OneTwentyEight,
+            });
+        }
+        if p.peek::<tok::i1>() {
+            p.parse::<tok::i1>()?;
+            return Ok(Type {
+                kind: Kind::Int,
+                bit_width: BitWidth::One,
+            });
+        }
+        if p.peek::<tok::i8>() {
+            p.parse::<tok::i8>()?;
+            return Ok(Type {
+                kind: Kind::Int,
+                bit_width: BitWidth::Eight,
+            });
+        }
+        if p.peek::<tok::i16>() {
+            p.parse::<tok::i16>()?;
+            return Ok(Type {
+                kind: Kind::Int,
+                bit_width: BitWidth::Sixteen,
+            });
+        }
+        if p.peek::<tok::i32>() {
+            p.parse::<tok::i32>()?;
+            return Ok(Type {
+                kind: Kind::Int,
+                bit_width: BitWidth::ThirtyTwo,
+            });
+        }
+        if p.peek::<tok::i64>() {
+            p.parse::<tok::i64>()?;
+            return Ok(Type {
+                kind: Kind::Int,
+                bit_width: BitWidth::SixtyFour,
+            });
+        }
+        if p.peek::<tok::i128>() {
+            p.parse::<tok::i128>()?;
+            return Ok(Type {
+                kind: Kind::Int,
+                bit_width: BitWidth::OneTwentyEight,
+            });
+        }
+        Err(p.error("expected an ascribed type"))
+    }
+}

cranelift/peepmatic/crates/test/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "peepmatic-test"
+version = "0.1.0"
+authors = ["Nick Fitzgerald <fitzgen@gmail.com>"]
+edition = "2018"
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+env_logger = "0.7.1"
+log = "0.4.8"
+peepmatic = { path = "../.." }
+peepmatic-runtime = { path = "../runtime" }

cranelift/peepmatic/crates/test/src/lib.rs

@@ -0,0 +1,535 @@
+//! Testing utilities and a testing-only instruction set for `peepmatic`.
+
+#![deny(missing_debug_implementations)]
+
+use peepmatic_runtime::{
+    cc::ConditionCode,
+    instruction_set::InstructionSet,
+    operator::Operator,
+    part::{Constant, Part},
+    paths::Path,
+    r#type::{BitWidth, Kind, Type},
+};
+use std::cell::RefCell;
+use std::collections::BTreeMap;
+use std::convert::TryFrom;
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
+pub struct Instruction(pub usize);
+
+#[derive(Debug)]
+pub struct InstructionData {
+    pub operator: Operator,
+    pub r#type: Type,
+    pub immediates: Vec<Immediate>,
+    pub arguments: Vec<Instruction>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Immediate {
+    Constant(Constant),
+    ConditionCode(ConditionCode),
+}
+
+impl Immediate {
+    fn unwrap_constant(&self) -> Constant {
+        match *self {
+            Immediate::Constant(c) => c,
+            _ => panic!("not a constant"),
+        }
+    }
+}
+
+impl From<Constant> for Immediate {
+    fn from(c: Constant) -> Immediate {
+        Immediate::Constant(c)
+    }
+}
+
+impl From<ConditionCode> for Immediate {
+    fn from(cc: ConditionCode) -> Immediate {
+        Immediate::ConditionCode(cc)
+    }
+}
+
+impl From<Immediate> for Part<Instruction> {
+    fn from(imm: Immediate) -> Part<Instruction> {
+        match imm {
+            Immediate::Constant(c) => Part::Constant(c),
+            Immediate::ConditionCode(cc) => Part::ConditionCode(cc),
+        }
+    }
+}
+
+impl TryFrom<Part<Instruction>> for Immediate {
+    type Error = &'static str;
+
+    fn try_from(part: Part<Instruction>) -> Result<Immediate, Self::Error> {
+        match part {
+            Part::Constant(c) => Ok(Immediate::Constant(c)),
+            Part::ConditionCode(c) => Ok(Immediate::ConditionCode(c)),
+            Part::Instruction(_) => Err("instruction parts cannot be converted into immediates"),
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct Program {
+    instr_counter: usize,
+    instruction_data: BTreeMap<Instruction, InstructionData>,
+    replacements: RefCell<BTreeMap<Instruction, Instruction>>,
+}
+
+impl Program {
+    /// Are `a` and `b` structurally equivalent, even if they use different
+    /// `Instruction`s for various arguments?
+    pub fn structurally_eq(&mut self, a: Instruction, b: Instruction) -> bool {
+        macro_rules! ensure_eq {
+            ($a:expr, $b:expr) => {{
+                let a = &$a;
+                let b = &$b;
+                if a != b {
+                    log::debug!(
+                        "{} != {} ({:?} != {:?})",
+                        stringify!($a),
+                        stringify!($b),
+                        a,
+                        b
+                    );
+                    return false;
+                }
+            }};
+        }
+
+        let a = self.resolve(a);
+        let b = self.resolve(b);
+        if a == b {
+            return true;
+        }
+
+        let a = self.data(a);
+        let b = self.data(b);
+        log::debug!("structurally_eq({:?}, {:?})", a, b);
+
+        ensure_eq!(a.operator, b.operator);
+        ensure_eq!(a.r#type, b.r#type);
+        ensure_eq!(a.immediates, b.immediates);
+        ensure_eq!(a.arguments.len(), b.arguments.len());
+        a.arguments
+            .clone()
+            .into_iter()
+            .zip(b.arguments.clone().into_iter())
+            .all(|(a, b)| self.structurally_eq(a, b))
+    }
+
+    pub fn instructions(&self) -> impl Iterator<Item = (Instruction, &InstructionData)> {
+        self.instruction_data.iter().map(|(k, v)| (*k, v))
+    }
+
+    pub fn replace_instruction(&mut self, old: Instruction, new: Instruction) {
+        log::debug!("replacing {:?} with {:?}", old, new);
+
+        let old = self.resolve(old);
+        let new = self.resolve(new);
+        if old == new {
+            return;
+        }
+
+        let mut replacements = self.replacements.borrow_mut();
+        let existing_replacement = replacements.insert(old, new);
+        assert!(existing_replacement.is_none());
+
+        let old_data = self.instruction_data.remove(&old);
+        assert!(old_data.is_some());
+    }
+
+    pub fn resolve(&self, inst: Instruction) -> Instruction {
+        let mut replacements = self.replacements.borrow_mut();
+        let mut replacements_followed = 0;
+        let mut resolved = inst;
+        while let Some(i) = replacements.get(&resolved).cloned() {
+            log::trace!("resolving replaced instruction: {:?} -> {:?}", resolved, i);
+            replacements_followed += 1;
+            assert!(
+                replacements_followed <= replacements.len(),
+                "cyclic replacements"
+            );
+
+            resolved = i;
+            continue;
+        }
+
+        if inst != resolved {
+            let old_replacement = replacements.insert(inst, resolved);
+            assert!(old_replacement.is_some());
+        }
+
+        resolved
+    }
+
+    pub fn data(&self, inst: Instruction) -> &InstructionData {
+        let inst = self.resolve(inst);
+        &self.instruction_data[&inst]
+    }
+
+    pub fn new_instruction(
+        &mut self,
+        operator: Operator,
+        r#type: Type,
+        immediates: Vec<Immediate>,
+        arguments: Vec<Instruction>,
+    ) -> Instruction {
+        assert_eq!(
+            operator.immediates_arity() as usize,
+            immediates.len(),
+            "wrong number of immediates for {:?}: expected {}, found {}",
+            operator,
+            operator.immediates_arity(),
+            immediates.len(),
+        );
+        assert_eq!(
+            operator.params_arity() as usize,
+            arguments.len(),
+            "wrong number of arguments for {:?}: expected {}, found {}",
+            operator,
+            operator.params_arity(),
+            arguments.len(),
+        );
+
+        assert!(!r#type.bit_width.is_polymorphic());
+        assert!(immediates.iter().all(|imm| match imm {
+            Immediate::Constant(Constant::Bool(_, w))
+            | Immediate::Constant(Constant::Int(_, w)) => !w.is_polymorphic(),
+            Immediate::ConditionCode(_) => true,
+        }));
+
+        let inst = Instruction(self.instr_counter);
+        self.instr_counter += 1;
+
+        let data = InstructionData {
+            operator,
+            r#type,
+            immediates,
+            arguments,
+        };
+
+        log::trace!("new instruction: {:?} = {:?}", inst, data);
+        self.instruction_data.insert(inst, data);
+        inst
+    }
+
+    pub fn r#const(&mut self, c: Constant, root_bit_width: BitWidth) -> Instruction {
+        assert!(!root_bit_width.is_polymorphic());
+        match c {
+            Constant::Bool(_, bit_width) => self.new_instruction(
+                Operator::Bconst,
+                if bit_width.is_polymorphic() {
+                    Type {
+                        kind: Kind::Bool,
+                        bit_width: root_bit_width,
+                    }
+                } else {
+                    Type {
+                        kind: Kind::Bool,
+                        bit_width,
+                    }
+                },
+                vec![c.into()],
+                vec![],
+            ),
+            Constant::Int(_, bit_width) => self.new_instruction(
+                Operator::Iconst,
+                if bit_width.is_polymorphic() {
+                    Type {
+                        kind: Kind::Int,
+                        bit_width: root_bit_width,
+                    }
+                } else {
+                    Type {
+                        kind: Kind::Int,
+                        bit_width,
+                    }
+                },
+                vec![c.into()],
+                vec![],
+            ),
+        }
+    }
+
+    fn instruction_to_constant(&mut self, inst: Instruction) -> Option<Constant> {
+        match self.data(inst) {
+            InstructionData {
+                operator: Operator::Iconst,
+                immediates,
+                ..
+            } => Some(immediates[0].unwrap_constant()),
+            InstructionData {
+                operator: Operator::Bconst,
+                immediates,
+                ..
+            } => Some(immediates[0].unwrap_constant()),
+            _ => None,
+        }
+    }
+
+    fn part_to_immediate(&mut self, part: Part<Instruction>) -> Result<Immediate, &'static str> {
+        match part {
+            Part::Instruction(i) => self
+                .instruction_to_constant(i)
+                .map(|c| c.into())
+                .ok_or("non-constant instructions cannot be converted into immediates"),
+            Part::Constant(c) => Ok(c.into()),
+            Part::ConditionCode(cc) => Ok(Immediate::ConditionCode(cc)),
+        }
+    }
+
+    fn part_to_instruction(
+        &mut self,
+        root: Instruction,
+        part: Part<Instruction>,
+    ) -> Result<Instruction, &'static str> {
+        match part {
+            Part::Instruction(inst) => {
+                let inst = self.resolve(inst);
+                Ok(inst)
+            }
+            Part::Constant(c) => {
+                let root_width = self.data(root).r#type.bit_width;
+                Ok(self.r#const(c, root_width))
+            }
+            Part::ConditionCode(_) => Err("condition codes cannot be converted into instructions"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct TestIsa {
+    pub native_word_size_in_bits: u8,
+}
+
+// Unsafe because we must ensure that `instruction_result_bit_width` never
+// returns zero.
+unsafe impl<'a> InstructionSet<'a> for TestIsa {
+    type Context = Program;
+
+    type Instruction = Instruction;
+
+    fn replace_instruction(
+        &self,
+        program: &mut Program,
+        old: Instruction,
+        new: Part<Instruction>,
+    ) -> Instruction {
+        log::debug!("replace_instruction({:?}, {:?})", old, new);
+        let new = program.part_to_instruction(old, new).unwrap();
+        program.replace_instruction(old, new);
+        new
+    }
+
+    fn get_part_at_path(
+        &self,
+        program: &mut Program,
+        root: Instruction,
+        path: Path,
+    ) -> Option<Part<Instruction>> {
+        log::debug!("get_part_at_path({:?})", path);
+
+        assert!(!path.0.is_empty());
+        assert_eq!(path.0[0], 0);
+
+        let mut part = Part::Instruction(root);
+        for p in &path.0[1..] {
+            if let Part::Instruction(inst) = part {
+                let data = program.data(inst);
+                let p = *p as usize;
+
+                if p < data.immediates.len() {
+                    part = data.immediates[p].into();
+                    continue;
+                }
+
+                if let Some(inst) = data.arguments.get(p - data.immediates.len()).copied() {
+                    part = Part::Instruction(inst);
+                    continue;
+                }
+            }
+
+            return None;
+        }
+
+        Some(part)
+    }
+
+    fn operator(&self, program: &mut Program, instr: Instruction) -> Option<Operator> {
+        log::debug!("operator({:?})", instr);
+        let data = program.data(instr);
+        Some(data.operator)
+    }
+
+    fn make_inst_1(
+        &self,
+        program: &mut Program,
+        root: Instruction,
+        operator: Operator,
+        r#type: Type,
+        a: Part<Instruction>,
+    ) -> Instruction {
+        log::debug!(
+            "make_inst_1(\n\toperator = {:?},\n\ttype = {},\n\ta = {:?},\n)",
+            operator,
+            r#type,
+            a,
+        );
+
+        let (imms, args) = match operator.immediates_arity() {
+            0 => {
+                assert_eq!(operator.params_arity(), 1);
+                (vec![], vec![program.part_to_instruction(root, a).unwrap()])
+            }
+            1 => {
+                assert_eq!(operator.params_arity(), 0);
+                (vec![program.part_to_immediate(a).unwrap()], vec![])
+            }
+            _ => unreachable!(),
+        };
+        program.new_instruction(operator, r#type, imms, args)
+    }
+
+    fn make_inst_2(
+        &self,
+        program: &mut Program,
+        root: Instruction,
+        operator: Operator,
+        r#type: Type,
+        a: Part<Instruction>,
+        b: Part<Instruction>,
+    ) -> Instruction {
+        log::debug!(
+            "make_inst_2(\n\toperator = {:?},\n\ttype = {},\n\ta = {:?},\n\tb = {:?},\n)",
+            operator,
+            r#type,
+            a,
+            b,
+        );
+
+        let (imms, args) = match operator.immediates_arity() {
+            0 => {
+                assert_eq!(operator.params_arity(), 2);
+                (
+                    vec![],
+                    vec![
+                        program.part_to_instruction(root, a).unwrap(),
+                        program.part_to_instruction(root, b).unwrap(),
+                    ],
+                )
+            }
+            1 => {
+                assert_eq!(operator.params_arity(), 1);
+                (
+                    vec![program.part_to_immediate(a).unwrap()],
+                    vec![program.part_to_instruction(root, b).unwrap()],
+                )
+            }
+            2 => {
+                assert_eq!(operator.params_arity(), 0);
+                (
+                    vec![
+                        program.part_to_immediate(a).unwrap(),
+                        program.part_to_immediate(b).unwrap(),
+                    ],
+                    vec![],
+                )
+            }
+            _ => unreachable!(),
+        };
+        program.new_instruction(operator, r#type, imms, args)
+    }
+
+    fn make_inst_3(
+        &self,
+        program: &mut Program,
+        root: Instruction,
+        operator: Operator,
+        r#type: Type,
+        a: Part<Instruction>,
+        b: Part<Instruction>,
+        c: Part<Instruction>,
+    ) -> Instruction {
+        log::debug!(
+            "make_inst_3(\n\toperator = {:?},\n\ttype = {},\n\ta = {:?},\n\tb = {:?},\n\tc = {:?},\n)",
+            operator,
+            r#type,
+            a,
+            b,
+            c,
+        );
+        let (imms, args) = match operator.immediates_arity() {
+            0 => {
+                assert_eq!(operator.params_arity(), 3);
+                (
+                    vec![],
+                    vec![
+                        program.part_to_instruction(root, a).unwrap(),
+                        program.part_to_instruction(root, b).unwrap(),
+                        program.part_to_instruction(root, c).unwrap(),
+                    ],
+                )
+            }
+            1 => {
+                assert_eq!(operator.params_arity(), 2);
+                (
+                    vec![program.part_to_immediate(a).unwrap()],
+                    vec![
+                        program.part_to_instruction(root, b).unwrap(),
+                        program.part_to_instruction(root, c).unwrap(),
+                    ],
+                )
+            }
+            2 => {
+                assert_eq!(operator.params_arity(), 1);
+                (
+                    vec![
+                        program.part_to_immediate(a).unwrap(),
+                        program.part_to_immediate(b).unwrap(),
+                    ],
+                    vec![program.part_to_instruction(root, c).unwrap()],
+                )
+            }
+            3 => {
+                assert_eq!(operator.params_arity(), 0);
+                (
+                    vec![
+                        program.part_to_immediate(a).unwrap(),
+                        program.part_to_immediate(b).unwrap(),
+                        program.part_to_immediate(c).unwrap(),
+                    ],
+                    vec![],
+                )
+            }
+            _ => unreachable!(),
+        };
+        program.new_instruction(operator, r#type, imms, args)
+    }
+
+    fn instruction_to_constant(
+        &self,
+        program: &mut Program,
+        inst: Instruction,
+    ) -> Option<Constant> {
+        log::debug!("instruction_to_constant({:?})", inst);
+        program.instruction_to_constant(inst)
+    }
+
+    fn instruction_result_bit_width(&self, program: &mut Program, inst: Instruction) -> u8 {
+        log::debug!("instruction_result_bit_width({:?})", inst);
+        let ty = program.data(inst).r#type;
+        let width = ty.bit_width.fixed_width().unwrap();
+        assert!(width != 0);
+        width
+    }
+
+    fn native_word_size_in_bits(&self, _program: &mut Program) -> u8 {
+        log::debug!("native_word_size_in_bits");
+        self.native_word_size_in_bits
+    }
+}

cranelift/peepmatic/crates/test/tests/tests.rs

@@ -0,0 +1,393 @@
+use peepmatic_runtime::{
+    cc::ConditionCode,
+    operator::Operator,
+    part::Constant,
+    r#type::{BitWidth, Type},
+};
+use peepmatic_test::*;
+
+const TEST_ISA: TestIsa = TestIsa {
+    native_word_size_in_bits: 32,
+};
+
+macro_rules! optimizer {
+    ($opts:ident, $source:expr) => {{
+        let _ = env_logger::try_init();
+        $opts = peepmatic::compile_str($source, std::path::Path::new("peepmatic-test")).unwrap();
+        $opts.optimizer(TEST_ISA)
+    }};
+}
+
+#[test]
+fn opcode() {
+    let opts;
+    let mut optimizer = optimizer!(opts, "(=> (iadd $x 0) $x)");
+
+    let mut program = Program::default();
+    let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let zero = program.r#const(Constant::Int(0, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let add = program.new_instruction(Operator::Iadd, Type::i32(), vec![], vec![five, zero]);
+
+    let new = optimizer.apply_one(&mut program, add);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, five));
+
+    let add = program.new_instruction(Operator::Iadd, Type::i32(), vec![], vec![five, five]);
+    let replacement = optimizer.apply_one(&mut program, add);
+    assert!(replacement.is_none());
+}
+
+#[test]
+fn constant() {
+    let opts;
+    let mut optimizer = optimizer!(opts, "(=> (iadd $C $x) (iadd_imm $C $x))");
+
+    let mut program = Program::default();
+    let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let zero = program.r#const(Constant::Int(0, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let add = program.new_instruction(Operator::Iadd, Type::i32(), vec![], vec![five, zero]);
+
+    let expected = program.new_instruction(
+        Operator::IaddImm,
+        Type::i32(),
+        vec![Constant::Int(5, BitWidth::ThirtyTwo).into()],
+        vec![zero],
+    );
+
+    let new = optimizer.apply_one(&mut program, add);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, expected));
+
+    let mul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, zero]);
+    let add = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![mul, five]);
+    let replacement = optimizer.apply_one(&mut program, add);
+    assert!(replacement.is_none());
+}
+
+#[test]
+fn boolean() {
+    let opts;
+    let mut optimizer = optimizer!(opts, "(=> (bint true) 1)");
+
+    let mut program = Program::default();
+    let t = program.r#const(Constant::Bool(true, BitWidth::One), BitWidth::One);
+    let bint = program.new_instruction(Operator::Bint, Type::i1(), vec![], vec![t]);
+    let one = program.r#const(Constant::Int(1, BitWidth::One), BitWidth::ThirtyTwo);
+
+    let new = optimizer.apply_one(&mut program, bint);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, one));
+
+    let f = program.r#const(Constant::Bool(false, BitWidth::One), BitWidth::One);
+    let bint = program.new_instruction(Operator::Bint, Type::i1(), vec![], vec![f]);
+    let replacement = optimizer.apply_one(&mut program, bint);
+    assert!(replacement.is_none());
+}
+
+#[test]
+fn condition_codes() {
+    let opts;
+    let mut optimizer = optimizer!(opts, "(=> (icmp eq $x $x) true)");
+
+    let mut program = Program::default();
+    let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::One);
+    let icmp_eq = program.new_instruction(
+        Operator::Icmp,
+        Type::b1(),
+        vec![ConditionCode::Eq.into()],
+        vec![five, five],
+    );
+    let t = program.r#const(Constant::Bool(true, BitWidth::One), BitWidth::One);
+
+    let new = optimizer.apply_one(&mut program, icmp_eq);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, t));
+
+    let icmp_ne = program.new_instruction(
+        Operator::Icmp,
+        Type::b1(),
+        vec![ConditionCode::Ne.into()],
+        vec![five, five],
+    );
+    let replacement = optimizer.apply_one(&mut program, icmp_ne);
+    assert!(replacement.is_none());
+}
+
+#[test]
+fn is_power_of_two() {
+    let opts;
+    let mut optimizer = optimizer!(
+        opts,
+        "
+(=> (when (imul $x $C)
+          (is-power-of-two $C))
+    (ishl $x $(log2 $C)))
+"
+    );
+
+    let mut program = Program::default();
+    let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let two = program.r#const(Constant::Int(2, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, two]);
+
+    let one = program.r#const(Constant::Int(1, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let ishl = program.new_instruction(Operator::Ishl, Type::i32(), vec![], vec![five, one]);
+
+    let new = optimizer.apply_one(&mut program, imul);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, ishl));
+
+    let three = program.r#const(Constant::Int(3, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, three]);
+
+    let replacement = optimizer.apply_one(&mut program, imul);
+    assert!(replacement.is_none());
+}
+
+#[test]
+fn bit_width() {
+    let opts;
+    let mut optimizer = optimizer!(
+        opts,
+        "
+(=> (when (imul $C $x)
+          (bit-width $C 32))
+    (imul_imm $C $x))
+"
+    );
+
+    let mut program = Program::default();
+    let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let two = program.r#const(Constant::Int(2, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, two]);
+
+    let imul_imm = program.new_instruction(
+        Operator::ImulImm,
+        Type::i32(),
+        vec![Constant::Int(5, BitWidth::ThirtyTwo).into()],
+        vec![two],
+    );
+
+    let new = optimizer.apply_one(&mut program, imul);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, imul_imm));
+
+    let five = program.r#const(Constant::Int(5, BitWidth::SixtyFour), BitWidth::SixtyFour);
+    let two = program.r#const(Constant::Int(2, BitWidth::SixtyFour), BitWidth::SixtyFour);
+    let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, two]);
+
+    let replacement = optimizer.apply_one(&mut program, imul);
+    assert!(replacement.is_none());
+}
+
+#[test]
+fn fits_in_native_word() {
+    let opts;
+    let mut optimizer = optimizer!(
+        opts,
+        "
+(=> (when (imul $C $x)
+          (fits-in-native-word $C))
+    (imul_imm $C $x))
+"
+    );
+
+    let mut program = Program::default();
+    let five = program.r#const(Constant::Int(5, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let two = program.r#const(Constant::Int(2, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let imul = program.new_instruction(Operator::Imul, Type::i32(), vec![], vec![five, two]);
+
+    let imul_imm = program.new_instruction(
+        Operator::ImulImm,
+        Type::i32(),
+        vec![Constant::Int(5, BitWidth::ThirtyTwo).into()],
+        vec![two],
+    );
+
+    let new = optimizer.apply_one(&mut program, imul);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, imul_imm));
+
+    let five = program.r#const(Constant::Int(5, BitWidth::SixtyFour), BitWidth::SixtyFour);
+    let two = program.r#const(Constant::Int(2, BitWidth::SixtyFour), BitWidth::SixtyFour);
+    let imul = program.new_instruction(Operator::Imul, Type::i64(), vec![], vec![five, two]);
+
+    let replacement = optimizer.apply_one(&mut program, imul);
+    assert!(replacement.is_none());
+}
+
+#[test]
+fn unquote_neg() {
+    let opts;
+    let mut optimizer = optimizer!(
+        opts,
+        "
+(=> (isub $x $C)
+    (iadd_imm $(neg $C) $x))
+"
+    );
+
+    let mut program = Program::default();
+    let five = program.r#const(Constant::Int(5, BitWidth::SixtyFour), BitWidth::SixtyFour);
+    let two = program.r#const(Constant::Int(2, BitWidth::SixtyFour), BitWidth::SixtyFour);
+    let isub = program.new_instruction(Operator::Isub, Type::i64(), vec![], vec![five, two]);
+
+    let iadd_imm = program.new_instruction(
+        Operator::IaddImm,
+        Type::i64(),
+        vec![Constant::Int(-2 as _, BitWidth::SixtyFour).into()],
+        vec![five],
+    );
+
+    let new = optimizer.apply_one(&mut program, isub);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, iadd_imm));
+}
+
+#[test]
+fn subsumption() {
+    let opts;
+    let mut optimizer = optimizer!(
+        opts,
+        "
+;; NB: the following optimizations are ordered from least to most general, so
+;; the first applicable optimization should be the one that is applied.
+
+(=> (iadd (iadd (iadd $w $x) $y) $z)
+    (iadd (iadd $w $x) (iadd $y $z)))
+
+(=> (iadd $C1 $C2)
+    $(iadd $C1 $C2))
+
+(=> (iadd $C $x)
+    (iadd_imm $C $x))
+
+(=> (iadd $x $x)
+    (ishl_imm 1 $x))
+"
+    );
+
+    let mut program = Program::default();
+
+    let w = program.r#const(Constant::Int(11, BitWidth::SixtyFour), BitWidth::SixtyFour);
+    let x = program.r#const(Constant::Int(22, BitWidth::SixtyFour), BitWidth::SixtyFour);
+    let y = program.r#const(Constant::Int(33, BitWidth::SixtyFour), BitWidth::SixtyFour);
+    let z = program.r#const(Constant::Int(44, BitWidth::SixtyFour), BitWidth::SixtyFour);
+
+    log::debug!("(iadd (iadd (iadd w x) y) z) => (iadd (iadd w x) (iadd y z))");
+
+    let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![w, x]);
+    let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![iadd, y]);
+    let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![iadd, z]);
+    let expected_lhs = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![w, x]);
+    let expected_rhs = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![y, z]);
+    let expected = program.new_instruction(
+        Operator::Iadd,
+        Type::i64(),
+        vec![],
+        vec![expected_lhs, expected_rhs],
+    );
+
+    let new = optimizer.apply_one(&mut program, iadd);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, expected));
+
+    log::debug!("(iadd w x) => y");
+
+    let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![w, x]);
+    let new = optimizer.apply_one(&mut program, iadd);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, y));
+
+    log::debug!("(iadd x (iadd y z)) => (iadd_imm x (iadd y z))");
+
+    let iadd_y_z = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![y, z]);
+    let iadd = program.new_instruction(Operator::Iadd, Type::i64(), vec![], vec![x, iadd_y_z]);
+    let iadd_imm = program.new_instruction(
+        Operator::IaddImm,
+        Type::i64(),
+        vec![Constant::Int(22, BitWidth::SixtyFour).into()],
+        vec![iadd_y_z],
+    );
+    let new = optimizer.apply_one(&mut program, iadd);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, iadd_imm));
+
+    log::debug!("(iadd (imul_imm x 1) (imul_imm x 1)) => (ishl_imm 1 (imul_imm x 1))");
+
+    let imul_imm = program.new_instruction(
+        Operator::ImulImm,
+        Type::i64(),
+        vec![Constant::Int(1, BitWidth::SixtyFour).into()],
+        vec![x],
+    );
+    let iadd = program.new_instruction(
+        Operator::Iadd,
+        Type::i64(),
+        vec![],
+        vec![imul_imm, imul_imm],
+    );
+    let ishl_imm = program.new_instruction(
+        Operator::IshlImm,
+        Type::i64(),
+        vec![Constant::Int(1, BitWidth::SixtyFour).into()],
+        vec![imul_imm],
+    );
+    let new = optimizer.apply_one(&mut program, iadd);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, ishl_imm));
+
+    log::debug!("(iadd (imul w x) (imul y z)) does not match any optimization.");
+
+    let imul_w_x = program.new_instruction(Operator::Imul, Type::i64(), vec![], vec![w, x]);
+    let imul_y_z = program.new_instruction(Operator::Imul, Type::i64(), vec![], vec![y, z]);
+    let iadd = program.new_instruction(
+        Operator::Iadd,
+        Type::i64(),
+        vec![],
+        vec![imul_w_x, imul_y_z],
+    );
+
+    let replacement = optimizer.apply_one(&mut program, iadd);
+    assert!(replacement.is_none());
+}
+
+#[test]
+fn polymorphic_bit_widths() {
+    let opts;
+    let mut optimizer = optimizer!(opts, "(=> (iadd $C $x) (iadd_imm $C $x))");
+
+    let mut program = Program::default();
+
+    // Applies to 32 bit adds.
+
+    let x = program.r#const(Constant::Int(42, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let y = program.r#const(Constant::Int(420, BitWidth::ThirtyTwo), BitWidth::ThirtyTwo);
+    let iadd = program.new_instruction(Operator::Iadd, Type::i32(), vec![], vec![x, y]);
+    let iadd_imm = program.new_instruction(
+        Operator::IaddImm,
+        Type::i32(),
+        vec![Constant::Int(42, BitWidth::ThirtyTwo).into()],
+        vec![y],
+    );
+
+    let new = optimizer.apply_one(&mut program, iadd);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, iadd_imm));
+
+    // Applies to 16 bit adds.
+
+    let x = program.r#const(Constant::Int(42, BitWidth::Sixteen), BitWidth::Sixteen);
+    let y = program.r#const(Constant::Int(420, BitWidth::Sixteen), BitWidth::Sixteen);
+    let iadd = program.new_instruction(Operator::Iadd, Type::i16(), vec![], vec![x, y]);
+    let iadd_imm = program.new_instruction(
+        Operator::IaddImm,
+        Type::i16(),
+        vec![Constant::Int(42, BitWidth::Sixteen).into()],
+        vec![y],
+    );
+
+    let new = optimizer.apply_one(&mut program, iadd);
+    let new = new.expect("optimization should have applied");
+    assert!(program.structurally_eq(new, iadd_imm));
+}

cranelift/peepmatic/examples/mul-by-pow2.peepmatic

@@ -0,0 +1,3 @@
+(=> (when (imul $x $C)
+          (is-power-of-two $C))
+    (ishl $x $(log2 $C)))

cranelift/peepmatic/examples/redundant-bor.peepmatic

@@ -0,0 +1,13 @@
+;; Remove redundant bitwise OR instructions that are no-ops.
+
+(=> (bor $x (bor $x $y))
+    (bor $x $y))
+
+(=> (bor $y (bor $x $y))
+    (bor $x $y))
+
+(=> (bor (bor $x $y) $x)
+    (bor $x $y))
+
+(=> (bor (bor $x $y) $y)
+    (bor $x $y))

cranelift/peepmatic/examples/simple.peepmatic

@@ -0,0 +1,11 @@
+(=> (bor $x (bor $x $y))
+    (bor $x $y))
+
+(=> (bor $y (bor $x $y))
+    (bor $x $y))
+
+(=> (bor (bor $x $y) $x)
+    (bor $x $y))
+
+(=> (bor (bor $x $y) $y)
+    (bor $x $y))

cranelift/peepmatic/src/ast.rs

@@ -0,0 +1,508 @@
+//! Abstract syntax tree type definitions.
+//!
+//! This file makes fairly heavy use of macros, which are defined in the
+//! `peepmatic_macro` crate that lives at `crates/macro`. Notably, the following
+//! traits are all derived via `derive(Ast)`:
+//!
+//! * `Span` -- access the `wast::Span` where an AST node was parsed from. For
+//!   `struct`s, there must be a `span: wast::Span` field, because the macro
+//!   always generates an implementation that returns `self.span` for
+//!   `struct`s. For `enum`s, every variant must have a single, unnamed field
+//!   which implements the `Span` trait. The macro will generate code to return
+//!   the span of whatever variant it is.
+//!
+//! * `ChildNodes` -- get each of the child AST nodes that a given node
+//!   references. Some fields in an AST type aren't actually considered an AST
+//!   node (like spans) and these are ignored via the `#[peepmatic(skip_child)]`
+//!   attribute. Some fields contain multiple AST nodes (like vectors of
+//!   operands) and these are flattened with `#[peepmatic(flatten)]`.
+//!
+//! * `From<&'a Self> for DynAstRef<'a>` -- convert a particular AST type into
+//!   `DynAstRef`, which is an `enum` of all the different kinds of AST nodes.
+
+use peepmatic_macro::Ast;
+use peepmatic_runtime::{
+    operator::{Operator, UnquoteOperator},
+    r#type::{BitWidth, Type},
+};
+use std::cell::Cell;
+use std::hash::{Hash, Hasher};
+use std::marker::PhantomData;
+use wast::Id;
+
+/// A reference to any AST node.
+#[derive(Debug, Clone, Copy)]
+pub enum DynAstRef<'a> {
+    /// A reference to an `Optimizations`.
+    Optimizations(&'a Optimizations<'a>),
+
+    /// A reference to an `Optimization`.
+    Optimization(&'a Optimization<'a>),
+
+    /// A reference to an `Lhs`.
+    Lhs(&'a Lhs<'a>),
+
+    /// A reference to an `Rhs`.
+    Rhs(&'a Rhs<'a>),
+
+    /// A reference to a `Pattern`.
+    Pattern(&'a Pattern<'a>),
+
+    /// A reference to a `Precondition`.
+    Precondition(&'a Precondition<'a>),
+
+    /// A reference to a `ConstraintOperand`.
+    ConstraintOperand(&'a ConstraintOperand<'a>),
+
+    /// A reference to a `ValueLiteral`.
+    ValueLiteral(&'a ValueLiteral<'a>),
+
+    /// A reference to a `Constant`.
+    Constant(&'a Constant<'a>),
+
+    /// A reference to a `PatternOperation`.
+    PatternOperation(&'a Operation<'a, Pattern<'a>>),
+
+    /// A reference to a `Variable`.
+    Variable(&'a Variable<'a>),
+
+    /// A reference to an `Integer`.
+    Integer(&'a Integer<'a>),
+
+    /// A reference to a `Boolean`.
+    Boolean(&'a Boolean<'a>),
+
+    /// A reference to a `ConditionCode`.
+    ConditionCode(&'a ConditionCode<'a>),
+
+    /// A reference to an `Unquote`.
+    Unquote(&'a Unquote<'a>),
+
+    /// A reference to an `RhsOperation`.
+    RhsOperation(&'a Operation<'a, Rhs<'a>>),
+}
+
+impl<'a, 'b> ChildNodes<'a, 'b> for DynAstRef<'a> {
+    fn child_nodes(&'b self, sink: &mut impl Extend<DynAstRef<'a>>) {
+        match self {
+            Self::Optimizations(x) => x.child_nodes(sink),
+            Self::Optimization(x) => x.child_nodes(sink),
+            Self::Lhs(x) => x.child_nodes(sink),
+            Self::Rhs(x) => x.child_nodes(sink),
+            Self::Pattern(x) => x.child_nodes(sink),
+            Self::Precondition(x) => x.child_nodes(sink),
+            Self::ConstraintOperand(x) => x.child_nodes(sink),
+            Self::ValueLiteral(x) => x.child_nodes(sink),
+            Self::Constant(x) => x.child_nodes(sink),
+            Self::PatternOperation(x) => x.child_nodes(sink),
+            Self::Variable(x) => x.child_nodes(sink),
+            Self::Integer(x) => x.child_nodes(sink),
+            Self::Boolean(x) => x.child_nodes(sink),
+            Self::ConditionCode(x) => x.child_nodes(sink),
+            Self::Unquote(x) => x.child_nodes(sink),
+            Self::RhsOperation(x) => x.child_nodes(sink),
+        }
+    }
+}
+
+/// A trait implemented by all AST nodes.
+///
+/// All AST nodes can:
+///
+/// * Enumerate their children via `ChildNodes`.
+///
+/// * Give you the `wast::Span` where they were defined.
+///
+/// * Be converted into a `DynAstRef`.
+///
+/// This trait is blanked implemented for everything that does those three
+/// things, and in practice those three thrings are all implemented by the
+/// `derive(Ast)` macro.
+pub trait Ast<'a>: 'a + ChildNodes<'a, 'a> + Span
+where
+    DynAstRef<'a>: From<&'a Self>,
+{
+}
+
+impl<'a, T> Ast<'a> for T
+where
+    T: 'a + ?Sized + ChildNodes<'a, 'a> + Span,
+    DynAstRef<'a>: From<&'a Self>,
+{
+}
+
+/// Enumerate the child AST nodes of a given node.
+pub trait ChildNodes<'a, 'b> {
+    /// Get each of this AST node's children, in order.
+    fn child_nodes(&'b self, sink: &mut impl Extend<DynAstRef<'a>>);
+}
+
+/// A trait for getting the span where an AST node was defined.
+pub trait Span {
+    /// Get the span where this AST node was defined.
+    fn span(&self) -> wast::Span;
+}
+
+/// A set of optimizations.
+///
+/// This is the root AST node.
+#[derive(Debug, Ast)]
+pub struct Optimizations<'a> {
+    /// Where these `Optimizations` were defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// The optimizations.
+    #[peepmatic(flatten)]
+    pub optimizations: Vec<Optimization<'a>>,
+}
+
+/// A complete optimization: a left-hand side to match against and a right-hand
+/// side replacement.
+#[derive(Debug, Ast)]
+pub struct Optimization<'a> {
+    /// Where this `Optimization` was defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// The left-hand side that matches when this optimization applies.
+    pub lhs: Lhs<'a>,
+
+    /// The new sequence of instructions to replace an old sequence that matches
+    /// the left-hand side with.
+    pub rhs: Rhs<'a>,
+}
+
+/// A left-hand side describes what is required for a particular optimization to
+/// apply.
+///
+/// A left-hand side has two parts: a structural pattern for describing
+/// candidate instruction sequences, and zero or more preconditions that add
+/// additional constraints upon instruction sequences matched by the pattern.
+#[derive(Debug, Ast)]
+pub struct Lhs<'a> {
+    /// Where this `Lhs` was defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// A pattern that describes sequences of instructions to match.
+    pub pattern: Pattern<'a>,
+
+    /// Additional constraints that a match must satisfy in addition to
+    /// structually matching the pattern, e.g. some constant must be a power of
+    /// two.
+    #[peepmatic(flatten)]
+    pub preconditions: Vec<Precondition<'a>>,
+}
+
+/// A structural pattern, potentially with wildcard variables for matching whole
+/// subtrees.
+#[derive(Debug, Ast)]
+pub enum Pattern<'a> {
+    /// A specific value. These are written as `1234` or `0x1234` or `true` or
+    /// `false`.
+    ValueLiteral(ValueLiteral<'a>),
+
+    /// A constant that matches any constant value. This subsumes value
+    /// patterns. These are upper-case identifiers like `$C`.
+    Constant(Constant<'a>),
+
+    /// An operation pattern with zero or more operand patterns. These are
+    /// s-expressions like `(iadd $x $y)`.
+    Operation(Operation<'a, Pattern<'a>>),
+
+    /// A variable that matches any kind of subexpression. This subsumes all
+    /// other patterns. These are lower-case identifiers like `$x`.
+    Variable(Variable<'a>),
+}
+
+/// An integer or boolean value literal.
+#[derive(Debug, Ast)]
+pub enum ValueLiteral<'a> {
+    /// An integer value.
+    Integer(Integer<'a>),
+
+    /// A boolean value: `true` or `false`.
+    Boolean(Boolean<'a>),
+
+    /// A condition code: `eq`, `ne`, etc...
+    ConditionCode(ConditionCode<'a>),
+}
+
+/// An integer literal.
+#[derive(Debug, PartialEq, Eq, Ast)]
+pub struct Integer<'a> {
+    /// Where this `Integer` was defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// The integer value.
+    ///
+    /// Note that although Cranelift allows 128 bits wide values, the widest
+    /// supported constants as immediates are 64 bits.
+    #[peepmatic(skip_child)]
+    pub value: i64,
+
+    /// The bit width of this integer.
+    ///
+    /// This is either a fixed bit width, or polymorphic over the width of the
+    /// optimization.
+    ///
+    /// This field is initialized from `None` to `Some` by the type checking
+    /// pass in `src/verify.rs`.
+    #[peepmatic(skip_child)]
+    pub bit_width: Cell<Option<BitWidth>>,
+
+    #[allow(missing_docs)]
+    #[peepmatic(skip_child)]
+    pub marker: PhantomData<&'a ()>,
+}
+
+impl Hash for Integer<'_> {
+    fn hash<H>(&self, state: &mut H)
+    where
+        H: Hasher,
+    {
+        let Integer {
+            span,
+            value,
+            bit_width,
+            marker: _,
+        } = self;
+        span.hash(state);
+        value.hash(state);
+        let bit_width = bit_width.get();
+        bit_width.hash(state);
+    }
+}
+
+/// A boolean literal.
+#[derive(Debug, PartialEq, Eq, Ast)]
+pub struct Boolean<'a> {
+    /// Where this `Boolean` was defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// The boolean value.
+    #[peepmatic(skip_child)]
+    pub value: bool,
+
+    /// The bit width of this boolean.
+    ///
+    /// This is either a fixed bit width, or polymorphic over the width of the
+    /// optimization.
+    ///
+    /// This field is initialized from `None` to `Some` by the type checking
+    /// pass in `src/verify.rs`.
+    #[peepmatic(skip_child)]
+    pub bit_width: Cell<Option<BitWidth>>,
+
+    #[allow(missing_docs)]
+    #[peepmatic(skip_child)]
+    pub marker: PhantomData<&'a ()>,
+}
+
+impl Hash for Boolean<'_> {
+    fn hash<H>(&self, state: &mut H)
+    where
+        H: Hasher,
+    {
+        let Boolean {
+            span,
+            value,
+            bit_width,
+            marker: _,
+        } = self;
+        span.hash(state);
+        value.hash(state);
+        let bit_width = bit_width.get();
+        bit_width.hash(state);
+    }
+}
+
+/// A condition code.
+#[derive(Debug, Ast)]
+pub struct ConditionCode<'a> {
+    /// Where this `ConditionCode` was defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// The actual condition code.
+    #[peepmatic(skip_child)]
+    pub cc: peepmatic_runtime::cc::ConditionCode,
+
+    #[allow(missing_docs)]
+    #[peepmatic(skip_child)]
+    pub marker: PhantomData<&'a ()>,
+}
+
+/// A symbolic constant.
+///
+/// These are identifiers containing uppercase letters: `$C`, `$MY-CONST`,
+/// `$CONSTANT1`.
+#[derive(Debug, Ast)]
+pub struct Constant<'a> {
+    /// Where this `Constant` was defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// This constant's identifier.
+    #[peepmatic(skip_child)]
+    pub id: Id<'a>,
+}
+
+/// A variable that matches any subtree.
+///
+/// Duplicate uses of the same variable constrain each occurrence's match to
+/// being the same as each other occurrence as well, e.g. `(iadd $x $x)` matches
+/// `(iadd 5 5)` but not `(iadd 1 2)`.
+#[derive(Debug, Ast)]
+pub struct Variable<'a> {
+    /// Where this `Variable` was defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// This variable's identifier.
+    #[peepmatic(skip_child)]
+    pub id: Id<'a>,
+}
+
+/// An operation with an operator, and operands of type `T`.
+#[derive(Debug, Ast)]
+#[peepmatic(no_into_dyn_node)]
+pub struct Operation<'a, T>
+where
+    T: 'a + Ast<'a>,
+    DynAstRef<'a>: From<&'a T>,
+{
+    /// The span where this operation was written.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// The operator for this operation, e.g. `imul` or `iadd`.
+    #[peepmatic(skip_child)]
+    pub operator: Operator,
+
+    /// An optional ascribed or inferred type for the operator.
+    #[peepmatic(skip_child)]
+    pub r#type: Cell<Option<Type>>,
+
+    /// This operation's operands.
+    ///
+    /// When `Operation` is used in a pattern, these are the sub-patterns for
+    /// the operands. When `Operation is used in a right-hand side replacement,
+    /// these are the sub-replacements for the operands.
+    #[peepmatic(flatten)]
+    pub operands: Vec<T>,
+
+    #[allow(missing_docs)]
+    #[peepmatic(skip_child)]
+    pub marker: PhantomData<&'a ()>,
+}
+
+impl<'a> From<&'a Operation<'a, Pattern<'a>>> for DynAstRef<'a> {
+    #[inline]
+    fn from(o: &'a Operation<'a, Pattern<'a>>) -> DynAstRef<'a> {
+        DynAstRef::PatternOperation(o)
+    }
+}
+
+impl<'a> From<&'a Operation<'a, Rhs<'a>>> for DynAstRef<'a> {
+    #[inline]
+    fn from(o: &'a Operation<'a, Rhs<'a>>) -> DynAstRef<'a> {
+        DynAstRef::RhsOperation(o)
+    }
+}
+
+/// A precondition adds additional constraints to a pattern, such as "$C must be
+/// a power of two".
+#[derive(Debug, Ast)]
+pub struct Precondition<'a> {
+    /// Where this `Precondition` was defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// The constraint operator.
+    #[peepmatic(skip_child)]
+    pub constraint: Constraint,
+
+    /// The operands of the constraint.
+    #[peepmatic(flatten)]
+    pub operands: Vec<ConstraintOperand<'a>>,
+}
+
+/// Contraint operators.
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
+pub enum Constraint {
+    /// Is the operand a power of two?
+    IsPowerOfTwo,
+
+    /// Check the bit width of a value.
+    BitWidth,
+
+    /// Does the argument fit within our target architecture's native word size?
+    FitsInNativeWord,
+}
+
+/// An operand of a precondition's constraint.
+#[derive(Debug, Ast)]
+pub enum ConstraintOperand<'a> {
+    /// A value literal operand.
+    ValueLiteral(ValueLiteral<'a>),
+
+    /// A constant operand.
+    Constant(Constant<'a>),
+
+    /// A variable operand.
+    Variable(Variable<'a>),
+}
+
+/// The right-hand side of an optimization that contains the instructions to
+/// replace any matched left-hand side with.
+#[derive(Debug, Ast)]
+pub enum Rhs<'a> {
+    /// A value literal right-hand side.
+    ValueLiteral(ValueLiteral<'a>),
+
+    /// A constant right-hand side (the constant must have been matched and
+    /// bound in the left-hand side's pattern).
+    Constant(Constant<'a>),
+
+    /// A variable right-hand side (the variable must have been matched and
+    /// bound in the left-hand side's pattern).
+    Variable(Variable<'a>),
+
+    /// An unquote expression that is evaluated while replacing the left-hand
+    /// side with the right-hand side. The result of the evaluation is used in
+    /// the replacement.
+    Unquote(Unquote<'a>),
+
+    /// A compound right-hand side consisting of an operation and subsequent
+    /// right-hand side operands.
+    Operation(Operation<'a, Rhs<'a>>),
+}
+
+/// An unquote operation.
+///
+/// Rather than replaciong a left-hand side, these are evaluated and then the
+/// result of the evaluation replaces the left-hand side. This allows for
+/// compile-time computation while replacing a matched left-hand side with a
+/// right-hand side.
+///
+/// For example, given the unqouted right-hand side `$(log2 $C)`, we replace any
+/// instructions that match its left-hand side with the compile-time result of
+/// `log2($C)` (the left-hand side must match and bind the constant `$C`).
+#[derive(Debug, Ast)]
+pub struct Unquote<'a> {
+    /// Where this `Unquote` was defined.
+    #[peepmatic(skip_child)]
+    pub span: wast::Span,
+
+    /// The operator for this unquote operation.
+    #[peepmatic(skip_child)]
+    pub operator: UnquoteOperator,
+
+    /// The operands for this unquote operation.
+    #[peepmatic(flatten)]
+    pub operands: Vec<Rhs<'a>>,
+}

cranelift/peepmatic/src/automatize.rs

@@ -0,0 +1,31 @@
+//! Compile a set of linear optimizations into an automaton.
+
+use peepmatic_automata::{Automaton, Builder};
+use peepmatic_runtime::linear;
+
+/// Construct an automaton from a set of linear optimizations.
+pub fn automatize(
+    opts: &linear::Optimizations,
+) -> Automaton<linear::MatchResult, linear::MatchOp, Box<[linear::Action]>> {
+    debug_assert!(crate::linear_passes::is_sorted_lexicographically(opts));
+
+    let mut builder = Builder::<linear::MatchResult, linear::MatchOp, Box<[linear::Action]>>::new();
+
+    for opt in &opts.optimizations {
+        let mut insertion = builder.insert();
+        for inc in &opt.increments {
+            // Ensure that this state's associated data is this increment's
+            // match operation.
+            if let Some(op) = insertion.get_state_data() {
+                assert_eq!(*op, inc.operation);
+            } else {
+                insertion.set_state_data(inc.operation);
+            }
+
+            insertion.next(inc.expected, inc.actions.clone().into_boxed_slice());
+        }
+        insertion.finish();
+    }
+
+    builder.finish()
+}

cranelift/peepmatic/src/dot_fmt.rs

@@ -0,0 +1,145 @@
+//! Formatting a peephole optimizer's automata for GraphViz Dot.
+//!
+//! See also `crates/automata/src/dot.rs`.
+
+use peepmatic_automata::dot::DotFmt;
+use peepmatic_runtime::{
+    cc::ConditionCode,
+    integer_interner::{IntegerId, IntegerInterner},
+    linear,
+    operator::Operator,
+    paths::{PathId, PathInterner},
+};
+use std::convert::{TryFrom, TryInto};
+use std::io::{self, Write};
+use std::num::NonZeroU16;
+
+#[derive(Debug)]
+pub(crate) struct PeepholeDotFmt<'a>(pub(crate) &'a PathInterner, pub(crate) &'a IntegerInterner);
+
+impl DotFmt<linear::MatchResult, linear::MatchOp, Box<[linear::Action]>> for PeepholeDotFmt<'_> {
+    fn fmt_transition(
+        &self,
+        w: &mut impl Write,
+        from: Option<&linear::MatchOp>,
+        input: &linear::MatchResult,
+        _to: Option<&linear::MatchOp>,
+    ) -> io::Result<()> {
+        let from = from.expect("we should have match op for every state");
+        if let Some(x) = input.ok().map(|x| x.get()) {
+            match from {
+                linear::MatchOp::Opcode { .. } => {
+                    let opcode =
+                        Operator::try_from(x).expect("we shouldn't generate non-opcode edges");
+                    write!(w, "{}", opcode)
+                }
+                linear::MatchOp::ConditionCode { .. } => {
+                    let cc =
+                        ConditionCode::try_from(x).expect("we shouldn't generate non-CC edges");
+                    write!(w, "{}", cc)
+                }
+                linear::MatchOp::IntegerValue { .. } => {
+                    let x = self
+                        .1
+                        .lookup(IntegerId(NonZeroU16::new(x.try_into().unwrap()).unwrap()));
+                    write!(w, "{}", x)
+                }
+                _ => write!(w, "Ok({})", x),
+            }
+        } else {
+            write!(w, "(else)")
+        }
+    }
+
+    fn fmt_state(&self, w: &mut impl Write, op: &linear::MatchOp) -> io::Result<()> {
+        use linear::MatchOp::*;
+
+        write!(w, r#"<font face="monospace">"#)?;
+
+        let p = p(self.0);
+        match op {
+            Opcode { path } => write!(w, "opcode @ {}", p(path))?,
+            IsConst { path } => write!(w, "is-const? @ {}", p(path))?,
+            IsPowerOfTwo { path } => write!(w, "is-power-of-two? @ {}", p(path))?,
+            BitWidth { path } => write!(w, "bit-width @ {}", p(path))?,
+            FitsInNativeWord { path } => write!(w, "fits-in-native-word @ {}", p(path))?,
+            Eq { path_a, path_b } => write!(w, "{} == {}", p(path_a), p(path_b))?,
+            IntegerValue { path } => write!(w, "integer-value @ {}", p(path))?,
+            BooleanValue { path } => write!(w, "boolean-value @ {}", p(path))?,
+            ConditionCode { path } => write!(w, "condition-code @ {}", p(path))?,
+            Nop => write!(w, "nop")?,
+        }
+
+        writeln!(w, "</font>")
+    }
+
+    fn fmt_output(&self, w: &mut impl Write, actions: &Box<[linear::Action]>) -> io::Result<()> {
+        use linear::Action::*;
+
+        if actions.is_empty() {
+            return writeln!(w, "(no output)");
+        }
+
+        write!(w, r#"<font face="monospace">"#)?;
+
+        let p = p(self.0);
+
+        for a in actions.iter() {
+            match a {
+                GetLhs { path } => write!(w, "get-lhs @ {}<br/>", p(path))?,
+                UnaryUnquote { operator, operand } => {
+                    write!(w, "eval {} $rhs{}<br/>", operator, operand.0)?
+                }
+                BinaryUnquote { operator, operands } => write!(
+                    w,
+                    "eval {} $rhs{}, $rhs{}<br/>",
+                    operator, operands[0].0, operands[1].0,
+                )?,
+                MakeIntegerConst {
+                    value,
+                    bit_width: _,
+                } => write!(w, "make {}<br/>", self.1.lookup(*value))?,
+                MakeBooleanConst {
+                    value,
+                    bit_width: _,
+                } => write!(w, "make {}<br/>", value)?,
+                MakeConditionCode { cc } => write!(w, "{}<br/>", cc)?,
+                MakeUnaryInst {
+                    operand,
+                    operator,
+                    r#type: _,
+                } => write!(w, "make {} $rhs{}<br/>", operator, operand.0,)?,
+                MakeBinaryInst {
+                    operator,
+                    operands,
+                    r#type: _,
+                } => write!(
+                    w,
+                    "make {} $rhs{}, $rhs{}<br/>",
+                    operator, operands[0].0, operands[1].0,
+                )?,
+                MakeTernaryInst {
+                    operator,
+                    operands,
+                    r#type: _,
+                } => write!(
+                    w,
+                    "make {} $rhs{}, $rhs{}, $rhs{}<br/>",
+                    operator, operands[0].0, operands[1].0, operands[2].0,
+                )?,
+            }
+        }
+
+        writeln!(w, "</font>")
+    }
+}
+
+fn p<'a>(paths: &'a PathInterner) -> impl Fn(&PathId) -> String + 'a {
+    move |path: &PathId| {
+        let mut s = vec![];
+        for b in paths.lookup(*path).0 {
+            s.push(b.to_string());
+        }
+        s.join(".")
+    }
+}

cranelift/peepmatic/src/lib.rs

@@ -0,0 +1,165 @@
+/*!
+
+`peepmatic` is a DSL and compiler for generating peephole optimizers.
+
+The user writes a set of optimizations in the DSL, and then `peepmatic` compiles
+the set of optimizations into an efficient peephole optimizer.
+
+ */
+
+#![deny(missing_docs)]
+#![deny(missing_debug_implementations)]
+
+mod ast;
+mod automatize;
+mod dot_fmt;
+mod linear_passes;
+mod linearize;
+mod parser;
+mod traversals;
+mod verify;
+pub use self::{
+    ast::*, automatize::*, linear_passes::*, linearize::*, parser::*, traversals::*, verify::*,
+};
+
+use peepmatic_runtime::PeepholeOptimizations;
+use std::fs;
+use std::path::Path;
+
+/// Compile the given DSL file into a compact peephole optimizations automaton!
+///
+/// ## Example
+///
+/// ```no_run
+/// # fn main() -> anyhow::Result<()> {
+/// use std::path::Path;
+///
+/// let peep_opts = peepmatic::compile_file(Path::new(
+///     "path/to/optimizations.peepmatic"
+/// ))?;
+///
+/// // Use the peephole optimizations or serialize them into bytes here...
+/// # Ok(())
+/// # }
+/// ```
+///
+/// ## Visualizing the Peephole Optimizer's Automaton
+///
+/// To visualize (or debug) the peephole optimizer's automaton, set the
+/// `PEEPMATIC_DOT` environment variable to a file path. A [GraphViz
+/// Dot]((https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf)) file showing the
+/// peephole optimizer's automaton will be written to that file path.
+pub fn compile_file(filename: &Path) -> anyhow::Result<PeepholeOptimizations> {
+    let source = fs::read_to_string(filename)?;
+    compile_str(&source, filename)
+}
+
+/// Compile the given DSL source text down into a compact peephole optimizations
+/// automaton.
+///
+/// This is like [compile_file][crate::compile_file] but you bring your own file
+/// I/O.
+///
+/// The `filename` parameter is used to provide better error messages.
+///
+/// ## Example
+///
+/// ```no_run
+/// # fn main() -> anyhow::Result<()> {
+/// use std::path::Path;
+///
+/// let peep_opts = peepmatic::compile_str(
+///     "
+///     (=> (iadd $x 0) $x)
+///     (=> (imul $x 0) 0)
+///     (=> (imul $x 1) $x)
+///     ",
+///     Path::new("my-optimizations"),
+/// )?;
+///
+/// // Use the peephole optimizations or serialize them into bytes here...
+/// # Ok(())
+/// # }
+/// ```
+///
+/// ## Visualizing the Peephole Optimizer's Automaton
+///
+/// To visualize (or debug) the peephole optimizer's automaton, set the
+/// `PEEPMATIC_DOT` environment variable to a file path. A [GraphViz
+/// Dot]((https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf)) file showing the
+/// peephole optimizer's automaton will be written to that file path.
+pub fn compile_str(source: &str, filename: &Path) -> anyhow::Result<PeepholeOptimizations> {
+    let buf = wast::parser::ParseBuffer::new(source).map_err(|mut e| {
+        e.set_path(filename);
+        e.set_text(source);
+        e
+    })?;
+
+    let opts = wast::parser::parse::<Optimizations>(&buf).map_err(|mut e| {
+        e.set_path(filename);
+        e.set_text(source);
+        e
+    })?;
+
+    verify(&opts).map_err(|mut e| {
+        e.set_path(filename);
+        e.set_text(source);
+        e
+    })?;
+
+    let mut opts = crate::linearize(&opts);
+    sort_least_to_most_general(&mut opts);
+    remove_unnecessary_nops(&mut opts);
+    match_in_same_order(&mut opts);
+    sort_lexicographically(&mut opts);
+
+    let automata = automatize(&opts);
+    let paths = opts.paths;
+    let integers = opts.integers;
+
+    if let Ok(path) = std::env::var("PEEPMATIC_DOT") {
+        let f = dot_fmt::PeepholeDotFmt(&paths, &integers);
+        if let Err(e) = automata.write_dot_file(&f, &path) {
+            panic!(
+                "failed to write GraphViz Dot file to PEEPMATIC_DOT={}; error: {}",
+                path, e
+            );
+        }
+    }
+
+    Ok(PeepholeOptimizations {
+        paths,
+        integers,
+        automata,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_compiles(path: &str) {
+        match compile_file(Path::new(path)) {
+            Ok(_) => return,
+            Err(e) => {
+                eprintln!("error: {}", e);
+                panic!("error: {}", e);
+            }
+        }
+    }
+
+    #[test]
+    fn compile_redundant_bor() {
+        assert_compiles("examples/redundant-bor.peepmatic");
+    }
+
+    #[test]
+    fn mul_by_pow2() {
+        assert_compiles("examples/mul-by-pow2.peepmatic");
+    }
+
+    #[test]
+    fn compile_preopt() {
+        assert_compiles("../codegen/src/preopt.peepmatic");
+    }
+}

cranelift/peepmatic/src/linear_passes.rs

@@ -0,0 +1,719 @@
+//! Passes over the linear IR.
+
+use peepmatic_runtime::{
+    linear,
+    paths::{PathId, PathInterner},
+};
+use std::cmp::Ordering;
+
+/// Sort a set of optimizations from least to most general.
+///
+/// This helps us ensure that we always match the least-general (aka
+/// most-specific) optimization that we can for a particular instruction
+/// sequence.
+///
+/// For example, if we have both of these optimizations:
+///
+/// ```lisp
+/// (=> (imul $C $x)
+///     (imul_imm $C $x))
+///
+/// (=> (when (imul $C $x))
+///           (is-power-of-two $C))
+///     (ishl $x $C))
+/// ```
+///
+/// and we are matching `(imul 4 (..))`, then we want to apply the second
+/// optimization, because it is more specific than the first.
+pub fn sort_least_to_most_general(opts: &mut linear::Optimizations) {
+    let linear::Optimizations {
+        ref mut optimizations,
+        ref paths,
+        ..
+    } = opts;
+
+    // NB: we *cannot* use an unstable sort here, because we want deterministic
+    // compilation of optimizations to automata.
+    optimizations.sort_by(|a, b| compare_optimization_generality(paths, a, b));
+    debug_assert!(is_sorted_by_generality(opts));
+}
+
+/// Sort the linear optimizations lexicographically.
+///
+/// This sort order is required for automata construction.
+pub fn sort_lexicographically(opts: &mut linear::Optimizations) {
+    let linear::Optimizations {
+        ref mut optimizations,
+        ref paths,
+        ..
+    } = opts;
+
+    // NB: we *cannot* use an unstable sort here, same as above.
+    optimizations
+        .sort_by(|a, b| compare_optimizations(paths, a, b, |a_len, b_len| a_len.cmp(&b_len)));
+}
+
+fn compare_optimizations(
+    paths: &PathInterner,
+    a: &linear::Optimization,
+    b: &linear::Optimization,
+    compare_lengths: impl Fn(usize, usize) -> Ordering,
+) -> Ordering {
+    for (a, b) in a.increments.iter().zip(b.increments.iter()) {
+        let c = compare_match_op_generality(paths, a.operation, b.operation);
+        if c != Ordering::Equal {
+            return c;
+        }
+
+        let c = match (a.expected, b.expected) {
+            (Ok(a), Ok(b)) => a.cmp(&b).reverse(),
+            (Err(_), Ok(_)) => Ordering::Greater,
+            (Ok(_), Err(_)) => Ordering::Less,
+            (Err(linear::Else), Err(linear::Else)) => Ordering::Equal,
+        };
+        if c != Ordering::Equal {
+            return c;
+        }
+    }
+
+    compare_lengths(a.increments.len(), b.increments.len())
+}
+
+fn compare_optimization_generality(
+    paths: &PathInterner,
+    a: &linear::Optimization,
+    b: &linear::Optimization,
+) -> Ordering {
+    compare_optimizations(paths, a, b, |a_len, b_len| {
+        // If they shared equivalent prefixes, then compare lengths and invert the
+        // result because longer patterns are less general than shorter patterns.
+        a_len.cmp(&b_len).reverse()
+    })
+}
+
+fn compare_match_op_generality(
+    paths: &PathInterner,
+    a: linear::MatchOp,
+    b: linear::MatchOp,
+) -> Ordering {
+    use linear::MatchOp::*;
+    match (a, b) {
+        (Opcode { path: a }, Opcode { path: b }) => compare_paths(paths, a, b),
+        (Opcode { .. }, _) => Ordering::Less,
+        (_, Opcode { .. }) => Ordering::Greater,
+
+        (IntegerValue { path: a }, IntegerValue { path: b }) => compare_paths(paths, a, b),
+        (IntegerValue { .. }, _) => Ordering::Less,
+        (_, IntegerValue { .. }) => Ordering::Greater,
+
+        (BooleanValue { path: a }, BooleanValue { path: b }) => compare_paths(paths, a, b),
+        (BooleanValue { .. }, _) => Ordering::Less,
+        (_, BooleanValue { .. }) => Ordering::Greater,
+
+        (ConditionCode { path: a }, ConditionCode { path: b }) => compare_paths(paths, a, b),
+        (ConditionCode { .. }, _) => Ordering::Less,
+        (_, ConditionCode { .. }) => Ordering::Greater,
+
+        (IsConst { path: a }, IsConst { path: b }) => compare_paths(paths, a, b),
+        (IsConst { .. }, _) => Ordering::Less,
+        (_, IsConst { .. }) => Ordering::Greater,
+
+        (
+            Eq {
+                path_a: pa1,
+                path_b: pb1,
+            },
+            Eq {
+                path_a: pa2,
+                path_b: pb2,
+            },
+        ) => compare_paths(paths, pa1, pa2).then(compare_paths(paths, pb1, pb2)),
+        (Eq { .. }, _) => Ordering::Less,
+        (_, Eq { .. }) => Ordering::Greater,
+
+        (IsPowerOfTwo { path: a }, IsPowerOfTwo { path: b }) => compare_paths(paths, a, b),
+        (IsPowerOfTwo { .. }, _) => Ordering::Less,
+        (_, IsPowerOfTwo { .. }) => Ordering::Greater,
+
+        (BitWidth { path: a }, BitWidth { path: b }) => compare_paths(paths, a, b),
+        (BitWidth { .. }, _) => Ordering::Less,
+        (_, BitWidth { .. }) => Ordering::Greater,
+
+        (FitsInNativeWord { path: a }, FitsInNativeWord { path: b }) => compare_paths(paths, a, b),
+        (FitsInNativeWord { .. }, _) => Ordering::Less,
+        (_, FitsInNativeWord { .. }) => Ordering::Greater,
+
+        (Nop, Nop) => Ordering::Equal,
+    }
+}
+
+fn compare_paths(paths: &PathInterner, a: PathId, b: PathId) -> Ordering {
+    if a == b {
+        Ordering::Equal
+    } else {
+        let a = paths.lookup(a);
+        let b = paths.lookup(b);
+        a.0.cmp(&b.0)
+    }
+}
+
+/// Are the given optimizations sorted from least to most general?
+pub(crate) fn is_sorted_by_generality(opts: &linear::Optimizations) -> bool {
+    opts.optimizations
+        .windows(2)
+        .all(|w| compare_optimization_generality(&opts.paths, &w[0], &w[1]) <= Ordering::Equal)
+}
+
+/// Are the given optimizations sorted lexicographically?
+pub(crate) fn is_sorted_lexicographically(opts: &linear::Optimizations) -> bool {
+    opts.optimizations.windows(2).all(|w| {
+        compare_optimizations(&opts.paths, &w[0], &w[1], |a_len, b_len| a_len.cmp(&b_len))
+            <= Ordering::Equal
+    })
+}
+
+/// Ensure that we emit match operations in a consistent order.
+///
+/// There are many linear optimizations, each of which have their own sequence
+/// of match operations that need to be tested. But when interpreting the
+/// automata against some instructions, we only perform a single sequence of
+/// match operations, and at any given moment, we only want one match operation
+/// to interpret next. This means that two optimizations that are next to each
+/// other in the sorting must have their shared prefixes diverge on an
+/// **expected result edge**, not on which match operation to preform next. And
+/// if they have zero shared prefix, then we need to create one, that
+/// immediately divereges on the expected result.
+///
+/// For example, consider these two patterns that don't have any shared prefix:
+///
+/// ```lisp
+/// (=> (iadd $x $y) ...)
+/// (=> $C ...)
+/// ```
+///
+/// These produce the following linear match operations and expected results:
+///
+/// ```text
+/// opcode @ 0 --iadd-->
+/// is-const? @ 0 --true-->
+/// ```
+///
+/// In order to ensure that we only have one match operation to interpret at any
+/// given time when evaluating the automata, this pass transforms the second
+/// optimization so that it shares a prefix match operation, but diverges on the
+/// expected result:
+///
+/// ```text
+/// opcode @ 0 --iadd-->
+/// opcode @ 0 --(else)--> is-const? @ 0 --true-->
+/// ```
+pub fn match_in_same_order(opts: &mut linear::Optimizations) {
+    assert!(!opts.optimizations.is_empty());
+
+    let mut prefix = vec![];
+
+    for opt in &mut opts.optimizations {
+        assert!(!opt.increments.is_empty());
+
+        let mut old_increments = opt.increments.iter().peekable();
+        let mut new_increments = vec![];
+
+        for (last_op, last_expected) in &prefix {
+            match old_increments.peek() {
+                None => {
+                    break;
+                }
+                Some(inc) if *last_op == inc.operation => {
+                    let inc = old_increments.next().unwrap();
+                    new_increments.push(inc.clone());
+                    if inc.expected != *last_expected {
+                        break;
+                    }
+                }
+                Some(_) => {
+                    new_increments.push(linear::Increment {
+                        operation: *last_op,
+                        expected: Err(linear::Else),
+                        actions: vec![],
+                    });
+                    if last_expected.is_ok() {
+                        break;
+                    }
+                }
+            }
+        }
+
+        new_increments.extend(old_increments.cloned());
+        assert!(new_increments.len() >= opt.increments.len());
+        opt.increments = new_increments;
+
+        prefix.clear();
+        prefix.extend(
+            opt.increments
+                .iter()
+                .map(|inc| (inc.operation, inc.expected)),
+        );
+    }
+
+    // Should still be sorted after this pass.
+    debug_assert!(is_sorted_by_generality(&opts));
+}
+
+/// 99.99% of nops are unnecessary; remove them.
+///
+/// They're only needed for when a LHS pattern is just a variable, and that's
+/// it. However, it is easier to have basically unused nop matching operations
+/// for the DSL's edge-cases than it is to try and statically eliminate their
+/// existence completely. So we just emit nop match operations for all variable
+/// patterns, and then in this post-processing pass, we fuse them and their
+/// actions with their preceding increment.
+pub fn remove_unnecessary_nops(opts: &mut linear::Optimizations) {
+    for opt in &mut opts.optimizations {
+        if opt.increments.len() < 2 {
+            debug_assert!(!opt.increments.is_empty());
+            continue;
+        }
+
+        for i in (1..opt.increments.len()).rev() {
+            if let linear::MatchOp::Nop = opt.increments[i].operation {
+                let nop = opt.increments.remove(i);
+                opt.increments[i - 1].actions.extend(nop.actions);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::ast::*;
+    use peepmatic_runtime::{
+        linear::{bool_to_match_result, Else, MatchOp::*, MatchResult},
+        operator::Operator,
+        paths::*,
+    };
+    use std::num::NonZeroU32;
+
+    #[test]
+    fn ok_non_zero_less_than_err_else() {
+        assert!(Ok(NonZeroU32::new(1).unwrap()) < Err(Else));
+    }
+
+    macro_rules! sorts_to {
+        ($test_name:ident, $source:expr, $make_expected:expr) => {
+            #[test]
+            #[allow(unused_variables)]
+            fn $test_name() {
+                let buf = wast::parser::ParseBuffer::new($source).expect("should lex OK");
+
+                let opts = match wast::parser::parse::<Optimizations>(&buf) {
+                    Ok(opts) => opts,
+                    Err(mut e) => {
+                        e.set_path(std::path::Path::new(stringify!($test_name)));
+                        e.set_text($source);
+                        eprintln!("{}", e);
+                        panic!("should parse OK")
+                    }
+                };
+
+                if let Err(mut e) = crate::verify(&opts) {
+                    e.set_path(std::path::Path::new(stringify!($test_name)));
+                    e.set_text($source);
+                    eprintln!("{}", e);
+                    panic!("should verify OK")
+                }
+
+                let mut opts = crate::linearize(&opts);
+
+                let before = opts
+                    .optimizations
+                    .iter()
+                    .map(|o| {
+                        o.increments
+                            .iter()
+                            .map(|i| format!("{:?} == {:?}", i.operation, i.expected))
+                            .collect::<Vec<_>>()
+                    })
+                    .collect::<Vec<_>>();
+                eprintln!("before = {:#?}", before);
+
+                sort_least_to_most_general(&mut opts);
+
+                let after = opts
+                    .optimizations
+                    .iter()
+                    .map(|o| {
+                        o.increments
+                            .iter()
+                            .map(|i| format!("{:?} == {:?}", i.operation, i.expected))
+                            .collect::<Vec<_>>()
+                    })
+                    .collect::<Vec<_>>();
+                eprintln!("after = {:#?}", before);
+
+                let linear::Optimizations {
+                    mut paths,
+                    mut integers,
+                    optimizations,
+                } = opts;
+
+                let actual: Vec<Vec<_>> = optimizations
+                    .iter()
+                    .map(|o| {
+                        o.increments
+                            .iter()
+                            .map(|i| (i.operation, i.expected))
+                            .collect()
+                    })
+                    .collect();
+
+                let mut p = |p: &[u8]| paths.intern(Path::new(&p));
+                let mut i = |i: u64| Ok(integers.intern(i).into());
+                let expected = $make_expected(&mut p, &mut i);
+
+                assert_eq!(expected, actual);
+            }
+        };
+    }
+
+    macro_rules! match_in_same_order {
+        ($test_name:ident, $source:expr, $make_expected:expr) => {
+            #[test]
+            #[allow(unused_variables)]
+            fn $test_name() {
+                let buf = wast::parser::ParseBuffer::new($source).expect("should lex OK");
+
+                let opts = match wast::parser::parse::<Optimizations>(&buf) {
+                    Ok(opts) => opts,
+                    Err(mut e) => {
+                        e.set_path(std::path::Path::new(stringify!($test_name)));
+                        e.set_text($source);
+                        eprintln!("{}", e);
+                        panic!("should parse OK")
+                    }
+                };
+
+                if let Err(mut e) = crate::verify(&opts) {
+                    e.set_path(std::path::Path::new(stringify!($test_name)));
+                    e.set_text($source);
+                    eprintln!("{}", e);
+                    panic!("should verify OK")
+                }
+
+                let mut opts = crate::linearize(&opts);
+                sort_least_to_most_general(&mut opts);
+
+                let before = opts
+                    .optimizations
+                    .iter()
+                    .map(|o| {
+                        o.increments
+                            .iter()
+                            .map(|i| format!("{:?} == {:?}", i.operation, i.expected))
+                            .collect::<Vec<_>>()
+                    })
+                    .collect::<Vec<_>>();
+                eprintln!("before = {:#?}", before);
+
+                match_in_same_order(&mut opts);
+
+                let after = opts
+                    .optimizations
+                    .iter()
+                    .map(|o| {
+                        o.increments
+                            .iter()
+                            .map(|i| format!("{:?} == {:?}", i.operation, i.expected))
+                            .collect::<Vec<_>>()
+                    })
+                    .collect::<Vec<_>>();
+                eprintln!("after = {:#?}", before);
+
+                let linear::Optimizations {
+                    mut paths,
+                    mut integers,
+                    optimizations,
+                } = opts;
+
+                let actual: Vec<Vec<_>> = optimizations
+                    .iter()
+                    .map(|o| {
+                        o.increments
+                            .iter()
+                            .map(|i| (i.operation, i.expected))
+                            .collect()
+                    })
+                    .collect();
+
+                let mut p = |p: &[u8]| paths.intern(Path::new(&p));
+                let mut i = |i: u64| Ok(integers.intern(i).into());
+                let expected = $make_expected(&mut p, &mut i);
+
+                assert_eq!(expected, actual);
+            }
+        };
+    }
+
+    sorts_to!(
+        test_sort_least_to_most_general,
+        "
+(=>       $x                                 0)
+(=>       (iadd $x $y)                       0)
+(=>       (iadd $x $x)                       0)
+(=>       (iadd $x $C)                       0)
+(=> (when (iadd $x $C) (is-power-of-two $C)) 0)
+(=> (when (iadd $x $C) (bit-width $x 32))    0)
+(=>       (iadd $x 42)                       0)
+(=>       (iadd $x (iadd $y $z))             0)
+",
+        |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (
+                    Opcode { path: p(&[0, 1]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (Nop, Err(Else)),
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (IntegerValue { path: p(&[0, 1]) }, i(42))
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (IsConst { path: p(&[0, 1]) }, bool_to_match_result(true)),
+                (
+                    IsPowerOfTwo { path: p(&[0, 1]) },
+                    bool_to_match_result(true)
+                )
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (IsConst { path: p(&[0, 1]) }, bool_to_match_result(true)),
+                (
+                    BitWidth { path: p(&[0, 0]) },
+                    Ok(NonZeroU32::new(32).unwrap())
+                )
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (IsConst { path: p(&[0, 1]) }, bool_to_match_result(true))
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (
+                    Eq {
+                        path_a: p(&[0, 1]),
+                        path_b: p(&[0, 0]),
+                    },
+                    bool_to_match_result(true)
+                )
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (Nop, Err(Else)),
+            ],
+            vec![(Nop, Err(Else))]
+        ]
+    );
+
+    sorts_to!(
+        expected_edges_are_sorted,
+        "
+(=> (iadd 0 $x) $x)
+(=> (iadd $x 0) $x)
+(=> (imul 1 $x) $x)
+(=> (imul $x 1) $x)
+(=> (imul 2 $x) (ishl $x 1))
+(=> (imul $x 2) (ishl $x 1))
+",
+        |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Imul as _).unwrap())
+                ),
+                (IntegerValue { path: p(&[0, 0]) }, i(2)),
+                (Nop, Err(Else))
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Imul as _).unwrap())
+                ),
+                (IntegerValue { path: p(&[0, 0]) }, i(1)),
+                (Nop, Err(Else))
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Imul as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (IntegerValue { path: p(&[0, 1]) }, i(2))
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Imul as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (IntegerValue { path: p(&[0, 1]) }, i(1))
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (IntegerValue { path: p(&[0, 0]) }, i(0)),
+                (Nop, Err(Else))
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Iadd as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (IntegerValue { path: p(&[0, 1]) }, i(0))
+            ]
+        ]
+    );
+
+    sorts_to!(
+        sort_redundant_bor,
+        "
+        (=> (bor (bor $x $y) $x)
+            (bor $x $y))
+
+        (=> (bor (bor $x $y) $y)
+            (bor $x $y))
+        ",
+        |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Bor as _).unwrap())
+                ),
+                (
+                    Opcode { path: p(&[0, 0]) },
+                    Ok(NonZeroU32::new(Operator::Bor as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (Nop, Err(Else)),
+                (
+                    Eq {
+                        path_a: p(&[0, 1]),
+                        path_b: p(&[0, 0, 0]),
+                    },
+                    bool_to_match_result(true)
+                ),
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Bor as _).unwrap())
+                ),
+                (
+                    Opcode { path: p(&[0, 0]) },
+                    Ok(NonZeroU32::new(Operator::Bor as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (Nop, Err(Else)),
+                (
+                    Eq {
+                        path_a: p(&[0, 1]),
+                        path_b: p(&[0, 0, 1]),
+                    },
+                    bool_to_match_result(true)
+                ),
+            ],
+        ]
+    );
+
+    match_in_same_order!(
+        match_in_same_order_redundant_bor,
+        "
+        (=> (bor (bor $x $y) $x)
+            (bor $x $y))
+
+        (=> (bor (bor $x $y) $y)
+            (bor $x $y))
+        ",
+        |p: &mut dyn FnMut(&[u8]) -> PathId, i: &mut dyn FnMut(u64) -> MatchResult| vec![
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Bor as _).unwrap())
+                ),
+                (
+                    Opcode { path: p(&[0, 0]) },
+                    Ok(NonZeroU32::new(Operator::Bor as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (Nop, Err(Else)),
+                (
+                    Eq {
+                        path_a: p(&[0, 1]),
+                        path_b: p(&[0, 0, 0]),
+                    },
+                    bool_to_match_result(true)
+                ),
+            ],
+            vec![
+                (
+                    Opcode { path: p(&[0]) },
+                    Ok(NonZeroU32::new(Operator::Bor as _).unwrap())
+                ),
+                (
+                    Opcode { path: p(&[0, 0]) },
+                    Ok(NonZeroU32::new(Operator::Bor as _).unwrap())
+                ),
+                (Nop, Err(Else)),
+                (Nop, Err(Else)),
+                (
+                    Eq {
+                        path_a: p(&[0, 1]),
+                        path_b: p(&[0, 0, 0]),
+                    },
+                    Err(Else),
+                ),
+                (
+                    Eq {
+                        path_a: p(&[0, 1]),
+                        path_b: p(&[0, 0, 1]),
+                    },
+                    bool_to_match_result(true)
+                ),
+            ],
+        ]
+    );
+}

cranelift/peepmatic/src/linearize.rs

@@ -0,0 +1,812 @@
+//! Convert an AST into its linear equivalent.
+//!
+//! Convert each optimization's left-hand side into a linear series of match
+//! operations. This makes it easy to create an automaton, because automatas
+//! typically deal with a linear sequence of inputs. The optimization's
+//! right-hand side is built incrementally inside actions that are taken on
+//! transitions between match operations.
+//!
+//! See `crates/runtime/src/linear.rs` for the linear datatype definitions.
+//!
+//! ## Example
+//!
+//! As an example, if we linearize this optimization:
+//!
+//! ```lisp
+//! (=> (when (imul $x $C)
+//!           (is-power-of-two $C))
+//!     (ishl $x $(log2 C)))
+//! ```
+//!
+//! Then we should get the following linear chain of "increments":
+//!
+//! ```ignore
+//! [
+//!   // ( Match Operation, Expected Value, Actions )
+//!   ( Opcode@0,           imul,           [$x = GetLhs@0.0, $C = GetLhs@0.1, ...] ),
+//!   ( IsConst(C),         true,           [] ),
+//!   ( IsPowerOfTwo(C),    true,           [] ),
+//! ]
+//! ```
+//!
+//! Each increment will essentially become a state and a transition out of that
+//! state in the final automata, along with the actions to perform when taking
+//! that transition. The actions record the scope of matches from the left-hand
+//! side and also incrementally build the right-hand side's instructions. (Note
+//! that we've elided the actions that build up the optimization's right-hand
+//! side in this example.)
+//!
+//! ## General Principles
+//!
+//! Here are the general principles that linearization should adhere to:
+//!
+//! * Actions should be pushed as early in the optimization's increment chain as
+//!   they can be. This means the tail has fewer side effects, and is therefore
+//!   more likely to be share-able with other optimizations in the automata that
+//!   we build.
+//!
+//! * RHS actions cannot reference matches from the LHS until they've been
+//!   defined. And finally, an RHS operation's operands must be defined before
+//!   the RHS operation itself. In general, definitions must come before uses!
+//!
+//! * Shorter increment chains are better! This means fewer tests when matching
+//!   left-hand sides, and a more-compact, more-cache-friendly automata, and
+//!   ultimately, a faster automata.
+//!
+//! * An increment's match operation should be a switch rather than a predicate
+//!   that returns a boolean. For example, we switch on an instruction's opcode,
+//!   rather than ask whether this operation is an `imul`. This allows for more
+//!   prefix sharing in the automata, which (again) makes it more compact and
+//!   more cache friendly.
+//!
+//! ## Implementation Overview
+//!
+//! We emit match operations for a left-hand side's pattern structure, followed
+//! by match operations for its preconditions on that structure. This ensures
+//! that anything bound in the pattern is defined before it is used in
+//! precondition.
+//!
+//! Within matching the pattern structure, we emit matching operations in a
+//! pre-order traversal of the pattern. This ensures that we've already matched
+//! an operation before we consider its operands, and therefore we already know
+//! the operands exist. See `PatternPreOrder` for details.
+//!
+//! As we define the match operations for a pattern, we remember the path where
+//! each LHS id first occurred. These will later be reused when building the RHS
+//! actions. See `LhsIdToPath` for details.
+//!
+//! After we've generated the match operations and expected result of those
+//! match operations, then we generate the right-hand side actions. The
+//! right-hand side is built up a post-order traversal, so that operands are
+//! defined before they are used. See `RhsPostOrder` and `RhsBuilder` for
+//! details.
+//!
+//! Finally, see `linearize_optimization` for the the main AST optimization into
+//! linear optimization translation function.
+
+use crate::ast::*;
+use crate::traversals::Dfs;
+use peepmatic_runtime::{
+    integer_interner::IntegerInterner,
+    linear,
+    paths::{Path, PathId, PathInterner},
+};
+use std::collections::BTreeMap;
+use std::convert::TryInto;
+use std::num::NonZeroU32;
+use wast::Id;
+
+/// Translate the given AST optimizations into linear optimizations.
+pub fn linearize(opts: &Optimizations) -> linear::Optimizations {
+    let mut optimizations = vec![];
+    let mut paths = PathInterner::new();
+    let mut integers = IntegerInterner::new();
+    for opt in &opts.optimizations {
+        let lin_opt = linearize_optimization(&mut paths, &mut integers, opt);
+        optimizations.push(lin_opt);
+    }
+    linear::Optimizations {
+        optimizations,
+        paths,
+        integers,
+    }
+}
+
+/// Translate an AST optimization into a linear optimization!
+fn linearize_optimization(
+    paths: &mut PathInterner,
+    integers: &mut IntegerInterner,
+    opt: &Optimization,
+) -> linear::Optimization {
+    let mut increments: Vec<linear::Increment> = vec![];
+
+    let mut lhs_id_to_path = LhsIdToPath::new();
+
+    // We do a pre-order traversal of the LHS because we don't know whether a
+    // child actually exists to match on until we've matched its parent, and we
+    // don't want to emit matching operations on things that might not exist!
+    let mut patterns = PatternPreOrder::new(&opt.lhs.pattern);
+    while let Some((path, pattern)) = patterns.next(paths) {
+        // Create the matching parts of an `Increment` for this part of the
+        // pattern, without any actions yet.
+        let (operation, expected) = pattern.to_linear_match_op(integers, &lhs_id_to_path, path);
+        increments.push(linear::Increment {
+            operation,
+            expected,
+            actions: vec![],
+        });
+
+        lhs_id_to_path.remember_path_to_pattern_ids(pattern, path);
+
+        // Some operations require type ascriptions for us to infer the correct
+        // bit width of their results: `ireduce`, `sextend`, `uextend`, etc.
+        // When there is such a type ascription in the pattern, insert another
+        // increment that checks the instruction-being-matched's bit width.
+        if let Pattern::Operation(Operation { r#type, .. }) = pattern {
+            if let Some(w) = r#type.get().and_then(|ty| ty.bit_width.fixed_width()) {
+                debug_assert!(w != 0, "All fixed-width bit widths are non-zero");
+                let expected = Ok(unsafe { NonZeroU32::new_unchecked(w as u32) });
+
+                increments.push(linear::Increment {
+                    operation: linear::MatchOp::BitWidth { path },
+                    expected,
+                    actions: vec![],
+                });
+            }
+        }
+    }
+
+    // Now that we've added all the increments for the LHS pattern, add the
+    // increments for its preconditions.
+    for pre in &opt.lhs.preconditions {
+        increments.push(pre.to_linear_increment(&lhs_id_to_path));
+    }
+
+    assert!(!increments.is_empty());
+
+    // Finally, generate the RHS-building actions and attach them to the first increment.
+    let mut rhs_builder = RhsBuilder::new(&opt.rhs);
+    rhs_builder.add_rhs_build_actions(integers, &lhs_id_to_path, &mut increments[0].actions);
+
+    linear::Optimization { increments }
+}
+
+/// A post-order, depth-first traversal of right-hand sides.
+///
+/// Does not maintain any extra state about the traversal, such as where in the
+/// tree each yielded node comes from.
+struct RhsPostOrder<'a> {
+    dfs: Dfs<'a>,
+}
+
+impl<'a> RhsPostOrder<'a> {
+    fn new(rhs: &'a Rhs<'a>) -> Self {
+        Self { dfs: Dfs::new(rhs) }
+    }
+}
+
+impl<'a> Iterator for RhsPostOrder<'a> {
+    type Item = &'a Rhs<'a>;
+
+    fn next(&mut self) -> Option<&'a Rhs<'a>> {
+        use crate::traversals::TraversalEvent as TE;
+        loop {
+            match self.dfs.next()? {
+                (TE::Exit, DynAstRef::Rhs(rhs)) => return Some(rhs),
+                _ => continue,
+            }
+        }
+    }
+}
+
+/// A pre-order, depth-first traversal of left-hand side patterns.
+///
+/// Keeps track of the path to each pattern, and yields it along side the
+/// pattern AST node.
+struct PatternPreOrder<'a> {
+    last_child: Option<u8>,
+    path: Vec<u8>,
+    dfs: Dfs<'a>,
+}
+
+impl<'a> PatternPreOrder<'a> {
+    fn new(pattern: &'a Pattern<'a>) -> Self {
+        Self {
+            last_child: None,
+            path: vec![],
+            dfs: Dfs::new(pattern),
+        }
+    }
+
+    fn next(&mut self, paths: &mut PathInterner) -> Option<(PathId, &'a Pattern<'a>)> {
+        use crate::traversals::TraversalEvent as TE;
+        loop {
+            match self.dfs.next()? {
+                (TE::Enter, DynAstRef::Pattern(pattern)) => {
+                    let last_child = self.last_child.take();
+                    self.path.push(match last_child {
+                        None => 0,
+                        Some(c) => {
+                            assert!(
+                                c < std::u8::MAX,
+                                "operators must have less than or equal u8::MAX arity"
+                            );
+                            c + 1
+                        }
+                    });
+                    let path = paths.intern(Path(&self.path));
+                    return Some((path, pattern));
+                }
+                (TE::Exit, DynAstRef::Pattern(_)) => {
+                    self.last_child = Some(
+                        self.path
+                            .pop()
+                            .expect("should always have a non-empty path during traversal"),
+                    );
+                }
+                _ => {}
+            }
+        }
+    }
+}
+
+/// A map from left-hand side identifiers to the path in the left-hand side
+/// where they first occurred.
+struct LhsIdToPath<'a> {
+    id_to_path: BTreeMap<&'a str, PathId>,
+}
+
+impl<'a> LhsIdToPath<'a> {
+    /// Construct a new, empty `LhsIdToPath`.
+    fn new() -> Self {
+        Self {
+            id_to_path: Default::default(),
+        }
+    }
+
+    /// Have we already seen the given identifier?
+    fn get_first_occurrence(&self, id: &Id) -> Option<PathId> {
+        self.id_to_path.get(id.name()).copied()
+    }
+
+    /// Get the path within the left-hand side pattern where we first saw the
+    /// given AST id.
+    ///
+    /// ## Panics
+    ///
+    /// Panics if the given AST id has not already been canonicalized.
+    fn unwrap_first_occurrence(&self, id: &Id) -> PathId {
+        self.id_to_path[id.name()]
+    }
+
+    /// Remember the path to any LHS ids used in the given pattern.
+    fn remember_path_to_pattern_ids(&mut self, pattern: &'a Pattern<'a>, path: PathId) {
+        match pattern {
+            // If this is the first time we've seen an identifier defined on the
+            // left-hand side, remember it.
+            Pattern::Variable(Variable { id, .. }) | Pattern::Constant(Constant { id, .. }) => {
+                self.id_to_path.entry(id.name()).or_insert(path);
+            }
+            _ => {}
+        }
+    }
+}
+
+/// An `RhsBuilder` emits the actions for building the right-hand side
+/// instructions.
+struct RhsBuilder<'a> {
+    // We do a post order traversal of the RHS because an RHS instruction cannot
+    // be created until after all of its operands are created.
+    rhs_post_order: RhsPostOrder<'a>,
+
+    // A map from a right-hand side's span to its `linear::RhsId`. This is used
+    // by RHS-construction actions to reference operands. In practice the
+    // `RhsId` is roughly equivalent to its index in the post-order traversal of
+    // the RHS.
+    rhs_span_to_id: BTreeMap<wast::Span, linear::RhsId>,
+}
+
+impl<'a> RhsBuilder<'a> {
+    /// Create a new builder for the given right-hand side.
+    fn new(rhs: &'a Rhs<'a>) -> Self {
+        let rhs_post_order = RhsPostOrder::new(rhs);
+        let rhs_span_to_id = Default::default();
+        Self {
+            rhs_post_order,
+            rhs_span_to_id,
+        }
+    }
+
+    /// Get the `linear::RhsId` for the given right-hand side.
+    ///
+    /// ## Panics
+    ///
+    /// Panics if we haven't already emitted the action for building this RHS's
+    /// instruction.
+    fn get_rhs_id(&self, rhs: &Rhs) -> linear::RhsId {
+        self.rhs_span_to_id[&rhs.span()]
+    }
+
+    /// Create actions for building up this right-hand side of an optimization.
+    ///
+    /// Because we are walking the right-hand side with a post-order traversal,
+    /// we know that we already created an instruction's operands that are
+    /// defined in the right-hand side, before we get to the parent instruction.
+    fn add_rhs_build_actions(
+        &mut self,
+        integers: &mut IntegerInterner,
+        lhs_id_to_path: &LhsIdToPath,
+        actions: &mut Vec<linear::Action>,
+    ) {
+        while let Some(rhs) = self.rhs_post_order.next() {
+            actions.push(self.rhs_to_linear_action(integers, lhs_id_to_path, rhs));
+            let id = linear::RhsId(self.rhs_span_to_id.len().try_into().unwrap());
+            self.rhs_span_to_id.insert(rhs.span(), id);
+        }
+    }
+
+    fn rhs_to_linear_action(
+        &self,
+        integers: &mut IntegerInterner,
+        lhs_id_to_path: &LhsIdToPath,
+        rhs: &Rhs,
+    ) -> linear::Action {
+        match rhs {
+            Rhs::ValueLiteral(ValueLiteral::Integer(i)) => linear::Action::MakeIntegerConst {
+                value: integers.intern(i.value as u64),
+                bit_width: i
+                    .bit_width
+                    .get()
+                    .expect("should be initialized after type checking"),
+            },
+            Rhs::ValueLiteral(ValueLiteral::Boolean(b)) => linear::Action::MakeBooleanConst {
+                value: b.value,
+                bit_width: b
+                    .bit_width
+                    .get()
+                    .expect("should be initialized after type checking"),
+            },
+            Rhs::ValueLiteral(ValueLiteral::ConditionCode(ConditionCode { cc, .. })) => {
+                linear::Action::MakeConditionCode { cc: *cc }
+            }
+            Rhs::Variable(Variable { id, .. }) | Rhs::Constant(Constant { id, .. }) => {
+                let path = lhs_id_to_path.unwrap_first_occurrence(id);
+                linear::Action::GetLhs { path }
+            }
+            Rhs::Unquote(unq) => match unq.operands.len() {
+                1 => linear::Action::UnaryUnquote {
+                    operator: unq.operator,
+                    operand: self.get_rhs_id(&unq.operands[0]),
+                },
+                2 => linear::Action::BinaryUnquote {
+                    operator: unq.operator,
+                    operands: [
+                        self.get_rhs_id(&unq.operands[0]),
+                        self.get_rhs_id(&unq.operands[1]),
+                    ],
+                },
+                n => unreachable!("no unquote operators of arity {}", n),
+            },
+            Rhs::Operation(op) => match op.operands.len() {
+                1 => linear::Action::MakeUnaryInst {
+                    operator: op.operator,
+                    r#type: op
+                        .r#type
+                        .get()
+                        .expect("should be initialized after type checking"),
+                    operand: self.get_rhs_id(&op.operands[0]),
+                },
+                2 => linear::Action::MakeBinaryInst {
+                    operator: op.operator,
+                    r#type: op
+                        .r#type
+                        .get()
+                        .expect("should be initialized after type checking"),
+                    operands: [
+                        self.get_rhs_id(&op.operands[0]),
+                        self.get_rhs_id(&op.operands[1]),
+                    ],
+                },
+                3 => linear::Action::MakeTernaryInst {
+                    operator: op.operator,
+                    r#type: op
+                        .r#type
+                        .get()
+                        .expect("should be initialized after type checking"),
+                    operands: [
+                        self.get_rhs_id(&op.operands[0]),
+                        self.get_rhs_id(&op.operands[1]),
+                        self.get_rhs_id(&op.operands[2]),
+                    ],
+                },
+                n => unreachable!("no instructions of arity {}", n),
+            },
+        }
+    }
+}
+
+impl Precondition<'_> {
+    /// Convert this precondition into a `linear::Increment`.
+    fn to_linear_increment(&self, lhs_id_to_path: &LhsIdToPath) -> linear::Increment {
+        match self.constraint {
+            Constraint::IsPowerOfTwo => {
+                let id = match &self.operands[0] {
+                    ConstraintOperand::Constant(Constant { id, .. }) => id,
+                    _ => unreachable!("checked in verification"),
+                };
+                let path = lhs_id_to_path.unwrap_first_occurrence(&id);
+                linear::Increment {
+                    operation: linear::MatchOp::IsPowerOfTwo { path },
+                    expected: linear::bool_to_match_result(true),
+                    actions: vec![],
+                }
+            }
+            Constraint::BitWidth => {
+                let id = match &self.operands[0] {
+                    ConstraintOperand::Constant(Constant { id, .. })
+                    | ConstraintOperand::Variable(Variable { id, .. }) => id,
+                    _ => unreachable!("checked in verification"),
+                };
+                let path = lhs_id_to_path.unwrap_first_occurrence(&id);
+
+                let width = match &self.operands[1] {
+                    ConstraintOperand::ValueLiteral(ValueLiteral::Integer(Integer {
+                        value,
+                        ..
+                    })) => *value,
+                    _ => unreachable!("checked in verification"),
+                };
+
+                assert!(0 < width && width <= 128);
+                assert!((width as u8).is_power_of_two());
+                let expected = Ok(unsafe { NonZeroU32::new_unchecked(width as u32) });
+
+                linear::Increment {
+                    operation: linear::MatchOp::BitWidth { path },
+                    expected,
+                    actions: vec![],
+                }
+            }
+            Constraint::FitsInNativeWord => {
+                let id = match &self.operands[0] {
+                    ConstraintOperand::Constant(Constant { id, .. })
+                    | ConstraintOperand::Variable(Variable { id, .. }) => id,
+                    _ => unreachable!("checked in verification"),
+                };
+                let path = lhs_id_to_path.unwrap_first_occurrence(&id);
+                linear::Increment {
+                    operation: linear::MatchOp::FitsInNativeWord { path },
+                    expected: linear::bool_to_match_result(true),
+                    actions: vec![],
+                }
+            }
+        }
+    }
+}
+
+impl Pattern<'_> {
+    /// Convert this pattern into its linear match operation and the expected
+    /// result of that operation.
+    ///
+    /// NB: these mappings to expected values need to stay sync'd with the
+    /// runtime!
+    fn to_linear_match_op(
+        &self,
+        integers: &mut IntegerInterner,
+        lhs_id_to_path: &LhsIdToPath,
+        path: PathId,
+    ) -> (linear::MatchOp, linear::MatchResult) {
+        match self {
+            Pattern::ValueLiteral(ValueLiteral::Integer(Integer { value, .. })) => (
+                linear::MatchOp::IntegerValue { path },
+                Ok(integers.intern(*value as u64).into()),
+            ),
+            Pattern::ValueLiteral(ValueLiteral::Boolean(Boolean { value, .. })) => (
+                linear::MatchOp::BooleanValue { path },
+                linear::bool_to_match_result(*value),
+            ),
+            Pattern::ValueLiteral(ValueLiteral::ConditionCode(ConditionCode { cc, .. })) => {
+                let cc = *cc as u32;
+                debug_assert!(cc != 0, "no `ConditionCode` variants are zero");
+                let expected = Ok(unsafe { NonZeroU32::new_unchecked(cc) });
+                (linear::MatchOp::ConditionCode { path }, expected)
+            }
+            Pattern::Constant(Constant { id, .. }) => {
+                if let Some(path_b) = lhs_id_to_path.get_first_occurrence(id) {
+                    debug_assert!(path != path_b);
+                    (
+                        linear::MatchOp::Eq {
+                            path_a: path,
+                            path_b,
+                        },
+                        linear::bool_to_match_result(true),
+                    )
+                } else {
+                    (
+                        linear::MatchOp::IsConst { path },
+                        linear::bool_to_match_result(true),
+                    )
+                }
+            }
+            Pattern::Variable(Variable { id, .. }) => {
+                if let Some(path_b) = lhs_id_to_path.get_first_occurrence(id) {
+                    debug_assert!(path != path_b);
+                    (
+                        linear::MatchOp::Eq {
+                            path_a: path,
+                            path_b,
+                        },
+                        linear::bool_to_match_result(true),
+                    )
+                } else {
+                    (linear::MatchOp::Nop, Err(linear::Else))
+                }
+            }
+            Pattern::Operation(op) => {
+                let op = op.operator as u32;
+                debug_assert!(op != 0, "no `Operator` variants are zero");
+                let expected = Ok(unsafe { NonZeroU32::new_unchecked(op) });
+                (linear::MatchOp::Opcode { path }, expected)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use peepmatic_runtime::{
+        integer_interner::IntegerId,
+        linear::{bool_to_match_result, Action::*, Else, MatchOp::*},
+        operator::Operator,
+        r#type::{BitWidth, Kind, Type},
+    };
+
+    macro_rules! linearizes_to {
+        ($name:ident, $source:expr, $make_expected:expr $(,)* ) => {
+            #[test]
+            fn $name() {
+                let buf = wast::parser::ParseBuffer::new($source).expect("should lex OK");
+
+                let opts = match wast::parser::parse::<Optimizations>(&buf) {
+                    Ok(opts) => opts,
+                    Err(mut e) => {
+                        e.set_path(std::path::Path::new(stringify!($name)));
+                        e.set_text($source);
+                        eprintln!("{}", e);
+                        panic!("should parse OK")
+                    }
+                };
+
+                assert_eq!(
+                    opts.optimizations.len(),
+                    1,
+                    "`linearizes_to!` only supports a single optimization; split the big test into \
+                     multiple small tests"
+                );
+
+                if let Err(mut e) = crate::verify(&opts) {
+                    e.set_path(std::path::Path::new(stringify!($name)));
+                    e.set_text($source);
+                    eprintln!("{}", e);
+                    panic!("should verify OK")
+                }
+
+                let mut paths = PathInterner::new();
+                let mut p = |p: &[u8]| paths.intern(Path::new(&p));
+
+                let mut integers = IntegerInterner::new();
+                let mut i = |i: u64| integers.intern(i);
+
+                #[allow(unused_variables)]
+                let make_expected: fn(
+                    &mut dyn FnMut(&[u8]) -> PathId,
+                    &mut dyn FnMut(u64) -> IntegerId,
+                ) -> Vec<linear::Increment> = $make_expected;
+                let expected = make_expected(&mut p, &mut i);
+                dbg!(&expected);
+
+                let actual = linearize_optimization(&mut paths, &mut integers, &opts.optimizations[0]);
+                dbg!(&actual.increments);
+
+                assert_eq!(expected, actual.increments);
+            }
+        };
+    }
+
+    linearizes_to!(
+        mul_by_pow2_into_shift,
+        "
+(=> (when (imul $x $C)
+          (is-power-of-two $C))
+    (ishl $x $C))
+        ",
+        |p, i| vec![
+            linear::Increment {
+                operation: Opcode { path: p(&[0]) },
+                expected: Ok(NonZeroU32::new(Operator::Imul as _).unwrap()),
+                actions: vec![
+                    GetLhs { path: p(&[0, 0]) },
+                    GetLhs { path: p(&[0, 1]) },
+                    MakeBinaryInst {
+                        operator: Operator::Ishl,
+                        r#type: Type {
+                            kind: Kind::Int,
+                            bit_width: BitWidth::Polymorphic,
+                        },
+                        operands: [linear::RhsId(0), linear::RhsId(1)],
+                    },
+                ],
+            },
+            linear::Increment {
+                operation: Nop,
+                expected: Err(Else),
+                actions: vec![],
+            },
+            linear::Increment {
+                operation: IsConst { path: p(&[0, 1]) },
+                expected: bool_to_match_result(true),
+                actions: vec![],
+            },
+            linear::Increment {
+                operation: IsPowerOfTwo { path: p(&[0, 1]) },
+                expected: bool_to_match_result(true),
+                actions: vec![],
+            },
+        ],
+    );
+
+    linearizes_to!(variable_pattern_id_optimization, "(=> $x $x)", |p, i| vec![
+        linear::Increment {
+            operation: Nop,
+            expected: Err(Else),
+            actions: vec![GetLhs { path: p(&[0]) }],
+        }
+    ]);
+
+    linearizes_to!(constant_pattern_id_optimization, "(=> $C $C)", |p, i| vec![
+        linear::Increment {
+            operation: IsConst { path: p(&[0]) },
+            expected: bool_to_match_result(true),
+            actions: vec![GetLhs { path: p(&[0]) }],
+        }
+    ]);
+
+    linearizes_to!(
+        boolean_literal_id_optimization,
+        "(=> true true)",
+        |p, i| vec![linear::Increment {
+            operation: BooleanValue { path: p(&[0]) },
+            expected: bool_to_match_result(true),
+            actions: vec![MakeBooleanConst {
+                value: true,
+                bit_width: BitWidth::Polymorphic,
+            }],
+        }]
+    );
+
+    linearizes_to!(number_literal_id_optimization, "(=> 5 5)", |p, i| vec![
+        linear::Increment {
+            operation: IntegerValue { path: p(&[0]) },
+            expected: Ok(i(5).into()),
+            actions: vec![MakeIntegerConst {
+                value: i(5),
+                bit_width: BitWidth::Polymorphic,
+            }],
+        }
+    ]);
+
+    linearizes_to!(
+        operation_id_optimization,
+        "(=> (iconst $C) (iconst $C))",
+        |p, i| vec![
+            linear::Increment {
+                operation: Opcode { path: p(&[0]) },
+                expected: Ok(NonZeroU32::new(Operator::Iconst as _).unwrap()),
+                actions: vec![
+                    GetLhs { path: p(&[0, 0]) },
+                    MakeUnaryInst {
+                        operator: Operator::Iconst,
+                        r#type: Type {
+                            kind: Kind::Int,
+                            bit_width: BitWidth::Polymorphic,
+                        },
+                        operand: linear::RhsId(0),
+                    },
+                ],
+            },
+            linear::Increment {
+                operation: IsConst { path: p(&[0, 0]) },
+                expected: bool_to_match_result(true),
+                actions: vec![],
+            },
+        ]
+    );
+
+    linearizes_to!(
+        redundant_bor,
+        "(=> (bor $x (bor $x $y)) (bor $x $y))",
+        |p, i| vec![
+            linear::Increment {
+                operation: Opcode { path: p(&[0]) },
+                expected: Ok(NonZeroU32::new(Operator::Bor as _).unwrap()),
+                actions: vec![
+                    GetLhs { path: p(&[0, 0]) },
+                    GetLhs {
+                        path: p(&[0, 1, 1]),
+                    },
+                    MakeBinaryInst {
+                        operator: Operator::Bor,
+                        r#type: Type {
+                            kind: Kind::Int,
+                            bit_width: BitWidth::Polymorphic,
+                        },
+                        operands: [linear::RhsId(0), linear::RhsId(1)],
+                    },
+                ],
+            },
+            linear::Increment {
+                operation: Nop,
+                expected: Err(Else),
+                actions: vec![],
+            },
+            linear::Increment {
+                operation: Opcode { path: p(&[0, 1]) },
+                expected: Ok(NonZeroU32::new(Operator::Bor as _).unwrap()),
+                actions: vec![],
+            },
+            linear::Increment {
+                operation: Eq {
+                    path_a: p(&[0, 1, 0]),
+                    path_b: p(&[0, 0]),
+                },
+                expected: bool_to_match_result(true),
+                actions: vec![],
+            },
+            linear::Increment {
+                operation: Nop,
+                expected: Err(Else),
+                actions: vec![],
+            },
+        ]
+    );
+
+    linearizes_to!(
+        large_integers,
+        // u64::MAX
+        "(=> 18446744073709551615 0)",
+        |p, i| vec![linear::Increment {
+            operation: IntegerValue { path: p(&[0]) },
+            expected: Ok(i(std::u64::MAX).into()),
+            actions: vec![MakeIntegerConst {
+                value: i(0),
+                bit_width: BitWidth::Polymorphic,
+            }],
+        }]
+    );
+
+    linearizes_to!(
+        ireduce_with_type_ascription,
+        "(=> (ireduce{i32} $x) 0)",
+        |p, i| vec![
+            linear::Increment {
+                operation: Opcode { path: p(&[0]) },
+                expected: Ok(NonZeroU32::new(Operator::Ireduce as _).unwrap()),
+                actions: vec![MakeIntegerConst {
+                    value: i(0),
+                    bit_width: BitWidth::ThirtyTwo,
+                }],
+            },
+            linear::Increment {
+                operation: linear::MatchOp::BitWidth { path: p(&[0]) },
+                expected: Ok(NonZeroU32::new(32).unwrap()),
+                actions: vec![],
+            },
+            linear::Increment {
+                operation: Nop,
+                expected: Err(Else),
+                actions: vec![],
+            },
+        ]
+    );
+}

cranelift/peepmatic/src/parser.rs

@@ -0,0 +1,932 @@
+/*!
+
+This module implements parsing the DSL text format. It implements the
+`wast::Parse` trait for all of our AST types.
+
+The grammar for the DSL is given below:
+
+```ebnf
+<optimizations> ::= <optimization>*
+
+<optimization> ::= '(' '=>' <lhs> <rhs> ')'
+
+<left-hand-side> ::= <pattern>
+                   | '(' 'when' <pattern> <precondition>* ')'
+
+<pattern> ::= <value-literal>
+            | <constant>
+            | <operation<pattern>>
+            | <variable>
+
+<value-literal> ::= <integer>
+                  | <boolean>
+
+<boolean> ::= 'true' | 'false'
+
+<operation<T>> ::= '(' <operator> [<type-ascription>] <T>* ')'
+
+<precondition> ::= '(' <constraint> <constraint-operands>* ')'
+
+<constraint-operand> ::= <value-literal>
+                       | <constant>
+                       | <variable>
+
+<rhs> ::= <value-literal>
+        | <constant>
+        | <variable>
+        | <unquote>
+        | <operation<rhs>>
+
+<unquote> ::= '$' '(' <unquote-operator> <unquote-operand>* ')'
+
+<unquote-operand> ::= <value-literal>
+                    | <constant>
+```
+
+ */
+
+use crate::ast::*;
+use peepmatic_runtime::r#type::Type;
+use std::cell::Cell;
+use std::marker::PhantomData;
+use wast::{
+    parser::{Cursor, Parse, Parser, Peek, Result as ParseResult},
+    Id, LParen,
+};
+
+mod tok {
+    use wast::{custom_keyword, custom_reserved};
+
+    custom_keyword!(bit_width = "bit-width");
+    custom_reserved!(dollar = "$");
+    custom_keyword!(r#false = "false");
+    custom_keyword!(fits_in_native_word = "fits-in-native-word");
+    custom_keyword!(is_power_of_two = "is-power-of-two");
+    custom_reserved!(left_curly = "{");
+    custom_keyword!(log2);
+    custom_keyword!(neg);
+    custom_reserved!(replace = "=>");
+    custom_reserved!(right_curly = "}");
+    custom_keyword!(r#true = "true");
+    custom_keyword!(when);
+
+    custom_keyword!(eq);
+    custom_keyword!(ne);
+    custom_keyword!(slt);
+    custom_keyword!(ult);
+    custom_keyword!(sge);
+    custom_keyword!(uge);
+    custom_keyword!(sgt);
+    custom_keyword!(ugt);
+    custom_keyword!(sle);
+    custom_keyword!(ule);
+    custom_keyword!(of);
+    custom_keyword!(nof);
+}
+
+impl<'a> Parse<'a> for Optimizations<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        let mut optimizations = vec![];
+        while !p.is_empty() {
+            optimizations.push(p.parse()?);
+        }
+        Ok(Optimizations {
+            span,
+            optimizations,
+        })
+    }
+}
+
+impl<'a> Parse<'a> for Optimization<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        p.parens(|p| {
+            p.parse::<tok::replace>()?;
+            let lhs = p.parse()?;
+            let rhs = p.parse()?;
+            Ok(Optimization { span, lhs, rhs })
+        })
+    }
+}
+
+impl<'a> Parse<'a> for Lhs<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        let mut preconditions = vec![];
+        if p.peek::<wast::LParen>() && p.peek2::<tok::when>() {
+            p.parens(|p| {
+                p.parse::<tok::when>()?;
+                let pattern = p.parse()?;
+                while p.peek::<LParen>() {
+                    preconditions.push(p.parse()?);
+                }
+                Ok(Lhs {
+                    span,
+                    pattern,
+                    preconditions,
+                })
+            })
+        } else {
+            let span = p.cur_span();
+            let pattern = p.parse()?;
+            Ok(Lhs {
+                span,
+                pattern,
+                preconditions,
+            })
+        }
+    }
+}
+
+impl<'a> Parse<'a> for Pattern<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        if p.peek::<ValueLiteral>() {
+            return Ok(Pattern::ValueLiteral(p.parse()?));
+        }
+        if p.peek::<Constant>() {
+            return Ok(Pattern::Constant(p.parse()?));
+        }
+        if p.peek::<Operation<Self>>() {
+            return Ok(Pattern::Operation(p.parse()?));
+        }
+        if p.peek::<Variable>() {
+            return Ok(Pattern::Variable(p.parse()?));
+        }
+        Err(p.error("expected a left-hand side pattern"))
+    }
+}
+
+impl<'a> Peek for Pattern<'a> {
+    fn peek(c: Cursor) -> bool {
+        ValueLiteral::peek(c)
+            || Constant::peek(c)
+            || Variable::peek(c)
+            || Operation::<Self>::peek(c)
+    }
+
+    fn display() -> &'static str {
+        "left-hand side pattern"
+    }
+}
+
+impl<'a> Parse<'a> for ValueLiteral<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        if let Ok(b) = p.parse::<Boolean>() {
+            return Ok(ValueLiteral::Boolean(b));
+        }
+        if let Ok(i) = p.parse::<Integer>() {
+            return Ok(ValueLiteral::Integer(i));
+        }
+        if let Ok(cc) = p.parse::<ConditionCode>() {
+            return Ok(ValueLiteral::ConditionCode(cc));
+        }
+        Err(p.error("expected an integer or boolean or condition code literal"))
+    }
+}
+
+impl<'a> Peek for ValueLiteral<'a> {
+    fn peek(c: Cursor) -> bool {
+        c.integer().is_some() || Boolean::peek(c) || ConditionCode::peek(c)
+    }
+
+    fn display() -> &'static str {
+        "value literal"
+    }
+}
+
+impl<'a> Parse<'a> for Integer<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        p.step(|c| {
+            if let Some((i, rest)) = c.integer() {
+                let (s, base) = i.val();
+                let val = i64::from_str_radix(s, base)
+                    .or_else(|_| u128::from_str_radix(s, base).map(|i| i as i64));
+                return match val {
+                    Ok(value) => Ok((
+                        Integer {
+                            span,
+                            value,
+                            bit_width: Default::default(),
+                            marker: PhantomData,
+                        },
+                        rest,
+                    )),
+                    Err(_) => Err(c.error("invalid integer: out of range")),
+                };
+            }
+            Err(c.error("expected an integer"))
+        })
+    }
+}
+
+impl<'a> Parse<'a> for Boolean<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        if p.parse::<tok::r#true>().is_ok() {
+            return Ok(Boolean {
+                span,
+                value: true,
+                bit_width: Default::default(),
+                marker: PhantomData,
+            });
+        }
+        if p.parse::<tok::r#false>().is_ok() {
+            return Ok(Boolean {
+                span,
+                value: false,
+                bit_width: Default::default(),
+                marker: PhantomData,
+            });
+        }
+        Err(p.error("expected `true` or `false`"))
+    }
+}
+
+impl<'a> Peek for Boolean<'a> {
+    fn peek(c: Cursor) -> bool {
+        <tok::r#true as Peek>::peek(c) || <tok::r#false as Peek>::peek(c)
+    }
+
+    fn display() -> &'static str {
+        "boolean `true` or `false`"
+    }
+}
+
+impl<'a> Parse<'a> for ConditionCode<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+
+        macro_rules! parse_cc {
+            ( $( $token:ident => $cc:ident, )* ) => {
+                $(
+                    if p.peek::<tok::$token>() {
+                        p.parse::<tok::$token>()?;
+                        return Ok(Self {
+                            span,
+                            cc: peepmatic_runtime::cc::ConditionCode::$cc,
+                            marker: PhantomData,
+                        });
+                    }
+                )*
+            }
+        }
+
+        parse_cc! {
+            eq => Eq,
+            ne => Ne,
+            slt => Slt,
+            ult => Ult,
+            sge => Sge,
+            uge => Uge,
+            sgt => Sgt,
+            ugt => Ugt,
+            sle => Sle,
+            ule => Ule,
+            of => Of,
+            nof => Nof,
+        }
+
+        Err(p.error("expected a condition code"))
+    }
+}
+
+impl<'a> Peek for ConditionCode<'a> {
+    fn peek(c: Cursor) -> bool {
+        macro_rules! peek_cc {
+            ( $( $token:ident, )* ) => {
+                false $( || <tok::$token as Peek>::peek(c) )*
+            }
+        }
+
+        peek_cc! {
+            eq,
+            ne,
+            slt,
+            ult,
+            sge,
+            uge,
+            sgt,
+            ugt,
+            sle,
+            ule,
+            of,
+            nof,
+        }
+    }
+
+    fn display() -> &'static str {
+        "condition code"
+    }
+}
+
+impl<'a> Parse<'a> for Constant<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        let id = Id::parse(p)?;
+        if id
+            .name()
+            .chars()
+            .all(|c| !c.is_alphabetic() || c.is_uppercase())
+        {
+            Ok(Constant { span, id })
+        } else {
+            let upper = id
+                .name()
+                .chars()
+                .flat_map(|c| c.to_uppercase())
+                .collect::<String>();
+            Err(p.error(format!(
+                "symbolic constants must start with an upper-case letter like ${}",
+                upper
+            )))
+        }
+    }
+}
+
+impl<'a> Peek for Constant<'a> {
+    fn peek(c: Cursor) -> bool {
+        if let Some((id, _rest)) = c.id() {
+            id.chars().all(|c| !c.is_alphabetic() || c.is_uppercase())
+        } else {
+            false
+        }
+    }
+
+    fn display() -> &'static str {
+        "symbolic constant"
+    }
+}
+
+impl<'a> Parse<'a> for Variable<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        let id = Id::parse(p)?;
+        if id
+            .name()
+            .chars()
+            .all(|c| !c.is_alphabetic() || c.is_lowercase())
+        {
+            Ok(Variable { span, id })
+        } else {
+            let lower = id
+                .name()
+                .chars()
+                .flat_map(|c| c.to_lowercase())
+                .collect::<String>();
+            Err(p.error(format!(
+                "variables must start with an lower-case letter like ${}",
+                lower
+            )))
+        }
+    }
+}
+
+impl<'a> Peek for Variable<'a> {
+    fn peek(c: Cursor) -> bool {
+        if let Some((id, _rest)) = c.id() {
+            id.chars().all(|c| !c.is_alphabetic() || c.is_lowercase())
+        } else {
+            false
+        }
+    }
+
+    fn display() -> &'static str {
+        "variable"
+    }
+}
+
+impl<'a, T> Parse<'a> for Operation<'a, T>
+where
+    T: 'a + Ast<'a> + Peek + Parse<'a>,
+    DynAstRef<'a>: From<&'a T>,
+{
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        p.parens(|p| {
+            let operator = p.parse()?;
+
+            let r#type = Cell::new(if p.peek::<tok::left_curly>() {
+                p.parse::<tok::left_curly>()?;
+                let ty = p.parse::<Type>()?;
+                p.parse::<tok::right_curly>()?;
+                Some(ty)
+            } else {
+                None
+            });
+
+            let mut operands = vec![];
+            while p.peek::<T>() {
+                operands.push(p.parse()?);
+            }
+            Ok(Operation {
+                span,
+                operator,
+                r#type,
+                operands,
+                marker: PhantomData,
+            })
+        })
+    }
+}
+
+impl<'a, T> Peek for Operation<'a, T>
+where
+    T: 'a + Ast<'a>,
+    DynAstRef<'a>: From<&'a T>,
+{
+    fn peek(c: Cursor) -> bool {
+        wast::LParen::peek(c)
+    }
+
+    fn display() -> &'static str {
+        "operation"
+    }
+}
+
+impl<'a> Parse<'a> for Precondition<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        p.parens(|p| {
+            let constraint = p.parse()?;
+            let mut operands = vec![];
+            while p.peek::<ConstraintOperand>() {
+                operands.push(p.parse()?);
+            }
+            Ok(Precondition {
+                span,
+                constraint,
+                operands,
+            })
+        })
+    }
+}
+
+impl<'a> Parse<'a> for Constraint {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        if p.peek::<tok::is_power_of_two>() {
+            p.parse::<tok::is_power_of_two>()?;
+            return Ok(Constraint::IsPowerOfTwo);
+        }
+        if p.peek::<tok::bit_width>() {
+            p.parse::<tok::bit_width>()?;
+            return Ok(Constraint::BitWidth);
+        }
+        if p.peek::<tok::fits_in_native_word>() {
+            p.parse::<tok::fits_in_native_word>()?;
+            return Ok(Constraint::FitsInNativeWord);
+        }
+        Err(p.error("expected a precondition constraint"))
+    }
+}
+
+impl<'a> Parse<'a> for ConstraintOperand<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        if p.peek::<ValueLiteral>() {
+            return Ok(ConstraintOperand::ValueLiteral(p.parse()?));
+        }
+        if p.peek::<Constant>() {
+            return Ok(ConstraintOperand::Constant(p.parse()?));
+        }
+        if p.peek::<Variable>() {
+            return Ok(ConstraintOperand::Variable(p.parse()?));
+        }
+        Err(p.error("expected an operand for precondition constraint"))
+    }
+}
+
+impl<'a> Peek for ConstraintOperand<'a> {
+    fn peek(c: Cursor) -> bool {
+        ValueLiteral::peek(c) || Constant::peek(c) || Variable::peek(c)
+    }
+
+    fn display() -> &'static str {
+        "operand for a precondition constraint"
+    }
+}
+
+impl<'a> Parse<'a> for Rhs<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        if p.peek::<ValueLiteral>() {
+            return Ok(Rhs::ValueLiteral(p.parse()?));
+        }
+        if p.peek::<Constant>() {
+            return Ok(Rhs::Constant(p.parse()?));
+        }
+        if p.peek::<Variable>() {
+            return Ok(Rhs::Variable(p.parse()?));
+        }
+        if p.peek::<Unquote>() {
+            return Ok(Rhs::Unquote(p.parse()?));
+        }
+        if p.peek::<Operation<Self>>() {
+            return Ok(Rhs::Operation(p.parse()?));
+        }
+        Err(p.error("expected a right-hand side replacement"))
+    }
+}
+
+impl<'a> Peek for Rhs<'a> {
+    fn peek(c: Cursor) -> bool {
+        ValueLiteral::peek(c)
+            || Constant::peek(c)
+            || Variable::peek(c)
+            || Unquote::peek(c)
+            || Operation::<Self>::peek(c)
+    }
+
+    fn display() -> &'static str {
+        "right-hand side replacement"
+    }
+}
+
+impl<'a> Parse<'a> for Unquote<'a> {
+    fn parse(p: Parser<'a>) -> ParseResult<Self> {
+        let span = p.cur_span();
+        p.parse::<tok::dollar>()?;
+        p.parens(|p| {
+            let operator = p.parse()?;
+            let mut operands = vec![];
+            while p.peek::<Rhs>() {
+                operands.push(p.parse()?);
+            }
+            Ok(Unquote {
+                span,
+                operator,
+                operands,
+            })
+        })
+    }
+}
+
+impl<'a> Peek for Unquote<'a> {
+    fn peek(c: Cursor) -> bool {
+        tok::dollar::peek(c)
+    }
+
+    fn display() -> &'static str {
+        "unquote expression"
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use peepmatic_runtime::operator::Operator;
+
+    macro_rules! test_parse {
+        (
+            $(
+                $name:ident < $ast:ty > {
+                    $( ok { $( $ok:expr , )* } )*
+                    $( err { $( $err:expr , )* } )*
+                }
+            )*
+        ) => {
+            $(
+                #[test]
+                #[allow(non_snake_case)]
+                fn $name() {
+                    $(
+                        $({
+                            let input = $ok;
+                            let buf = wast::parser::ParseBuffer::new(input).unwrap_or_else(|e| {
+                                panic!("should lex OK, got error:\n\n{}\n\nInput:\n\n{}", e, input)
+                            });
+                            if let Err(e) = wast::parser::parse::<$ast>(&buf) {
+                                panic!(
+                                    "expected to parse OK, got error:\n\n{}\n\nInput:\n\n{}",
+                                    e, input
+                                );
+                            }
+                        })*
+                    )*
+
+                    $(
+                        $({
+                            let input = $err;
+                            let buf = wast::parser::ParseBuffer::new(input).unwrap_or_else(|e| {
+                                panic!("should lex OK, got error:\n\n{}\n\nInput:\n\n{}", e, input)
+                            });
+                            if let Ok(ast) = wast::parser::parse::<$ast>(&buf) {
+                                panic!(
+                                    "expected a parse error, got:\n\n{:?}\n\nInput:\n\n{}",
+                                    ast, input
+                                );
+                            }
+                        })*
+                    )*
+                }
+            )*
+        }
+    }
+
+    test_parse! {
+        parse_boolean<Boolean> {
+            ok {
+                "true",
+                "false",
+            }
+            err {
+                "",
+                "t",
+                "tr",
+                "tru",
+                "truezzz",
+                "f",
+                "fa",
+                "fal",
+                "fals",
+                "falsezzz",
+            }
+        }
+        parse_cc<ConditionCode> {
+            ok {
+                "eq",
+                "ne",
+                "slt",
+                "ult",
+                "sge",
+                "uge",
+                "sgt",
+                "ugt",
+                "sle",
+                "ule",
+                "of",
+                "nof",
+            }
+            err {
+                "",
+                "neq",
+            }
+        }
+        parse_constant<Constant> {
+            ok {
+                "$C",
+                "$C1",
+                "$C2",
+                "$X",
+                "$Y",
+                "$SOME-CONSTANT",
+                "$SOME_CONSTANT",
+            }
+            err {
+                "",
+                "zzz",
+                "$",
+                "$variable",
+                "$Some-Constant",
+                "$Some_Constant",
+                "$Some_constant",
+            }
+        }
+        parse_constraint<Constraint> {
+            ok {
+                "is-power-of-two",
+                "bit-width",
+                "fits-in-native-word",
+            }
+            err {
+                "",
+                "iadd",
+                "imul",
+            }
+        }
+        parse_constraint_operand<ConstraintOperand> {
+            ok {
+                "1234",
+                "true",
+                "$CONSTANT",
+                "$variable",
+            }
+            err {
+                "",
+                "is-power-of-two",
+                "(is-power-of-two $C)",
+                "(iadd 1 2)",
+            }
+        }
+        parse_integer<Integer> {
+            ok {
+                "0",
+                "1",
+                "12",
+                "123",
+                "1234",
+                "12345",
+                "123456",
+                "1234567",
+                "12345678",
+                "123456789",
+                "1234567890",
+                "0x0",
+                "0x1",
+                "0x12",
+                "0x123",
+                "0x1234",
+                "0x12345",
+                "0x123456",
+                "0x1234567",
+                "0x12345678",
+                "0x123456789",
+                "0x123456789a",
+                "0x123456789ab",
+                "0x123456789abc",
+                "0x123456789abcd",
+                "0x123456789abcde",
+                "0x123456789abcdef",
+                "0xffff_ffff_ffff_ffff",
+            }
+            err {
+                "",
+                "abcdef",
+                "01234567890abcdef",
+                "0xgggg",
+                "0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+            }
+        }
+        parse_lhs<Lhs> {
+            ok {
+                "(when (imul $C1 $C2) (is-power-of-two $C1) (is-power-of-two $C2))",
+                "(when (imul $x $C) (is-power-of-two $C))",
+                "(imul $x $y)",
+                "(imul $x)",
+                "(imul)",
+                "$C",
+                "$x",
+            }
+            err {
+                "",
+                "()",
+                "abc",
+            }
+        }
+        parse_operation_pattern<Operation<Pattern>> {
+            ok {
+                "(iadd)",
+                "(iadd 1)",
+                "(iadd 1 2)",
+                "(iadd $x $C)",
+                "(iadd{i32} $x $y)",
+                "(icmp eq $x $y)",
+            }
+            err {
+                "",
+                "()",
+                "$var",
+                "$CONST",
+                "(ishl $x $(log2 $C))",
+            }
+        }
+        parse_operation_rhs<Operation<Rhs>> {
+            ok {
+                "(iadd)",
+                "(iadd 1)",
+                "(iadd 1 2)",
+                "(ishl $x $(log2 $C))",
+            }
+            err {
+                "",
+                "()",
+                "$var",
+                "$CONST",
+            }
+        }
+        parse_operator<Operator> {
+            ok {
+                "bor",
+                "iadd",
+                "iadd_imm",
+                "iconst",
+                "imul",
+                "imul_imm",
+                "ishl",
+                "sdiv",
+                "sdiv_imm",
+                "sshr",
+            }
+            err {
+                "",
+                "iadd.i32",
+                "iadd{i32}",
+            }
+        }
+        parse_optimization<Optimization> {
+            ok {
+                "(=> (when (iadd $x $C) (is-power-of-two $C) (is-power-of-two $C)) (iadd $C $x))",
+                "(=> (when (iadd $x $C)) (iadd $C $x))",
+                "(=> (iadd $x $C) (iadd $C $x))",
+            }
+            err {
+                "",
+                "()",
+                "(=>)",
+                "(=> () ())",
+            }
+        }
+        parse_optimizations<Optimizations> {
+            ok {
+                "",
+                r#"
+                ;; Canonicalize `a + (b + c)` into `(a + b) + c`.
+                (=> (iadd $a (iadd $b $c))
+                    (iadd (iadd $a $b) $c))
+
+                ;; Combine a `const` and an `iadd` into a `iadd_imm`.
+                (=> (iadd (iconst $C) $x)
+                    (iadd_imm $C $x))
+
+                ;; When `C` is a power of two, replace `x * C` with `x << log2(C)`.
+                (=> (when (imul $x $C)
+                          (is-power-of-two $C))
+                    (ishl $x $(log2 $C)))
+                "#,
+            }
+        }
+        parse_pattern<Pattern> {
+            ok {
+                "1234",
+                "$C",
+                "$x",
+                "(iadd $x $y)",
+            }
+            err {
+                "",
+                "()",
+                "abc",
+            }
+        }
+        parse_precondition<Precondition> {
+            ok {
+                "(is-power-of-two)",
+                "(is-power-of-two $C)",
+                "(is-power-of-two $C1 $C2)",
+            }
+            err {
+                "",
+                "1234",
+                "()",
+                "$var",
+                "$CONST",
+            }
+        }
+        parse_rhs<Rhs> {
+            ok {
+                "5",
+                "$C",
+                "$x",
+                "$(log2 $C)",
+                "(iadd $x 1)",
+            }
+            err {
+                "",
+                "()",
+            }
+        }
+        parse_unquote<Unquote> {
+            ok {
+                "$(log2)",
+                "$(log2 $C)",
+                "$(log2 $C1 1)",
+                "$(neg)",
+                "$(neg $C)",
+                "$(neg $C 1)",
+            }
+            err {
+                "",
+                "(log2 $C)",
+                "$()",
+            }
+        }
+        parse_value_literal<ValueLiteral> {
+            ok {
+                "12345",
+                "true",
+            }
+            err {
+                "",
+                "'c'",
+                "\"hello\"",
+                "12.34",
+            }
+        }
+        parse_variable<Variable> {
+            ok {
+                "$v",
+                "$v1",
+                "$v2",
+                "$x",
+                "$y",
+                "$some-var",
+                "$another_var",
+            }
+            err {
+                "zzz",
+                "$",
+                "$CONSTANT",
+                "$fooBar",
+            }
+        }
+    }
+}

cranelift/peepmatic/src/traversals.rs

@@ -0,0 +1,278 @@
+//! Traversals over the AST.
+
+use crate::ast::*;
+
+/// A low-level DFS traversal event: either entering or exiting the traversal of
+/// an AST node.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum TraversalEvent {
+    /// Entering traversal of an AST node.
+    ///
+    /// Processing an AST node upon this event corresponds to a pre-order
+    /// DFS traversal.
+    Enter,
+
+    /// Exiting traversal of an AST node.
+    ///
+    /// Processing an AST node upon this event corresponds to a post-order DFS
+    /// traversal.
+    Exit,
+}
+
+/// A depth-first traversal of an AST.
+///
+/// This is a fairly low-level traversal type, and is intended to be used as a
+/// building block for making specific pre-order or post-order traversals for
+/// whatever problem is at hand.
+///
+/// This implementation is not recursive, and exposes an `Iterator` interface
+/// that yields pairs of `(TraversalEvent, DynAstRef)` items.
+///
+/// The traversal can walk a whole set of `Optimization`s or just a subtree of
+/// the AST, because the `new` constructor takes anything that can convert into
+/// a `DynAstRef`.
+#[derive(Debug, Clone)]
+pub struct Dfs<'a> {
+    stack: Vec<(TraversalEvent, DynAstRef<'a>)>,
+}
+
+impl<'a> Dfs<'a> {
+    /// Construct a new `Dfs` traversal starting at the given `start` AST node.
+    pub fn new(start: impl Into<DynAstRef<'a>>) -> Self {
+        let start = start.into();
+        Dfs {
+            stack: vec![
+                (TraversalEvent::Exit, start),
+                (TraversalEvent::Enter, start),
+            ],
+        }
+    }
+
+    /// Peek at the next traversal event and AST node pair, if any.
+    pub fn peek(&self) -> Option<(TraversalEvent, DynAstRef<'a>)> {
+        self.stack.last().cloned()
+    }
+}
+
+impl<'a> Iterator for Dfs<'a> {
+    type Item = (TraversalEvent, DynAstRef<'a>);
+
+    fn next(&mut self) -> Option<(TraversalEvent, DynAstRef<'a>)> {
+        let (event, node) = self.stack.pop()?;
+        if let TraversalEvent::Enter = event {
+            let mut enqueue_children = EnqueueChildren(self);
+            node.child_nodes(&mut enqueue_children)
+        }
+        return Some((event, node));
+
+        struct EnqueueChildren<'a, 'b>(&'b mut Dfs<'a>)
+        where
+            'a: 'b;
+
+        impl<'a, 'b> Extend<DynAstRef<'a>> for EnqueueChildren<'a, 'b>
+        where
+            'a: 'b,
+        {
+            fn extend<T: IntoIterator<Item = DynAstRef<'a>>>(&mut self, iter: T) {
+                let iter = iter.into_iter();
+
+                let (min, max) = iter.size_hint();
+                self.0.stack.reserve(max.unwrap_or(min) * 2);
+
+                let start = self.0.stack.len();
+
+                for node in iter {
+                    self.0.stack.push((TraversalEvent::Enter, node));
+                    self.0.stack.push((TraversalEvent::Exit, node));
+                }
+
+                // Reverse to make it so that we visit children in order
+                // (e.g. operands are visited in order).
+                self.0.stack[start..].reverse();
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use DynAstRef::*;
+
+    #[test]
+    fn test_dfs_traversal() {
+        let input = "
+(=> (when (imul $x $C)
+          (is-power-of-two $C))
+    (ishl $x $(log2 $C)))
+";
+        let buf = wast::parser::ParseBuffer::new(input).expect("input should lex OK");
+        let ast = match wast::parser::parse::<crate::ast::Optimizations>(&buf) {
+            Ok(ast) => ast,
+            Err(e) => panic!("expected to parse OK, got error:\n\n{}", e),
+        };
+
+        let mut dfs = Dfs::new(&ast);
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Optimizations(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Optimization(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Lhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Pattern(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, PatternOperation(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Pattern(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Variable(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Variable(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Pattern(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Pattern(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Constant(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Constant(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Pattern(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, PatternOperation(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Pattern(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Precondition(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, ConstraintOperand(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Constant(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Constant(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, ConstraintOperand(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Precondition(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Lhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Rhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, RhsOperation(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Rhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Variable(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Variable(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Rhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Rhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Unquote(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Rhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Enter, Constant(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Constant(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Rhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Unquote(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Rhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, RhsOperation(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Rhs(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Optimization(..)))
+        ));
+        assert!(matches!(
+            dbg!(dfs.next()),
+            Some((TraversalEvent::Exit, Optimizations(..)))
+        ));
+        assert!(dfs.next().is_none());
+    }
+}

fuzz/Cargo.toml

@@ -14,6 +14,7 @@ cranelift-reader = { path = "../cranelift/reader" }
 cranelift-wasm = { path = "../cranelift/wasm" }
 libfuzzer-sys = "0.3.2"
 target-lexicon = "0.10"
+peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing" }
 wasmtime = { path = "../crates/wasmtime" }
 wasmtime-fuzzing = { path = "../crates/fuzzing" }
 
@@ -56,5 +57,35 @@ path = "fuzz_targets/spectests.rs"
 test = false
 doc = false
 
+[[bin]]
+name = "peepmatic_simple_automata"
+path = "fuzz_targets/peepmatic_simple_automata.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "peepmatic_fst_differential"
+path = "fuzz_targets/peepmatic_fst_differential.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "peepmatic_parser"
+path = "fuzz_targets/peepmatic_parser.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "peepmatic_compile"
+path = "fuzz_targets/peepmatic_compile.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "peepmatic_interp"
+path = "fuzz_targets/peepmatic_interp.rs"
+test = false
+doc = false
+
 [features]
 binaryen = ['wasmtime-fuzzing/binaryen']

fuzz/fuzz_targets/peepmatic_compile.rs

@@ -0,0 +1,8 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use peepmatic_fuzzing::compile::compile;
+
+fuzz_target!(|data: &[u8]| {
+    compile(data);
+});

fuzz/fuzz_targets/peepmatic_fst_differential.rs

@@ -0,0 +1,8 @@
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+use peepmatic_fuzzing::automata::fst_differential;
+use std::collections::HashMap;
+
+fuzz_target!(|map: HashMap<Vec<u8>, u64>| {
+    fst_differential(map);
+});

fuzz/fuzz_targets/peepmatic_interp.rs

@@ -0,0 +1,8 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use peepmatic_fuzzing::interp::interp;
+
+fuzz_target!(|data: &[u8]| {
+    interp(data);
+});

fuzz/fuzz_targets/peepmatic_parser.rs

@@ -0,0 +1,8 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use peepmatic_fuzzing::parser::parse;
+
+fuzz_target!(|data: &[u8]| {
+    parse(data);
+});

fuzz/fuzz_targets/peepmatic_simple_automata.rs

@@ -0,0 +1,7 @@
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+use peepmatic_fuzzing::automata::simple_automata;
+
+fuzz_target!(|input_output_pairs: Vec<Vec<(u8, Vec<u8>)>>| {
+    simple_automata(input_output_pairs);
+});