diff --git a/.gitattributes b/.gitattributes
index df94c94722..928feac126 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -5,3 +5,10 @@
 *.png binary
 *.ico binary
 *.wasm binary
+
+# ISLE should use lisp syntax highlighting.
+*.isle linguist-language=lisp
+
+# Tell GitHub this is generated code, and doesn't need to be shown in diffs by
+# default.
+cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs linguist-generated
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3f7b2f01c9..b880bfafb3 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -189,6 +189,21 @@ jobs:
       working-directory: ./fuzz
     - run: cargo fuzz build --dev
 
+  rebuild_isle:
+    name: Rebuild ISLE
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: ./.github/actions/install-rust
+    - name: Rebuild ISLE DSL files
+      run: cargo build -p cranelift-codegen --features "rebuild-isle"
+    - name: Reformat
+      run: cargo fmt -p cranelift-codegen
+    - name: Check that the ISLE DSL files are up-to-date
+      run: git diff --exit-code
+
   rebuild_peephole_optimizers:
     name: Rebuild Peephole Optimizers
     runs-on: ubuntu-latest
diff --git a/Cargo.lock b/Cargo.lock
index 85405f14b2..7ff21a314a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -461,7 +461,7 @@ dependencies = [
  "atty",
  "bitflags",
  "strsim",
- "textwrap",
+ "textwrap 0.11.0",
  "unicode-width",
  "vec_map",
 ]
@@ -551,7 +551,9 @@ dependencies = [
  "criterion",
  "gimli",
  "hashbrown",
+ "isle",
  "log",
+ "miette",
  "peepmatic",
  "peepmatic-runtime",
  "peepmatic-traits",
@@ -1093,6 +1095,15 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "env_logger"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3"
+dependencies = [
+ "log",
+]
+
 [[package]]
 name = "errno"
 version = "0.2.8"
@@ -1422,6 +1433,42 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135"
 
+[[package]]
+name = "is_ci"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb"
+
+[[package]]
+name = "isle"
+version = "0.78.0"
+dependencies = [
+ "log",
+ "miette",
+ "thiserror",
+]
+
+[[package]]
+name = "isle-fuzz"
+version = "0.0.0"
+dependencies = [
+ "env_logger 0.9.0",
+ "isle",
+ "libfuzzer-sys",
+ "log",
+]
+
+[[package]]
+name = "islec"
+version = "0.1.0"
+dependencies = [
+ "env_logger 0.8.3",
+ "isle",
+ "log",
+ "miette",
+ "structopt",
+]
+
 [[package]]
 name = "itertools"
 version = "0.9.0"
@@ -1608,6 +1655,36 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882"
 
+[[package]]
+name = "miette"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec47e61dc212c43f44dcd1f2841ccba79c6ec10da357cab7a7859b5f87bd27a9"
+dependencies = [
+ "atty",
+ "backtrace",
+ "miette-derive",
+ "once_cell",
+ "owo-colors",
+ "supports-color",
+ "supports-hyperlinks",
+ "supports-unicode",
+ "term_size",
+ "textwrap 0.14.2",
+ "thiserror",
+]
+
+[[package]]
+name = "miette-derive"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c0f0b6f999b9a9f7e86322125583a437cf015054b7aaa9926dff0ff13005b7e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.4.4"
@@ -1811,9 +1888,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.7.2"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
+checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
 
 [[package]]
 name = "oorandom"
@@ -1865,6 +1942,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "owo-colors"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a61765925aec40abdb23812a3a1a01fafc6ffb9da22768b2ce665a9e84e527c"
+
 [[package]]
 name = "p256"
 version = "0.9.0"
@@ -2674,6 +2757,12 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
 
+[[package]]
+name = "smawk"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043"
+
 [[package]]
 name = "souper-ir"
 version = "2.1.0"
@@ -2718,9 +2807,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "structopt"
-version = "0.3.21"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c"
+checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"
 dependencies = [
  "clap",
  "lazy_static",
@@ -2729,9 +2818,9 @@ dependencies = [
 
 [[package]]
 name = "structopt-derive"
-version = "0.4.14"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90"
+checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -2746,6 +2835,34 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
 
+[[package]]
+name = "supports-color"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4872ced36b91d47bae8a214a683fe54e7078875b399dfa251df346c9b547d1f9"
+dependencies = [
+ "atty",
+ "is_ci",
+]
+
+[[package]]
+name = "supports-hyperlinks"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "590b34f7c5f01ecc9d78dba4b3f445f31df750a67621cf31626f3b7441ce6406"
+dependencies = [
+ "atty",
+]
+
+[[package]]
+name = "supports-unicode"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8b945e45b417b125a8ec51f1b7df2f8df7920367700d1f98aedd21e5735f8b2"
+dependencies = [
+ "atty",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.72"
@@ -2812,6 +2929,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "term_size"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.2"
@@ -2860,19 +2987,30 @@ dependencies = [
 ]
 
 [[package]]
-name = "thiserror"
-version = "1.0.25"
+name = "textwrap"
+version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6"
+checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
+dependencies = [
+ "smawk",
+ "unicode-linebreak",
+ "unicode-width",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.25"
+version = "1.0.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d"
+checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2999,6 +3137,15 @@ version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06"
 
+[[package]]
+name = "unicode-linebreak"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a52dcaab0c48d931f7cc8ef826fa51690a08e1ea55117ef26f89864f532383f"
+dependencies = [
+ "regex",
+]
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.7.1"
diff --git a/Cargo.toml b/Cargo.toml
index 50cd622a93..0dc3964444 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -73,6 +73,8 @@ opt-level = 0
 resolver = '2'
 members = [
   "cranelift",
+  "cranelift/isle/fuzz",
+  "cranelift/isle/islec",
   "cranelift/serde",
   "crates/bench-api",
   "crates/c-api",
diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml
index 5bc01a92ce..86306d7f5c 100644
--- a/cranelift/codegen/Cargo.toml
+++ b/cranelift/codegen/Cargo.toml
@@ -39,6 +39,8 @@ criterion = "0.3"
 
 [build-dependencies]
 cranelift-codegen-meta = { path = "meta", version = "0.78.0" }
+isle = { path = "../isle/isle", version = "0.78.0", optional = true }
+miette = { version = "3", features = ["fancy"] }
 
 [features]
 default = ["std", "unwind"]
@@ -98,6 +100,9 @@ enable-peepmatic = ["peepmatic-runtime", "peepmatic-traits", "serde"]
 # Enable support for the Souper harvester.
 souper-harvest = ["souper-ir", "souper-ir/stringify"]
 
+# Recompile ISLE DSL source files into their generated Rust code.
+rebuild-isle = ["isle", "cranelift-codegen-meta/rebuild-isle"]
+
 [badges]
 maintenance = { status = "experimental" }
 
diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs
index 51fcf4cbaa..47b63c50a1 100644
--- a/cranelift/codegen/build.rs
+++ b/cranelift/codegen/build.rs
@@ -46,9 +46,12 @@ fn main() {
         isa_targets
     };
 
+    let cur_dir = env::current_dir().expect("Can't access current working directory");
+    let crate_dir = cur_dir.as_path();
+
     println!("cargo:rerun-if-changed=build.rs");
 
-    if let Err(err) = meta::generate(&isas, &out_dir) {
+    if let Err(err) = meta::generate(&isas, &out_dir, crate_dir) {
         eprintln!("Error: {}", err);
         process::exit(1);
     }
@@ -74,6 +77,19 @@ fn main() {
         .unwrap()
     }
 
+    #[cfg(feature = "rebuild-isle")]
+    {
+        if let Err(e) = rebuild_isle(crate_dir) {
+            eprintln!("Error building ISLE files: {:?}", e);
+            let mut source = e.source();
+            while let Some(e) = source {
+                eprintln!("{:?}", e);
+                source = e.source();
+            }
+            std::process::abort();
+        }
+    }
+
     let pkg_version = env::var("CARGO_PKG_VERSION").unwrap();
     let mut cmd = std::process::Command::new("git");
     cmd.arg("rev-parse")
@@ -110,3 +126,136 @@ fn main() {
     )
     .unwrap();
 }
+
+/// Rebuild ISLE DSL source text into generated Rust code.
+///
+/// NB: This must happen *after* the `cranelift-codegen-meta` functions, since
+/// it consumes files generated by them.
+#[cfg(feature = "rebuild-isle")]
+fn rebuild_isle(crate_dir: &std::path::Path) -> Result<(), Box<dyn std::error::Error + 'static>> {
+    use std::sync::Once;
+    static SET_MIETTE_HOOK: Once = Once::new();
+    SET_MIETTE_HOOK.call_once(|| {
+        let _ = miette::set_hook(Box::new(|_| {
+            Box::new(
+                miette::MietteHandlerOpts::new()
+                    // This is necessary for `miette` to properly display errors
+                    // until https://github.com/zkat/miette/issues/93 is fixed.
+                    .force_graphical(true)
+                    .build(),
+            )
+        }));
+    });
+
+    let clif_isle = crate_dir.join("src").join("clif.isle");
+    let prelude_isle = crate_dir.join("src").join("prelude.isle");
+    let src_isa_x64 = crate_dir.join("src").join("isa").join("x64");
+
+    // This is a set of ISLE compilation units.
+    //
+    // The format of each entry is:
+    //
+    //     (output Rust code file, input ISLE source files)
+    //
+    // There should be one entry for each backend that uses ISLE for lowering,
+    // and if/when we replace our peephole optimization passes with ISLE, there
+    // should be an entry for each of those as well.
+    let isle_compilations = vec![
+        // The x86-64 instruction selector.
+        (
+            src_isa_x64
+                .join("lower")
+                .join("isle")
+                .join("generated_code.rs"),
+            vec![
+                clif_isle,
+                prelude_isle,
+                src_isa_x64.join("inst.isle"),
+                src_isa_x64.join("lower.isle"),
+            ],
+        ),
+    ];
+
+    let cur_dir = std::env::current_dir()?;
+    for (out_file, mut files) in isle_compilations {
+        for file in files.iter_mut() {
+            println!("cargo:rerun-if-changed={}", file.display());
+
+            // Strip the current directory from the file paths, because `islec`
+            // includes them in the generated source, and this helps us maintain
+            // deterministic builds that don't include those local file paths.
+            if let Ok(suffix) = file.strip_prefix(&cur_dir) {
+                *file = suffix.to_path_buf();
+            }
+        }
+
+        let code = (|| {
+            let lexer = isle::lexer::Lexer::from_files(files)?;
+            let defs = isle::parser::parse(lexer)?;
+            isle::compile::compile(&defs)
+        })()
+        .map_err(|e| {
+            // Make sure to include the source snippets location info along with
+            // the error messages.
+
+            let report = miette::Report::new(e);
+            return DebugReport(report);
+
+            struct DebugReport(miette::Report);
+
+            impl std::fmt::Display for DebugReport {
+                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+                    self.0.handler().debug(&*self.0, f)
+                }
+            }
+
+            impl std::fmt::Debug for DebugReport {
+                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+                    std::fmt::Display::fmt(self, f)
+                }
+            }
+
+            impl std::error::Error for DebugReport {}
+        })?;
+
+        let code = rustfmt(&code).unwrap_or_else(|e| {
+            println!(
+                "cargo:warning=Failed to run `rustfmt` on ISLE-generated code: {:?}",
+                e
+            );
+            code
+        });
+
+        println!("Writing ISLE-generated Rust code to {}", out_file.display());
+        std::fs::write(out_file, code)?;
+    }
+
+    return Ok(());
+
+    fn rustfmt(code: &str) -> std::io::Result<String> {
+        use std::io::Write;
+
+        let mut rustfmt = std::process::Command::new("rustfmt")
+            .stdin(std::process::Stdio::piped())
+            .stdout(std::process::Stdio::piped())
+            .spawn()?;
+
+        let mut stdin = rustfmt.stdin.take().unwrap();
+        stdin.write_all(code.as_bytes())?;
+        drop(stdin);
+
+        let mut stdout = rustfmt.stdout.take().unwrap();
+        let mut data = vec![];
+        stdout.read_to_end(&mut data)?;
+
+        let status = rustfmt.wait()?;
+        if !status.success() {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!("`rustfmt` exited with status {}", status),
+            ));
+        }
+
+        Ok(String::from_utf8(data).expect("rustfmt always writs utf-8 to stdout"))
+    }
+}
diff --git a/cranelift/codegen/meta/Cargo.toml b/cranelift/codegen/meta/Cargo.toml
index 374f4f63db..041d59befd 100644
--- a/cranelift/codegen/meta/Cargo.toml
+++ b/cranelift/codegen/meta/Cargo.toml
@@ -17,3 +17,6 @@ cranelift-codegen-shared = { path = "../shared", version = "0.78.0" }
 
 [badges]
 maintenance = { status = "experimental" }
+
+[features]
+rebuild-isle = []
diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs
index 5e28e0b24b..1817bb7937 100644
--- a/cranelift/codegen/meta/src/gen_inst.rs
+++ b/cranelift/codegen/meta/src/gen_inst.rs
@@ -1,5 +1,6 @@
 //! Generate instruction data (including opcodes, formats, builders, etc.).
 use std::fmt;
+use std::path::Path;
 
 use cranelift_codegen_shared::constant_hash;
 
@@ -1084,6 +1085,243 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo
     fmtln!(fmt, "}")
 }
 
+#[cfg(feature = "rebuild-isle")]
+fn gen_isle(formats: &[&InstructionFormat], instructions: &AllInstructions, fmt: &mut Formatter) {
+    use std::collections::BTreeSet;
+    use std::fmt::Write;
+
+    fmt.multi_line(
+        r#"
+;; GENERATED BY `gen_isle`. DO NOT EDIT!!!
+;;
+;; This ISLE file defines all the external type declarations for Cranelift's
+;; data structures that ISLE will process, such as `InstructionData` and
+;; `Opcode`.
+        "#,
+    );
+    fmt.empty_line();
+
+    // Generate all the extern type declarations we need for various immediates.
+    fmt.line(";;;; Extern type declarations for immediates ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;");
+    fmt.empty_line();
+    let imm_tys: BTreeSet<_> = formats
+        .iter()
+        .flat_map(|f| {
+            f.imm_fields
+                .iter()
+                .map(|i| i.kind.rust_type.rsplit("::").next().unwrap())
+                .collect::<Vec<_>>()
+        })
+        .collect();
+    for ty in imm_tys {
+        fmtln!(fmt, "(type {} (primitive {}))", ty, ty);
+    }
+    fmt.empty_line();
+
+    // Generate all of the value arrays we need for `InstructionData` as well as
+    // the constructors and extractors for them.
+    fmt.line(";;;; Value Arrays ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;");
+    fmt.empty_line();
+    let value_array_arities: BTreeSet<_> = formats
+        .iter()
+        .filter(|f| f.typevar_operand.is_some() && !f.has_value_list && f.num_value_operands != 1)
+        .map(|f| f.num_value_operands)
+        .collect();
+    for n in value_array_arities {
+        fmtln!(fmt, ";; ISLE representation of `[Value; {}]`.", n);
+        fmtln!(fmt, "(type ValueArray{} extern (enum))", n);
+        fmt.empty_line();
+
+        fmtln!(
+            fmt,
+            "(decl value_array_{} ({}) ValueArray{})",
+            n,
+            (0..n).map(|_| "Value").collect::<Vec<_>>().join(" "),
+            n
+        );
+        fmtln!(
+            fmt,
+            "(extern constructor value_array_{} pack_value_array_{})",
+            n,
+            n
+        );
+        fmtln!(
+            fmt,
+            "(extern extractor infallible value_array_{} unpack_value_array_{})",
+            n,
+            n
+        );
+        fmt.empty_line();
+    }
+
+    // Generate the extern type declaration for `Opcode`.
+    fmt.line(";;;; `Opcode` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;");
+    fmt.empty_line();
+    fmt.line("(type Opcode extern");
+    fmt.indent(|fmt| {
+        fmt.line("(enum");
+        fmt.indent(|fmt| {
+            for inst in instructions {
+                fmtln!(fmt, "{}", inst.camel_name);
+            }
+        });
+        fmt.line(")");
+    });
+    fmt.line(")");
+    fmt.empty_line();
+
+    // Generate the extern type declaration for `InstructionData`.
+    fmt.line(";;;; `InstructionData` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;");
+    fmt.empty_line();
+    fmt.line("(type InstructionData extern");
+    fmt.indent(|fmt| {
+        fmt.line("(enum");
+        fmt.indent(|fmt| {
+            for format in formats {
+                let mut s = format!("({} (opcode Opcode)", format.name);
+                if format.typevar_operand.is_some() {
+                    if format.has_value_list {
+                        s.push_str(" (args ValueList)");
+                    } else if format.num_value_operands == 1 {
+                        s.push_str(" (arg Value)");
+                    } else {
+                        write!(&mut s, " (args ValueArray{})", format.num_value_operands).unwrap();
+                    }
+                }
+                for field in &format.imm_fields {
+                    write!(
+                        &mut s,
+                        " ({} {})",
+                        field.member,
+                        field.kind.rust_type.rsplit("::").next().unwrap()
+                    )
+                    .unwrap();
+                }
+                s.push(')');
+                fmt.line(&s);
+            }
+        });
+        fmt.line(")");
+    });
+    fmt.line(")");
+    fmt.empty_line();
+
+    // Generate the helper extractors for each opcode's full instruction.
+    //
+    // TODO: if/when we port our peephole optimization passes to ISLE we will
+    // want helper constructors as well.
+    fmt.line(";;;; Extracting Opcode, Operands, and Immediates from `InstructionData` ;;;;;;;;");
+    fmt.empty_line();
+    for inst in instructions {
+        fmtln!(
+            fmt,
+            "(decl {} ({}) Inst)",
+            inst.name,
+            inst.operands_in
+                .iter()
+                .map(|o| {
+                    let ty = o.kind.rust_type;
+                    if ty == "&[Value]" {
+                        "ValueSlice"
+                    } else {
+                        ty.rsplit("::").next().unwrap()
+                    }
+                })
+                .collect::<Vec<_>>()
+                .join(" ")
+        );
+        fmtln!(fmt, "(extractor");
+        fmt.indent(|fmt| {
+            fmtln!(
+                fmt,
+                "({} {})",
+                inst.name,
+                inst.operands_in
+                    .iter()
+                    .map(|o| { o.name })
+                    .collect::<Vec<_>>()
+                    .join(" ")
+            );
+            let mut s = format!(
+                "(inst_data (InstructionData.{} (Opcode.{})",
+                inst.format.name, inst.camel_name
+            );
+
+            // Immediates.
+            let imm_operands: Vec<_> = inst
+                .operands_in
+                .iter()
+                .filter(|o| !o.is_value() && !o.is_varargs())
+                .collect();
+            assert_eq!(imm_operands.len(), inst.format.imm_fields.len());
+            for op in imm_operands {
+                write!(&mut s, " {}", op.name).unwrap();
+            }
+
+            // Value and varargs operands.
+            if inst.format.typevar_operand.is_some() {
+                if inst.format.has_value_list {
+                    // The instruction format uses a value list, but the
+                    // instruction itself might have not only a `&[Value]`
+                    // varargs operand, but also one or more `Value` operands as
+                    // well. If this is the case, then we need to read them off
+                    // the front of the `ValueList`.
+                    let values: Vec<_> = inst
+                        .operands_in
+                        .iter()
+                        .filter(|o| o.is_value())
+                        .map(|o| o.name)
+                        .collect();
+                    let varargs = inst
+                        .operands_in
+                        .iter()
+                        .find(|o| o.is_varargs())
+                        .unwrap()
+                        .name;
+                    if values.is_empty() {
+                        write!(&mut s, " (value_list_slice {})", varargs).unwrap();
+                    } else {
+                        write!(
+                            &mut s,
+                            " (unwrap_head_value_list_{} {} {})",
+                            values.len(),
+                            values.join(" "),
+                            varargs
+                        )
+                        .unwrap();
+                    }
+                } else if inst.format.num_value_operands == 1 {
+                    write!(
+                        &mut s,
+                        " {}",
+                        inst.operands_in.iter().find(|o| o.is_value()).unwrap().name
+                    )
+                    .unwrap();
+                } else {
+                    let values = inst
+                        .operands_in
+                        .iter()
+                        .filter(|o| o.is_value())
+                        .map(|o| o.name)
+                        .collect::<Vec<_>>();
+                    assert_eq!(values.len(), inst.format.num_value_operands);
+                    let values = values.join(" ");
+                    write!(
+                        &mut s,
+                        " (value_array_{} {})",
+                        inst.format.num_value_operands, values,
+                    )
+                    .unwrap();
+                }
+            }
+            s.push_str("))");
+            fmt.line(&s);
+        });
+        fmt.line(")");
+        fmt.empty_line();
+    }
+}
+
 /// Generate a Builder trait with methods for all instructions.
 fn gen_builder(
     instructions: &AllInstructions,
@@ -1128,7 +1366,9 @@ pub(crate) fn generate(
     all_inst: &AllInstructions,
     opcode_filename: &str,
     inst_builder_filename: &str,
+    isle_filename: &str,
     out_dir: &str,
+    crate_dir: &Path,
 ) -> Result<(), error::Error> {
     // Opcodes.
     let mut fmt = Formatter::new();
@@ -1144,6 +1384,20 @@ pub(crate) fn generate(
     gen_try_from(all_inst, &mut fmt);
     fmt.update_file(opcode_filename, out_dir)?;
 
+    // ISLE DSL.
+    #[cfg(feature = "rebuild-isle")]
+    {
+        let mut fmt = Formatter::new();
+        gen_isle(&formats, all_inst, &mut fmt);
+        let crate_src_dir = crate_dir.join("src");
+        fmt.update_file(isle_filename, &crate_src_dir.display().to_string())?;
+    }
+    #[cfg(not(feature = "rebuild-isle"))]
+    {
+        // Silence unused variable warnings.
+        let _ = (isle_filename, crate_dir);
+    }
+
     // Instruction builder.
     let mut fmt = Formatter::new();
     gen_builder(all_inst, &formats, &mut fmt);
diff --git a/cranelift/codegen/meta/src/lib.rs b/cranelift/codegen/meta/src/lib.rs
index 30e5acc48c..e688f89b7d 100644
--- a/cranelift/codegen/meta/src/lib.rs
+++ b/cranelift/codegen/meta/src/lib.rs
@@ -1,5 +1,7 @@
 //! This crate generates Rust sources for use by
 //! [`cranelift_codegen`](../cranelift_codegen/index.html).
+
+use std::path::Path;
 #[macro_use]
 mod cdsl;
 mod srcgen;
@@ -21,7 +23,7 @@ pub fn isa_from_arch(arch: &str) -> Result<isa::Isa, String> {
 }
 
 /// Generates all the Rust source files used in Cranelift from the meta-language.
-pub fn generate(isas: &[isa::Isa], out_dir: &str) -> Result<(), error::Error> {
+pub fn generate(isas: &[isa::Isa], out_dir: &str, crate_dir: &Path) -> Result<(), error::Error> {
     // Create all the definitions:
     // - common definitions.
     let mut shared_defs = shared::define();
@@ -46,7 +48,9 @@ pub fn generate(isas: &[isa::Isa], out_dir: &str) -> Result<(), error::Error> {
         &shared_defs.all_instructions,
         "opcodes.rs",
         "inst_builder.rs",
+        "clif.isle",
         &out_dir,
+        crate_dir,
     )?;
 
     for isa in target_isas {
diff --git a/cranelift/codegen/meta/src/srcgen.rs b/cranelift/codegen/meta/src/srcgen.rs
index 21e3d5e904..c4fde06623 100644
--- a/cranelift/codegen/meta/src/srcgen.rs
+++ b/cranelift/codegen/meta/src/srcgen.rs
@@ -100,6 +100,7 @@ impl Formatter {
         let path_str = format!("{}/{}", directory, filename.as_ref());
 
         let path = path::Path::new(&path_str);
+        println!("Writing generated file: {}", path.display());
         let mut f = fs::File::create(path)?;
 
         for l in self.lines.iter().map(|l| l.as_bytes()) {
diff --git a/cranelift/codegen/src/clif.isle b/cranelift/codegen/src/clif.isle
new file mode 100644
index 0000000000..04601d5ea5
--- /dev/null
+++ b/cranelift/codegen/src/clif.isle
@@ -0,0 +1,1635 @@
+;; GENERATED BY `gen_isle`. DO NOT EDIT!!!
+;;
+;; This ISLE file defines all the external type declarations for Cranelift's
+;; data structures that ISLE will process, such as `InstructionData` and
+;; `Opcode`.
+
+;;;; Extern type declarations for immediates ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(type AtomicRmwOp (primitive AtomicRmwOp))
+(type Block (primitive Block))
+(type Constant (primitive Constant))
+(type FloatCC (primitive FloatCC))
+(type FuncRef (primitive FuncRef))
+(type GlobalValue (primitive GlobalValue))
+(type Heap (primitive Heap))
+(type Ieee32 (primitive Ieee32))
+(type Ieee64 (primitive Ieee64))
+(type Imm64 (primitive Imm64))
+(type Immediate (primitive Immediate))
+(type IntCC (primitive IntCC))
+(type JumpTable (primitive JumpTable))
+(type MemFlags (primitive MemFlags))
+(type Offset32 (primitive Offset32))
+(type SigRef (primitive SigRef))
+(type StackSlot (primitive StackSlot))
+(type Table (primitive Table))
+(type TrapCode (primitive TrapCode))
+(type Uimm32 (primitive Uimm32))
+(type Uimm8 (primitive Uimm8))
+(type bool (primitive bool))
+
+;;;; Value Arrays ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; ISLE representation of `[Value; 2]`.
+(type ValueArray2 extern (enum))
+
+(decl value_array_2 (Value Value) ValueArray2)
+(extern constructor value_array_2 pack_value_array_2)
+(extern extractor infallible value_array_2 unpack_value_array_2)
+
+;; ISLE representation of `[Value; 3]`.
+(type ValueArray3 extern (enum))
+
+(decl value_array_3 (Value Value Value) ValueArray3)
+(extern constructor value_array_3 pack_value_array_3)
+(extern extractor infallible value_array_3 unpack_value_array_3)
+
+;;;; `Opcode` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(type Opcode extern
+    (enum
+        Jump
+        Brz
+        Brnz
+        BrIcmp
+        Brif
+        Brff
+        BrTable
+        Debugtrap
+        Trap
+        Trapz
+        ResumableTrap
+        Trapnz
+        ResumableTrapnz
+        Trapif
+        Trapff
+        Return
+        FallthroughReturn
+        Call
+        CallIndirect
+        FuncAddr
+        Splat
+        Swizzle
+        Insertlane
+        Extractlane
+        Imin
+        Umin
+        Imax
+        Umax
+        AvgRound
+        UaddSat
+        SaddSat
+        UsubSat
+        SsubSat
+        Load
+        LoadComplex
+        Store
+        StoreComplex
+        Uload8
+        Uload8Complex
+        Sload8
+        Sload8Complex
+        Istore8
+        Istore8Complex
+        Uload16
+        Uload16Complex
+        Sload16
+        Sload16Complex
+        Istore16
+        Istore16Complex
+        Uload32
+        Uload32Complex
+        Sload32
+        Sload32Complex
+        Istore32
+        Istore32Complex
+        Uload8x8
+        Uload8x8Complex
+        Sload8x8
+        Sload8x8Complex
+        Uload16x4
+        Uload16x4Complex
+        Sload16x4
+        Sload16x4Complex
+        Uload32x2
+        Uload32x2Complex
+        Sload32x2
+        Sload32x2Complex
+        StackLoad
+        StackStore
+        StackAddr
+        GlobalValue
+        SymbolValue
+        TlsValue
+        HeapAddr
+        GetPinnedReg
+        SetPinnedReg
+        TableAddr
+        Iconst
+        F32const
+        F64const
+        Bconst
+        Vconst
+        ConstAddr
+        Shuffle
+        Null
+        Nop
+        Select
+        Selectif
+        SelectifSpectreGuard
+        Bitselect
+        Copy
+        IfcmpSp
+        Vsplit
+        Vconcat
+        Vselect
+        VanyTrue
+        VallTrue
+        VhighBits
+        Icmp
+        IcmpImm
+        Ifcmp
+        IfcmpImm
+        Iadd
+        Isub
+        Ineg
+        Iabs
+        Imul
+        Umulhi
+        Smulhi
+        SqmulRoundSat
+        Udiv
+        Sdiv
+        Urem
+        Srem
+        IaddImm
+        ImulImm
+        UdivImm
+        SdivImm
+        UremImm
+        SremImm
+        IrsubImm
+        IaddCin
+        IaddIfcin
+        IaddCout
+        IaddIfcout
+        IaddCarry
+        IaddIfcarry
+        IsubBin
+        IsubIfbin
+        IsubBout
+        IsubIfbout
+        IsubBorrow
+        IsubIfborrow
+        Band
+        Bor
+        Bxor
+        Bnot
+        BandNot
+        BorNot
+        BxorNot
+        BandImm
+        BorImm
+        BxorImm
+        Rotl
+        Rotr
+        RotlImm
+        RotrImm
+        Ishl
+        Ushr
+        Sshr
+        IshlImm
+        UshrImm
+        SshrImm
+        Bitrev
+        Clz
+        Cls
+        Ctz
+        Popcnt
+        Fcmp
+        Ffcmp
+        Fadd
+        Fsub
+        Fmul
+        Fdiv
+        Sqrt
+        Fma
+        Fneg
+        Fabs
+        Fcopysign
+        Fmin
+        FminPseudo
+        Fmax
+        FmaxPseudo
+        Ceil
+        Floor
+        Trunc
+        Nearest
+        IsNull
+        IsInvalid
+        Trueif
+        Trueff
+        Bitcast
+        RawBitcast
+        ScalarToVector
+        Breduce
+        Bextend
+        Bint
+        Bmask
+        Ireduce
+        Snarrow
+        Unarrow
+        Uunarrow
+        SwidenLow
+        SwidenHigh
+        UwidenLow
+        UwidenHigh
+        IaddPairwise
+        WideningPairwiseDotProductS
+        Uextend
+        Sextend
+        Fpromote
+        Fdemote
+        Fvdemote
+        FvpromoteLow
+        FcvtToUint
+        FcvtToUintSat
+        FcvtToSint
+        FcvtToSintSat
+        FcvtFromUint
+        FcvtFromSint
+        FcvtLowFromSint
+        Isplit
+        Iconcat
+        AtomicRmw
+        AtomicCas
+        AtomicLoad
+        AtomicStore
+        Fence
+    )
+)
+
+;;;; `InstructionData` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(type InstructionData extern
+    (enum
+        (AtomicCas (opcode Opcode) (args ValueArray3) (flags MemFlags))
+        (AtomicRmw (opcode Opcode) (args ValueArray2) (flags MemFlags) (op AtomicRmwOp))
+        (Binary (opcode Opcode) (args ValueArray2))
+        (BinaryImm64 (opcode Opcode) (arg Value) (imm Imm64))
+        (BinaryImm8 (opcode Opcode) (arg Value) (imm Uimm8))
+        (Branch (opcode Opcode) (args ValueList) (destination Block))
+        (BranchFloat (opcode Opcode) (args ValueList) (cond FloatCC) (destination Block))
+        (BranchIcmp (opcode Opcode) (args ValueList) (cond IntCC) (destination Block))
+        (BranchInt (opcode Opcode) (args ValueList) (cond IntCC) (destination Block))
+        (BranchTable (opcode Opcode) (arg Value) (destination Block) (table JumpTable))
+        (Call (opcode Opcode) (func_ref FuncRef))
+        (CallIndirect (opcode Opcode) (args ValueList) (sig_ref SigRef))
+        (CondTrap (opcode Opcode) (arg Value) (code TrapCode))
+        (FloatCompare (opcode Opcode) (args ValueArray2) (cond FloatCC))
+        (FloatCond (opcode Opcode) (arg Value) (cond FloatCC))
+        (FloatCondTrap (opcode Opcode) (arg Value) (cond FloatCC) (code TrapCode))
+        (FuncAddr (opcode Opcode) (func_ref FuncRef))
+        (HeapAddr (opcode Opcode) (arg Value) (heap Heap) (imm Uimm32))
+        (IntCompare (opcode Opcode) (args ValueArray2) (cond IntCC))
+        (IntCompareImm (opcode Opcode) (arg Value) (cond IntCC) (imm Imm64))
+        (IntCond (opcode Opcode) (arg Value) (cond IntCC))
+        (IntCondTrap (opcode Opcode) (arg Value) (cond IntCC) (code TrapCode))
+        (IntSelect (opcode Opcode) (args ValueArray3) (cond IntCC))
+        (Jump (opcode Opcode) (destination Block))
+        (Load (opcode Opcode) (arg Value) (flags MemFlags) (offset Offset32))
+        (LoadComplex (opcode Opcode) (flags MemFlags) (offset Offset32))
+        (LoadNoOffset (opcode Opcode) (arg Value) (flags MemFlags))
+        (MultiAry (opcode Opcode))
+        (NullAry (opcode Opcode))
+        (Shuffle (opcode Opcode) (args ValueArray2) (imm Immediate))
+        (StackLoad (opcode Opcode) (stack_slot StackSlot) (offset Offset32))
+        (StackStore (opcode Opcode) (arg Value) (stack_slot StackSlot) (offset Offset32))
+        (Store (opcode Opcode) (args ValueArray2) (flags MemFlags) (offset Offset32))
+        (StoreComplex (opcode Opcode) (args ValueList) (flags MemFlags) (offset Offset32))
+        (StoreNoOffset (opcode Opcode) (args ValueArray2) (flags MemFlags))
+        (TableAddr (opcode Opcode) (arg Value) (table Table) (offset Offset32))
+        (Ternary (opcode Opcode) (args ValueArray3))
+        (TernaryImm8 (opcode Opcode) (args ValueArray2) (imm Uimm8))
+        (Trap (opcode Opcode) (code TrapCode))
+        (Unary (opcode Opcode) (arg Value))
+        (UnaryBool (opcode Opcode) (imm bool))
+        (UnaryConst (opcode Opcode) (constant_handle Constant))
+        (UnaryGlobalValue (opcode Opcode) (global_value GlobalValue))
+        (UnaryIeee32 (opcode Opcode) (imm Ieee32))
+        (UnaryIeee64 (opcode Opcode) (imm Ieee64))
+        (UnaryImm (opcode Opcode) (imm Imm64))
+    )
+)
+
+;;;; Extracting Opcode, Operands, and Immediates from `InstructionData` ;;;;;;;;
+
+(decl jump (Block ValueSlice) Inst)
+(extractor
+    (jump block args)
+    (inst_data (InstructionData.Jump (Opcode.Jump) block))
+)
+
+(decl brz (Value Block ValueSlice) Inst)
+(extractor
+    (brz c block args)
+    (inst_data (InstructionData.Branch (Opcode.Brz) block (unwrap_head_value_list_1 c args)))
+)
+
+(decl brnz (Value Block ValueSlice) Inst)
+(extractor
+    (brnz c block args)
+    (inst_data (InstructionData.Branch (Opcode.Brnz) block (unwrap_head_value_list_1 c args)))
+)
+
+(decl br_icmp (IntCC Value Value Block ValueSlice) Inst)
+(extractor
+    (br_icmp Cond x y block args)
+    (inst_data (InstructionData.BranchIcmp (Opcode.BrIcmp) Cond block (unwrap_head_value_list_2 x y args)))
+)
+
+(decl brif (IntCC Value Block ValueSlice) Inst)
+(extractor
+    (brif Cond f block args)
+    (inst_data (InstructionData.BranchInt (Opcode.Brif) Cond block (unwrap_head_value_list_1 f args)))
+)
+
+(decl brff (FloatCC Value Block ValueSlice) Inst)
+(extractor
+    (brff Cond f block args)
+    (inst_data (InstructionData.BranchFloat (Opcode.Brff) Cond block (unwrap_head_value_list_1 f args)))
+)
+
+(decl br_table (Value Block JumpTable) Inst)
+(extractor
+    (br_table x block JT)
+    (inst_data (InstructionData.BranchTable (Opcode.BrTable) block JT x))
+)
+
+(decl debugtrap () Inst)
+(extractor
+    (debugtrap )
+    (inst_data (InstructionData.NullAry (Opcode.Debugtrap)))
+)
+
+(decl trap (TrapCode) Inst)
+(extractor
+    (trap code)
+    (inst_data (InstructionData.Trap (Opcode.Trap) code))
+)
+
+(decl trapz (Value TrapCode) Inst)
+(extractor
+    (trapz c code)
+    (inst_data (InstructionData.CondTrap (Opcode.Trapz) code c))
+)
+
+(decl resumable_trap (TrapCode) Inst)
+(extractor
+    (resumable_trap code)
+    (inst_data (InstructionData.Trap (Opcode.ResumableTrap) code))
+)
+
+(decl trapnz (Value TrapCode) Inst)
+(extractor
+    (trapnz c code)
+    (inst_data (InstructionData.CondTrap (Opcode.Trapnz) code c))
+)
+
+(decl resumable_trapnz (Value TrapCode) Inst)
+(extractor
+    (resumable_trapnz c code)
+    (inst_data (InstructionData.CondTrap (Opcode.ResumableTrapnz) code c))
+)
+
+(decl trapif (IntCC Value TrapCode) Inst)
+(extractor
+    (trapif Cond f code)
+    (inst_data (InstructionData.IntCondTrap (Opcode.Trapif) Cond code f))
+)
+
+(decl trapff (FloatCC Value TrapCode) Inst)
+(extractor
+    (trapff Cond f code)
+    (inst_data (InstructionData.FloatCondTrap (Opcode.Trapff) Cond code f))
+)
+
+(decl return (ValueSlice) Inst)
+(extractor
+    (return rvals)
+    (inst_data (InstructionData.MultiAry (Opcode.Return)))
+)
+
+(decl fallthrough_return (ValueSlice) Inst)
+(extractor
+    (fallthrough_return rvals)
+    (inst_data (InstructionData.MultiAry (Opcode.FallthroughReturn)))
+)
+
+(decl call (FuncRef ValueSlice) Inst)
+(extractor
+    (call FN args)
+    (inst_data (InstructionData.Call (Opcode.Call) FN))
+)
+
+(decl call_indirect (SigRef Value ValueSlice) Inst)
+(extractor
+    (call_indirect SIG callee args)
+    (inst_data (InstructionData.CallIndirect (Opcode.CallIndirect) SIG (unwrap_head_value_list_1 callee args)))
+)
+
+(decl func_addr (FuncRef) Inst)
+(extractor
+    (func_addr FN)
+    (inst_data (InstructionData.FuncAddr (Opcode.FuncAddr) FN))
+)
+
+(decl splat (Value) Inst)
+(extractor
+    (splat x)
+    (inst_data (InstructionData.Unary (Opcode.Splat) x))
+)
+
+(decl swizzle (Value Value) Inst)
+(extractor
+    (swizzle x y)
+    (inst_data (InstructionData.Binary (Opcode.Swizzle) (value_array_2 x y)))
+)
+
+(decl insertlane (Value Value Uimm8) Inst)
+(extractor
+    (insertlane x y Idx)
+    (inst_data (InstructionData.TernaryImm8 (Opcode.Insertlane) Idx (value_array_2 x y)))
+)
+
+(decl extractlane (Value Uimm8) Inst)
+(extractor
+    (extractlane x Idx)
+    (inst_data (InstructionData.BinaryImm8 (Opcode.Extractlane) Idx x))
+)
+
+(decl imin (Value Value) Inst)
+(extractor
+    (imin x y)
+    (inst_data (InstructionData.Binary (Opcode.Imin) (value_array_2 x y)))
+)
+
+(decl umin (Value Value) Inst)
+(extractor
+    (umin x y)
+    (inst_data (InstructionData.Binary (Opcode.Umin) (value_array_2 x y)))
+)
+
+(decl imax (Value Value) Inst)
+(extractor
+    (imax x y)
+    (inst_data (InstructionData.Binary (Opcode.Imax) (value_array_2 x y)))
+)
+
+(decl umax (Value Value) Inst)
+(extractor
+    (umax x y)
+    (inst_data (InstructionData.Binary (Opcode.Umax) (value_array_2 x y)))
+)
+
+(decl avg_round (Value Value) Inst)
+(extractor
+    (avg_round x y)
+    (inst_data (InstructionData.Binary (Opcode.AvgRound) (value_array_2 x y)))
+)
+
+(decl uadd_sat (Value Value) Inst)
+(extractor
+    (uadd_sat x y)
+    (inst_data (InstructionData.Binary (Opcode.UaddSat) (value_array_2 x y)))
+)
+
+(decl sadd_sat (Value Value) Inst)
+(extractor
+    (sadd_sat x y)
+    (inst_data (InstructionData.Binary (Opcode.SaddSat) (value_array_2 x y)))
+)
+
+(decl usub_sat (Value Value) Inst)
+(extractor
+    (usub_sat x y)
+    (inst_data (InstructionData.Binary (Opcode.UsubSat) (value_array_2 x y)))
+)
+
+(decl ssub_sat (Value Value) Inst)
+(extractor
+    (ssub_sat x y)
+    (inst_data (InstructionData.Binary (Opcode.SsubSat) (value_array_2 x y)))
+)
+
+(decl load (MemFlags Value Offset32) Inst)
+(extractor
+    (load MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Load) MemFlags Offset p))
+)
+
+(decl load_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (load_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.LoadComplex) MemFlags Offset))
+)
+
+(decl store (MemFlags Value Value Offset32) Inst)
+(extractor
+    (store MemFlags x p Offset)
+    (inst_data (InstructionData.Store (Opcode.Store) MemFlags Offset (value_array_2 x p)))
+)
+
+(decl store_complex (MemFlags Value ValueSlice Offset32) Inst)
+(extractor
+    (store_complex MemFlags x args Offset)
+    (inst_data (InstructionData.StoreComplex (Opcode.StoreComplex) MemFlags Offset (unwrap_head_value_list_1 x args)))
+)
+
+(decl uload8 (MemFlags Value Offset32) Inst)
+(extractor
+    (uload8 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Uload8) MemFlags Offset p))
+)
+
+(decl uload8_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (uload8_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Uload8Complex) MemFlags Offset))
+)
+
+(decl sload8 (MemFlags Value Offset32) Inst)
+(extractor
+    (sload8 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Sload8) MemFlags Offset p))
+)
+
+(decl sload8_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (sload8_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Sload8Complex) MemFlags Offset))
+)
+
+(decl istore8 (MemFlags Value Value Offset32) Inst)
+(extractor
+    (istore8 MemFlags x p Offset)
+    (inst_data (InstructionData.Store (Opcode.Istore8) MemFlags Offset (value_array_2 x p)))
+)
+
+(decl istore8_complex (MemFlags Value ValueSlice Offset32) Inst)
+(extractor
+    (istore8_complex MemFlags x args Offset)
+    (inst_data (InstructionData.StoreComplex (Opcode.Istore8Complex) MemFlags Offset (unwrap_head_value_list_1 x args)))
+)
+
+(decl uload16 (MemFlags Value Offset32) Inst)
+(extractor
+    (uload16 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Uload16) MemFlags Offset p))
+)
+
+(decl uload16_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (uload16_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Uload16Complex) MemFlags Offset))
+)
+
+(decl sload16 (MemFlags Value Offset32) Inst)
+(extractor
+    (sload16 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Sload16) MemFlags Offset p))
+)
+
+(decl sload16_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (sload16_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Sload16Complex) MemFlags Offset))
+)
+
+(decl istore16 (MemFlags Value Value Offset32) Inst)
+(extractor
+    (istore16 MemFlags x p Offset)
+    (inst_data (InstructionData.Store (Opcode.Istore16) MemFlags Offset (value_array_2 x p)))
+)
+
+(decl istore16_complex (MemFlags Value ValueSlice Offset32) Inst)
+(extractor
+    (istore16_complex MemFlags x args Offset)
+    (inst_data (InstructionData.StoreComplex (Opcode.Istore16Complex) MemFlags Offset (unwrap_head_value_list_1 x args)))
+)
+
+(decl uload32 (MemFlags Value Offset32) Inst)
+(extractor
+    (uload32 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Uload32) MemFlags Offset p))
+)
+
+(decl uload32_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (uload32_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Uload32Complex) MemFlags Offset))
+)
+
+(decl sload32 (MemFlags Value Offset32) Inst)
+(extractor
+    (sload32 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Sload32) MemFlags Offset p))
+)
+
+(decl sload32_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (sload32_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Sload32Complex) MemFlags Offset))
+)
+
+(decl istore32 (MemFlags Value Value Offset32) Inst)
+(extractor
+    (istore32 MemFlags x p Offset)
+    (inst_data (InstructionData.Store (Opcode.Istore32) MemFlags Offset (value_array_2 x p)))
+)
+
+(decl istore32_complex (MemFlags Value ValueSlice Offset32) Inst)
+(extractor
+    (istore32_complex MemFlags x args Offset)
+    (inst_data (InstructionData.StoreComplex (Opcode.Istore32Complex) MemFlags Offset (unwrap_head_value_list_1 x args)))
+)
+
+(decl uload8x8 (MemFlags Value Offset32) Inst)
+(extractor
+    (uload8x8 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Uload8x8) MemFlags Offset p))
+)
+
+(decl uload8x8_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (uload8x8_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Uload8x8Complex) MemFlags Offset))
+)
+
+(decl sload8x8 (MemFlags Value Offset32) Inst)
+(extractor
+    (sload8x8 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Sload8x8) MemFlags Offset p))
+)
+
+(decl sload8x8_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (sload8x8_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Sload8x8Complex) MemFlags Offset))
+)
+
+(decl uload16x4 (MemFlags Value Offset32) Inst)
+(extractor
+    (uload16x4 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Uload16x4) MemFlags Offset p))
+)
+
+(decl uload16x4_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (uload16x4_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Uload16x4Complex) MemFlags Offset))
+)
+
+(decl sload16x4 (MemFlags Value Offset32) Inst)
+(extractor
+    (sload16x4 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Sload16x4) MemFlags Offset p))
+)
+
+(decl sload16x4_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (sload16x4_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Sload16x4Complex) MemFlags Offset))
+)
+
+(decl uload32x2 (MemFlags Value Offset32) Inst)
+(extractor
+    (uload32x2 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Uload32x2) MemFlags Offset p))
+)
+
+(decl uload32x2_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (uload32x2_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Uload32x2Complex) MemFlags Offset))
+)
+
+(decl sload32x2 (MemFlags Value Offset32) Inst)
+(extractor
+    (sload32x2 MemFlags p Offset)
+    (inst_data (InstructionData.Load (Opcode.Sload32x2) MemFlags Offset p))
+)
+
+(decl sload32x2_complex (MemFlags ValueSlice Offset32) Inst)
+(extractor
+    (sload32x2_complex MemFlags args Offset)
+    (inst_data (InstructionData.LoadComplex (Opcode.Sload32x2Complex) MemFlags Offset))
+)
+
+(decl stack_load (StackSlot Offset32) Inst)
+(extractor
+    (stack_load SS Offset)
+    (inst_data (InstructionData.StackLoad (Opcode.StackLoad) SS Offset))
+)
+
+(decl stack_store (Value StackSlot Offset32) Inst)
+(extractor
+    (stack_store x SS Offset)
+    (inst_data (InstructionData.StackStore (Opcode.StackStore) SS Offset x))
+)
+
+(decl stack_addr (StackSlot Offset32) Inst)
+(extractor
+    (stack_addr SS Offset)
+    (inst_data (InstructionData.StackLoad (Opcode.StackAddr) SS Offset))
+)
+
+(decl global_value (GlobalValue) Inst)
+(extractor
+    (global_value GV)
+    (inst_data (InstructionData.UnaryGlobalValue (Opcode.GlobalValue) GV))
+)
+
+(decl symbol_value (GlobalValue) Inst)
+(extractor
+    (symbol_value GV)
+    (inst_data (InstructionData.UnaryGlobalValue (Opcode.SymbolValue) GV))
+)
+
+(decl tls_value (GlobalValue) Inst)
+(extractor
+    (tls_value GV)
+    (inst_data (InstructionData.UnaryGlobalValue (Opcode.TlsValue) GV))
+)
+
+(decl heap_addr (Heap Value Uimm32) Inst)
+(extractor
+    (heap_addr H p Size)
+    (inst_data (InstructionData.HeapAddr (Opcode.HeapAddr) H Size p))
+)
+
+(decl get_pinned_reg () Inst)
+(extractor
+    (get_pinned_reg )
+    (inst_data (InstructionData.NullAry (Opcode.GetPinnedReg)))
+)
+
+(decl set_pinned_reg (Value) Inst)
+(extractor
+    (set_pinned_reg addr)
+    (inst_data (InstructionData.Unary (Opcode.SetPinnedReg) addr))
+)
+
+(decl table_addr (Table Value Offset32) Inst)
+(extractor
+    (table_addr T p Offset)
+    (inst_data (InstructionData.TableAddr (Opcode.TableAddr) T Offset p))
+)
+
+(decl iconst (Imm64) Inst)
+(extractor
+    (iconst N)
+    (inst_data (InstructionData.UnaryImm (Opcode.Iconst) N))
+)
+
+(decl f32const (Ieee32) Inst)
+(extractor
+    (f32const N)
+    (inst_data (InstructionData.UnaryIeee32 (Opcode.F32const) N))
+)
+
+(decl f64const (Ieee64) Inst)
+(extractor
+    (f64const N)
+    (inst_data (InstructionData.UnaryIeee64 (Opcode.F64const) N))
+)
+
+(decl bconst (bool) Inst)
+(extractor
+    (bconst N)
+    (inst_data (InstructionData.UnaryBool (Opcode.Bconst) N))
+)
+
+(decl vconst (Constant) Inst)
+(extractor
+    (vconst N)
+    (inst_data (InstructionData.UnaryConst (Opcode.Vconst) N))
+)
+
+(decl const_addr (Constant) Inst)
+(extractor
+    (const_addr constant)
+    (inst_data (InstructionData.UnaryConst (Opcode.ConstAddr) constant))
+)
+
+(decl shuffle (Value Value Immediate) Inst)
+(extractor
+    (shuffle a b mask)
+    (inst_data (InstructionData.Shuffle (Opcode.Shuffle) mask (value_array_2 a b)))
+)
+
+(decl null () Inst)
+(extractor
+    (null )
+    (inst_data (InstructionData.NullAry (Opcode.Null)))
+)
+
+(decl nop () Inst)
+(extractor
+    (nop )
+    (inst_data (InstructionData.NullAry (Opcode.Nop)))
+)
+
+(decl select (Value Value Value) Inst)
+(extractor
+    (select c x y)
+    (inst_data (InstructionData.Ternary (Opcode.Select) (value_array_3 c x y)))
+)
+
+(decl selectif (IntCC Value Value Value) Inst)
+(extractor
+    (selectif cc flags x y)
+    (inst_data (InstructionData.IntSelect (Opcode.Selectif) cc (value_array_3 flags x y)))
+)
+
+(decl selectif_spectre_guard (IntCC Value Value Value) Inst)
+(extractor
+    (selectif_spectre_guard cc flags x y)
+    (inst_data (InstructionData.IntSelect (Opcode.SelectifSpectreGuard) cc (value_array_3 flags x y)))
+)
+
+(decl bitselect (Value Value Value) Inst)
+(extractor
+    (bitselect c x y)
+    (inst_data (InstructionData.Ternary (Opcode.Bitselect) (value_array_3 c x y)))
+)
+
+(decl copy (Value) Inst)
+(extractor
+    (copy x)
+    (inst_data (InstructionData.Unary (Opcode.Copy) x))
+)
+
+(decl ifcmp_sp (Value) Inst)
+(extractor
+    (ifcmp_sp addr)
+    (inst_data (InstructionData.Unary (Opcode.IfcmpSp) addr))
+)
+
+(decl vsplit (Value) Inst)
+(extractor
+    (vsplit x)
+    (inst_data (InstructionData.Unary (Opcode.Vsplit) x))
+)
+
+(decl vconcat (Value Value) Inst)
+(extractor
+    (vconcat x y)
+    (inst_data (InstructionData.Binary (Opcode.Vconcat) (value_array_2 x y)))
+)
+
+(decl vselect (Value Value Value) Inst)
+(extractor
+    (vselect c x y)
+    (inst_data (InstructionData.Ternary (Opcode.Vselect) (value_array_3 c x y)))
+)
+
+(decl vany_true (Value) Inst)
+(extractor
+    (vany_true a)
+    (inst_data (InstructionData.Unary (Opcode.VanyTrue) a))
+)
+
+(decl vall_true (Value) Inst)
+(extractor
+    (vall_true a)
+    (inst_data (InstructionData.Unary (Opcode.VallTrue) a))
+)
+
+(decl vhigh_bits (Value) Inst)
+(extractor
+    (vhigh_bits a)
+    (inst_data (InstructionData.Unary (Opcode.VhighBits) a))
+)
+
+(decl icmp (IntCC Value Value) Inst)
+(extractor
+    (icmp Cond x y)
+    (inst_data (InstructionData.IntCompare (Opcode.Icmp) Cond (value_array_2 x y)))
+)
+
+(decl icmp_imm (IntCC Value Imm64) Inst)
+(extractor
+    (icmp_imm Cond x Y)
+    (inst_data (InstructionData.IntCompareImm (Opcode.IcmpImm) Cond Y x))
+)
+
+(decl ifcmp (Value Value) Inst)
+(extractor
+    (ifcmp x y)
+    (inst_data (InstructionData.Binary (Opcode.Ifcmp) (value_array_2 x y)))
+)
+
+(decl ifcmp_imm (Value Imm64) Inst)
+(extractor
+    (ifcmp_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.IfcmpImm) Y x))
+)
+
+(decl iadd (Value Value) Inst)
+(extractor
+    (iadd x y)
+    (inst_data (InstructionData.Binary (Opcode.Iadd) (value_array_2 x y)))
+)
+
+(decl isub (Value Value) Inst)
+(extractor
+    (isub x y)
+    (inst_data (InstructionData.Binary (Opcode.Isub) (value_array_2 x y)))
+)
+
+(decl ineg (Value) Inst)
+(extractor
+    (ineg x)
+    (inst_data (InstructionData.Unary (Opcode.Ineg) x))
+)
+
+(decl iabs (Value) Inst)
+(extractor
+    (iabs x)
+    (inst_data (InstructionData.Unary (Opcode.Iabs) x))
+)
+
+(decl imul (Value Value) Inst)
+(extractor
+    (imul x y)
+    (inst_data (InstructionData.Binary (Opcode.Imul) (value_array_2 x y)))
+)
+
+(decl umulhi (Value Value) Inst)
+(extractor
+    (umulhi x y)
+    (inst_data (InstructionData.Binary (Opcode.Umulhi) (value_array_2 x y)))
+)
+
+(decl smulhi (Value Value) Inst)
+(extractor
+    (smulhi x y)
+    (inst_data (InstructionData.Binary (Opcode.Smulhi) (value_array_2 x y)))
+)
+
+(decl sqmul_round_sat (Value Value) Inst)
+(extractor
+    (sqmul_round_sat x y)
+    (inst_data (InstructionData.Binary (Opcode.SqmulRoundSat) (value_array_2 x y)))
+)
+
+(decl udiv (Value Value) Inst)
+(extractor
+    (udiv x y)
+    (inst_data (InstructionData.Binary (Opcode.Udiv) (value_array_2 x y)))
+)
+
+(decl sdiv (Value Value) Inst)
+(extractor
+    (sdiv x y)
+    (inst_data (InstructionData.Binary (Opcode.Sdiv) (value_array_2 x y)))
+)
+
+(decl urem (Value Value) Inst)
+(extractor
+    (urem x y)
+    (inst_data (InstructionData.Binary (Opcode.Urem) (value_array_2 x y)))
+)
+
+(decl srem (Value Value) Inst)
+(extractor
+    (srem x y)
+    (inst_data (InstructionData.Binary (Opcode.Srem) (value_array_2 x y)))
+)
+
+(decl iadd_imm (Value Imm64) Inst)
+(extractor
+    (iadd_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.IaddImm) Y x))
+)
+
+(decl imul_imm (Value Imm64) Inst)
+(extractor
+    (imul_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.ImulImm) Y x))
+)
+
+(decl udiv_imm (Value Imm64) Inst)
+(extractor
+    (udiv_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.UdivImm) Y x))
+)
+
+(decl sdiv_imm (Value Imm64) Inst)
+(extractor
+    (sdiv_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.SdivImm) Y x))
+)
+
+(decl urem_imm (Value Imm64) Inst)
+(extractor
+    (urem_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.UremImm) Y x))
+)
+
+(decl srem_imm (Value Imm64) Inst)
+(extractor
+    (srem_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.SremImm) Y x))
+)
+
+(decl irsub_imm (Value Imm64) Inst)
+(extractor
+    (irsub_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.IrsubImm) Y x))
+)
+
+(decl iadd_cin (Value Value Value) Inst)
+(extractor
+    (iadd_cin x y c_in)
+    (inst_data (InstructionData.Ternary (Opcode.IaddCin) (value_array_3 x y c_in)))
+)
+
+(decl iadd_ifcin (Value Value Value) Inst)
+(extractor
+    (iadd_ifcin x y c_in)
+    (inst_data (InstructionData.Ternary (Opcode.IaddIfcin) (value_array_3 x y c_in)))
+)
+
+(decl iadd_cout (Value Value) Inst)
+(extractor
+    (iadd_cout x y)
+    (inst_data (InstructionData.Binary (Opcode.IaddCout) (value_array_2 x y)))
+)
+
+(decl iadd_ifcout (Value Value) Inst)
+(extractor
+    (iadd_ifcout x y)
+    (inst_data (InstructionData.Binary (Opcode.IaddIfcout) (value_array_2 x y)))
+)
+
+(decl iadd_carry (Value Value Value) Inst)
+(extractor
+    (iadd_carry x y c_in)
+    (inst_data (InstructionData.Ternary (Opcode.IaddCarry) (value_array_3 x y c_in)))
+)
+
+(decl iadd_ifcarry (Value Value Value) Inst)
+(extractor
+    (iadd_ifcarry x y c_in)
+    (inst_data (InstructionData.Ternary (Opcode.IaddIfcarry) (value_array_3 x y c_in)))
+)
+
+(decl isub_bin (Value Value Value) Inst)
+(extractor
+    (isub_bin x y b_in)
+    (inst_data (InstructionData.Ternary (Opcode.IsubBin) (value_array_3 x y b_in)))
+)
+
+(decl isub_ifbin (Value Value Value) Inst)
+(extractor
+    (isub_ifbin x y b_in)
+    (inst_data (InstructionData.Ternary (Opcode.IsubIfbin) (value_array_3 x y b_in)))
+)
+
+(decl isub_bout (Value Value) Inst)
+(extractor
+    (isub_bout x y)
+    (inst_data (InstructionData.Binary (Opcode.IsubBout) (value_array_2 x y)))
+)
+
+(decl isub_ifbout (Value Value) Inst)
+(extractor
+    (isub_ifbout x y)
+    (inst_data (InstructionData.Binary (Opcode.IsubIfbout) (value_array_2 x y)))
+)
+
+(decl isub_borrow (Value Value Value) Inst)
+(extractor
+    (isub_borrow x y b_in)
+    (inst_data (InstructionData.Ternary (Opcode.IsubBorrow) (value_array_3 x y b_in)))
+)
+
+(decl isub_ifborrow (Value Value Value) Inst)
+(extractor
+    (isub_ifborrow x y b_in)
+    (inst_data (InstructionData.Ternary (Opcode.IsubIfborrow) (value_array_3 x y b_in)))
+)
+
+(decl band (Value Value) Inst)
+(extractor
+    (band x y)
+    (inst_data (InstructionData.Binary (Opcode.Band) (value_array_2 x y)))
+)
+
+(decl bor (Value Value) Inst)
+(extractor
+    (bor x y)
+    (inst_data (InstructionData.Binary (Opcode.Bor) (value_array_2 x y)))
+)
+
+(decl bxor (Value Value) Inst)
+(extractor
+    (bxor x y)
+    (inst_data (InstructionData.Binary (Opcode.Bxor) (value_array_2 x y)))
+)
+
+(decl bnot (Value) Inst)
+(extractor
+    (bnot x)
+    (inst_data (InstructionData.Unary (Opcode.Bnot) x))
+)
+
+(decl band_not (Value Value) Inst)
+(extractor
+    (band_not x y)
+    (inst_data (InstructionData.Binary (Opcode.BandNot) (value_array_2 x y)))
+)
+
+(decl bor_not (Value Value) Inst)
+(extractor
+    (bor_not x y)
+    (inst_data (InstructionData.Binary (Opcode.BorNot) (value_array_2 x y)))
+)
+
+(decl bxor_not (Value Value) Inst)
+(extractor
+    (bxor_not x y)
+    (inst_data (InstructionData.Binary (Opcode.BxorNot) (value_array_2 x y)))
+)
+
+(decl band_imm (Value Imm64) Inst)
+(extractor
+    (band_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.BandImm) Y x))
+)
+
+(decl bor_imm (Value Imm64) Inst)
+(extractor
+    (bor_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.BorImm) Y x))
+)
+
+(decl bxor_imm (Value Imm64) Inst)
+(extractor
+    (bxor_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.BxorImm) Y x))
+)
+
+(decl rotl (Value Value) Inst)
+(extractor
+    (rotl x y)
+    (inst_data (InstructionData.Binary (Opcode.Rotl) (value_array_2 x y)))
+)
+
+(decl rotr (Value Value) Inst)
+(extractor
+    (rotr x y)
+    (inst_data (InstructionData.Binary (Opcode.Rotr) (value_array_2 x y)))
+)
+
+(decl rotl_imm (Value Imm64) Inst)
+(extractor
+    (rotl_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.RotlImm) Y x))
+)
+
+(decl rotr_imm (Value Imm64) Inst)
+(extractor
+    (rotr_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.RotrImm) Y x))
+)
+
+(decl ishl (Value Value) Inst)
+(extractor
+    (ishl x y)
+    (inst_data (InstructionData.Binary (Opcode.Ishl) (value_array_2 x y)))
+)
+
+(decl ushr (Value Value) Inst)
+(extractor
+    (ushr x y)
+    (inst_data (InstructionData.Binary (Opcode.Ushr) (value_array_2 x y)))
+)
+
+(decl sshr (Value Value) Inst)
+(extractor
+    (sshr x y)
+    (inst_data (InstructionData.Binary (Opcode.Sshr) (value_array_2 x y)))
+)
+
+(decl ishl_imm (Value Imm64) Inst)
+(extractor
+    (ishl_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.IshlImm) Y x))
+)
+
+(decl ushr_imm (Value Imm64) Inst)
+(extractor
+    (ushr_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.UshrImm) Y x))
+)
+
+(decl sshr_imm (Value Imm64) Inst)
+(extractor
+    (sshr_imm x Y)
+    (inst_data (InstructionData.BinaryImm64 (Opcode.SshrImm) Y x))
+)
+
+(decl bitrev (Value) Inst)
+(extractor
+    (bitrev x)
+    (inst_data (InstructionData.Unary (Opcode.Bitrev) x))
+)
+
+(decl clz (Value) Inst)
+(extractor
+    (clz x)
+    (inst_data (InstructionData.Unary (Opcode.Clz) x))
+)
+
+(decl cls (Value) Inst)
+(extractor
+    (cls x)
+    (inst_data (InstructionData.Unary (Opcode.Cls) x))
+)
+
+(decl ctz (Value) Inst)
+(extractor
+    (ctz x)
+    (inst_data (InstructionData.Unary (Opcode.Ctz) x))
+)
+
+(decl popcnt (Value) Inst)
+(extractor
+    (popcnt x)
+    (inst_data (InstructionData.Unary (Opcode.Popcnt) x))
+)
+
+(decl fcmp (FloatCC Value Value) Inst)
+(extractor
+    (fcmp Cond x y)
+    (inst_data (InstructionData.FloatCompare (Opcode.Fcmp) Cond (value_array_2 x y)))
+)
+
+(decl ffcmp (Value Value) Inst)
+(extractor
+    (ffcmp x y)
+    (inst_data (InstructionData.Binary (Opcode.Ffcmp) (value_array_2 x y)))
+)
+
+(decl fadd (Value Value) Inst)
+(extractor
+    (fadd x y)
+    (inst_data (InstructionData.Binary (Opcode.Fadd) (value_array_2 x y)))
+)
+
+(decl fsub (Value Value) Inst)
+(extractor
+    (fsub x y)
+    (inst_data (InstructionData.Binary (Opcode.Fsub) (value_array_2 x y)))
+)
+
+(decl fmul (Value Value) Inst)
+(extractor
+    (fmul x y)
+    (inst_data (InstructionData.Binary (Opcode.Fmul) (value_array_2 x y)))
+)
+
+(decl fdiv (Value Value) Inst)
+(extractor
+    (fdiv x y)
+    (inst_data (InstructionData.Binary (Opcode.Fdiv) (value_array_2 x y)))
+)
+
+(decl sqrt (Value) Inst)
+(extractor
+    (sqrt x)
+    (inst_data (InstructionData.Unary (Opcode.Sqrt) x))
+)
+
+(decl fma (Value Value Value) Inst)
+(extractor
+    (fma x y z)
+    (inst_data (InstructionData.Ternary (Opcode.Fma) (value_array_3 x y z)))
+)
+
+(decl fneg (Value) Inst)
+(extractor
+    (fneg x)
+    (inst_data (InstructionData.Unary (Opcode.Fneg) x))
+)
+
+(decl fabs (Value) Inst)
+(extractor
+    (fabs x)
+    (inst_data (InstructionData.Unary (Opcode.Fabs) x))
+)
+
+(decl fcopysign (Value Value) Inst)
+(extractor
+    (fcopysign x y)
+    (inst_data (InstructionData.Binary (Opcode.Fcopysign) (value_array_2 x y)))
+)
+
+(decl fmin (Value Value) Inst)
+(extractor
+    (fmin x y)
+    (inst_data (InstructionData.Binary (Opcode.Fmin) (value_array_2 x y)))
+)
+
+(decl fmin_pseudo (Value Value) Inst)
+(extractor
+    (fmin_pseudo x y)
+    (inst_data (InstructionData.Binary (Opcode.FminPseudo) (value_array_2 x y)))
+)
+
+(decl fmax (Value Value) Inst)
+(extractor
+    (fmax x y)
+    (inst_data (InstructionData.Binary (Opcode.Fmax) (value_array_2 x y)))
+)
+
+(decl fmax_pseudo (Value Value) Inst)
+(extractor
+    (fmax_pseudo x y)
+    (inst_data (InstructionData.Binary (Opcode.FmaxPseudo) (value_array_2 x y)))
+)
+
+(decl ceil (Value) Inst)
+(extractor
+    (ceil x)
+    (inst_data (InstructionData.Unary (Opcode.Ceil) x))
+)
+
+(decl floor (Value) Inst)
+(extractor
+    (floor x)
+    (inst_data (InstructionData.Unary (Opcode.Floor) x))
+)
+
+(decl trunc (Value) Inst)
+(extractor
+    (trunc x)
+    (inst_data (InstructionData.Unary (Opcode.Trunc) x))
+)
+
+(decl nearest (Value) Inst)
+(extractor
+    (nearest x)
+    (inst_data (InstructionData.Unary (Opcode.Nearest) x))
+)
+
+(decl is_null (Value) Inst)
+(extractor
+    (is_null x)
+    (inst_data (InstructionData.Unary (Opcode.IsNull) x))
+)
+
+(decl is_invalid (Value) Inst)
+(extractor
+    (is_invalid x)
+    (inst_data (InstructionData.Unary (Opcode.IsInvalid) x))
+)
+
+(decl trueif (IntCC Value) Inst)
+(extractor
+    (trueif Cond f)
+    (inst_data (InstructionData.IntCond (Opcode.Trueif) Cond f))
+)
+
+(decl trueff (FloatCC Value) Inst)
+(extractor
+    (trueff Cond f)
+    (inst_data (InstructionData.FloatCond (Opcode.Trueff) Cond f))
+)
+
+(decl bitcast (Value) Inst)
+(extractor
+    (bitcast x)
+    (inst_data (InstructionData.Unary (Opcode.Bitcast) x))
+)
+
+(decl raw_bitcast (Value) Inst)
+(extractor
+    (raw_bitcast x)
+    (inst_data (InstructionData.Unary (Opcode.RawBitcast) x))
+)
+
+(decl scalar_to_vector (Value) Inst)
+(extractor
+    (scalar_to_vector s)
+    (inst_data (InstructionData.Unary (Opcode.ScalarToVector) s))
+)
+
+(decl breduce (Value) Inst)
+(extractor
+    (breduce x)
+    (inst_data (InstructionData.Unary (Opcode.Breduce) x))
+)
+
+(decl bextend (Value) Inst)
+(extractor
+    (bextend x)
+    (inst_data (InstructionData.Unary (Opcode.Bextend) x))
+)
+
+(decl bint (Value) Inst)
+(extractor
+    (bint x)
+    (inst_data (InstructionData.Unary (Opcode.Bint) x))
+)
+
+(decl bmask (Value) Inst)
+(extractor
+    (bmask x)
+    (inst_data (InstructionData.Unary (Opcode.Bmask) x))
+)
+
+(decl ireduce (Value) Inst)
+(extractor
+    (ireduce x)
+    (inst_data (InstructionData.Unary (Opcode.Ireduce) x))
+)
+
+(decl snarrow (Value Value) Inst)
+(extractor
+    (snarrow x y)
+    (inst_data (InstructionData.Binary (Opcode.Snarrow) (value_array_2 x y)))
+)
+
+(decl unarrow (Value Value) Inst)
+(extractor
+    (unarrow x y)
+    (inst_data (InstructionData.Binary (Opcode.Unarrow) (value_array_2 x y)))
+)
+
+(decl uunarrow (Value Value) Inst)
+(extractor
+    (uunarrow x y)
+    (inst_data (InstructionData.Binary (Opcode.Uunarrow) (value_array_2 x y)))
+)
+
+(decl swiden_low (Value) Inst)
+(extractor
+    (swiden_low x)
+    (inst_data (InstructionData.Unary (Opcode.SwidenLow) x))
+)
+
+(decl swiden_high (Value) Inst)
+(extractor
+    (swiden_high x)
+    (inst_data (InstructionData.Unary (Opcode.SwidenHigh) x))
+)
+
+(decl uwiden_low (Value) Inst)
+(extractor
+    (uwiden_low x)
+    (inst_data (InstructionData.Unary (Opcode.UwidenLow) x))
+)
+
+(decl uwiden_high (Value) Inst)
+(extractor
+    (uwiden_high x)
+    (inst_data (InstructionData.Unary (Opcode.UwidenHigh) x))
+)
+
+(decl iadd_pairwise (Value Value) Inst)
+(extractor
+    (iadd_pairwise x y)
+    (inst_data (InstructionData.Binary (Opcode.IaddPairwise) (value_array_2 x y)))
+)
+
+(decl widening_pairwise_dot_product_s (Value Value) Inst)
+(extractor
+    (widening_pairwise_dot_product_s x y)
+    (inst_data (InstructionData.Binary (Opcode.WideningPairwiseDotProductS) (value_array_2 x y)))
+)
+
+(decl uextend (Value) Inst)
+(extractor
+    (uextend x)
+    (inst_data (InstructionData.Unary (Opcode.Uextend) x))
+)
+
+(decl sextend (Value) Inst)
+(extractor
+    (sextend x)
+    (inst_data (InstructionData.Unary (Opcode.Sextend) x))
+)
+
+(decl fpromote (Value) Inst)
+(extractor
+    (fpromote x)
+    (inst_data (InstructionData.Unary (Opcode.Fpromote) x))
+)
+
+(decl fdemote (Value) Inst)
+(extractor
+    (fdemote x)
+    (inst_data (InstructionData.Unary (Opcode.Fdemote) x))
+)
+
+(decl fvdemote (Value) Inst)
+(extractor
+    (fvdemote x)
+    (inst_data (InstructionData.Unary (Opcode.Fvdemote) x))
+)
+
+(decl fvpromote_low (Value) Inst)
+(extractor
+    (fvpromote_low a)
+    (inst_data (InstructionData.Unary (Opcode.FvpromoteLow) a))
+)
+
+(decl fcvt_to_uint (Value) Inst)
+(extractor
+    (fcvt_to_uint x)
+    (inst_data (InstructionData.Unary (Opcode.FcvtToUint) x))
+)
+
+(decl fcvt_to_uint_sat (Value) Inst)
+(extractor
+    (fcvt_to_uint_sat x)
+    (inst_data (InstructionData.Unary (Opcode.FcvtToUintSat) x))
+)
+
+(decl fcvt_to_sint (Value) Inst)
+(extractor
+    (fcvt_to_sint x)
+    (inst_data (InstructionData.Unary (Opcode.FcvtToSint) x))
+)
+
+(decl fcvt_to_sint_sat (Value) Inst)
+(extractor
+    (fcvt_to_sint_sat x)
+    (inst_data (InstructionData.Unary (Opcode.FcvtToSintSat) x))
+)
+
+(decl fcvt_from_uint (Value) Inst)
+(extractor
+    (fcvt_from_uint x)
+    (inst_data (InstructionData.Unary (Opcode.FcvtFromUint) x))
+)
+
+(decl fcvt_from_sint (Value) Inst)
+(extractor
+    (fcvt_from_sint x)
+    (inst_data (InstructionData.Unary (Opcode.FcvtFromSint) x))
+)
+
+(decl fcvt_low_from_sint (Value) Inst)
+(extractor
+    (fcvt_low_from_sint x)
+    (inst_data (InstructionData.Unary (Opcode.FcvtLowFromSint) x))
+)
+
+(decl isplit (Value) Inst)
+(extractor
+    (isplit x)
+    (inst_data (InstructionData.Unary (Opcode.Isplit) x))
+)
+
+(decl iconcat (Value Value) Inst)
+(extractor
+    (iconcat lo hi)
+    (inst_data (InstructionData.Binary (Opcode.Iconcat) (value_array_2 lo hi)))
+)
+
+(decl atomic_rmw (MemFlags AtomicRmwOp Value Value) Inst)
+(extractor
+    (atomic_rmw MemFlags AtomicRmwOp p x)
+    (inst_data (InstructionData.AtomicRmw (Opcode.AtomicRmw) MemFlags AtomicRmwOp (value_array_2 p x)))
+)
+
+(decl atomic_cas (MemFlags Value Value Value) Inst)
+(extractor
+    (atomic_cas MemFlags p e x)
+    (inst_data (InstructionData.AtomicCas (Opcode.AtomicCas) MemFlags (value_array_3 p e x)))
+)
+
+(decl atomic_load (MemFlags Value) Inst)
+(extractor
+    (atomic_load MemFlags p)
+    (inst_data (InstructionData.LoadNoOffset (Opcode.AtomicLoad) MemFlags p))
+)
+
+(decl atomic_store (MemFlags Value Value) Inst)
+(extractor
+    (atomic_store MemFlags x p)
+    (inst_data (InstructionData.StoreNoOffset (Opcode.AtomicStore) MemFlags (value_array_2 x p)))
+)
+
+(decl fence () Inst)
+(extractor
+    (fence )
+    (inst_data (InstructionData.NullAry (Opcode.Fence)))
+)
+
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
new file mode 100644
index 0000000000..dc00a84cc4
--- /dev/null
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -0,0 +1,977 @@
+;; Extern type definitions and constructors for the x64 `MachInst` type.
+
+;;;; `MInst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(type MInst extern
+      (enum (Nop (len u8))
+            (AluRmiR (size OperandSize)
+                     (op AluRmiROpcode)
+                     (src1 Reg)
+                     (src2 RegMemImm)
+                     (dst WritableReg))
+            (MulHi (size OperandSize)
+                   (signed bool)
+                   (src1 Reg)
+                   (src2 RegMem)
+                   (dst_lo WritableReg)
+                   (dst_hi WritableReg))
+            (XmmRmR (op SseOpcode)
+                    (src1 Reg)
+                    (src2 RegMem)
+                    (dst WritableReg))
+            (XmmUnaryRmR (op SseOpcode)
+                         (src RegMem)
+                         (dst WritableReg))
+            (XmmRmiReg (opcode SseOpcode)
+                       (src1 Reg)
+                       (src2 RegMemImm)
+                       (dst WritableReg))
+            (XmmRmRImm (op SseOpcode)
+                       (src1 Reg)
+                       (src2 RegMem)
+                       (dst WritableReg)
+                       (imm u8)
+                       (size OperandSize))
+            (CmpRmiR (size OperandSize)
+                     (opcode CmpOpcode)
+                     (src RegMemImm)
+                     (dst Reg))
+            (Imm (dst_size OperandSize)
+                 (simm64 u64)
+                 (dst WritableReg))
+            (ShiftR (size OperandSize)
+                    (kind ShiftKind)
+                    (src Reg)
+                    (num_bits Imm8Reg)
+                    (dst WritableReg))
+            (MovzxRmR (ext_mode ExtMode)
+                      (src RegMem)
+                      (dst WritableReg))
+            (MovsxRmR (ext_mode ExtMode)
+                      (src RegMem)
+                      (dst WritableReg))
+            (Cmove (size OperandSize)
+                   (cc CC)
+                   (consequent RegMem)
+                   (alternative Reg)
+                   (dst WritableReg))
+            (XmmRmREvex (op Avx512Opcode)
+                        (src1 RegMem)
+                        (src2 Reg)
+                        (dst WritableReg))))
+
+(type OperandSize extern
+      (enum Size8
+            Size16
+            Size32
+            Size64))
+
+;; Get the `OperandSize` for a given `Type`.
+(decl operand_size_of_type (Type) OperandSize)
+(extern constructor operand_size_of_type operand_size_of_type)
+
+;; Get the bit width of an `OperandSize`.
+(decl operand_size_bits (OperandSize) u16)
+(rule (operand_size_bits (OperandSize.Size8)) 8)
+(rule (operand_size_bits (OperandSize.Size16)) 16)
+(rule (operand_size_bits (OperandSize.Size32)) 32)
+(rule (operand_size_bits (OperandSize.Size64)) 64)
+
+(type AluRmiROpcode extern
+      (enum Add
+            Adc
+            Sub
+            Sbb
+            And
+            Or
+            Xor
+            Mul
+            And8
+            Or8))
+
+(type SseOpcode extern
+      (enum Addps
+            Addpd
+            Addss
+            Addsd
+            Andps
+            Andpd
+            Andnps
+            Andnpd
+            Blendvpd
+            Blendvps
+            Comiss
+            Comisd
+            Cmpps
+            Cmppd
+            Cmpss
+            Cmpsd
+            Cvtdq2ps
+            Cvtdq2pd
+            Cvtpd2ps
+            Cvtps2pd
+            Cvtsd2ss
+            Cvtsd2si
+            Cvtsi2ss
+            Cvtsi2sd
+            Cvtss2si
+            Cvtss2sd
+            Cvttpd2dq
+            Cvttps2dq
+            Cvttss2si
+            Cvttsd2si
+            Divps
+            Divpd
+            Divss
+            Divsd
+            Insertps
+            Maxps
+            Maxpd
+            Maxss
+            Maxsd
+            Minps
+            Minpd
+            Minss
+            Minsd
+            Movaps
+            Movapd
+            Movd
+            Movdqa
+            Movdqu
+            Movlhps
+            Movmskps
+            Movmskpd
+            Movq
+            Movss
+            Movsd
+            Movups
+            Movupd
+            Mulps
+            Mulpd
+            Mulss
+            Mulsd
+            Orps
+            Orpd
+            Pabsb
+            Pabsw
+            Pabsd
+            Packssdw
+            Packsswb
+            Packusdw
+            Packuswb
+            Paddb
+            Paddd
+            Paddq
+            Paddw
+            Paddsb
+            Paddsw
+            Paddusb
+            Paddusw
+            Palignr
+            Pand
+            Pandn
+            Pavgb
+            Pavgw
+            Pblendvb
+            Pcmpeqb
+            Pcmpeqw
+            Pcmpeqd
+            Pcmpeqq
+            Pcmpgtb
+            Pcmpgtw
+            Pcmpgtd
+            Pcmpgtq
+            Pextrb
+            Pextrw
+            Pextrd
+            Pinsrb
+            Pinsrw
+            Pinsrd
+            Pmaddubsw
+            Pmaddwd
+            Pmaxsb
+            Pmaxsw
+            Pmaxsd
+            Pmaxub
+            Pmaxuw
+            Pmaxud
+            Pminsb
+            Pminsw
+            Pminsd
+            Pminub
+            Pminuw
+            Pminud
+            Pmovmskb
+            Pmovsxbd
+            Pmovsxbw
+            Pmovsxbq
+            Pmovsxwd
+            Pmovsxwq
+            Pmovsxdq
+            Pmovzxbd
+            Pmovzxbw
+            Pmovzxbq
+            Pmovzxwd
+            Pmovzxwq
+            Pmovzxdq
+            Pmuldq
+            Pmulhw
+            Pmulhuw
+            Pmulhrsw
+            Pmulld
+            Pmullw
+            Pmuludq
+            Por
+            Pshufb
+            Pshufd
+            Psllw
+            Pslld
+            Psllq
+            Psraw
+            Psrad
+            Psrlw
+            Psrld
+            Psrlq
+            Psubb
+            Psubd
+            Psubq
+            Psubw
+            Psubsb
+            Psubsw
+            Psubusb
+            Psubusw
+            Ptest
+            Punpckhbw
+            Punpckhwd
+            Punpcklbw
+            Punpcklwd
+            Pxor
+            Rcpss
+            Roundps
+            Roundpd
+            Roundss
+            Roundsd
+            Rsqrtss
+            Shufps
+            Sqrtps
+            Sqrtpd
+            Sqrtss
+            Sqrtsd
+            Subps
+            Subpd
+            Subss
+            Subsd
+            Ucomiss
+            Ucomisd
+            Unpcklps
+            Xorps
+            Xorpd))
+
+(type CmpOpcode extern
+      (enum Cmp
+            Test))
+
+(type RegMemImm extern
+      (enum
+       (Reg (reg Reg))
+       (Mem (addr SyntheticAmode))
+       (Imm (simm32 u32))))
+
+(type RegMem extern
+      (enum
+       (Reg (reg Reg))
+       (Mem (addr SyntheticAmode))))
+
+;; Put the given clif value into a `RegMem` operand.
+;;
+;; Asserts that the value fits into a single register, and doesn't require
+;; multiple registers for its representation (like `i128` for example).
+;;
+;; As a side effect, this marks the value as used.
+(decl put_in_reg_mem (Value) RegMem)
+(extern constructor put_in_reg_mem put_in_reg_mem)
+
+(type SyntheticAmode extern (enum))
+
+(type ShiftKind extern
+      (enum ShiftLeft
+            ShiftRightLogical
+            ShiftRightArithmetic
+            RotateLeft
+            RotateRight))
+
+(type Imm8Reg extern
+      (enum (Imm8 (imm u8))
+            (Reg (reg Reg))))
+
+(type CC extern
+      (enum O
+            NO
+            B
+            NB
+            Z
+            NZ
+            BE
+            NBE
+            S
+            NS
+            L
+            NL
+            LE
+            NLE
+            P
+            NP))
+
+(type Avx512Opcode extern
+      (enum Vcvtudq2ps
+            Vpabsq
+            Vpermi2b
+            Vpmullq
+            Vpopcntb))
+
+;;;; Helpers for Querying Enabled ISA Extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl avx512vl_enabled () Type)
+(extern extractor avx512vl_enabled avx512vl_enabled)
+
+(decl avx512dq_enabled () Type)
+(extern extractor avx512dq_enabled avx512dq_enabled)
+
+;;;; Helpers for Merging and Sinking Immediates/Loads  ;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Extract a constant `Imm8Reg.Imm8` from a value operand.
+(decl imm8_from_value (Imm8Reg) Value)
+(extern extractor imm8_from_value imm8_from_value)
+
+;; Extract a constant `RegMemImm.Imm` from a value operand.
+(decl simm32_from_value (RegMemImm) Value)
+(extern extractor simm32_from_value simm32_from_value)
+
+;; Extract a constant `RegMemImm.Imm` from an `Imm64` immediate.
+(decl simm32_from_imm64 (RegMemImm) Imm64)
+(extern extractor simm32_from_imm64 simm32_from_imm64)
+
+;; A load that can be sunk into another operation.
+(type SinkableLoad extern (enum))
+
+;; Extract a `SinkableLoad` that works with `RegMemImm.Mem` from a value
+;; operand.
+(decl sinkable_load (SinkableLoad) Value)
+(extern extractor sinkable_load sinkable_load)
+
+;; Sink a `SinkableLoad` into a `RegMemImm.Mem`.
+;;
+;; This is a side-effectful operation that notifies the context that the
+;; instruction that produced the `SinkableImm` has been sunk into another
+;; instruction, and no longer needs to be lowered.
+(decl sink_load (SinkableLoad) RegMemImm)
+(extern constructor sink_load sink_load)
+
+;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Newtype wrapper around `MInst` for instructions that are used for their
+;; effect on flags.
+(type ProducesFlags (enum (ProducesFlags (inst MInst) (result Reg))))
+
+;; Newtype wrapper around `MInst` for instructions that consume flags.
+(type ConsumesFlags (enum (ConsumesFlags (inst MInst) (result Reg))))
+
+;; Combine flags-producing and -consuming instructions together, ensuring that
+;; they are emitted back-to-back and no other instructions can be emitted
+;; between them and potentially clobber the flags.
+;;
+;; Returns a `ValueRegs` where the first register is the result of the
+;; `ProducesFlags` instruction and the second is the result of the
+;; `ConsumesFlags` instruction.
+(decl with_flags (ProducesFlags ConsumesFlags) ValueRegs)
+(rule (with_flags (ProducesFlags.ProducesFlags producer_inst producer_result)
+                  (ConsumesFlags.ConsumesFlags consumer_inst consumer_result))
+      (let ((_x Unit (emit producer_inst))
+            (_y Unit (emit consumer_inst)))
+        (value_regs producer_result consumer_result)))
+
+;; Like `with_flags` but returns only the result of the consumer operation.
+(decl with_flags_1 (ProducesFlags ConsumesFlags) Reg)
+(rule (with_flags_1 (ProducesFlags.ProducesFlags producer_inst _producer_result)
+                    (ConsumesFlags.ConsumesFlags consumer_inst consumer_result))
+      (let ((_x Unit (emit producer_inst))
+            (_y Unit (emit consumer_inst)))
+        consumer_result))
+
+;; Like `with_flags` but allows two consumers of the same flags. The result is a
+;; `ValueRegs` containing the first consumer's result and then the second
+;; consumer's result.
+(decl with_flags_2 (ProducesFlags ConsumesFlags ConsumesFlags) ValueRegs)
+(rule (with_flags_2 (ProducesFlags.ProducesFlags producer_inst producer_result)
+                    (ConsumesFlags.ConsumesFlags consumer_inst_1 consumer_result_1)
+                    (ConsumesFlags.ConsumesFlags consumer_inst_2 consumer_result_2))
+      (let ((_x Unit (emit producer_inst))
+            (_y Unit (emit consumer_inst_1))
+            (_z Unit (emit consumer_inst_2)))
+        (value_regs consumer_result_1 consumer_result_2)))
+
+;;;; Helpers for Sign/Zero Extending ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(type ExtendKind (enum Sign Zero))
+
+(type ExtMode extern (enum BL BQ WL WQ LQ))
+
+;; `ExtMode::new`
+(decl ext_mode (u16 u16) ExtMode)
+(extern constructor ext_mode ext_mode)
+
+;; Put the given value into a register, but extended as the given type.
+(decl extend_to_reg (Value Type ExtendKind) Reg)
+
+;; If the value is already of the requested type, no extending is necessary.
+(rule (extend_to_reg (and val (value_type ty)) =ty _kind)
+      (put_in_reg val))
+
+(rule (extend_to_reg (and val (value_type from_ty))
+                     to_ty
+                     kind)
+      (let ((from_bits u16 (ty_bits from_ty))
+            ;; Use `operand_size_of_type` so that the we clamp the output to 32-
+            ;; or 64-bit width types.
+            (to_bits u16 (operand_size_bits (operand_size_of_type to_ty))))
+        (extend kind
+                to_ty
+                (ext_mode from_bits to_bits)
+                (put_in_reg_mem val))))
+
+;; Do a sign or zero extension of the given `RegMem`.
+(decl extend (ExtendKind Type ExtMode RegMem) Reg)
+
+;; Zero extending uses `movzx`.
+(rule (extend (ExtendKind.Zero) ty mode src)
+      (movzx ty mode src))
+
+;; Sign extending uses `movsx`.
+(rule (extend (ExtendKind.Sign) ty mode src)
+      (movsx ty mode src))
+
+;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; These constructors create SSA-style `MInst`s. It is their responsibility to
+;; maintain the invariant that each temporary register they allocate and define
+;; only gets defined the once.
+
+;; Emit an instruction.
+;;
+;; This is low-level and side-effectful; it should only be used as an
+;; implementation detail by helpers that preserve the SSA facade themselves.
+(decl emit (MInst) Unit)
+(extern constructor emit emit)
+
+;; Helper for emitting `MInst.AluRmiR` instructions.
+(decl alu_rmi_r (Type AluRmiROpcode Reg RegMemImm) Reg)
+(rule (alu_rmi_r ty opcode src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (size OperandSize (operand_size_of_type ty))
+            (_ Unit (emit (MInst.AluRmiR size opcode src1 src2 dst))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `add` instructions.
+(decl add (Type Reg RegMemImm) Reg)
+(rule (add ty src1 src2)
+      (alu_rmi_r ty
+                 (AluRmiROpcode.Add)
+                 src1
+                 src2))
+
+;; Helper for creating `add` instructions whose flags are also used.
+(decl add_with_flags (Type Reg RegMemImm) ProducesFlags)
+(rule (add_with_flags ty src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type ty)
+                                                    (AluRmiROpcode.Add)
+                                                    src1
+                                                    src2
+                                                    dst)
+                                     (writable_reg_to_reg dst))))
+
+;; Helper for creating `adc` instructions.
+(decl adc (Type Reg RegMemImm) ConsumesFlags)
+(rule (adc ty src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type ty)
+                                                    (AluRmiROpcode.Adc)
+                                                    src1
+                                                    src2
+                                                    dst)
+                                     (writable_reg_to_reg dst))))
+
+;; Helper for emitting `sub` instructions.
+(decl sub (Type Reg RegMemImm) Reg)
+(rule (sub ty src1 src2)
+      (alu_rmi_r ty
+                 (AluRmiROpcode.Sub)
+                 src1
+                 src2))
+
+;; Helper for creating `sub` instructions whose flags are also used.
+(decl sub_with_flags (Type Reg RegMemImm) ProducesFlags)
+(rule (sub_with_flags ty src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ProducesFlags.ProducesFlags (MInst.AluRmiR (operand_size_of_type ty)
+                                                    (AluRmiROpcode.Sub)
+                                                    src1
+                                                    src2
+                                                    dst)
+                                     (writable_reg_to_reg dst))))
+
+;; Helper for creating `sbb` instructions.
+(decl sbb (Type Reg RegMemImm) ConsumesFlags)
+(rule (sbb ty src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ConsumesFlags.ConsumesFlags (MInst.AluRmiR (operand_size_of_type ty)
+                                                    (AluRmiROpcode.Sbb)
+                                                    src1
+                                                    src2
+                                                    dst)
+                                     (writable_reg_to_reg dst))))
+
+;; Helper for creating `mul` instructions.
+(decl mul (Type Reg RegMemImm) Reg)
+(rule (mul ty src1 src2)
+      (alu_rmi_r ty
+                 (AluRmiROpcode.Mul)
+                 src1
+                 src2))
+
+;; Helper for emitting `and` instructions.
+;;
+;; Use `m_` prefix (short for "mach inst") to disambiguate with the ISLE-builtin
+;; `and` operator.
+(decl m_and (Type Reg RegMemImm) Reg)
+(rule (m_and ty src1 src2)
+      (alu_rmi_r ty
+                 (AluRmiROpcode.And)
+                 src1
+                 src2))
+
+;; Helper for emitting `or` instructions.
+(decl or (Type Reg RegMemImm) Reg)
+(rule (or ty src1 src2)
+      (alu_rmi_r ty
+                 (AluRmiROpcode.Or)
+                 src1
+                 src2))
+
+;; Helper for emitting `xor` instructions.
+(decl xor (Type Reg RegMemImm) Reg)
+(rule (xor ty src1 src2)
+      (alu_rmi_r ty
+                 (AluRmiROpcode.Xor)
+                 src1
+                 src2))
+
+;; Helper for emitting immediates.
+(decl imm (Type u64) Reg)
+(rule (imm ty simm64)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (size OperandSize (operand_size_of_type ty))
+            (_ Unit (emit (MInst.Imm size simm64 dst))))
+        (writable_reg_to_reg dst)))
+
+(decl nonzero_u64_fits_in_u32 (u64) u64)
+(extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32)
+
+;; Special case for when a 64-bit immediate fits into 32-bits. We can use a
+;; 32-bit move that zero-extends the value, which has a smaller encoding.
+(rule (imm $I64 (nonzero_u64_fits_in_u32 x))
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.Imm (OperandSize.Size32) x dst))))
+        (writable_reg_to_reg dst)))
+
+;; Special case for zero immediates: turn them into an `xor r, r`.
+(rule (imm ty 0)
+      (let ((wr WritableReg (temp_writable_reg ty))
+            (r Reg (writable_reg_to_reg wr))
+            (size OperandSize (operand_size_of_type ty))
+            (_ Unit (emit (MInst.AluRmiR size
+                                         (AluRmiROpcode.Xor)
+                                         r
+                                         (RegMemImm.Reg r)
+                                         wr))))
+        r))
+
+;; Helper for creating `MInst.ShifR` instructions.
+(decl shift_r (Type ShiftKind Reg Imm8Reg) Reg)
+(rule (shift_r ty kind src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (size OperandSize (operand_size_of_type ty))
+            (_ Unit (emit (MInst.ShiftR size kind src1 src2 dst))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for creating `rotl` instructions (prefixed with "m_", short for "mach
+;; inst", to disambiguate this from clif's `rotl`).
+(decl m_rotl (Type Reg Imm8Reg) Reg)
+(rule (m_rotl ty src1 src2)
+      (shift_r ty (ShiftKind.RotateLeft) src1 src2))
+
+;; Helper for creating `shl` instructions.
+(decl shl (Type Reg Imm8Reg) Reg)
+(rule (shl ty src1 src2)
+      (shift_r ty (ShiftKind.ShiftLeft) src1 src2))
+
+;; Helper for creating logical shift-right instructions.
+(decl shr (Type Reg Imm8Reg) Reg)
+(rule (shr ty src1 src2)
+      (shift_r ty (ShiftKind.ShiftRightLogical) src1 src2))
+
+;; Helper for creating arithmetic shift-right instructions.
+(decl sar (Type Reg Imm8Reg) Reg)
+(rule (sar ty src1 src2)
+      (shift_r ty (ShiftKind.ShiftRightArithmetic) src1 src2))
+
+;; Helper for creating `MInst.CmpRmiR` instructions.
+(decl cmp_rmi_r (OperandSize CmpOpcode RegMemImm Reg) ProducesFlags)
+(rule (cmp_rmi_r size opcode src1 src2)
+      (ProducesFlags.ProducesFlags (MInst.CmpRmiR size
+                                                  opcode
+                                                  src1
+                                                  src2)
+                                   (invalid_reg)))
+
+;; Helper for creating `cmp` instructions.
+(decl cmp (OperandSize RegMemImm Reg) ProducesFlags)
+(rule (cmp size src1 src2)
+      (cmp_rmi_r size (CmpOpcode.Cmp) src1 src2))
+
+;; Helper for creating `test` instructions.
+(decl test (OperandSize RegMemImm Reg) ProducesFlags)
+(rule (test size src1 src2)
+      (cmp_rmi_r size (CmpOpcode.Test) src1 src2))
+
+;; Helper for creating `MInst.Cmove` instructions.
+(decl cmove (Type CC RegMem Reg) ConsumesFlags)
+(rule (cmove ty cc consequent alternative)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (size OperandSize (operand_size_of_type ty)))
+        (ConsumesFlags.ConsumesFlags (MInst.Cmove size cc consequent alternative dst)
+                                     (writable_reg_to_reg dst))))
+
+;; Helper for creating `MInst.MovzxRmR` instructions.
+(decl movzx (Type ExtMode RegMem) Reg)
+(rule (movzx ty mode src)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.MovzxRmR mode src dst))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for creating `MInst.MovsxRmR` instructions.
+(decl movsx (Type ExtMode RegMem) Reg)
+(rule (movsx ty mode src)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.MovsxRmR mode src dst))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for creating `MInst.XmmRmR` instructions.
+(decl xmm_rm_r (Type SseOpcode Reg RegMem) Reg)
+(rule (xmm_rm_r ty op src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.XmmRmR op src1 src2 dst))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for creating `paddb` instructions.
+(decl paddb (Reg RegMem) Reg)
+(rule (paddb src1 src2)
+      (xmm_rm_r $I8X16 (SseOpcode.Paddb) src1 src2))
+
+;; Helper for creating `paddw` instructions.
+(decl paddw (Reg RegMem) Reg)
+(rule (paddw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Paddw) src1 src2))
+
+;; Helper for creating `paddd` instructions.
+(decl paddd (Reg RegMem) Reg)
+(rule (paddd src1 src2)
+      (xmm_rm_r $I32X4 (SseOpcode.Paddd) src1 src2))
+
+;; Helper for creating `paddq` instructions.
+(decl paddq (Reg RegMem) Reg)
+(rule (paddq src1 src2)
+      (xmm_rm_r $I64X2 (SseOpcode.Paddq) src1 src2))
+
+;; Helper for creating `paddsb` instructions.
+(decl paddsb (Reg RegMem) Reg)
+(rule (paddsb src1 src2)
+      (xmm_rm_r $I8X16 (SseOpcode.Paddsb) src1 src2))
+
+;; Helper for creating `paddsw` instructions.
+(decl paddsw (Reg RegMem) Reg)
+(rule (paddsw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Paddsw) src1 src2))
+
+;; Helper for creating `paddusb` instructions.
+(decl paddusb (Reg RegMem) Reg)
+(rule (paddusb src1 src2)
+      (xmm_rm_r $I8X16 (SseOpcode.Paddusb) src1 src2))
+
+;; Helper for creating `paddusw` instructions.
+(decl paddusw (Reg RegMem) Reg)
+(rule (paddusw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Paddusw) src1 src2))
+
+;; Helper for creating `psubb` instructions.
+(decl psubb (Reg RegMem) Reg)
+(rule (psubb src1 src2)
+      (xmm_rm_r $I8X16 (SseOpcode.Psubb) src1 src2))
+
+;; Helper for creating `psubw` instructions.
+(decl psubw (Reg RegMem) Reg)
+(rule (psubw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Psubw) src1 src2))
+
+;; Helper for creating `psubd` instructions.
+(decl psubd (Reg RegMem) Reg)
+(rule (psubd src1 src2)
+      (xmm_rm_r $I32X4 (SseOpcode.Psubd) src1 src2))
+
+;; Helper for creating `psubq` instructions.
+(decl psubq (Reg RegMem) Reg)
+(rule (psubq src1 src2)
+      (xmm_rm_r $I64X2 (SseOpcode.Psubq) src1 src2))
+
+;; Helper for creating `psubsb` instructions.
+(decl psubsb (Reg RegMem) Reg)
+(rule (psubsb src1 src2)
+      (xmm_rm_r $I8X16 (SseOpcode.Psubsb) src1 src2))
+
+;; Helper for creating `psubsw` instructions.
+(decl psubsw (Reg RegMem) Reg)
+(rule (psubsw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Psubsw) src1 src2))
+
+;; Helper for creating `psubusb` instructions.
+(decl psubusb (Reg RegMem) Reg)
+(rule (psubusb src1 src2)
+      (xmm_rm_r $I8X16 (SseOpcode.Psubusb) src1 src2))
+
+;; Helper for creating `psubusw` instructions.
+(decl psubusw (Reg RegMem) Reg)
+(rule (psubusw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Psubusw) src1 src2))
+
+;; Helper for creating `pavgb` instructions.
+(decl pavgb (Reg RegMem) Reg)
+(rule (pavgb src1 src2)
+      (xmm_rm_r $I8X16 (SseOpcode.Pavgb) src1 src2))
+
+;; Helper for creating `pavgw` instructions.
+(decl pavgw (Reg RegMem) Reg)
+(rule (pavgw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Pavgw) src1 src2))
+
+;; Helper for creating `pand` instructions.
+(decl pand (Reg RegMem) Reg)
+(rule (pand src1 src2)
+      (xmm_rm_r $F32X4 (SseOpcode.Pand) src1 src2))
+
+;; Helper for creating `andps` instructions.
+(decl andps (Reg RegMem) Reg)
+(rule (andps src1 src2)
+      (xmm_rm_r $F32X4 (SseOpcode.Andps) src1 src2))
+
+;; Helper for creating `andpd` instructions.
+(decl andpd (Reg RegMem) Reg)
+(rule (andpd src1 src2)
+      (xmm_rm_r $F64X2 (SseOpcode.Andpd) src1 src2))
+
+;; Helper for creating `por` instructions.
+(decl por (Reg RegMem) Reg)
+(rule (por src1 src2)
+      (xmm_rm_r $F32X4 (SseOpcode.Por) src1 src2))
+
+;; Helper for creating `orps` instructions.
+(decl orps (Reg RegMem) Reg)
+(rule (orps src1 src2)
+      (xmm_rm_r $F32X4 (SseOpcode.Orps) src1 src2))
+
+;; Helper for creating `orpd` instructions.
+(decl orpd (Reg RegMem) Reg)
+(rule (orpd src1 src2)
+      (xmm_rm_r $F64X2 (SseOpcode.Orpd) src1 src2))
+
+;; Helper for creating `pxor` instructions.
+(decl pxor (Reg RegMem) Reg)
+(rule (pxor src1 src2)
+      (xmm_rm_r $I8X16 (SseOpcode.Pxor) src1 src2))
+
+;; Helper for creating `xorps` instructions.
+(decl xorps (Reg RegMem) Reg)
+(rule (xorps src1 src2)
+      (xmm_rm_r $F32X4 (SseOpcode.Xorps) src1 src2))
+
+;; Helper for creating `xorpd` instructions.
+(decl xorpd (Reg RegMem) Reg)
+(rule (xorpd src1 src2)
+      (xmm_rm_r $F64X2 (SseOpcode.Xorpd) src1 src2))
+
+;; Helper for creating `pmullw` instructions.
+(decl pmullw (Reg RegMem) Reg)
+(rule (pmullw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Pmullw) src1 src2))
+
+;; Helper for creating `pmulld` instructions.
+(decl pmulld (Reg RegMem) Reg)
+(rule (pmulld src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Pmulld) src1 src2))
+
+;; Helper for creating `pmulhw` instructions.
+(decl pmulhw (Reg RegMem) Reg)
+(rule (pmulhw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Pmulhw) src1 src2))
+
+;; Helper for creating `pmulhuw` instructions.
+(decl pmulhuw (Reg RegMem) Reg)
+(rule (pmulhuw src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Pmulhuw) src1 src2))
+
+;; Helper for creating `pmuldq` instructions.
+(decl pmuldq (Reg RegMem) Reg)
+(rule (pmuldq src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Pmuldq) src1 src2))
+
+;; Helper for creating `pmuludq` instructions.
+(decl pmuludq (Reg RegMem) Reg)
+(rule (pmuludq src1 src2)
+      (xmm_rm_r $I64X2 (SseOpcode.Pmuludq) src1 src2))
+
+;; Helper for creating `punpckhwd` instructions.
+(decl punpckhwd (Reg RegMem) Reg)
+(rule (punpckhwd src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Punpckhwd) src1 src2))
+
+;; Helper for creating `punpcklwd` instructions.
+(decl punpcklwd (Reg RegMem) Reg)
+(rule (punpcklwd src1 src2)
+      (xmm_rm_r $I16X8 (SseOpcode.Punpcklwd) src1 src2))
+
+;; Helper for creating `andnps` instructions.
+(decl andnps (Reg RegMem) Reg)
+(rule (andnps src1 src2)
+      (xmm_rm_r $F32X4 (SseOpcode.Andnps) src1 src2))
+
+;; Helper for creating `andnpd` instructions.
+(decl andnpd (Reg RegMem) Reg)
+(rule (andnpd src1 src2)
+      (xmm_rm_r $F64X2 (SseOpcode.Andnpd) src1 src2))
+
+;; Helper for creating `pandn` instructions.
+(decl pandn (Reg RegMem) Reg)
+(rule (pandn src1 src2)
+      (xmm_rm_r $F64X2 (SseOpcode.Pandn) src1 src2))
+
+;; Helper for creating `MInst.XmmRmRImm` instructions.
+(decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Reg)
+(rule (xmm_rm_r_imm op src1 src2 imm size)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.XmmRmRImm op
+                                           src1
+                                           src2
+                                           dst
+                                           imm
+                                           size))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for creating `palignr` instructions.
+(decl palignr (Reg RegMem u8 OperandSize) Reg)
+(rule (palignr src1 src2 imm size)
+      (xmm_rm_r_imm (SseOpcode.Palignr)
+                    src1
+                    src2
+                    imm
+                    size))
+
+;; Helper for creating `pshufd` instructions.
+(decl pshufd (RegMem u8 OperandSize) Reg)
+(rule (pshufd src imm size)
+      (let ((w_dst WritableReg (temp_writable_reg $I8X16))
+            (dst Reg (writable_reg_to_reg w_dst))
+            (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pshufd)
+                                           dst
+                                           src
+                                           w_dst
+                                           imm
+                                           size))))
+        dst))
+
+;; Helper for creating `MInst.XmmUnaryRmR` instructions.
+(decl xmm_unary_rm_r (SseOpcode RegMem) Reg)
+(rule (xmm_unary_rm_r op src)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.XmmUnaryRmR op src dst))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for creating `pmovsxbw` instructions.
+(decl pmovsxbw (RegMem) Reg)
+(rule (pmovsxbw src)
+      (xmm_unary_rm_r (SseOpcode.Pmovsxbw) src))
+
+;; Helper for creating `pmovzxbw` instructions.
+(decl pmovzxbw (RegMem) Reg)
+(rule (pmovzxbw src)
+      (xmm_unary_rm_r (SseOpcode.Pmovzxbw) src))
+
+;; Helper for creating `MInst.XmmRmREvex` instructions.
+(decl xmm_rm_r_evex (Avx512Opcode RegMem Reg) Reg)
+(rule (xmm_rm_r_evex op src1 src2)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.XmmRmREvex op
+                                            src1
+                                            src2
+                                            dst))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for creating `vpmullq` instructions.
+;;
+;; Requires AVX-512 vl and dq.
+(decl vpmullq (RegMem Reg) Reg)
+(rule (vpmullq src1 src2)
+      (xmm_rm_r_evex (Avx512Opcode.Vpmullq)
+                     src1
+                     src2))
+
+;; Helper for creating `MInst.XmmRmiReg` instructions.
+(decl xmm_rmi_reg (SseOpcode Reg RegMemImm) Reg)
+(rule (xmm_rmi_reg op src1 src2)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.XmmRmiReg op
+                                           src1
+                                           src2
+                                           dst))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for creating `psllq` instructions.
+(decl psllq (Reg RegMemImm) Reg)
+(rule (psllq src1 src2)
+      (xmm_rmi_reg (SseOpcode.Psllq) src1 src2))
+
+;; Helper for creating `psrlq` instructions.
+(decl psrlq (Reg RegMemImm) Reg)
+(rule (psrlq src1 src2)
+      (xmm_rmi_reg (SseOpcode.Psrlq) src1 src2))
+
+;; Helper for creating `MInst.MulHi` instructions.
+;;
+;; Returns the (lo, hi) register halves of the multiplication.
+(decl mul_hi (Type bool Reg RegMem) ValueRegs)
+(rule (mul_hi ty signed src1 src2)
+      (let ((dst_lo WritableReg (temp_writable_reg ty))
+            (dst_hi WritableReg (temp_writable_reg ty))
+            (size OperandSize (operand_size_of_type ty))
+            (_ Unit (emit (MInst.MulHi size
+                                       signed
+                                       src1
+                                       src2
+                                       dst_lo
+                                       dst_hi))))
+        (value_regs (writable_reg_to_reg dst_lo)
+                    (writable_reg_to_reg dst_hi))))
+
+;; Helper for creating `mul` instructions that return both the lower and
+;; (unsigned) higher halves of the result.
+(decl mulhi_u (Type Reg RegMem) ValueRegs)
+(rule (mulhi_u ty src1 src2)
+      (mul_hi ty $false src1 src2))
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index f279ee9096..e7aaf108fe 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -1,14 +1,13 @@
 //! Instruction operand sub-components (aka "parts"): definitions and printing.
 
 use super::regs::{self, show_ireg_sized};
-use super::EmitState;
+use super::{EmitState, RegMapper};
 use crate::ir::condcodes::{FloatCC, IntCC};
 use crate::ir::{MemFlags, Type};
 use crate::isa::x64::inst::Inst;
 use crate::machinst::*;
 use regalloc::{
-    PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector,
-    RegUsageMapper, Writable,
+    PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, Writable,
 };
 use smallvec::{smallvec, SmallVec};
 use std::fmt;
@@ -175,7 +174,7 @@ impl SyntheticAmode {
         }
     }
 
-    pub(crate) fn map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM) {
+    pub(crate) fn map_uses<RM: RegMapper>(&mut self, map: &RM) {
         match self {
             SyntheticAmode::Real(addr) => addr.map_uses(map),
             SyntheticAmode::NominalSPOffset { .. } => {
@@ -285,6 +284,25 @@ impl PrettyPrintSized for RegMemImm {
     }
 }
 
+/// An operand which is either an 8-bit integer immediate or a register.
+#[derive(Clone, Debug)]
+pub enum Imm8Reg {
+    Imm8 { imm: u8 },
+    Reg { reg: Reg },
+}
+
+impl From<u8> for Imm8Reg {
+    fn from(imm: u8) -> Self {
+        Self::Imm8 { imm }
+    }
+}
+
+impl From<Reg> for Imm8Reg {
+    fn from(reg: Reg) -> Self {
+        Self::Reg { reg }
+    }
+}
+
 /// An operand which is either an integer Register or a value in Memory.  This can denote an 8, 16,
 /// 32, 64, or 128 bit value.
 #[derive(Clone, Debug)]
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 93132a2aab..53f67d7346 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -147,14 +147,16 @@ pub(crate) fn emit(
         Inst::AluRmiR {
             size,
             op,
-            src,
+            src1,
+            src2,
             dst: reg_g,
         } => {
+            debug_assert_eq!(*src1, reg_g.to_reg());
             let mut rex = RexFlags::from(*size);
             if *op == AluRmiROpcode::Mul {
                 // We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so
                 // we have to special-case it.
-                match src {
+                match src2 {
                     RegMemImm::Reg { reg: reg_e } => {
                         emit_std_reg_reg(
                             sink,
@@ -213,7 +215,7 @@ pub(crate) fn emit(
                 };
                 assert!(!(is_8bit && *size == OperandSize::Size64));
 
-                match src {
+                match src2 {
                     RegMemImm::Reg { reg: reg_e } => {
                         if is_8bit {
                             rex.always_emit_if_8bit_needed(*reg_e);
@@ -323,8 +325,9 @@ pub(crate) fn emit(
             }
         }
 
-        Inst::Not { size, src } => {
-            let rex_flags = RexFlags::from((*size, src.to_reg()));
+        Inst::Not { size, src, dst } => {
+            debug_assert_eq!(*src, dst.to_reg());
+            let rex_flags = RexFlags::from((*size, dst.to_reg()));
             let (opcode, prefix) = match size {
                 OperandSize::Size8 => (0xF6, LegacyPrefixes::None),
                 OperandSize::Size16 => (0xF7, LegacyPrefixes::_66),
@@ -333,12 +336,13 @@ pub(crate) fn emit(
             };
 
             let subopcode = 2;
-            let enc_src = int_reg_enc(src.to_reg());
+            let enc_src = int_reg_enc(dst.to_reg());
             emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_src, rex_flags)
         }
 
-        Inst::Neg { size, src } => {
-            let rex_flags = RexFlags::from((*size, src.to_reg()));
+        Inst::Neg { size, src, dst } => {
+            debug_assert_eq!(*src, dst.to_reg());
+            let rex_flags = RexFlags::from((*size, dst.to_reg()));
             let (opcode, prefix) = match size {
                 OperandSize::Size8 => (0xF6, LegacyPrefixes::None),
                 OperandSize::Size16 => (0xF7, LegacyPrefixes::_66),
@@ -347,15 +351,21 @@ pub(crate) fn emit(
             };
 
             let subopcode = 3;
-            let enc_src = int_reg_enc(src.to_reg());
+            let enc_src = int_reg_enc(dst.to_reg());
             emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_src, rex_flags)
         }
 
         Inst::Div {
             size,
             signed,
+            dividend,
             divisor,
+            dst_quotient,
+            dst_remainder,
         } => {
+            debug_assert_eq!(*dividend, regs::rax());
+            debug_assert_eq!(dst_quotient.to_reg(), regs::rax());
+            debug_assert_eq!(dst_remainder.to_reg(), regs::rdx());
             let (opcode, prefix) = match size {
                 OperandSize::Size8 => (0xF6, LegacyPrefixes::None),
                 OperandSize::Size16 => (0xF7, LegacyPrefixes::_66),
@@ -397,7 +407,18 @@ pub(crate) fn emit(
             }
         }
 
-        Inst::MulHi { size, signed, rhs } => {
+        Inst::MulHi {
+            size,
+            signed,
+            src1,
+            src2,
+            dst_lo,
+            dst_hi,
+        } => {
+            debug_assert_eq!(*src1, regs::rax());
+            debug_assert_eq!(dst_lo.to_reg(), regs::rax());
+            debug_assert_eq!(dst_hi.to_reg(), regs::rdx());
+
             let rex_flags = RexFlags::from(*size);
             let prefix = match size {
                 OperandSize::Size16 => LegacyPrefixes::_66,
@@ -407,7 +428,7 @@ pub(crate) fn emit(
             };
 
             let subopcode = if *signed { 5 } else { 4 };
-            match rhs {
+            match src2 {
                 RegMem::Reg { reg } => {
                     let src = int_reg_enc(*reg);
                     emit_std_enc_enc(sink, prefix, 0xF7, 1, subopcode, src, rex_flags)
@@ -421,28 +442,39 @@ pub(crate) fn emit(
             }
         }
 
-        Inst::SignExtendData { size } => match size {
-            OperandSize::Size8 => {
-                sink.put1(0x66);
-                sink.put1(0x98);
+        Inst::SignExtendData { size, src, dst } => {
+            debug_assert_eq!(*src, regs::rax());
+            debug_assert_eq!(dst.to_reg(), regs::rdx());
+            match size {
+                OperandSize::Size8 => {
+                    sink.put1(0x66);
+                    sink.put1(0x98);
+                }
+                OperandSize::Size16 => {
+                    sink.put1(0x66);
+                    sink.put1(0x99);
+                }
+                OperandSize::Size32 => sink.put1(0x99),
+                OperandSize::Size64 => {
+                    sink.put1(0x48);
+                    sink.put1(0x99);
+                }
             }
-            OperandSize::Size16 => {
-                sink.put1(0x66);
-                sink.put1(0x99);
-            }
-            OperandSize::Size32 => sink.put1(0x99),
-            OperandSize::Size64 => {
-                sink.put1(0x48);
-                sink.put1(0x99);
-            }
-        },
+        }
 
         Inst::CheckedDivOrRemSeq {
             kind,
             size,
+            dividend,
             divisor,
             tmp,
+            dst_quotient,
+            dst_remainder,
         } => {
+            debug_assert_eq!(*dividend, regs::rax());
+            debug_assert_eq!(dst_quotient.to_reg(), regs::rax());
+            debug_assert_eq!(dst_remainder.to_reg(), regs::rdx());
+
             // Generates the following code sequence:
             //
             // ;; check divide by zero:
@@ -792,9 +824,11 @@ pub(crate) fn emit(
         Inst::ShiftR {
             size,
             kind,
+            src,
             num_bits,
             dst,
         } => {
+            debug_assert_eq!(*src, dst.to_reg());
             let subopcode = match kind {
                 ShiftKind::RotateLeft => 0,
                 ShiftKind::RotateRight => 1,
@@ -805,7 +839,8 @@ pub(crate) fn emit(
             let enc_dst = int_reg_enc(dst.to_reg());
             let rex_flags = RexFlags::from((*size, dst.to_reg()));
             match num_bits {
-                None => {
+                Imm8Reg::Reg { reg } => {
+                    debug_assert_eq!(*reg, regs::rcx());
                     let (opcode, prefix) = match size {
                         OperandSize::Size8 => (0xD2, LegacyPrefixes::None),
                         OperandSize::Size16 => (0xD3, LegacyPrefixes::_66),
@@ -820,7 +855,7 @@ pub(crate) fn emit(
                     emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_dst, rex_flags);
                 }
 
-                Some(num_bits) => {
+                Imm8Reg::Imm8 { imm: num_bits } => {
                     let (opcode, prefix) = match size {
                         OperandSize::Size8 => (0xC0, LegacyPrefixes::None),
                         OperandSize::Size16 => (0xC1, LegacyPrefixes::_66),
@@ -840,10 +875,16 @@ pub(crate) fn emit(
             }
         }
 
-        Inst::XmmRmiReg { opcode, src, dst } => {
+        Inst::XmmRmiReg {
+            opcode,
+            src1,
+            src2,
+            dst,
+        } => {
+            debug_assert_eq!(*src1, dst.to_reg());
             let rex = RexFlags::clear_w();
             let prefix = LegacyPrefixes::_66;
-            if let RegMemImm::Imm { simm32 } = src {
+            if let RegMemImm::Imm { simm32 } = src2 {
                 let (opcode_bytes, reg_digit) = match opcode {
                     SseOpcode::Psllw => (0x0F71, 6),
                     SseOpcode::Pslld => (0x0F72, 6),
@@ -874,7 +915,7 @@ pub(crate) fn emit(
                     _ => panic!("invalid opcode: {}", opcode),
                 };
 
-                match src {
+                match src2 {
                     RegMemImm::Reg { reg } => {
                         emit_std_reg_reg(sink, prefix, opcode_bytes, 2, dst.to_reg(), *reg, rex);
                     }
@@ -993,9 +1034,11 @@ pub(crate) fn emit(
         Inst::Cmove {
             size,
             cc,
-            src,
+            consequent,
+            alternative,
             dst: reg_g,
         } => {
+            debug_assert_eq!(*alternative, reg_g.to_reg());
             let rex_flags = RexFlags::from(*size);
             let prefix = match size {
                 OperandSize::Size16 => LegacyPrefixes::_66,
@@ -1004,7 +1047,7 @@ pub(crate) fn emit(
                 _ => unreachable!("invalid size spec for cmove"),
             };
             let opcode = 0x0F40 + cc.get_enc() as u32;
-            match src {
+            match consequent {
                 RegMem::Reg { reg: reg_e } => {
                     emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex_flags);
                 }
@@ -1433,9 +1476,11 @@ pub(crate) fn emit(
 
         Inst::XmmRmR {
             op,
-            src: src_e,
+            src1,
+            src2: src_e,
             dst: reg_g,
         } => {
+            debug_assert_eq!(*src1, reg_g.to_reg());
             let rex = RexFlags::clear_w();
             let (prefix, opcode, length) = match op {
                 SseOpcode::Addps => (LegacyPrefixes::None, 0x0F58, 2),
@@ -1678,11 +1723,13 @@ pub(crate) fn emit(
 
         Inst::XmmRmRImm {
             op,
-            src,
+            src1,
+            src2,
             dst,
             imm,
             size,
         } => {
+            debug_assert_eq!(*src1, dst.to_reg());
             let (prefix, opcode, len) = match op {
                 SseOpcode::Cmpps => (LegacyPrefixes::None, 0x0FC2, 2),
                 SseOpcode::Cmppd => (LegacyPrefixes::_66, 0x0FC2, 2),
@@ -1713,7 +1760,7 @@ pub(crate) fn emit(
                 // `src` in ModRM's r/m field.
                 _ => false,
             };
-            match src {
+            match src2 {
                 RegMem::Reg { reg } => {
                     if regs_swapped {
                         emit_std_reg_reg(sink, prefix, opcode, len, *reg, dst.to_reg(), rex);
@@ -2403,8 +2450,17 @@ pub(crate) fn emit(
             }
         }
 
-        Inst::LockCmpxchg { ty, src, dst } => {
-            // lock cmpxchg{b,w,l,q} %src, (dst)
+        Inst::LockCmpxchg {
+            ty,
+            replacement,
+            expected,
+            mem,
+            dst_old,
+        } => {
+            debug_assert_eq!(*expected, regs::rax());
+            debug_assert_eq!(dst_old.to_reg(), regs::rax());
+
+            // lock cmpxchg{b,w,l,q} %replacement, (mem)
             // Note that 0xF0 is the Lock prefix.
             let (prefix, opcodes) = match *ty {
                 types::I8 => (LegacyPrefixes::_F0, 0x0FB0),
@@ -2413,12 +2469,34 @@ pub(crate) fn emit(
                 types::I64 => (LegacyPrefixes::_F0, 0x0FB1),
                 _ => unreachable!(),
             };
-            let rex = RexFlags::from((OperandSize::from_ty(*ty), *src));
-            let amode = dst.finalize(state, sink);
-            emit_std_reg_mem(sink, state, info, prefix, opcodes, 2, *src, &amode, rex);
+            let rex = RexFlags::from((OperandSize::from_ty(*ty), *replacement));
+            let amode = mem.finalize(state, sink);
+            emit_std_reg_mem(
+                sink,
+                state,
+                info,
+                prefix,
+                opcodes,
+                2,
+                *replacement,
+                &amode,
+                rex,
+            );
         }
 
-        Inst::AtomicRmwSeq { ty, op } => {
+        Inst::AtomicRmwSeq {
+            ty,
+            op,
+            address,
+            operand,
+            temp,
+            dst_old,
+        } => {
+            debug_assert_eq!(*address, regs::r9());
+            debug_assert_eq!(*operand, regs::r10());
+            debug_assert_eq!(temp.to_reg(), regs::r11());
+            debug_assert_eq!(dst_old.to_reg(), regs::rax());
+
             // Emit this:
             //
             //    mov{zbq,zwq,zlq,q}     (%r9), %rax  // rax = old value
@@ -2516,8 +2594,10 @@ pub(crate) fn emit(
             // No need to call `add_trap` here, since the `i4` emit will do that.
             let i4 = Inst::LockCmpxchg {
                 ty: *ty,
-                src: r11,
-                dst: amode.into(),
+                replacement: r11,
+                expected: regs::rax(),
+                mem: amode.into(),
+                dst_old: Writable::from_reg(regs::rax()),
             };
             i4.emit(sink, info, state);
 
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
index 1a81191141..2f753ace1a 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
@@ -4199,8 +4199,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I8,
-            src: rbx,
-            dst: am1,
+            mem: am1,
+            replacement: rbx,
+            expected: rax,
+            dst_old: w_rax,
         },
         "F0410FB09C9241010000",
         "lock cmpxchgb %bl, 321(%r10,%rdx,4)",
@@ -4209,8 +4211,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I8,
-            src: rdx,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: rdx,
+            expected: rax,
+            dst_old: w_rax,
         },
         "F00FB094F1C7CFFFFF",
         "lock cmpxchgb %dl, -12345(%rcx,%rsi,8)",
@@ -4218,8 +4222,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I8,
-            src: rsi,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: rsi,
+            expected: rax,
+            dst_old: w_rax,
         },
         "F0400FB0B4F1C7CFFFFF",
         "lock cmpxchgb %sil, -12345(%rcx,%rsi,8)",
@@ -4227,8 +4233,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I8,
-            src: r10,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: r10,
+            expected: rax,
+            dst_old: w_rax,
         },
         "F0440FB094F1C7CFFFFF",
         "lock cmpxchgb %r10b, -12345(%rcx,%rsi,8)",
@@ -4236,8 +4244,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I8,
-            src: r15,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: r15,
+            expected: rax,
+            dst_old: w_rax,
         },
         "F0440FB0BCF1C7CFFFFF",
         "lock cmpxchgb %r15b, -12345(%rcx,%rsi,8)",
@@ -4246,8 +4256,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I16,
-            src: rsi,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: rsi,
+            expected: rax,
+            dst_old: w_rax,
         },
         "66F00FB1B4F1C7CFFFFF",
         "lock cmpxchgw %si, -12345(%rcx,%rsi,8)",
@@ -4255,8 +4267,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I16,
-            src: r10,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: r10,
+            expected: rax,
+            dst_old: w_rax,
         },
         "66F0440FB194F1C7CFFFFF",
         "lock cmpxchgw %r10w, -12345(%rcx,%rsi,8)",
@@ -4265,8 +4279,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I32,
-            src: rsi,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: rsi,
+            expected: rax,
+            dst_old: w_rax,
         },
         "F00FB1B4F1C7CFFFFF",
         "lock cmpxchgl %esi, -12345(%rcx,%rsi,8)",
@@ -4274,8 +4290,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I32,
-            src: r10,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: r10,
+            expected: rax,
+            dst_old: w_rax,
         },
         "F0440FB194F1C7CFFFFF",
         "lock cmpxchgl %r10d, -12345(%rcx,%rsi,8)",
@@ -4284,8 +4302,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I64,
-            src: rsi,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: rsi,
+            expected: rax,
+            dst_old: w_rax,
         },
         "F0480FB1B4F1C7CFFFFF",
         "lock cmpxchgq %rsi, -12345(%rcx,%rsi,8)",
@@ -4293,8 +4313,10 @@ fn test_x64_emit() {
     insns.push((
         Inst::LockCmpxchg {
             ty: types::I64,
-            src: r10,
-            dst: am2.clone(),
+            mem: am2.clone(),
+            replacement: r10,
+            expected: rax,
+            dst_old: w_rax,
         },
         "F04C0FB194F1C7CFFFFF",
         "lock cmpxchgq %r10, -12345(%rcx,%rsi,8)",
@@ -4302,27 +4324,62 @@ fn test_x64_emit() {
 
     // AtomicRmwSeq
     insns.push((
-        Inst::AtomicRmwSeq { ty: types::I8, op: inst_common::AtomicRmwOp::Or, },
+        Inst::AtomicRmwSeq {
+            ty: types::I8,
+            op: inst_common::AtomicRmwOp::Or,
+            address: r9,
+            operand: r10,
+            temp: w_r11,
+            dst_old: w_rax
+        },
         "490FB6014989C34D09D3F0450FB0190F85EFFFFFFF",
         "atomically { 8_bits_at_[%r9]) Or= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }"
     ));
     insns.push((
-        Inst::AtomicRmwSeq { ty: types::I16, op: inst_common::AtomicRmwOp::And, },
+        Inst::AtomicRmwSeq {
+            ty: types::I16,
+            op: inst_common::AtomicRmwOp::And,
+            address: r9,
+            operand: r10,
+            temp: w_r11,
+            dst_old: w_rax
+        },
         "490FB7014989C34D21D366F0450FB1190F85EEFFFFFF",
         "atomically { 16_bits_at_[%r9]) And= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }"
     ));
     insns.push((
-        Inst::AtomicRmwSeq { ty: types::I32, op: inst_common::AtomicRmwOp::Xchg, },
+        Inst::AtomicRmwSeq {
+            ty: types::I32,
+            op: inst_common::AtomicRmwOp::Xchg,
+            address: r9,
+            operand: r10,
+            temp: w_r11,
+            dst_old: w_rax
+        },
         "418B014989C34D89D3F0450FB1190F85EFFFFFFF",
         "atomically { 32_bits_at_[%r9]) Xchg= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }"
     ));
     insns.push((
-        Inst::AtomicRmwSeq { ty: types::I32, op: inst_common::AtomicRmwOp::Umin, },
+        Inst::AtomicRmwSeq {
+            ty: types::I32,
+            op: inst_common::AtomicRmwOp::Umin,
+            address: r9,
+            operand: r10,
+            temp: w_r11,
+            dst_old: w_rax
+        },
         "418B014989C34539DA4D0F46DAF0450FB1190F85EBFFFFFF",
         "atomically { 32_bits_at_[%r9]) Umin= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }"
     ));
     insns.push((
-        Inst::AtomicRmwSeq { ty: types::I64, op: inst_common::AtomicRmwOp::Add, },
+        Inst::AtomicRmwSeq {
+            ty: types::I64,
+            op: inst_common::AtomicRmwOp::Add,
+            address: r9,
+            operand: r10,
+            temp: w_r11,
+            dst_old: w_rax
+        },
         "498B014989C34D01D3F04D0FB1190F85EFFFFFFF",
         "atomically { 64_bits_at_[%r9]) Add= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }"
     ));
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index e682c6f51c..d011cd2b97 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -11,8 +11,8 @@ use crate::{settings, settings::Flags, CodegenError, CodegenResult};
 use alloc::boxed::Box;
 use alloc::vec::Vec;
 use regalloc::{
-    PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector,
-    RegUsageMapper, SpillSlot, VirtualReg, Writable,
+    PrettyPrint, PrettyPrintSized, RealRegUniverse, Reg, RegClass, RegUsageCollector, SpillSlot,
+    VirtualReg, Writable,
 };
 use smallvec::{smallvec, SmallVec};
 use std::fmt;
@@ -33,7 +33,7 @@ use regs::{create_reg_universe_systemv, show_ireg_sized};
 
 // Don't build these directly.  Instead use the Inst:: functions to create them.
 
-/// Instructions.  Destinations are on the RIGHT (a la AT&T syntax).
+/// Instructions.
 #[derive(Clone)]
 pub enum Inst {
     /// Nops of various sizes, including zero.
@@ -45,7 +45,8 @@ pub enum Inst {
     AluRmiR {
         size: OperandSize, // 4 or 8
         op: AluRmiROpcode,
-        src: RegMemImm,
+        src1: Reg,
+        src2: RegMemImm,
         dst: Writable<Reg>,
     },
 
@@ -60,13 +61,15 @@ pub enum Inst {
     /// Bitwise not
     Not {
         size: OperandSize, // 1, 2, 4 or 8
-        src: Writable<Reg>,
+        src: Reg,
+        dst: Writable<Reg>,
     },
 
     /// Integer negation
     Neg {
         size: OperandSize, // 1, 2, 4 or 8
-        src: Writable<Reg>,
+        src: Reg,
+        dst: Writable<Reg>,
     },
 
     /// Integer quotient and remainder: (div idiv) $rax $rdx (reg addr)
@@ -74,19 +77,27 @@ pub enum Inst {
         size: OperandSize, // 1, 2, 4 or 8
         signed: bool,
         divisor: RegMem,
+        dividend: Reg,
+        dst_quotient: Writable<Reg>,
+        dst_remainder: Writable<Reg>,
     },
 
     /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs.
     MulHi {
         size: OperandSize, // 2, 4, or 8
         signed: bool,
-        rhs: RegMem,
+        src1: Reg,
+        src2: RegMem,
+        dst_lo: Writable<Reg>,
+        dst_hi: Writable<Reg>,
     },
 
     /// A synthetic sequence to implement the right inline checks for remainder and division,
     /// assuming the dividend is in %rax.
+    ///
     /// Puts the result back into %rax if is_div, %rdx if !is_div, to mimic what the div
     /// instruction does.
+    ///
     /// The generated code sequence is described in the emit's function match arm for this
     /// instruction.
     ///
@@ -97,9 +108,12 @@ pub enum Inst {
     CheckedDivOrRemSeq {
         kind: DivOrRemKind,
         size: OperandSize,
+        dividend: Reg,
         /// The divisor operand. Note it's marked as modified so that it gets assigned a register
         /// different from the temporary.
         divisor: Writable<Reg>,
+        dst_quotient: Writable<Reg>,
+        dst_remainder: Writable<Reg>,
         tmp: Option<Writable<Reg>>,
     },
 
@@ -107,9 +121,12 @@ pub enum Inst {
     /// or al into ah: (cbw)
     SignExtendData {
         size: OperandSize, // 1, 2, 4 or 8
+        src: Reg,
+        dst: Writable<Reg>,
     },
 
     /// Constant materialization: (imm32 imm64) reg.
+    ///
     /// Either: movl $imm32, %reg32 or movabsq $imm64, %reg32.
     Imm {
         dst_size: OperandSize, // 4 or 8
@@ -163,15 +180,17 @@ pub enum Inst {
     ShiftR {
         size: OperandSize, // 1, 2, 4 or 8
         kind: ShiftKind,
+        src: Reg,
         /// shift count: Some(0 .. #bits-in-type - 1), or None to mean "%cl".
-        num_bits: Option<u8>,
+        num_bits: Imm8Reg,
         dst: Writable<Reg>,
     },
 
     /// Arithmetic SIMD shifts.
     XmmRmiReg {
         opcode: SseOpcode,
-        src: RegMemImm,
+        src1: Reg,
+        src2: RegMemImm,
         dst: Writable<Reg>,
     },
 
@@ -191,7 +210,8 @@ pub enum Inst {
     Cmove {
         size: OperandSize, // 2, 4, or 8
         cc: CC,
-        src: RegMem,
+        consequent: RegMem,
+        alternative: Reg,
         dst: Writable<Reg>,
     },
 
@@ -208,7 +228,8 @@ pub enum Inst {
     /// XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg
     XmmRmR {
         op: SseOpcode,
-        src: RegMem,
+        src1: Reg,
+        src2: RegMem,
         dst: Writable<Reg>,
     },
 
@@ -337,7 +358,8 @@ pub enum Inst {
     /// A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm (reg addr) reg
     XmmRmRImm {
         op: SseOpcode,
-        src: RegMem,
+        src1: Reg,
+        src2: RegMem,
         dst: Writable<Reg>,
         imm: u8,
         size: OperandSize, // 4 or 8
@@ -428,17 +450,19 @@ pub enum Inst {
     // Instructions pertaining to atomic memory accesses.
     /// A standard (native) `lock cmpxchg src, (amode)`, with register conventions:
     ///
-    /// `dst`  (read) address
-    /// `src`  (read) replacement value
-    /// %rax   (modified) in: expected value, out: value that was actually at `dst`
+    /// `mem`          (read) address
+    /// `replacement`  (read) replacement value
+    /// %rax           (modified) in: expected value, out: value that was actually at `dst`
     /// %rflags is written.  Do not assume anything about it after the instruction.
     ///
     /// The instruction "succeeded" iff the lowest `ty` bits of %rax afterwards are the same as
     /// they were before.
     LockCmpxchg {
         ty: Type, // I8, I16, I32 or I64
-        src: Reg,
-        dst: SyntheticAmode,
+        replacement: Reg,
+        expected: Reg,
+        mem: SyntheticAmode,
+        dst_old: Writable<Reg>,
     },
 
     /// A synthetic instruction, based on a loop around a native `lock cmpxchg` instruction.
@@ -467,6 +491,10 @@ pub enum Inst {
     AtomicRmwSeq {
         ty: Type, // I8, I16, I32 or I64
         op: inst_common::AtomicRmwOp,
+        address: Reg,
+        operand: Reg,
+        temp: Writable<Reg>,
+        dst_old: Writable<Reg>,
     },
 
     /// A memory fence (mfence, lfence or sfence).
@@ -606,7 +634,13 @@ impl Inst {
         debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
         src.assert_regclass_is(RegClass::I64);
         debug_assert!(dst.to_reg().get_class() == RegClass::I64);
-        Self::AluRmiR { size, op, src, dst }
+        Self::AluRmiR {
+            size,
+            op,
+            src1: dst.to_reg(),
+            src2: src,
+            dst,
+        }
     }
 
     pub(crate) fn unary_rm_r(
@@ -627,12 +661,20 @@ impl Inst {
 
     pub(crate) fn not(size: OperandSize, src: Writable<Reg>) -> Inst {
         debug_assert_eq!(src.to_reg().get_class(), RegClass::I64);
-        Inst::Not { size, src }
+        Inst::Not {
+            size,
+            src: src.to_reg(),
+            dst: src,
+        }
     }
 
     pub(crate) fn neg(size: OperandSize, src: Writable<Reg>) -> Inst {
         debug_assert_eq!(src.to_reg().get_class(), RegClass::I64);
-        Inst::Neg { size, src }
+        Inst::Neg {
+            size,
+            src: src.to_reg(),
+            dst: src,
+        }
     }
 
     pub(crate) fn div(size: OperandSize, signed: bool, divisor: RegMem) -> Inst {
@@ -641,6 +683,9 @@ impl Inst {
             size,
             signed,
             divisor,
+            dividend: regs::rax(),
+            dst_quotient: Writable::from_reg(regs::rax()),
+            dst_remainder: Writable::from_reg(regs::rdx()),
         }
     }
 
@@ -651,7 +696,14 @@ impl Inst {
             OperandSize::Size64
         ]));
         rhs.assert_regclass_is(RegClass::I64);
-        Inst::MulHi { size, signed, rhs }
+        Inst::MulHi {
+            size,
+            signed,
+            src1: regs::rax(),
+            src2: rhs,
+            dst_lo: Writable::from_reg(regs::rax()),
+            dst_hi: Writable::from_reg(regs::rdx()),
+        }
     }
 
     pub(crate) fn checked_div_or_rem_seq(
@@ -668,12 +720,19 @@ impl Inst {
             kind,
             size,
             divisor,
+            dividend: regs::rax(),
+            dst_quotient: Writable::from_reg(regs::rax()),
+            dst_remainder: Writable::from_reg(regs::rdx()),
             tmp,
         }
     }
 
     pub(crate) fn sign_extend_data(size: OperandSize) -> Inst {
-        Inst::SignExtendData { size }
+        Inst::SignExtendData {
+            size,
+            src: regs::rax(),
+            dst: Writable::from_reg(regs::rdx()),
+        }
     }
 
     pub(crate) fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
@@ -728,7 +787,12 @@ impl Inst {
     pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self {
         src.assert_regclass_is(RegClass::V128);
         debug_assert!(dst.to_reg().get_class() == RegClass::V128);
-        Inst::XmmRmR { op, src, dst }
+        Inst::XmmRmR {
+            op,
+            src1: dst.to_reg(),
+            src2: src,
+            dst,
+        }
     }
 
     pub(crate) fn xmm_rm_r_evex(
@@ -902,7 +966,8 @@ impl Inst {
         debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
         Inst::XmmRmRImm {
             op,
-            src,
+            src1: dst.to_reg(),
+            src2: src,
             dst,
             imm,
             size,
@@ -918,7 +983,12 @@ impl Inst {
     pub(crate) fn xmm_rmi_reg(opcode: SseOpcode, src: RegMemImm, dst: Writable<Reg>) -> Inst {
         src.assert_regclass_is(RegClass::V128);
         debug_assert!(dst.to_reg().get_class() == RegClass::V128);
-        Inst::XmmRmiReg { opcode, src, dst }
+        Inst::XmmRmiReg {
+            opcode,
+            src1: dst.to_reg(),
+            src2: src,
+            dst,
+        }
     }
 
     pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
@@ -976,7 +1046,11 @@ impl Inst {
         Inst::ShiftR {
             size,
             kind,
-            num_bits,
+            src: dst.to_reg(),
+            num_bits: match num_bits {
+                Some(imm) => Imm8Reg::Imm8 { imm },
+                None => Imm8Reg::Reg { reg: regs::rcx() },
+            },
             dst,
         }
     }
@@ -1024,7 +1098,13 @@ impl Inst {
             OperandSize::Size64
         ]));
         debug_assert!(dst.to_reg().get_class() == RegClass::I64);
-        Inst::Cmove { size, cc, src, dst }
+        Inst::Cmove {
+            size,
+            cc,
+            consequent: src,
+            alternative: dst.to_reg(),
+            dst,
+        }
     }
 
     pub(crate) fn xmm_cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst {
@@ -1193,13 +1273,13 @@ impl Inst {
     /// same as the first register (already handled).
     fn produces_const(&self) -> bool {
         match self {
-            Self::AluRmiR { op, src, dst, .. } => {
-                src.to_reg() == Some(dst.to_reg())
+            Self::AluRmiR { op, src2, dst, .. } => {
+                src2.to_reg() == Some(dst.to_reg())
                     && (*op == AluRmiROpcode::Xor || *op == AluRmiROpcode::Sub)
             }
 
-            Self::XmmRmR { op, src, dst, .. } => {
-                src.to_reg() == Some(dst.to_reg())
+            Self::XmmRmR { op, src2, dst, .. } => {
+                src2.to_reg() == Some(dst.to_reg())
                     && (*op == SseOpcode::Xorps
                         || *op == SseOpcode::Xorpd
                         || *op == SseOpcode::Pxor
@@ -1210,9 +1290,9 @@ impl Inst {
             }
 
             Self::XmmRmRImm {
-                op, src, dst, imm, ..
+                op, src2, dst, imm, ..
             } => {
-                src.to_reg() == Some(dst.to_reg())
+                src2.to_reg() == Some(dst.to_reg())
                     && (*op == SseOpcode::Cmppd || *op == SseOpcode::Cmpps)
                     && *imm == FcmpImm::Equal.encode()
             }
@@ -1285,6 +1365,265 @@ impl Inst {
             _ => unimplemented!("unimplemented type for Inst::xor: {}", ty),
         }
     }
+
+    /// Translate three-operand instructions into a sequence of two-operand
+    /// instructions.
+    ///
+    /// For example:
+    ///
+    /// ```text
+    /// x = add a, b
+    /// ```
+    ///
+    /// Becomes:
+    ///
+    /// ```text
+    /// mov x, a
+    /// add x, b
+    /// ```
+    ///
+    /// The three-operand form for instructions allows our ISLE DSL code to have
+    /// a value-based, SSA view of the world. This method is responsible for
+    /// undoing that.
+    ///
+    /// Note that register allocation cleans up most of these inserted `mov`s
+    /// with its move coalescing.
+    pub(crate) fn mov_mitosis(mut self) -> impl Iterator<Item = Self> {
+        log::trace!("mov_mitosis({:?})", self);
+
+        let mut insts = SmallVec::<[Self; 4]>::new();
+
+        match &mut self {
+            Inst::AluRmiR { src1, dst, .. } => {
+                if *src1 != dst.to_reg() {
+                    debug_assert!(src1.is_virtual());
+                    insts.push(Self::gen_move(*dst, *src1, types::I64));
+                    *src1 = dst.to_reg();
+                }
+                insts.push(self);
+            }
+            Inst::XmmRmiReg { src1, dst, .. } => {
+                if *src1 != dst.to_reg() {
+                    debug_assert!(src1.is_virtual());
+                    insts.push(Self::gen_move(*dst, *src1, types::I8X16));
+                    *src1 = dst.to_reg();
+                }
+                insts.push(self);
+            }
+            Inst::XmmRmR { src1, dst, .. } => {
+                if *src1 != dst.to_reg() {
+                    debug_assert!(src1.is_virtual());
+                    insts.push(Self::gen_move(*dst, *src1, types::I8X16));
+                    *src1 = dst.to_reg();
+                }
+                insts.push(self);
+            }
+            Inst::XmmRmRImm { src1, dst, .. } => {
+                if *src1 != dst.to_reg() {
+                    debug_assert!(src1.is_virtual());
+                    insts.push(Self::gen_move(*dst, *src1, types::I8X16));
+                    *src1 = dst.to_reg();
+                }
+                insts.push(self);
+            }
+            Inst::Cmove {
+                size,
+                alternative,
+                dst,
+                ..
+            } => {
+                if *alternative != dst.to_reg() {
+                    debug_assert!(alternative.is_virtual());
+                    insts.push(Self::mov_r_r(*size, *alternative, *dst));
+                    *alternative = dst.to_reg();
+                }
+                insts.push(self);
+            }
+            Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => {
+                if *src != dst.to_reg() {
+                    debug_assert!(src.is_virtual());
+                    insts.push(Self::gen_move(*dst, *src, types::I64));
+                    *src = dst.to_reg();
+                }
+                insts.push(self);
+            }
+            Inst::Div {
+                dividend,
+                dst_quotient,
+                dst_remainder,
+                ..
+            }
+            | Inst::CheckedDivOrRemSeq {
+                dividend,
+                dst_quotient,
+                dst_remainder,
+                ..
+            } => {
+                if *dividend != regs::rax() {
+                    debug_assert!(dividend.is_virtual());
+                    insts.push(Self::gen_move(
+                        Writable::from_reg(regs::rax()),
+                        *dividend,
+                        types::I64,
+                    ));
+                    *dividend = regs::rax();
+                }
+                let mut quotient_mov = None;
+                if dst_quotient.to_reg() != regs::rax() {
+                    debug_assert!(dst_quotient.to_reg().is_virtual());
+                    quotient_mov = Some(Self::gen_move(*dst_quotient, regs::rax(), types::I64));
+                    *dst_quotient = Writable::from_reg(regs::rax());
+                }
+                let mut remainder_mov = None;
+                if dst_remainder.to_reg() != regs::rdx() {
+                    debug_assert!(dst_remainder.to_reg().is_virtual());
+                    remainder_mov = Some(Self::gen_move(*dst_remainder, regs::rdx(), types::I64));
+                    *dst_remainder = Writable::from_reg(regs::rdx());
+                }
+                insts.push(self);
+                insts.extend(quotient_mov);
+                insts.extend(remainder_mov);
+            }
+            Inst::MulHi {
+                src1,
+                dst_lo,
+                dst_hi,
+                ..
+            } => {
+                if *src1 != regs::rax() {
+                    debug_assert!(src1.is_virtual());
+                    insts.push(Self::gen_move(
+                        Writable::from_reg(regs::rax()),
+                        *src1,
+                        types::I64,
+                    ));
+                    *src1 = regs::rax();
+                }
+                let mut dst_lo_mov = None;
+                if dst_lo.to_reg() != regs::rax() {
+                    debug_assert!(dst_lo.to_reg().is_virtual());
+                    dst_lo_mov = Some(Self::gen_move(*dst_lo, regs::rax(), types::I64));
+                    *dst_lo = Writable::from_reg(regs::rax());
+                }
+                let mut dst_hi_mov = None;
+                if dst_hi.to_reg() != regs::rdx() {
+                    debug_assert!(dst_hi.to_reg().is_virtual());
+                    dst_hi_mov = Some(Self::gen_move(*dst_hi, regs::rdx(), types::I64));
+                    *dst_hi = Writable::from_reg(regs::rdx());
+                }
+                insts.push(self);
+                insts.extend(dst_lo_mov);
+                insts.extend(dst_hi_mov);
+            }
+            Inst::SignExtendData { src, dst, .. } => {
+                if *src != regs::rax() {
+                    debug_assert!(src.is_virtual());
+                    insts.push(Self::gen_move(
+                        Writable::from_reg(regs::rax()),
+                        *src,
+                        types::I64,
+                    ));
+                    *src = regs::rax();
+                }
+                let mut dst_mov = None;
+                if dst.to_reg() != regs::rax() {
+                    debug_assert!(dst.to_reg().is_virtual());
+                    dst_mov = Some(Self::gen_move(*dst, dst.to_reg(), types::I64));
+                    *dst = Writable::from_reg(regs::rax());
+                }
+                insts.push(self);
+                insts.extend(dst_mov);
+            }
+            Inst::ShiftR {
+                src, num_bits, dst, ..
+            } => {
+                if *src != dst.to_reg() {
+                    debug_assert!(src.is_virtual());
+                    insts.push(Self::gen_move(*dst, *src, types::I64));
+                    *src = dst.to_reg();
+                }
+                if let Imm8Reg::Reg { reg } = num_bits {
+                    if *reg != regs::rcx() {
+                        debug_assert!(reg.is_virtual());
+                        insts.push(Self::gen_move(
+                            Writable::from_reg(regs::rcx()),
+                            *reg,
+                            types::I64,
+                        ));
+                        *reg = regs::rcx();
+                    }
+                }
+                insts.push(self);
+            }
+            Inst::LockCmpxchg {
+                ty,
+                expected,
+                dst_old,
+                ..
+            } => {
+                if *expected != regs::rax() {
+                    debug_assert!(expected.is_virtual());
+                    insts.push(Self::gen_move(
+                        Writable::from_reg(regs::rax()),
+                        *expected,
+                        *ty,
+                    ));
+                }
+                let mut dst_old_mov = None;
+                if dst_old.to_reg() != regs::rax() {
+                    debug_assert!(dst_old.to_reg().is_virtual());
+                    dst_old_mov = Some(Self::gen_move(*dst_old, regs::rax(), *ty));
+                    *dst_old = Writable::from_reg(regs::rax());
+                }
+                insts.push(self);
+                insts.extend(dst_old_mov);
+            }
+            Inst::AtomicRmwSeq {
+                ty,
+                address,
+                operand,
+                dst_old,
+                ..
+            } => {
+                if *address != regs::r9() {
+                    debug_assert!(address.is_virtual());
+                    insts.push(Self::gen_move(
+                        Writable::from_reg(regs::r9()),
+                        *address,
+                        types::I64,
+                    ));
+                    *address = regs::r9();
+                }
+                if *operand != regs::r10() {
+                    debug_assert!(operand.is_virtual());
+                    insts.push(Self::gen_move(
+                        Writable::from_reg(regs::r10()),
+                        *operand,
+                        *ty,
+                    ));
+                    *address = regs::r10();
+                }
+                let mut dst_old_mov = None;
+                if dst_old.to_reg() != regs::rax() {
+                    debug_assert!(dst_old.to_reg().is_virtual());
+                    dst_old_mov = Some(Self::gen_move(*dst_old, regs::rax(), *ty));
+                    *dst_old = Writable::from_reg(regs::rax());
+                }
+                insts.push(self);
+                insts.extend(dst_old_mov);
+            }
+            // No other instruction needs 3-operand to 2-operand legalization.
+            _ => insts.push(self),
+        }
+
+        if log::log_enabled!(log::Level::Trace) {
+            for inst in &insts {
+                log::trace!("  -> {:?}", inst);
+            }
+        }
+
+        insts.into_iter()
+    }
 }
 
 //=============================================================================
@@ -1344,10 +1683,16 @@ impl PrettyPrint for Inst {
         match self {
             Inst::Nop { len } => format!("{} len={}", ljustify("nop".to_string()), len),
 
-            Inst::AluRmiR { size, op, src, dst } => format!(
+            Inst::AluRmiR {
+                size,
+                op,
+                src1: _,
+                src2,
+                dst,
+            } => format!(
                 "{} {}, {}",
                 ljustify2(op.to_string(), suffix_lqb(*size, op.is_8bit())),
-                src.show_rru_sized(mb_rru, size_lqb(*size, op.is_8bit())),
+                src2.show_rru_sized(mb_rru, size_lqb(*size, op.is_8bit())),
                 show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*size, op.is_8bit())),
             ),
 
@@ -1358,16 +1703,16 @@ impl PrettyPrint for Inst {
                 show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes()),
             ),
 
-            Inst::Not { size, src } => format!(
+            Inst::Not { size, src: _, dst } => format!(
                 "{} {}",
                 ljustify2("not".to_string(), suffix_bwlq(*size)),
-                show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes())
+                show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
             ),
 
-            Inst::Neg { size, src } => format!(
+            Inst::Neg { size, src: _, dst } => format!(
                 "{} {}",
                 ljustify2("neg".to_string(), suffix_bwlq(*size)),
-                show_ireg_sized(src.to_reg(), mb_rru, size.to_bytes())
+                show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
             ),
 
             Inst::Div {
@@ -1386,7 +1731,7 @@ impl PrettyPrint for Inst {
             ),
 
             Inst::MulHi {
-                size, signed, rhs, ..
+                size, signed, src2, ..
             } => format!(
                 "{} {}",
                 ljustify(if *signed {
@@ -1394,7 +1739,7 @@ impl PrettyPrint for Inst {
                 } else {
                     "mul".to_string()
                 }),
-                rhs.show_rru_sized(mb_rru, size.to_bytes())
+                src2.show_rru_sized(mb_rru, size.to_bytes())
             ),
 
             Inst::CheckedDivOrRemSeq {
@@ -1413,7 +1758,7 @@ impl PrettyPrint for Inst {
                 show_ireg_sized(divisor.to_reg(), mb_rru, size.to_bytes()),
             ),
 
-            Inst::SignExtendData { size } => match size {
+            Inst::SignExtendData { size, .. } => match size {
                 OperandSize::Size8 => "cbw",
                 OperandSize::Size16 => "cwd",
                 OperandSize::Size32 => "cdq",
@@ -1442,10 +1787,10 @@ impl PrettyPrint for Inst {
                 dst.show_rru(mb_rru),
             ),
 
-            Inst::XmmRmR { op, src, dst, .. } => format!(
+            Inst::XmmRmR { op, src2, dst, .. } => format!(
                 "{} {}, {}",
                 ljustify(op.to_string()),
-                src.show_rru_sized(mb_rru, 8),
+                src2.show_rru_sized(mb_rru, 8),
                 show_ireg_sized(dst.to_reg(), mb_rru, 8),
             ),
 
@@ -1484,7 +1829,7 @@ impl PrettyPrint for Inst {
 
             Inst::XmmRmRImm {
                 op,
-                src,
+                src2,
                 dst,
                 imm,
                 size,
@@ -1501,7 +1846,7 @@ impl PrettyPrint for Inst {
                     }
                 )),
                 imm,
-                src.show_rru(mb_rru),
+                src2.show_rru(mb_rru),
                 dst.show_rru(mb_rru),
             ),
 
@@ -1681,14 +2026,16 @@ impl PrettyPrint for Inst {
                 kind,
                 num_bits,
                 dst,
+                ..
             } => match num_bits {
-                None => format!(
-                    "{} %cl, {}",
+                Imm8Reg::Reg { reg } => format!(
+                    "{} {}, {}",
                     ljustify2(kind.to_string(), suffix_bwlq(*size)),
+                    show_ireg_sized(*reg, mb_rru, 1),
                     show_ireg_sized(dst.to_reg(), mb_rru, size.to_bytes())
                 ),
 
-                Some(num_bits) => format!(
+                Imm8Reg::Imm8 { imm: num_bits } => format!(
                     "{} ${}, {}",
                     ljustify2(kind.to_string(), suffix_bwlq(*size)),
                     num_bits,
@@ -1696,10 +2043,12 @@ impl PrettyPrint for Inst {
                 ),
             },
 
-            Inst::XmmRmiReg { opcode, src, dst } => format!(
+            Inst::XmmRmiReg {
+                opcode, src2, dst, ..
+            } => format!(
                 "{} {}, {}",
                 ljustify(opcode.to_string()),
-                src.show_rru(mb_rru),
+                src2.show_rru(mb_rru),
                 dst.to_reg().show_rru(mb_rru)
             ),
 
@@ -1727,7 +2076,13 @@ impl PrettyPrint for Inst {
                 show_ireg_sized(dst.to_reg(), mb_rru, 1)
             ),
 
-            Inst::Cmove { size, cc, src, dst } => format!(
+            Inst::Cmove {
+                size,
+                cc,
+                consequent: src,
+                alternative: _,
+                dst,
+            } => format!(
                 "{} {}, {}",
                 ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size))),
                 src.show_rru_sized(mb_rru, size.to_bytes()),
@@ -1813,13 +2168,18 @@ impl PrettyPrint for Inst {
                 show_ireg_sized(dst.to_reg(), mb_rru, 8),
             ),
 
-            Inst::LockCmpxchg { ty, src, dst, .. } => {
+            Inst::LockCmpxchg {
+                ty,
+                replacement,
+                mem,
+                ..
+            } => {
                 let size = ty.bytes() as u8;
                 format!(
                     "lock cmpxchg{} {}, {}",
                     suffix_bwlq(OperandSize::from_bytes(size as u32)),
-                    show_ireg_sized(*src, mb_rru, size),
-                    dst.show_rru(mb_rru)
+                    show_ireg_sized(*replacement, mb_rru, size),
+                    mem.show_rru(mb_rru)
                 )
             }
 
@@ -1874,36 +2234,74 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
     // regalloc.rs will "fix" this for us by removing the modified set from the use and def
     // sets.
     match inst {
-        Inst::AluRmiR { src, dst, .. } => {
+        Inst::AluRmiR {
+            src1, src2, dst, ..
+        } => {
+            debug_assert_eq!(*src1, dst.to_reg());
             if inst.produces_const() {
-                // No need to account for src, since src == dst.
+                // No need to account for src2, since src2 == dst.
                 collector.add_def(*dst);
             } else {
-                src.get_regs_as_uses(collector);
+                src2.get_regs_as_uses(collector);
                 collector.add_mod(*dst);
             }
         }
-        Inst::Not { src, .. } => {
-            collector.add_mod(*src);
+        Inst::Not { src, dst, .. } => {
+            debug_assert_eq!(*src, dst.to_reg());
+            collector.add_mod(*dst);
         }
-        Inst::Neg { src, .. } => {
-            collector.add_mod(*src);
+        Inst::Neg { src, dst, .. } => {
+            debug_assert_eq!(*src, dst.to_reg());
+            collector.add_mod(*dst);
         }
-        Inst::Div { size, divisor, .. } => {
+        Inst::Div {
+            size,
+            divisor,
+            dividend,
+            dst_quotient,
+            dst_remainder,
+            ..
+        } => {
+            debug_assert_eq!(*dividend, regs::rax());
+            debug_assert_eq!(dst_quotient.to_reg(), regs::rax());
             collector.add_mod(Writable::from_reg(regs::rax()));
+
+            debug_assert_eq!(dst_remainder.to_reg(), regs::rdx());
             if *size == OperandSize::Size8 {
                 collector.add_def(Writable::from_reg(regs::rdx()));
             } else {
                 collector.add_mod(Writable::from_reg(regs::rdx()));
             }
+
             divisor.get_regs_as_uses(collector);
         }
-        Inst::MulHi { rhs, .. } => {
+        Inst::MulHi {
+            src1,
+            src2,
+            dst_lo,
+            dst_hi,
+            ..
+        } => {
+            debug_assert_eq!(*src1, regs::rax());
+            debug_assert_eq!(dst_lo.to_reg(), regs::rax());
             collector.add_mod(Writable::from_reg(regs::rax()));
+
+            debug_assert_eq!(dst_hi.to_reg(), regs::rdx());
             collector.add_def(Writable::from_reg(regs::rdx()));
-            rhs.get_regs_as_uses(collector);
+
+            src2.get_regs_as_uses(collector);
         }
-        Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => {
+        Inst::CheckedDivOrRemSeq {
+            divisor,
+            dividend,
+            dst_quotient,
+            dst_remainder,
+            tmp,
+            ..
+        } => {
+            debug_assert_eq!(*dividend, regs::rax());
+            debug_assert_eq!(dst_quotient.to_reg(), regs::rax());
+            debug_assert_eq!(dst_remainder.to_reg(), regs::rdx());
             // Mark both fixed registers as mods, to avoid an early clobber problem in codegen
             // (i.e. the temporary is allocated one of the fixed registers). This requires writing
             // the rdx register *before* the instruction, which is not too bad.
@@ -1914,25 +2312,36 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
                 collector.add_def(*tmp);
             }
         }
-        Inst::SignExtendData { size } => match size {
-            OperandSize::Size8 => collector.add_mod(Writable::from_reg(regs::rax())),
-            _ => {
-                collector.add_use(regs::rax());
-                collector.add_def(Writable::from_reg(regs::rdx()));
+        Inst::SignExtendData { size, src, dst } => {
+            debug_assert_eq!(*src, regs::rax());
+            debug_assert_eq!(dst.to_reg(), regs::rdx());
+            match size {
+                OperandSize::Size8 => collector.add_mod(Writable::from_reg(regs::rax())),
+                _ => {
+                    collector.add_use(regs::rax());
+                    collector.add_def(Writable::from_reg(regs::rdx()));
+                }
             }
-        },
+        }
         Inst::UnaryRmR { src, dst, .. }
         | Inst::XmmUnaryRmR { src, dst, .. }
         | Inst::XmmUnaryRmREvex { src, dst, .. } => {
             src.get_regs_as_uses(collector);
             collector.add_def(*dst);
         }
-        Inst::XmmRmR { src, dst, op, .. } => {
+        Inst::XmmRmR {
+            src1,
+            src2,
+            dst,
+            op,
+            ..
+        } => {
+            debug_assert_eq!(*src1, dst.to_reg());
             if inst.produces_const() {
                 // No need to account for src, since src == dst.
                 collector.add_def(*dst);
             } else {
-                src.get_regs_as_uses(collector);
+                src2.get_regs_as_uses(collector);
                 collector.add_mod(*dst);
                 // Some instructions have an implicit use of XMM0.
                 if *op == SseOpcode::Blendvpd
@@ -1957,9 +2366,17 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
                 _ => collector.add_def(*dst),
             }
         }
-        Inst::XmmRmRImm { op, src, dst, .. } => {
+        Inst::XmmRmRImm {
+            op,
+            src1,
+            src2,
+            dst,
+            ..
+        } => {
+            debug_assert_eq!(*src1, dst.to_reg());
             if inst.produces_const() {
-                // No need to account for src, since src == dst.
+                // No need to account for src2, since src2 == dst.
+                debug_assert_eq!(src2.to_reg(), Some(dst.to_reg()));
                 collector.add_def(*dst);
             } else if *op == SseOpcode::Pextrb
                 || *op == SseOpcode::Pextrw
@@ -1970,10 +2387,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
                 || *op == SseOpcode::Roundps
                 || *op == SseOpcode::Roundpd
             {
-                src.get_regs_as_uses(collector);
+                src2.get_regs_as_uses(collector);
                 collector.add_def(*dst);
             } else {
-                src.get_regs_as_uses(collector);
+                src2.get_regs_as_uses(collector);
                 collector.add_mod(*dst);
             }
         }
@@ -1983,8 +2400,11 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
             collector.add_use(*lhs);
             collector.add_mod(*rhs_dst);
         }
-        Inst::XmmRmiReg { src, dst, .. } => {
-            src.get_regs_as_uses(collector);
+        Inst::XmmRmiReg {
+            src1, src2, dst, ..
+        } => {
+            debug_assert_eq!(*src1, dst.to_reg());
+            src2.get_regs_as_uses(collector);
             collector.add_mod(*dst);
         }
         Inst::XmmMovRM { src, dst, .. } => {
@@ -2054,7 +2474,8 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
             dst.get_regs_as_uses(collector);
         }
         Inst::ShiftR { num_bits, dst, .. } => {
-            if num_bits.is_none() {
+            if let Imm8Reg::Reg { reg } = num_bits {
+                debug_assert_eq!(*reg, regs::rcx());
                 collector.add_use(regs::rcx());
             }
             collector.add_mod(*dst);
@@ -2066,7 +2487,12 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
         Inst::Setcc { dst, .. } => {
             collector.add_def(*dst);
         }
-        Inst::Cmove { src, dst, .. } | Inst::XmmCmove { src, dst, .. } => {
+        Inst::Cmove {
+            consequent: src,
+            dst,
+            ..
+        }
+        | Inst::XmmCmove { src, dst, .. } => {
             src.get_regs_as_uses(collector);
             collector.add_mod(*dst);
         }
@@ -2115,9 +2541,18 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
             collector.add_def(*dst);
         }
 
-        Inst::LockCmpxchg { src, dst, .. } => {
-            dst.get_regs_as_uses(collector);
-            collector.add_use(*src);
+        Inst::LockCmpxchg {
+            replacement,
+            expected,
+            mem,
+            dst_old,
+            ..
+        } => {
+            mem.get_regs_as_uses(collector);
+            collector.add_use(*replacement);
+
+            debug_assert_eq!(*expected, regs::rax());
+            debug_assert_eq!(dst_old.to_reg(), regs::rax());
             collector.add_mod(Writable::from_reg(regs::rax()));
         }
 
@@ -2165,29 +2600,55 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
 //=============================================================================
 // Instructions and subcomponents: map_regs
 
-fn map_use<RUM: RegUsageMapper>(m: &RUM, r: &mut Reg) {
-    if let Some(reg) = r.as_virtual_reg() {
-        let new = m.get_use(reg).unwrap().to_reg();
+// Define our own register-mapping trait so we can do arbitrary register
+// renaming that are more free form than what `regalloc` constrains us to with
+// its `RegUsageMapper` trait definition.
+pub trait RegMapper {
+    fn get_use(&self, reg: Reg) -> Option<Reg>;
+    fn get_def(&self, reg: Reg) -> Option<Reg>;
+    fn get_mod(&self, reg: Reg) -> Option<Reg>;
+}
+
+impl<T> RegMapper for T
+where
+    T: regalloc::RegUsageMapper,
+{
+    fn get_use(&self, reg: Reg) -> Option<Reg> {
+        let v = reg.as_virtual_reg()?;
+        self.get_use(v).map(|r| r.to_reg())
+    }
+
+    fn get_def(&self, reg: Reg) -> Option<Reg> {
+        let v = reg.as_virtual_reg()?;
+        self.get_def(v).map(|r| r.to_reg())
+    }
+
+    fn get_mod(&self, reg: Reg) -> Option<Reg> {
+        let v = reg.as_virtual_reg()?;
+        self.get_mod(v).map(|r| r.to_reg())
+    }
+}
+
+fn map_use<RM: RegMapper>(m: &RM, r: &mut Reg) {
+    if let Some(new) = m.get_use(*r) {
         *r = new;
     }
 }
 
-fn map_def<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>) {
-    if let Some(reg) = r.to_reg().as_virtual_reg() {
-        let new = m.get_def(reg).unwrap().to_reg();
+fn map_def<RM: RegMapper>(m: &RM, r: &mut Writable<Reg>) {
+    if let Some(new) = m.get_def(r.to_reg()) {
         *r = Writable::from_reg(new);
     }
 }
 
-fn map_mod<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>) {
-    if let Some(reg) = r.to_reg().as_virtual_reg() {
-        let new = m.get_mod(reg).unwrap().to_reg();
+fn map_mod<RM: RegMapper>(m: &RM, r: &mut Writable<Reg>) {
+    if let Some(new) = m.get_mod(r.to_reg()) {
         *r = Writable::from_reg(new);
     }
 }
 
 impl Amode {
-    fn map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM) {
+    fn map_uses<RM: RegMapper>(&mut self, map: &RM) {
         match self {
             Amode::ImmReg { ref mut base, .. } => map_use(map, base),
             Amode::ImmRegRegShift {
@@ -2217,7 +2678,7 @@ impl Amode {
 }
 
 impl RegMemImm {
-    fn map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM) {
+    fn map_uses<RM: RegMapper>(&mut self, map: &RM) {
         match self {
             RegMemImm::Reg { ref mut reg } => map_use(map, reg),
             RegMemImm::Mem { ref mut addr } => addr.map_uses(map),
@@ -2225,7 +2686,7 @@ impl RegMemImm {
         }
     }
 
-    fn map_as_def<RUM: RegUsageMapper>(&mut self, mapper: &RUM) {
+    fn map_as_def<RM: RegMapper>(&mut self, mapper: &RM) {
         match self {
             Self::Reg { reg } => {
                 let mut writable_src = Writable::from_reg(*reg);
@@ -2238,14 +2699,14 @@ impl RegMemImm {
 }
 
 impl RegMem {
-    fn map_uses<RUM: RegUsageMapper>(&mut self, map: &RUM) {
+    fn map_uses<RM: RegMapper>(&mut self, map: &RM) {
         match self {
             RegMem::Reg { ref mut reg } => map_use(map, reg),
             RegMem::Mem { ref mut addr, .. } => addr.map_uses(map),
         }
     }
 
-    fn map_as_def<RUM: RegUsageMapper>(&mut self, mapper: &RUM) {
+    fn map_as_def<RM: RegMapper>(&mut self, mapper: &RM) {
         match self {
             Self::Reg { reg } => {
                 let mut writable_src = Writable::from_reg(*reg);
@@ -2257,28 +2718,36 @@ impl RegMem {
     }
 }
 
-fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
+pub(crate) fn x64_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
     // Note this must be carefully synchronized with x64_get_regs.
     let produces_const = inst.produces_const();
 
     match inst {
         // ** Nop
         Inst::AluRmiR {
-            ref mut src,
+            ref mut src1,
+            ref mut src2,
             ref mut dst,
             ..
         } => {
+            debug_assert_eq!(*src1, dst.to_reg());
             if produces_const {
-                src.map_as_def(mapper);
+                src2.map_as_def(mapper);
                 map_def(mapper, dst);
+                *src1 = dst.to_reg();
             } else {
-                src.map_uses(mapper);
+                src2.map_uses(mapper);
                 map_mod(mapper, dst);
+                *src1 = dst.to_reg();
             }
         }
-        Inst::Not { src, .. } | Inst::Neg { src, .. } => map_mod(mapper, src),
+        Inst::Not { src, dst, .. } | Inst::Neg { src, dst, .. } => {
+            debug_assert_eq!(*src, dst.to_reg());
+            map_mod(mapper, dst);
+            *src = dst.to_reg();
+        }
         Inst::Div { divisor, .. } => divisor.map_uses(mapper),
-        Inst::MulHi { rhs, .. } => rhs.map_uses(mapper),
+        Inst::MulHi { src2, .. } => src2.map_uses(mapper),
         Inst::CheckedDivOrRemSeq { divisor, tmp, .. } => {
             map_mod(mapper, divisor);
             if let Some(tmp) = tmp {
@@ -2306,13 +2775,16 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
         }
         Inst::XmmRmRImm {
             ref op,
-            ref mut src,
+            ref mut src1,
+            ref mut src2,
             ref mut dst,
             ..
         } => {
+            debug_assert_eq!(*src1, dst.to_reg());
             if produces_const {
-                src.map_as_def(mapper);
+                src2.map_as_def(mapper);
                 map_def(mapper, dst);
+                *src1 = dst.to_reg();
             } else if *op == SseOpcode::Pextrb
                 || *op == SseOpcode::Pextrw
                 || *op == SseOpcode::Pextrd
@@ -2322,24 +2794,30 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
                 || *op == SseOpcode::Roundps
                 || *op == SseOpcode::Roundpd
             {
-                src.map_uses(mapper);
+                src2.map_uses(mapper);
                 map_def(mapper, dst);
+                *src1 = dst.to_reg();
             } else {
-                src.map_uses(mapper);
+                src2.map_uses(mapper);
                 map_mod(mapper, dst);
+                *src1 = dst.to_reg();
             }
         }
         Inst::XmmRmR {
-            ref mut src,
+            ref mut src1,
+            ref mut src2,
             ref mut dst,
             ..
         } => {
+            debug_assert_eq!(*src1, dst.to_reg());
             if produces_const {
-                src.map_as_def(mapper);
+                src2.map_as_def(mapper);
                 map_def(mapper, dst);
+                *src1 = dst.to_reg();
             } else {
-                src.map_uses(mapper);
+                src2.map_uses(mapper);
                 map_mod(mapper, dst);
+                *src1 = dst.to_reg();
             }
         }
         Inst::XmmRmREvex {
@@ -2357,12 +2835,15 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             }
         }
         Inst::XmmRmiReg {
-            ref mut src,
+            ref mut src1,
+            ref mut src2,
             ref mut dst,
             ..
         } => {
-            src.map_uses(mapper);
+            debug_assert_eq!(*src1, dst.to_reg());
+            src2.map_uses(mapper);
             map_mod(mapper, dst);
+            *src1 = dst.to_reg();
         }
         Inst::XmmUninitializedValue { ref mut dst, .. } => {
             map_def(mapper, dst);
@@ -2475,8 +2956,14 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             map_use(mapper, src);
             dst.map_uses(mapper);
         }
-        Inst::ShiftR { ref mut dst, .. } => {
+        Inst::ShiftR {
+            ref mut src,
+            ref mut dst,
+            ..
+        } => {
+            debug_assert_eq!(*src, dst.to_reg());
             map_mod(mapper, dst);
+            *src = dst.to_reg();
         }
         Inst::CmpRmiR {
             ref mut src,
@@ -2488,17 +2975,22 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
         }
         Inst::Setcc { ref mut dst, .. } => map_def(mapper, dst),
         Inst::Cmove {
-            ref mut src,
+            consequent: ref mut src,
             ref mut dst,
+            ref mut alternative,
             ..
+        } => {
+            src.map_uses(mapper);
+            map_mod(mapper, dst);
+            *alternative = dst.to_reg();
         }
-        | Inst::XmmCmove {
+        Inst::XmmCmove {
             ref mut src,
             ref mut dst,
             ..
         } => {
             src.map_uses(mapper);
-            map_mod(mapper, dst)
+            map_mod(mapper, dst);
         }
         Inst::Push64 { ref mut src } => src.map_uses(mapper),
         Inst::Pop64 { ref mut dst } => {
@@ -2549,12 +3041,12 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
         Inst::LoadExtName { ref mut dst, .. } => map_def(mapper, dst),
 
         Inst::LockCmpxchg {
-            ref mut src,
-            ref mut dst,
+            ref mut replacement,
+            ref mut mem,
             ..
         } => {
-            map_use(mapper, src);
-            dst.map_uses(mapper);
+            map_use(mapper, replacement);
+            mem.map_uses(mapper);
         }
 
         Inst::ValueLabelMarker { ref mut reg, .. } => map_use(mapper, reg),
@@ -2588,7 +3080,10 @@ impl MachInst for Inst {
         x64_get_regs(&self, collector)
     }
 
-    fn map_regs<RUM: RegUsageMapper>(&mut self, mapper: &RUM) {
+    fn map_regs<RUM>(&mut self, mapper: &RUM)
+    where
+        RUM: regalloc::RegUsageMapper,
+    {
         x64_map_regs(self, mapper);
     }
 
@@ -2742,16 +3237,30 @@ impl MachInst for Inst {
     ) -> SmallVec<[Self; 4]> {
         let mut ret = SmallVec::new();
         if ty == types::I128 {
-            ret.push(Inst::imm(
-                OperandSize::Size64,
-                value as u64,
-                to_regs.regs()[0],
-            ));
-            ret.push(Inst::imm(
-                OperandSize::Size64,
-                (value >> 64) as u64,
-                to_regs.regs()[1],
-            ));
+            let lo = value as u64;
+            let hi = (value >> 64) as u64;
+            let lo_reg = to_regs.regs()[0];
+            let hi_reg = to_regs.regs()[1];
+            if lo == 0 {
+                ret.push(Inst::alu_rmi_r(
+                    OperandSize::Size64,
+                    AluRmiROpcode::Xor,
+                    RegMemImm::reg(lo_reg.to_reg()),
+                    lo_reg,
+                ));
+            } else {
+                ret.push(Inst::imm(OperandSize::Size64, lo, lo_reg));
+            }
+            if hi == 0 {
+                ret.push(Inst::alu_rmi_r(
+                    OperandSize::Size64,
+                    AluRmiROpcode::Xor,
+                    RegMemImm::reg(hi_reg.to_reg()),
+                    hi_reg,
+                ));
+            } else {
+                ret.push(Inst::imm(OperandSize::Size64, hi, hi_reg));
+            }
         } else {
             let to_reg = to_regs
                 .only_reg()
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
new file mode 100644
index 0000000000..ee67b81f11
--- /dev/null
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -0,0 +1,947 @@
+;; x86-64 instruction selection and CLIF-to-MachInst lowering.
+
+;; The main lowering constructor term: takes a clif `Inst` and returns the
+;; register(s) within which the lowered instruction's result values live.
+(decl lower (Inst) ValueRegs)
+
+;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `i64` and smaller.
+(rule (lower (has_type (fits_in_64 ty)
+                       (iconst (u64_from_imm64 x))))
+      (value_reg (imm ty x)))
+
+;; `i128`
+(rule (lower (has_type $I128
+                       (iconst (u64_from_imm64 x))))
+      (value_regs (imm $I64 x)
+                  (imm $I64 0)))
+
+;;;; Rules for `bconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `b64` and smaller.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bconst $false)))
+      (value_reg (imm ty 0)))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bconst $true)))
+      (value_reg (imm ty 1)))
+
+;; `b128`
+
+(rule (lower (has_type $B128
+                       (bconst $false)))
+      (value_regs (imm $B64 0)
+                  (imm $B64 0)))
+
+(rule (lower (has_type $B128
+                       (bconst $true)))
+      (value_regs (imm $B64 1)
+                  (imm $B64 0)))
+
+;;;; Rules for `null` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type ty (null)))
+      (value_reg (imm ty 0)))
+
+;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `i64` and smaller.
+
+;; Add two registers.
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd x y)))
+      (value_reg (add ty
+                      (put_in_reg x)
+                      (RegMemImm.Reg (put_in_reg y)))))
+
+;; Add a register and an immediate.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd x (simm32_from_value y))))
+      (value_reg (add ty (put_in_reg x) y)))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd (simm32_from_value x) y)))
+      (value_reg (add ty (put_in_reg y) x)))
+
+;; Add a register and memory.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd x (sinkable_load y))))
+      (value_reg (add ty
+                      (put_in_reg x)
+                      (sink_load y))))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd (sinkable_load x) y)))
+      (value_reg (add ty
+                      (put_in_reg y)
+                      (sink_load x))))
+
+;; SSE.
+
+(rule (lower (has_type (multi_lane 8 16)
+                       (iadd x y)))
+      (value_reg (paddb (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 16 8)
+                       (iadd x y)))
+      (value_reg (paddw (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 32 4)
+                       (iadd x y)))
+      (value_reg (paddd (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 64 2)
+                       (iadd x y)))
+      (value_reg (paddq (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+;; `i128`
+(rule (lower (has_type $I128 (iadd x y)))
+      ;; Get the high/low registers for `x`.
+      (let ((x_regs ValueRegs (put_in_regs x))
+            (x_lo Reg (value_regs_get x_regs 0))
+            (x_hi Reg (value_regs_get x_regs 1)))
+        ;; Get the high/low registers for `y`.
+        (let ((y_regs ValueRegs (put_in_regs y))
+              (y_lo Reg (value_regs_get y_regs 0))
+              (y_hi Reg (value_regs_get y_regs 1)))
+          ;; Do an add followed by an add-with-carry.
+          (with_flags (add_with_flags $I64 x_lo (RegMemImm.Reg y_lo))
+                      (adc $I64 x_hi (RegMemImm.Reg y_hi))))))
+
+;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (multi_lane 8 16)
+                       (sadd_sat x y)))
+      (value_reg (paddsb (put_in_reg x)
+                         (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 16 8)
+                       (sadd_sat x y)))
+      (value_reg (paddsw (put_in_reg x)
+                         (put_in_reg_mem y))))
+
+;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (multi_lane 8 16)
+                       (uadd_sat x y)))
+      (value_reg (paddusb (put_in_reg x)
+                          (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 16 8)
+                       (uadd_sat x y)))
+      (value_reg (paddusw (put_in_reg x)
+                          (put_in_reg_mem y))))
+
+;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Add two registers.
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd_ifcout x y)))
+      (value_reg (add ty
+                      (put_in_reg x)
+                      (RegMemImm.Reg (put_in_reg y)))))
+
+;; Add a register and an immediate.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd_ifcout x (simm32_from_value y))))
+      (value_reg (add ty (put_in_reg x) y)))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd_ifcout (simm32_from_value x) y)))
+      (value_reg (add ty (put_in_reg y) x)))
+
+;; Add a register and memory.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd_ifcout x (sinkable_load y))))
+      (value_reg (add ty
+                      (put_in_reg x)
+                      (sink_load y))))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (iadd_ifcout (sinkable_load x) y)))
+      (value_reg (add ty
+                      (put_in_reg y)
+                      (sink_load x))))
+
+;; (No `iadd_ifcout` for `i128`.)
+
+;;;; Rules for `iadd_imm` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `i64` and smaller.
+
+;; When the immediate fits in a `RegMemImm.Imm`, use that.
+(rule (lower (has_type (fits_in_64 ty) (iadd_imm (simm32_from_imm64 x) y)))
+      (value_reg (add ty (put_in_reg y) x)))
+
+;; Otherwise, put the immediate into a register.
+(rule (lower (has_type (fits_in_64 ty) (iadd_imm (u64_from_imm64 x) y)))
+      (value_reg (add ty (put_in_reg y) (RegMemImm.Reg (imm ty x)))))
+
+;; `i128`
+
+;; When the immediate fits in a `RegMemImm.Imm`, use that.
+(rule (lower (has_type $I128 (iadd_imm (simm32_from_imm64 x) y)))
+      (let ((y_regs ValueRegs (put_in_regs y))
+            (y_lo Reg (value_regs_get y_regs 0))
+            (y_hi Reg (value_regs_get y_regs 1)))
+        (with_flags (add_with_flags $I64 y_lo x)
+                    (adc $I64 y_hi (RegMemImm.Imm 0)))))
+
+;; Otherwise, put the immediate into a register.
+(rule (lower (has_type $I128 (iadd_imm (u64_from_imm64 x) y)))
+      (let ((y_regs ValueRegs (put_in_regs y))
+            (y_lo Reg (value_regs_get y_regs 0))
+            (y_hi Reg (value_regs_get y_regs 1))
+            (x_lo Reg (imm $I64 x)))
+        (with_flags (add_with_flags $I64 y_lo (RegMemImm.Reg x_lo))
+                    (adc $I64 y_hi (RegMemImm.Imm 0)))))
+
+;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `i64` and smaller.
+
+;; Sub two registers.
+(rule (lower (has_type (fits_in_64 ty)
+                       (isub x y)))
+      (value_reg (sub ty
+                      (put_in_reg x)
+                      (RegMemImm.Reg (put_in_reg y)))))
+
+;; Sub a register and an immediate.
+(rule (lower (has_type (fits_in_64 ty)
+                       (isub x (simm32_from_value y))))
+      (value_reg (sub ty (put_in_reg x) y)))
+
+;; Sub a register and memory.
+(rule (lower (has_type (fits_in_64 ty)
+                       (isub x (sinkable_load y))))
+      (value_reg (sub ty
+                      (put_in_reg x)
+                      (sink_load y))))
+
+;; SSE.
+
+(rule (lower (has_type (multi_lane 8 16)
+                       (isub x y)))
+      (value_reg (psubb (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 16 8)
+                       (isub x y)))
+      (value_reg (psubw (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 32 4)
+                       (isub x y)))
+      (value_reg (psubd (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 64 2)
+                       (isub x y)))
+      (value_reg (psubq (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+;; `i128`
+(rule (lower (has_type $I128 (isub x y)))
+      ;; Get the high/low registers for `x`.
+      (let ((x_regs ValueRegs (put_in_regs x))
+            (x_lo Reg (value_regs_get x_regs 0))
+            (x_hi Reg (value_regs_get x_regs 1)))
+        ;; Get the high/low registers for `y`.
+        (let ((y_regs ValueRegs (put_in_regs y))
+              (y_lo Reg (value_regs_get y_regs 0))
+              (y_hi Reg (value_regs_get y_regs 1)))
+          ;; Do a sub followed by an sub-with-borrow.
+          (with_flags (sub_with_flags $I64 x_lo (RegMemImm.Reg y_lo))
+                      (sbb $I64 x_hi (RegMemImm.Reg y_hi))))))
+
+;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (multi_lane 8 16)
+                       (ssub_sat x y)))
+      (value_reg (psubsb (put_in_reg x)
+                         (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 16 8)
+                       (ssub_sat x y)))
+      (value_reg (psubsw (put_in_reg x)
+                         (put_in_reg_mem y))))
+
+;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (multi_lane 8 16)
+                       (usub_sat x y)))
+      (value_reg (psubusb (put_in_reg x)
+                          (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 16 8)
+                       (usub_sat x y)))
+      (value_reg (psubusw (put_in_reg x)
+                          (put_in_reg_mem y))))
+
+;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `{i,b}64` and smaller.
+
+;; And two registers.
+(rule (lower (has_type (fits_in_64 ty) (band x y)))
+      (value_reg (m_and ty
+                        (put_in_reg x)
+                        (RegMemImm.Reg (put_in_reg y)))))
+
+;; And with a memory operand.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (band x (sinkable_load y))))
+      (value_reg (m_and ty
+                        (put_in_reg x)
+                        (sink_load y))))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (band (sinkable_load x) y)))
+      (value_reg (m_and ty
+                        (put_in_reg y)
+                        (sink_load x))))
+
+;; And with an immediate.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (band x (simm32_from_value y))))
+      (value_reg (m_and ty
+                        (put_in_reg x)
+                        y)))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (band (simm32_from_value x) y)))
+      (value_reg (m_and ty
+                        (put_in_reg y)
+                        x)))
+
+;; SSE.
+
+(rule (lower (has_type $F32X4 (band x y)))
+      (value_reg (andps (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type $F64X2 (band x y)))
+      (value_reg (andpd (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane _bits _lanes)
+                       (band x y)))
+      (value_reg (pand (put_in_reg x)
+                       (put_in_reg_mem y))))
+
+;; `{i,b}128`.
+
+(rule (lower (has_type $I128 (band x y)))
+      (let ((x_regs ValueRegs (put_in_regs x))
+            (x_lo Reg (value_regs_get x_regs 0))
+            (x_hi Reg (value_regs_get x_regs 1))
+            (y_regs ValueRegs (put_in_regs y))
+            (y_lo Reg (value_regs_get y_regs 0))
+            (y_hi Reg (value_regs_get y_regs 1)))
+        (value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo))
+                    (m_and $I64 x_hi (RegMemImm.Reg y_hi)))))
+
+(rule (lower (has_type $B128 (band x y)))
+      ;; Booleans are always `0` or `1`, so we only need to do the `and` on the
+      ;; low half. The high half is always zero but, rather than generate a new
+      ;; zero, we just reuse `x`'s high half which is already zero.
+      (let ((x_regs ValueRegs (put_in_regs x))
+            (x_lo Reg (value_regs_get x_regs 0))
+            (x_hi Reg (value_regs_get x_regs 1))
+            (y_lo Reg (lo_reg y)))
+        (value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo))
+                    x_hi)))
+
+;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `{i,b}64` and smaller.
+
+;; Or two registers.
+(rule (lower (has_type (fits_in_64 ty) (bor x y)))
+      (value_reg (or ty
+                     (put_in_reg x)
+                     (RegMemImm.Reg (put_in_reg y)))))
+
+;; Or with a memory operand.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bor x (sinkable_load y))))
+      (value_reg (or ty
+                     (put_in_reg x)
+                     (sink_load y))))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bor (sinkable_load x) y)))
+      (value_reg (or ty
+                     (put_in_reg y)
+                     (sink_load x))))
+
+;; Or with an immediate.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bor x (simm32_from_value y))))
+      (value_reg (or ty
+                     (put_in_reg x)
+                     y)))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bor (simm32_from_value x) y)))
+      (value_reg (or ty
+                     (put_in_reg y)
+                     x)))
+
+;; SSE.
+
+(rule (lower (has_type $F32X4 (bor x y)))
+      (value_reg (orps (put_in_reg x)
+                       (put_in_reg_mem y))))
+
+(rule (lower (has_type $F64X2 (bor x y)))
+      (value_reg (orpd (put_in_reg x)
+                       (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane _bits _lanes)
+                       (bor x y)))
+      (value_reg (por (put_in_reg x)
+                      (put_in_reg_mem y))))
+
+;; `{i,b}128`.
+
+(decl or_i128 (ValueRegs ValueRegs) ValueRegs)
+(rule (or_i128 x y)
+      (let ((x_lo Reg (value_regs_get x 0))
+            (x_hi Reg (value_regs_get x 1))
+            (y_lo Reg (value_regs_get y 0))
+            (y_hi Reg (value_regs_get y 1)))
+        (value_regs (or $I64 x_lo (RegMemImm.Reg y_lo))
+                    (or $I64 x_hi (RegMemImm.Reg y_hi)))))
+
+(rule (lower (has_type $I128 (bor x y)))
+      (or_i128 (put_in_regs x) (put_in_regs y)))
+
+(rule (lower (has_type $B128 (bor x y)))
+      ;; Booleans are always `0` or `1`, so we only need to do the `or` on the
+      ;; low half. The high half is always zero but, rather than generate a new
+      ;; zero, we just reuse `x`'s high half which is already zero.
+      (let ((x_regs ValueRegs (put_in_regs x))
+            (x_lo Reg (value_regs_get x_regs 0))
+            (x_hi Reg (value_regs_get x_regs 1))
+            (y_lo Reg (lo_reg y)))
+        (value_regs (or $I64 x_lo (RegMemImm.Reg y_lo))
+                    x_hi)))
+
+;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `{i,b}64` and smaller.
+
+;; Xor two registers.
+(rule (lower (has_type (fits_in_64 ty) (bxor x y)))
+      (value_reg (xor ty
+                      (put_in_reg x)
+                      (RegMemImm.Reg (put_in_reg y)))))
+
+;; Xor with a memory operand.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bxor x (sinkable_load y))))
+      (value_reg (xor ty
+                      (put_in_reg x)
+                      (sink_load y))))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bxor (sinkable_load x) y)))
+      (value_reg (xor ty
+                      (put_in_reg y)
+                      (sink_load x))))
+
+;; Xor with an immediate.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bxor x (simm32_from_value y))))
+      (value_reg (xor ty
+                      (put_in_reg x)
+                      y)))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (bxor (simm32_from_value x) y)))
+      (value_reg (xor ty
+                      (put_in_reg y)
+                      x)))
+
+;; SSE.
+
+(rule (lower (has_type $F32X4 (bxor x y)))
+      (value_reg (xorps (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type $F64X2 (bxor x y)))
+      (value_reg (xorpd (put_in_reg x)
+                        (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane _bits _lanes)
+                       (bxor x y)))
+      (value_reg (pxor (put_in_reg x)
+                       (put_in_reg_mem y))))
+
+;; `{i,b}128`.
+
+(rule (lower (has_type $I128 (bxor x y)))
+      (let ((x_regs ValueRegs (put_in_regs x))
+            (x_lo Reg (value_regs_get x_regs 0))
+            (x_hi Reg (value_regs_get x_regs 1))
+            (y_regs ValueRegs (put_in_regs y))
+            (y_lo Reg (value_regs_get y_regs 0))
+            (y_hi Reg (value_regs_get y_regs 1)))
+        (value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo))
+                    (xor $I64 x_hi (RegMemImm.Reg y_hi)))))
+
+(rule (lower (has_type $B128 (bxor x y)))
+      ;; Booleans are always `0` or `1`, so we only need to do the `xor` on the
+      ;; low half. The high half is always zero but, rather than generate a new
+      ;; zero, we just reuse `x`'s high half which is already zero.
+      (let ((x_regs ValueRegs (put_in_regs x))
+            (x_lo Reg (value_regs_get x_regs 0))
+            (x_hi Reg (value_regs_get x_regs 1))
+            (y_lo Reg (lo_reg y)))
+        (value_regs (xor $I64 x_lo (RegMemImm.Reg y_lo))
+                    x_hi)))
+
+;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `i64` and smaller.
+
+(rule (lower (has_type (fits_in_64 ty) (ishl src amt)))
+      ;; NB: Only the low bits of `amt` matter since we logically mask the shift
+      ;; amount to the value's bit width.
+      (let ((amt_ Reg (lo_reg amt)))
+        (value_reg (shl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
+
+(rule (lower (has_type (fits_in_64 ty) (ishl src (imm8_from_value amt))))
+      (value_reg (shl ty (put_in_reg src) amt)))
+
+;; `i128`.
+
+(decl shl_i128 (ValueRegs Reg) ValueRegs)
+(rule (shl_i128 src amt)
+      ;; Unpack the registers that make up the 128-bit value being shifted.
+      (let ((src_lo Reg (value_regs_get src 0))
+            (src_hi Reg (value_regs_get src 1))
+            ;; Do two 64-bit shifts.
+            (lo_shifted Reg (shl $I64 src_lo (Imm8Reg.Reg amt)))
+            (hi_shifted Reg (shl $I64 src_hi (Imm8Reg.Reg amt)))
+            ;; `src_lo >> (64 - amt)` are the bits to carry over from the lo
+            ;; into the hi.
+            (carry Reg (shr $I64 src_lo (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt)))))
+            (zero Reg (imm $I64 0))
+            ;; Nullify the carry if we are shifting in by a multiple of 128.
+            (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
+                                      (cmove $I64 (CC.Z) (RegMem.Reg zero) carry)))
+            ;; Add the carry into the high half.
+            (hi_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg hi_shifted))))
+        ;; Combine the two shifted halves. However, if we are shifting by >= 64
+        ;; (modulo 128), then the low bits are zero and the high bits are our
+        ;; low bits.
+        (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
+                      (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted) zero)
+                      (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted_) lo_shifted))))
+
+(rule (lower (has_type $I128 (ishl src amt)))
+      ;; NB: Only the low bits of `amt` matter since we logically mask the shift
+      ;; amount to the value's bit width.
+      (let ((amt_ Reg (lo_reg amt)))
+        (shl_i128 (put_in_regs src) amt_)))
+
+;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `i64` and smaller.
+
+(rule (lower (has_type (fits_in_64 ty) (ushr src amt)))
+      (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Zero)))
+            ;; NB: Only the low bits of `amt` matter since we logically mask the
+            ;; shift amount to the value's bit width.
+            (amt_ Reg (lo_reg amt)))
+        (value_reg (shr ty src_ (Imm8Reg.Reg amt_)))))
+
+(rule (lower (has_type (fits_in_64 ty) (ushr src (imm8_from_value amt))))
+      (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Zero))))
+        (value_reg (shr ty src_ amt))))
+
+;; `i128`.
+
+(decl shr_i128 (ValueRegs Reg) ValueRegs)
+(rule (shr_i128 src amt)
+      ;; Unpack the lo/hi halves of `src`.
+      (let ((src_lo Reg (value_regs_get src 0))
+            (src_hi Reg (value_regs_get src 1))
+            ;; Do a shift on each half.
+            (lo_shifted Reg (shr $I64 src_lo (Imm8Reg.Reg amt)))
+            (hi_shifted Reg (shr $I64 src_hi (Imm8Reg.Reg amt)))
+            ;; `src_hi << (64 - amt)` are the bits to carry over from the hi
+            ;; into the lo.
+            (carry Reg (shl $I64 src_hi (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt)))))
+            ;; Nullify the carry if we are shifting by a multiple of 128.
+            (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt)
+                                      (cmove $I64 (CC.Z) (RegMem.Reg (imm $I64 0)) carry)))
+            ;; Add the carry bits into the lo.
+            (lo_shifted_ Reg (or $I64 carry_ (RegMemImm.Reg lo_shifted))))
+        ;; Combine the two shifted halves. However, if we are shifting by >= 64
+        ;; (modulo 128), then the hi bits are zero and the lo bits are what
+        ;; would otherwise be our hi bits.
+        (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt)
+                      (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted_) hi_shifted)
+                      (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted) (imm $I64 0)))))
+
+(rule (lower (has_type $I128 (ushr src amt)))
+      ;; NB: Only the low bits of `amt` matter since we logically mask the shift
+      ;; amount to the value's bit width.
+      (let ((amt_ Reg (lo_reg amt)))
+        (shr_i128 (put_in_regs src) amt_)))
+
+;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `i64` and smaller.
+
+(rule (lower (has_type (fits_in_64 ty) (rotl src amt)))
+      ;; NB: Only the low bits of `amt` matter since we logically mask the
+      ;; shift amount to the value's bit width.
+      (let ((amt_ Reg (lo_reg amt)))
+        (value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
+
+(rule (lower (has_type (fits_in_64 ty) (rotl src (imm8_from_value amt))))
+      (value_reg (m_rotl ty (put_in_reg src) amt)))
+
+;; `i128`.
+
+(rule (lower (has_type $I128 (rotl src amt)))
+      (let ((src_ ValueRegs (put_in_regs src))
+            ;; NB: Only the low bits of `amt` matter since we logically mask the
+            ;; rotation amount to the value's bit width.
+            (amt_ Reg (lo_reg amt)))
+        (or_i128 (shl_i128 src_ amt_)
+                 (shr_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_))))))
+
+;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type (multi_lane 8 16)
+                       (avg_round x y)))
+      (value_reg (pavgb (put_in_reg x) (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 16 8)
+                       (avg_round x y)))
+      (value_reg (pavgw (put_in_reg x) (put_in_reg_mem y))))
+
+;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `i64` and smaller.
+
+;; Multiply two registers.
+(rule (lower (has_type (fits_in_64 ty) (imul x y)))
+      (value_reg (mul ty
+                      (put_in_reg x)
+                      (RegMemImm.Reg (put_in_reg y)))))
+
+;; Multiply a register and an immediate.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (imul x (simm32_from_value y))))
+      (value_reg (mul ty (put_in_reg x) y)))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (imul (simm32_from_value x) y)))
+      (value_reg (mul ty (put_in_reg y) x)))
+
+;; Multiply a register and a memory load.
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (imul x (sinkable_load y))))
+      (value_reg (mul ty
+                      (put_in_reg x)
+                      (sink_load y))))
+
+(rule (lower (has_type (fits_in_64 ty)
+                       (imul (sinkable_load x) y)))
+      (value_reg (mul ty
+                      (put_in_reg y)
+                      (sink_load x))))
+
+;; `i128`.
+
+;; mul:
+;;   dst_lo = lhs_lo * rhs_lo
+;;   dst_hi = umulhi(lhs_lo, rhs_lo) +
+;;            lhs_lo * rhs_hi +
+;;            lhs_hi * rhs_lo
+;;
+;; so we emit:
+;;   lo_hi = mul x_lo, y_hi
+;;   hi_lo = mul x_hi, y_lo
+;;   hilo_hilo = add lo_hi, hi_lo
+;;   dst_lo:hi_lolo = mulhi_u x_lo, y_lo
+;;   dst_hi = add hilo_hilo, hi_lolo
+;;   return (dst_lo, dst_hi)
+(rule (lower (has_type $I128 (imul x y)))
+      ;; Put `x` into registers and unpack its hi/lo halves.
+      (let ((x_regs ValueRegs (put_in_regs x))
+            (x_lo Reg (value_regs_get x_regs 0))
+            (x_hi Reg (value_regs_get x_regs 1))
+            ;; Put `y` into registers and unpack its hi/lo halves.
+            (y_regs ValueRegs (put_in_regs y))
+            (y_lo Reg (value_regs_get y_regs 0))
+            (y_hi Reg (value_regs_get y_regs 1))
+            ;; lo_hi = mul x_lo, y_hi
+            (lo_hi Reg (mul $I64 x_lo (RegMemImm.Reg y_hi)))
+            ;; hi_lo = mul x_hi, y_lo
+            (hi_lo Reg (mul $I64 x_hi (RegMemImm.Reg y_lo)))
+            ;; hilo_hilo = add lo_hi, hi_lo
+            (hilo_hilo Reg (add $I64 lo_hi (RegMemImm.Reg hi_lo)))
+            ;; dst_lo:hi_lolo = mulhi_u x_lo, y_lo
+            (mul_regs ValueRegs (mulhi_u $I64 x_lo (RegMem.Reg y_lo)))
+            (dst_lo Reg (value_regs_get mul_regs 0))
+            (hi_lolo Reg (value_regs_get mul_regs 1))
+            ;; dst_hi = add hilo_hilo, hi_lolo
+            (dst_hi Reg (add $I64 hilo_hilo (RegMemImm.Reg hi_lolo))))
+        (value_regs dst_lo dst_hi)))
+
+;; SSE.
+
+;; (No i8x16 multiply.)
+
+(rule (lower (has_type (multi_lane 16 8) (imul x y)))
+      (value_reg (pmullw (put_in_reg x) (put_in_reg_mem y))))
+
+(rule (lower (has_type (multi_lane 32 4) (imul x y)))
+      (value_reg (pmulld (put_in_reg x) (put_in_reg_mem y))))
+
+;; With AVX-512 we can implement `i64x2` multiplication with a single
+;; instruction.
+(rule (lower (has_type (and (avx512vl_enabled)
+                            (avx512dq_enabled)
+                            (multi_lane 64 2))
+                       (imul x y)))
+      (value_reg (vpmullq (put_in_reg_mem x) (put_in_reg y))))
+
+;; Otherwise, for i64x2 multiplication we describe a lane A as being composed of
+;; a 32-bit upper half "Ah" and a 32-bit lower half "Al". The 32-bit long hand
+;; multiplication can then be written as:
+;;
+;;    Ah Al
+;; *  Bh Bl
+;;    -----
+;;    Al * Bl
+;; + (Ah * Bl) << 32
+;; + (Al * Bh) << 32
+;;
+;; So for each lane we will compute:
+;;
+;;   A * B  = (Al * Bl) + ((Ah * Bl) + (Al * Bh)) << 32
+;;
+;; Note, the algorithm will use `pmuldq` which operates directly on the lower
+;; 32-bit (`Al` or `Bl`) of a lane and writes the result to the full 64-bits of
+;; the lane of the destination. For this reason we don't need shifts to isolate
+;; the lower 32-bits, however, we will need to use shifts to isolate the high
+;; 32-bits when doing calculations, i.e., `Ah == A >> 32`.
+(rule (lower (has_type (multi_lane 64 2)
+                       (imul a b)))
+      (let ((a0 Reg (put_in_reg a))
+            (b0 Reg (put_in_reg b))
+            ;; a_hi = A >> 32
+            (a_hi Reg (psrlq a0 (RegMemImm.Imm 32)))
+            ;; ah_bl = Ah * Bl
+            (ah_bl Reg (pmuludq a_hi (RegMem.Reg b0)))
+            ;; b_hi = B >> 32
+            (b_hi Reg (psrlq b0 (RegMemImm.Imm 32)))
+            ;; al_bh = Al * Bh
+            (al_bh Reg (pmuludq a0 (RegMem.Reg b_hi)))
+            ;; aa_bb = ah_bl + al_bh
+            (aa_bb Reg (paddq ah_bl (RegMem.Reg al_bh)))
+            ;; aa_bb_shifted = aa_bb << 32
+            (aa_bb_shifted Reg (psllq aa_bb (RegMemImm.Imm 32)))
+            ;; al_bl = Al * Bl
+            (al_bl Reg (pmuludq a0 (RegMem.Reg b0))))
+        ;; al_bl + aa_bb_shifted
+        (value_reg (paddq al_bl (RegMem.Reg aa_bb_shifted)))))
+
+;; Special case for `i16x8.extmul_high_i8x16_s`.
+(rule (lower (has_type (multi_lane 16 8)
+                       (imul (def_inst (swiden_high (and (value_type (multi_lane 8 16))
+                                                         x)))
+                             (def_inst (swiden_high (and (value_type (multi_lane 8 16))
+                                                         y))))))
+      (let ((x1 Reg (put_in_reg x))
+            (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32)))
+            (x3 Reg (pmovsxbw (RegMem.Reg x2)))
+            (y1 Reg (put_in_reg y))
+            (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32)))
+            (y3 Reg (pmovsxbw (RegMem.Reg y2))))
+        (value_reg (pmullw x3 (RegMem.Reg y3)))))
+
+;; Special case for `i32x4.extmul_high_i16x8_s`.
+(rule (lower (has_type (multi_lane 32 4)
+                       (imul (def_inst (swiden_high (and (value_type (multi_lane 16 8))
+                                                         x)))
+                             (def_inst (swiden_high (and (value_type (multi_lane 16 8))
+                                                         y))))))
+      (let ((x2 Reg (put_in_reg x))
+            (y2 Reg (put_in_reg y))
+            (lo Reg (pmullw x2 (RegMem.Reg y2)))
+            (hi Reg (pmulhw x2 (RegMem.Reg y2))))
+        (value_reg (punpckhwd lo (RegMem.Reg hi)))))
+
+;; Special case for `i64x2.extmul_high_i32x4_s`.
+(rule (lower (has_type (multi_lane 64 2)
+                       (imul (def_inst (swiden_high (and (value_type (multi_lane 32 4))
+                                                         x)))
+                             (def_inst (swiden_high (and (value_type (multi_lane 32 4))
+                                                         y))))))
+      (let ((x2 Reg (pshufd (put_in_reg_mem x)
+                            0xFA
+                            (OperandSize.Size32)))
+            (y2 Reg (pshufd (put_in_reg_mem y)
+                            0xFA
+                            (OperandSize.Size32))))
+        (value_reg (pmuldq x2 (RegMem.Reg y2)))))
+
+;; Special case for `i16x8.extmul_low_i8x16_s`.
+(rule (lower (has_type (multi_lane 16 8)
+                       (imul (def_inst (swiden_low (and (value_type (multi_lane 8 16))
+                                                        x)))
+                             (def_inst (swiden_low (and (value_type (multi_lane 8 16))
+                                                        y))))))
+      (let ((x2 Reg (pmovsxbw (put_in_reg_mem x)))
+            (y2 Reg (pmovsxbw (put_in_reg_mem y))))
+        (value_reg (pmullw x2 (RegMem.Reg y2)))))
+
+;; Special case for `i32x4.extmul_low_i16x8_s`.
+(rule (lower (has_type (multi_lane 32 4)
+                       (imul (def_inst (swiden_low (and (value_type (multi_lane 16 8))
+                                                        x)))
+                             (def_inst (swiden_low (and (value_type (multi_lane 16 8))
+                                                        y))))))
+      (let ((x2 Reg (put_in_reg x))
+            (y2 Reg (put_in_reg y))
+            (lo Reg (pmullw x2 (RegMem.Reg y2)))
+            (hi Reg (pmulhw x2 (RegMem.Reg y2))))
+        (value_reg (punpcklwd lo (RegMem.Reg hi)))))
+
+;; Special case for `i64x2.extmul_low_i32x4_s`.
+(rule (lower (has_type (multi_lane 64 2)
+                       (imul (def_inst (swiden_low (and (value_type (multi_lane 32 4))
+                                                        x)))
+                             (def_inst (swiden_low (and (value_type (multi_lane 32 4))
+                                                        y))))))
+      (let ((x2 Reg (pshufd (put_in_reg_mem x)
+                            0x50
+                            (OperandSize.Size32)))
+            (y2 Reg (pshufd (put_in_reg_mem y)
+                            0x50
+                            (OperandSize.Size32))))
+        (value_reg (pmuldq x2 (RegMem.Reg y2)))))
+
+;; Special case for `i16x8.extmul_high_i8x16_u`.
+(rule (lower (has_type (multi_lane 16 8)
+                       (imul (def_inst (uwiden_high (and (value_type (multi_lane 8 16))
+                                                         x)))
+                             (def_inst (uwiden_high (and (value_type (multi_lane 8 16))
+                                                         y))))))
+      (let ((x1 Reg (put_in_reg x))
+            (x2 Reg (palignr x1 (RegMem.Reg x1) 8 (OperandSize.Size32)))
+            (x3 Reg (pmovzxbw (RegMem.Reg x2)))
+            (y1 Reg (put_in_reg y))
+            (y2 Reg (palignr y1 (RegMem.Reg y1) 8 (OperandSize.Size32)))
+            (y3 Reg (pmovzxbw (RegMem.Reg y2))))
+        (value_reg (pmullw x3 (RegMem.Reg y3)))))
+
+;; Special case for `i32x4.extmul_high_i16x8_u`.
+(rule (lower (has_type (multi_lane 32 4)
+                       (imul (def_inst (uwiden_high (and (value_type (multi_lane 16 8))
+                                                         x)))
+                             (def_inst (uwiden_high (and (value_type (multi_lane 16 8))
+                                                         y))))))
+      (let ((x2 Reg (put_in_reg x))
+            (y2 Reg (put_in_reg y))
+            (lo Reg (pmullw x2 (RegMem.Reg y2)))
+            (hi Reg (pmulhuw x2 (RegMem.Reg y2))))
+        (value_reg (punpckhwd lo (RegMem.Reg hi)))))
+
+;; Special case for `i64x2.extmul_high_i32x4_u`.
+(rule (lower (has_type (multi_lane 64 2)
+                       (imul (def_inst (uwiden_high (and (value_type (multi_lane 32 4))
+                                                         x)))
+                             (def_inst (uwiden_high (and (value_type (multi_lane 32 4))
+                                                         y))))))
+      (let ((x2 Reg (pshufd (put_in_reg_mem x)
+                            0xFA
+                            (OperandSize.Size32)))
+            (y2 Reg (pshufd (put_in_reg_mem y)
+                            0xFA
+                            (OperandSize.Size32))))
+        (value_reg (pmuludq x2 (RegMem.Reg y2)))))
+
+;; Special case for `i16x8.extmul_low_i8x16_u`.
+(rule (lower (has_type (multi_lane 16 8)
+                       (imul (def_inst (uwiden_low (and (value_type (multi_lane 8 16))
+                                                        x)))
+                             (def_inst (uwiden_low (and (value_type (multi_lane 8 16))
+                                                        y))))))
+      (let ((x2 Reg (pmovzxbw (put_in_reg_mem x)))
+            (y2 Reg (pmovzxbw (put_in_reg_mem y))))
+        (value_reg (pmullw x2 (RegMem.Reg y2)))))
+
+;; Special case for `i32x4.extmul_low_i16x8_u`.
+(rule (lower (has_type (multi_lane 32 4)
+                       (imul (def_inst (uwiden_low (and (value_type (multi_lane 16 8))
+                                                        x)))
+                             (def_inst (uwiden_low (and (value_type (multi_lane 16 8))
+                                                        y))))))
+      (let ((x2 Reg (put_in_reg x))
+            (y2 Reg (put_in_reg y))
+            (lo Reg (pmullw x2 (RegMem.Reg y2)))
+            (hi Reg (pmulhuw x2 (RegMem.Reg y2))))
+        (value_reg (punpcklwd lo (RegMem.Reg hi)))))
+
+;; Special case for `i64x2.extmul_low_i32x4_u`.
+(rule (lower (has_type (multi_lane 64 2)
+                       (imul (def_inst (uwiden_low (and (value_type (multi_lane 32 4))
+                                                        x)))
+                             (def_inst (uwiden_low (and (value_type (multi_lane 32 4))
+                                                        y))))))
+      (let ((x2 Reg (pshufd (put_in_reg_mem x)
+                            0x50
+                            (OperandSize.Size32)))
+            (y2 Reg (pshufd (put_in_reg_mem y)
+                            0x50
+                            (OperandSize.Size32))))
+        (value_reg (pmuludq x2 (RegMem.Reg y2)))))
+
+;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Note the flipping of operands below. CLIF specifies
+;;
+;;   band_not(x, y) = and(x, not(y))
+;;
+;; while x86 does
+;;
+;;   pandn(x, y) = and(not(x), y)
+
+(rule (lower (has_type $F32X4 (band_not x y)))
+      (value_reg (andnps (put_in_reg y) (put_in_reg_mem x))))
+
+(rule (lower (has_type $F64X2 (band_not x y)))
+      (value_reg (andnpd (put_in_reg y) (put_in_reg_mem x))))
+
+(rule (lower (has_type (multi_lane _bits _lanes) (band_not x y)))
+      (value_reg (pandn (put_in_reg y) (put_in_reg_mem x))))
diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs
index c67dab3dde..71500f7c7d 100644
--- a/cranelift/codegen/src/isa/x64/lower.rs
+++ b/cranelift/codegen/src/isa/x64/lower.rs
@@ -1,5 +1,8 @@
 //! Lowering rules for X64.
 
+// ISLE integration glue.
+mod isle;
+
 use crate::data_value::DataValue;
 use crate::ir::{
     condcodes::{CondCode, FloatCC, IntCC},
@@ -1497,20 +1500,15 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         None
     };
 
-    match op {
-        Opcode::Iconst | Opcode::Bconst | Opcode::Null => {
-            let value = ctx
-                .get_constant(insn)
-                .expect("constant value for iconst et al");
-            let dst = get_output_reg(ctx, outputs[0]);
-            for inst in Inst::gen_constant(dst, value as u128, ty.unwrap(), |ty| {
-                ctx.alloc_tmp(ty).only_reg().unwrap()
-            }) {
-                ctx.emit(inst);
-            }
-        }
+    if let Ok(()) = isle::lower(ctx, isa_flags, &outputs, insn) {
+        return Ok(());
+    }
 
-        Opcode::Iadd
+    match op {
+        Opcode::Iconst
+        | Opcode::Bconst
+        | Opcode::Null
+        | Opcode::Iadd
         | Opcode::IaddIfcout
         | Opcode::SaddSat
         | Opcode::UaddSat
@@ -1520,755 +1518,14 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         | Opcode::AvgRound
         | Opcode::Band
         | Opcode::Bor
-        | Opcode::Bxor => {
-            let ty = ty.unwrap();
-            if ty.lane_count() > 1 {
-                let sse_op = match op {
-                    Opcode::Iadd => match ty {
-                        types::I8X16 => SseOpcode::Paddb,
-                        types::I16X8 => SseOpcode::Paddw,
-                        types::I32X4 => SseOpcode::Paddd,
-                        types::I64X2 => SseOpcode::Paddq,
-                        _ => panic!("Unsupported type for packed iadd instruction: {}", ty),
-                    },
-                    Opcode::SaddSat => match ty {
-                        types::I8X16 => SseOpcode::Paddsb,
-                        types::I16X8 => SseOpcode::Paddsw,
-                        _ => panic!("Unsupported type for packed sadd_sat instruction: {}", ty),
-                    },
-                    Opcode::UaddSat => match ty {
-                        types::I8X16 => SseOpcode::Paddusb,
-                        types::I16X8 => SseOpcode::Paddusw,
-                        _ => panic!("Unsupported type for packed uadd_sat instruction: {}", ty),
-                    },
-                    Opcode::Isub => match ty {
-                        types::I8X16 => SseOpcode::Psubb,
-                        types::I16X8 => SseOpcode::Psubw,
-                        types::I32X4 => SseOpcode::Psubd,
-                        types::I64X2 => SseOpcode::Psubq,
-                        _ => panic!("Unsupported type for packed isub instruction: {}", ty),
-                    },
-                    Opcode::SsubSat => match ty {
-                        types::I8X16 => SseOpcode::Psubsb,
-                        types::I16X8 => SseOpcode::Psubsw,
-                        _ => panic!("Unsupported type for packed ssub_sat instruction: {}", ty),
-                    },
-                    Opcode::UsubSat => match ty {
-                        types::I8X16 => SseOpcode::Psubusb,
-                        types::I16X8 => SseOpcode::Psubusw,
-                        _ => panic!("Unsupported type for packed usub_sat instruction: {}", ty),
-                    },
-                    Opcode::AvgRound => match ty {
-                        types::I8X16 => SseOpcode::Pavgb,
-                        types::I16X8 => SseOpcode::Pavgw,
-                        _ => panic!("Unsupported type for packed avg_round instruction: {}", ty),
-                    },
-                    Opcode::Band => match ty {
-                        types::F32X4 => SseOpcode::Andps,
-                        types::F64X2 => SseOpcode::Andpd,
-                        _ => SseOpcode::Pand,
-                    },
-                    Opcode::Bor => match ty {
-                        types::F32X4 => SseOpcode::Orps,
-                        types::F64X2 => SseOpcode::Orpd,
-                        _ => SseOpcode::Por,
-                    },
-                    Opcode::Bxor => match ty {
-                        types::F32X4 => SseOpcode::Xorps,
-                        types::F64X2 => SseOpcode::Xorpd,
-                        _ => SseOpcode::Pxor,
-                    },
-                    _ => panic!("Unsupported packed instruction: {}", op),
-                };
-                let lhs = put_input_in_reg(ctx, inputs[0]);
-                let rhs = input_to_reg_mem(ctx, inputs[1]);
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-                // Move the `lhs` to the same register as `dst`.
-                ctx.emit(Inst::gen_move(dst, lhs, ty));
-                ctx.emit(Inst::xmm_rm_r(sse_op, rhs, dst));
-            } else if ty == types::I128 || ty == types::B128 {
-                let alu_ops = match op {
-                    Opcode::Iadd => (AluRmiROpcode::Add, AluRmiROpcode::Adc),
-                    Opcode::Isub => (AluRmiROpcode::Sub, AluRmiROpcode::Sbb),
-                    Opcode::Band => (AluRmiROpcode::And, AluRmiROpcode::And),
-                    Opcode::Bor => (AluRmiROpcode::Or, AluRmiROpcode::Or),
-                    Opcode::Bxor => (AluRmiROpcode::Xor, AluRmiROpcode::Xor),
-                    _ => panic!("Unsupported opcode with 128-bit integers: {:?}", op),
-                };
-                let lhs = put_input_in_regs(ctx, inputs[0]);
-                let rhs = put_input_in_regs(ctx, inputs[1]);
-                let dst = get_output_reg(ctx, outputs[0]);
-                assert_eq!(lhs.len(), 2);
-                assert_eq!(rhs.len(), 2);
-                assert_eq!(dst.len(), 2);
-
-                // For add, sub, and, or, xor: just do ops on lower then upper
-                // half. Carry-flag propagation is implicit (add/adc, sub/sbb).
-                ctx.emit(Inst::gen_move(dst.regs()[0], lhs.regs()[0], types::I64));
-                ctx.emit(Inst::gen_move(dst.regs()[1], lhs.regs()[1], types::I64));
-                ctx.emit(Inst::alu_rmi_r(
-                    OperandSize::Size64,
-                    alu_ops.0,
-                    RegMemImm::reg(rhs.regs()[0]),
-                    dst.regs()[0],
-                ));
-                ctx.emit(Inst::alu_rmi_r(
-                    OperandSize::Size64,
-                    alu_ops.1,
-                    RegMemImm::reg(rhs.regs()[1]),
-                    dst.regs()[1],
-                ));
-            } else {
-                let size = if ty == types::I64 {
-                    OperandSize::Size64
-                } else {
-                    OperandSize::Size32
-                };
-                let alu_op = match op {
-                    Opcode::Iadd | Opcode::IaddIfcout => AluRmiROpcode::Add,
-                    Opcode::Isub => AluRmiROpcode::Sub,
-                    Opcode::Band => AluRmiROpcode::And,
-                    Opcode::Bor => AluRmiROpcode::Or,
-                    Opcode::Bxor => AluRmiROpcode::Xor,
-                    _ => unreachable!(),
-                };
-
-                let (lhs, rhs) = match op {
-                    Opcode::Iadd
-                    | Opcode::IaddIfcout
-                    | Opcode::Band
-                    | Opcode::Bor
-                    | Opcode::Bxor => {
-                        // For commutative operations, try to commute operands if one is an
-                        // immediate or direct memory reference. Do so by converting LHS to RMI; if
-                        // reg, then always convert RHS to RMI; else, use LHS as RMI and convert
-                        // RHS to reg.
-                        let lhs = input_to_reg_mem_imm(ctx, inputs[0]);
-                        if let RegMemImm::Reg { reg: lhs_reg } = lhs {
-                            let rhs = input_to_reg_mem_imm(ctx, inputs[1]);
-                            (lhs_reg, rhs)
-                        } else {
-                            let rhs_reg = put_input_in_reg(ctx, inputs[1]);
-                            (rhs_reg, lhs)
-                        }
-                    }
-                    Opcode::Isub => (
-                        put_input_in_reg(ctx, inputs[0]),
-                        input_to_reg_mem_imm(ctx, inputs[1]),
-                    ),
-                    _ => unreachable!(),
-                };
-
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                ctx.emit(Inst::mov_r_r(OperandSize::Size64, lhs, dst));
-                ctx.emit(Inst::alu_rmi_r(size, alu_op, rhs, dst));
-            }
-        }
-
-        Opcode::Imul => {
-            let ty = ty.unwrap();
-
-            // Check for ext_mul_* instructions which are being shared here under imul. We must
-            // check first for operands that are opcodes since checking for types is not enough.
-            if let Some(_) = matches_input_any(
-                ctx,
-                inputs[0],
-                &[
-                    Opcode::SwidenHigh,
-                    Opcode::SwidenLow,
-                    Opcode::UwidenHigh,
-                    Opcode::UwidenLow,
-                ],
-            ) {
-                // Optimized ext_mul_* lowerings are based on optimized lowerings
-                // here: https://github.com/WebAssembly/simd/pull/376
-                if let Some(swiden0_high) = matches_input(ctx, inputs[0], Opcode::SwidenHigh) {
-                    if let Some(swiden1_high) = matches_input(ctx, inputs[1], Opcode::SwidenHigh) {
-                        let swiden_input = &[
-                            InsnInput {
-                                insn: swiden0_high,
-                                input: 0,
-                            },
-                            InsnInput {
-                                insn: swiden1_high,
-                                input: 0,
-                            },
-                        ];
-                        let input0_ty = ctx.input_ty(swiden0_high, 0);
-                        let input1_ty = ctx.input_ty(swiden1_high, 0);
-                        let output_ty = ctx.output_ty(insn, 0);
-                        let lhs = put_input_in_reg(ctx, swiden_input[0]);
-                        let rhs = put_input_in_reg(ctx, swiden_input[1]);
-                        let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-                        match (input0_ty, input1_ty, output_ty) {
-                            (types::I8X16, types::I8X16, types::I16X8) => {
-                                // i16x8.extmul_high_i8x16_s
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Palignr,
-                                    RegMem::reg(lhs),
-                                    Writable::from_reg(lhs),
-                                    8,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_mov(
-                                    SseOpcode::Pmovsxbw,
-                                    RegMem::reg(lhs),
-                                    Writable::from_reg(lhs),
-                                ));
-
-                                ctx.emit(Inst::gen_move(dst, rhs, output_ty));
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Palignr,
-                                    RegMem::reg(rhs),
-                                    dst,
-                                    8,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_mov(
-                                    SseOpcode::Pmovsxbw,
-                                    RegMem::reg(dst.to_reg()),
-                                    dst,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(lhs), dst));
-                            }
-                            (types::I16X8, types::I16X8, types::I32X4) => {
-                                // i32x4.extmul_high_i16x8_s
-                                ctx.emit(Inst::gen_move(dst, lhs, input0_ty));
-                                let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap();
-                                ctx.emit(Inst::gen_move(tmp_reg, lhs, input0_ty));
-                                ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(rhs), dst));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmulhw,
-                                    RegMem::reg(rhs),
-                                    tmp_reg,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Punpckhwd,
-                                    RegMem::from(tmp_reg),
-                                    dst,
-                                ));
-                            }
-                            (types::I32X4, types::I32X4, types::I64X2) => {
-                                // i64x2.extmul_high_i32x4_s
-                                let tmp_reg = ctx.alloc_tmp(types::I32X4).only_reg().unwrap();
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Pshufd,
-                                    RegMem::reg(lhs),
-                                    tmp_reg,
-                                    0xFA,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Pshufd,
-                                    RegMem::reg(rhs),
-                                    dst,
-                                    0xFA,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmuldq,
-                                    RegMem::reg(tmp_reg.to_reg()),
-                                    dst,
-                                ));
-                            }
-                            // Note swiden_high only allows types: I8X16, I16X8, and I32X4
-                            _ => panic!("Unsupported extmul_low_signed type"),
-                        }
-                    }
-                } else if let Some(swiden0_low) = matches_input(ctx, inputs[0], Opcode::SwidenLow) {
-                    if let Some(swiden1_low) = matches_input(ctx, inputs[1], Opcode::SwidenLow) {
-                        let swiden_input = &[
-                            InsnInput {
-                                insn: swiden0_low,
-                                input: 0,
-                            },
-                            InsnInput {
-                                insn: swiden1_low,
-                                input: 0,
-                            },
-                        ];
-                        let input0_ty = ctx.input_ty(swiden0_low, 0);
-                        let input1_ty = ctx.input_ty(swiden1_low, 0);
-                        let output_ty = ctx.output_ty(insn, 0);
-                        let lhs = put_input_in_reg(ctx, swiden_input[0]);
-                        let rhs = put_input_in_reg(ctx, swiden_input[1]);
-                        let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-                        match (input0_ty, input1_ty, output_ty) {
-                            (types::I8X16, types::I8X16, types::I16X8) => {
-                                // i32x4.extmul_low_i8x16_s
-                                let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap();
-                                ctx.emit(Inst::xmm_mov(
-                                    SseOpcode::Pmovsxbw,
-                                    RegMem::reg(lhs),
-                                    tmp_reg,
-                                ));
-                                ctx.emit(Inst::xmm_mov(SseOpcode::Pmovsxbw, RegMem::reg(rhs), dst));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmullw,
-                                    RegMem::reg(tmp_reg.to_reg()),
-                                    dst,
-                                ));
-                            }
-                            (types::I16X8, types::I16X8, types::I32X4) => {
-                                // i32x4.extmul_low_i16x8_s
-                                ctx.emit(Inst::gen_move(dst, lhs, input0_ty));
-                                let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap();
-                                ctx.emit(Inst::gen_move(tmp_reg, lhs, input0_ty));
-                                ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(rhs), dst));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmulhw,
-                                    RegMem::reg(rhs),
-                                    tmp_reg,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Punpcklwd,
-                                    RegMem::from(tmp_reg),
-                                    dst,
-                                ));
-                            }
-                            (types::I32X4, types::I32X4, types::I64X2) => {
-                                // i64x2.extmul_low_i32x4_s
-                                let tmp_reg = ctx.alloc_tmp(types::I32X4).only_reg().unwrap();
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Pshufd,
-                                    RegMem::reg(lhs),
-                                    tmp_reg,
-                                    0x50,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Pshufd,
-                                    RegMem::reg(rhs),
-                                    dst,
-                                    0x50,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmuldq,
-                                    RegMem::reg(tmp_reg.to_reg()),
-                                    dst,
-                                ));
-                            }
-                            // Note swiden_low only allows types: I8X16, I16X8, and I32X4
-                            _ => panic!("Unsupported extmul_low_signed type"),
-                        }
-                    }
-                } else if let Some(uwiden0_high) = matches_input(ctx, inputs[0], Opcode::UwidenHigh)
-                {
-                    if let Some(uwiden1_high) = matches_input(ctx, inputs[1], Opcode::UwidenHigh) {
-                        let uwiden_input = &[
-                            InsnInput {
-                                insn: uwiden0_high,
-                                input: 0,
-                            },
-                            InsnInput {
-                                insn: uwiden1_high,
-                                input: 0,
-                            },
-                        ];
-                        let input0_ty = ctx.input_ty(uwiden0_high, 0);
-                        let input1_ty = ctx.input_ty(uwiden1_high, 0);
-                        let output_ty = ctx.output_ty(insn, 0);
-                        let lhs = put_input_in_reg(ctx, uwiden_input[0]);
-                        let rhs = put_input_in_reg(ctx, uwiden_input[1]);
-                        let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-                        match (input0_ty, input1_ty, output_ty) {
-                            (types::I8X16, types::I8X16, types::I16X8) => {
-                                // i16x8.extmul_high_i8x16_u
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Palignr,
-                                    RegMem::reg(lhs),
-                                    Writable::from_reg(lhs),
-                                    8,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_mov(
-                                    SseOpcode::Pmovzxbw,
-                                    RegMem::reg(lhs),
-                                    Writable::from_reg(lhs),
-                                ));
-                                ctx.emit(Inst::gen_move(dst, rhs, output_ty));
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Palignr,
-                                    RegMem::reg(rhs),
-                                    dst,
-                                    8,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_mov(
-                                    SseOpcode::Pmovzxbw,
-                                    RegMem::reg(dst.to_reg()),
-                                    dst,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(lhs), dst));
-                            }
-                            (types::I16X8, types::I16X8, types::I32X4) => {
-                                // i32x4.extmul_high_i16x8_u
-                                ctx.emit(Inst::gen_move(dst, lhs, input0_ty));
-                                let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap();
-                                ctx.emit(Inst::gen_move(tmp_reg, lhs, input0_ty));
-                                ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(rhs), dst));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmulhuw,
-                                    RegMem::reg(rhs),
-                                    tmp_reg,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Punpckhwd,
-                                    RegMem::from(tmp_reg),
-                                    dst,
-                                ));
-                            }
-                            (types::I32X4, types::I32X4, types::I64X2) => {
-                                // i64x2.extmul_high_i32x4_u
-                                let tmp_reg = ctx.alloc_tmp(types::I32X4).only_reg().unwrap();
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Pshufd,
-                                    RegMem::reg(lhs),
-                                    tmp_reg,
-                                    0xFA,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Pshufd,
-                                    RegMem::reg(rhs),
-                                    dst,
-                                    0xFA,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmuludq,
-                                    RegMem::reg(tmp_reg.to_reg()),
-                                    dst,
-                                ));
-                            }
-                            // Note uwiden_high only allows types: I8X16, I16X8, and I32X4
-                            _ => panic!("Unsupported extmul_high_unsigned type"),
-                        }
-                    }
-                } else if let Some(uwiden0_low) = matches_input(ctx, inputs[0], Opcode::UwidenLow) {
-                    if let Some(uwiden1_low) = matches_input(ctx, inputs[1], Opcode::UwidenLow) {
-                        let uwiden_input = &[
-                            InsnInput {
-                                insn: uwiden0_low,
-                                input: 0,
-                            },
-                            InsnInput {
-                                insn: uwiden1_low,
-                                input: 0,
-                            },
-                        ];
-
-                        let input0_ty = ctx.input_ty(uwiden0_low, 0);
-                        let input1_ty = ctx.input_ty(uwiden1_low, 0);
-                        let output_ty = ctx.output_ty(insn, 0);
-                        let lhs = put_input_in_reg(ctx, uwiden_input[0]);
-                        let rhs = put_input_in_reg(ctx, uwiden_input[1]);
-                        let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-                        match (input0_ty, input1_ty, output_ty) {
-                            (types::I8X16, types::I8X16, types::I16X8) => {
-                                // i16x8.extmul_low_i8x16_u
-                                let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap();
-                                ctx.emit(Inst::xmm_mov(
-                                    SseOpcode::Pmovzxbw,
-                                    RegMem::reg(lhs),
-                                    tmp_reg,
-                                ));
-                                ctx.emit(Inst::xmm_mov(SseOpcode::Pmovzxbw, RegMem::reg(rhs), dst));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmullw,
-                                    RegMem::reg(tmp_reg.to_reg()),
-                                    dst,
-                                ));
-                            }
-                            (types::I16X8, types::I16X8, types::I32X4) => {
-                                // i32x4.extmul_low_i16x8_u
-                                ctx.emit(Inst::gen_move(dst, lhs, input0_ty));
-                                let tmp_reg = ctx.alloc_tmp(types::I16X8).only_reg().unwrap();
-                                ctx.emit(Inst::gen_move(tmp_reg, lhs, input0_ty));
-                                ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmullw, RegMem::reg(rhs), dst));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmulhuw,
-                                    RegMem::reg(rhs),
-                                    tmp_reg,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Punpcklwd,
-                                    RegMem::from(tmp_reg),
-                                    dst,
-                                ));
-                            }
-                            (types::I32X4, types::I32X4, types::I64X2) => {
-                                // i64x2.extmul_low_i32x4_u
-                                let tmp_reg = ctx.alloc_tmp(types::I32X4).only_reg().unwrap();
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Pshufd,
-                                    RegMem::reg(lhs),
-                                    tmp_reg,
-                                    0x50,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r_imm(
-                                    SseOpcode::Pshufd,
-                                    RegMem::reg(rhs),
-                                    dst,
-                                    0x50,
-                                    OperandSize::Size32,
-                                ));
-                                ctx.emit(Inst::xmm_rm_r(
-                                    SseOpcode::Pmuludq,
-                                    RegMem::reg(tmp_reg.to_reg()),
-                                    dst,
-                                ));
-                            }
-                            // Note uwiden_low only allows types: I8X16, I16X8, and I32X4
-                            _ => panic!("Unsupported extmul_low_unsigned type"),
-                        }
-                    }
-                } else {
-                    panic!("Unsupported imul operation for type: {}", ty);
-                }
-            } else if ty == types::I64X2 {
-                // Eventually one of these should be `input_to_reg_mem` (TODO).
-                let lhs = put_input_in_reg(ctx, inputs[0]);
-                let rhs = put_input_in_reg(ctx, inputs[1]);
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-                if isa_flags.use_avx512vl_simd() && isa_flags.use_avx512dq_simd() {
-                    // With the right AVX512 features (VL + DQ) this operation
-                    // can lower to a single operation.
-                    ctx.emit(Inst::xmm_rm_r_evex(
-                        Avx512Opcode::Vpmullq,
-                        RegMem::reg(rhs),
-                        lhs,
-                        dst,
-                    ));
-                } else {
-                    // Otherwise, for I64X2 multiplication we describe a lane A as being
-                    // composed of a 32-bit upper half "Ah" and a 32-bit lower half
-                    // "Al". The 32-bit long hand multiplication can then be written
-                    // as:
-                    //    Ah Al
-                    // *  Bh Bl
-                    //    -----
-                    //    Al * Bl
-                    // + (Ah * Bl) << 32
-                    // + (Al * Bh) << 32
-                    //
-                    // So for each lane we will compute:
-                    //   A * B  = (Al * Bl) + ((Ah * Bl) + (Al * Bh)) << 32
-                    //
-                    // Note, the algorithm will use pmuldq which operates directly
-                    // on the lower 32-bit (Al or Bl) of a lane and writes the
-                    // result to the full 64-bits of the lane of the destination.
-                    // For this reason we don't need shifts to isolate the lower
-                    // 32-bits, however, we will need to use shifts to isolate the
-                    // high 32-bits when doing calculations, i.e., Ah == A >> 32.
-                    //
-                    // The full sequence then is as follows:
-                    //   A' = A
-                    //   A' = A' >> 32
-                    //   A' = Ah' * Bl
-                    //   B' = B
-                    //   B' = B' >> 32
-                    //   B' = Bh' * Al
-                    //   B' = B' + A'
-                    //   B' = B' << 32
-                    //   A' = A
-                    //   A' = Al' * Bl
-                    //   A' = A' + B'
-                    //   dst = A'
-
-                    // A' = A
-                    let rhs_1 = ctx.alloc_tmp(types::I64X2).only_reg().unwrap();
-                    ctx.emit(Inst::gen_move(rhs_1, rhs, ty));
-
-                    // A' = A' >> 32
-                    // A' = Ah' * Bl
-                    ctx.emit(Inst::xmm_rmi_reg(
-                        SseOpcode::Psrlq,
-                        RegMemImm::imm(32),
-                        rhs_1,
-                    ));
-                    ctx.emit(Inst::xmm_rm_r(
-                        SseOpcode::Pmuludq,
-                        RegMem::reg(lhs.clone()),
-                        rhs_1,
-                    ));
-
-                    // B' = B
-                    let lhs_1 = ctx.alloc_tmp(types::I64X2).only_reg().unwrap();
-                    ctx.emit(Inst::gen_move(lhs_1, lhs, ty));
-
-                    // B' = B' >> 32
-                    // B' = Bh' * Al
-                    ctx.emit(Inst::xmm_rmi_reg(
-                        SseOpcode::Psrlq,
-                        RegMemImm::imm(32),
-                        lhs_1,
-                    ));
-                    ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmuludq, RegMem::reg(rhs), lhs_1));
-
-                    // B' = B' + A'
-                    // B' = B' << 32
-                    ctx.emit(Inst::xmm_rm_r(
-                        SseOpcode::Paddq,
-                        RegMem::reg(rhs_1.to_reg()),
-                        lhs_1,
-                    ));
-                    ctx.emit(Inst::xmm_rmi_reg(
-                        SseOpcode::Psllq,
-                        RegMemImm::imm(32),
-                        lhs_1,
-                    ));
-
-                    // A' = A
-                    // A' = Al' * Bl
-                    // A' = A' + B'
-                    // dst = A'
-                    ctx.emit(Inst::gen_move(rhs_1, rhs, ty));
-                    ctx.emit(Inst::xmm_rm_r(
-                        SseOpcode::Pmuludq,
-                        RegMem::reg(lhs.clone()),
-                        rhs_1,
-                    ));
-                    ctx.emit(Inst::xmm_rm_r(
-                        SseOpcode::Paddq,
-                        RegMem::reg(lhs_1.to_reg()),
-                        rhs_1,
-                    ));
-                    ctx.emit(Inst::gen_move(dst, rhs_1.to_reg(), ty));
-                }
-            } else if ty.lane_count() > 1 {
-                // Emit single instruction lowerings for the remaining vector
-                // multiplications.
-                let sse_op = match ty {
-                    types::I16X8 => SseOpcode::Pmullw,
-                    types::I32X4 => SseOpcode::Pmulld,
-                    _ => panic!("Unsupported type for packed imul instruction: {}", ty),
-                };
-                let lhs = put_input_in_reg(ctx, inputs[0]);
-                let rhs = input_to_reg_mem(ctx, inputs[1]);
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-                // Move the `lhs` to the same register as `dst`.
-                ctx.emit(Inst::gen_move(dst, lhs, ty));
-                ctx.emit(Inst::xmm_rm_r(sse_op, rhs, dst));
-            } else if ty == types::I128 || ty == types::B128 {
-                // Handle 128-bit multiplications.
-                let lhs = put_input_in_regs(ctx, inputs[0]);
-                let rhs = put_input_in_regs(ctx, inputs[1]);
-                let dst = get_output_reg(ctx, outputs[0]);
-                assert_eq!(lhs.len(), 2);
-                assert_eq!(rhs.len(), 2);
-                assert_eq!(dst.len(), 2);
-
-                // mul:
-                //   dst_lo = lhs_lo * rhs_lo
-                //   dst_hi = umulhi(lhs_lo, rhs_lo) + lhs_lo * rhs_hi + lhs_hi * rhs_lo
-                //
-                // so we emit:
-                //   mov dst_lo, lhs_lo
-                //   mul dst_lo, rhs_lo
-                //   mov dst_hi, lhs_lo
-                //   mul dst_hi, rhs_hi
-                //   mov tmp, lhs_hi
-                //   mul tmp, rhs_lo
-                //   add dst_hi, tmp
-                //   mov rax, lhs_lo
-                //   umulhi rhs_lo  // implicit rax arg/dst
-                //   add dst_hi, rax
-                let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                ctx.emit(Inst::gen_move(dst.regs()[0], lhs.regs()[0], types::I64));
-                ctx.emit(Inst::alu_rmi_r(
-                    OperandSize::Size64,
-                    AluRmiROpcode::Mul,
-                    RegMemImm::reg(rhs.regs()[0]),
-                    dst.regs()[0],
-                ));
-                ctx.emit(Inst::gen_move(dst.regs()[1], lhs.regs()[0], types::I64));
-                ctx.emit(Inst::alu_rmi_r(
-                    OperandSize::Size64,
-                    AluRmiROpcode::Mul,
-                    RegMemImm::reg(rhs.regs()[1]),
-                    dst.regs()[1],
-                ));
-                ctx.emit(Inst::gen_move(tmp, lhs.regs()[1], types::I64));
-                ctx.emit(Inst::alu_rmi_r(
-                    OperandSize::Size64,
-                    AluRmiROpcode::Mul,
-                    RegMemImm::reg(rhs.regs()[0]),
-                    tmp,
-                ));
-                ctx.emit(Inst::alu_rmi_r(
-                    OperandSize::Size64,
-                    AluRmiROpcode::Add,
-                    RegMemImm::reg(tmp.to_reg()),
-                    dst.regs()[1],
-                ));
-                ctx.emit(Inst::gen_move(
-                    Writable::from_reg(regs::rax()),
-                    lhs.regs()[0],
-                    types::I64,
-                ));
-                ctx.emit(Inst::mul_hi(
-                    OperandSize::Size64,
-                    /* signed = */ false,
-                    RegMem::reg(rhs.regs()[0]),
-                ));
-                ctx.emit(Inst::alu_rmi_r(
-                    OperandSize::Size64,
-                    AluRmiROpcode::Add,
-                    RegMemImm::reg(regs::rdx()),
-                    dst.regs()[1],
-                ));
-            } else {
-                let size = if ty == types::I64 {
-                    OperandSize::Size64
-                } else {
-                    OperandSize::Size32
-                };
-                let alu_op = AluRmiROpcode::Mul;
-
-                // For commutative operations, try to commute operands if one is
-                // an immediate or direct memory reference. Do so by converting
-                // LHS to RMI; if reg, then always convert RHS to RMI; else, use
-                // LHS as RMI and convert RHS to reg.
-                let lhs = input_to_reg_mem_imm(ctx, inputs[0]);
-                let (lhs, rhs) = if let RegMemImm::Reg { reg: lhs_reg } = lhs {
-                    let rhs = input_to_reg_mem_imm(ctx, inputs[1]);
-                    (lhs_reg, rhs)
-                } else {
-                    let rhs_reg = put_input_in_reg(ctx, inputs[1]);
-                    (rhs_reg, lhs)
-                };
-
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                ctx.emit(Inst::mov_r_r(OperandSize::Size64, lhs, dst));
-                ctx.emit(Inst::alu_rmi_r(size, alu_op, rhs, dst));
-            }
-        }
-
-        Opcode::BandNot => {
-            let ty = ty.unwrap();
-            debug_assert!(ty.is_vector() && ty.bytes() == 16);
-            let lhs = input_to_reg_mem(ctx, inputs[0]);
-            let rhs = put_input_in_reg(ctx, inputs[1]);
-            let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let sse_op = match ty {
-                types::F32X4 => SseOpcode::Andnps,
-                types::F64X2 => SseOpcode::Andnpd,
-                _ => SseOpcode::Pandn,
-            };
-            // Note the flipping of operands: the `rhs` operand is used as the destination instead
-            // of the `lhs` as in the other bit operations above (e.g. `band`).
-            ctx.emit(Inst::gen_move(dst, rhs, ty));
-            ctx.emit(Inst::xmm_rm_r(sse_op, lhs, dst));
+        | Opcode::Bxor
+        | Opcode::Imul
+        | Opcode::BandNot => {
+            unreachable!(
+                "implemented in ISLE: inst = `{}`, type = `{:?}`",
+                ctx.dfg().display_inst(insn),
+                ty
+            );
         }
 
         Opcode::Iabs => {
@@ -5801,7 +5058,14 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
             // Now the AtomicRmwSeq (pseudo-) instruction itself
             let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap());
-            ctx.emit(Inst::AtomicRmwSeq { ty: ty_access, op });
+            ctx.emit(Inst::AtomicRmwSeq {
+                ty: ty_access,
+                op,
+                address: regs::r9(),
+                operand: regs::r10(),
+                temp: Writable::from_reg(regs::r11()),
+                dst_old: Writable::from_reg(regs::rax()),
+            });
 
             // And finally, copy the preordained AtomicRmwSeq output reg to its destination.
             ctx.emit(Inst::gen_move(dst, regs::rax(), types::I64));
@@ -5827,8 +5091,10 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             ));
             ctx.emit(Inst::LockCmpxchg {
                 ty: ty_access,
-                src: replacement,
-                dst: addr.into(),
+                mem: addr.into(),
+                replacement,
+                expected: regs::rax(),
+                dst_old: Writable::from_reg(regs::rax()),
             });
             // And finally, copy the old value at the location to its destination reg.
             ctx.emit(Inst::gen_move(dst, regs::rax(), types::I64));
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
new file mode 100644
index 0000000000..1fe15ea976
--- /dev/null
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -0,0 +1,414 @@
+//! ISLE integration glue code for x64 lowering.
+
+// Pull in the ISLE generated code.
+mod generated_code;
+
+// Types that the generated ISLE code uses via `use super::*`.
+use super::{
+    is_mergeable_load, lower_to_amode, AluRmiROpcode, Inst as MInst, OperandSize, Reg, RegMemImm,
+    Writable,
+};
+use crate::isa::x64::inst::args::SyntheticAmode;
+use crate::isa::x64::settings as x64_settings;
+use crate::{
+    ir::{immediates::*, types::*, Inst, InstructionData, Opcode, Value, ValueList},
+    isa::x64::inst::{
+        args::{Avx512Opcode, CmpOpcode, ExtMode, Imm8Reg, RegMem, ShiftKind, SseOpcode, CC},
+        x64_map_regs, RegMapper,
+    },
+    machinst::{get_output_reg, InsnInput, InsnOutput, LowerCtx},
+};
+use smallvec::SmallVec;
+use std::convert::TryFrom;
+
+type Unit = ();
+type ValueSlice<'a> = &'a [Value];
+type ValueArray2 = [Value; 2];
+type ValueArray3 = [Value; 3];
+type WritableReg = Writable<Reg>;
+type ValueRegs = crate::machinst::ValueRegs<Reg>;
+
+pub struct SinkableLoad {
+    inst: Inst,
+    addr_input: InsnInput,
+    offset: i32,
+}
+
+#[derive(Default)]
+struct RegRenamer {
+    // Map of `(old, new)` register names. Use a `SmallVec` because we typically
+    // only have one or two renamings.
+    renames: SmallVec<[(Reg, Reg); 2]>,
+}
+
+impl RegRenamer {
+    fn add_rename(&mut self, old: Reg, new: Reg) {
+        self.renames.push((old, new));
+    }
+
+    fn get_rename(&self, reg: Reg) -> Option<Reg> {
+        self.renames
+            .iter()
+            .find(|(old, _)| reg == *old)
+            .map(|(_, new)| *new)
+    }
+}
+
+impl RegMapper for RegRenamer {
+    fn get_use(&self, reg: Reg) -> Option<Reg> {
+        self.get_rename(reg)
+    }
+
+    fn get_def(&self, reg: Reg) -> Option<Reg> {
+        self.get_rename(reg)
+    }
+
+    fn get_mod(&self, reg: Reg) -> Option<Reg> {
+        self.get_rename(reg)
+    }
+}
+
+/// The main entry point for lowering with ISLE.
+pub(crate) fn lower<C>(
+    lower_ctx: &mut C,
+    isa_flags: &x64_settings::Flags,
+    outputs: &[InsnOutput],
+    inst: Inst,
+) -> Result<(), ()>
+where
+    C: LowerCtx<I = MInst>,
+{
+    // TODO: reuse the ISLE context across lowerings so we can reuse its
+    // internal heap allocations.
+    let mut isle_ctx = IsleContext::new(lower_ctx, isa_flags);
+
+    let temp_regs = generated_code::constructor_lower(&mut isle_ctx, inst).ok_or(())?;
+    let mut temp_regs = temp_regs.regs().iter();
+
+    // The ISLE generated code emits its own registers to define the
+    // instruction's lowered values in. We rename those registers to the
+    // registers they were assigned when their value was used as an operand in
+    // earlier lowerings.
+    let mut renamer = RegRenamer::default();
+    for output in outputs {
+        let dsts = get_output_reg(isle_ctx.lower_ctx, *output);
+        for (temp, dst) in temp_regs.by_ref().zip(dsts.regs()) {
+            renamer.add_rename(*temp, dst.to_reg());
+        }
+    }
+
+    for mut inst in isle_ctx.into_emitted_insts() {
+        x64_map_regs(&mut inst, &renamer);
+        lower_ctx.emit(inst);
+    }
+
+    Ok(())
+}
+
+pub struct IsleContext<'a, C> {
+    lower_ctx: &'a mut C,
+    isa_flags: &'a x64_settings::Flags,
+    emitted_insts: SmallVec<[MInst; 6]>,
+}
+
+impl<'a, C> IsleContext<'a, C> {
+    pub fn new(lower_ctx: &'a mut C, isa_flags: &'a x64_settings::Flags) -> Self {
+        IsleContext {
+            lower_ctx,
+            isa_flags,
+            emitted_insts: SmallVec::new(),
+        }
+    }
+
+    pub fn into_emitted_insts(self) -> SmallVec<[MInst; 6]> {
+        self.emitted_insts
+    }
+}
+
+impl<'a, C> generated_code::Context for IsleContext<'a, C>
+where
+    C: LowerCtx<I = MInst>,
+{
+    #[inline]
+    fn unpack_value_array_2(&mut self, arr: &ValueArray2) -> (Value, Value) {
+        let [a, b] = *arr;
+        (a, b)
+    }
+
+    #[inline]
+    fn pack_value_array_2(&mut self, a: Value, b: Value) -> ValueArray2 {
+        [a, b]
+    }
+
+    #[inline]
+    fn unpack_value_array_3(&mut self, arr: &ValueArray3) -> (Value, Value, Value) {
+        let [a, b, c] = *arr;
+        (a, b, c)
+    }
+
+    #[inline]
+    fn pack_value_array_3(&mut self, a: Value, b: Value, c: Value) -> ValueArray3 {
+        [a, b, c]
+    }
+
+    #[inline]
+    fn value_reg(&mut self, reg: Reg) -> ValueRegs {
+        ValueRegs::one(reg)
+    }
+
+    #[inline]
+    fn value_regs(&mut self, r1: Reg, r2: Reg) -> ValueRegs {
+        ValueRegs::two(r1, r2)
+    }
+
+    #[inline]
+    fn temp_writable_reg(&mut self, ty: Type) -> WritableReg {
+        let value_regs = self.lower_ctx.alloc_tmp(ty);
+        value_regs.only_reg().unwrap()
+    }
+
+    #[inline]
+    fn invalid_reg(&mut self) -> Reg {
+        Reg::invalid()
+    }
+
+    #[inline]
+    fn put_in_reg(&mut self, val: Value) -> Reg {
+        self.lower_ctx.put_value_in_regs(val).only_reg().unwrap()
+    }
+
+    #[inline]
+    fn put_in_regs(&mut self, val: Value) -> ValueRegs {
+        self.lower_ctx.put_value_in_regs(val)
+    }
+
+    #[inline]
+    fn value_regs_get(&mut self, regs: ValueRegs, i: usize) -> Reg {
+        regs.regs()[i]
+    }
+
+    #[inline]
+    fn u8_as_u64(&mut self, x: u8) -> u64 {
+        x.into()
+    }
+
+    #[inline]
+    fn u16_as_u64(&mut self, x: u16) -> u64 {
+        x.into()
+    }
+
+    #[inline]
+    fn u32_as_u64(&mut self, x: u32) -> u64 {
+        x.into()
+    }
+
+    #[inline]
+    fn ty_bits(&mut self, ty: Type) -> u16 {
+        ty.bits()
+    }
+
+    #[inline]
+    fn fits_in_64(&mut self, ty: Type) -> Option<Type> {
+        if ty.bits() <= 64 {
+            Some(ty)
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn value_list_slice(&mut self, list: ValueList) -> ValueSlice {
+        list.as_slice(&self.lower_ctx.dfg().value_lists)
+    }
+
+    #[inline]
+    fn unwrap_head_value_list_1(&mut self, list: ValueList) -> (Value, ValueSlice) {
+        match self.value_list_slice(list) {
+            [head, tail @ ..] => (*head, tail),
+            _ => out_of_line_panic("`unwrap_head_value_list_1` on empty `ValueList`"),
+        }
+    }
+
+    #[inline]
+    fn unwrap_head_value_list_2(&mut self, list: ValueList) -> (Value, Value, ValueSlice) {
+        match self.value_list_slice(list) {
+            [head1, head2, tail @ ..] => (*head1, *head2, tail),
+            _ => out_of_line_panic(
+                "`unwrap_head_value_list_2` on list without at least two elements",
+            ),
+        }
+    }
+
+    #[inline]
+    fn writable_reg_to_reg(&mut self, r: WritableReg) -> Reg {
+        r.to_reg()
+    }
+
+    #[inline]
+    fn u64_from_imm64(&mut self, imm: Imm64) -> u64 {
+        imm.bits() as u64
+    }
+
+    #[inline]
+    fn inst_results(&mut self, inst: Inst) -> ValueSlice {
+        self.lower_ctx.dfg().inst_results(inst)
+    }
+
+    #[inline]
+    fn first_result(&mut self, inst: Inst) -> Option<Value> {
+        self.lower_ctx.dfg().inst_results(inst).first().copied()
+    }
+
+    #[inline]
+    fn inst_data(&mut self, inst: Inst) -> InstructionData {
+        self.lower_ctx.dfg()[inst].clone()
+    }
+
+    #[inline]
+    fn value_type(&mut self, val: Value) -> Type {
+        self.lower_ctx.dfg().value_type(val)
+    }
+
+    #[inline]
+    fn multi_lane(&mut self, ty: Type) -> Option<(u8, u16)> {
+        if ty.lane_count() > 1 {
+            Some((ty.lane_bits(), ty.lane_count()))
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn def_inst(&mut self, val: Value) -> Option<Inst> {
+        self.lower_ctx.dfg().value_def(val).inst()
+    }
+
+    #[inline]
+    fn operand_size_of_type(&mut self, ty: Type) -> OperandSize {
+        if ty.bits() == 64 {
+            OperandSize::Size64
+        } else {
+            OperandSize::Size32
+        }
+    }
+
+    fn put_in_reg_mem(&mut self, val: Value) -> RegMem {
+        let inputs = self.lower_ctx.get_value_as_source_or_const(val);
+
+        if let Some(c) = inputs.constant {
+            // Generate constants fresh at each use to minimize long-range
+            // register pressure.
+            let ty = self.value_type(val);
+            return RegMem::reg(generated_code::constructor_imm(self, ty, c).unwrap());
+        }
+
+        if let Some((src_insn, 0)) = inputs.inst {
+            if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, src_insn) {
+                self.lower_ctx.sink_inst(src_insn);
+                let amode = lower_to_amode(self.lower_ctx, addr_input, offset);
+                return RegMem::mem(amode);
+            }
+        }
+
+        RegMem::reg(self.put_in_reg(val))
+    }
+
+    #[inline]
+    fn avx512vl_enabled(&mut self, _: Type) -> Option<()> {
+        if self.isa_flags.use_avx512vl_simd() {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn avx512dq_enabled(&mut self, _: Type) -> Option<()> {
+        if self.isa_flags.use_avx512dq_simd() {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn imm8_from_value(&mut self, val: Value) -> Option<Imm8Reg> {
+        let inst = self.lower_ctx.dfg().value_def(val).inst()?;
+        let constant = self.lower_ctx.get_constant(inst)?;
+        let imm = u8::try_from(constant).ok()?;
+        Some(Imm8Reg::Imm8 { imm })
+    }
+
+    #[inline]
+    fn simm32_from_value(&mut self, val: Value) -> Option<RegMemImm> {
+        let inst = self.lower_ctx.dfg().value_def(val).inst()?;
+        let constant: u64 = self.lower_ctx.get_constant(inst)?;
+        let constant = constant as i64;
+        to_simm32(constant)
+    }
+
+    #[inline]
+    fn simm32_from_imm64(&mut self, imm: Imm64) -> Option<RegMemImm> {
+        to_simm32(imm.bits())
+    }
+
+    fn sinkable_load(&mut self, val: Value) -> Option<SinkableLoad> {
+        let input = self.lower_ctx.get_value_as_source_or_const(val);
+        if let Some((inst, 0)) = input.inst {
+            if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, inst) {
+                return Some(SinkableLoad {
+                    inst,
+                    addr_input,
+                    offset,
+                });
+            }
+        }
+        None
+    }
+
+    fn sink_load(&mut self, load: &SinkableLoad) -> RegMemImm {
+        self.lower_ctx.sink_inst(load.inst);
+        let addr = lower_to_amode(self.lower_ctx, load.addr_input, load.offset);
+        RegMemImm::Mem {
+            addr: SyntheticAmode::Real(addr),
+        }
+    }
+
+    #[inline]
+    fn ext_mode(&mut self, from_bits: u16, to_bits: u16) -> ExtMode {
+        ExtMode::new(from_bits, to_bits).unwrap()
+    }
+
+    fn emit(&mut self, inst: &MInst) -> Unit {
+        for inst in inst.clone().mov_mitosis() {
+            self.emitted_insts.push(inst);
+        }
+    }
+
+    #[inline]
+    fn nonzero_u64_fits_in_u32(&mut self, x: u64) -> Option<u64> {
+        if x != 0 && x < u64::from(u32::MAX) {
+            Some(x)
+        } else {
+            None
+        }
+    }
+}
+
+#[inline]
+fn to_simm32(constant: i64) -> Option<RegMemImm> {
+    if constant == ((constant << 32) >> 32) {
+        Some(RegMemImm::Imm {
+            simm32: constant as u32,
+        })
+    } else {
+        None
+    }
+}
+
+#[inline(never)]
+#[cold]
+#[track_caller]
+fn out_of_line_panic(msg: &str) -> ! {
+    panic!("{}", msg);
+}
diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs
new file mode 100644
index 0000000000..da0932027e
--- /dev/null
+++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs
@@ -0,0 +1,3639 @@
+// GENERATED BY ISLE. DO NOT EDIT!
+//
+// Generated automatically from the instruction-selection DSL code in:
+// - src/clif.isle
+// - src/prelude.isle
+// - src/isa/x64/inst.isle
+// - src/isa/x64/lower.isle
+
+#![allow(dead_code, unreachable_code, unreachable_patterns)]
+#![allow(unused_imports, unused_variables, non_snake_case)]
+#![allow(irrefutable_let_patterns)]
+
+use super::*; // Pulls in all external types.
+
+/// Context during lowering: an implementation of this trait
+/// must be provided with all external constructors and extractors.
+/// A mutable borrow is passed along through all lowering logic.
+pub trait Context {
+    fn unpack_value_array_2(&mut self, arg0: &ValueArray2) -> (Value, Value);
+    fn pack_value_array_2(&mut self, arg0: Value, arg1: Value) -> ValueArray2;
+    fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value);
+    fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3;
+    fn value_reg(&mut self, arg0: Reg) -> ValueRegs;
+    fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs;
+    fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg;
+    fn invalid_reg(&mut self) -> Reg;
+    fn put_in_reg(&mut self, arg0: Value) -> Reg;
+    fn put_in_regs(&mut self, arg0: Value) -> ValueRegs;
+    fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg;
+    fn u8_as_u64(&mut self, arg0: u8) -> u64;
+    fn u16_as_u64(&mut self, arg0: u16) -> u64;
+    fn u32_as_u64(&mut self, arg0: u32) -> u64;
+    fn ty_bits(&mut self, arg0: Type) -> u16;
+    fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
+    fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
+    fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice);
+    fn unwrap_head_value_list_2(&mut self, arg0: ValueList) -> (Value, Value, ValueSlice);
+    fn writable_reg_to_reg(&mut self, arg0: WritableReg) -> Reg;
+    fn u64_from_imm64(&mut self, arg0: Imm64) -> u64;
+    fn inst_results(&mut self, arg0: Inst) -> ValueSlice;
+    fn first_result(&mut self, arg0: Inst) -> Option<Value>;
+    fn inst_data(&mut self, arg0: Inst) -> InstructionData;
+    fn value_type(&mut self, arg0: Value) -> Type;
+    fn multi_lane(&mut self, arg0: Type) -> Option<(u8, u16)>;
+    fn def_inst(&mut self, arg0: Value) -> Option<Inst>;
+    fn operand_size_of_type(&mut self, arg0: Type) -> OperandSize;
+    fn put_in_reg_mem(&mut self, arg0: Value) -> RegMem;
+    fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>;
+    fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>;
+    fn imm8_from_value(&mut self, arg0: Value) -> Option<Imm8Reg>;
+    fn simm32_from_value(&mut self, arg0: Value) -> Option<RegMemImm>;
+    fn simm32_from_imm64(&mut self, arg0: Imm64) -> Option<RegMemImm>;
+    fn sinkable_load(&mut self, arg0: Value) -> Option<SinkableLoad>;
+    fn sink_load(&mut self, arg0: &SinkableLoad) -> RegMemImm;
+    fn ext_mode(&mut self, arg0: u16, arg1: u16) -> ExtMode;
+    fn emit(&mut self, arg0: &MInst) -> Unit;
+    fn nonzero_u64_fits_in_u32(&mut self, arg0: u64) -> Option<u64>;
+}
+
+/// Internal type ProducesFlags: defined at src/isa/x64/inst.isle line 374.
+#[derive(Clone, Debug)]
+pub enum ProducesFlags {
+    ProducesFlags { inst: MInst, result: Reg },
+}
+
+/// Internal type ConsumesFlags: defined at src/isa/x64/inst.isle line 377.
+#[derive(Clone, Debug)]
+pub enum ConsumesFlags {
+    ConsumesFlags { inst: MInst, result: Reg },
+}
+
+/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 415.
+#[derive(Clone, Debug)]
+pub enum ExtendKind {
+    Sign,
+    Zero,
+}
+
+// Generated as internal constructor for term temp_reg.
+pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/prelude.isle line 57.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term lo_reg.
+pub fn constructor_lo_reg<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/prelude.isle line 92.
+    let expr0_0 = C::put_in_regs(ctx, pattern0_0);
+    let expr1_0: usize = 0;
+    let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term operand_size_bits.
+pub fn constructor_operand_size_bits<C: Context>(ctx: &mut C, arg0: &OperandSize) -> Option<u16> {
+    let pattern0_0 = arg0;
+    match pattern0_0 {
+        &OperandSize::Size8 => {
+            // Rule at src/isa/x64/inst.isle line 75.
+            let expr0_0: u16 = 8;
+            return Some(expr0_0);
+        }
+        &OperandSize::Size16 => {
+            // Rule at src/isa/x64/inst.isle line 76.
+            let expr0_0: u16 = 16;
+            return Some(expr0_0);
+        }
+        &OperandSize::Size32 => {
+            // Rule at src/isa/x64/inst.isle line 77.
+            let expr0_0: u16 = 32;
+            return Some(expr0_0);
+        }
+        &OperandSize::Size64 => {
+            // Rule at src/isa/x64/inst.isle line 78.
+            let expr0_0: u16 = 64;
+            return Some(expr0_0);
+        }
+        _ => {}
+    }
+    return None;
+}
+
+// Generated as internal constructor for term with_flags.
+pub fn constructor_with_flags<C: Context>(
+    ctx: &mut C,
+    arg0: &ProducesFlags,
+    arg1: &ConsumesFlags,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    if let &ProducesFlags::ProducesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        let pattern2_0 = arg1;
+        if let &ConsumesFlags::ConsumesFlags {
+            inst: ref pattern3_0,
+            result: pattern3_1,
+        } = pattern2_0
+        {
+            // Rule at src/isa/x64/inst.isle line 387.
+            let expr0_0 = C::emit(ctx, &pattern1_0);
+            let expr1_0 = C::emit(ctx, &pattern3_0);
+            let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
+            return Some(expr2_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term with_flags_1.
+pub fn constructor_with_flags_1<C: Context>(
+    ctx: &mut C,
+    arg0: &ProducesFlags,
+    arg1: &ConsumesFlags,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if let &ProducesFlags::ProducesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        let pattern2_0 = arg1;
+        if let &ConsumesFlags::ConsumesFlags {
+            inst: ref pattern3_0,
+            result: pattern3_1,
+        } = pattern2_0
+        {
+            // Rule at src/isa/x64/inst.isle line 395.
+            let expr0_0 = C::emit(ctx, &pattern1_0);
+            let expr1_0 = C::emit(ctx, &pattern3_0);
+            return Some(pattern3_1);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term with_flags_2.
+pub fn constructor_with_flags_2<C: Context>(
+    ctx: &mut C,
+    arg0: &ProducesFlags,
+    arg1: &ConsumesFlags,
+    arg2: &ConsumesFlags,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    if let &ProducesFlags::ProducesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        let pattern2_0 = arg1;
+        if let &ConsumesFlags::ConsumesFlags {
+            inst: ref pattern3_0,
+            result: pattern3_1,
+        } = pattern2_0
+        {
+            let pattern4_0 = arg2;
+            if let &ConsumesFlags::ConsumesFlags {
+                inst: ref pattern5_0,
+                result: pattern5_1,
+            } = pattern4_0
+            {
+                // Rule at src/isa/x64/inst.isle line 405.
+                let expr0_0 = C::emit(ctx, &pattern1_0);
+                let expr1_0 = C::emit(ctx, &pattern3_0);
+                let expr2_0 = C::emit(ctx, &pattern5_0);
+                let expr3_0 = C::value_regs(ctx, pattern3_1, pattern5_1);
+                return Some(expr3_0);
+            }
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term extend_to_reg.
+pub fn constructor_extend_to_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Value,
+    arg1: Type,
+    arg2: &ExtendKind,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::value_type(ctx, pattern0_0);
+    let pattern2_0 = arg1;
+    if pattern2_0 == pattern1_0 {
+        let pattern4_0 = arg2;
+        // Rule at src/isa/x64/inst.isle line 427.
+        let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+        return Some(expr0_0);
+    }
+    let pattern3_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 430.
+    let expr0_0 = C::ty_bits(ctx, pattern1_0);
+    let expr1_0 = C::operand_size_of_type(ctx, pattern2_0);
+    let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?;
+    let expr3_0 = C::ext_mode(ctx, expr0_0, expr2_0);
+    let expr4_0 = C::put_in_reg_mem(ctx, pattern0_0);
+    let expr5_0 = constructor_extend(ctx, pattern3_0, pattern2_0, &expr3_0, &expr4_0)?;
+    return Some(expr5_0);
+}
+
+// Generated as internal constructor for term extend.
+pub fn constructor_extend<C: Context>(
+    ctx: &mut C,
+    arg0: &ExtendKind,
+    arg1: Type,
+    arg2: &ExtMode,
+    arg3: &RegMem,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    match pattern0_0 {
+        &ExtendKind::Sign => {
+            let pattern2_0 = arg1;
+            let pattern3_0 = arg2;
+            let pattern4_0 = arg3;
+            // Rule at src/isa/x64/inst.isle line 450.
+            let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?;
+            return Some(expr0_0);
+        }
+        &ExtendKind::Zero => {
+            let pattern2_0 = arg1;
+            let pattern3_0 = arg2;
+            let pattern4_0 = arg3;
+            // Rule at src/isa/x64/inst.isle line 446.
+            let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?;
+            return Some(expr0_0);
+        }
+        _ => {}
+    }
+    return None;
+}
+
+// Generated as internal constructor for term alu_rmi_r.
+pub fn constructor_alu_rmi_r<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &AluRmiROpcode,
+    arg2: Reg,
+    arg3: &RegMemImm,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/x64/inst.isle line 468.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
+    let expr2_0 = MInst::AluRmiR {
+        size: expr1_0,
+        op: pattern1_0.clone(),
+        src1: pattern2_0,
+        src2: pattern3_0.clone(),
+        dst: expr0_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term add.
+pub fn constructor_add<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 476.
+    let expr0_0 = AluRmiROpcode::Add;
+    let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_with_flags.
+pub fn constructor_add_with_flags<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 484.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
+    let expr2_0 = AluRmiROpcode::Add;
+    let expr3_0 = MInst::AluRmiR {
+        size: expr1_0,
+        op: expr2_0,
+        src1: pattern1_0,
+        src2: pattern2_0.clone(),
+        dst: expr0_0,
+    };
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    let expr5_0 = ProducesFlags::ProducesFlags {
+        inst: expr3_0,
+        result: expr4_0,
+    };
+    return Some(expr5_0);
+}
+
+// Generated as internal constructor for term adc.
+pub fn constructor_adc<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<ConsumesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 495.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
+    let expr2_0 = AluRmiROpcode::Adc;
+    let expr3_0 = MInst::AluRmiR {
+        size: expr1_0,
+        op: expr2_0,
+        src1: pattern1_0,
+        src2: pattern2_0.clone(),
+        dst: expr0_0,
+    };
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    let expr5_0 = ConsumesFlags::ConsumesFlags {
+        inst: expr3_0,
+        result: expr4_0,
+    };
+    return Some(expr5_0);
+}
+
+// Generated as internal constructor for term sub.
+pub fn constructor_sub<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 506.
+    let expr0_0 = AluRmiROpcode::Sub;
+    let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sub_with_flags.
+pub fn constructor_sub_with_flags<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 514.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
+    let expr2_0 = AluRmiROpcode::Sub;
+    let expr3_0 = MInst::AluRmiR {
+        size: expr1_0,
+        op: expr2_0,
+        src1: pattern1_0,
+        src2: pattern2_0.clone(),
+        dst: expr0_0,
+    };
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    let expr5_0 = ProducesFlags::ProducesFlags {
+        inst: expr3_0,
+        result: expr4_0,
+    };
+    return Some(expr5_0);
+}
+
+// Generated as internal constructor for term sbb.
+pub fn constructor_sbb<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<ConsumesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 525.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
+    let expr2_0 = AluRmiROpcode::Sbb;
+    let expr3_0 = MInst::AluRmiR {
+        size: expr1_0,
+        op: expr2_0,
+        src1: pattern1_0,
+        src2: pattern2_0.clone(),
+        dst: expr0_0,
+    };
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    let expr5_0 = ConsumesFlags::ConsumesFlags {
+        inst: expr3_0,
+        result: expr4_0,
+    };
+    return Some(expr5_0);
+}
+
+// Generated as internal constructor for term mul.
+pub fn constructor_mul<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 536.
+    let expr0_0 = AluRmiROpcode::Mul;
+    let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term m_and.
+pub fn constructor_m_and<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 547.
+    let expr0_0 = AluRmiROpcode::And;
+    let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term or.
+pub fn constructor_or<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 555.
+    let expr0_0 = AluRmiROpcode::Or;
+    let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term xor.
+pub fn constructor_xor<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 563.
+    let expr0_0 = AluRmiROpcode::Xor;
+    let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term imm.
+pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) {
+            // Rule at src/isa/x64/inst.isle line 582.
+            let expr0_0: Type = I64;
+            let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+            let expr2_0 = OperandSize::Size32;
+            let expr3_0 = MInst::Imm {
+                dst_size: expr2_0,
+                simm64: pattern3_0,
+                dst: expr1_0,
+            };
+            let expr4_0 = C::emit(ctx, &expr3_0);
+            let expr5_0 = C::writable_reg_to_reg(ctx, expr1_0);
+            return Some(expr5_0);
+        }
+    }
+    let pattern1_0 = arg1;
+    if pattern1_0 == 0 {
+        // Rule at src/isa/x64/inst.isle line 588.
+        let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+        let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
+        let expr2_0 = C::operand_size_of_type(ctx, pattern0_0);
+        let expr3_0 = AluRmiROpcode::Xor;
+        let expr4_0 = RegMemImm::Reg { reg: expr1_0 };
+        let expr5_0 = MInst::AluRmiR {
+            size: expr2_0,
+            op: expr3_0,
+            src1: expr1_0,
+            src2: expr4_0,
+            dst: expr0_0,
+        };
+        let expr6_0 = C::emit(ctx, &expr5_0);
+        return Some(expr1_0);
+    }
+    // Rule at src/isa/x64/inst.isle line 571.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
+    let expr2_0 = MInst::Imm {
+        dst_size: expr1_0,
+        simm64: pattern1_0,
+        dst: expr0_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term shift_r.
+pub fn constructor_shift_r<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ShiftKind,
+    arg2: Reg,
+    arg3: &Imm8Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/x64/inst.isle line 601.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
+    let expr2_0 = MInst::ShiftR {
+        size: expr1_0,
+        kind: pattern1_0.clone(),
+        src: pattern2_0,
+        num_bits: pattern3_0.clone(),
+        dst: expr0_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term m_rotl.
+pub fn constructor_m_rotl<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &Imm8Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 610.
+    let expr0_0 = ShiftKind::RotateLeft;
+    let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term shl.
+pub fn constructor_shl<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &Imm8Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 615.
+    let expr0_0 = ShiftKind::ShiftLeft;
+    let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term shr.
+pub fn constructor_shr<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &Imm8Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 620.
+    let expr0_0 = ShiftKind::ShiftRightLogical;
+    let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sar.
+pub fn constructor_sar<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &Imm8Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 625.
+    let expr0_0 = ShiftKind::ShiftRightArithmetic;
+    let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term cmp_rmi_r.
+pub fn constructor_cmp_rmi_r<C: Context>(
+    ctx: &mut C,
+    arg0: &OperandSize,
+    arg1: &CmpOpcode,
+    arg2: &RegMemImm,
+    arg3: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/x64/inst.isle line 630.
+    let expr0_0 = MInst::CmpRmiR {
+        size: pattern0_0.clone(),
+        opcode: pattern1_0.clone(),
+        src: pattern2_0.clone(),
+        dst: pattern3_0,
+    };
+    let expr1_0 = C::invalid_reg(ctx);
+    let expr2_0 = ProducesFlags::ProducesFlags {
+        inst: expr0_0,
+        result: expr1_0,
+    };
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term cmp.
+pub fn constructor_cmp<C: Context>(
+    ctx: &mut C,
+    arg0: &OperandSize,
+    arg1: &RegMemImm,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 639.
+    let expr0_0 = CmpOpcode::Cmp;
+    let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term test.
+pub fn constructor_test<C: Context>(
+    ctx: &mut C,
+    arg0: &OperandSize,
+    arg1: &RegMemImm,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 644.
+    let expr0_0 = CmpOpcode::Test;
+    let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term cmove.
+pub fn constructor_cmove<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &CC,
+    arg2: &RegMem,
+    arg3: Reg,
+) -> Option<ConsumesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/x64/inst.isle line 649.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::operand_size_of_type(ctx, pattern0_0);
+    let expr2_0 = MInst::Cmove {
+        size: expr1_0,
+        cc: pattern1_0.clone(),
+        consequent: pattern2_0.clone(),
+        alternative: pattern3_0,
+        dst: expr0_0,
+    };
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    let expr4_0 = ConsumesFlags::ConsumesFlags {
+        inst: expr2_0,
+        result: expr3_0,
+    };
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term movzx.
+pub fn constructor_movzx<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ExtMode,
+    arg2: &RegMem,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 657.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::MovzxRmR {
+        ext_mode: pattern1_0.clone(),
+        src: pattern2_0.clone(),
+        dst: expr0_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term movsx.
+pub fn constructor_movsx<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ExtMode,
+    arg2: &RegMem,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 664.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::MovsxRmR {
+        ext_mode: pattern1_0.clone(),
+        src: pattern2_0.clone(),
+        dst: expr0_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term xmm_rm_r.
+pub fn constructor_xmm_rm_r<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &SseOpcode,
+    arg2: Reg,
+    arg3: &RegMem,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/x64/inst.isle line 671.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::XmmRmR {
+        op: pattern1_0.clone(),
+        src1: pattern2_0,
+        src2: pattern3_0.clone(),
+        dst: expr0_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term paddb.
+pub fn constructor_paddb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 678.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = SseOpcode::Paddb;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term paddw.
+pub fn constructor_paddw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 683.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Paddw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term paddd.
+pub fn constructor_paddd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 688.
+    let expr0_0: Type = I32X4;
+    let expr1_0 = SseOpcode::Paddd;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term paddq.
+pub fn constructor_paddq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 693.
+    let expr0_0: Type = I64X2;
+    let expr1_0 = SseOpcode::Paddq;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term paddsb.
+pub fn constructor_paddsb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 698.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = SseOpcode::Paddsb;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term paddsw.
+pub fn constructor_paddsw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 703.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Paddsw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term paddusb.
+pub fn constructor_paddusb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 708.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = SseOpcode::Paddusb;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term paddusw.
+pub fn constructor_paddusw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 713.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Paddusw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term psubb.
+pub fn constructor_psubb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 718.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = SseOpcode::Psubb;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term psubw.
+pub fn constructor_psubw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 723.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Psubw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term psubd.
+pub fn constructor_psubd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 728.
+    let expr0_0: Type = I32X4;
+    let expr1_0 = SseOpcode::Psubd;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term psubq.
+pub fn constructor_psubq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 733.
+    let expr0_0: Type = I64X2;
+    let expr1_0 = SseOpcode::Psubq;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term psubsb.
+pub fn constructor_psubsb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 738.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = SseOpcode::Psubsb;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term psubsw.
+pub fn constructor_psubsw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 743.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Psubsw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term psubusb.
+pub fn constructor_psubusb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 748.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = SseOpcode::Psubusb;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term psubusw.
+pub fn constructor_psubusw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 753.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Psubusw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pavgb.
+pub fn constructor_pavgb<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 758.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = SseOpcode::Pavgb;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pavgw.
+pub fn constructor_pavgw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 763.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Pavgw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pand.
+pub fn constructor_pand<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 768.
+    let expr0_0: Type = F32X4;
+    let expr1_0 = SseOpcode::Pand;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term andps.
+pub fn constructor_andps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 773.
+    let expr0_0: Type = F32X4;
+    let expr1_0 = SseOpcode::Andps;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term andpd.
+pub fn constructor_andpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 778.
+    let expr0_0: Type = F64X2;
+    let expr1_0 = SseOpcode::Andpd;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term por.
+pub fn constructor_por<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 783.
+    let expr0_0: Type = F32X4;
+    let expr1_0 = SseOpcode::Por;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term orps.
+pub fn constructor_orps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 788.
+    let expr0_0: Type = F32X4;
+    let expr1_0 = SseOpcode::Orps;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term orpd.
+pub fn constructor_orpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 793.
+    let expr0_0: Type = F64X2;
+    let expr1_0 = SseOpcode::Orpd;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pxor.
+pub fn constructor_pxor<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 798.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = SseOpcode::Pxor;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term xorps.
+pub fn constructor_xorps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 803.
+    let expr0_0: Type = F32X4;
+    let expr1_0 = SseOpcode::Xorps;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term xorpd.
+pub fn constructor_xorpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 808.
+    let expr0_0: Type = F64X2;
+    let expr1_0 = SseOpcode::Xorpd;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pmullw.
+pub fn constructor_pmullw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 813.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Pmullw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pmulld.
+pub fn constructor_pmulld<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 818.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Pmulld;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pmulhw.
+pub fn constructor_pmulhw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 823.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Pmulhw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pmulhuw.
+pub fn constructor_pmulhuw<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 828.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Pmulhuw;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pmuldq.
+pub fn constructor_pmuldq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 833.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Pmuldq;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pmuludq.
+pub fn constructor_pmuludq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 838.
+    let expr0_0: Type = I64X2;
+    let expr1_0 = SseOpcode::Pmuludq;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term punpckhwd.
+pub fn constructor_punpckhwd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 843.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Punpckhwd;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term punpcklwd.
+pub fn constructor_punpcklwd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 848.
+    let expr0_0: Type = I16X8;
+    let expr1_0 = SseOpcode::Punpcklwd;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term andnps.
+pub fn constructor_andnps<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 853.
+    let expr0_0: Type = F32X4;
+    let expr1_0 = SseOpcode::Andnps;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term andnpd.
+pub fn constructor_andnpd<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 858.
+    let expr0_0: Type = F64X2;
+    let expr1_0 = SseOpcode::Andnpd;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term pandn.
+pub fn constructor_pandn<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 863.
+    let expr0_0: Type = F64X2;
+    let expr1_0 = SseOpcode::Pandn;
+    let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term xmm_rm_r_imm.
+pub fn constructor_xmm_rm_r_imm<C: Context>(
+    ctx: &mut C,
+    arg0: &SseOpcode,
+    arg1: Reg,
+    arg2: &RegMem,
+    arg3: u8,
+    arg4: &OperandSize,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    let pattern4_0 = arg4;
+    // Rule at src/isa/x64/inst.isle line 868.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::XmmRmRImm {
+        op: pattern0_0.clone(),
+        src1: pattern1_0,
+        src2: pattern2_0.clone(),
+        dst: expr1_0,
+        imm: pattern3_0,
+        size: pattern4_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term palignr.
+pub fn constructor_palignr<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &RegMem,
+    arg2: u8,
+    arg3: &OperandSize,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/x64/inst.isle line 880.
+    let expr0_0 = SseOpcode::Palignr;
+    let expr1_0 = constructor_xmm_rm_r_imm(
+        ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0,
+    )?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term pshufd.
+pub fn constructor_pshufd<C: Context>(
+    ctx: &mut C,
+    arg0: &RegMem,
+    arg1: u8,
+    arg2: &OperandSize,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 889.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    let expr3_0 = SseOpcode::Pshufd;
+    let expr4_0 = MInst::XmmRmRImm {
+        op: expr3_0,
+        src1: expr2_0,
+        src2: pattern0_0.clone(),
+        dst: expr1_0,
+        imm: pattern1_0,
+        size: pattern2_0.clone(),
+    };
+    let expr5_0 = C::emit(ctx, &expr4_0);
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term xmm_unary_rm_r.
+pub fn constructor_xmm_unary_rm_r<C: Context>(
+    ctx: &mut C,
+    arg0: &SseOpcode,
+    arg1: &RegMem,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 902.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::XmmUnaryRmR {
+        op: pattern0_0.clone(),
+        src: pattern1_0.clone(),
+        dst: expr1_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term pmovsxbw.
+pub fn constructor_pmovsxbw<C: Context>(ctx: &mut C, arg0: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/x64/inst.isle line 909.
+    let expr0_0 = SseOpcode::Pmovsxbw;
+    let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term pmovzxbw.
+pub fn constructor_pmovzxbw<C: Context>(ctx: &mut C, arg0: &RegMem) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/x64/inst.isle line 914.
+    let expr0_0 = SseOpcode::Pmovzxbw;
+    let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term xmm_rm_r_evex.
+pub fn constructor_xmm_rm_r_evex<C: Context>(
+    ctx: &mut C,
+    arg0: &Avx512Opcode,
+    arg1: &RegMem,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 919.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::XmmRmREvex {
+        op: pattern0_0.clone(),
+        src1: pattern1_0.clone(),
+        src2: pattern2_0,
+        dst: expr1_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term vpmullq.
+pub fn constructor_vpmullq<C: Context>(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 931.
+    let expr0_0 = Avx512Opcode::Vpmullq;
+    let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term xmm_rmi_reg.
+pub fn constructor_xmm_rmi_reg<C: Context>(
+    ctx: &mut C,
+    arg0: &SseOpcode,
+    arg1: Reg,
+    arg2: &RegMemImm,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 938.
+    let expr0_0: Type = I8X16;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::XmmRmiReg {
+        opcode: pattern0_0.clone(),
+        src1: pattern1_0,
+        src2: pattern2_0.clone(),
+        dst: expr1_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term psllq.
+pub fn constructor_psllq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 948.
+    let expr0_0 = SseOpcode::Psllq;
+    let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term psrlq.
+pub fn constructor_psrlq<C: Context>(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/inst.isle line 953.
+    let expr0_0 = SseOpcode::Psrlq;
+    let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term mul_hi.
+pub fn constructor_mul_hi<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: bool,
+    arg2: Reg,
+    arg3: &RegMem,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/x64/inst.isle line 960.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr2_0 = C::operand_size_of_type(ctx, pattern0_0);
+    let expr3_0 = MInst::MulHi {
+        size: expr2_0,
+        signed: pattern1_0,
+        src1: pattern2_0,
+        src2: pattern3_0.clone(),
+        dst_lo: expr0_0,
+        dst_hi: expr1_0,
+    };
+    let expr4_0 = C::emit(ctx, &expr3_0);
+    let expr5_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    let expr6_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    let expr7_0 = C::value_regs(ctx, expr5_0, expr6_0);
+    return Some(expr7_0);
+}
+
+// Generated as internal constructor for term mulhi_u.
+pub fn constructor_mulhi_u<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &RegMem,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/x64/inst.isle line 976.
+    let expr0_0: bool = false;
+    let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term lower.
+pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::first_result(ctx, pattern0_0) {
+        let pattern2_0 = C::value_type(ctx, pattern1_0);
+        if pattern2_0 == B128 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::UnaryBool {
+                    opcode: ref pattern5_0,
+                    imm: pattern5_1,
+                } => {
+                    if let &Opcode::Bconst = &pattern5_0 {
+                        if pattern5_1 == true {
+                            // Rule at src/isa/x64/lower.isle line 39.
+                            let expr0_0: Type = B64;
+                            let expr1_0: u64 = 1;
+                            let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?;
+                            let expr3_0: Type = B64;
+                            let expr4_0: u64 = 0;
+                            let expr5_0 = constructor_imm(ctx, expr3_0, expr4_0)?;
+                            let expr6_0 = C::value_regs(ctx, expr2_0, expr5_0);
+                            return Some(expr6_0);
+                        }
+                        if pattern5_1 == false {
+                            // Rule at src/isa/x64/lower.isle line 34.
+                            let expr0_0: Type = B64;
+                            let expr1_0: u64 = 0;
+                            let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?;
+                            let expr3_0: Type = B64;
+                            let expr4_0: u64 = 0;
+                            let expr5_0 = constructor_imm(ctx, expr3_0, expr4_0)?;
+                            let expr6_0 = C::value_regs(ctx, expr2_0, expr5_0);
+                            return Some(expr6_0);
+                        }
+                    }
+                }
+                &InstructionData::Binary {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Band => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 358.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                            let expr3_0: usize = 1;
+                            let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                            let expr5_0 = constructor_lo_reg(ctx, pattern7_1)?;
+                            let expr6_0: Type = I64;
+                            let expr7_0 = RegMemImm::Reg { reg: expr5_0 };
+                            let expr8_0 = constructor_m_and(ctx, expr6_0, expr2_0, &expr7_0)?;
+                            let expr9_0 = C::value_regs(ctx, expr8_0, expr4_0);
+                            return Some(expr9_0);
+                        }
+                        &Opcode::Bor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 436.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                            let expr3_0: usize = 1;
+                            let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                            let expr5_0 = constructor_lo_reg(ctx, pattern7_1)?;
+                            let expr6_0: Type = I64;
+                            let expr7_0 = RegMemImm::Reg { reg: expr5_0 };
+                            let expr8_0 = constructor_or(ctx, expr6_0, expr2_0, &expr7_0)?;
+                            let expr9_0 = C::value_regs(ctx, expr8_0, expr4_0);
+                            return Some(expr9_0);
+                        }
+                        &Opcode::Bxor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 512.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                            let expr3_0: usize = 1;
+                            let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                            let expr5_0 = constructor_lo_reg(ctx, pattern7_1)?;
+                            let expr6_0: Type = I64;
+                            let expr7_0 = RegMemImm::Reg { reg: expr5_0 };
+                            let expr8_0 = constructor_xor(ctx, expr6_0, expr2_0, &expr7_0)?;
+                            let expr9_0 = C::value_regs(ctx, expr8_0, expr4_0);
+                            return Some(expr9_0);
+                        }
+                        _ => {}
+                    }
+                }
+                _ => {}
+            }
+        }
+        if pattern2_0 == I128 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::UnaryImm {
+                    opcode: ref pattern5_0,
+                    imm: pattern5_1,
+                } => {
+                    if let &Opcode::Iconst = &pattern5_0 {
+                        let pattern7_0 = C::u64_from_imm64(ctx, pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 15.
+                        let expr0_0: Type = I64;
+                        let expr1_0 = constructor_imm(ctx, expr0_0, pattern7_0)?;
+                        let expr2_0: Type = I64;
+                        let expr3_0: u64 = 0;
+                        let expr4_0 = constructor_imm(ctx, expr2_0, expr3_0)?;
+                        let expr5_0 = C::value_regs(ctx, expr1_0, expr4_0);
+                        return Some(expr5_0);
+                    }
+                }
+                &InstructionData::Binary {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Iadd => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 107.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                            let expr3_0: usize = 1;
+                            let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                            let expr5_0 = C::put_in_regs(ctx, pattern7_1);
+                            let expr6_0: usize = 0;
+                            let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0);
+                            let expr8_0: usize = 1;
+                            let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0);
+                            let expr10_0: Type = I64;
+                            let expr11_0 = RegMemImm::Reg { reg: expr7_0 };
+                            let expr12_0 =
+                                constructor_add_with_flags(ctx, expr10_0, expr2_0, &expr11_0)?;
+                            let expr13_0: Type = I64;
+                            let expr14_0 = RegMemImm::Reg { reg: expr9_0 };
+                            let expr15_0 = constructor_adc(ctx, expr13_0, expr4_0, &expr14_0)?;
+                            let expr16_0 = constructor_with_flags(ctx, &expr12_0, &expr15_0)?;
+                            return Some(expr16_0);
+                        }
+                        &Opcode::Isub => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 256.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                            let expr3_0: usize = 1;
+                            let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                            let expr5_0 = C::put_in_regs(ctx, pattern7_1);
+                            let expr6_0: usize = 0;
+                            let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0);
+                            let expr8_0: usize = 1;
+                            let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0);
+                            let expr10_0: Type = I64;
+                            let expr11_0 = RegMemImm::Reg { reg: expr7_0 };
+                            let expr12_0 =
+                                constructor_sub_with_flags(ctx, expr10_0, expr2_0, &expr11_0)?;
+                            let expr13_0: Type = I64;
+                            let expr14_0 = RegMemImm::Reg { reg: expr9_0 };
+                            let expr15_0 = constructor_sbb(ctx, expr13_0, expr4_0, &expr14_0)?;
+                            let expr16_0 = constructor_with_flags(ctx, &expr12_0, &expr15_0)?;
+                            return Some(expr16_0);
+                        }
+                        &Opcode::Imul => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 696.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                            let expr3_0: usize = 1;
+                            let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                            let expr5_0 = C::put_in_regs(ctx, pattern7_1);
+                            let expr6_0: usize = 0;
+                            let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0);
+                            let expr8_0: usize = 1;
+                            let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0);
+                            let expr10_0: Type = I64;
+                            let expr11_0 = RegMemImm::Reg { reg: expr9_0 };
+                            let expr12_0 = constructor_mul(ctx, expr10_0, expr2_0, &expr11_0)?;
+                            let expr13_0: Type = I64;
+                            let expr14_0 = RegMemImm::Reg { reg: expr7_0 };
+                            let expr15_0 = constructor_mul(ctx, expr13_0, expr4_0, &expr14_0)?;
+                            let expr16_0: Type = I64;
+                            let expr17_0 = RegMemImm::Reg { reg: expr15_0 };
+                            let expr18_0 = constructor_add(ctx, expr16_0, expr12_0, &expr17_0)?;
+                            let expr19_0: Type = I64;
+                            let expr20_0 = RegMem::Reg { reg: expr7_0 };
+                            let expr21_0 = constructor_mulhi_u(ctx, expr19_0, expr2_0, &expr20_0)?;
+                            let expr22_0: usize = 0;
+                            let expr23_0 = C::value_regs_get(ctx, expr21_0, expr22_0);
+                            let expr24_0: usize = 1;
+                            let expr25_0 = C::value_regs_get(ctx, expr21_0, expr24_0);
+                            let expr26_0: Type = I64;
+                            let expr27_0 = RegMemImm::Reg { reg: expr25_0 };
+                            let expr28_0 = constructor_add(ctx, expr26_0, expr18_0, &expr27_0)?;
+                            let expr29_0 = C::value_regs(ctx, expr23_0, expr28_0);
+                            return Some(expr29_0);
+                        }
+                        &Opcode::Band => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 348.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                            let expr3_0: usize = 1;
+                            let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                            let expr5_0 = C::put_in_regs(ctx, pattern7_1);
+                            let expr6_0: usize = 0;
+                            let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0);
+                            let expr8_0: usize = 1;
+                            let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0);
+                            let expr10_0: Type = I64;
+                            let expr11_0 = RegMemImm::Reg { reg: expr7_0 };
+                            let expr12_0 = constructor_m_and(ctx, expr10_0, expr2_0, &expr11_0)?;
+                            let expr13_0: Type = I64;
+                            let expr14_0 = RegMemImm::Reg { reg: expr9_0 };
+                            let expr15_0 = constructor_m_and(ctx, expr13_0, expr4_0, &expr14_0)?;
+                            let expr16_0 = C::value_regs(ctx, expr12_0, expr15_0);
+                            return Some(expr16_0);
+                        }
+                        &Opcode::Bor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 433.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_regs(ctx, pattern7_1);
+                            let expr2_0 = constructor_or_i128(ctx, expr0_0, expr1_0)?;
+                            return Some(expr2_0);
+                        }
+                        &Opcode::Bxor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 502.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0: usize = 0;
+                            let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                            let expr3_0: usize = 1;
+                            let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                            let expr5_0 = C::put_in_regs(ctx, pattern7_1);
+                            let expr6_0: usize = 0;
+                            let expr7_0 = C::value_regs_get(ctx, expr5_0, expr6_0);
+                            let expr8_0: usize = 1;
+                            let expr9_0 = C::value_regs_get(ctx, expr5_0, expr8_0);
+                            let expr10_0: Type = I64;
+                            let expr11_0 = RegMemImm::Reg { reg: expr7_0 };
+                            let expr12_0 = constructor_xor(ctx, expr10_0, expr2_0, &expr11_0)?;
+                            let expr13_0: Type = I64;
+                            let expr14_0 = RegMemImm::Reg { reg: expr9_0 };
+                            let expr15_0 = constructor_xor(ctx, expr13_0, expr4_0, &expr14_0)?;
+                            let expr16_0 = C::value_regs(ctx, expr12_0, expr15_0);
+                            return Some(expr16_0);
+                        }
+                        &Opcode::Rotl => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 629.
+                            let expr0_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?;
+                            let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?;
+                            let expr3_0: Type = I64;
+                            let expr4_0: Type = I64;
+                            let expr5_0: u64 = 128;
+                            let expr6_0 = constructor_imm(ctx, expr4_0, expr5_0)?;
+                            let expr7_0 = RegMemImm::Reg { reg: expr1_0 };
+                            let expr8_0 = constructor_sub(ctx, expr3_0, expr6_0, &expr7_0)?;
+                            let expr9_0 = constructor_shr_i128(ctx, expr0_0, expr8_0)?;
+                            let expr10_0 = constructor_or_i128(ctx, expr2_0, expr9_0)?;
+                            return Some(expr10_0);
+                        }
+                        &Opcode::Ishl => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 562.
+                            let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?;
+                            let expr1_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr2_0 = constructor_shl_i128(ctx, expr1_0, expr0_0)?;
+                            return Some(expr2_0);
+                        }
+                        &Opcode::Ushr => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/x64/lower.isle line 608.
+                            let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?;
+                            let expr1_0 = C::put_in_regs(ctx, pattern7_0);
+                            let expr2_0 = constructor_shr_i128(ctx, expr1_0, expr0_0)?;
+                            return Some(expr2_0);
+                        }
+                        _ => {}
+                    }
+                }
+                &InstructionData::BinaryImm64 {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    imm: pattern5_2,
+                } => {
+                    if let &Opcode::IaddImm = &pattern5_0 {
+                        let pattern7_0 = C::u64_from_imm64(ctx, pattern5_2);
+                        // Rule at src/isa/x64/lower.isle line 202.
+                        let expr0_0 = C::put_in_regs(ctx, pattern5_1);
+                        let expr1_0: usize = 0;
+                        let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+                        let expr3_0: usize = 1;
+                        let expr4_0 = C::value_regs_get(ctx, expr0_0, expr3_0);
+                        let expr5_0: Type = I64;
+                        let expr6_0 = constructor_imm(ctx, expr5_0, pattern7_0)?;
+                        let expr7_0: Type = I64;
+                        let expr8_0 = RegMemImm::Reg { reg: expr6_0 };
+                        let expr9_0 = constructor_add_with_flags(ctx, expr7_0, expr2_0, &expr8_0)?;
+                        let expr10_0: Type = I64;
+                        let expr11_0: u32 = 0;
+                        let expr12_0 = RegMemImm::Imm { simm32: expr11_0 };
+                        let expr13_0 = constructor_adc(ctx, expr10_0, expr4_0, &expr12_0)?;
+                        let expr14_0 = constructor_with_flags(ctx, &expr9_0, &expr13_0)?;
+                        return Some(expr14_0);
+                    }
+                }
+                _ => {}
+            }
+        }
+        if pattern2_0 == F32X4 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::Binary {
+                opcode: ref pattern5_0,
+                args: ref pattern5_1,
+            } = &pattern4_0
+            {
+                match &pattern5_0 {
+                    &Opcode::Band => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 333.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
+                        let expr2_0 = constructor_andps(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Bor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 409.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
+                        let expr2_0 = constructor_orps(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Bxor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 487.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
+                        let expr2_0 = constructor_xorps(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::BandNot => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 940.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0);
+                        let expr2_0 = constructor_andnps(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    _ => {}
+                }
+            }
+        }
+        if pattern2_0 == F64X2 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::Binary {
+                opcode: ref pattern5_0,
+                args: ref pattern5_1,
+            } = &pattern4_0
+            {
+                match &pattern5_0 {
+                    &Opcode::Band => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 337.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
+                        let expr2_0 = constructor_andpd(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Bor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 413.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
+                        let expr2_0 = constructor_orpd(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Bxor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 491.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
+                        let expr2_0 = constructor_xorpd(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::BandNot => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 943.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0);
+                        let expr2_0 = constructor_andnpd(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    _ => {}
+                }
+            }
+        }
+        let pattern3_0 = C::inst_data(ctx, pattern0_0);
+        if let &InstructionData::NullAry {
+            opcode: ref pattern4_0,
+        } = &pattern3_0
+        {
+            if let &Opcode::Null = &pattern4_0 {
+                // Rule at src/isa/x64/lower.isle line 46.
+                let expr0_0: u64 = 0;
+                let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?;
+                let expr2_0 = C::value_reg(ctx, expr1_0);
+                return Some(expr2_0);
+            }
+        }
+        if let Some(()) = C::avx512vl_enabled(ctx, pattern2_0) {
+            if let Some(()) = C::avx512dq_enabled(ctx, pattern2_0) {
+                if let Some((pattern5_0, pattern5_1)) = C::multi_lane(ctx, pattern2_0) {
+                    if pattern5_0 == 64 {
+                        if pattern5_1 == 2 {
+                            let pattern8_0 = C::inst_data(ctx, pattern0_0);
+                            if let &InstructionData::Binary {
+                                opcode: ref pattern9_0,
+                                args: ref pattern9_1,
+                            } = &pattern8_0
+                            {
+                                if let &Opcode::Imul = &pattern9_0 {
+                                    let (pattern11_0, pattern11_1) =
+                                        C::unpack_value_array_2(ctx, &pattern9_1);
+                                    // Rule at src/isa/x64/lower.isle line 731.
+                                    let expr0_0 = C::put_in_reg_mem(ctx, pattern11_0);
+                                    let expr1_0 = C::put_in_reg(ctx, pattern11_1);
+                                    let expr2_0 = constructor_vpmullq(ctx, &expr0_0, expr1_0)?;
+                                    let expr3_0 = C::value_reg(ctx, expr2_0);
+                                    return Some(expr3_0);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        if let Some((pattern3_0, pattern3_1)) = C::multi_lane(ctx, pattern2_0) {
+            if pattern3_0 == 8 {
+                if pattern3_1 == 16 {
+                    let pattern6_0 = C::inst_data(ctx, pattern0_0);
+                    if let &InstructionData::Binary {
+                        opcode: ref pattern7_0,
+                        args: ref pattern7_1,
+                    } = &pattern6_0
+                    {
+                        match &pattern7_0 {
+                            &Opcode::AvgRound => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 639.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::UaddSat => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 134.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_paddusb(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::SaddSat => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 122.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_paddsb(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::UsubSat => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 283.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_psubusb(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::SsubSat => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 271.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_psubsb(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::Iadd => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 86.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_paddb(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::Isub => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 235.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_psubb(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+            }
+            if pattern3_0 == 16 {
+                if pattern3_1 == 8 {
+                    let pattern6_0 = C::inst_data(ctx, pattern0_0);
+                    if let &InstructionData::Binary {
+                        opcode: ref pattern7_0,
+                        args: ref pattern7_1,
+                    } = &pattern6_0
+                    {
+                        match &pattern7_0 {
+                            &Opcode::AvgRound => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 643.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::UaddSat => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 139.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_paddusw(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::SaddSat => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 127.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_paddsw(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::UsubSat => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 288.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_psubusw(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::SsubSat => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 276.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_psubsw(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::Iadd => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 91.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_paddw(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::Isub => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 240.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_psubw(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::Imul => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                if let Some(pattern10_0) = C::def_inst(ctx, pattern9_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Unary {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                    } = &pattern11_0
+                                    {
+                                        match &pattern12_0 {
+                                            &Opcode::SwidenLow => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 8 {
+                                                        if pattern15_1 == 16 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::SwidenLow =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 8 {
+                                                                                if pattern23_1 == 16
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 819.
+                                                                                    let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
+                                                                                    let expr1_0 = constructor_pmovsxbw(ctx, &expr0_0)?;
+                                                                                    let expr2_0 = C::put_in_reg_mem(ctx, pattern20_1);
+                                                                                    let expr3_0 = constructor_pmovsxbw(ctx, &expr2_0)?;
+                                                                                    let expr4_0 = RegMem::Reg {
+                                                                                        reg: expr3_0,
+                                                                                    };
+                                                                                    let expr5_0 = constructor_pmullw(ctx, expr1_0, &expr4_0)?;
+                                                                                    let expr6_0 = C::value_reg(ctx, expr5_0);
+                                                                                    return Some(
+                                                                                        expr6_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            &Opcode::SwidenHigh => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 8 {
+                                                        if pattern15_1 == 16 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::SwidenHigh =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 8 {
+                                                                                if pattern23_1 == 16
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 779.
+                                                                                    let expr0_0 = C::put_in_reg(ctx, pattern12_1);
+                                                                                    let expr1_0 = RegMem::Reg {
+                                                                                        reg: expr0_0,
+                                                                                    };
+                                                                                    let expr2_0: u8 = 8;
+                                                                                    let expr3_0 = OperandSize::Size32;
+                                                                                    let expr4_0 = constructor_palignr(ctx, expr0_0, &expr1_0, expr2_0, &expr3_0)?;
+                                                                                    let expr5_0 = RegMem::Reg {
+                                                                                        reg: expr4_0,
+                                                                                    };
+                                                                                    let expr6_0 = constructor_pmovsxbw(ctx, &expr5_0)?;
+                                                                                    let expr7_0 = C::put_in_reg(ctx, pattern20_1);
+                                                                                    let expr8_0 = RegMem::Reg {
+                                                                                        reg: expr7_0,
+                                                                                    };
+                                                                                    let expr9_0: u8 = 8;
+                                                                                    let expr10_0 = OperandSize::Size32;
+                                                                                    let expr11_0 = constructor_palignr(ctx, expr7_0, &expr8_0, expr9_0, &expr10_0)?;
+                                                                                    let expr12_0 = RegMem::Reg {
+                                                                                        reg: expr11_0,
+                                                                                    };
+                                                                                    let expr13_0 = constructor_pmovsxbw(ctx, &expr12_0)?;
+                                                                                    let expr14_0 = RegMem::Reg {
+                                                                                        reg: expr13_0,
+                                                                                    };
+                                                                                    let expr15_0 = constructor_pmullw(ctx, expr6_0, &expr14_0)?;
+                                                                                    let expr16_0 = C::value_reg(ctx, expr15_0);
+                                                                                    return Some(
+                                                                                        expr16_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            &Opcode::UwidenLow => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 8 {
+                                                        if pattern15_1 == 16 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::UwidenLow =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 8 {
+                                                                                if pattern23_1 == 16
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 895.
+                                                                                    let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
+                                                                                    let expr1_0 = constructor_pmovzxbw(ctx, &expr0_0)?;
+                                                                                    let expr2_0 = C::put_in_reg_mem(ctx, pattern20_1);
+                                                                                    let expr3_0 = constructor_pmovzxbw(ctx, &expr2_0)?;
+                                                                                    let expr4_0 = RegMem::Reg {
+                                                                                        reg: expr3_0,
+                                                                                    };
+                                                                                    let expr5_0 = constructor_pmullw(ctx, expr1_0, &expr4_0)?;
+                                                                                    let expr6_0 = C::value_reg(ctx, expr5_0);
+                                                                                    return Some(
+                                                                                        expr6_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            &Opcode::UwidenHigh => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 8 {
+                                                        if pattern15_1 == 16 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::UwidenHigh =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 8 {
+                                                                                if pattern23_1 == 16
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 855.
+                                                                                    let expr0_0 = C::put_in_reg(ctx, pattern12_1);
+                                                                                    let expr1_0 = RegMem::Reg {
+                                                                                        reg: expr0_0,
+                                                                                    };
+                                                                                    let expr2_0: u8 = 8;
+                                                                                    let expr3_0 = OperandSize::Size32;
+                                                                                    let expr4_0 = constructor_palignr(ctx, expr0_0, &expr1_0, expr2_0, &expr3_0)?;
+                                                                                    let expr5_0 = RegMem::Reg {
+                                                                                        reg: expr4_0,
+                                                                                    };
+                                                                                    let expr6_0 = constructor_pmovzxbw(ctx, &expr5_0)?;
+                                                                                    let expr7_0 = C::put_in_reg(ctx, pattern20_1);
+                                                                                    let expr8_0 = RegMem::Reg {
+                                                                                        reg: expr7_0,
+                                                                                    };
+                                                                                    let expr9_0: u8 = 8;
+                                                                                    let expr10_0 = OperandSize::Size32;
+                                                                                    let expr11_0 = constructor_palignr(ctx, expr7_0, &expr8_0, expr9_0, &expr10_0)?;
+                                                                                    let expr12_0 = RegMem::Reg {
+                                                                                        reg: expr11_0,
+                                                                                    };
+                                                                                    let expr13_0 = constructor_pmovzxbw(ctx, &expr12_0)?;
+                                                                                    let expr14_0 = RegMem::Reg {
+                                                                                        reg: expr13_0,
+                                                                                    };
+                                                                                    let expr15_0 = constructor_pmullw(ctx, expr6_0, &expr14_0)?;
+                                                                                    let expr16_0 = C::value_reg(ctx, expr15_0);
+                                                                                    return Some(
+                                                                                        expr16_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            _ => {}
+                                        }
+                                    }
+                                }
+                                // Rule at src/isa/x64/lower.isle line 723.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_pmullw(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+            }
+            if pattern3_0 == 32 {
+                if pattern3_1 == 4 {
+                    let pattern6_0 = C::inst_data(ctx, pattern0_0);
+                    if let &InstructionData::Binary {
+                        opcode: ref pattern7_0,
+                        args: ref pattern7_1,
+                    } = &pattern6_0
+                    {
+                        match &pattern7_0 {
+                            &Opcode::Iadd => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 96.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_paddd(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::Isub => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 245.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_psubd(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::Imul => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                if let Some(pattern10_0) = C::def_inst(ctx, pattern9_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Unary {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                    } = &pattern11_0
+                                    {
+                                        match &pattern12_0 {
+                                            &Opcode::SwidenLow => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 16 {
+                                                        if pattern15_1 == 8 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::SwidenLow =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 16 {
+                                                                                if pattern23_1 == 8
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 829.
+                                                                                    let expr0_0 = C::put_in_reg(ctx, pattern12_1);
+                                                                                    let expr1_0 = C::put_in_reg(ctx, pattern20_1);
+                                                                                    let expr2_0 = RegMem::Reg {
+                                                                                        reg: expr1_0,
+                                                                                    };
+                                                                                    let expr3_0 = constructor_pmullw(ctx, expr0_0, &expr2_0)?;
+                                                                                    let expr4_0 = RegMem::Reg {
+                                                                                        reg: expr1_0,
+                                                                                    };
+                                                                                    let expr5_0 = constructor_pmulhw(ctx, expr0_0, &expr4_0)?;
+                                                                                    let expr6_0 = RegMem::Reg {
+                                                                                        reg: expr5_0,
+                                                                                    };
+                                                                                    let expr7_0 = constructor_punpcklwd(ctx, expr3_0, &expr6_0)?;
+                                                                                    let expr8_0 = C::value_reg(ctx, expr7_0);
+                                                                                    return Some(
+                                                                                        expr8_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            &Opcode::SwidenHigh => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 16 {
+                                                        if pattern15_1 == 8 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::SwidenHigh =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 16 {
+                                                                                if pattern23_1 == 8
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 793.
+                                                                                    let expr0_0 = C::put_in_reg(ctx, pattern12_1);
+                                                                                    let expr1_0 = C::put_in_reg(ctx, pattern20_1);
+                                                                                    let expr2_0 = RegMem::Reg {
+                                                                                        reg: expr1_0,
+                                                                                    };
+                                                                                    let expr3_0 = constructor_pmullw(ctx, expr0_0, &expr2_0)?;
+                                                                                    let expr4_0 = RegMem::Reg {
+                                                                                        reg: expr1_0,
+                                                                                    };
+                                                                                    let expr5_0 = constructor_pmulhw(ctx, expr0_0, &expr4_0)?;
+                                                                                    let expr6_0 = RegMem::Reg {
+                                                                                        reg: expr5_0,
+                                                                                    };
+                                                                                    let expr7_0 = constructor_punpckhwd(ctx, expr3_0, &expr6_0)?;
+                                                                                    let expr8_0 = C::value_reg(ctx, expr7_0);
+                                                                                    return Some(
+                                                                                        expr8_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            &Opcode::UwidenLow => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 16 {
+                                                        if pattern15_1 == 8 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::UwidenLow =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 16 {
+                                                                                if pattern23_1 == 8
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 905.
+                                                                                    let expr0_0 = C::put_in_reg(ctx, pattern12_1);
+                                                                                    let expr1_0 = C::put_in_reg(ctx, pattern20_1);
+                                                                                    let expr2_0 = RegMem::Reg {
+                                                                                        reg: expr1_0,
+                                                                                    };
+                                                                                    let expr3_0 = constructor_pmullw(ctx, expr0_0, &expr2_0)?;
+                                                                                    let expr4_0 = RegMem::Reg {
+                                                                                        reg: expr1_0,
+                                                                                    };
+                                                                                    let expr5_0 = constructor_pmulhuw(ctx, expr0_0, &expr4_0)?;
+                                                                                    let expr6_0 = RegMem::Reg {
+                                                                                        reg: expr5_0,
+                                                                                    };
+                                                                                    let expr7_0 = constructor_punpcklwd(ctx, expr3_0, &expr6_0)?;
+                                                                                    let expr8_0 = C::value_reg(ctx, expr7_0);
+                                                                                    return Some(
+                                                                                        expr8_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            &Opcode::UwidenHigh => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 16 {
+                                                        if pattern15_1 == 8 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::UwidenHigh =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 16 {
+                                                                                if pattern23_1 == 8
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 869.
+                                                                                    let expr0_0 = C::put_in_reg(ctx, pattern12_1);
+                                                                                    let expr1_0 = C::put_in_reg(ctx, pattern20_1);
+                                                                                    let expr2_0 = RegMem::Reg {
+                                                                                        reg: expr1_0,
+                                                                                    };
+                                                                                    let expr3_0 = constructor_pmullw(ctx, expr0_0, &expr2_0)?;
+                                                                                    let expr4_0 = RegMem::Reg {
+                                                                                        reg: expr1_0,
+                                                                                    };
+                                                                                    let expr5_0 = constructor_pmulhuw(ctx, expr0_0, &expr4_0)?;
+                                                                                    let expr6_0 = RegMem::Reg {
+                                                                                        reg: expr5_0,
+                                                                                    };
+                                                                                    let expr7_0 = constructor_punpckhwd(ctx, expr3_0, &expr6_0)?;
+                                                                                    let expr8_0 = C::value_reg(ctx, expr7_0);
+                                                                                    return Some(
+                                                                                        expr8_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            _ => {}
+                                        }
+                                    }
+                                }
+                                // Rule at src/isa/x64/lower.isle line 726.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_pmulld(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+            }
+            if pattern3_0 == 64 {
+                if pattern3_1 == 2 {
+                    let pattern6_0 = C::inst_data(ctx, pattern0_0);
+                    if let &InstructionData::Binary {
+                        opcode: ref pattern7_0,
+                        args: ref pattern7_1,
+                    } = &pattern6_0
+                    {
+                        match &pattern7_0 {
+                            &Opcode::Iadd => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 101.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_paddq(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::Isub => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                // Rule at src/isa/x64/lower.isle line 250.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1);
+                                let expr2_0 = constructor_psubq(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::Imul => {
+                                let (pattern9_0, pattern9_1) =
+                                    C::unpack_value_array_2(ctx, &pattern7_1);
+                                if let Some(pattern10_0) = C::def_inst(ctx, pattern9_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Unary {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                    } = &pattern11_0
+                                    {
+                                        match &pattern12_0 {
+                                            &Opcode::SwidenLow => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 32 {
+                                                        if pattern15_1 == 4 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::SwidenLow =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 32 {
+                                                                                if pattern23_1 == 4
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 841.
+                                                                                    let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
+                                                                                    let expr1_0: u8 = 80;
+                                                                                    let expr2_0 = OperandSize::Size32;
+                                                                                    let expr3_0 = constructor_pshufd(ctx, &expr0_0, expr1_0, &expr2_0)?;
+                                                                                    let expr4_0 = C::put_in_reg_mem(ctx, pattern20_1);
+                                                                                    let expr5_0: u8 = 80;
+                                                                                    let expr6_0 = OperandSize::Size32;
+                                                                                    let expr7_0 = constructor_pshufd(ctx, &expr4_0, expr5_0, &expr6_0)?;
+                                                                                    let expr8_0 = RegMem::Reg {
+                                                                                        reg: expr7_0,
+                                                                                    };
+                                                                                    let expr9_0 = constructor_pmuldq(ctx, expr3_0, &expr8_0)?;
+                                                                                    let expr10_0 = C::value_reg(ctx, expr9_0);
+                                                                                    return Some(
+                                                                                        expr10_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            &Opcode::SwidenHigh => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 32 {
+                                                        if pattern15_1 == 4 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::SwidenHigh =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 32 {
+                                                                                if pattern23_1 == 4
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 805.
+                                                                                    let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
+                                                                                    let expr1_0: u8 = 250;
+                                                                                    let expr2_0 = OperandSize::Size32;
+                                                                                    let expr3_0 = constructor_pshufd(ctx, &expr0_0, expr1_0, &expr2_0)?;
+                                                                                    let expr4_0 = C::put_in_reg_mem(ctx, pattern20_1);
+                                                                                    let expr5_0: u8 = 250;
+                                                                                    let expr6_0 = OperandSize::Size32;
+                                                                                    let expr7_0 = constructor_pshufd(ctx, &expr4_0, expr5_0, &expr6_0)?;
+                                                                                    let expr8_0 = RegMem::Reg {
+                                                                                        reg: expr7_0,
+                                                                                    };
+                                                                                    let expr9_0 = constructor_pmuldq(ctx, expr3_0, &expr8_0)?;
+                                                                                    let expr10_0 = C::value_reg(ctx, expr9_0);
+                                                                                    return Some(
+                                                                                        expr10_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            &Opcode::UwidenLow => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 32 {
+                                                        if pattern15_1 == 4 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::UwidenLow =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 32 {
+                                                                                if pattern23_1 == 4
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 917.
+                                                                                    let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
+                                                                                    let expr1_0: u8 = 80;
+                                                                                    let expr2_0 = OperandSize::Size32;
+                                                                                    let expr3_0 = constructor_pshufd(ctx, &expr0_0, expr1_0, &expr2_0)?;
+                                                                                    let expr4_0 = C::put_in_reg_mem(ctx, pattern20_1);
+                                                                                    let expr5_0: u8 = 80;
+                                                                                    let expr6_0 = OperandSize::Size32;
+                                                                                    let expr7_0 = constructor_pshufd(ctx, &expr4_0, expr5_0, &expr6_0)?;
+                                                                                    let expr8_0 = RegMem::Reg {
+                                                                                        reg: expr7_0,
+                                                                                    };
+                                                                                    let expr9_0 = constructor_pmuludq(ctx, expr3_0, &expr8_0)?;
+                                                                                    let expr10_0 = C::value_reg(ctx, expr9_0);
+                                                                                    return Some(
+                                                                                        expr10_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            &Opcode::UwidenHigh => {
+                                                let pattern14_0 = C::value_type(ctx, pattern12_1);
+                                                if let Some((pattern15_0, pattern15_1)) =
+                                                    C::multi_lane(ctx, pattern14_0)
+                                                {
+                                                    if pattern15_0 == 32 {
+                                                        if pattern15_1 == 4 {
+                                                            if let Some(pattern18_0) =
+                                                                C::def_inst(ctx, pattern9_1)
+                                                            {
+                                                                let pattern19_0 =
+                                                                    C::inst_data(ctx, pattern18_0);
+                                                                if let &InstructionData::Unary {
+                                                                    opcode: ref pattern20_0,
+                                                                    arg: pattern20_1,
+                                                                } = &pattern19_0
+                                                                {
+                                                                    if let &Opcode::UwidenHigh =
+                                                                        &pattern20_0
+                                                                    {
+                                                                        let pattern22_0 =
+                                                                            C::value_type(
+                                                                                ctx,
+                                                                                pattern20_1,
+                                                                            );
+                                                                        if let Some((
+                                                                            pattern23_0,
+                                                                            pattern23_1,
+                                                                        )) = C::multi_lane(
+                                                                            ctx,
+                                                                            pattern22_0,
+                                                                        ) {
+                                                                            if pattern23_0 == 32 {
+                                                                                if pattern23_1 == 4
+                                                                                {
+                                                                                    // Rule at src/isa/x64/lower.isle line 881.
+                                                                                    let expr0_0 = C::put_in_reg_mem(ctx, pattern12_1);
+                                                                                    let expr1_0: u8 = 250;
+                                                                                    let expr2_0 = OperandSize::Size32;
+                                                                                    let expr3_0 = constructor_pshufd(ctx, &expr0_0, expr1_0, &expr2_0)?;
+                                                                                    let expr4_0 = C::put_in_reg_mem(ctx, pattern20_1);
+                                                                                    let expr5_0: u8 = 250;
+                                                                                    let expr6_0 = OperandSize::Size32;
+                                                                                    let expr7_0 = constructor_pshufd(ctx, &expr4_0, expr5_0, &expr6_0)?;
+                                                                                    let expr8_0 = RegMem::Reg {
+                                                                                        reg: expr7_0,
+                                                                                    };
+                                                                                    let expr9_0 = constructor_pmuludq(ctx, expr3_0, &expr8_0)?;
+                                                                                    let expr10_0 = C::value_reg(ctx, expr9_0);
+                                                                                    return Some(
+                                                                                        expr10_0,
+                                                                                    );
+                                                                                }
+                                                                            }
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                            _ => {}
+                                        }
+                                    }
+                                }
+                                // Rule at src/isa/x64/lower.isle line 757.
+                                let expr0_0 = C::put_in_reg(ctx, pattern9_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern9_1);
+                                let expr2_0: u32 = 32;
+                                let expr3_0 = RegMemImm::Imm { simm32: expr2_0 };
+                                let expr4_0 = constructor_psrlq(ctx, expr0_0, &expr3_0)?;
+                                let expr5_0 = RegMem::Reg { reg: expr1_0 };
+                                let expr6_0 = constructor_pmuludq(ctx, expr4_0, &expr5_0)?;
+                                let expr7_0: u32 = 32;
+                                let expr8_0 = RegMemImm::Imm { simm32: expr7_0 };
+                                let expr9_0 = constructor_psrlq(ctx, expr1_0, &expr8_0)?;
+                                let expr10_0 = RegMem::Reg { reg: expr9_0 };
+                                let expr11_0 = constructor_pmuludq(ctx, expr0_0, &expr10_0)?;
+                                let expr12_0 = RegMem::Reg { reg: expr11_0 };
+                                let expr13_0 = constructor_paddq(ctx, expr6_0, &expr12_0)?;
+                                let expr14_0: u32 = 32;
+                                let expr15_0 = RegMemImm::Imm { simm32: expr14_0 };
+                                let expr16_0 = constructor_psllq(ctx, expr13_0, &expr15_0)?;
+                                let expr17_0 = RegMem::Reg { reg: expr1_0 };
+                                let expr18_0 = constructor_pmuludq(ctx, expr0_0, &expr17_0)?;
+                                let expr19_0 = RegMem::Reg { reg: expr16_0 };
+                                let expr20_0 = constructor_paddq(ctx, expr18_0, &expr19_0)?;
+                                let expr21_0 = C::value_reg(ctx, expr20_0);
+                                return Some(expr21_0);
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+            }
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::Binary {
+                opcode: ref pattern5_0,
+                args: ref pattern5_1,
+            } = &pattern4_0
+            {
+                match &pattern5_0 {
+                    &Opcode::Band => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 341.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
+                        let expr2_0 = constructor_pand(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Bor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 417.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
+                        let expr2_0 = constructor_por(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Bxor => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 495.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1);
+                        let expr2_0 = constructor_pxor(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::BandNot => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 946.
+                        let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                        let expr1_0 = C::put_in_reg_mem(ctx, pattern7_0);
+                        let expr2_0 = constructor_pandn(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    _ => {}
+                }
+            }
+        }
+        if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::UnaryImm {
+                    opcode: ref pattern5_0,
+                    imm: pattern5_1,
+                } => {
+                    if let &Opcode::Iconst = &pattern5_0 {
+                        let pattern7_0 = C::u64_from_imm64(ctx, pattern5_1);
+                        // Rule at src/isa/x64/lower.isle line 10.
+                        let expr0_0 = constructor_imm(ctx, pattern3_0, pattern7_0)?;
+                        let expr1_0 = C::value_reg(ctx, expr0_0);
+                        return Some(expr1_0);
+                    }
+                }
+                &InstructionData::UnaryBool {
+                    opcode: ref pattern5_0,
+                    imm: pattern5_1,
+                } => {
+                    if let &Opcode::Bconst = &pattern5_0 {
+                        if pattern5_1 == true {
+                            // Rule at src/isa/x64/lower.isle line 28.
+                            let expr0_0: u64 = 1;
+                            let expr1_0 = constructor_imm(ctx, pattern3_0, expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                        if pattern5_1 == false {
+                            // Rule at src/isa/x64/lower.isle line 24.
+                            let expr0_0: u64 = 0;
+                            let expr1_0 = constructor_imm(ctx, pattern3_0, expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                }
+                &InstructionData::Binary {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Iadd => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 66.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 78.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 62.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 72.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 54.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
+                            let expr3_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Isub => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 222.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_sub(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 227.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_sub(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 215.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
+                            let expr3_0 = constructor_sub(ctx, pattern3_0, expr0_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Imul => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 663.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_mul(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 675.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 659.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_mul(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 669.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 652.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
+                            let expr3_0 = constructor_mul(ctx, pattern3_0, expr0_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::IaddIfcout => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 159.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 171.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 155.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_add(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 165.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 147.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
+                            let expr3_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Band => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 325.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_m_and(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 311.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 =
+                                    constructor_m_and(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 319.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_m_and(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 305.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 =
+                                    constructor_m_and(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 298.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
+                            let expr3_0 = constructor_m_and(ctx, pattern3_0, expr0_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Bor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 401.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_or(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 387.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 395.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_or(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 381.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 374.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
+                            let expr3_0 = constructor_or(ctx, pattern3_0, expr0_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Bxor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 479.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_xor(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_0) {
+                                // Rule at src/isa/x64/lower.isle line 465.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            if let Some(pattern8_0) = C::simm32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 473.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_xor(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::sinkable_load(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 459.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = C::sink_load(ctx, &pattern8_0);
+                                let expr2_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 452.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
+                            let expr3_0 = constructor_xor(ctx, pattern3_0, expr0_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Rotl => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 624.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_m_rotl(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 618.
+                            let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?;
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr2_0 = Imm8Reg::Reg { reg: expr0_0 };
+                            let expr3_0 = constructor_m_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Ishl => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 533.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_shl(ctx, pattern3_0, expr0_0, &pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 527.
+                            let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?;
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr2_0 = Imm8Reg::Reg { reg: expr0_0 };
+                            let expr3_0 = constructor_shl(ctx, pattern3_0, expr1_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Ushr => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/x64/lower.isle line 579.
+                                let expr0_0 = ExtendKind::Zero;
+                                let expr1_0 = constructor_extend_to_reg(
+                                    ctx, pattern7_0, pattern3_0, &expr0_0,
+                                )?;
+                                let expr2_0 =
+                                    constructor_shr(ctx, pattern3_0, expr1_0, &pattern8_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/x64/lower.isle line 572.
+                            let expr0_0 = ExtendKind::Zero;
+                            let expr1_0 =
+                                constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?;
+                            let expr2_0 = constructor_lo_reg(ctx, pattern7_1)?;
+                            let expr3_0 = Imm8Reg::Reg { reg: expr2_0 };
+                            let expr4_0 = constructor_shr(ctx, pattern3_0, expr1_0, &expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                        _ => {}
+                    }
+                }
+                &InstructionData::BinaryImm64 {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    imm: pattern5_2,
+                } => {
+                    if let &Opcode::IaddImm = &pattern5_0 {
+                        let pattern7_0 = C::u64_from_imm64(ctx, pattern5_2);
+                        // Rule at src/isa/x64/lower.isle line 188.
+                        let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                        let expr1_0 = constructor_imm(ctx, pattern3_0, pattern7_0)?;
+                        let expr2_0 = RegMemImm::Reg { reg: expr1_0 };
+                        let expr3_0 = constructor_add(ctx, pattern3_0, expr0_0, &expr2_0)?;
+                        let expr4_0 = C::value_reg(ctx, expr3_0);
+                        return Some(expr4_0);
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term or_i128.
+pub fn constructor_or_i128<C: Context>(
+    ctx: &mut C,
+    arg0: ValueRegs,
+    arg1: ValueRegs,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/lower.isle line 425.
+    let expr0_0: usize = 0;
+    let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0);
+    let expr2_0: usize = 1;
+    let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0);
+    let expr4_0: usize = 0;
+    let expr5_0 = C::value_regs_get(ctx, pattern1_0, expr4_0);
+    let expr6_0: usize = 1;
+    let expr7_0 = C::value_regs_get(ctx, pattern1_0, expr6_0);
+    let expr8_0: Type = I64;
+    let expr9_0 = RegMemImm::Reg { reg: expr5_0 };
+    let expr10_0 = constructor_or(ctx, expr8_0, expr1_0, &expr9_0)?;
+    let expr11_0: Type = I64;
+    let expr12_0 = RegMemImm::Reg { reg: expr7_0 };
+    let expr13_0 = constructor_or(ctx, expr11_0, expr3_0, &expr12_0)?;
+    let expr14_0 = C::value_regs(ctx, expr10_0, expr13_0);
+    return Some(expr14_0);
+}
+
+// Generated as internal constructor for term shl_i128.
+pub fn constructor_shl_i128<C: Context>(
+    ctx: &mut C,
+    arg0: ValueRegs,
+    arg1: Reg,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/lower.isle line 539.
+    let expr0_0: usize = 0;
+    let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0);
+    let expr2_0: usize = 1;
+    let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0);
+    let expr4_0: Type = I64;
+    let expr5_0 = Imm8Reg::Reg { reg: pattern1_0 };
+    let expr6_0 = constructor_shl(ctx, expr4_0, expr1_0, &expr5_0)?;
+    let expr7_0: Type = I64;
+    let expr8_0 = Imm8Reg::Reg { reg: pattern1_0 };
+    let expr9_0 = constructor_shl(ctx, expr7_0, expr3_0, &expr8_0)?;
+    let expr10_0: Type = I64;
+    let expr11_0: Type = I64;
+    let expr12_0: Type = I64;
+    let expr13_0: u64 = 64;
+    let expr14_0 = constructor_imm(ctx, expr12_0, expr13_0)?;
+    let expr15_0 = RegMemImm::Reg { reg: pattern1_0 };
+    let expr16_0 = constructor_sub(ctx, expr11_0, expr14_0, &expr15_0)?;
+    let expr17_0 = Imm8Reg::Reg { reg: expr16_0 };
+    let expr18_0 = constructor_shr(ctx, expr10_0, expr1_0, &expr17_0)?;
+    let expr19_0: Type = I64;
+    let expr20_0: u64 = 0;
+    let expr21_0 = constructor_imm(ctx, expr19_0, expr20_0)?;
+    let expr22_0 = OperandSize::Size64;
+    let expr23_0: u32 = 127;
+    let expr24_0 = RegMemImm::Imm { simm32: expr23_0 };
+    let expr25_0 = constructor_test(ctx, &expr22_0, &expr24_0, pattern1_0)?;
+    let expr26_0: Type = I64;
+    let expr27_0 = CC::Z;
+    let expr28_0 = RegMem::Reg { reg: expr21_0 };
+    let expr29_0 = constructor_cmove(ctx, expr26_0, &expr27_0, &expr28_0, expr18_0)?;
+    let expr30_0 = constructor_with_flags_1(ctx, &expr25_0, &expr29_0)?;
+    let expr31_0: Type = I64;
+    let expr32_0 = RegMemImm::Reg { reg: expr9_0 };
+    let expr33_0 = constructor_or(ctx, expr31_0, expr30_0, &expr32_0)?;
+    let expr34_0 = OperandSize::Size64;
+    let expr35_0: u32 = 64;
+    let expr36_0 = RegMemImm::Imm { simm32: expr35_0 };
+    let expr37_0 = constructor_test(ctx, &expr34_0, &expr36_0, pattern1_0)?;
+    let expr38_0: Type = I64;
+    let expr39_0 = CC::Z;
+    let expr40_0 = RegMem::Reg { reg: expr6_0 };
+    let expr41_0 = constructor_cmove(ctx, expr38_0, &expr39_0, &expr40_0, expr21_0)?;
+    let expr42_0: Type = I64;
+    let expr43_0 = CC::Z;
+    let expr44_0 = RegMem::Reg { reg: expr33_0 };
+    let expr45_0 = constructor_cmove(ctx, expr42_0, &expr43_0, &expr44_0, expr6_0)?;
+    let expr46_0 = constructor_with_flags_2(ctx, &expr37_0, &expr41_0, &expr45_0)?;
+    return Some(expr46_0);
+}
+
+// Generated as internal constructor for term shr_i128.
+pub fn constructor_shr_i128<C: Context>(
+    ctx: &mut C,
+    arg0: ValueRegs,
+    arg1: Reg,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/x64/lower.isle line 586.
+    let expr0_0: usize = 0;
+    let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0);
+    let expr2_0: usize = 1;
+    let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0);
+    let expr4_0: Type = I64;
+    let expr5_0 = Imm8Reg::Reg { reg: pattern1_0 };
+    let expr6_0 = constructor_shr(ctx, expr4_0, expr1_0, &expr5_0)?;
+    let expr7_0: Type = I64;
+    let expr8_0 = Imm8Reg::Reg { reg: pattern1_0 };
+    let expr9_0 = constructor_shr(ctx, expr7_0, expr3_0, &expr8_0)?;
+    let expr10_0: Type = I64;
+    let expr11_0: Type = I64;
+    let expr12_0: Type = I64;
+    let expr13_0: u64 = 64;
+    let expr14_0 = constructor_imm(ctx, expr12_0, expr13_0)?;
+    let expr15_0 = RegMemImm::Reg { reg: pattern1_0 };
+    let expr16_0 = constructor_sub(ctx, expr11_0, expr14_0, &expr15_0)?;
+    let expr17_0 = Imm8Reg::Reg { reg: expr16_0 };
+    let expr18_0 = constructor_shl(ctx, expr10_0, expr3_0, &expr17_0)?;
+    let expr19_0 = OperandSize::Size64;
+    let expr20_0: u32 = 127;
+    let expr21_0 = RegMemImm::Imm { simm32: expr20_0 };
+    let expr22_0 = constructor_test(ctx, &expr19_0, &expr21_0, pattern1_0)?;
+    let expr23_0: Type = I64;
+    let expr24_0 = CC::Z;
+    let expr25_0: Type = I64;
+    let expr26_0: u64 = 0;
+    let expr27_0 = constructor_imm(ctx, expr25_0, expr26_0)?;
+    let expr28_0 = RegMem::Reg { reg: expr27_0 };
+    let expr29_0 = constructor_cmove(ctx, expr23_0, &expr24_0, &expr28_0, expr18_0)?;
+    let expr30_0 = constructor_with_flags_1(ctx, &expr22_0, &expr29_0)?;
+    let expr31_0: Type = I64;
+    let expr32_0 = RegMemImm::Reg { reg: expr6_0 };
+    let expr33_0 = constructor_or(ctx, expr31_0, expr30_0, &expr32_0)?;
+    let expr34_0 = OperandSize::Size64;
+    let expr35_0: u32 = 64;
+    let expr36_0 = RegMemImm::Imm { simm32: expr35_0 };
+    let expr37_0 = constructor_test(ctx, &expr34_0, &expr36_0, pattern1_0)?;
+    let expr38_0: Type = I64;
+    let expr39_0 = CC::Z;
+    let expr40_0 = RegMem::Reg { reg: expr33_0 };
+    let expr41_0 = constructor_cmove(ctx, expr38_0, &expr39_0, &expr40_0, expr9_0)?;
+    let expr42_0: Type = I64;
+    let expr43_0 = CC::Z;
+    let expr44_0 = RegMem::Reg { reg: expr9_0 };
+    let expr45_0: Type = I64;
+    let expr46_0: u64 = 0;
+    let expr47_0 = constructor_imm(ctx, expr45_0, expr46_0)?;
+    let expr48_0 = constructor_cmove(ctx, expr42_0, &expr43_0, &expr44_0, expr47_0)?;
+    let expr49_0 = constructor_with_flags_2(ctx, &expr37_0, &expr41_0, &expr48_0)?;
+    return Some(expr49_0);
+}
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index 0a35ad3ae6..8f65449cde 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -11,13 +11,14 @@ use crate::fx::{FxHashMap, FxHashSet};
 use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit};
 use crate::ir::instructions::BranchInfo;
 use crate::ir::{
-    ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst,
-    InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef,
-    ValueLabelAssignments, ValueLabelStart,
+    ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, ExternalName, Function,
+    GlobalValueData, Inst, InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value,
+    ValueDef, ValueLabelAssignments, ValueLabelStart,
 };
 use crate::machinst::{
-    writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode,
-    VCodeBuilder, VCodeConstant, VCodeConstantData, VCodeConstants, VCodeInst, ValueRegs,
+    non_writable_value_regs, writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder,
+    LoweredBlock, MachLabel, VCode, VCodeBuilder, VCodeConstant, VCodeConstantData, VCodeConstants,
+    VCodeInst, ValueRegs,
 };
 use crate::CodegenResult;
 use alloc::boxed::Box;
@@ -61,6 +62,8 @@ pub trait LowerCtx {
     /// The instruction type for which this lowering framework is instantiated.
     type I: VCodeInst;
 
+    fn dfg(&self) -> &DataFlowGraph;
+
     // Function-level queries:
 
     /// Get the `ABICallee`.
@@ -124,8 +127,12 @@ pub trait LowerCtx {
     /// instruction's result(s) must have *no* uses remaining, because it will
     /// not be codegen'd (it has been integrated into the current instruction).
     fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput;
+    /// Like `get_input_as_source_or_const` but with a `Value`.
+    fn get_value_as_source_or_const(&self, value: Value) -> NonRegInput;
     /// Put the `idx`th input into register(s) and return the assigned register.
     fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg>;
+    /// Put the given value into register(s) and return the assigned register.
+    fn put_value_in_regs(&mut self, value: Value) -> ValueRegs<Reg>;
     /// Get the `idx`th output register(s) of the given IR instruction. When
     /// `backend.lower_inst_to_regs(ctx, inst)` is called, it is expected that
     /// the backend will write results to these output register(s).  This
@@ -1002,101 +1009,15 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
 
         Ok((vcode, stack_map_info))
     }
-
-    fn put_value_in_regs(&mut self, val: Value) -> ValueRegs<Reg> {
-        log::trace!("put_value_in_reg: val {}", val);
-        let mut regs = self.value_regs[val];
-        log::trace!(" -> regs {:?}", regs);
-        assert!(regs.is_valid());
-
-        self.value_lowered_uses[val] += 1;
-
-        // Pinned-reg hack: if backend specifies a fixed pinned register, use it
-        // directly when we encounter a GetPinnedReg op, rather than lowering
-        // the actual op, and do not return the source inst to the caller; the
-        // value comes "out of the ether" and we will not force generation of
-        // the superfluous move.
-        if let ValueDef::Result(i, 0) = self.f.dfg.value_def(val) {
-            if self.f.dfg[i].opcode() == Opcode::GetPinnedReg {
-                if let Some(pr) = self.pinned_reg {
-                    regs = ValueRegs::one(pr);
-                }
-            }
-        }
-
-        regs
-    }
-
-    /// Get the actual inputs for a value. This is the implementation for
-    /// `get_input()` but starting from the SSA value, which is not exposed to
-    /// the backend.
-    fn get_value_as_source_or_const(&self, val: Value) -> NonRegInput {
-        log::trace!(
-            "get_input_for_val: val {} at cur_inst {:?} cur_scan_entry_color {:?}",
-            val,
-            self.cur_inst,
-            self.cur_scan_entry_color,
-        );
-        let inst = match self.f.dfg.value_def(val) {
-            // OK to merge source instruction if (i) we have a source
-            // instruction, and:
-            // - It has no side-effects, OR
-            // - It has a side-effect, has one output value, that one output has
-            //   only one use (this one), and the instruction's color is *one less
-            //   than* the current scan color.
-            //
-            //   This latter set of conditions is testing whether a
-            //   side-effecting instruction can sink to the current scan
-            //   location; this is possible if the in-color of this inst is
-            //   equal to the out-color of the producing inst, so no other
-            //   side-effecting ops occur between them (which will only be true
-            //   if they are in the same BB, because color increments at each BB
-            //   start).
-            //
-            //   If it is actually sunk, then in `merge_inst()`, we update the
-            //   scan color so that as we scan over the range past which the
-            //   instruction was sunk, we allow other instructions (that came
-            //   prior to the sunk instruction) to sink.
-            ValueDef::Result(src_inst, result_idx) => {
-                let src_side_effect = has_lowering_side_effect(self.f, src_inst);
-                log::trace!(" -> src inst {}", src_inst);
-                log::trace!(" -> has lowering side effect: {}", src_side_effect);
-                if !src_side_effect {
-                    // Pure instruction: always possible to sink.
-                    Some((src_inst, result_idx))
-                } else {
-                    // Side-effect: test whether this is the only use of the
-                    // only result of the instruction, and whether colors allow
-                    // the code-motion.
-                    if self.cur_scan_entry_color.is_some()
-                        && self.value_uses[val] == 1
-                        && self.value_lowered_uses[val] == 0
-                        && self.num_outputs(src_inst) == 1
-                        && self
-                            .side_effect_inst_entry_colors
-                            .get(&src_inst)
-                            .unwrap()
-                            .get()
-                            + 1
-                            == self.cur_scan_entry_color.unwrap().get()
-                    {
-                        Some((src_inst, 0))
-                    } else {
-                        None
-                    }
-                }
-            }
-            _ => None,
-        };
-        let constant = inst.and_then(|(inst, _)| self.get_constant(inst));
-
-        NonRegInput { inst, constant }
-    }
 }
 
 impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
     type I = I;
 
+    fn dfg(&self) -> &DataFlowGraph {
+        &self.f.dfg
+    }
+
     fn abi(&mut self) -> &mut dyn ABICallee<I = I> {
         self.vcode.abi()
     }
@@ -1207,12 +1128,124 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
         self.get_value_as_source_or_const(val)
     }
 
+    fn get_value_as_source_or_const(&self, val: Value) -> NonRegInput {
+        log::trace!(
+            "get_input_for_val: val {} at cur_inst {:?} cur_scan_entry_color {:?}",
+            val,
+            self.cur_inst,
+            self.cur_scan_entry_color,
+        );
+        let inst = match self.f.dfg.value_def(val) {
+            // OK to merge source instruction if (i) we have a source
+            // instruction, and:
+            // - It has no side-effects, OR
+            // - It has a side-effect, has one output value, that one output has
+            //   only one use (this one), and the instruction's color is *one less
+            //   than* the current scan color.
+            //
+            //   This latter set of conditions is testing whether a
+            //   side-effecting instruction can sink to the current scan
+            //   location; this is possible if the in-color of this inst is
+            //   equal to the out-color of the producing inst, so no other
+            //   side-effecting ops occur between them (which will only be true
+            //   if they are in the same BB, because color increments at each BB
+            //   start).
+            //
+            //   If it is actually sunk, then in `merge_inst()`, we update the
+            //   scan color so that as we scan over the range past which the
+            //   instruction was sunk, we allow other instructions (that came
+            //   prior to the sunk instruction) to sink.
+            ValueDef::Result(src_inst, result_idx) => {
+                let src_side_effect = has_lowering_side_effect(self.f, src_inst);
+                log::trace!(" -> src inst {}", src_inst);
+                log::trace!(" -> has lowering side effect: {}", src_side_effect);
+                if !src_side_effect {
+                    // Pure instruction: always possible to sink.
+                    Some((src_inst, result_idx))
+                } else {
+                    // Side-effect: test whether this is the only use of the
+                    // only result of the instruction, and whether colors allow
+                    // the code-motion.
+                    if self.cur_scan_entry_color.is_some()
+                        && self.value_uses[val] == 1
+                        && self.value_lowered_uses[val] == 0
+                        && self.num_outputs(src_inst) == 1
+                        && self
+                            .side_effect_inst_entry_colors
+                            .get(&src_inst)
+                            .unwrap()
+                            .get()
+                            + 1
+                            == self.cur_scan_entry_color.unwrap().get()
+                    {
+                        Some((src_inst, 0))
+                    } else {
+                        None
+                    }
+                }
+            }
+            _ => None,
+        };
+        let constant = inst.and_then(|(inst, _)| self.get_constant(inst));
+
+        NonRegInput { inst, constant }
+    }
+
     fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs<Reg> {
         let val = self.f.dfg.inst_args(ir_inst)[idx];
-        let val = self.f.dfg.resolve_aliases(val);
         self.put_value_in_regs(val)
     }
 
+    fn put_value_in_regs(&mut self, val: Value) -> ValueRegs<Reg> {
+        let val = self.f.dfg.resolve_aliases(val);
+        log::trace!("put_value_in_regs: val {}", val);
+
+        // If the value is a constant, then (re)materialize it at each use. This
+        // lowers register pressure.
+        if let Some(c) = self
+            .f
+            .dfg
+            .value_def(val)
+            .inst()
+            .and_then(|inst| self.get_constant(inst))
+        {
+            let ty = self.f.dfg.value_type(val);
+
+            let regs = self.alloc_tmp(ty);
+            log::trace!(" -> regs {:?}", regs);
+            assert!(regs.is_valid());
+
+            let insts = I::gen_constant(regs, c.into(), ty, |ty| {
+                self.alloc_tmp(ty).only_reg().unwrap()
+            });
+            for inst in insts {
+                self.emit(inst);
+            }
+            return non_writable_value_regs(regs);
+        }
+
+        let mut regs = self.value_regs[val];
+        log::trace!(" -> regs {:?}", regs);
+        assert!(regs.is_valid());
+
+        self.value_lowered_uses[val] += 1;
+
+        // Pinned-reg hack: if backend specifies a fixed pinned register, use it
+        // directly when we encounter a GetPinnedReg op, rather than lowering
+        // the actual op, and do not return the source inst to the caller; the
+        // value comes "out of the ether" and we will not force generation of
+        // the superfluous move.
+        if let ValueDef::Result(i, 0) = self.f.dfg.value_def(val) {
+            if self.f.dfg[i].opcode() == Opcode::GetPinnedReg {
+                if let Some(pr) = self.pinned_reg {
+                    regs = ValueRegs::one(pr);
+                }
+            }
+        }
+
+        regs
+    }
+
     fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs<Writable<Reg>> {
         let val = self.f.dfg.inst_results(ir_inst)[idx];
         writable_value_regs(self.value_regs[val])
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
new file mode 100644
index 0000000000..07cc34e87e
--- /dev/null
+++ b/cranelift/codegen/src/prelude.isle
@@ -0,0 +1,202 @@
+;; This is a prelude of standard definitions for ISLE, the instruction-selector
+;; DSL, as we use it bound to our interfaces.
+
+;;;; Primitive and External Types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; `()`
+(type Unit (primitive Unit))
+
+;; `bool` is declared in `clif.isle`.
+(extern const $true bool)
+(extern const $false bool)
+
+(type u8 (primitive u8))
+(type u16 (primitive u16))
+(type u32 (primitive u32))
+(type u64 (primitive u64))
+(type u128 (primitive u128))
+(type usize (primitive usize))
+
+(type i8 (primitive i8))
+(type i16 (primitive i16))
+(type i32 (primitive i32))
+(type i64 (primitive i64))
+(type i128 (primitive i128))
+(type isize (primitive isize))
+
+;; `cranelift-entity`-based identifiers.
+(type Inst (primitive Inst))
+(type Type (primitive Type))
+(type Value (primitive Value))
+
+;; ISLE representation of `&[Value]`.
+(type ValueSlice (primitive ValueSlice))
+
+(type ValueList (primitive ValueList))
+(type ValueRegs (primitive ValueRegs))
+
+;;;; Registers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(type Reg (primitive Reg))
+(type WritableReg (primitive WritableReg))
+
+;; Construct a `ValueRegs` of one register.
+(decl value_reg (Reg) ValueRegs)
+(extern constructor value_reg value_reg)
+
+;; Construct a `ValueRegs` of two registers.
+(decl value_regs (Reg Reg) ValueRegs)
+(extern constructor value_regs value_regs)
+
+;; Get a temporary register for writing.
+(decl temp_writable_reg (Type) WritableReg)
+(extern constructor temp_writable_reg temp_writable_reg)
+
+;; Get a temporary register for reading.
+(decl temp_reg (Type) Reg)
+(rule (temp_reg ty)
+      (writable_reg_to_reg (temp_writable_reg ty)))
+
+;; Get the invalid register.
+(decl invalid_reg () Reg)
+(extern constructor invalid_reg invalid_reg)
+
+;; Put the given value into a register.
+;;
+;; Asserts that the value fits into a single register, and doesn't require
+;; multiple registers for its representation (like `i128` on x64 for example).
+;;
+;; As a side effect, this marks the value as used.
+(decl put_in_reg (Value) Reg)
+(extern constructor put_in_reg put_in_reg)
+
+;; Put the given value into one or more registers.
+;;
+;; As a side effect, this marks the value as used.
+(decl put_in_regs (Value) ValueRegs)
+(extern constructor put_in_regs put_in_regs)
+
+;; Get the `n`th register inside a `ValueRegs`.
+(decl value_regs_get (ValueRegs usize) Reg)
+(extern constructor value_regs_get value_regs_get)
+
+;; Put the value into one or more registers and return the first register.
+;;
+;; Unlike `put_in_reg`, this does not assert that the value fits in a single
+;; register. This is useful for things like a `i128` shift amount, where we mask
+;; the shift amount to the bit width of the value being shifted, and so the high
+;; half of the `i128` won't ever be used.
+;;
+;; As a side efect, this marks that value as used.
+(decl lo_reg (Value) Reg)
+(rule (lo_reg val)
+      (let ((regs ValueRegs (put_in_regs val)))
+        (value_regs_get regs 0)))
+
+;;;; Primitive Type Conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl u8_as_u64 (u8) u64)
+(extern constructor u8_as_u64 u8_as_u64)
+
+(decl u16_as_u64 (u16) u64)
+(extern constructor u16_as_u64 u16_as_u64)
+
+(decl u32_as_u64 (u32) u64)
+(extern constructor u32_as_u64 u32_as_u64)
+
+;;;; `cranelift_codegen::ir::Type` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(extern const $B1 Type)
+(extern const $B8 Type)
+(extern const $B16 Type)
+(extern const $B32 Type)
+(extern const $B64 Type)
+(extern const $B128 Type)
+
+(extern const $I8 Type)
+(extern const $I16 Type)
+(extern const $I32 Type)
+(extern const $I64 Type)
+(extern const $I128 Type)
+
+(extern const $B8X16 Type)
+(extern const $B16X8 Type)
+(extern const $B32X4 Type)
+(extern const $B64X2 Type)
+
+(extern const $I8X16 Type)
+(extern const $I16X8 Type)
+(extern const $I32X4 Type)
+(extern const $I64X2 Type)
+
+(extern const $F32X4 Type)
+(extern const $F64X2 Type)
+
+;; Get the bit width of a given type.
+(decl ty_bits (Type) u16)
+(extern constructor ty_bits ty_bits)
+
+;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; An extractor that only matches types that can fit in 64 bits.
+(decl fits_in_64 (Type) Type)
+(extern extractor fits_in_64 fits_in_64)
+
+;; Extractor to get a `ValueSlice` out of a `ValueList`.
+(decl value_list_slice (ValueSlice) ValueList)
+(extern extractor infallible value_list_slice value_list_slice)
+
+;; Extractor to get the first element from a value list, along with its tail as
+;; a `ValueSlice`.
+(decl unwrap_head_value_list_1 (Value ValueSlice) ValueList)
+(extern extractor infallible unwrap_head_value_list_1 unwrap_head_value_list_1)
+
+;; Extractor to get the first two elements from a value list, along with its
+;; tail as a `ValueSlice`.
+(decl unwrap_head_value_list_2 (Value Value ValueSlice) ValueList)
+(extern extractor infallible unwrap_head_value_list_2 unwrap_head_value_list_2)
+
+;; Turn a `Writable<Reg>` into a `Reg` via `Writable::to_reg`.
+(decl writable_reg_to_reg (WritableReg) Reg)
+(extern constructor writable_reg_to_reg writable_reg_to_reg)
+
+;; Extract a `u64` from an `Imm64`.
+(decl u64_from_imm64 (u64) Imm64)
+(extern extractor infallible u64_from_imm64 u64_from_imm64)
+
+;; Extract the result values for the given instruction.
+(decl inst_results (ValueSlice) Inst)
+(extern extractor infallible inst_results inst_results)
+
+;; Extract the first result value of the given instruction.
+(decl first_result (Value) Inst)
+(extern extractor first_result first_result)
+
+;; Extract the `InstructionData` for an `Inst`.
+(decl inst_data (InstructionData) Inst)
+(extern extractor infallible inst_data inst_data)
+
+;; Extract the type of a `Value`.
+(decl value_type (Type) Value)
+(extern extractor infallible value_type value_type)
+
+;; Extract the type of the instruction's first result.
+(decl result_type (Type) Inst)
+(extractor (result_type ty)
+           (first_result (value_type ty)))
+
+;; Extract the type of the instruction's first result and pass along the
+;; instruction as well.
+(decl has_type (Type Inst) Inst)
+(extractor (has_type ty inst)
+           (and (result_type ty)
+                inst))
+
+;; Match a multi-lane type, extracting (# bits per lane, # lanes) from the given
+;; type. Will only match when there is more than one lane.
+(decl multi_lane (u8 u16) Type)
+(extern extractor multi_lane multi_lane)
+
+;; Match the instruction that defines the given value, if any.
+(decl def_inst (Inst) Value)
+(extern extractor def_inst def_inst)
diff --git a/cranelift/entity/src/list.rs b/cranelift/entity/src/list.rs
index d37ea8ae09..d4a057bf4e 100644
--- a/cranelift/entity/src/list.rs
+++ b/cranelift/entity/src/list.rs
@@ -62,7 +62,7 @@ use serde::{Deserialize, Serialize};
 ///
 /// The index stored in an `EntityList` points to part 2, the list elements. The value 0 is
 /// reserved for the empty list which isn't allocated in the vector.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct EntityList<T: EntityRef + ReservedValue> {
     index: u32,
@@ -271,7 +271,7 @@ impl<T: EntityRef + ReservedValue> EntityList<T> {
     }
 
     /// Get the list as a slice.
-    pub fn as_slice<'a>(&'a self, pool: &'a ListPool<T>) -> &'a [T] {
+    pub fn as_slice<'a>(&self, pool: &'a ListPool<T>) -> &'a [T] {
         let idx = self.index as usize;
         match pool.len_of(self) {
             None => &[],
diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif
index 96520af906..5066404fe0 100644
--- a/cranelift/filetests/filetests/isa/x64/i128.clif
+++ b/cranelift/filetests/filetests/isa/x64/i128.clif
@@ -122,18 +122,14 @@ block0(v0: i128, v1: i128):
 
     v2 = imul v0, v1
 ; nextln:  movq    %rsi, %rax
-; nextln:  movq    %rcx, %r8
 ; nextln:  movq    %rdi, %rsi
-; nextln:  imulq   %rdx, %rsi
-; nextln:  movq    %rdi, %rcx
-; nextln:  imulq   %r8, %rcx
+; nextln:  imulq   %rcx, %rsi
 ; nextln:  imulq   %rdx, %rax
-; nextln:  addq    %rax, %rcx
+; nextln:  addq    %rax, %rsi
 ; nextln:  movq    %rdi, %rax
 ; nextln:  mul     %rdx
-; nextln:  addq    %rdx, %rcx
-; nextln:  movq    %rsi, %rax
-; nextln:  movq    %rcx, %rdx
+; nextln:  addq    %rdx, %rsi
+; nextln:  movq    %rsi, %rdx
 
     return v2
 ; nextln:  movq    %rbp, %rsp
@@ -700,34 +696,35 @@ block2(v6: i128):
     v8 = iadd.i128 v6, v7
     return v8
 
-; check: pushq   %rbp
-; nextln: movq    %rsp, %rbp
-; nextln: testb   $$1, %dl
-; nextln: jnz     label1; j label2
+; check: Block 0:
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+; nextln:  testb   $$1, %dl
+; nextln:  jnz     label1; j label2
 ; check: Block 1:
-; check:  movl    $$0, %esi
-; nextln: movl    $$0, %edi
-; nextln: movl    $$1, %eax
-; nextln: movl    $$0, %ecx
-; nextln: addq    %rax, %rsi
-; nextln: adcq    %rcx, %rdi
-; nextln: movq    %rsi, %rax
-; nextln: movq    %rdi, %rdx
-; nextln: movq    %rbp, %rsp
-; nextln: popq    %rbp
-; nextln: ret
+; check:   xorq    %rdi, %rdi
+; nextln:  xorq    %rsi, %rsi
+; nextln:  movl    $$1, %ecx
+; nextln:  xorq    %rax, %rax
+; nextln:  addq    %rcx, %rdi
+; nextln:  adcq    %rax, %rsi
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
 ; check: Block 2:
-; check:  movl    $$0, %esi
-; nextln: movl    $$0, %edi
-; nextln: movl    $$2, %eax
-; nextln: movl    $$0, %ecx
-; nextln: addq    %rax, %rsi
-; nextln: adcq    %rcx, %rdi
-; nextln: movq    %rsi, %rax
-; nextln: movq    %rdi, %rdx
-; nextln: movq    %rbp, %rsp
-; nextln: popq    %rbp
-; nextln: ret
+; check:   xorq    %rdi, %rdi
+; nextln:  xorq    %rsi, %rsi
+; nextln:  movl    $$2, %ecx
+; nextln:  xorq    %rax, %rax
+; nextln:  addq    %rcx, %rdi
+; nextln:  adcq    %rax, %rsi
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
 
 }
 
@@ -744,34 +741,32 @@ block0(v0: i128, v1: i128, v2: i64, v3: i128, v4: i128, v5: i128):
 
 ; check:  pushq   %rbp
 ; nextln: movq    %rsp, %rbp
-; nextln: subq    $$32, %rsp
+; nextln: subq    $$16, %rsp
 ; nextln: movq    %r12, 0(%rsp)
 ; nextln: movq    %r13, 8(%rsp)
-; nextln: movq    %r14, 16(%rsp)
-; nextln: movq    %r8, %r14
-; nextln: movq    16(%rbp), %r10
+; nextln: movq    %r9, %r11
+; nextln: movq    16(%rbp), %r13
 ; nextln: movq    24(%rbp), %r12
-; nextln: movq    32(%rbp), %r11
-; nextln: movq    40(%rbp), %rax
-; nextln: movq    48(%rbp), %r13
-; nextln: movq    %rsi, %r8
+; nextln: movq    32(%rbp), %r10
+; nextln: movq    40(%rbp), %r9
+; nextln: movq    48(%rbp), %rax
 ; nextln: addq    %rdx, %rdi
-; nextln: adcq    %rcx, %r8
+; nextln: movq    %rsi, %rdx
+; nextln: adcq    %rcx, %rdx
 ; nextln: xorq    %rsi, %rsi
-; nextln: addq    %r14, %r9
-; nextln: adcq    %rsi, %r10
-; nextln: addq    %rax, %r12
-; nextln: adcq    %r13, %r11
-; nextln: addq    %r9, %rdi
-; nextln: adcq    %r10, %r8
+; nextln: addq    %r8, %r11
+; nextln: adcq    %rsi, %r13
+; nextln: addq    %r9, %r12
+; nextln: adcq    %rax, %r10
+; nextln: addq    %r11, %rdi
+; nextln: adcq    %r13, %rdx
 ; nextln: addq    %rdi, %r12
-; nextln: adcq    %r8, %r11
+; nextln: adcq    %rdx, %r10
 ; nextln: movq    %r12, %rax
-; nextln: movq    %r11, %rdx
+; nextln: movq    %r10, %rdx
 ; nextln: movq    0(%rsp), %r12
 ; nextln: movq    8(%rsp), %r13
-; nextln: movq    16(%rsp), %r14
-; nextln: addq    $$32, %rsp
+; nextln: addq    $$16, %rsp
 ; nextln: movq    %rbp, %rsp
 ; nextln: popq    %rbp
 ; nextln: ret
@@ -907,26 +902,25 @@ block0(v0: i128, v1: i128):
 
 ; check:  pushq   %rbp
 ; nextln: movq    %rsp, %rbp
-; nextln: movq    %rsi, %rax
-; nextln: movq    %rdi, %rsi
+; nextln: movq    %rdi, %rax
+; nextln: movq    %rsi, %rdi
+; nextln: movq    %rax, %rsi
 ; nextln: movq    %rdx, %rcx
 ; nextln: shlq    %cl, %rsi
 ; nextln: movq    %rdx, %rcx
-; nextln: shlq    %cl, %rax
+; nextln: shlq    %cl, %rdi
 ; nextln: movl    $$64, %ecx
 ; nextln: subq    %rdx, %rcx
-; nextln: shrq    %cl, %rdi
+; nextln: shrq    %cl, %rax
 ; nextln: xorq    %rcx, %rcx
 ; nextln: testq   $$127, %rdx
-; nextln: cmovzq  %rcx, %rdi
-; nextln: orq     %rax, %rdi
-; nextln: xorq    %rax, %rax
-; nextln: andq    $$64, %rdx
-; nextln: cmovzq  %rdi, %rax
+; nextln: cmovzq  %rcx, %rax
+; nextln: orq     %rdi, %rax
+; nextln: testq   $$64, %rdx
 ; nextln: cmovzq  %rsi, %rcx
-; nextln: cmovnzq %rsi, %rax
-; nextln: movq    %rax, %rdx
+; nextln: cmovzq  %rax, %rsi
 ; nextln: movq    %rcx, %rax
+; nextln: movq    %rsi, %rdx
 ; nextln: movq    %rbp, %rsp
 ; nextln: popq    %rbp
 ; nextln: ret
@@ -939,28 +933,26 @@ block0(v0: i128, v1: i128):
 
 ; check:  pushq   %rbp
 ; nextln: movq    %rsp, %rbp
-; nextln: movq    %rdi, %rax
-; nextln: movq    %rsi, %rdi
-; nextln: movq    %rdi, %rsi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rdx, %rcx
+; nextln: shrq    %cl, %rdi
+; nextln: movq    %rax, %rsi
 ; nextln: movq    %rdx, %rcx
 ; nextln: shrq    %cl, %rsi
-; nextln: movq    %rdx, %rcx
-; nextln: shrq    %cl, %rax
 ; nextln: movl    $$64, %ecx
 ; nextln: subq    %rdx, %rcx
-; nextln: shlq    %cl, %rdi
+; nextln: shlq    %cl, %rax
 ; nextln: xorq    %rcx, %rcx
 ; nextln: testq   $$127, %rdx
-; nextln: cmovzq  %rcx, %rdi
-; nextln: orq     %rax, %rdi
-; nextln: xorq    %rax, %rax
+; nextln: cmovzq  %rcx, %rax
+; nextln: orq     %rdi, %rax
 ; nextln: xorq    %rcx, %rcx
-; nextln: andq    $$64, %rdx
-; nextln: cmovzq  %rsi, %rax
-; nextln: cmovzq  %rdi, %rcx
-; nextln: cmovnzq %rsi, %rcx
-; nextln: movq    %rax, %rdx
-; nextln: movq    %rcx, %rax
+; nextln: testq   $$64, %rdx
+; nextln: movq    %rsi, %rdi
+; nextln: cmovzq  %rax, %rdi
+; nextln: cmovzq  %rsi, %rcx
+; nextln: movq    %rdi, %rax
+; nextln: movq    %rcx, %rdx
 ; nextln: movq    %rbp, %rsp
 ; nextln: popq    %rbp
 ; nextln: ret
@@ -1006,53 +998,51 @@ block0(v0: i128, v1: i128):
     return v2
 }
 
-; check: pushq   %rbp
+; check:  pushq   %rbp
 ; nextln: movq    %rsp, %rbp
-; nextln: movq    %rdi, %r8
-; nextln: movq    %r8, %r9
-; nextln: movq    %rdx, %rcx
-; nextln: shlq    %cl, %r9
-; nextln: movq    %rsi, %rax
+; nextln: movq    %rdi, %rax
 ; nextln: movq    %rdx, %rcx
 ; nextln: shlq    %cl, %rax
+; nextln: movq    %rsi, %r8
+; nextln: movq    %rdx, %rcx
+; nextln: shlq    %cl, %r8
 ; nextln: movl    $$64, %ecx
 ; nextln: subq    %rdx, %rcx
-; nextln: movq    %r8, %r10
-; nextln: shrq    %cl, %r10
-; nextln: xorq    %rdi, %rdi
+; nextln: movq    %rdi, %r9
+; nextln: shrq    %cl, %r9
+; nextln: xorq    %rcx, %rcx
 ; nextln: testq   $$127, %rdx
-; nextln: cmovzq  %rdi, %r10
-; nextln: orq     %rax, %r10
-; nextln: xorq    %rax, %rax
-; nextln: movq    %rdx, %rcx
-; nextln: andq    $$64, %rcx
-; nextln: cmovzq  %r10, %rax
-; nextln: cmovzq  %r9, %rdi
-; nextln: cmovnzq %r9, %rax
+; nextln: cmovzq  %rcx, %r9
+; nextln: orq     %r8, %r9
+; nextln: testq   $$64, %rdx
+; nextln: movq    %rcx, %r8
+; nextln: cmovzq  %rax, %r8
+; nextln: cmovzq  %r9, %rax
 ; nextln: movl    $$128, %r9d
 ; nextln: subq    %rdx, %r9
-; nextln: movq    %rsi, %rdx
+; nextln: movq    %rdi, %rdx
 ; nextln: movq    %r9, %rcx
 ; nextln: shrq    %cl, %rdx
+; nextln: movq    %rsi, %rdi
 ; nextln: movq    %r9, %rcx
-; nextln: shrq    %cl, %r8
+; nextln: shrq    %cl, %rdi
 ; nextln: movl    $$64, %ecx
 ; nextln: subq    %r9, %rcx
 ; nextln: shlq    %cl, %rsi
 ; nextln: xorq    %rcx, %rcx
 ; nextln: testq   $$127, %r9
 ; nextln: cmovzq  %rcx, %rsi
-; nextln: orq     %r8, %rsi
-; nextln: xorq    %rcx, %rcx
-; nextln: xorq    %r8, %r8
-; nextln: andq    $$64, %r9
-; nextln: cmovzq  %rdx, %rcx
-; nextln: cmovzq  %rsi, %r8
-; nextln: cmovnzq %rdx, %r8
-; nextln: orq     %rdi, %r8
-; nextln: orq     %rax, %rcx
+; nextln: orq     %rdx, %rsi
+; nextln: xorq    %rdx, %rdx
+; nextln: testq   $$64, %r9
+; nextln: movq    %rdi, %rcx
+; nextln: cmovzq  %rsi, %rcx
+; nextln: movq    %rdx, %rsi
+; nextln: cmovzq  %rdi, %rsi
+; nextln: orq     %rcx, %r8
+; nextln: orq     %rsi, %rax
+; nextln: movq    %rax, %rdx
 ; nextln: movq    %r8, %rax
-; nextln: movq    %rcx, %rdx
 ; nextln: movq    %rbp, %rsp
 ; nextln: popq    %rbp
 ; nextln: ret
diff --git a/cranelift/isle/.gitignore b/cranelift/isle/.gitignore
new file mode 100644
index 0000000000..3110c83344
--- /dev/null
+++ b/cranelift/isle/.gitignore
@@ -0,0 +1,3 @@
+/target
+*~
+.*.swp
diff --git a/cranelift/isle/README.md b/cranelift/isle/README.md
new file mode 100644
index 0000000000..635fe863f8
--- /dev/null
+++ b/cranelift/isle/README.md
@@ -0,0 +1,530 @@
+# ISLE: Instruction Selection/Lowering Expressions DSL
+
+## Table of Contents
+
+* [Introduction](#introduction)
+* [Example Usage](#example-usage)
+* [Tutorial](#tutorial)
+* [Implementation](#implementation)
+* [Sketch of Instruction Selector](#sketch-of-instruction-selector)
+
+## Introduction
+
+ISLE is a DSL that allows one to write instruction-lowering rules for a
+compiler backend. It is based on a "term-rewriting" paradigm in which the input
+-- some sort of compiler IR -- is, conceptually, a tree of terms, and we have a
+set of rewrite rules that turn this into another tree of terms.
+
+This repository contains a prototype meta-compiler that compiles ISLE rules
+down to an instruction selector implementation in generated Rust code. The
+generated code operates efficiently in a single pass over the input, and merges
+all rules into a decision tree, sharing work where possible, while respecting
+user-configurable priorities on each rule.
+
+The ISLE language is designed so that the rules can both be compiled into an
+efficient compiler backend and can be used in formal reasoning about the
+compiler. The compiler in this repository implements the former. The latter
+use-case is future work and outside the scope of this prototype, but at a high
+level, the rules can be seen as simple equivalences between values in two
+languages, and so should be translatable to formal constraints or other logical
+specification languages.
+
+Some more details and motivation are in [BA RFC
+#15](https://github.com/bytecodealliance/rfcs/pull/15); additional
+documentation will eventually be added to carefully specify the language
+semantics.
+
+## Example Usage
+
+Build `islec`, the ISLE compiler:
+
+```shell
+$ cargo build --release
+```
+
+Compile a `.isle` source file into Rust code:
+
+```shell
+$ target/release/islec -i isle_examples/test.isle -o isle_examples/test.rs
+```
+
+Include that Rust code in your crate and compile it:
+
+```shell
+$ rustc isle_examples/test_main.rs
+```
+
+## Tutorial
+
+This tutorial walks through defining an instruction selection and lowering pass
+for a simple, RISC-y, high-level IR down to low-level, CISC-y machine
+instructions. It is intentionally somewhat similar to CLIF to MachInst lowering,
+although it restricts the input and output languages to only adds, loads, and
+constants so that we can focus on ISLE itself.
+
+> The full ISLE source code for this tutorial is available at
+> `isle_examples/tutorial.isle`.
+
+The ISLE language is based around rules for translating a term (i.e. expression)
+into another term. Terms are typed, so before we can write rules for translating
+some type of term into another type of term, we have to define those types:
+
+```lisp
+;; Declare that we are using the `i32` primitive type from Rust.
+(type i32 (primitive i32))
+
+;; Our high-level, RISC-y input IR.
+(type HighLevelInst
+  (enum (Add (a Value) (b Value))
+        (Load (addr Value))
+        (Const (c i32))))
+
+;; A value in our high-level IR is a Rust `Copy` type. Values are either defined
+;; by an instruction, or are a basic block argument.
+(type Value (primitive Value))
+
+;; Our low-level, CISC-y machine instructions.
+(type LowLevelInst
+  (enum (Add (mode AddrMode))
+        (Load (offset i32) (addr Reg))
+        (Const (c i32))))
+
+;; Different kinds of addressing modes for operands to our low-level machine
+;; instructions.
+(type AddrMode
+  (enum
+    ;; Both operands in registers.
+    (RegReg (a Reg) (b Reg))
+    ;; The destination/first operand is a register; the second operand is in
+    ;; memory at `[b + offset]`.
+    (RegMem (a Reg) (b Reg) (offset i32))
+    ;; The destination/first operand is a register, second operand is an
+    ;; immediate.
+    (RegImm (a Reg) (imm i32))))
+
+;; The register type is a Rust `Copy` type.
+(type Reg (primitive Reg))
+```
+
+Now we can start writing some basic lowering rules! We declare the top-level
+lowering function (a "constructor term" in ISLE terminology) and attach rules to
+it. The simplest case is matching a high-level `Const` instruction and lowering
+that to a low-level `Const` instruction, since there isn't any translation we
+really have to do.
+
+```lisp
+;; Declare our top-level lowering function. We will attach rules to this
+;; declaration for lowering various patterns of `HighLevelInst` inputs.
+(decl lower (HighLevelInst) LowLevelInst)
+
+;; Simple rule for lowering constants.
+(rule (lower (HighLevelInst.Const c))
+      (LowLevelInst.Const c))
+```
+
+Each rule has the form `(rule <left-hand side> <right-hand-side>)`. The
+left-hand side (LHS) is a *pattern* and the right-hand side (RHS) is an
+*expression*. When the LHS pattern matches the input, then we evaluate the RHS
+expression. The LHS pattern can bind variables from the input that are then
+available in the right-hand side. For example, in our `Const`-lowering rule, the
+variable `c` is bound from the LHS and then reused in the RHS.
+
+Now we can compile this code by running
+
+```shell
+$ islec isle_examples/tutorial.isle
+```
+
+and we'll get the following output <sup>(ignoring any minor code generation
+changes in the future)</sup>:
+
+```rust
+// GENERATED BY ISLE. DO NOT EDIT!
+//
+// Generated automatically from the instruction-selection DSL code in:
+// - isle_examples/tutorial.isle
+
+// [Type and `Context` definitions removed for brevity...]
+
+// Generated as internal constructor for term lower.
+pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: &HighLevelInst) -> Option<LowLevelInst> {
+    let pattern0_0 = arg0;
+    if let &HighLevelInst::Const { c: pattern1_0 } = pattern0_0 {
+        // Rule at isle_examples/tutorial.isle line 45.
+        let expr0_0 = LowLevelInst::Const {
+            c: pattern1_0,
+        };
+        return Some(expr0_0);
+    }
+    return None;
+}
+```
+
+There are a few things to notice about this generated Rust code:
+
+* The `lower` constructor term becomes the `constructor_lower` function in the
+  generated code.
+
+* The function returns a value of type `Option<LowLevelInst>` and returns `None`
+  when it doesn't know how to lower an input `HighLevelInst`. This is useful for
+  incrementally porting hand-written lowering code to ISLE.
+
+* There is a helpful comment documenting where in the ISLE source code a rule
+  was defined. The goal is to ISLE more transparent and less magical.
+
+* The code is parameterized by a type that implements a `Context`
+  trait. Implementing this trait is how you glue the generated code into your
+  compiler. Right now this is an empty trait; more on `Context` later.
+
+* Lastly, and most importantly, this generated Rust code is basically what we
+  would have written by hand to do the same thing, other than things like
+  variable names. It checks if the input is a `Const`, and if so, translates it
+  into a `LowLevelInst::Const`.
+
+Okay, one rule isn't very impressive, but in order to start writing more rules
+we need to be able to put the result of a lowered instruction into a `Reg`. This
+might internally have to do arbitrary things like update use counts or anything
+else that Cranelift's existing `LowerCtx::put_input_in_reg` does for different
+target architectures. To allow for plugging in this kind of arbitrary logic,
+ISLE supports *external constructors*. These end up as methods of the `Context`
+trait in the generated Rust code, and you can implement them however you want
+with custom Rust code.
+
+Here is how we declare an external helper to put a value into a register:
+
+```lisp
+;; Declare an external constructor that puts a high-level `Value` into a
+;; low-level `Reg`.
+(decl put_in_reg (Value) Reg)
+(extern constructor put_in_reg put_in_reg)
+```
+
+If we rerun `islec` on our ISLE source, instead of an empty `Context` trait, now
+we will get this trait definition:
+
+```rust
+pub trait Context {
+    fn put_in_reg(&mut self, arg0: Value) -> (Reg,);
+}
+```
+
+With the `put_in_reg` helper available, we can define rules for lowering loads
+and adds:
+
+```lisp
+;; Simple rule for lowering adds.
+(rule (lower (HighLevelInst.Add a b))
+      (LowLevelInst.Add
+        (AddrMode.RegReg (put_in_reg a) (put_in_reg b))))
+
+;; Simple rule for lowering loads.
+(rule (lower (HighLevelInst.Load addr))
+      (LowLevelInst.Load 0 (put_in_reg addr)))
+```
+
+If we compile our ISLE source into Rust code once again, the generated code for
+`lower` now looks like this:
+
+```rust
+// Generated as internal constructor for term lower.
+pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: &HighLevelInst) -> Option<LowLevelInst> {
+    let pattern0_0 = arg0;
+    match pattern0_0 {
+        &HighLevelInst::Const { c: pattern1_0 } => {
+            // Rule at isle_examples/tutorial.isle line 45.
+            let expr0_0 = LowLevelInst::Const {
+                c: pattern1_0,
+            };
+            return Some(expr0_0);
+        }
+        &HighLevelInst::Load { addr: pattern1_0 } => {
+            // Rule at isle_examples/tutorial.isle line 59.
+            let expr0_0: i32 = 0;
+            let expr1_0 = C::put_in_reg(ctx, pattern1_0);
+            let expr2_0 = LowLevelInst::Load {
+                offset: expr0_0,
+                addr: expr1_0,
+            };
+            return Some(expr2_0);
+        }
+        &HighLevelInst::Add { a: pattern1_0, b: pattern1_1 } => {
+            // Rule at isle_examples/tutorial.isle line 54.
+            let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+            let expr1_0 = C::put_in_reg(ctx, pattern1_1);
+            let expr2_0 = AddrMode::RegReg {
+                a: expr0_0,
+                b: expr1_0,
+            };
+            let expr3_0 = LowLevelInst::Add {
+                mode: expr2_0,
+            };
+            return Some(expr3_0);
+        }
+        _ => {}
+    }
+    return None;
+}
+```
+
+As you can see, each of our rules was collapsed into a single, efficient `match`
+expression. Just like we would have otherwise written by hand. And wherever we
+need to get a high-level operand as a low-level register, there is a call to the
+`Context::put_in_reg` trait method, allowing us to hook whatever arbitrary logic
+we need to when putting a value into a register when we implement the `Context`
+trait.
+
+Things start to get more interesting when we want to do things like sink a load
+into the add's addressing mode. This is only desirable when our add is the only
+use of the loaded value. Furthermore, it is only valid to do when there isn't
+any store that might write to the same address we are loading from in between
+the load and the add. Otherwise, moving the load across the store could result
+in a miscompilation where we load the wrong value to add:
+
+```text
+x = load addr
+store 42 -> addr
+y = add x, 1
+
+==/==>
+
+store 42 -> addr
+x = load addr
+y = add x, 1
+```
+
+We can encode these kinds of preconditions in an *external extractor*. An
+extractor is like our regular constructor functions, but it is used inside LHS
+patterns, rather than RHS expressions, and its arguments and results flipped
+around: instead of taking arguments and producing results, it takes a result and
+(fallibly) produces the arguments. This allows us to write custom preconditions
+for matching code.
+
+Let's make this more clear with a concrete example. Here is the declaration of
+an external extractor to match on the high-level instruction that defined a
+given operand `Value`, along with a new rule to sink loads into adds:
+
+```lisp
+;; Declare an external extractor for extracting the instruction that defined a
+;; given operand value.
+(decl inst_result (HighLevelInst) Value)
+(extern extractor inst_result inst_result)
+
+;; Rule to sink loads into adds.
+(rule (lower (HighLevelInst.Add a (inst_result (HighLevelInst.Load addr))))
+      (LowLevelInst.Add
+        (AddrMode.RegMem (put_in_reg a)
+                         (put_in_reg addr)
+                         0)))
+```
+
+Note that the operand `Value` passed into this extractor might be a basic block
+parameter, in which case there is no such instruction. Or there might be a store
+or function call instruction in between the current instruction and the
+instruction that defines the given operand value, in which case we want to
+"hide" the instruction so that we don't illegally sink loads into adds they
+shouldn't be sunk into. So this extractor might fail to return an instruction
+for a given operand `Value`.
+
+If we recompile our ISLE source into Rust code once again, we see a new
+`inst_result` method defined on our `Context` trait, we notice that its
+arguments and returns are flipped around from the `decl` in the ISLE source
+because it is an extractor, and finally that it returns an `Option` because it
+isn't guaranteed that we can extract a defining instruction for the given
+operand `Value`:
+
+```rust
+pub trait Context {
+    fn put_in_reg(&mut self, arg0: Value) -> (Reg,);
+    fn inst_result(&mut self, arg0: Value) -> Option<(HighLevelInst,)>;
+}
+```
+
+And if we look at the generated code for our `lower` function, there is a new,
+nested case for sinking loads into adds that uses the `Context::inst_result`
+trait method to see if our new rule can be applied:
+
+```rust
+// Generated as internal constructor for term lower.
+pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: &HighLevelInst) -> Option<LowLevelInst> {
+    let pattern0_0 = arg0;
+    match pattern0_0 {
+        &HighLevelInst::Const { c: pattern1_0 } => {
+            // [...]
+        }
+        &HighLevelInst::Load { addr: pattern1_0 } => {
+            // [...]
+        }
+        &HighLevelInst::Add { a: pattern1_0, b: pattern1_1 } => {
+            if let Some((pattern2_0,)) = C::inst_result(ctx, pattern1_1) {
+                if let &HighLevelInst::Load { addr: pattern3_0 } = &pattern2_0 {
+                    // Rule at isle_examples/tutorial.isle line 68.
+                    let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+                    let expr1_0 = C::put_in_reg(ctx, pattern3_0);
+                    let expr2_0: i32 = 0;
+                    let expr3_0 = AddrMode::RegMem {
+                        a: expr0_0,
+                        b: expr1_0,
+                        offset: expr2_0,
+                    };
+                    let expr4_0 = LowLevelInst::Add {
+                        mode: expr3_0,
+                    };
+                    return Some(expr4_0);
+                }
+            }
+            // Rule at isle_examples/tutorial.isle line 54.
+            let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+            let expr1_0 = C::put_in_reg(ctx, pattern1_1);
+            let expr2_0 = AddrMode::RegReg {
+                a: expr0_0,
+                b: expr1_0,
+            };
+            let expr3_0 = LowLevelInst::Add {
+                mode: expr2_0,
+            };
+            return Some(expr3_0);
+        }
+        _ => {}
+    }
+    return None;
+}
+```
+
+Once again, this is pretty much the code you would have otherwise written by
+hand to sink the load into the add.
+
+At this point we can start defining a whole bunch of even-more-complicated
+lowering rules that do things like take advantage of folding static offsets into
+loads into adds:
+
+```lisp
+;; Rule to sink a load of a base address with a static offset into a single add.
+(rule (lower (HighLevelInst.Add
+               a
+               (inst_result (HighLevelInst.Load
+                              (inst_result (HighLevelInst.Add
+                                             base
+                                             (inst_result (HighLevelInst.Const offset))))))))
+      (LowLevelInst.Add
+        (AddrMode.RegMem (put_in_reg a)
+                         (put_in_reg base)
+                         offset)))
+
+;; Rule for sinking an immediate into an add.
+(rule (lower (HighLevelInst.Add a (inst_result (HighLevelInst.Const c))))
+      (LowLevelInst.Add
+        (AddrMode.RegImm (put_in_reg a) c)))
+
+;; Rule for lowering loads of a base address with a static offset.
+(rule (lower (HighLevelInst.Load
+               (inst_result (HighLevelInst.Add
+                              base
+                              (inst_result (HighLevelInst.Const offset))))))
+      (LowLevelInst.Load offset (put_in_reg base)))
+```
+
+I'm not going to show the generated Rust code for these new rules here because
+it is starting to get a bit too big. But you can compile
+`isle_examples/tutorial.isle` and verify yourself that it generates the code you
+expect it to.
+
+In conclusion, adding new lowering rules is easy with ISLE. And you still get
+that efficient, compact tree of `match` expressions in the generated Rust code
+that you would otherwise write by hand.
+
+## Implementation
+
+This is an overview of `islec`'s passes and data structures:
+
+```text
+    +------------------+
+    | ISLE Source Text |
+    +------------------+
+             |
+             | Lex
+             V
+         +--------+
+         | Tokens |
+         +--------+
+             |
+             | Parse
+             V
+   +----------------------+
+   | Abstract Syntax Tree |
+   +----------------------+
+             |
+             | Semantic Analysis
+             V
++----------------------------+
+| Term and Type Environments |
++----------------------------+
+             |
+             | Trie Construction
+             V
+       +-----------+
+       | Term Trie |
+       +-----------+
+             |
+             | Code Generation
+             V
+    +------------------+
+    | Rust Source Code |
+    +------------------+
+```
+
+### Lexing
+
+Lexing breaks up the input ISLE source text into a stream of tokens. Our lexer
+is pull-based, meaning that we don't eagerly construct the full stream of
+tokens. Instead, we wait until the next token is requested, at which point we
+lazily lex it.
+
+Relevant source files:
+
+* `isle/src/lexer.rs`
+
+### Parsing
+
+Parsing translates the stream of tokens into an abstract syntax tree (AST). Our
+parser is a simple, hand-written, recursive-descent parser.
+
+Relevant source files:
+
+* `isle/src/ast.rs`
+* `isle/src/parser.rs`
+
+### Semantic Analysis
+
+Semantic analysis performs type checking, figures out which rules apply to which
+terms, etc. It creates a type environment and a term environment that we can use
+to get information about our terms throughout the rest of the pipeline.
+
+Relevant source files:
+
+* `isle/src/sema.rs`
+
+### Trie Construction
+
+The trie construction phase linearizes each rule's LHS pattern and inserts them
+into a trie that maps LHS patterns to RHS expressions. This trie is the skeleton
+of the decision tree that will be emitted during code generation.
+
+Relevant source files:
+
+* `isle/src/ir.rs`
+* `isle/src/trie.rs`
+
+### Code Generation
+
+Code generation takes in the term trie and emits Rust source code that
+implements it.
+
+Relevant source files:
+
+* `isle/src/codegen.rs`
+
+## Sketch of Instruction Selector
+
+Please see [this Cranelift
+branch](https://github.com/cfallin/wasmtime/tree/isle) for an ongoing sketch of
+an instruction selector backend in Cranelift that uses ISLE.
diff --git a/cranelift/isle/TODO b/cranelift/isle/TODO
new file mode 100644
index 0000000000..dfd7fce33b
--- /dev/null
+++ b/cranelift/isle/TODO
@@ -0,0 +1,22 @@
+- Document the semantics of the DSL!
+
+- Clean up and factor the codegen properly.
+
+- Get rid of the expression syntax `<EXPR` in patterns; do a type-dependent
+  parse instead where we know the polarity of pattern-term args and parse
+  in-args as exprs.
+
+- Look into whether optimizations are possible:
+  - More in-depth fallibility analysis (avoid failure edges where possible)
+
+- Slightly nicer human-readable generated code
+  - Include full rule body (S-expression) in comment, not just line number
+  - Inline some expressions (no more `let val23 = 1234; ... f(val23);`)
+
+- Build inlining and simplification: inline invocations of internal
+  constructors, and eliminate ctor-etor or makevariant-matchvariant pairs.
+
+- Ideas from discussion with fitzgen
+  - Turn arg-polarity and exprs on extractors into purer "InstFormat"
+  - Emit two contexts: an immutable context for inputs and a mutable context for
+    outputs
diff --git a/cranelift/isle/fuzz/.gitignore b/cranelift/isle/fuzz/.gitignore
new file mode 100644
index 0000000000..a0925114d6
--- /dev/null
+++ b/cranelift/isle/fuzz/.gitignore
@@ -0,0 +1,3 @@
+target
+corpus
+artifacts
diff --git a/cranelift/isle/fuzz/Cargo.toml b/cranelift/isle/fuzz/Cargo.toml
new file mode 100644
index 0000000000..51a593006c
--- /dev/null
+++ b/cranelift/isle/fuzz/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "isle-fuzz"
+version = "0.0.0"
+authors = ["Automatically generated"]
+publish = false
+edition = "2018"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+env_logger = { version = "0.9.0", default-features = false }
+isle = { path = "../isle" }
+libfuzzer-sys = "0.4"
+log = "0.4.14"
+
+[[bin]]
+name = "compile"
+path = "fuzz_targets/compile.rs"
+test = false
+doc = false
diff --git a/cranelift/isle/fuzz/README.md b/cranelift/isle/fuzz/README.md
new file mode 100644
index 0000000000..a6d695df10
--- /dev/null
+++ b/cranelift/isle/fuzz/README.md
@@ -0,0 +1,4 @@
+# ISLE Fuzz Targets
+
+These are separate from the top-level `wasmtime/fuzz` fuzz targets because we
+don't intend to run them on OSS-Fuzz. They are just for local ISLE hacking.
diff --git a/cranelift/isle/fuzz/fuzz_targets/compile.rs b/cranelift/isle/fuzz/fuzz_targets/compile.rs
new file mode 100644
index 0000000000..8526c29cce
--- /dev/null
+++ b/cranelift/isle/fuzz/fuzz_targets/compile.rs
@@ -0,0 +1,32 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|s: &str| {
+    let _ = env_logger::try_init();
+
+    let lexer = isle::lexer::Lexer::from_str(s, "fuzz-input.isle");
+    log::debug!("lexer = {:?}", lexer);
+    let lexer = match lexer {
+        Ok(l) => l,
+        Err(_) => return,
+    };
+
+    let defs = isle::parser::parse(lexer);
+    log::debug!("defs = {:?}", defs);
+    let defs = match defs {
+        Ok(d) => d,
+        Err(_) => return,
+    };
+
+    let code = isle::compile::compile(&defs);
+    log::debug!("code = {:?}", code);
+    let code = match code {
+        Ok(c) => c,
+        Err(_) => return,
+    };
+
+    // TODO: check that the generated code is valid Rust. This will require
+    // stubbing out extern types, extractors, and constructors.
+    drop(code);
+});
diff --git a/cranelift/isle/isle/Cargo.toml b/cranelift/isle/isle/Cargo.toml
new file mode 100644
index 0000000000..c57c6f43dc
--- /dev/null
+++ b/cranelift/isle/isle/Cargo.toml
@@ -0,0 +1,14 @@
+[package]
+authors = ["The Cranelift Project Developers"]
+description = "ISLE: Instruction Selection and Lowering Expressions. A domain-specific language for instruction selection in Cranelift."
+edition = "2018"
+license = "Apache-2.0 WITH LLVM-exception"
+name = "isle"
+readme = "../README.md"
+repository = "https://github.com/bytecodealliance/wasmtime/tree/main/cranelift/isle"
+version = "0.78.0"
+
+[dependencies]
+log = "0.4"
+miette = "3.0.0"
+thiserror = "1.0.29"
diff --git a/cranelift/isle/isle/README.md b/cranelift/isle/isle/README.md
new file mode 100644
index 0000000000..fbd1d48e08
--- /dev/null
+++ b/cranelift/isle/isle/README.md
@@ -0,0 +1,9 @@
+# ISLE: Instruction Selection / Lowering Expressions
+
+ISLE is a domain specific language (DSL) for instruction selection and lowering
+clif instructions to vcode's `MachInst`s in Cranelift.
+
+ISLE is a statically-typed term-rewriting language. You define rewriting rules
+that map input terms (clif instructions) into output terms (`MachInst`s). These
+rules get compiled down into Rust source test that uses a tree of `match`
+expressions that is as good or better than what you would have written by hand.
diff --git a/cranelift/isle/isle/src/ast.rs b/cranelift/isle/isle/src/ast.rs
new file mode 100644
index 0000000000..57f257a892
--- /dev/null
+++ b/cranelift/isle/isle/src/ast.rs
@@ -0,0 +1,420 @@
+//! Abstract syntax tree (AST) created from parsed ISLE.
+
+#![allow(missing_docs)]
+
+use crate::lexer::Pos;
+use std::sync::Arc;
+
+/// The parsed form of an ISLE file.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Defs {
+    pub defs: Vec<Def>,
+    pub filenames: Vec<Arc<str>>,
+    pub file_texts: Vec<Arc<str>>,
+}
+
+/// One toplevel form in an ISLE file.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum Def {
+    Type(Type),
+    Rule(Rule),
+    Extractor(Extractor),
+    Decl(Decl),
+    Extern(Extern),
+}
+
+/// An identifier -- a variable, term symbol, or type.
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Ident(pub String, pub Pos);
+
+/// A declaration of a type.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Type {
+    pub name: Ident,
+    pub is_extern: bool,
+    pub ty: TypeValue,
+    pub pos: Pos,
+}
+
+/// The actual type-value: a primitive or an enum with variants.
+///
+/// TODO: add structs as well?
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum TypeValue {
+    Primitive(Ident, Pos),
+    Enum(Vec<Variant>, Pos),
+}
+
+/// One variant of an enum type.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Variant {
+    pub name: Ident,
+    pub fields: Vec<Field>,
+    pub pos: Pos,
+}
+
+/// One field of an enum variant.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Field {
+    pub name: Ident,
+    pub ty: Ident,
+    pub pos: Pos,
+}
+
+/// A declaration of a term with its argument and return types.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Decl {
+    pub term: Ident,
+    pub arg_tys: Vec<Ident>,
+    pub ret_ty: Ident,
+    pub pos: Pos,
+}
+
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Rule {
+    pub pattern: Pattern,
+    pub expr: Expr,
+    pub pos: Pos,
+    pub prio: Option<i64>,
+}
+
+/// An extractor macro: (A x y) becomes (B x _ y ...). Expanded during
+/// ast-to-sema pass.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Extractor {
+    pub term: Ident,
+    pub args: Vec<Ident>,
+    pub template: Pattern,
+    pub pos: Pos,
+}
+
+/// A pattern: the left-hand side of a rule.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum Pattern {
+    /// An operator that binds a variable to a subterm and match the
+    /// subpattern.
+    BindPattern {
+        var: Ident,
+        subpat: Box<Pattern>,
+        pos: Pos,
+    },
+    /// A variable that has already been bound (`=x` syntax).
+    Var { var: Ident, pos: Pos },
+    /// An operator that matches a constant integer value.
+    ConstInt { val: i64, pos: Pos },
+    /// An operator that matches an external constant value.
+    ConstPrim { val: Ident, pos: Pos },
+    /// An application of a type variant or term.
+    Term {
+        sym: Ident,
+        args: Vec<TermArgPattern>,
+        pos: Pos,
+    },
+    /// An operator that matches anything.
+    Wildcard { pos: Pos },
+    /// N sub-patterns that must all match.
+    And { subpats: Vec<Pattern>, pos: Pos },
+    /// Internal use only: macro argument in a template.
+    MacroArg { index: usize, pos: Pos },
+}
+
+impl Pattern {
+    pub fn root_term(&self) -> Option<&Ident> {
+        match self {
+            &Pattern::BindPattern { ref subpat, .. } => subpat.root_term(),
+            &Pattern::Term { ref sym, .. } => Some(sym),
+            _ => None,
+        }
+    }
+
+    /// Call `f` for each of the terms in this pattern.
+    pub fn terms(&self, f: &mut dyn FnMut(Pos, &Ident)) {
+        match self {
+            Pattern::Term { sym, args, pos } => {
+                f(*pos, sym);
+                for arg in args {
+                    if let TermArgPattern::Pattern(p) = arg {
+                        p.terms(f);
+                    }
+                }
+            }
+            Pattern::And { subpats, .. } => {
+                for p in subpats {
+                    p.terms(f);
+                }
+            }
+            Pattern::BindPattern { subpat, .. } => {
+                subpat.terms(f);
+            }
+            Pattern::Var { .. }
+            | Pattern::ConstInt { .. }
+            | Pattern::ConstPrim { .. }
+            | Pattern::Wildcard { .. }
+            | Pattern::MacroArg { .. } => {}
+        }
+    }
+
+    pub fn make_macro_template(&self, macro_args: &[Ident]) -> Pattern {
+        log::trace!("make_macro_template: {:?} with {:?}", self, macro_args);
+        match self {
+            &Pattern::BindPattern {
+                ref var,
+                ref subpat,
+                pos,
+                ..
+            } if matches!(&**subpat, &Pattern::Wildcard { .. }) => {
+                if let Some(i) = macro_args.iter().position(|arg| arg.0 == var.0) {
+                    Pattern::MacroArg { index: i, pos }
+                } else {
+                    self.clone()
+                }
+            }
+            &Pattern::BindPattern {
+                ref var,
+                ref subpat,
+                pos,
+            } => Pattern::BindPattern {
+                var: var.clone(),
+                subpat: Box::new(subpat.make_macro_template(macro_args)),
+                pos,
+            },
+            &Pattern::And { ref subpats, pos } => {
+                let subpats = subpats
+                    .iter()
+                    .map(|subpat| subpat.make_macro_template(macro_args))
+                    .collect::<Vec<_>>();
+                Pattern::And { subpats, pos }
+            }
+            &Pattern::Term {
+                ref sym,
+                ref args,
+                pos,
+            } => {
+                let args = args
+                    .iter()
+                    .map(|arg| arg.make_macro_template(macro_args))
+                    .collect::<Vec<_>>();
+                Pattern::Term {
+                    sym: sym.clone(),
+                    args,
+                    pos,
+                }
+            }
+
+            &Pattern::Var { .. }
+            | &Pattern::Wildcard { .. }
+            | &Pattern::ConstInt { .. }
+            | &Pattern::ConstPrim { .. } => self.clone(),
+            &Pattern::MacroArg { .. } => unreachable!(),
+        }
+    }
+
+    pub fn subst_macro_args(&self, macro_args: &[Pattern]) -> Option<Pattern> {
+        log::trace!("subst_macro_args: {:?} with {:?}", self, macro_args);
+        match self {
+            &Pattern::BindPattern {
+                ref var,
+                ref subpat,
+                pos,
+            } => Some(Pattern::BindPattern {
+                var: var.clone(),
+                subpat: Box::new(subpat.subst_macro_args(macro_args)?),
+                pos,
+            }),
+            &Pattern::And { ref subpats, pos } => {
+                let subpats = subpats
+                    .iter()
+                    .map(|subpat| subpat.subst_macro_args(macro_args))
+                    .collect::<Option<Vec<_>>>()?;
+                Some(Pattern::And { subpats, pos })
+            }
+            &Pattern::Term {
+                ref sym,
+                ref args,
+                pos,
+            } => {
+                let args = args
+                    .iter()
+                    .map(|arg| arg.subst_macro_args(macro_args))
+                    .collect::<Option<Vec<_>>>()?;
+                Some(Pattern::Term {
+                    sym: sym.clone(),
+                    args,
+                    pos,
+                })
+            }
+
+            &Pattern::Var { .. }
+            | &Pattern::Wildcard { .. }
+            | &Pattern::ConstInt { .. }
+            | &Pattern::ConstPrim { .. } => Some(self.clone()),
+            &Pattern::MacroArg { index, .. } => macro_args.get(index).cloned(),
+        }
+    }
+
+    pub fn pos(&self) -> Pos {
+        match self {
+            &Pattern::ConstInt { pos, .. }
+            | &Pattern::ConstPrim { pos, .. }
+            | &Pattern::And { pos, .. }
+            | &Pattern::Term { pos, .. }
+            | &Pattern::BindPattern { pos, .. }
+            | &Pattern::Var { pos, .. }
+            | &Pattern::Wildcard { pos, .. }
+            | &Pattern::MacroArg { pos, .. } => pos,
+        }
+    }
+}
+
+/// A pattern in a term argument. Adds "evaluated expression" to kinds
+/// of patterns in addition to all options in `Pattern`.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum TermArgPattern {
+    /// A regular pattern that must match the existing value in the term's argument.
+    Pattern(Pattern),
+    /// An expression that is evaluated during the match phase and can
+    /// be given into an extractor. This is essentially a limited form
+    /// of unification or bidirectional argument flow (a la Prolog):
+    /// we can pass an arg *into* an extractor rather than getting the
+    /// arg *out of* it.
+    Expr(Expr),
+}
+
+impl TermArgPattern {
+    fn make_macro_template(&self, args: &[Ident]) -> TermArgPattern {
+        log::trace!("repplace_macro_args: {:?} with {:?}", self, args);
+        match self {
+            &TermArgPattern::Pattern(ref pat) => {
+                TermArgPattern::Pattern(pat.make_macro_template(args))
+            }
+            &TermArgPattern::Expr(_) => self.clone(),
+        }
+    }
+
+    fn subst_macro_args(&self, args: &[Pattern]) -> Option<TermArgPattern> {
+        match self {
+            &TermArgPattern::Pattern(ref pat) => {
+                Some(TermArgPattern::Pattern(pat.subst_macro_args(args)?))
+            }
+            &TermArgPattern::Expr(_) => Some(self.clone()),
+        }
+    }
+}
+
+/// An expression: the right-hand side of a rule.
+///
+/// Note that this *almost* looks like a core Lisp or lambda calculus,
+/// except that there is no abstraction (lambda). This first-order
+/// limit is what makes it analyzable.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum Expr {
+    /// A term: `(sym args...)`.
+    Term {
+        sym: Ident,
+        args: Vec<Expr>,
+        pos: Pos,
+    },
+    /// A variable use.
+    Var { name: Ident, pos: Pos },
+    /// A constant integer.
+    ConstInt { val: i64, pos: Pos },
+    /// A constant of some other primitive type.
+    ConstPrim { val: Ident, pos: Pos },
+    /// The `(let ((var ty val)*) body)` form.
+    Let {
+        defs: Vec<LetDef>,
+        body: Box<Expr>,
+        pos: Pos,
+    },
+}
+
+impl Expr {
+    pub fn pos(&self) -> Pos {
+        match self {
+            &Expr::Term { pos, .. }
+            | &Expr::Var { pos, .. }
+            | &Expr::ConstInt { pos, .. }
+            | &Expr::ConstPrim { pos, .. }
+            | &Expr::Let { pos, .. } => pos,
+        }
+    }
+
+    /// Call `f` for each of the terms in this expression.
+    pub fn terms(&self, f: &mut dyn FnMut(Pos, &Ident)) {
+        match self {
+            Expr::Term { sym, args, pos } => {
+                f(*pos, sym);
+                for arg in args {
+                    arg.terms(f);
+                }
+            }
+            Expr::Let { defs, body, .. } => {
+                for def in defs {
+                    def.val.terms(f);
+                }
+                body.terms(f);
+            }
+            Expr::Var { .. } | Expr::ConstInt { .. } | Expr::ConstPrim { .. } => {}
+        }
+    }
+}
+
+/// One variable locally bound in a `(let ...)` expression.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct LetDef {
+    pub var: Ident,
+    pub ty: Ident,
+    pub val: Box<Expr>,
+    pub pos: Pos,
+}
+
+/// An external binding: an extractor or constructor function attached
+/// to a term.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum Extern {
+    /// An external extractor: `(extractor Term rustfunc)` form.
+    Extractor {
+        /// The term to which this external extractor is attached.
+        term: Ident,
+        /// The Rust function name.
+        func: Ident,
+        /// The position of this decl.
+        pos: Pos,
+        /// Poliarity of args: whether values are inputs or outputs to
+        /// the external extractor function. This is a sort of
+        /// statically-defined approximation to Prolog-style
+        /// unification; we allow for the same flexible directionality
+        /// but fix it at DSL-definition time. By default, every arg
+        /// is an *output* from the extractor (and the 'retval", or
+        /// more precisely the term value that we are extracting, is
+        /// an "input").
+        arg_polarity: Option<Vec<ArgPolarity>>,
+        /// Infallibility: if an external extractor returns `(T1, T2,
+        /// ...)` rather than `Option<(T1, T2, ...)>`, and hence can
+        /// never fail, it is declared as such and allows for slightly
+        /// better code to be generated.
+        infallible: bool,
+    },
+    /// An external constructor: `(constructor Term rustfunc)` form.
+    Constructor {
+        /// The term to which this external constructor is attached.
+        term: Ident,
+        /// The Rust function name.
+        func: Ident,
+        /// The position of this decl.
+        pos: Pos,
+    },
+    /// An external constant: `(const $IDENT type)` form.
+    Const { name: Ident, ty: Ident, pos: Pos },
+}
+
+/// Whether an argument is an input or an output.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ArgPolarity {
+    /// An arg that must be given an Expr in the pattern and passes data *to*
+    /// the extractor op.
+    Input,
+    /// An arg that must be given a regular pattern (not Expr) and receives data
+    /// *from* the extractor op.
+    Output,
+}
diff --git a/cranelift/isle/isle/src/codegen.rs b/cranelift/isle/isle/src/codegen.rs
new file mode 100644
index 0000000000..74874b6a1d
--- /dev/null
+++ b/cranelift/isle/isle/src/codegen.rs
@@ -0,0 +1,852 @@
+//! Generate Rust code from a series of Sequences.
+
+use crate::ir::{ExprInst, InstId, PatternInst, Value};
+use crate::sema::ExternalSig;
+use crate::sema::{TermEnv, TermId, Type, TypeEnv, TypeId, Variant};
+use crate::trie::{TrieEdge, TrieNode, TrieSymbol};
+use std::collections::{BTreeMap, BTreeSet};
+use std::fmt::Write;
+
+/// Emit Rust source code for the given type and term environments.
+pub fn codegen(typeenv: &TypeEnv, termenv: &TermEnv, tries: &BTreeMap<TermId, TrieNode>) -> String {
+    Codegen::compile(typeenv, termenv, tries).generate_rust()
+}
+
+#[derive(Clone, Debug)]
+struct Codegen<'a> {
+    typeenv: &'a TypeEnv,
+    termenv: &'a TermEnv,
+    functions_by_term: &'a BTreeMap<TermId, TrieNode>,
+}
+
+#[derive(Clone, Debug, Default)]
+struct BodyContext {
+    /// For each value: (is_ref, ty).
+    values: BTreeMap<Value, (bool, TypeId)>,
+}
+
+impl<'a> Codegen<'a> {
+    fn compile(
+        typeenv: &'a TypeEnv,
+        termenv: &'a TermEnv,
+        tries: &'a BTreeMap<TermId, TrieNode>,
+    ) -> Codegen<'a> {
+        Codegen {
+            typeenv,
+            termenv,
+            functions_by_term: tries,
+        }
+    }
+
+    fn generate_rust(&self) -> String {
+        let mut code = String::new();
+
+        self.generate_header(&mut code);
+        self.generate_ctx_trait(&mut code);
+        self.generate_internal_types(&mut code);
+        self.generate_internal_term_constructors(&mut code);
+
+        code
+    }
+
+    fn generate_header(&self, code: &mut String) {
+        writeln!(code, "// GENERATED BY ISLE. DO NOT EDIT!").unwrap();
+        writeln!(code, "//").unwrap();
+        writeln!(
+            code,
+            "// Generated automatically from the instruction-selection DSL code in:",
+        )
+        .unwrap();
+        for file in &self.typeenv.filenames {
+            writeln!(code, "// - {}", file).unwrap();
+        }
+
+        writeln!(
+            code,
+            "\n#![allow(dead_code, unreachable_code, unreachable_patterns)]"
+        )
+        .unwrap();
+        writeln!(
+            code,
+            "#![allow(unused_imports, unused_variables, non_snake_case)]"
+        )
+        .unwrap();
+        writeln!(code, "#![allow(irrefutable_let_patterns)]").unwrap();
+
+        writeln!(code, "\nuse super::*;  // Pulls in all external types.").unwrap();
+    }
+
+    fn generate_trait_sig(&self, code: &mut String, indent: &str, sig: &ExternalSig) {
+        writeln!(
+            code,
+            "{indent}fn {name}(&mut self, {params}) -> {opt_start}{open_paren}{rets}{close_paren}{opt_end};",
+            indent = indent,
+            name = sig.func_name,
+            params = sig.param_tys
+                .iter()
+                .enumerate()
+                .map(|(i, &ty)| format!("arg{}: {}", i, self.type_name(ty, /* by_ref = */ true)))
+                .collect::<Vec<_>>()
+                .join(", "),
+            opt_start = if sig.infallible { "" } else { "Option<" },
+            open_paren = if sig.ret_tys.len() != 1 { "(" } else { "" },
+            rets = sig.ret_tys
+                .iter()
+                .map(|&ty| self.type_name(ty, /* by_ref = */ false))
+                .collect::<Vec<_>>()
+                .join(", "),
+            close_paren = if sig.ret_tys.len() != 1 { ")" } else { "" },
+            opt_end = if sig.infallible { "" } else { ">" },
+        )
+        .unwrap();
+    }
+
+    fn generate_ctx_trait(&self, code: &mut String) {
+        writeln!(code, "").unwrap();
+        writeln!(
+            code,
+            "/// Context during lowering: an implementation of this trait"
+        )
+        .unwrap();
+        writeln!(
+            code,
+            "/// must be provided with all external constructors and extractors."
+        )
+        .unwrap();
+        writeln!(
+            code,
+            "/// A mutable borrow is passed along through all lowering logic."
+        )
+        .unwrap();
+        writeln!(code, "pub trait Context {{").unwrap();
+        for term in &self.termenv.terms {
+            if term.has_external_extractor() {
+                let ext_sig = term.extractor_sig(self.typeenv).unwrap();
+                self.generate_trait_sig(code, "    ", &ext_sig);
+            }
+            if term.has_external_constructor() {
+                let ext_sig = term.constructor_sig(self.typeenv).unwrap();
+                self.generate_trait_sig(code, "    ", &ext_sig);
+            }
+        }
+        writeln!(code, "}}").unwrap();
+    }
+
+    fn generate_internal_types(&self, code: &mut String) {
+        for ty in &self.typeenv.types {
+            match ty {
+                &Type::Enum {
+                    name,
+                    is_extern,
+                    ref variants,
+                    pos,
+                    ..
+                } if !is_extern => {
+                    let name = &self.typeenv.syms[name.index()];
+                    writeln!(
+                        code,
+                        "\n/// Internal type {}: defined at {}.",
+                        name,
+                        pos.pretty_print_line(&self.typeenv.filenames[..])
+                    )
+                    .unwrap();
+                    writeln!(code, "#[derive(Clone, Debug)]").unwrap();
+                    writeln!(code, "pub enum {} {{", name).unwrap();
+                    for variant in variants {
+                        let name = &self.typeenv.syms[variant.name.index()];
+                        if variant.fields.is_empty() {
+                            writeln!(code, "    {},", name).unwrap();
+                        } else {
+                            writeln!(code, "    {} {{", name).unwrap();
+                            for field in &variant.fields {
+                                let name = &self.typeenv.syms[field.name.index()];
+                                let ty_name =
+                                    self.typeenv.types[field.ty.index()].name(&self.typeenv);
+                                writeln!(code, "        {}: {},", name, ty_name).unwrap();
+                            }
+                            writeln!(code, "    }},").unwrap();
+                        }
+                    }
+                    writeln!(code, "}}").unwrap();
+                }
+                _ => {}
+            }
+        }
+    }
+
+    fn type_name(&self, typeid: TypeId, by_ref: bool) -> String {
+        match &self.typeenv.types[typeid.index()] {
+            &Type::Primitive(_, sym, _) => self.typeenv.syms[sym.index()].clone(),
+            &Type::Enum { name, .. } => {
+                let r = if by_ref { "&" } else { "" };
+                format!("{}{}", r, self.typeenv.syms[name.index()])
+            }
+        }
+    }
+
+    fn value_name(&self, value: &Value) -> String {
+        match value {
+            &Value::Pattern { inst, output } => format!("pattern{}_{}", inst.index(), output),
+            &Value::Expr { inst, output } => format!("expr{}_{}", inst.index(), output),
+        }
+    }
+
+    fn ty_prim(&self, ty: TypeId) -> bool {
+        self.typeenv.types[ty.index()].is_prim()
+    }
+
+    fn value_binder(&self, value: &Value, is_ref: bool, ty: TypeId) -> String {
+        let prim = self.ty_prim(ty);
+        if prim || !is_ref {
+            format!("{}", self.value_name(value))
+        } else {
+            format!("ref {}", self.value_name(value))
+        }
+    }
+
+    fn value_by_ref(&self, value: &Value, ctx: &BodyContext) -> String {
+        let raw_name = self.value_name(value);
+        let &(is_ref, ty) = ctx.values.get(value).unwrap();
+        let prim = self.ty_prim(ty);
+        if is_ref || prim {
+            raw_name
+        } else {
+            format!("&{}", raw_name)
+        }
+    }
+
+    fn value_by_val(&self, value: &Value, ctx: &BodyContext) -> String {
+        let raw_name = self.value_name(value);
+        let &(is_ref, _) = ctx.values.get(value).unwrap();
+        if is_ref {
+            format!("{}.clone()", raw_name)
+        } else {
+            raw_name
+        }
+    }
+
+    fn define_val(&self, value: &Value, ctx: &mut BodyContext, is_ref: bool, ty: TypeId) {
+        let is_ref = !self.ty_prim(ty) && is_ref;
+        ctx.values.insert(value.clone(), (is_ref, ty));
+    }
+
+    fn const_int(&self, val: i64, ty: TypeId) -> String {
+        let is_bool = match &self.typeenv.types[ty.index()] {
+            &Type::Primitive(_, name, _) => &self.typeenv.syms[name.index()] == "bool",
+            _ => unreachable!(),
+        };
+        if is_bool {
+            format!("{}", val != 0)
+        } else {
+            format!("{}", val)
+        }
+    }
+
+    fn generate_internal_term_constructors(&self, code: &mut String) {
+        for (&termid, trie) in self.functions_by_term {
+            let termdata = &self.termenv.terms[termid.index()];
+
+            // Skip terms that are enum variants or that have external
+            // constructors/extractors.
+            if !termdata.has_constructor() || termdata.has_external_constructor() {
+                continue;
+            }
+
+            let sig = termdata.constructor_sig(self.typeenv).unwrap();
+
+            let args = sig
+                .param_tys
+                .iter()
+                .enumerate()
+                .map(|(i, &ty)| format!("arg{}: {}", i, self.type_name(ty, true)))
+                .collect::<Vec<_>>()
+                .join(", ");
+            assert_eq!(sig.ret_tys.len(), 1);
+            let ret = self.type_name(sig.ret_tys[0], false);
+
+            writeln!(
+                code,
+                "\n// Generated as internal constructor for term {}.",
+                self.typeenv.syms[termdata.name.index()],
+            )
+            .unwrap();
+            writeln!(
+                code,
+                "pub fn {}<C: Context>(ctx: &mut C, {}) -> Option<{}> {{",
+                sig.func_name, args, ret,
+            )
+            .unwrap();
+
+            let mut body_ctx: BodyContext = Default::default();
+            let returned =
+                self.generate_body(code, /* depth = */ 0, trie, "    ", &mut body_ctx);
+            if !returned {
+                writeln!(code, "    return None;").unwrap();
+            }
+
+            writeln!(code, "}}").unwrap();
+        }
+    }
+
+    fn generate_expr_inst(
+        &self,
+        code: &mut String,
+        id: InstId,
+        inst: &ExprInst,
+        indent: &str,
+        ctx: &mut BodyContext,
+        returns: &mut Vec<(usize, String)>,
+    ) {
+        log::trace!("generate_expr_inst: {:?}", inst);
+        match inst {
+            &ExprInst::ConstInt { ty, val } => {
+                let value = Value::Expr {
+                    inst: id,
+                    output: 0,
+                };
+                self.define_val(&value, ctx, /* is_ref = */ false, ty);
+                let name = self.value_name(&value);
+                let ty_name = self.type_name(ty, /* by_ref = */ false);
+                writeln!(
+                    code,
+                    "{}let {}: {} = {};",
+                    indent,
+                    name,
+                    ty_name,
+                    self.const_int(val, ty)
+                )
+                .unwrap();
+            }
+            &ExprInst::ConstPrim { ty, val } => {
+                let value = Value::Expr {
+                    inst: id,
+                    output: 0,
+                };
+                self.define_val(&value, ctx, /* is_ref = */ false, ty);
+                let name = self.value_name(&value);
+                let ty_name = self.type_name(ty, /* by_ref = */ false);
+                writeln!(
+                    code,
+                    "{}let {}: {} = {};",
+                    indent,
+                    name,
+                    ty_name,
+                    self.typeenv.syms[val.index()],
+                )
+                .unwrap();
+            }
+            &ExprInst::CreateVariant {
+                ref inputs,
+                ty,
+                variant,
+            } => {
+                let variantinfo = match &self.typeenv.types[ty.index()] {
+                    &Type::Primitive(..) => panic!("CreateVariant with primitive type"),
+                    &Type::Enum { ref variants, .. } => &variants[variant.index()],
+                };
+                let mut input_fields = vec![];
+                for ((input_value, _), field) in inputs.iter().zip(variantinfo.fields.iter()) {
+                    let field_name = &self.typeenv.syms[field.name.index()];
+                    let value_expr = self.value_by_val(input_value, ctx);
+                    input_fields.push(format!("{}: {}", field_name, value_expr));
+                }
+
+                let output = Value::Expr {
+                    inst: id,
+                    output: 0,
+                };
+                let outputname = self.value_name(&output);
+                let full_variant_name = format!(
+                    "{}::{}",
+                    self.type_name(ty, false),
+                    self.typeenv.syms[variantinfo.name.index()]
+                );
+                if input_fields.is_empty() {
+                    writeln!(
+                        code,
+                        "{}let {} = {};",
+                        indent, outputname, full_variant_name
+                    )
+                    .unwrap();
+                } else {
+                    writeln!(
+                        code,
+                        "{}let {} = {} {{",
+                        indent, outputname, full_variant_name
+                    )
+                    .unwrap();
+                    for input_field in input_fields {
+                        writeln!(code, "{}    {},", indent, input_field).unwrap();
+                    }
+                    writeln!(code, "{}}};", indent).unwrap();
+                }
+                self.define_val(&output, ctx, /* is_ref = */ false, ty);
+            }
+            &ExprInst::Construct {
+                ref inputs,
+                term,
+                infallible,
+                ..
+            } => {
+                let mut input_exprs = vec![];
+                for (input_value, input_ty) in inputs {
+                    let value_expr = if self.typeenv.types[input_ty.index()].is_prim() {
+                        self.value_by_val(input_value, ctx)
+                    } else {
+                        self.value_by_ref(input_value, ctx)
+                    };
+                    input_exprs.push(value_expr);
+                }
+
+                let output = Value::Expr {
+                    inst: id,
+                    output: 0,
+                };
+                let outputname = self.value_name(&output);
+                let termdata = &self.termenv.terms[term.index()];
+                let sig = termdata.constructor_sig(self.typeenv).unwrap();
+                assert_eq!(input_exprs.len(), sig.param_tys.len());
+                let fallible_try = if infallible { "" } else { "?" };
+                writeln!(
+                    code,
+                    "{}let {} = {}(ctx, {}){};",
+                    indent,
+                    outputname,
+                    sig.full_name,
+                    input_exprs.join(", "),
+                    fallible_try,
+                )
+                .unwrap();
+                self.define_val(&output, ctx, /* is_ref = */ false, termdata.ret_ty);
+            }
+            &ExprInst::Return {
+                index, ref value, ..
+            } => {
+                let value_expr = self.value_by_val(value, ctx);
+                returns.push((index, value_expr));
+            }
+        }
+    }
+
+    fn match_variant_binders(
+        &self,
+        variant: &Variant,
+        arg_tys: &[TypeId],
+        id: InstId,
+        ctx: &mut BodyContext,
+    ) -> Vec<String> {
+        arg_tys
+            .iter()
+            .zip(variant.fields.iter())
+            .enumerate()
+            .map(|(i, (&ty, field))| {
+                let value = Value::Pattern {
+                    inst: id,
+                    output: i,
+                };
+                let valuename = self.value_binder(&value, /* is_ref = */ true, ty);
+                let fieldname = &self.typeenv.syms[field.name.index()];
+                self.define_val(&value, ctx, /* is_ref = */ false, field.ty);
+                format!("{}: {}", fieldname, valuename)
+            })
+            .collect::<Vec<_>>()
+    }
+
+    /// Returns a `bool` indicating whether this pattern inst is
+    /// infallible.
+    fn generate_pattern_inst(
+        &self,
+        code: &mut String,
+        id: InstId,
+        inst: &PatternInst,
+        indent: &str,
+        ctx: &mut BodyContext,
+    ) -> bool {
+        match inst {
+            &PatternInst::Arg { index, ty } => {
+                let output = Value::Pattern {
+                    inst: id,
+                    output: 0,
+                };
+                let outputname = self.value_name(&output);
+                let is_ref = match &self.typeenv.types[ty.index()] {
+                    &Type::Primitive(..) => false,
+                    _ => true,
+                };
+                writeln!(code, "{}let {} = arg{};", indent, outputname, index).unwrap();
+                self.define_val(
+                    &Value::Pattern {
+                        inst: id,
+                        output: 0,
+                    },
+                    ctx,
+                    is_ref,
+                    ty,
+                );
+                true
+            }
+            &PatternInst::MatchEqual { ref a, ref b, .. } => {
+                let a = self.value_by_ref(a, ctx);
+                let b = self.value_by_ref(b, ctx);
+                writeln!(code, "{}if {} == {} {{", indent, a, b).unwrap();
+                false
+            }
+            &PatternInst::MatchInt {
+                ref input, int_val, ..
+            } => {
+                let input = self.value_by_val(input, ctx);
+                writeln!(code, "{}if {} == {} {{", indent, input, int_val).unwrap();
+                false
+            }
+            &PatternInst::MatchPrim { ref input, val, .. } => {
+                let input = self.value_by_val(input, ctx);
+                let sym = &self.typeenv.syms[val.index()];
+                writeln!(code, "{}if {} == {} {{", indent, input, sym).unwrap();
+                false
+            }
+            &PatternInst::MatchVariant {
+                ref input,
+                input_ty,
+                variant,
+                ref arg_tys,
+            } => {
+                let input = self.value_by_ref(input, ctx);
+                let variants = match &self.typeenv.types[input_ty.index()] {
+                    &Type::Primitive(..) => panic!("primitive type input to MatchVariant"),
+                    &Type::Enum { ref variants, .. } => variants,
+                };
+                let ty_name = self.type_name(input_ty, /* is_ref = */ true);
+                let variant = &variants[variant.index()];
+                let variantname = &self.typeenv.syms[variant.name.index()];
+                let args = self.match_variant_binders(variant, &arg_tys[..], id, ctx);
+                let args = if args.is_empty() {
+                    "".to_string()
+                } else {
+                    format!("{{ {} }}", args.join(", "))
+                };
+                writeln!(
+                    code,
+                    "{}if let {}::{} {} = {} {{",
+                    indent, ty_name, variantname, args, input
+                )
+                .unwrap();
+                false
+            }
+            &PatternInst::Extract {
+                ref inputs,
+                ref output_tys,
+                term,
+                infallible,
+                ..
+            } => {
+                let termdata = &self.termenv.terms[term.index()];
+                let sig = termdata.extractor_sig(self.typeenv).unwrap();
+
+                let input_values = inputs
+                    .iter()
+                    .map(|input| self.value_by_ref(input, ctx))
+                    .collect::<Vec<_>>();
+                let output_binders = output_tys
+                    .iter()
+                    .enumerate()
+                    .map(|(i, &ty)| {
+                        let output_val = Value::Pattern {
+                            inst: id,
+                            output: i,
+                        };
+                        self.define_val(&output_val, ctx, /* is_ref = */ false, ty);
+                        self.value_binder(&output_val, /* is_ref = */ false, ty)
+                    })
+                    .collect::<Vec<_>>();
+
+                if infallible {
+                    writeln!(
+                        code,
+                        "{indent}let {open_paren}{vars}{close_paren} = {name}(ctx, {args});",
+                        indent = indent,
+                        open_paren = if output_binders.len() == 1 { "" } else { "(" },
+                        vars = output_binders.join(", "),
+                        close_paren = if output_binders.len() == 1 { "" } else { ")" },
+                        name = sig.full_name,
+                        args = input_values.join(", "),
+                    )
+                    .unwrap();
+                    true
+                } else {
+                    writeln!(
+                        code,
+                        "{indent}if let Some({open_paren}{vars}{close_paren}) = {name}(ctx, {args}) {{",
+                        indent = indent,
+                        open_paren = if output_binders.len() == 1 { "" } else { "(" },
+                        vars = output_binders.join(", "),
+                        close_paren = if output_binders.len() == 1 { "" } else { ")" },
+                        name = sig.full_name,
+                        args = input_values.join(", "),
+                    )
+                    .unwrap();
+                    false
+                }
+            }
+            &PatternInst::Expr {
+                ref seq, output_ty, ..
+            } if seq.is_const_int().is_some() => {
+                let (ty, val) = seq.is_const_int().unwrap();
+                assert_eq!(ty, output_ty);
+
+                let output = Value::Pattern {
+                    inst: id,
+                    output: 0,
+                };
+                writeln!(
+                    code,
+                    "{}let {} = {};",
+                    indent,
+                    self.value_name(&output),
+                    self.const_int(val, ty),
+                )
+                .unwrap();
+                self.define_val(&output, ctx, /* is_ref = */ false, ty);
+                true
+            }
+            &PatternInst::Expr {
+                ref seq, output_ty, ..
+            } => {
+                let closure_name = format!("closure{}", id.index());
+                writeln!(code, "{}let {} = || {{", indent, closure_name).unwrap();
+                let subindent = format!("{}    ", indent);
+                let mut subctx = ctx.clone();
+                let mut returns = vec![];
+                for (id, inst) in seq.insts.iter().enumerate() {
+                    let id = InstId(id);
+                    self.generate_expr_inst(code, id, inst, &subindent, &mut subctx, &mut returns);
+                }
+                assert_eq!(returns.len(), 1);
+                writeln!(code, "{}return Some({});", subindent, returns[0].1).unwrap();
+                writeln!(code, "{}}};", indent).unwrap();
+
+                let output = Value::Pattern {
+                    inst: id,
+                    output: 0,
+                };
+                writeln!(
+                    code,
+                    "{}if let Some({}) = {}() {{",
+                    indent,
+                    self.value_binder(&output, /* is_ref = */ false, output_ty),
+                    closure_name
+                )
+                .unwrap();
+                self.define_val(&output, ctx, /* is_ref = */ false, output_ty);
+
+                false
+            }
+        }
+    }
+
+    fn generate_body(
+        &self,
+        code: &mut String,
+        depth: usize,
+        trie: &TrieNode,
+        indent: &str,
+        ctx: &mut BodyContext,
+    ) -> bool {
+        log::trace!("generate_body:\n{}", trie.pretty());
+        let mut returned = false;
+        match trie {
+            &TrieNode::Empty => {}
+
+            &TrieNode::Leaf { ref output, .. } => {
+                writeln!(
+                    code,
+                    "{}// Rule at {}.",
+                    indent,
+                    output.pos.pretty_print_line(&self.typeenv.filenames[..])
+                )
+                .unwrap();
+                // If this is a leaf node, generate the ExprSequence and return.
+                let mut returns = vec![];
+                for (id, inst) in output.insts.iter().enumerate() {
+                    let id = InstId(id);
+                    self.generate_expr_inst(code, id, inst, indent, ctx, &mut returns);
+                }
+
+                assert_eq!(returns.len(), 1);
+                writeln!(code, "{}return Some({});", indent, returns[0].1).unwrap();
+
+                returned = true;
+            }
+
+            &TrieNode::Decision { ref edges } => {
+                let subindent = format!("{}    ", indent);
+                // if this is a decision node, generate each match op
+                // in turn (in priority order). Sort the ops within
+                // each priority, and gather together adjacent
+                // MatchVariant ops with the same input and disjoint
+                // variants in order to create a `match` rather than a
+                // chain of if-lets.
+                let mut edges = edges.clone();
+                edges.sort_by(|e1, e2| {
+                    (-e1.range.min, &e1.symbol).cmp(&(-e2.range.min, &e2.symbol))
+                });
+
+                let mut i = 0;
+                while i < edges.len() {
+                    // Gather adjacent match variants so that we can turn these
+                    // into a `match` rather than a sequence of `if let`s.
+                    let mut last = i;
+                    let mut adjacent_variants = BTreeSet::new();
+                    let mut adjacent_variant_input = None;
+                    log::trace!(
+                        "edge: range = {:?}, symbol = {:?}",
+                        edges[i].range,
+                        edges[i].symbol
+                    );
+                    while last < edges.len() {
+                        match &edges[last].symbol {
+                            &TrieSymbol::Match {
+                                op: PatternInst::MatchVariant { input, variant, .. },
+                            } => {
+                                if adjacent_variant_input.is_none() {
+                                    adjacent_variant_input = Some(input);
+                                }
+                                if adjacent_variant_input == Some(input)
+                                    && !adjacent_variants.contains(&variant)
+                                {
+                                    adjacent_variants.insert(variant);
+                                    last += 1;
+                                } else {
+                                    break;
+                                }
+                            }
+                            _ => {
+                                break;
+                            }
+                        }
+                    }
+
+                    // Now `edges[i..last]` is a run of adjacent `MatchVariants`
+                    // (possibly an empty one). Only use a `match` form if there
+                    // are at least two adjacent options.
+                    if last - i > 1 {
+                        self.generate_body_matches(code, depth, &edges[i..last], indent, ctx);
+                        i = last;
+                        continue;
+                    } else {
+                        let &TrieEdge {
+                            ref symbol,
+                            ref node,
+                            ..
+                        } = &edges[i];
+                        i += 1;
+
+                        match symbol {
+                            &TrieSymbol::EndOfMatch => {
+                                returned = self.generate_body(code, depth + 1, node, indent, ctx);
+                            }
+                            &TrieSymbol::Match { ref op } => {
+                                let id = InstId(depth);
+                                let infallible =
+                                    self.generate_pattern_inst(code, id, op, indent, ctx);
+                                let i = if infallible { indent } else { &subindent[..] };
+                                let sub_returned =
+                                    self.generate_body(code, depth + 1, node, i, ctx);
+                                if !infallible {
+                                    writeln!(code, "{}}}", indent).unwrap();
+                                }
+                                if infallible && sub_returned {
+                                    returned = true;
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        returned
+    }
+
+    fn generate_body_matches(
+        &self,
+        code: &mut String,
+        depth: usize,
+        edges: &[TrieEdge],
+        indent: &str,
+        ctx: &mut BodyContext,
+    ) {
+        let (input, input_ty) = match &edges[0].symbol {
+            &TrieSymbol::Match {
+                op:
+                    PatternInst::MatchVariant {
+                        input, input_ty, ..
+                    },
+            } => (input, input_ty),
+            _ => unreachable!(),
+        };
+        let (input_ty_sym, variants) = match &self.typeenv.types[input_ty.index()] {
+            &Type::Enum {
+                ref name,
+                ref variants,
+                ..
+            } => (name, variants),
+            _ => unreachable!(),
+        };
+        let input_ty_name = &self.typeenv.syms[input_ty_sym.index()];
+
+        // Emit the `match`.
+        writeln!(
+            code,
+            "{}match {} {{",
+            indent,
+            self.value_by_ref(&input, ctx)
+        )
+        .unwrap();
+
+        // Emit each case.
+        for &TrieEdge {
+            ref symbol,
+            ref node,
+            ..
+        } in edges
+        {
+            let id = InstId(depth);
+            let (variant, arg_tys) = match symbol {
+                &TrieSymbol::Match {
+                    op:
+                        PatternInst::MatchVariant {
+                            variant,
+                            ref arg_tys,
+                            ..
+                        },
+                } => (variant, arg_tys),
+                _ => unreachable!(),
+            };
+
+            let variantinfo = &variants[variant.index()];
+            let variantname = &self.typeenv.syms[variantinfo.name.index()];
+            let fields = self.match_variant_binders(variantinfo, arg_tys, id, ctx);
+            let fields = if fields.is_empty() {
+                "".to_string()
+            } else {
+                format!("{{ {} }}", fields.join(", "))
+            };
+            writeln!(
+                code,
+                "{}    &{}::{} {} => {{",
+                indent, input_ty_name, variantname, fields,
+            )
+            .unwrap();
+            let subindent = format!("{}        ", indent);
+            self.generate_body(code, depth + 1, node, &subindent, ctx);
+            writeln!(code, "{}    }}", indent).unwrap();
+        }
+
+        // Always add a catchall, because we don't do exhaustiveness
+        // checking on the MatcHVariants.
+        writeln!(code, "{}    _ => {{}}", indent).unwrap();
+
+        writeln!(code, "{}}}", indent).unwrap();
+    }
+}
diff --git a/cranelift/isle/isle/src/compile.rs b/cranelift/isle/isle/src/compile.rs
new file mode 100644
index 0000000000..8d9a1bb754
--- /dev/null
+++ b/cranelift/isle/isle/src/compile.rs
@@ -0,0 +1,12 @@
+//! Compilation process, from AST to Sema to Sequences of Insts.
+
+use crate::error::Result;
+use crate::{ast, codegen, sema, trie};
+
+/// Compile the given AST definitions into Rust source code.
+pub fn compile(defs: &ast::Defs) -> Result<String> {
+    let mut typeenv = sema::TypeEnv::from_ast(defs)?;
+    let termenv = sema::TermEnv::from_ast(&mut typeenv, defs)?;
+    let tries = trie::build_tries(&typeenv, &termenv);
+    Ok(codegen::codegen(&typeenv, &termenv, &tries))
+}
diff --git a/cranelift/isle/isle/src/error.rs b/cranelift/isle/isle/src/error.rs
new file mode 100644
index 0000000000..4aee4c0e2e
--- /dev/null
+++ b/cranelift/isle/isle/src/error.rs
@@ -0,0 +1,148 @@
+//! Error types.
+
+use miette::{Diagnostic, SourceCode, SourceSpan};
+use std::sync::Arc;
+
+/// Either `Ok(T)` or `Err(isle::Error)`.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Errors produced by ISLE.
+#[derive(thiserror::Error, Diagnostic, Clone, Debug)]
+pub enum Error {
+    /// An I/O error.
+    #[error("{context}")]
+    IoError {
+        /// The underlying I/O error.
+        #[source]
+        error: Arc<std::io::Error>,
+        /// The context explaining what caused the I/O error.
+        context: String,
+    },
+
+    /// The input ISLE source has a parse error.
+    #[error("parse error: {msg}")]
+    #[diagnostic()]
+    ParseError {
+        /// The error message.
+        msg: String,
+
+        /// The input ISLE source.
+        #[source_code]
+        src: Source,
+
+        /// The location of the parse error.
+        #[label("{msg}")]
+        span: SourceSpan,
+    },
+
+    /// The input ISLE source has a type error.
+    #[error("type error: {msg}")]
+    #[diagnostic()]
+    TypeError {
+        /// The error message.
+        msg: String,
+
+        /// The input ISLE source.
+        #[source_code]
+        src: Source,
+
+        /// The location of the type error.
+        #[label("{msg}")]
+        span: SourceSpan,
+    },
+
+    /// Multiple errors.
+    #[error("Found {} errors:\n\n{}",
+            self.unwrap_errors().len(),
+            DisplayErrors(self.unwrap_errors()))]
+    #[diagnostic()]
+    Errors(#[related] Vec<Error>),
+}
+
+impl Error {
+    /// Create a `isle::Error` from the given I/O error and context.
+    pub fn from_io(error: std::io::Error, context: impl Into<String>) -> Self {
+        Error::IoError {
+            error: Arc::new(error),
+            context: context.into(),
+        }
+    }
+}
+
+impl From<Vec<Error>> for Error {
+    fn from(es: Vec<Error>) -> Self {
+        Error::Errors(es)
+    }
+}
+
+impl Error {
+    fn unwrap_errors(&self) -> &[Error] {
+        match self {
+            Error::Errors(e) => e,
+            _ => panic!("`isle::Error::unwrap_errors` on non-`isle::Error::Errors`"),
+        }
+    }
+}
+
+struct DisplayErrors<'a>(&'a [Error]);
+impl std::fmt::Display for DisplayErrors<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for e in self.0 {
+            writeln!(f, "{}", e)?;
+        }
+        Ok(())
+    }
+}
+
+/// A source file and its contents.
+#[derive(Clone)]
+pub struct Source {
+    name: Arc<str>,
+    text: Arc<str>,
+}
+
+impl std::fmt::Debug for Source {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Source")
+            .field("name", &self.name)
+            .field("source", &"<redacted>");
+        Ok(())
+    }
+}
+
+impl Source {
+    pub(crate) fn new(name: Arc<str>, text: Arc<str>) -> Self {
+        Self { name, text }
+    }
+
+    /// Get this source's file name.
+    pub fn name(&self) -> &Arc<str> {
+        &self.name
+    }
+
+    /// Get this source's text contents.
+    pub fn text(&self) -> &Arc<str> {
+        &self.name
+    }
+}
+
+impl SourceCode for Source {
+    fn read_span<'a>(
+        &'a self,
+        span: &SourceSpan,
+        context_lines_before: usize,
+        context_lines_after: usize,
+    ) -> std::result::Result<Box<dyn miette::SpanContents<'a> + 'a>, miette::MietteError> {
+        let contents = self
+            .text
+            .read_span(span, context_lines_before, context_lines_after)?;
+        Ok(Box::new(miette::MietteSpanContents::new_named(
+            self.name.to_string(),
+            contents.data(),
+            contents.span().clone(),
+            contents.line(),
+            contents.column(),
+            contents.line_count(),
+        )))
+    }
+}
diff --git a/cranelift/isle/isle/src/ir.rs b/cranelift/isle/isle/src/ir.rs
new file mode 100644
index 0000000000..caa0090864
--- /dev/null
+++ b/cranelift/isle/isle/src/ir.rs
@@ -0,0 +1,685 @@
+//! Lowered matching IR.
+
+use crate::lexer::Pos;
+use crate::sema::*;
+use std::collections::BTreeMap;
+
+declare_id!(
+    /// The id of an instruction in a `PatternSequence`.
+    InstId
+);
+
+/// A value produced by a LHS or RHS instruction.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Value {
+    /// A value produced by an instruction in the Pattern (LHS).
+    Pattern {
+        /// The instruction that produces this value.
+        inst: InstId,
+        /// This value is the `output`th value produced by this pattern.
+        output: usize,
+    },
+    /// A value produced by an instruction in the Expr (RHS).
+    Expr {
+        /// The instruction that produces this value.
+        inst: InstId,
+        /// This value is the `output`th value produced by this expression.
+        output: usize,
+    },
+}
+
+/// A single Pattern instruction.
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum PatternInst {
+    /// Match a value as equal to another value. Produces no values.
+    MatchEqual {
+        /// The first value.
+        a: Value,
+        /// The second value.
+        b: Value,
+        /// The type of the values.
+        ty: TypeId,
+    },
+
+    /// Try matching the given value as the given integer. Produces no values.
+    MatchInt {
+        /// The value to match on.
+        input: Value,
+        /// The value's type.
+        ty: TypeId,
+        /// The integer to match against the value.
+        int_val: i64,
+    },
+
+    /// Try matching the given value as the given constant. Produces no values.
+    MatchPrim {
+        /// The value to match on.
+        input: Value,
+        /// The type of the value.
+        ty: TypeId,
+        /// The primitive to match against the value.
+        val: Sym,
+    },
+
+    /// Try matching the given value as the given variant, producing `|arg_tys|`
+    /// values as output.
+    MatchVariant {
+        /// The value to match on.
+        input: Value,
+        /// The type of the value.
+        input_ty: TypeId,
+        /// The types of values produced upon a successful match.
+        arg_tys: Vec<TypeId>,
+        /// The value type's variant that we are matching against.
+        variant: VariantId,
+    },
+
+    /// Evaluate an expression and provide the given value as the result of this
+    /// match instruction. The expression has access to the pattern-values up to
+    /// this point in the sequence.
+    Expr {
+        /// The expression to evaluate.
+        seq: ExprSequence,
+        /// The value produced by the expression.
+        output: Value,
+        /// The type of the output value.
+        output_ty: TypeId,
+    },
+
+    // NB: this has to come second-to-last, because it might be infallible, for
+    // the same reasons that `Arg` has to be last.
+    //
+    /// Invoke an extractor, taking the given values as input (the first is the
+    /// value to extract, the other are the `Input`-polarity extractor args) and
+    /// producing an output value for each `Output`-polarity extractor arg.
+    Extract {
+        /// The value to extract, followed by polarity extractor args.
+        inputs: Vec<Value>,
+        /// The types of the inputs.
+        input_tys: Vec<TypeId>,
+        /// The types of the output values produced upon a successful match.
+        output_tys: Vec<TypeId>,
+        /// This extractor's term.
+        term: TermId,
+        /// Whether this extraction is infallible or not.
+        infallible: bool,
+    },
+
+    // NB: This has to go last, since it is infallible, so that when we sort
+    // edges in the trie, we visit infallible edges after first having tried the
+    // more-specific fallible options.
+    //
+    /// Get the Nth input argument, which corresponds to the Nth field
+    /// of the root term.
+    Arg {
+        /// The index of the argument to get.
+        index: usize,
+        /// The type of the argument.
+        ty: TypeId,
+    },
+}
+
+/// A single Expr instruction.
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum ExprInst {
+    /// Produce a constant integer.
+    ConstInt {
+        /// This integer type.
+        ty: TypeId,
+        /// The integer value. Must fit within the type.
+        val: i64,
+    },
+
+    /// Produce a constant extern value.
+    ConstPrim {
+        /// The primitive type.
+        ty: TypeId,
+        /// The primitive value.
+        val: Sym,
+    },
+
+    /// Create a variant.
+    CreateVariant {
+        /// The input arguments that will make up this variant's fields.
+        ///
+        /// These must be in the same order as the variant's fields.
+        inputs: Vec<(Value, TypeId)>,
+        /// The enum type.
+        ty: TypeId,
+        /// The variant within the enum that we are contructing.
+        variant: VariantId,
+    },
+
+    /// Invoke a constructor.
+    Construct {
+        /// The arguments to the constructor.
+        inputs: Vec<(Value, TypeId)>,
+        /// The type of the constructor.
+        ty: TypeId,
+        /// The constructor term.
+        term: TermId,
+        /// Whether this constructor is infallible or not.
+        infallible: bool,
+    },
+
+    /// Set the Nth return value. Produces no values.
+    Return {
+        /// The index of the return value to set.
+        index: usize,
+        /// The type of the return value.
+        ty: TypeId,
+        /// The value to set as the `index`th return value.
+        value: Value,
+    },
+}
+
+impl ExprInst {
+    /// Invoke `f` for each value in this expression.
+    pub fn visit_values<F: FnMut(Value)>(&self, mut f: F) {
+        match self {
+            &ExprInst::ConstInt { .. } => {}
+            &ExprInst::ConstPrim { .. } => {}
+            &ExprInst::Construct { ref inputs, .. }
+            | &ExprInst::CreateVariant { ref inputs, .. } => {
+                for (input, _ty) in inputs {
+                    f(*input);
+                }
+            }
+            &ExprInst::Return { value, .. } => {
+                f(value);
+            }
+        }
+    }
+}
+
+/// A linear sequence of instructions that match on and destructure an
+/// argument. A pattern is fallible (may not match). If it does not fail, its
+/// result consists of the values produced by the `PatternInst`s, which may be
+/// used by a subsequent `Expr`.
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)]
+pub struct PatternSequence {
+    /// Instruction sequence for pattern.
+    ///
+    /// `InstId` indexes into this sequence for `Value::Pattern` values.
+    pub insts: Vec<PatternInst>,
+}
+
+/// A linear sequence of instructions that produce a new value from the
+/// right-hand side of a rule, given bindings that come from a `Pattern` derived
+/// from the left-hand side.
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Default, PartialOrd, Ord)]
+pub struct ExprSequence {
+    /// Instruction sequence for expression.
+    ///
+    /// `InstId` indexes into this sequence for `Value::Expr` values.
+    pub insts: Vec<ExprInst>,
+    /// Position at which the rule producing this sequence was located.
+    pub pos: Pos,
+}
+
+impl ExprSequence {
+    /// Is this expression sequence producing a constant integer?
+    ///
+    /// If so, return the integer type and the constant.
+    pub fn is_const_int(&self) -> Option<(TypeId, i64)> {
+        if self.insts.len() == 2 && matches!(&self.insts[1], &ExprInst::Return { .. }) {
+            match &self.insts[0] {
+                &ExprInst::ConstInt { ty, val } => Some((ty, val)),
+                _ => None,
+            }
+        } else {
+            None
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+enum ValueOrArgs {
+    Value(Value),
+    ImplicitTermFromArgs(TermId),
+}
+
+impl ValueOrArgs {
+    fn to_value(&self) -> Option<Value> {
+        match self {
+            &ValueOrArgs::Value(v) => Some(v),
+            _ => None,
+        }
+    }
+}
+
+impl PatternSequence {
+    fn add_inst(&mut self, inst: PatternInst) -> InstId {
+        let id = InstId(self.insts.len());
+        self.insts.push(inst);
+        id
+    }
+
+    fn add_arg(&mut self, index: usize, ty: TypeId) -> Value {
+        let inst = InstId(self.insts.len());
+        self.add_inst(PatternInst::Arg { index, ty });
+        Value::Pattern { inst, output: 0 }
+    }
+
+    fn add_match_equal(&mut self, a: Value, b: Value, ty: TypeId) {
+        self.add_inst(PatternInst::MatchEqual { a, b, ty });
+    }
+
+    fn add_match_int(&mut self, input: Value, ty: TypeId, int_val: i64) {
+        self.add_inst(PatternInst::MatchInt { input, ty, int_val });
+    }
+
+    fn add_match_prim(&mut self, input: Value, ty: TypeId, val: Sym) {
+        self.add_inst(PatternInst::MatchPrim { input, ty, val });
+    }
+
+    fn add_match_variant(
+        &mut self,
+        input: Value,
+        input_ty: TypeId,
+        arg_tys: &[TypeId],
+        variant: VariantId,
+    ) -> Vec<Value> {
+        let inst = InstId(self.insts.len());
+        let mut outs = vec![];
+        for (i, _arg_ty) in arg_tys.iter().enumerate() {
+            let val = Value::Pattern { inst, output: i };
+            outs.push(val);
+        }
+        let arg_tys = arg_tys.iter().cloned().collect();
+        self.add_inst(PatternInst::MatchVariant {
+            input,
+            input_ty,
+            arg_tys,
+            variant,
+        });
+        outs
+    }
+
+    fn add_extract(
+        &mut self,
+        inputs: Vec<Value>,
+        input_tys: Vec<TypeId>,
+        output_tys: Vec<TypeId>,
+        term: TermId,
+        infallible: bool,
+    ) -> Vec<Value> {
+        let inst = InstId(self.insts.len());
+        let mut outs = vec![];
+        for i in 0..output_tys.len() {
+            let val = Value::Pattern { inst, output: i };
+            outs.push(val);
+        }
+        let output_tys = output_tys.iter().cloned().collect();
+        self.add_inst(PatternInst::Extract {
+            inputs,
+            input_tys,
+            output_tys,
+            term,
+            infallible,
+        });
+        outs
+    }
+
+    fn add_expr_seq(&mut self, seq: ExprSequence, output: Value, output_ty: TypeId) -> Value {
+        let inst = self.add_inst(PatternInst::Expr {
+            seq,
+            output,
+            output_ty,
+        });
+
+        // Create values for all outputs.
+        Value::Pattern { inst, output: 0 }
+    }
+
+    /// Generate PatternInsts to match the given (sub)pattern. Works
+    /// recursively down the AST.
+    fn gen_pattern(
+        &mut self,
+        input: ValueOrArgs,
+        typeenv: &TypeEnv,
+        termenv: &TermEnv,
+        pat: &Pattern,
+        vars: &mut BTreeMap<VarId, Value>,
+    ) {
+        match pat {
+            &Pattern::BindPattern(_ty, var, ref subpat) => {
+                // Bind the appropriate variable and recurse.
+                assert!(!vars.contains_key(&var));
+                if let Some(v) = input.to_value() {
+                    vars.insert(var, v);
+                }
+                let root_term = self.gen_pattern(input, typeenv, termenv, &*subpat, vars);
+                root_term
+            }
+            &Pattern::Var(ty, var) => {
+                // Assert that the value matches the existing bound var.
+                let var_val = vars
+                    .get(&var)
+                    .cloned()
+                    .expect("Variable should already be bound");
+                let input_val = input
+                    .to_value()
+                    .expect("Cannot match an =var pattern against root term");
+                self.add_match_equal(input_val, var_val, ty);
+            }
+            &Pattern::ConstInt(ty, value) => {
+                // Assert that the value matches the constant integer.
+                let input_val = input
+                    .to_value()
+                    .expect("Cannot match an integer pattern against root term");
+                self.add_match_int(input_val, ty, value);
+            }
+            &Pattern::ConstPrim(ty, value) => {
+                let input_val = input
+                    .to_value()
+                    .expect("Cannot match a constant-primitive pattern against root term");
+                self.add_match_prim(input_val, ty, value);
+            }
+            &Pattern::Term(ty, term, ref args) => {
+                match input {
+                    ValueOrArgs::ImplicitTermFromArgs(termid) => {
+                        assert_eq!(
+                            termid, term,
+                            "Cannot match a different term against root pattern"
+                        );
+                        let termdata = &termenv.terms[term.index()];
+                        let arg_tys = &termdata.arg_tys[..];
+                        for (i, subpat) in args.iter().enumerate() {
+                            let value = self.add_arg(i, arg_tys[i]);
+                            let subpat = match subpat {
+                                &TermArgPattern::Expr(..) => {
+                                    panic!("Should have been caught in typechecking")
+                                }
+                                &TermArgPattern::Pattern(ref pat) => pat,
+                            };
+                            self.gen_pattern(
+                                ValueOrArgs::Value(value),
+                                typeenv,
+                                termenv,
+                                subpat,
+                                vars,
+                            );
+                        }
+                    }
+                    ValueOrArgs::Value(input) => {
+                        // Determine whether the term has an external extractor or not.
+                        let termdata = &termenv.terms[term.index()];
+                        let arg_tys = &termdata.arg_tys[..];
+                        match &termdata.kind {
+                            TermKind::EnumVariant { variant } => {
+                                let arg_values =
+                                    self.add_match_variant(input, ty, arg_tys, *variant);
+                                for (subpat, value) in args.iter().zip(arg_values.into_iter()) {
+                                    let subpat = match subpat {
+                                        &TermArgPattern::Pattern(ref pat) => pat,
+                                        _ => unreachable!("Should have been caught by sema"),
+                                    };
+                                    self.gen_pattern(
+                                        ValueOrArgs::Value(value),
+                                        typeenv,
+                                        termenv,
+                                        subpat,
+                                        vars,
+                                    );
+                                }
+                            }
+                            TermKind::Decl {
+                                extractor_kind: None,
+                                ..
+                            } => {
+                                panic!("Pattern invocation of undefined term body")
+                            }
+                            TermKind::Decl {
+                                extractor_kind: Some(ExtractorKind::InternalExtractor { .. }),
+                                ..
+                            } => {
+                                panic!("Should have been expanded away")
+                            }
+                            TermKind::Decl {
+                                extractor_kind:
+                                    Some(ExtractorKind::ExternalExtractor {
+                                        ref arg_polarity,
+                                        infallible,
+                                        ..
+                                    }),
+                                ..
+                            } => {
+                                // Evaluate all `input` args.
+                                let mut inputs = vec![];
+                                let mut input_tys = vec![];
+                                let mut output_tys = vec![];
+                                let mut output_pats = vec![];
+                                inputs.push(input);
+                                input_tys.push(termdata.ret_ty);
+                                for (arg, pol) in args.iter().zip(arg_polarity.iter()) {
+                                    match pol {
+                                        &ArgPolarity::Input => {
+                                            let expr = match arg {
+                                                &TermArgPattern::Expr(ref expr) => expr,
+                                                _ => panic!(
+                                                    "Should have been caught by typechecking"
+                                                ),
+                                            };
+                                            let mut seq = ExprSequence::default();
+                                            let value = seq.gen_expr(typeenv, termenv, expr, vars);
+                                            seq.add_return(expr.ty(), value);
+                                            let value = self.add_expr_seq(seq, value, expr.ty());
+                                            inputs.push(value);
+                                            input_tys.push(expr.ty());
+                                        }
+                                        &ArgPolarity::Output => {
+                                            let pat = match arg {
+                                                &TermArgPattern::Pattern(ref pat) => pat,
+                                                _ => panic!(
+                                                    "Should have been caught by typechecking"
+                                                ),
+                                            };
+                                            output_tys.push(pat.ty());
+                                            output_pats.push(pat);
+                                        }
+                                    }
+                                }
+
+                                // Invoke the extractor.
+                                let arg_values = self.add_extract(
+                                    inputs,
+                                    input_tys,
+                                    output_tys,
+                                    term,
+                                    *infallible,
+                                );
+
+                                for (pat, &val) in output_pats.iter().zip(arg_values.iter()) {
+                                    self.gen_pattern(
+                                        ValueOrArgs::Value(val),
+                                        typeenv,
+                                        termenv,
+                                        pat,
+                                        vars,
+                                    );
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            &Pattern::And(_ty, ref children) => {
+                for child in children {
+                    self.gen_pattern(input, typeenv, termenv, child, vars);
+                }
+            }
+            &Pattern::Wildcard(_ty) => {
+                // Nothing!
+            }
+        }
+    }
+}
+
+impl ExprSequence {
+    fn add_inst(&mut self, inst: ExprInst) -> InstId {
+        let id = InstId(self.insts.len());
+        self.insts.push(inst);
+        id
+    }
+
+    fn add_const_int(&mut self, ty: TypeId, val: i64) -> Value {
+        let inst = InstId(self.insts.len());
+        self.add_inst(ExprInst::ConstInt { ty, val });
+        Value::Expr { inst, output: 0 }
+    }
+
+    fn add_const_prim(&mut self, ty: TypeId, val: Sym) -> Value {
+        let inst = InstId(self.insts.len());
+        self.add_inst(ExprInst::ConstPrim { ty, val });
+        Value::Expr { inst, output: 0 }
+    }
+
+    fn add_create_variant(
+        &mut self,
+        inputs: &[(Value, TypeId)],
+        ty: TypeId,
+        variant: VariantId,
+    ) -> Value {
+        let inst = InstId(self.insts.len());
+        let inputs = inputs.iter().cloned().collect();
+        self.add_inst(ExprInst::CreateVariant {
+            inputs,
+            ty,
+            variant,
+        });
+        Value::Expr { inst, output: 0 }
+    }
+
+    fn add_construct(
+        &mut self,
+        inputs: &[(Value, TypeId)],
+        ty: TypeId,
+        term: TermId,
+        infallible: bool,
+    ) -> Value {
+        let inst = InstId(self.insts.len());
+        let inputs = inputs.iter().cloned().collect();
+        self.add_inst(ExprInst::Construct {
+            inputs,
+            ty,
+            term,
+            infallible,
+        });
+        Value::Expr { inst, output: 0 }
+    }
+
+    fn add_return(&mut self, ty: TypeId, value: Value) {
+        self.add_inst(ExprInst::Return {
+            index: 0,
+            ty,
+            value,
+        });
+    }
+
+    /// Creates a sequence of ExprInsts to generate the given
+    /// expression value. Returns the value ID as well as the root
+    /// term ID, if any.
+    fn gen_expr(
+        &mut self,
+        typeenv: &TypeEnv,
+        termenv: &TermEnv,
+        expr: &Expr,
+        vars: &BTreeMap<VarId, Value>,
+    ) -> Value {
+        log::trace!("gen_expr: expr {:?}", expr);
+        match expr {
+            &Expr::ConstInt(ty, val) => self.add_const_int(ty, val),
+            &Expr::ConstPrim(ty, val) => self.add_const_prim(ty, val),
+            &Expr::Let {
+                ty: _ty,
+                ref bindings,
+                ref body,
+            } => {
+                let mut vars = vars.clone();
+                for &(var, _var_ty, ref var_expr) in bindings {
+                    let var_value = self.gen_expr(typeenv, termenv, &*var_expr, &vars);
+                    vars.insert(var, var_value);
+                }
+                self.gen_expr(typeenv, termenv, body, &vars)
+            }
+            &Expr::Var(_ty, var_id) => vars.get(&var_id).cloned().unwrap(),
+            &Expr::Term(ty, term, ref arg_exprs) => {
+                let termdata = &termenv.terms[term.index()];
+                let mut arg_values_tys = vec![];
+                for (arg_ty, arg_expr) in termdata.arg_tys.iter().cloned().zip(arg_exprs.iter()) {
+                    arg_values_tys
+                        .push((self.gen_expr(typeenv, termenv, &*arg_expr, &vars), arg_ty));
+                }
+                match &termdata.kind {
+                    TermKind::EnumVariant { variant } => {
+                        self.add_create_variant(&arg_values_tys[..], ty, *variant)
+                    }
+                    TermKind::Decl {
+                        constructor_kind: Some(ConstructorKind::InternalConstructor),
+                        ..
+                    } => {
+                        self.add_construct(
+                            &arg_values_tys[..],
+                            ty,
+                            term,
+                            /* infallible = */ false,
+                        )
+                    }
+                    TermKind::Decl {
+                        constructor_kind: Some(ConstructorKind::ExternalConstructor { .. }),
+                        ..
+                    } => {
+                        self.add_construct(
+                            &arg_values_tys[..],
+                            ty,
+                            term,
+                            /* infallible = */ true,
+                        )
+                    }
+                    TermKind::Decl {
+                        constructor_kind: None,
+                        ..
+                    } => panic!("Should have been caught by typechecking"),
+                }
+            }
+        }
+    }
+}
+
+/// Build a sequence from a rule.
+pub fn lower_rule(
+    tyenv: &TypeEnv,
+    termenv: &TermEnv,
+    rule: RuleId,
+) -> (PatternSequence, ExprSequence) {
+    let mut pattern_seq: PatternSequence = Default::default();
+    let mut expr_seq: ExprSequence = Default::default();
+    expr_seq.pos = termenv.rules[rule.index()].pos;
+
+    let ruledata = &termenv.rules[rule.index()];
+    let mut vars = BTreeMap::new();
+    let root_term = ruledata
+        .lhs
+        .root_term()
+        .expect("Pattern must have a term at the root");
+
+    log::trace!("lower_rule: ruledata {:?}", ruledata,);
+
+    // Lower the pattern, starting from the root input value.
+    pattern_seq.gen_pattern(
+        ValueOrArgs::ImplicitTermFromArgs(root_term),
+        tyenv,
+        termenv,
+        &ruledata.lhs,
+        &mut vars,
+    );
+
+    // Lower the expression, making use of the bound variables
+    // from the pattern.
+    let rhs_root_val = expr_seq.gen_expr(tyenv, termenv, &ruledata.rhs, &vars);
+    // Return the root RHS value.
+    let output_ty = ruledata.rhs.ty();
+    expr_seq.add_return(output_ty, rhs_root_val);
+    (pattern_seq, expr_seq)
+}
diff --git a/cranelift/isle/isle/src/lexer.rs b/cranelift/isle/isle/src/lexer.rs
new file mode 100644
index 0000000000..e9a4e7b063
--- /dev/null
+++ b/cranelift/isle/isle/src/lexer.rs
@@ -0,0 +1,405 @@
+//! Lexer for the ISLE language.
+
+use crate::error::{Error, Result, Source};
+use std::borrow::Cow;
+use std::path::Path;
+use std::sync::Arc;
+
+/// The lexer.
+///
+/// Breaks source text up into a sequence of tokens (with source positions).
+#[derive(Clone, Debug)]
+pub struct Lexer<'a> {
+    /// Arena of filenames from the input source.
+    ///
+    /// Indexed via `Pos::file`.
+    pub filenames: Vec<Arc<str>>,
+
+    /// Arena of file source texts.
+    ///
+    /// Indexed via `Pos::file`.
+    pub file_texts: Vec<Arc<str>>,
+
+    file_starts: Vec<usize>,
+    buf: Cow<'a, [u8]>,
+    pos: Pos,
+    lookahead: Option<(Pos, Token)>,
+}
+
+/// A source position.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Hash, PartialOrd, Ord)]
+pub struct Pos {
+    /// This source position's file.
+    ///
+    /// Indexes into `Lexer::filenames` early in the compiler pipeline, and
+    /// later into `TypeEnv::filenames` once we get into semantic analysis.
+    pub file: usize,
+    /// This source position's byte offset in the file.
+    pub offset: usize,
+    /// This source position's line number in the file.
+    pub line: usize,
+    /// This source position's column number in the file.
+    pub col: usize,
+}
+
+impl Pos {
+    /// Print this source position as `file.isle:12:34`.
+    pub fn pretty_print(&self, filenames: &[Arc<str>]) -> String {
+        format!("{}:{}:{}", filenames[self.file], self.line, self.col)
+    }
+    /// Print this source position as `file.isle line 12`.
+    pub fn pretty_print_line(&self, filenames: &[Arc<str>]) -> String {
+        format!("{} line {}", filenames[self.file], self.line)
+    }
+}
+
+/// A token of ISLE source.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Token {
+    /// Left paren.
+    LParen,
+    /// Right paren.
+    RParen,
+    /// A symbol, e.g. `Foo`.
+    Symbol(String),
+    /// An integer.
+    Int(i64),
+    /// `@`
+    At,
+    /// `<`
+    Lt,
+}
+
+impl<'a> Lexer<'a> {
+    /// Create a new lexer for the given source contents and filename.
+    pub fn from_str(s: &'a str, filename: &'a str) -> Result<Lexer<'a>> {
+        let mut l = Lexer {
+            filenames: vec![filename.into()],
+            file_texts: vec![s.into()],
+            file_starts: vec![0],
+            buf: Cow::Borrowed(s.as_bytes()),
+            pos: Pos {
+                file: 0,
+                offset: 0,
+                line: 1,
+                col: 0,
+            },
+            lookahead: None,
+        };
+        l.reload()?;
+        Ok(l)
+    }
+
+    /// Create a new lexer from the given files.
+    pub fn from_files<P>(file_paths: impl IntoIterator<Item = P>) -> Result<Lexer<'a>>
+    where
+        P: AsRef<Path>,
+    {
+        let mut filenames = Vec::<Arc<str>>::new();
+        let mut file_texts = Vec::<Arc<str>>::new();
+
+        for f in file_paths {
+            let f = f.as_ref();
+
+            filenames.push(f.display().to_string().into());
+
+            let s = std::fs::read_to_string(f)
+                .map_err(|e| Error::from_io(e, format!("failed to read file: {}", f.display())))?;
+            file_texts.push(s.into());
+        }
+
+        assert!(!filenames.is_empty());
+
+        let mut file_starts = vec![];
+        let mut buf = String::new();
+        for text in &file_texts {
+            file_starts.push(buf.len());
+            buf += &text;
+            buf += "\n";
+        }
+
+        let mut l = Lexer {
+            filenames,
+            file_texts,
+            buf: Cow::Owned(buf.into_bytes()),
+            file_starts,
+            pos: Pos {
+                file: 0,
+                offset: 0,
+                line: 1,
+                col: 0,
+            },
+            lookahead: None,
+        };
+        l.reload()?;
+        Ok(l)
+    }
+
+    /// Get the lexer's current source position.
+    pub fn pos(&self) -> Pos {
+        Pos {
+            file: self.pos.file,
+            offset: self.pos.offset - self.file_starts[self.pos.file],
+            line: self.pos.line,
+            col: self.pos.file,
+        }
+    }
+
+    fn advance_pos(&mut self) {
+        self.pos.col += 1;
+        if self.buf[self.pos.offset] == b'\n' {
+            self.pos.line += 1;
+            self.pos.col = 0;
+        }
+        self.pos.offset += 1;
+        if self.pos.file + 1 < self.file_starts.len() {
+            let next_start = self.file_starts[self.pos.file + 1];
+            if self.pos.offset >= next_start {
+                assert!(self.pos.offset == next_start);
+                self.pos.file += 1;
+                self.pos.line = 1;
+            }
+        }
+    }
+
+    fn error(&self, pos: Pos, msg: impl Into<String>) -> Error {
+        Error::ParseError {
+            msg: msg.into(),
+            src: Source::new(
+                self.filenames[pos.file].clone(),
+                self.file_texts[pos.file].clone(),
+            ),
+            span: miette::SourceSpan::from((self.pos().offset, 1)),
+        }
+    }
+
+    fn next_token(&mut self) -> Result<Option<(Pos, Token)>> {
+        fn is_sym_first_char(c: u8) -> bool {
+            match c {
+                b'-' | b'0'..=b'9' | b'(' | b')' | b';' => false,
+                c if c.is_ascii_whitespace() => false,
+                _ => true,
+            }
+        }
+        fn is_sym_other_char(c: u8) -> bool {
+            match c {
+                b'(' | b')' | b';' | b'@' | b'<' => false,
+                c if c.is_ascii_whitespace() => false,
+                _ => true,
+            }
+        }
+
+        // Skip any whitespace and any comments.
+        while self.pos.offset < self.buf.len() {
+            if self.buf[self.pos.offset].is_ascii_whitespace() {
+                self.advance_pos();
+                continue;
+            }
+            if self.buf[self.pos.offset] == b';' {
+                while self.pos.offset < self.buf.len() && self.buf[self.pos.offset] != b'\n' {
+                    self.advance_pos();
+                }
+                continue;
+            }
+            break;
+        }
+
+        if self.pos.offset == self.buf.len() {
+            return Ok(None);
+        }
+
+        let char_pos = self.pos();
+        match self.buf[self.pos.offset] {
+            b'(' => {
+                self.advance_pos();
+                Ok(Some((char_pos, Token::LParen)))
+            }
+            b')' => {
+                self.advance_pos();
+                Ok(Some((char_pos, Token::RParen)))
+            }
+            b'@' => {
+                self.advance_pos();
+                Ok(Some((char_pos, Token::At)))
+            }
+            b'<' => {
+                self.advance_pos();
+                Ok(Some((char_pos, Token::Lt)))
+            }
+            c if is_sym_first_char(c) => {
+                let start = self.pos.offset;
+                let start_pos = self.pos();
+                while self.pos.offset < self.buf.len()
+                    && is_sym_other_char(self.buf[self.pos.offset])
+                {
+                    self.advance_pos();
+                }
+                let end = self.pos.offset;
+                let s = std::str::from_utf8(&self.buf[start..end])
+                    .expect("Only ASCII characters, should be UTF-8");
+                debug_assert!(!s.is_empty());
+                Ok(Some((start_pos, Token::Symbol(s.to_string()))))
+            }
+            c if (c >= b'0' && c <= b'9') || c == b'-' => {
+                let start_pos = self.pos();
+                let neg = if c == b'-' {
+                    self.advance_pos();
+                    true
+                } else {
+                    false
+                };
+
+                let mut radix = 10;
+
+                // Check for hex literals.
+                if self.buf.get(self.pos.offset).copied() == Some(b'0')
+                    && self.buf.get(self.pos.offset + 1).copied() == Some(b'x')
+                {
+                    self.advance_pos();
+                    self.advance_pos();
+                    radix = 16;
+                }
+
+                // Find the range in the buffer for this integer literal. We'll
+                // pass this range to `i64::from_str_radix` to do the actual
+                // string-to-integer conversion.
+                let start_offset = self.pos.offset;
+                while self.pos.offset < self.buf.len()
+                    && ((radix == 10
+                        && self.buf[self.pos.offset] >= b'0'
+                        && self.buf[self.pos.offset] <= b'9')
+                        || (radix == 16
+                            && ((self.buf[self.pos.offset] >= b'0'
+                                && self.buf[self.pos.offset] <= b'9')
+                                || (self.buf[self.pos.offset] >= b'a'
+                                    && self.buf[self.pos.offset] <= b'f')
+                                || (self.buf[self.pos.offset] >= b'A'
+                                    && self.buf[self.pos.offset] <= b'F'))))
+                {
+                    self.advance_pos();
+                }
+                let end_offset = self.pos.offset;
+
+                let num = i64::from_str_radix(
+                    std::str::from_utf8(&self.buf[start_offset..end_offset]).unwrap(),
+                    radix,
+                )
+                .map_err(|e| self.error(start_pos, e.to_string()))?;
+
+                let tok = if neg {
+                    Token::Int(num.checked_neg().ok_or_else(|| {
+                        self.error(start_pos, "integer literal cannot fit in i64")
+                    })?)
+                } else {
+                    Token::Int(num)
+                };
+                Ok(Some((start_pos, tok)))
+            }
+            c => panic!("Unexpected character '{}' at offset {}", c, self.pos.offset),
+        }
+    }
+
+    /// Get the next token from this lexer's token stream, if any.
+    pub fn next(&mut self) -> Result<Option<(Pos, Token)>> {
+        let tok = self.lookahead.take();
+        self.reload()?;
+        Ok(tok)
+    }
+
+    fn reload(&mut self) -> Result<()> {
+        if self.lookahead.is_none() && self.pos.offset < self.buf.len() {
+            self.lookahead = self.next_token()?;
+        }
+        Ok(())
+    }
+
+    /// Peek ahead at the next token.
+    pub fn peek(&self) -> Option<&(Pos, Token)> {
+        self.lookahead.as_ref()
+    }
+
+    /// Are we at the end of the source input?
+    pub fn eof(&self) -> bool {
+        self.lookahead.is_none()
+    }
+}
+
+impl Token {
+    /// Is this an `Int` token?
+    pub fn is_int(&self) -> bool {
+        match self {
+            Token::Int(_) => true,
+            _ => false,
+        }
+    }
+
+    /// Is this a `Sym` token?
+    pub fn is_sym(&self) -> bool {
+        match self {
+            Token::Symbol(_) => true,
+            _ => false,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    fn lex(s: &str, file: &str) -> Vec<Token> {
+        let mut toks = vec![];
+        let mut lexer = Lexer::from_str(s, file).unwrap();
+        while let Some((_, tok)) = lexer.next().unwrap() {
+            toks.push(tok);
+        }
+        toks
+    }
+
+    #[test]
+    fn lexer_basic() {
+        assert_eq!(
+            lex(
+                ";; comment\n; another\r\n   \t(one two three 23 -568  )\n",
+                "lexer_basic"
+            ),
+            vec![
+                Token::LParen,
+                Token::Symbol("one".to_string()),
+                Token::Symbol("two".to_string()),
+                Token::Symbol("three".to_string()),
+                Token::Int(23),
+                Token::Int(-568),
+                Token::RParen
+            ]
+        );
+    }
+
+    #[test]
+    fn ends_with_sym() {
+        assert_eq!(
+            lex("asdf", "ends_with_sym"),
+            vec![Token::Symbol("asdf".to_string()),]
+        );
+    }
+
+    #[test]
+    fn ends_with_num() {
+        assert_eq!(lex("23", "ends_with_num"), vec![Token::Int(23)],);
+    }
+
+    #[test]
+    fn weird_syms() {
+        assert_eq!(
+            lex("(+ [] => !! _test!;comment\n)", "weird_syms"),
+            vec![
+                Token::LParen,
+                Token::Symbol("+".to_string()),
+                Token::Symbol("[]".to_string()),
+                Token::Symbol("=>".to_string()),
+                Token::Symbol("!!".to_string()),
+                Token::Symbol("_test!".to_string()),
+                Token::RParen,
+            ]
+        );
+    }
+}
diff --git a/cranelift/isle/isle/src/lib.rs b/cranelift/isle/isle/src/lib.rs
new file mode 100644
index 0000000000..c516a78d9e
--- /dev/null
+++ b/cranelift/isle/isle/src/lib.rs
@@ -0,0 +1,29 @@
+#![doc = include_str!("../README.md")]
+#![deny(missing_docs)]
+
+macro_rules! declare_id {
+    (
+        $(#[$attr:meta])*
+            $name:ident
+    ) => {
+        $(#[$attr])*
+            #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+        pub struct $name(pub usize);
+        impl $name {
+            /// Get the index of this id.
+            pub fn index(self) -> usize {
+                self.0
+            }
+        }
+    };
+}
+
+pub mod ast;
+pub mod codegen;
+pub mod compile;
+pub mod error;
+pub mod ir;
+pub mod lexer;
+pub mod parser;
+pub mod sema;
+pub mod trie;
diff --git a/cranelift/isle/isle/src/parser.rs b/cranelift/isle/isle/src/parser.rs
new file mode 100644
index 0000000000..8d8bbad5d8
--- /dev/null
+++ b/cranelift/isle/isle/src/parser.rs
@@ -0,0 +1,504 @@
+//! Parser for ISLE language.
+
+use crate::ast::*;
+use crate::error::*;
+use crate::lexer::{Lexer, Pos, Token};
+
+/// Parse the top-level ISLE definitions and return their AST.
+pub fn parse(lexer: Lexer) -> Result<Defs> {
+    let parser = Parser::new(lexer);
+    parser.parse_defs()
+}
+
+/// The ISLE parser.
+///
+/// Takes in a lexer and creates an AST.
+#[derive(Clone, Debug)]
+struct Parser<'a> {
+    lexer: Lexer<'a>,
+}
+
+impl<'a> Parser<'a> {
+    /// Construct a new parser from the given lexer.
+    pub fn new(lexer: Lexer<'a>) -> Parser<'a> {
+        Parser { lexer }
+    }
+
+    fn error(&self, pos: Pos, msg: String) -> Error {
+        Error::ParseError {
+            msg,
+            src: Source::new(
+                self.lexer.filenames[pos.file].clone(),
+                self.lexer.file_texts[pos.file].clone(),
+            ),
+            span: miette::SourceSpan::from((pos.offset, 1)),
+        }
+    }
+
+    fn take<F: Fn(&Token) -> bool>(&mut self, f: F) -> Result<Token> {
+        if let Some(&(pos, ref peek)) = self.lexer.peek() {
+            if !f(peek) {
+                return Err(self.error(pos, format!("Unexpected token {:?}", peek)));
+            }
+            Ok(self.lexer.next()?.unwrap().1)
+        } else {
+            Err(self.error(self.lexer.pos(), "Unexpected EOF".to_string()))
+        }
+    }
+
+    fn is<F: Fn(&Token) -> bool>(&self, f: F) -> bool {
+        if let Some(&(_, ref peek)) = self.lexer.peek() {
+            f(peek)
+        } else {
+            false
+        }
+    }
+
+    fn pos(&self) -> Pos {
+        self.lexer
+            .peek()
+            .map_or_else(|| self.lexer.pos(), |(pos, _)| *pos)
+    }
+
+    fn is_lparen(&self) -> bool {
+        self.is(|tok| *tok == Token::LParen)
+    }
+    fn is_rparen(&self) -> bool {
+        self.is(|tok| *tok == Token::RParen)
+    }
+    fn is_at(&self) -> bool {
+        self.is(|tok| *tok == Token::At)
+    }
+    fn is_lt(&self) -> bool {
+        self.is(|tok| *tok == Token::Lt)
+    }
+    fn is_sym(&self) -> bool {
+        self.is(|tok| tok.is_sym())
+    }
+    fn is_int(&self) -> bool {
+        self.is(|tok| tok.is_int())
+    }
+    fn is_sym_str(&self, s: &str) -> bool {
+        self.is(|tok| match tok {
+            &Token::Symbol(ref tok_s) if tok_s == s => true,
+            _ => false,
+        })
+    }
+
+    fn is_const(&self) -> bool {
+        self.is(|tok| match tok {
+            &Token::Symbol(ref tok_s) if tok_s.starts_with("$") => true,
+            _ => false,
+        })
+    }
+
+    fn lparen(&mut self) -> Result<()> {
+        self.take(|tok| *tok == Token::LParen).map(|_| ())
+    }
+    fn rparen(&mut self) -> Result<()> {
+        self.take(|tok| *tok == Token::RParen).map(|_| ())
+    }
+    fn at(&mut self) -> Result<()> {
+        self.take(|tok| *tok == Token::At).map(|_| ())
+    }
+    fn lt(&mut self) -> Result<()> {
+        self.take(|tok| *tok == Token::Lt).map(|_| ())
+    }
+
+    fn symbol(&mut self) -> Result<String> {
+        match self.take(|tok| tok.is_sym())? {
+            Token::Symbol(s) => Ok(s),
+            _ => unreachable!(),
+        }
+    }
+
+    fn int(&mut self) -> Result<i64> {
+        match self.take(|tok| tok.is_int())? {
+            Token::Int(i) => Ok(i),
+            _ => unreachable!(),
+        }
+    }
+
+    fn parse_defs(mut self) -> Result<Defs> {
+        let mut defs = vec![];
+        while !self.lexer.eof() {
+            defs.push(self.parse_def()?);
+        }
+        Ok(Defs {
+            defs,
+            filenames: self.lexer.filenames,
+            file_texts: self.lexer.file_texts,
+        })
+    }
+
+    fn parse_def(&mut self) -> Result<Def> {
+        self.lparen()?;
+        let pos = self.pos();
+        let def = match &self.symbol()?[..] {
+            "type" => Def::Type(self.parse_type()?),
+            "decl" => Def::Decl(self.parse_decl()?),
+            "rule" => Def::Rule(self.parse_rule()?),
+            "extractor" => Def::Extractor(self.parse_etor()?),
+            "extern" => Def::Extern(self.parse_extern()?),
+            s => {
+                return Err(self.error(pos, format!("Unexpected identifier: {}", s)));
+            }
+        };
+        self.rparen()?;
+        Ok(def)
+    }
+
+    fn str_to_ident(&self, pos: Pos, s: &str) -> Result<Ident> {
+        let first = s
+            .chars()
+            .next()
+            .ok_or_else(|| self.error(pos, "empty symbol".into()))?;
+        if !first.is_alphabetic() && first != '_' && first != '$' {
+            return Err(self.error(
+                pos,
+                format!("Identifier '{}' does not start with letter or _ or $", s),
+            ));
+        }
+        if s.chars()
+            .skip(1)
+            .any(|c| !c.is_alphanumeric() && c != '_' && c != '.' && c != '$')
+        {
+            return Err(self.error(
+                pos,
+                format!(
+                    "Identifier '{}' contains invalid character (not a-z, A-Z, 0-9, _, ., $)",
+                    s
+                ),
+            ));
+        }
+        Ok(Ident(s.to_string(), pos))
+    }
+
+    fn parse_ident(&mut self) -> Result<Ident> {
+        let pos = self.pos();
+        let s = self.symbol()?;
+        self.str_to_ident(pos, &s)
+    }
+
+    fn parse_const(&mut self) -> Result<Ident> {
+        let pos = self.pos();
+        let ident = self.parse_ident()?;
+        if ident.0.starts_with("$") {
+            let s = &ident.0[1..];
+            Ok(Ident(s.to_string(), ident.1))
+        } else {
+            Err(self.error(
+                pos,
+                "Not a constant identifier; must start with a '$'".to_string(),
+            ))
+        }
+    }
+
+    fn parse_type(&mut self) -> Result<Type> {
+        let pos = self.pos();
+        let name = self.parse_ident()?;
+        let mut is_extern = false;
+        if self.is_sym_str("extern") {
+            self.symbol()?;
+            is_extern = true;
+        }
+        let ty = self.parse_typevalue()?;
+        Ok(Type {
+            name,
+            is_extern,
+            ty,
+            pos,
+        })
+    }
+
+    fn parse_typevalue(&mut self) -> Result<TypeValue> {
+        let pos = self.pos();
+        self.lparen()?;
+        if self.is_sym_str("primitive") {
+            self.symbol()?;
+            let primitive_ident = self.parse_ident()?;
+            self.rparen()?;
+            Ok(TypeValue::Primitive(primitive_ident, pos))
+        } else if self.is_sym_str("enum") {
+            self.symbol()?;
+            let mut variants = vec![];
+            while !self.is_rparen() {
+                let variant = self.parse_type_variant()?;
+                variants.push(variant);
+            }
+            self.rparen()?;
+            Ok(TypeValue::Enum(variants, pos))
+        } else {
+            Err(self.error(pos, "Unknown type definition".to_string()))
+        }
+    }
+
+    fn parse_type_variant(&mut self) -> Result<Variant> {
+        if self.is_sym() {
+            let pos = self.pos();
+            let name = self.parse_ident()?;
+            Ok(Variant {
+                name,
+                fields: vec![],
+                pos,
+            })
+        } else {
+            let pos = self.pos();
+            self.lparen()?;
+            let name = self.parse_ident()?;
+            let mut fields = vec![];
+            while !self.is_rparen() {
+                fields.push(self.parse_type_field()?);
+            }
+            self.rparen()?;
+            Ok(Variant { name, fields, pos })
+        }
+    }
+
+    fn parse_type_field(&mut self) -> Result<Field> {
+        let pos = self.pos();
+        self.lparen()?;
+        let name = self.parse_ident()?;
+        let ty = self.parse_ident()?;
+        self.rparen()?;
+        Ok(Field { name, ty, pos })
+    }
+
+    fn parse_decl(&mut self) -> Result<Decl> {
+        let pos = self.pos();
+        let term = self.parse_ident()?;
+
+        self.lparen()?;
+        let mut arg_tys = vec![];
+        while !self.is_rparen() {
+            arg_tys.push(self.parse_ident()?);
+        }
+        self.rparen()?;
+
+        let ret_ty = self.parse_ident()?;
+
+        Ok(Decl {
+            term,
+            arg_tys,
+            ret_ty,
+            pos,
+        })
+    }
+
+    fn parse_extern(&mut self) -> Result<Extern> {
+        let pos = self.pos();
+        if self.is_sym_str("constructor") {
+            self.symbol()?;
+            let term = self.parse_ident()?;
+            let func = self.parse_ident()?;
+            Ok(Extern::Constructor { term, func, pos })
+        } else if self.is_sym_str("extractor") {
+            self.symbol()?;
+
+            let infallible = if self.is_sym_str("infallible") {
+                self.symbol()?;
+                true
+            } else {
+                false
+            };
+
+            let term = self.parse_ident()?;
+            let func = self.parse_ident()?;
+
+            let arg_polarity = if self.is_lparen() {
+                let mut pol = vec![];
+                self.lparen()?;
+                while !self.is_rparen() {
+                    if self.is_sym_str("in") {
+                        self.symbol()?;
+                        pol.push(ArgPolarity::Input);
+                    } else if self.is_sym_str("out") {
+                        self.symbol()?;
+                        pol.push(ArgPolarity::Output);
+                    } else {
+                        return Err(self.error(pos, "Invalid argument polarity".to_string()));
+                    }
+                }
+                self.rparen()?;
+                Some(pol)
+            } else {
+                None
+            };
+            Ok(Extern::Extractor {
+                term,
+                func,
+                pos,
+                arg_polarity,
+                infallible,
+            })
+        } else if self.is_sym_str("const") {
+            self.symbol()?;
+            let pos = self.pos();
+            let name = self.parse_const()?;
+            let ty = self.parse_ident()?;
+            Ok(Extern::Const { name, ty, pos })
+        } else {
+            Err(self.error(
+                pos,
+                "Invalid extern: must be (extern constructor ...) or (extern extractor ...)"
+                    .to_string(),
+            ))
+        }
+    }
+
+    fn parse_etor(&mut self) -> Result<Extractor> {
+        let pos = self.pos();
+        self.lparen()?;
+        let term = self.parse_ident()?;
+        let mut args = vec![];
+        while !self.is_rparen() {
+            args.push(self.parse_ident()?);
+        }
+        self.rparen()?;
+        let template = self.parse_pattern()?;
+        Ok(Extractor {
+            term,
+            args,
+            template,
+            pos,
+        })
+    }
+
+    fn parse_rule(&mut self) -> Result<Rule> {
+        let pos = self.pos();
+        let prio = if self.is_int() {
+            Some(self.int()?)
+        } else {
+            None
+        };
+        let pattern = self.parse_pattern()?;
+        let expr = self.parse_expr()?;
+        Ok(Rule {
+            pattern,
+            expr,
+            pos,
+            prio,
+        })
+    }
+
+    fn parse_pattern(&mut self) -> Result<Pattern> {
+        let pos = self.pos();
+        if self.is_int() {
+            Ok(Pattern::ConstInt {
+                val: self.int()?,
+                pos,
+            })
+        } else if self.is_const() {
+            let val = self.parse_const()?;
+            Ok(Pattern::ConstPrim { val, pos })
+        } else if self.is_sym_str("_") {
+            self.symbol()?;
+            Ok(Pattern::Wildcard { pos })
+        } else if self.is_sym() {
+            let s = self.symbol()?;
+            if s.starts_with("=") {
+                let s = &s[1..];
+                let var = self.str_to_ident(pos, s)?;
+                Ok(Pattern::Var { var, pos })
+            } else {
+                let var = self.str_to_ident(pos, &s)?;
+                if self.is_at() {
+                    self.at()?;
+                    let subpat = Box::new(self.parse_pattern()?);
+                    Ok(Pattern::BindPattern { var, subpat, pos })
+                } else {
+                    Ok(Pattern::BindPattern {
+                        var,
+                        subpat: Box::new(Pattern::Wildcard { pos }),
+                        pos,
+                    })
+                }
+            }
+        } else if self.is_lparen() {
+            self.lparen()?;
+            if self.is_sym_str("and") {
+                self.symbol()?;
+                let mut subpats = vec![];
+                while !self.is_rparen() {
+                    subpats.push(self.parse_pattern()?);
+                }
+                self.rparen()?;
+                Ok(Pattern::And { subpats, pos })
+            } else {
+                let sym = self.parse_ident()?;
+                let mut args = vec![];
+                while !self.is_rparen() {
+                    args.push(self.parse_pattern_term_arg()?);
+                }
+                self.rparen()?;
+                Ok(Pattern::Term { sym, args, pos })
+            }
+        } else {
+            Err(self.error(pos, "Unexpected pattern".into()))
+        }
+    }
+
+    fn parse_pattern_term_arg(&mut self) -> Result<TermArgPattern> {
+        if self.is_lt() {
+            self.lt()?;
+            Ok(TermArgPattern::Expr(self.parse_expr()?))
+        } else {
+            Ok(TermArgPattern::Pattern(self.parse_pattern()?))
+        }
+    }
+
+    fn parse_expr(&mut self) -> Result<Expr> {
+        let pos = self.pos();
+        if self.is_lparen() {
+            self.lparen()?;
+            if self.is_sym_str("let") {
+                self.symbol()?;
+                self.lparen()?;
+                let mut defs = vec![];
+                while !self.is_rparen() {
+                    let def = self.parse_letdef()?;
+                    defs.push(def);
+                }
+                self.rparen()?;
+                let body = Box::new(self.parse_expr()?);
+                self.rparen()?;
+                Ok(Expr::Let { defs, body, pos })
+            } else {
+                let sym = self.parse_ident()?;
+                let mut args = vec![];
+                while !self.is_rparen() {
+                    args.push(self.parse_expr()?);
+                }
+                self.rparen()?;
+                Ok(Expr::Term { sym, args, pos })
+            }
+        } else if self.is_sym_str("#t") {
+            self.symbol()?;
+            Ok(Expr::ConstInt { val: 1, pos })
+        } else if self.is_sym_str("#f") {
+            self.symbol()?;
+            Ok(Expr::ConstInt { val: 0, pos })
+        } else if self.is_const() {
+            let val = self.parse_const()?;
+            Ok(Expr::ConstPrim { val, pos })
+        } else if self.is_sym() {
+            let name = self.parse_ident()?;
+            Ok(Expr::Var { name, pos })
+        } else if self.is_int() {
+            let val = self.int()?;
+            Ok(Expr::ConstInt { val, pos })
+        } else {
+            Err(self.error(pos, "Invalid expression".into()))
+        }
+    }
+
+    fn parse_letdef(&mut self) -> Result<LetDef> {
+        let pos = self.pos();
+        self.lparen()?;
+        let var = self.parse_ident()?;
+        let ty = self.parse_ident()?;
+        let val = Box::new(self.parse_expr()?);
+        self.rparen()?;
+        Ok(LetDef { var, ty, val, pos })
+    }
+}
diff --git a/cranelift/isle/isle/src/sema.rs b/cranelift/isle/isle/src/sema.rs
new file mode 100644
index 0000000000..fab78e6b7c
--- /dev/null
+++ b/cranelift/isle/isle/src/sema.rs
@@ -0,0 +1,1966 @@
+//! Semantic analysis.
+//!
+//! This module primarily contains the type environment and term environment.
+//!
+//! The type environment is constructed by analyzing an input AST. The type
+//! environment records the types used in the input source and the types of our
+//! various rules and symbols. ISLE's type system is intentionally easy to
+//! check, only requires a single pass over the AST, and doesn't require any
+//! unification or anything like that.
+//!
+//! The term environment is constructed from both the AST and type
+//! envionment. It is sort of a typed and reorganized AST that more directly
+//! reflects ISLE semantics than the input ISLE source code (where as the AST is
+//! the opposite).
+
+use crate::ast;
+use crate::error::*;
+use crate::lexer::Pos;
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
+use std::sync::Arc;
+
+declare_id!(
+    /// The id of an interned symbol.
+    Sym
+);
+declare_id!(
+    /// The id of an interned type inside the `TypeEnv`.
+    TypeId
+);
+declare_id!(
+    /// The id of a variant inside an enum.
+    VariantId
+);
+declare_id!(
+    /// The id of a field inside a variant.
+    FieldId
+);
+declare_id!(
+    /// The id of an interned term inside the `TermEnv`.
+    TermId
+);
+declare_id!(
+    /// The id of an interned rule inside the `TermEnv`.
+    RuleId
+);
+declare_id!(
+    /// The id of a bound variable inside a `Bindings`.
+    VarId
+);
+
+/// The type environment.
+///
+/// Keeps track of which symbols and rules have which types.
+#[derive(Clone, Debug)]
+pub struct TypeEnv {
+    /// Arena of input ISLE source filenames.
+    ///
+    /// We refer to these indirectly through the `Pos::file` indices.
+    pub filenames: Vec<Arc<str>>,
+
+    /// Arena of input ISLE source contents.
+    ///
+    /// We refer to these indirectly through the `Pos::file` indices.
+    pub file_texts: Vec<Arc<str>>,
+
+    /// Arena of interned symbol names.
+    ///
+    /// Referred to indirectly via `Sym` indices.
+    pub syms: Vec<String>,
+
+    /// Map of already-interned symbol names to their `Sym` ids.
+    pub sym_map: BTreeMap<String, Sym>,
+
+    /// Arena of type definitions.
+    ///
+    /// Referred to indirectly via `TypeId`s.
+    pub types: Vec<Type>,
+
+    /// A map from a type name symbol to its `TypeId`.
+    pub type_map: BTreeMap<Sym, TypeId>,
+
+    /// The types of constant symbols.
+    pub const_types: BTreeMap<Sym, TypeId>,
+
+    /// Type errors that we've found so far during type checking.
+    pub errors: Vec<Error>,
+}
+
+/// A type.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Type {
+    /// A primitive, `Copy` type.
+    ///
+    /// These are always defined externally, and we allow literals of these
+    /// types to pass through from ISLE source code to the emitted Rust code.
+    Primitive(TypeId, Sym, Pos),
+
+    /// A sum type.
+    ///
+    /// Note that enums with only one variant are equivalent to a "struct".
+    Enum {
+        /// The name of this enum.
+        name: Sym,
+        /// This `enum`'s type id.
+        id: TypeId,
+        /// Is this `enum` defined in external Rust code?
+        ///
+        /// If so, ISLE will not emit a definition for it. If not, then it will
+        /// emit a Rust definition for it.
+        is_extern: bool,
+        /// The different variants for this enum.
+        variants: Vec<Variant>,
+        /// The ISLE source position where this `enum` is defined.
+        pos: Pos,
+    },
+}
+
+impl Type {
+    /// Get the name of this `Type`.
+    pub fn name<'a>(&self, tyenv: &'a TypeEnv) -> &'a str {
+        match self {
+            Self::Primitive(_, name, _) | Self::Enum { name, .. } => &tyenv.syms[name.index()],
+        }
+    }
+
+    /// Get the position where this type was defined.
+    pub fn pos(&self) -> Pos {
+        match self {
+            Self::Primitive(_, _, pos) | Self::Enum { pos, .. } => *pos,
+        }
+    }
+
+    /// Is this a primitive type?
+    pub fn is_prim(&self) -> bool {
+        matches!(self, Type::Primitive(..))
+    }
+}
+
+/// A variant of an enum.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Variant {
+    /// The name of this variant.
+    pub name: Sym,
+
+    /// The full, prefixed-with-the-enum's-name name of this variant.
+    ///
+    /// E.g. if the enum is `Foo` and this variant is `Bar`, then the
+    /// `fullname` is `Foo.Bar`.
+    pub fullname: Sym,
+
+    /// The id of this variant, i.e. the index of this variant within its
+    /// enum's `Type::Enum::variants`.
+    pub id: VariantId,
+
+    /// The data fields of this enum variant.
+    pub fields: Vec<Field>,
+}
+
+/// A field of a `Variant`.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Field {
+    /// The name of this field.
+    pub name: Sym,
+    /// This field's id.
+    pub id: FieldId,
+    /// The type of this field.
+    pub ty: TypeId,
+}
+
+/// The term environment.
+///
+/// This is sort of a typed and reorganized AST that more directly reflects ISLE
+/// semantics than the input ISLE source code (where as the AST is the
+/// opposite).
+#[derive(Clone, Debug)]
+pub struct TermEnv {
+    /// Arena of interned terms defined in this ISLE program.
+    ///
+    /// This is indexed by `TermId`.
+    pub terms: Vec<Term>,
+
+    /// A map from am interned `Term`'s name to its `TermId`.
+    pub term_map: BTreeMap<Sym, TermId>,
+
+    /// Arena of interned rules defined in this ISLE program.
+    ///
+    /// This is indexed by `RuleId`.
+    pub rules: Vec<Rule>,
+}
+
+/// A term.
+///
+/// Maps parameter types to result types if this is a constructor term, or
+/// result types to parameter types if this is an extractor term. Or both if
+/// this term can be either a constructor or an extractor.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Term {
+    /// This term's id.
+    pub id: TermId,
+    /// The source position where this term was declared.
+    pub decl_pos: Pos,
+    /// The name of this term.
+    pub name: Sym,
+    /// The parameter types to this term.
+    pub arg_tys: Vec<TypeId>,
+    /// The result types of this term.
+    pub ret_ty: TypeId,
+    /// The kind of this term.
+    pub kind: TermKind,
+}
+
+/// The kind of a term.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum TermKind {
+    /// An enum variant constructor or extractor.
+    EnumVariant {
+        /// Which variant of the enum: e.g. for enum type `A` if a term is
+        /// `(A.A1 ...)` then the variant ID corresponds to `A1`.
+        variant: VariantId,
+    },
+    /// A term declared via a `(decl ...)` form.
+    Decl {
+        /// The kind of this term's constructor, if any.
+        constructor_kind: Option<ConstructorKind>,
+        /// The kind of this term's extractor, if any.
+        extractor_kind: Option<ExtractorKind>,
+    },
+}
+
+/// The kind of a constructor for a term.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ConstructorKind {
+    /// A term with "internal" rules that work in the forward direction. Becomes
+    /// a compiled Rust function in the generated code.
+    InternalConstructor,
+    /// A term defined solely by an external constructor function.
+    ExternalConstructor {
+        /// The external name of the constructor function.
+        name: Sym,
+    },
+}
+
+/// The kind of an extractor for a term.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ExtractorKind {
+    /// A term that defines an "extractor macro" in the LHS of a pattern. Its
+    /// arguments take patterns and are simply substituted with the given
+    /// patterns when used.
+    InternalExtractor {
+        /// This extractor's pattern.
+        template: ast::Pattern,
+    },
+    /// A term defined solely by an external extractor function.
+    ExternalExtractor {
+        /// The external name of the extractor function.
+        name: Sym,
+        /// Which arguments of the extractor are inputs and which are outputs?
+        arg_polarity: Vec<ArgPolarity>,
+        /// Is the external extractor infallible?
+        infallible: bool,
+        /// The position where this external extractor was declared.
+        pos: Pos,
+    },
+}
+
+pub use crate::ast::ArgPolarity;
+
+/// An external function signature.
+#[derive(Clone, Debug)]
+pub struct ExternalSig {
+    /// The name of the external function.
+    pub func_name: String,
+    /// The name of the external function, prefixed with the context trait.
+    pub full_name: String,
+    /// The types of this function signature's parameters.
+    pub param_tys: Vec<TypeId>,
+    /// The types of this function signature's results.
+    pub ret_tys: Vec<TypeId>,
+    /// Whether this signature is infallible or not.
+    pub infallible: bool,
+}
+
+impl Term {
+    /// Get this term's type.
+    pub fn ty(&self) -> TypeId {
+        self.ret_ty
+    }
+
+    /// Is this term an enum variant?
+    pub fn is_enum_variant(&self) -> bool {
+        matches!(self.kind, TermKind::EnumVariant { .. })
+    }
+
+    /// Does this term have a constructor?
+    pub fn has_constructor(&self) -> bool {
+        matches!(
+            self.kind,
+            TermKind::EnumVariant { .. }
+                | TermKind::Decl {
+                    constructor_kind: Some(_),
+                    ..
+                }
+        )
+    }
+
+    /// Does this term have an extractor?
+    pub fn has_extractor(&self) -> bool {
+        matches!(
+            self.kind,
+            TermKind::EnumVariant { .. }
+                | TermKind::Decl {
+                    extractor_kind: Some(_),
+                    ..
+                }
+        )
+    }
+
+    /// Is this term's extractor external?
+    pub fn has_external_extractor(&self) -> bool {
+        matches!(
+            self.kind,
+            TermKind::Decl {
+                extractor_kind: Some(ExtractorKind::ExternalExtractor { .. }),
+                ..
+            }
+        )
+    }
+
+    /// Is this term's constructor external?
+    pub fn has_external_constructor(&self) -> bool {
+        matches!(
+            self.kind,
+            TermKind::Decl {
+                constructor_kind: Some(ConstructorKind::ExternalConstructor { .. }),
+                ..
+            }
+        )
+    }
+
+    /// Get this term's extractor's external function signature, if any.
+    pub fn extractor_sig(&self, tyenv: &TypeEnv) -> Option<ExternalSig> {
+        match &self.kind {
+            TermKind::Decl {
+                extractor_kind:
+                    Some(ExtractorKind::ExternalExtractor {
+                        name,
+                        ref arg_polarity,
+                        infallible,
+                        ..
+                    }),
+                ..
+            } => {
+                let mut arg_tys = vec![];
+                let mut ret_tys = vec![];
+                arg_tys.push(self.ret_ty);
+                for (&arg, polarity) in self.arg_tys.iter().zip(arg_polarity.iter()) {
+                    match polarity {
+                        &ArgPolarity::Input => {
+                            arg_tys.push(arg);
+                        }
+                        &ArgPolarity::Output => {
+                            ret_tys.push(arg);
+                        }
+                    }
+                }
+                Some(ExternalSig {
+                    func_name: tyenv.syms[name.index()].clone(),
+                    full_name: format!("C::{}", tyenv.syms[name.index()]),
+                    param_tys: arg_tys,
+                    ret_tys,
+                    infallible: *infallible,
+                })
+            }
+            _ => None,
+        }
+    }
+
+    /// Get this term's constructor's external function signature, if any.
+    pub fn constructor_sig(&self, tyenv: &TypeEnv) -> Option<ExternalSig> {
+        match &self.kind {
+            TermKind::Decl {
+                constructor_kind: Some(ConstructorKind::ExternalConstructor { name }),
+                ..
+            } => Some(ExternalSig {
+                func_name: tyenv.syms[name.index()].clone(),
+                full_name: format!("C::{}", tyenv.syms[name.index()]),
+                param_tys: self.arg_tys.clone(),
+                ret_tys: vec![self.ret_ty],
+                infallible: true,
+            }),
+            TermKind::Decl {
+                constructor_kind: Some(ConstructorKind::InternalConstructor { .. }),
+                ..
+            } => {
+                let name = format!("constructor_{}", tyenv.syms[self.name.index()]);
+                Some(ExternalSig {
+                    func_name: name.clone(),
+                    full_name: name,
+                    param_tys: self.arg_tys.clone(),
+                    ret_tys: vec![self.ret_ty],
+                    infallible: false,
+                })
+            }
+            _ => None,
+        }
+    }
+}
+
+/// A term rewrite rule.
+#[derive(Clone, Debug)]
+pub struct Rule {
+    /// This rule's id.
+    pub id: RuleId,
+    /// The left-hand side pattern that this rule matches.
+    pub lhs: Pattern,
+    /// The right-hand side expression that this rule evaluates upon successful
+    /// match.
+    pub rhs: Expr,
+    /// The priority of this rule, if any.
+    pub prio: Option<i64>,
+    /// The source position where this rule is defined.
+    pub pos: Pos,
+}
+
+/// A left-hand side pattern of some rule.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Pattern {
+    /// Bind a variable of the given type from the current value.
+    ///
+    /// Keep matching on the value with the subpattern.
+    BindPattern(TypeId, VarId, Box<Pattern>),
+
+    /// Match the current value against an already bound variable with the given
+    /// type.
+    Var(TypeId, VarId),
+
+    /// Match the current value against a constant integer of the given integer
+    /// type.
+    ConstInt(TypeId, i64),
+
+    /// Match the current value against a constant primitive value of the given
+    /// primitive type.
+    ConstPrim(TypeId, Sym),
+
+    /// Match the current value against the given extractor term with the given
+    /// arguments.
+    Term(TypeId, TermId, Vec<TermArgPattern>),
+
+    /// Match anything of the given type successfully.
+    Wildcard(TypeId),
+
+    /// Match all of the following patterns of the given type.
+    And(TypeId, Vec<Pattern>),
+}
+
+/// Arguments to a term inside a pattern (i.e. an extractor).
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum TermArgPattern {
+    /// A pattern to match sub-values (i.e. the extractor's results) against.
+    Pattern(Pattern),
+    /// An expression to generate a value that is passed into the extractor.
+    Expr(Expr),
+}
+
+/// A right-hand side expression of some rule.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Expr {
+    /// Invoke this term constructor with the given arguments.
+    Term(TypeId, TermId, Vec<Expr>),
+    /// Get the value of a variable that was bound in the left-hand side.
+    Var(TypeId, VarId),
+    /// Get a constant integer.
+    ConstInt(TypeId, i64),
+    /// Get a constant primitive.
+    ConstPrim(TypeId, Sym),
+    /// Evaluate the nested expressions and bind their results to the given
+    /// variables, then evaluate the body expression.
+    Let {
+        /// The type of the result of this let expression.
+        ty: TypeId,
+        /// The expressions that are evaluated and bound to the given variables.
+        bindings: Vec<(VarId, TypeId, Box<Expr>)>,
+        /// The body expression that is evaluated after the bindings.
+        body: Box<Expr>,
+    },
+}
+
+impl Pattern {
+    /// Get this pattern's type.
+    pub fn ty(&self) -> TypeId {
+        match self {
+            &Self::BindPattern(t, ..) => t,
+            &Self::Var(t, ..) => t,
+            &Self::ConstInt(t, ..) => t,
+            &Self::ConstPrim(t, ..) => t,
+            &Self::Term(t, ..) => t,
+            &Self::Wildcard(t, ..) => t,
+            &Self::And(t, ..) => t,
+        }
+    }
+
+    /// Get the root term of this pattern, if any.
+    pub fn root_term(&self) -> Option<TermId> {
+        match self {
+            &Pattern::Term(_, term, _) => Some(term),
+            &Pattern::BindPattern(_, _, ref subpat) => subpat.root_term(),
+            _ => None,
+        }
+    }
+}
+
+impl Expr {
+    /// Get this expression's type.
+    pub fn ty(&self) -> TypeId {
+        match self {
+            &Self::Term(t, ..) => t,
+            &Self::Var(t, ..) => t,
+            &Self::ConstInt(t, ..) => t,
+            &Self::ConstPrim(t, ..) => t,
+            &Self::Let { ty: t, .. } => t,
+        }
+    }
+}
+
+/// Given an `Option<T>`, unwrap the inner `T` value, or `continue` if it is
+/// `None`.
+///
+/// Useful for when we encountered an error earlier in our analysis but kept
+/// going to find more errors, and now we've run into some missing data that
+/// would have been filled in if we didn't hit that original error, but we want
+/// to keep going to find more errors.
+macro_rules! unwrap_or_continue {
+    ($e:expr) => {
+        match $e {
+            Some(x) => x,
+            None => continue,
+        }
+    };
+}
+
+impl TypeEnv {
+    /// Construct the type environment from the AST.
+    pub fn from_ast(defs: &ast::Defs) -> Result<TypeEnv> {
+        let mut tyenv = TypeEnv {
+            filenames: defs.filenames.clone(),
+            file_texts: defs.file_texts.clone(),
+            syms: vec![],
+            sym_map: BTreeMap::new(),
+            types: vec![],
+            type_map: BTreeMap::new(),
+            const_types: BTreeMap::new(),
+            errors: vec![],
+        };
+
+        // Traverse defs, assigning type IDs to type names. We'll fill
+        // in types on a second pass.
+        for def in &defs.defs {
+            match def {
+                &ast::Def::Type(ref td) => {
+                    let tid = TypeId(tyenv.type_map.len());
+                    let name = tyenv.intern_mut(&td.name);
+
+                    if let Some(existing) = tyenv.type_map.get(&name).copied() {
+                        tyenv.report_error(
+                            td.pos,
+                            format!("Type with name '{}' defined more than once", td.name.0),
+                        );
+                        let pos = unwrap_or_continue!(tyenv.types.get(existing.index())).pos();
+                        tyenv.report_error(
+                            pos,
+                            format!("Type with name '{}' already defined here", td.name.0),
+                        );
+                        continue;
+                    }
+
+                    tyenv.type_map.insert(name, tid);
+                }
+                _ => {}
+            }
+        }
+
+        // Now lower AST nodes to type definitions, raising errors
+        // where typenames of fields are undefined or field names are
+        // duplicated.
+        let mut tid = 0;
+        for def in &defs.defs {
+            match def {
+                &ast::Def::Type(ref td) => {
+                    let ty = unwrap_or_continue!(tyenv.type_from_ast(TypeId(tid), td));
+                    tyenv.types.push(ty);
+                    tid += 1;
+                }
+                _ => {}
+            }
+        }
+
+        // Now collect types for extern constants.
+        for def in &defs.defs {
+            match def {
+                &ast::Def::Extern(ast::Extern::Const {
+                    ref name,
+                    ref ty,
+                    pos,
+                }) => {
+                    let ty = tyenv.intern_mut(ty);
+                    let ty = match tyenv.type_map.get(&ty) {
+                        Some(ty) => *ty,
+                        None => {
+                            tyenv.report_error(pos, "Unknown type for constant".into());
+                            continue;
+                        }
+                    };
+                    let name = tyenv.intern_mut(name);
+                    tyenv.const_types.insert(name, ty);
+                }
+                _ => {}
+            }
+        }
+
+        tyenv.return_errors()?;
+
+        Ok(tyenv)
+    }
+
+    fn return_errors(&mut self) -> Result<()> {
+        match self.errors.len() {
+            0 => Ok(()),
+            1 => Err(self.errors.pop().unwrap()),
+            _ => Err(Error::Errors(std::mem::take(&mut self.errors))),
+        }
+    }
+
+    fn type_from_ast(&mut self, tid: TypeId, ty: &ast::Type) -> Option<Type> {
+        let name = self.intern(&ty.name).unwrap();
+        match &ty.ty {
+            &ast::TypeValue::Primitive(ref id, ..) => {
+                Some(Type::Primitive(tid, self.intern_mut(id), ty.pos))
+            }
+            &ast::TypeValue::Enum(ref ty_variants, ..) => {
+                let mut variants = vec![];
+                for variant in ty_variants {
+                    let combined_ident =
+                        ast::Ident(format!("{}.{}", ty.name.0, variant.name.0), variant.name.1);
+                    let fullname = self.intern_mut(&combined_ident);
+                    let name = self.intern_mut(&variant.name);
+                    let id = VariantId(variants.len());
+                    if variants.iter().any(|v: &Variant| v.name == name) {
+                        self.report_error(
+                            variant.pos,
+                            format!("Duplicate variant name in type: '{}'", variant.name.0),
+                        );
+                        return None;
+                    }
+                    let mut fields = vec![];
+                    for field in &variant.fields {
+                        let field_name = self.intern_mut(&field.name);
+                        if fields.iter().any(|f: &Field| f.name == field_name) {
+                            self.report_error(
+                                field.pos,
+                                format!(
+                                    "Duplicate field name '{}' in variant '{}' of type",
+                                    field.name.0, variant.name.0
+                                ),
+                            );
+                            return None;
+                        }
+                        let field_ty = self.intern_mut(&field.ty);
+                        let field_tid = match self.type_map.get(&field_ty) {
+                            Some(tid) => *tid,
+                            None => {
+                                self.report_error(
+                                    field.ty.1,
+                                    format!(
+                                        "Unknown type '{}' for field '{}' in variant '{}'",
+                                        field.ty.0, field.name.0, variant.name.0
+                                    ),
+                                );
+                                return None;
+                            }
+                        };
+                        fields.push(Field {
+                            name: field_name,
+                            id: FieldId(fields.len()),
+                            ty: field_tid,
+                        });
+                    }
+                    variants.push(Variant {
+                        name,
+                        fullname,
+                        id,
+                        fields,
+                    });
+                }
+                Some(Type::Enum {
+                    name,
+                    id: tid,
+                    is_extern: ty.is_extern,
+                    variants,
+                    pos: ty.pos,
+                })
+            }
+        }
+    }
+
+    fn error(&self, pos: Pos, msg: String) -> Error {
+        let e = Error::TypeError {
+            msg,
+            src: Source::new(
+                self.filenames[pos.file].clone(),
+                self.file_texts[pos.file].clone(),
+            ),
+            span: miette::SourceSpan::from((pos.offset, 1)),
+        };
+        log::trace!("{}", e);
+        e
+    }
+
+    fn report_error(&mut self, pos: Pos, msg: String) {
+        let err = self.error(pos, msg);
+        self.errors.push(err);
+    }
+
+    fn intern_mut(&mut self, ident: &ast::Ident) -> Sym {
+        if let Some(s) = self.sym_map.get(&ident.0).copied() {
+            s
+        } else {
+            let s = Sym(self.syms.len());
+            self.syms.push(ident.0.clone());
+            self.sym_map.insert(ident.0.clone(), s);
+            s
+        }
+    }
+
+    fn intern(&self, ident: &ast::Ident) -> Option<Sym> {
+        self.sym_map.get(&ident.0).cloned()
+    }
+}
+
+#[derive(Clone, Debug)]
+struct Bindings {
+    next_var: usize,
+    vars: Vec<BoundVar>,
+}
+
+#[derive(Clone, Debug)]
+struct BoundVar {
+    name: Sym,
+    id: VarId,
+    ty: TypeId,
+}
+
+impl TermEnv {
+    /// Construct the term environment from the AST and the type environment.
+    pub fn from_ast(tyenv: &mut TypeEnv, defs: &ast::Defs) -> Result<TermEnv> {
+        let mut env = TermEnv {
+            terms: vec![],
+            term_map: BTreeMap::new(),
+            rules: vec![],
+        };
+
+        env.collect_term_sigs(tyenv, defs);
+        env.collect_enum_variant_terms(tyenv);
+        tyenv.return_errors()?;
+        env.collect_constructors(tyenv, defs);
+        env.collect_extractor_templates(tyenv, defs);
+        tyenv.return_errors()?;
+        env.collect_rules(tyenv, defs);
+        env.check_for_undefined_decls(tyenv, defs);
+        env.check_for_expr_terms_without_constructors(tyenv, defs);
+        tyenv.return_errors()?;
+
+        Ok(env)
+    }
+
+    fn collect_term_sigs(&mut self, tyenv: &mut TypeEnv, defs: &ast::Defs) {
+        for def in &defs.defs {
+            match def {
+                &ast::Def::Decl(ref decl) => {
+                    let tid = TermId(self.terms.len());
+                    let name = tyenv.intern_mut(&decl.term);
+                    if let Some(tid) = self.term_map.get(&name) {
+                        tyenv.report_error(
+                            decl.pos,
+                            format!("Duplicate decl for '{}'", decl.term.0),
+                        );
+                        tyenv.report_error(
+                            self.terms[tid.index()].decl_pos,
+                            format!("Duplicate decl for '{}'", decl.term.0),
+                        );
+                    }
+                    self.term_map.insert(name, tid);
+
+                    let arg_tys = decl
+                        .arg_tys
+                        .iter()
+                        .map(|id| {
+                            let sym = tyenv.intern_mut(id);
+                            tyenv.type_map.get(&sym).cloned().ok_or_else(|| {
+                                tyenv.report_error(id.1, format!("Unknown arg type: '{}'", id.0));
+                                ()
+                            })
+                        })
+                        .collect::<std::result::Result<Vec<_>, _>>();
+                    let arg_tys = match arg_tys {
+                        Ok(a) => a,
+                        Err(_) => {
+                            continue;
+                        }
+                    };
+                    let ret_ty = {
+                        let sym = tyenv.intern_mut(&decl.ret_ty);
+                        match tyenv.type_map.get(&sym).cloned() {
+                            Some(t) => t,
+                            None => {
+                                tyenv.report_error(
+                                    decl.ret_ty.1,
+                                    format!("Unknown return type: '{}'", decl.ret_ty.0),
+                                );
+                                continue;
+                            }
+                        }
+                    };
+
+                    self.terms.push(Term {
+                        id: tid,
+                        decl_pos: decl.pos,
+                        name,
+                        arg_tys,
+                        ret_ty,
+                        kind: TermKind::Decl {
+                            constructor_kind: None,
+                            extractor_kind: None,
+                        },
+                    });
+                }
+                _ => {}
+            }
+        }
+    }
+
+    fn collect_enum_variant_terms(&mut self, tyenv: &mut TypeEnv) {
+        'types: for i in 0..tyenv.types.len() {
+            let ty = &tyenv.types[i];
+            match ty {
+                &Type::Enum {
+                    pos,
+                    id,
+                    ref variants,
+                    ..
+                } => {
+                    for variant in variants {
+                        if self.term_map.contains_key(&variant.fullname) {
+                            let variant_name = tyenv.syms[variant.fullname.index()].clone();
+                            tyenv.report_error(
+                                pos,
+                                format!("Duplicate enum variant constructor: '{}'", variant_name,),
+                            );
+                            continue 'types;
+                        }
+                        let tid = TermId(self.terms.len());
+                        let arg_tys = variant.fields.iter().map(|fld| fld.ty).collect::<Vec<_>>();
+                        let ret_ty = id;
+                        self.terms.push(Term {
+                            id: tid,
+                            decl_pos: pos,
+                            name: variant.fullname,
+                            arg_tys,
+                            ret_ty,
+                            kind: TermKind::EnumVariant {
+                                variant: variant.id,
+                            },
+                        });
+                        self.term_map.insert(variant.fullname, tid);
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+
+    fn collect_constructors(&mut self, tyenv: &mut TypeEnv, defs: &ast::Defs) {
+        for def in &defs.defs {
+            log::debug!("collect_constructors from def: {:?}", def);
+            match def {
+                &ast::Def::Rule(ref rule) => {
+                    let pos = rule.pos;
+                    let term = match rule.pattern.root_term() {
+                        Some(t) => t,
+                        None => {
+                            tyenv.report_error(
+                                pos,
+                                "Rule does not have a term at the LHS root".to_string(),
+                            );
+                            continue;
+                        }
+                    };
+                    let sym = tyenv.intern_mut(&term);
+                    let term = match self.term_map.get(&sym) {
+                        Some(&tid) => tid,
+                        None => {
+                            tyenv
+                                .report_error(pos, "Rule LHS root term is not defined".to_string());
+                            continue;
+                        }
+                    };
+                    let termdata = &mut self.terms[term.index()];
+                    match &mut termdata.kind {
+                        TermKind::Decl {
+                            constructor_kind, ..
+                        } => {
+                            match constructor_kind {
+                                None => {
+                                    *constructor_kind = Some(ConstructorKind::InternalConstructor);
+                                }
+                                Some(ConstructorKind::InternalConstructor) => {
+                                    // OK, no error; multiple rules can apply to
+                                    // one internal constructor term.
+                                }
+                                Some(ConstructorKind::ExternalConstructor { .. }) => {
+                                    tyenv.report_error(
+                                        pos,
+                                        "Rule LHS root term is incorrect kind; cannot \
+                                         be external constructor"
+                                            .to_string(),
+                                    );
+                                    continue;
+                                }
+                            }
+                        }
+                        TermKind::EnumVariant { .. } => {
+                            tyenv.report_error(
+                                pos,
+                                "Rule LHS root term is incorrect kind; cannot be enum variant"
+                                    .to_string(),
+                            );
+                            continue;
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+
+    fn collect_extractor_templates(&mut self, tyenv: &mut TypeEnv, defs: &ast::Defs) {
+        let mut extractor_call_graph = BTreeMap::new();
+
+        for def in &defs.defs {
+            if let &ast::Def::Extractor(ref ext) = def {
+                let sym = tyenv.intern_mut(&ext.term);
+                let term = match self.term_map.get(&sym) {
+                    Some(x) => x,
+                    None => {
+                        tyenv.report_error(
+                            ext.pos,
+                            "Extractor macro body definition on a non-existent term".to_string(),
+                        );
+                        return;
+                    }
+                };
+
+                let template = ext.template.make_macro_template(&ext.args[..]);
+                log::trace!("extractor def: {:?} becomes template {:?}", def, template);
+
+                let mut callees = BTreeSet::new();
+                template.terms(&mut |pos, t| {
+                    let t = tyenv.intern_mut(t);
+                    callees.insert(t);
+
+                    if !self.term_map.contains_key(&t) {
+                        tyenv.report_error(
+                            pos,
+                            format!(
+                                "`{}` extractor definition references unknown term `{}`",
+                                ext.term.0,
+                                tyenv.syms[t.index()]
+                            ),
+                        );
+                    }
+                });
+                extractor_call_graph.insert(sym, callees);
+
+                let termdata = &mut self.terms[term.index()];
+                match &mut termdata.kind {
+                    TermKind::EnumVariant { .. } => {
+                        tyenv.report_error(
+                            ext.pos,
+                            "Extractor macro body defined on term of incorrect kind; cannot be an \
+                             enum variant"
+                                .into(),
+                        );
+                        continue;
+                    }
+                    TermKind::Decl { extractor_kind, .. } => match extractor_kind {
+                        None => {
+                            *extractor_kind = Some(ExtractorKind::InternalExtractor { template });
+                        }
+                        Some(ext_kind) => {
+                            tyenv.report_error(
+                                ext.pos,
+                                "Duplicate extractor definition".to_string(),
+                            );
+                            let pos = match ext_kind {
+                                ExtractorKind::InternalExtractor { template } => template.pos(),
+                                ExtractorKind::ExternalExtractor { pos, .. } => *pos,
+                            };
+                            tyenv.report_error(
+                                pos,
+                                "Extractor was already defined here".to_string(),
+                            );
+                            continue;
+                        }
+                    },
+                }
+            }
+        }
+
+        // Check for cycles in the extractor call graph.
+        let mut seen = BTreeSet::new();
+        let mut stack = vec![];
+        'outer: for root in extractor_call_graph.keys().copied() {
+            seen.clear();
+            stack.clear();
+            stack.push((root, vec![root]));
+
+            while let Some((caller, path)) = stack.pop() {
+                let is_new = seen.insert(caller);
+                if is_new {
+                    if let Some(callees) = extractor_call_graph.get(&caller) {
+                        stack.extend(callees.iter().map(|callee| {
+                            let mut path = path.clone();
+                            path.push(*callee);
+                            (*callee, path)
+                        }));
+                    }
+                } else {
+                    let term = match self.term_map.get(&caller) {
+                        Some(t) => t,
+                        None => {
+                            // Some other error must have already been recorded
+                            // if we don't have the caller's term data.
+                            assert!(!tyenv.errors.is_empty());
+                            continue 'outer;
+                        }
+                    };
+                    let pos = match &self.terms[term.index()].kind {
+                        TermKind::Decl {
+                            extractor_kind: Some(ExtractorKind::InternalExtractor { template }),
+                            ..
+                        } => template.pos(),
+                        _ => {
+                            // Again, there must have already been errors
+                            // recorded.
+                            assert!(!tyenv.errors.is_empty());
+                            continue 'outer;
+                        }
+                    };
+
+                    let path: Vec<_> = path
+                        .iter()
+                        .map(|sym| tyenv.syms[sym.index()].as_str())
+                        .collect();
+                    let msg = format!(
+                        "`{}` extractor definition is recursive: {}",
+                        tyenv.syms[root.index()],
+                        path.join(" -> ")
+                    );
+                    tyenv.report_error(pos, msg);
+                    continue 'outer;
+                }
+            }
+        }
+    }
+
+    fn collect_rules(&mut self, tyenv: &mut TypeEnv, defs: &ast::Defs) {
+        for def in &defs.defs {
+            match def {
+                &ast::Def::Rule(ref rule) => {
+                    let pos = rule.pos;
+                    let mut bindings = Bindings {
+                        next_var: 0,
+                        vars: vec![],
+                    };
+
+                    let rule_term = match rule.pattern.root_term() {
+                        Some(name) => {
+                            let sym = tyenv.intern_mut(name);
+                            match self.term_map.get(&sym) {
+                                Some(term) => *term,
+                                None => {
+                                    tyenv.report_error(
+                                        pos,
+                                        "Cannot define a rule for an unknown term".to_string(),
+                                    );
+                                    continue;
+                                }
+                            }
+                        }
+                        None => {
+                            tyenv.report_error(
+                                pos,
+                                "Rule does not have a term at the root of its left-hand side"
+                                    .to_string(),
+                            );
+                            continue;
+                        }
+                    };
+
+                    let (lhs, ty) = unwrap_or_continue!(self.translate_pattern(
+                        tyenv,
+                        rule_term,
+                        &rule.pattern,
+                        None,
+                        &mut bindings,
+                        /* is_root = */ true,
+                    ));
+                    let rhs = unwrap_or_continue!(self.translate_expr(
+                        tyenv,
+                        &rule.expr,
+                        ty,
+                        &mut bindings
+                    ));
+
+                    let rid = RuleId(self.rules.len());
+                    self.rules.push(Rule {
+                        id: rid,
+                        lhs,
+                        rhs,
+                        prio: rule.prio,
+                        pos,
+                    });
+                }
+                &ast::Def::Extern(ast::Extern::Constructor {
+                    ref term,
+                    ref func,
+                    pos,
+                }) => {
+                    let term_sym = tyenv.intern_mut(term);
+                    let func_sym = tyenv.intern_mut(func);
+                    let term_id = match self.term_map.get(&term_sym) {
+                        Some(term) => term,
+                        None => {
+                            tyenv.report_error(
+                                pos,
+                                format!("Constructor declared on undefined term '{}'", term.0),
+                            );
+                            continue;
+                        }
+                    };
+                    let termdata = &mut self.terms[term_id.index()];
+                    match &mut termdata.kind {
+                        TermKind::Decl {
+                            constructor_kind, ..
+                        } => match constructor_kind {
+                            None => {
+                                *constructor_kind =
+                                    Some(ConstructorKind::ExternalConstructor { name: func_sym });
+                            }
+                            Some(ConstructorKind::InternalConstructor) => {
+                                tyenv.report_error(
+                                    pos,
+                                    format!(
+                                        "External constructor declared on term that already has rules: {}",
+                                        term.0,
+                                    ),
+                                );
+                            }
+                            Some(ConstructorKind::ExternalConstructor { .. }) => {
+                                tyenv.report_error(
+                                    pos,
+                                    "Duplicate external constructor definition".to_string(),
+                                );
+                            }
+                        },
+                        TermKind::EnumVariant { .. } => {
+                            tyenv.report_error(
+                                pos,
+                                format!(
+                                    "External constructor cannot be defined on enum variant: {}",
+                                    term.0,
+                                ),
+                            );
+                        }
+                    }
+                }
+                &ast::Def::Extern(ast::Extern::Extractor {
+                    ref term,
+                    ref func,
+                    pos,
+                    ref arg_polarity,
+                    infallible,
+                }) => {
+                    let term_sym = tyenv.intern_mut(term);
+                    let func_sym = tyenv.intern_mut(func);
+                    let term_id = match self.term_map.get(&term_sym) {
+                        Some(term) => term,
+                        None => {
+                            tyenv.report_error(
+                                pos,
+                                format!("Extractor declared on undefined term '{}'", term.0),
+                            );
+                            continue;
+                        }
+                    };
+
+                    let termdata = &mut self.terms[term_id.index()];
+
+                    let arg_polarity = if let Some(pol) = arg_polarity.as_ref() {
+                        if pol.len() != termdata.arg_tys.len() {
+                            tyenv.report_error(pos, "Incorrect number of argument-polarity directions in extractor definition".to_string());
+                            continue;
+                        }
+                        pol.clone()
+                    } else {
+                        vec![ArgPolarity::Output; termdata.arg_tys.len()]
+                    };
+
+                    match &mut termdata.kind {
+                        TermKind::Decl { extractor_kind, .. } => match extractor_kind {
+                            None => {
+                                *extractor_kind = Some(ExtractorKind::ExternalExtractor {
+                                    name: func_sym,
+                                    arg_polarity,
+                                    infallible,
+                                    pos,
+                                });
+                            }
+                            Some(ExtractorKind::ExternalExtractor { pos: pos2, .. }) => {
+                                tyenv.report_error(
+                                    pos,
+                                    "Duplicate external extractor definition".to_string(),
+                                );
+                                tyenv.report_error(
+                                    *pos2,
+                                    "External extractor already defined".to_string(),
+                                );
+                                continue;
+                            }
+                            Some(ExtractorKind::InternalExtractor { template }) => {
+                                tyenv.report_error(
+                                    pos,
+                                    "Cannot define external extractor for term that already has an \
+                                     internal extractor macro body defined"
+                                        .to_string(),
+                                );
+                                tyenv.report_error(
+                                    template.pos(),
+                                    "Internal extractor macro body already defined".to_string(),
+                                );
+                                continue;
+                            }
+                        },
+                        TermKind::EnumVariant { .. } => {
+                            tyenv.report_error(
+                                pos,
+                                format!("Cannot define extractor for enum variant '{}'", term.0),
+                            );
+                            continue;
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+
+    fn check_for_undefined_decls(&self, tyenv: &mut TypeEnv, defs: &ast::Defs) {
+        for def in &defs.defs {
+            if let ast::Def::Decl(decl) = def {
+                let sym = tyenv.intern_mut(&decl.term);
+                let term = self.term_map[&sym];
+                let term = &self.terms[term.index()];
+                if !term.has_constructor() && !term.has_extractor() {
+                    tyenv.report_error(
+                        decl.pos,
+                        format!(
+                            "no rules, extractor, or external definition for declaration '{}'",
+                            decl.term.0
+                        ),
+                    );
+                }
+            }
+        }
+    }
+
+    fn check_for_expr_terms_without_constructors(&self, tyenv: &mut TypeEnv, defs: &ast::Defs) {
+        for def in &defs.defs {
+            if let ast::Def::Rule(rule) = def {
+                rule.expr.terms(&mut |pos, ident| {
+                    let sym = tyenv.intern_mut(ident);
+                    let term = match self.term_map.get(&sym) {
+                        None => {
+                            debug_assert!(!tyenv.errors.is_empty());
+                            return;
+                        }
+                        Some(t) => t,
+                    };
+                    let term = &self.terms[term.index()];
+                    if !term.has_constructor() {
+                        tyenv.report_error(
+                            pos,
+                            format!(
+                                "term `{}` cannot be used in an expression because \
+                                 it does not have a constructor",
+                                ident.0
+                            ),
+                        )
+                    }
+                });
+            }
+        }
+    }
+
+    fn translate_pattern(
+        &self,
+        tyenv: &mut TypeEnv,
+        rule_term: TermId,
+        pat: &ast::Pattern,
+        expected_ty: Option<TypeId>,
+        bindings: &mut Bindings,
+        is_root: bool,
+    ) -> Option<(Pattern, TypeId)> {
+        log::trace!("translate_pattern: {:?}", pat);
+        log::trace!("translate_pattern: bindings = {:?}", bindings);
+        match pat {
+            // TODO: flag on primitive type decl indicating it's an integer type?
+            &ast::Pattern::ConstInt { val, pos } => {
+                let ty = match expected_ty {
+                    Some(t) => t,
+                    None => {
+                        tyenv.report_error(
+                            pos,
+                            "Need an implied type for an integer constant".into(),
+                        );
+                        return None;
+                    }
+                };
+                if !tyenv.types[ty.index()].is_prim() {
+                    tyenv.report_error(
+                        pos,
+                        format!(
+                            "expected non-primitive type {}, but found integer literal '{}'",
+                            tyenv.types[ty.index()].name(tyenv),
+                            val,
+                        ),
+                    );
+                }
+                Some((Pattern::ConstInt(ty, val), ty))
+            }
+            &ast::Pattern::ConstPrim { ref val, pos } => {
+                let val = tyenv.intern_mut(val);
+                let const_ty = match tyenv.const_types.get(&val) {
+                    Some(ty) => *ty,
+                    None => {
+                        tyenv.report_error(pos, "Unknown constant".into());
+                        return None;
+                    }
+                };
+                if expected_ty.is_some() && expected_ty != Some(const_ty) {
+                    tyenv.report_error(pos, "Type mismatch for constant".into());
+                }
+                Some((Pattern::ConstPrim(const_ty, val), const_ty))
+            }
+            &ast::Pattern::Wildcard { pos } => {
+                let ty = match expected_ty {
+                    Some(t) => t,
+                    None => {
+                        tyenv.report_error(pos, "Need an implied type for a wildcard".into());
+                        return None;
+                    }
+                };
+                Some((Pattern::Wildcard(ty), ty))
+            }
+            &ast::Pattern::And { ref subpats, pos } => {
+                let mut expected_ty = expected_ty;
+                let mut children = vec![];
+                for subpat in subpats {
+                    let (subpat, ty) = unwrap_or_continue!(self.translate_pattern(
+                        tyenv,
+                        rule_term,
+                        &*subpat,
+                        expected_ty,
+                        bindings,
+                        /* is_root = */ false,
+                    ));
+                    expected_ty = expected_ty.or(Some(ty));
+                    children.push(subpat);
+                }
+                if expected_ty.is_none() {
+                    tyenv.report_error(pos, "No type for (and ...) form.".to_string());
+                    return None;
+                }
+                let ty = expected_ty.unwrap();
+                Some((Pattern::And(ty, children), ty))
+            }
+            &ast::Pattern::BindPattern {
+                ref var,
+                ref subpat,
+                pos,
+            } => {
+                // Do the subpattern first so we can resolve the type for sure.
+                let (subpat, ty) = self.translate_pattern(
+                    tyenv,
+                    rule_term,
+                    &*subpat,
+                    expected_ty,
+                    bindings,
+                    /* is_root = */ false,
+                )?;
+
+                let name = tyenv.intern_mut(var);
+                if bindings.vars.iter().any(|bv| bv.name == name) {
+                    tyenv.report_error(
+                        pos,
+                        format!("Re-bound variable name in LHS pattern: '{}'", var.0),
+                    );
+                    // Try to keep going.
+                }
+                let id = VarId(bindings.next_var);
+                bindings.next_var += 1;
+                log::trace!("binding var {:?}", var.0);
+                bindings.vars.push(BoundVar { name, id, ty });
+
+                Some((Pattern::BindPattern(ty, id, Box::new(subpat)), ty))
+            }
+            &ast::Pattern::Var { ref var, pos } => {
+                // Look up the variable; it must already have been bound.
+                let name = tyenv.intern_mut(var);
+                let bv = match bindings.vars.iter().rev().find(|bv| bv.name == name) {
+                    None => {
+                        tyenv.report_error(
+                            pos,
+                            format!(
+                                "Unknown variable '{}' in bound-var pattern '={}'",
+                                var.0, var.0
+                            ),
+                        );
+                        return None;
+                    }
+                    Some(bv) => bv,
+                };
+                let ty = match expected_ty {
+                    None => bv.ty,
+                    Some(expected_ty) if expected_ty == bv.ty => bv.ty,
+                    Some(expected_ty) => {
+                        tyenv.report_error(
+                            pos,
+                            format!(
+                                "Mismatched types: pattern expects type '{}' but already-bound var '{}' has type '{}'",
+                                tyenv.types[expected_ty.index()].name(tyenv),
+                                var.0,
+                                tyenv.types[bv.ty.index()].name(tyenv)));
+                        bv.ty // Try to keep going for more errors.
+                    }
+                };
+                Some((Pattern::Var(ty, bv.id), ty))
+            }
+            &ast::Pattern::Term {
+                ref sym,
+                ref args,
+                pos,
+            } => {
+                let name = tyenv.intern_mut(&sym);
+                // Look up the term.
+                let tid = match self.term_map.get(&name) {
+                    Some(t) => t,
+                    None => {
+                        tyenv.report_error(pos, format!("Unknown term in pattern: '{}'", sym.0));
+                        return None;
+                    }
+                };
+
+                // Get the return type and arg types. Verify the
+                // expected type of this pattern, if any, against the
+                // return type of the term.
+                let ret_ty = self.terms[tid.index()].ret_ty;
+                let ty = match expected_ty {
+                    None => ret_ty,
+                    Some(expected_ty) if expected_ty == ret_ty => ret_ty,
+                    Some(expected_ty) => {
+                        tyenv.report_error(
+                            pos,
+                            format!(
+                                "Mismatched types: pattern expects type '{}' but term has return type '{}'",
+                                tyenv.types[expected_ty.index()].name(tyenv),
+                                tyenv.types[ret_ty.index()].name(tyenv)));
+                        ret_ty // Try to keep going for more errors.
+                    }
+                };
+
+                // Check that we have the correct argument count.
+                if self.terms[tid.index()].arg_tys.len() != args.len() {
+                    tyenv.report_error(
+                        pos,
+                        format!(
+                            "Incorrect argument count for term '{}': got {}, expect {}",
+                            sym.0,
+                            args.len(),
+                            self.terms[tid.index()].arg_tys.len()
+                        ),
+                    );
+                }
+
+                let termdata = &self.terms[tid.index()];
+
+                match &termdata.kind {
+                    TermKind::Decl {
+                        constructor_kind: Some(ConstructorKind::InternalConstructor),
+                        ..
+                    } if is_root && *tid == rule_term => {
+                        // This is just the `(foo ...)` pseudo-pattern inside a
+                        // `(rule (foo ...) ...)` form. Just keep checking the
+                        // sub-patterns.
+                        for arg in args {
+                            if let ast::TermArgPattern::Expr(e) = arg {
+                                tyenv.report_error(
+                                    e.pos(),
+                                    "cannot use output-polarity expression with top-level rules"
+                                        .to_string(),
+                                );
+                            }
+                        }
+                    }
+                    TermKind::EnumVariant { .. } => {
+                        for arg in args {
+                            if let &ast::TermArgPattern::Expr(_) = arg {
+                                tyenv.report_error(
+                                    pos,
+                                    format!(
+                                        "Term in pattern '{}' cannot have an injected expr, because \
+                                         it is an enum variant",
+                                        sym.0
+                                    )
+                                );
+                            }
+                        }
+                    }
+                    TermKind::Decl {
+                        extractor_kind:
+                            Some(ExtractorKind::ExternalExtractor {
+                                ref arg_polarity, ..
+                            }),
+                        ..
+                    } => {
+                        for (arg, pol) in args.iter().zip(arg_polarity.iter()) {
+                            match (arg, pol) {
+                                (&ast::TermArgPattern::Expr(..), &ArgPolarity::Input) => {}
+                                (&ast::TermArgPattern::Expr(ref e), &ArgPolarity::Output) => {
+                                    tyenv.report_error(
+                                        e.pos(),
+                                        "Expression used for output-polarity extractor arg"
+                                            .to_string(),
+                                    );
+                                }
+                                (_, &ArgPolarity::Output) => {}
+                                (&ast::TermArgPattern::Pattern(ref p), &ArgPolarity::Input) => {
+                                    tyenv.report_error(
+                                        p.pos(),
+                                        "Non-expression used in pattern but expression required for \
+                                         input-polarity extractor arg"
+                                            .to_string()
+                                    );
+                                }
+                            }
+                        }
+                    }
+                    TermKind::Decl {
+                        extractor_kind: Some(ExtractorKind::InternalExtractor { ref template }),
+                        ..
+                    } => {
+                        // Expand the extractor macro! We create a map
+                        // from macro args to AST pattern trees and
+                        // then evaluate the template with these
+                        // substitutions.
+                        let mut macro_args: Vec<ast::Pattern> = vec![];
+                        for template_arg in args {
+                            let sub_ast = match template_arg {
+                                &ast::TermArgPattern::Pattern(ref pat) => pat.clone(),
+                                &ast::TermArgPattern::Expr(_) => {
+                                    tyenv.report_error(
+                                        pos,
+                                        "Cannot expand an extractor macro with an expression in a \
+                                         macro argument"
+                                            .to_string(),
+                                    );
+                                    return None;
+                                }
+                            };
+                            macro_args.push(sub_ast.clone());
+                        }
+                        log::trace!("internal extractor macro args = {:?}", args);
+                        let pat = template.subst_macro_args(&macro_args[..])?;
+                        return self.translate_pattern(
+                            tyenv,
+                            rule_term,
+                            &pat,
+                            expected_ty,
+                            bindings,
+                            /* is_root = */ false,
+                        );
+                    }
+                    TermKind::Decl {
+                        extractor_kind: None,
+                        ..
+                    } => {
+                        tyenv.report_error(
+                            pos,
+                            format!(
+                                "Cannot use term '{}' that does not have a defined extractor in a \
+                                 left-hand side pattern",
+                                sym.0
+                            ),
+                        );
+                    }
+                }
+
+                // Resolve subpatterns.
+                let mut subpats = vec![];
+                for (i, arg) in args.iter().enumerate() {
+                    let term = unwrap_or_continue!(self.terms.get(tid.index()));
+                    let arg_ty = unwrap_or_continue!(term.arg_tys.get(i).copied());
+                    let (subpat, _) = unwrap_or_continue!(self.translate_pattern_term_arg(
+                        tyenv,
+                        rule_term,
+                        pos,
+                        arg,
+                        Some(arg_ty),
+                        bindings,
+                    ));
+                    subpats.push(subpat);
+                }
+
+                Some((Pattern::Term(ty, *tid, subpats), ty))
+            }
+            &ast::Pattern::MacroArg { .. } => unreachable!(),
+        }
+    }
+
+    fn translate_pattern_term_arg(
+        &self,
+        tyenv: &mut TypeEnv,
+        rule_term: TermId,
+        pos: Pos,
+        pat: &ast::TermArgPattern,
+        expected_ty: Option<TypeId>,
+        bindings: &mut Bindings,
+    ) -> Option<(TermArgPattern, TypeId)> {
+        match pat {
+            &ast::TermArgPattern::Pattern(ref pat) => {
+                let (subpat, ty) = self.translate_pattern(
+                    tyenv,
+                    rule_term,
+                    pat,
+                    expected_ty,
+                    bindings,
+                    /* is_root = */ false,
+                )?;
+                Some((TermArgPattern::Pattern(subpat), ty))
+            }
+            &ast::TermArgPattern::Expr(ref expr) => {
+                if expected_ty.is_none() {
+                    tyenv.report_error(
+                        pos,
+                        "Expression in pattern must have expected type".to_string(),
+                    );
+                    return None;
+                }
+                let ty = expected_ty.unwrap();
+                let expr = self.translate_expr(tyenv, expr, expected_ty.unwrap(), bindings)?;
+                Some((TermArgPattern::Expr(expr), ty))
+            }
+        }
+    }
+
+    fn translate_expr(
+        &self,
+        tyenv: &mut TypeEnv,
+        expr: &ast::Expr,
+        ty: TypeId,
+        bindings: &mut Bindings,
+    ) -> Option<Expr> {
+        log::trace!("translate_expr: {:?}", expr);
+        match expr {
+            &ast::Expr::Term {
+                ref sym,
+                ref args,
+                pos,
+            } => {
+                // Look up the term.
+                let name = tyenv.intern_mut(&sym);
+                // Look up the term.
+                let tid = match self.term_map.get(&name) {
+                    Some(t) => t,
+                    None => {
+                        tyenv.report_error(pos, format!("Unknown term in pattern: '{}'", sym.0));
+                        return None;
+                    }
+                };
+
+                // Get the return type and arg types. Verify the
+                // expected type of this pattern, if any, against the
+                // return type of the term.
+                let ret_ty = self.terms[tid.index()].ret_ty;
+                if ret_ty != ty {
+                    tyenv.report_error(pos, format!("Mismatched types: expression expects type '{}' but term has return type '{}'", tyenv.types[ty.index()].name(tyenv), tyenv.types[ret_ty.index()].name(tyenv)));
+                }
+
+                // Check that we have the correct argument count.
+                if self.terms[tid.index()].arg_tys.len() != args.len() {
+                    tyenv.report_error(
+                        pos,
+                        format!(
+                            "Incorrect argument count for term '{}': got {}, expect {}",
+                            sym.0,
+                            args.len(),
+                            self.terms[tid.index()].arg_tys.len()
+                        ),
+                    );
+                }
+
+                // Resolve subexpressions.
+                let mut subexprs = vec![];
+                for (i, arg) in args.iter().enumerate() {
+                    let term = unwrap_or_continue!(self.terms.get(tid.index()));
+                    let arg_ty = unwrap_or_continue!(term.arg_tys.get(i).copied());
+                    let subexpr =
+                        unwrap_or_continue!(self.translate_expr(tyenv, arg, arg_ty, bindings));
+                    subexprs.push(subexpr);
+                }
+
+                Some(Expr::Term(ty, *tid, subexprs))
+            }
+            &ast::Expr::Var { ref name, pos } => {
+                let sym = tyenv.intern_mut(name);
+                // Look through bindings, innermost (most recent) first.
+                let bv = match bindings.vars.iter().rev().find(|b| b.name == sym) {
+                    None => {
+                        tyenv.report_error(pos, format!("Unknown variable '{}'", name.0));
+                        return None;
+                    }
+                    Some(bv) => bv,
+                };
+
+                // Verify type.
+                if bv.ty != ty {
+                    tyenv.report_error(
+                        pos,
+                        format!(
+                            "Variable '{}' has type {} but we need {} in context",
+                            name.0,
+                            tyenv.types[bv.ty.index()].name(tyenv),
+                            tyenv.types[ty.index()].name(tyenv)
+                        ),
+                    );
+                }
+
+                Some(Expr::Var(bv.ty, bv.id))
+            }
+            &ast::Expr::ConstInt { val, pos } => {
+                if !tyenv.types[ty.index()].is_prim() {
+                    tyenv.report_error(
+                        pos,
+                        format!(
+                            "expected non-primitive type {}, but found integer literal '{}'",
+                            tyenv.types[ty.index()].name(tyenv),
+                            val,
+                        ),
+                    );
+                }
+                Some(Expr::ConstInt(ty, val))
+            }
+            &ast::Expr::ConstPrim { ref val, pos } => {
+                let val = tyenv.intern_mut(val);
+                let const_ty = match tyenv.const_types.get(&val) {
+                    Some(ty) => *ty,
+                    None => {
+                        tyenv.report_error(pos, "Unknown constant".into());
+                        return None;
+                    }
+                };
+                if const_ty != ty {
+                    tyenv.report_error(
+                        pos,
+                        format!(
+                            "Constant '{}' has wrong type: expected {}, but is actually {}",
+                            tyenv.syms[val.index()],
+                            tyenv.types[ty.index()].name(tyenv),
+                            tyenv.types[const_ty.index()].name(tyenv)
+                        ),
+                    );
+                    return None;
+                }
+                Some(Expr::ConstPrim(ty, val))
+            }
+            &ast::Expr::Let {
+                ref defs,
+                ref body,
+                pos,
+            } => {
+                let orig_binding_len = bindings.vars.len();
+
+                // For each new binding...
+                let mut let_defs = vec![];
+                for def in defs {
+                    // Check that the given variable name does not already exist.
+                    let name = tyenv.intern_mut(&def.var);
+                    if bindings.vars.iter().any(|bv| bv.name == name) {
+                        tyenv.report_error(pos, format!("Variable '{}' already bound", def.var.0));
+                    }
+
+                    // Look up the type.
+                    let tysym = match tyenv.intern(&def.ty) {
+                        Some(ty) => ty,
+                        None => {
+                            tyenv.report_error(
+                                pos,
+                                format!("Unknown type {} for variable '{}'", def.ty.0, def.var.0),
+                            );
+                            continue;
+                        }
+                    };
+                    let tid = match tyenv.type_map.get(&tysym) {
+                        Some(tid) => *tid,
+                        None => {
+                            tyenv.report_error(
+                                pos,
+                                format!("Unknown type {} for variable '{}'", def.ty.0, def.var.0),
+                            );
+                            continue;
+                        }
+                    };
+
+                    // Evaluate the variable's value.
+                    let val = Box::new(unwrap_or_continue!(
+                        self.translate_expr(tyenv, &def.val, tid, bindings)
+                    ));
+
+                    // Bind the var with the given type.
+                    let id = VarId(bindings.next_var);
+                    bindings.next_var += 1;
+                    bindings.vars.push(BoundVar { name, id, ty: tid });
+
+                    let_defs.push((id, ty, val));
+                }
+
+                // Evaluate the body, expecting the type of the overall let-expr.
+                let body = Box::new(self.translate_expr(tyenv, body, ty, bindings)?);
+                let body_ty = body.ty();
+
+                // Pop the bindings.
+                bindings.vars.truncate(orig_binding_len);
+
+                Some(Expr::Let {
+                    ty: body_ty,
+                    bindings: let_defs,
+                    body,
+                })
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::ast::Ident;
+    use crate::lexer::{Lexer, Pos};
+    use crate::parser::parse;
+
+    #[test]
+    fn build_type_env() {
+        let text = r"
+            (type u32 (primitive u32))
+            (type A extern (enum (B (f1 u32) (f2 u32)) (C (f1 u32))))
+        ";
+        let ast = parse(Lexer::from_str(text, "file.isle").unwrap()).expect("should parse");
+        let tyenv = TypeEnv::from_ast(&ast).expect("should not have type-definition errors");
+
+        let sym_a = tyenv
+            .intern(&Ident("A".to_string(), Default::default()))
+            .unwrap();
+        let sym_b = tyenv
+            .intern(&Ident("B".to_string(), Default::default()))
+            .unwrap();
+        let sym_c = tyenv
+            .intern(&Ident("C".to_string(), Default::default()))
+            .unwrap();
+        let sym_a_b = tyenv
+            .intern(&Ident("A.B".to_string(), Default::default()))
+            .unwrap();
+        let sym_a_c = tyenv
+            .intern(&Ident("A.C".to_string(), Default::default()))
+            .unwrap();
+        let sym_u32 = tyenv
+            .intern(&Ident("u32".to_string(), Default::default()))
+            .unwrap();
+        let sym_f1 = tyenv
+            .intern(&Ident("f1".to_string(), Default::default()))
+            .unwrap();
+        let sym_f2 = tyenv
+            .intern(&Ident("f2".to_string(), Default::default()))
+            .unwrap();
+
+        assert_eq!(tyenv.type_map.get(&sym_u32).unwrap(), &TypeId(0));
+        assert_eq!(tyenv.type_map.get(&sym_a).unwrap(), &TypeId(1));
+
+        let expected_types = vec![
+            Type::Primitive(
+                TypeId(0),
+                sym_u32,
+                Pos {
+                    file: 0,
+                    offset: 19,
+                    line: 2,
+                    col: 0,
+                },
+            ),
+            Type::Enum {
+                name: sym_a,
+                id: TypeId(1),
+                is_extern: true,
+                variants: vec![
+                    Variant {
+                        name: sym_b,
+                        fullname: sym_a_b,
+                        id: VariantId(0),
+                        fields: vec![
+                            Field {
+                                name: sym_f1,
+                                id: FieldId(0),
+                                ty: TypeId(0),
+                            },
+                            Field {
+                                name: sym_f2,
+                                id: FieldId(1),
+                                ty: TypeId(0),
+                            },
+                        ],
+                    },
+                    Variant {
+                        name: sym_c,
+                        fullname: sym_a_c,
+                        id: VariantId(1),
+                        fields: vec![Field {
+                            name: sym_f1,
+                            id: FieldId(0),
+                            ty: TypeId(0),
+                        }],
+                    },
+                ],
+                pos: Pos {
+                    file: 0,
+                    offset: 58,
+                    line: 3,
+                    col: 0,
+                },
+            },
+        ];
+
+        assert_eq!(tyenv.types.len(), expected_types.len());
+        for (i, (actual, expected)) in tyenv.types.iter().zip(&expected_types).enumerate() {
+            assert_eq!(expected, actual, "`{}`th type is not equal!", i);
+        }
+    }
+}
diff --git a/cranelift/isle/isle/src/trie.rs b/cranelift/isle/isle/src/trie.rs
new file mode 100644
index 0000000000..5acb08b0ab
--- /dev/null
+++ b/cranelift/isle/isle/src/trie.rs
@@ -0,0 +1,587 @@
+//! Trie construction.
+
+use crate::ir::{lower_rule, ExprSequence, PatternInst, PatternSequence};
+use crate::sema::{RuleId, TermEnv, TermId, TypeEnv};
+use std::collections::BTreeMap;
+
+/// Construct the tries for each term.
+pub fn build_tries(typeenv: &TypeEnv, termenv: &TermEnv) -> BTreeMap<TermId, TrieNode> {
+    let mut builder = TermFunctionsBuilder::new(typeenv, termenv);
+    builder.build();
+    log::trace!("builder: {:?}", builder);
+    builder.finalize()
+}
+
+/// One "input symbol" for the decision tree that handles matching on
+/// a term. Each symbol represents one step: we either run a match op,
+/// or we finish the match.
+///
+/// Note that in the original Peepmatic scheme, the input-symbol to
+/// the FSM was specified slightly differently. The automaton
+/// responded to alphabet symbols that corresponded only to match
+/// results, and the "extra state" was used at each automaton node to
+/// represent the op to run next. This extra state differentiated
+/// nodes that would otherwise be merged together by
+/// deduplication. That scheme works well enough, but the "extra
+/// state" is slightly confusing and diverges slightly from a pure
+/// automaton.
+///
+/// Instead, here, we imagine that the user of the automaton/trie can
+/// query the possible transition edges out of the current state. Each
+/// of these edges corresponds to one possible match op to run. After
+/// running a match op, we reach a new state corresponding to
+/// successful matches up to that point.
+///
+/// However, it's a bit more subtle than this. Consider the
+/// prioritization problem. We want to give the DSL user the ability
+/// to change the order in which rules apply, for example to have a
+/// tier of "fallback rules" that apply only if more custom rules do
+/// not match.
+///
+/// A somewhat simplistic answer to this problem is "more specific
+/// rule wins". However, this implies the existence of a total
+/// ordering of linearized match sequences that may not fully capture
+/// the intuitive meaning of "more specific". Consider three left-hand
+/// sides:
+///
+/// - (A _ _)
+/// - (A (B _) _)
+/// - (A _ (B _))
+///
+/// Intuitively, the first is the least specific. Given the input `(A
+/// (B 1) (B 2))`, we can say for sure that the first should not be
+/// chosen, because either the second or third would match "more" of
+/// the input tree. But which of the second and third should be
+/// chosen? A "lexicographic ordering" rule would say that we sort
+/// left-hand sides such that the `(B _)` sub-pattern comes before the
+/// wildcard `_`, so the second rule wins. But that is arbitrarily
+/// privileging one over the other based on the order of the
+/// arguments.
+///
+/// Instead, we can accept explicit priorities from the user to allow
+/// either choice. So we need a data structure that can associate
+/// matching inputs *with priorities* to outputs.
+///
+/// Next, we build a decision tree rather than an FSM. Why? Because
+/// we're compiling to a structured language, Rust, and states become
+/// *program points* rather than *data*, we cannot easily support a
+/// DAG structure. In other words, we are not producing a FSM that we
+/// can interpret at runtime; rather we are compiling code in which
+/// each state corresponds to a sequence of statements and
+/// control-flow that branches to a next state, we naturally need
+/// nesting; we cannot codegen arbitrary state transitions in an
+/// efficient manner. We could support a limited form of DAG that
+/// reifies "diamonds" (two alternate paths that reconverge), but
+/// supporting this in a way that lets the output refer to values from
+/// either side is very complex (we need to invent phi-nodes), and the
+/// cases where we want to do this rather than invoke a sub-term (that
+/// is compiled to a separate function) are rare. Finally, note that
+/// one reason to deduplicate nodes and turn a tree back into a DAG --
+/// "output-suffix sharing" as some other instruction-rewriter
+/// engines, such as Peepmatic, do -- is not done, because all
+/// "output" occurs at leaf nodes; this is necessary because we do not
+/// want to start invoking external constructors until we are sure of
+/// the match. Some of the code-sharing advantages of the "suffix
+/// sharing" scheme can be obtained in a more flexible and
+/// user-controllable way (with less understanding of internal
+/// compiler logic needed) by factoring logic into different internal
+/// terms, which become different compiled functions. This is likely
+/// to happen anyway as part of good software engineering practice.
+///
+/// We prepare for codegen by building a "prioritized trie", where the
+/// trie associates input strings with priorities to output values.
+/// Each input string is a sequence of match operators followed by an
+/// "end of match" token, and each output is a sequence of ops that
+/// build the output expression. Each input-output mapping is
+/// associated with a priority. The goal of the trie is to generate a
+/// decision-tree procedure that lets us execute match ops in a
+/// deterministic way, eventually landing at a state that corresponds
+/// to the highest-priority matching rule and can produce the output.
+///
+/// To build this trie, we construct nodes with edges to child nodes;
+/// each edge consists of (i) one input token (a `PatternInst` or
+/// EOM), and (ii) the minimum and maximum priorities of rules along
+/// this edge. In a way this resembles an interval tree, though the
+/// intervals of children need not be disjoint.
+///
+/// To add a rule to this trie, we perform the usual trie-insertion
+/// logic, creating edges and subnodes where necessary, and updating
+/// the priority-range of each edge that we traverse to include the
+/// priority of the inserted rule.
+///
+/// However, we need to be a little bit careful, because with only
+/// priority ranges in place and the potential for overlap, we have
+/// something that resembles an NFA. For example, consider the case
+/// where we reach a node in the trie and have two edges with two
+/// match ops, one corresponding to a rule with priority 10, and the
+/// other corresponding to two rules, with priorities 20 and 0. The
+/// final match could lie along *either* path, so we have to traverse
+/// both.
+///
+/// So, to avoid this, we perform a sort of moral equivalent to the
+/// NFA-to-DFA conversion "on the fly" as we insert nodes by
+/// duplicating subtrees. At any node, when inserting with a priority
+/// P and when outgoing edges lie in a range [P_lo, P_hi] such that P
+/// >= P_lo and P <= P_hi, we "priority-split the edges" at priority
+/// P.
+///
+/// To priority-split the edges in a node at priority P:
+///
+/// - For each out-edge with priority [P_lo, P_hi] s.g. P \in [P_lo,
+///   P_hi], and token T:
+///   - Trim the subnode at P, yielding children C_lo and C_hi.
+///   - Both children must be non-empty (have at least one leaf)
+///     because the original node must have had a leaf at P_lo
+///     and a leaf at P_hi.
+///   - Replace the one edge with two edges, one for each child, with
+///     the original match op, and with ranges calculated according to
+///     the trimmed children.
+///
+/// To trim a node into range [P_lo, P_hi]:
+///
+/// - For a decision node:
+///   - If any edges have a range outside the bounds of the trimming
+///     range, trim the bounds of the edge, and trim the subtree under the
+///     edge into the trimmed edge's range. If the subtree is trimmed
+///     to `None`, remove the edge.
+///   - If all edges are removed, the decision node becomes `None`.
+/// - For a leaf node:
+///   - If the priority is outside the range, the node becomes `None`.
+///
+/// As we descend a path to insert a leaf node, we (i) priority-split
+/// if any edges' priority ranges overlap the insertion priority
+/// range, and (ii) expand priority ranges on edges to include the new
+/// leaf node's priority.
+///
+/// As long as we do this, we ensure the two key priority-trie
+/// invariants:
+///
+/// 1. At a given node, no two edges exist with priority ranges R_1,
+///    R_2 such that R_1 ∩ R_2 ≠ ∅, unless R_1 and R_2 are unit ranges
+///    ([x, x]) and are on edges with different match-ops.
+/// 2. Along the path from the root to any leaf node with priority P,
+///    each edge has a priority range R such that P ∈ R.
+///
+/// Note that this means that multiple edges with a single match-op
+/// may exist, with different priorities.
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum TrieSymbol {
+    /// Run a match operation to continue matching a LHS.
+    Match {
+        /// The match operation to run.
+        op: PatternInst,
+    },
+    /// We successfully matched a LHS.
+    EndOfMatch,
+}
+
+impl TrieSymbol {
+    fn is_eom(&self) -> bool {
+        match self {
+            TrieSymbol::EndOfMatch => true,
+            _ => false,
+        }
+    }
+}
+
+/// A priority.
+pub type Prio = i64;
+
+/// An inclusive range of priorities.
+#[derive(Clone, Copy, Debug)]
+pub struct PrioRange {
+    /// The minimum of this range.
+    pub min: Prio,
+    /// The maximum of this range.
+    pub max: Prio,
+}
+
+impl PrioRange {
+    fn contains(&self, prio: Prio) -> bool {
+        prio >= self.min && prio <= self.max
+    }
+
+    fn is_unit(&self) -> bool {
+        self.min == self.max
+    }
+
+    fn overlaps(&self, other: PrioRange) -> bool {
+        // This can be derived via DeMorgan: !(self.begin > other.end
+        // OR other.begin > self.end).
+        self.min <= other.max && other.min <= self.max
+    }
+
+    fn intersect(&self, other: PrioRange) -> PrioRange {
+        PrioRange {
+            min: std::cmp::max(self.min, other.min),
+            max: std::cmp::min(self.max, other.max),
+        }
+    }
+
+    fn union(&self, other: PrioRange) -> PrioRange {
+        PrioRange {
+            min: std::cmp::min(self.min, other.min),
+            max: std::cmp::max(self.max, other.max),
+        }
+    }
+
+    fn split_at(&self, prio: Prio) -> (PrioRange, PrioRange) {
+        assert!(self.contains(prio));
+        assert!(!self.is_unit());
+        if prio == self.min {
+            (
+                PrioRange {
+                    min: self.min,
+                    max: self.min,
+                },
+                PrioRange {
+                    min: self.min + 1,
+                    max: self.max,
+                },
+            )
+        } else {
+            (
+                PrioRange {
+                    min: self.min,
+                    max: prio - 1,
+                },
+                PrioRange {
+                    min: prio,
+                    max: self.max,
+                },
+            )
+        }
+    }
+}
+
+/// An edge in our term trie.
+#[derive(Clone, Debug)]
+pub struct TrieEdge {
+    /// The priority range for this edge's sub-trie.
+    pub range: PrioRange,
+    /// The match operation to perform for this edge.
+    pub symbol: TrieSymbol,
+    /// This edge's sub-trie.
+    pub node: TrieNode,
+}
+
+/// A node in the term trie.
+#[derive(Clone, Debug)]
+pub enum TrieNode {
+    /// One or more patterns could match.
+    ///
+    /// Maybe one pattern already has matched, but there are more (higher
+    /// priority and/or same priority but more specific) patterns that could
+    /// still match.
+    Decision {
+        /// The child sub-tries that we can match from this point on.
+        edges: Vec<TrieEdge>,
+    },
+
+    /// The successful match of an LHS pattern, and here is its RHS expression.
+    Leaf {
+        /// The priority of this rule.
+        prio: Prio,
+        /// The RHS expression to evaluate upon a successful LHS pattern match.
+        output: ExprSequence,
+    },
+
+    /// No LHS pattern matches.
+    Empty,
+}
+
+impl TrieNode {
+    fn is_empty(&self) -> bool {
+        matches!(self, &TrieNode::Empty)
+    }
+
+    fn insert(
+        &mut self,
+        prio: Prio,
+        mut input: impl Iterator<Item = TrieSymbol>,
+        output: ExprSequence,
+    ) -> bool {
+        // Take one input symbol. There must be *at least* one, EOM if
+        // nothing else.
+        let op = input
+            .next()
+            .expect("Cannot insert into trie with empty input sequence");
+        let is_last = op.is_eom();
+
+        // If we are empty, turn into a decision node.
+        if self.is_empty() {
+            *self = TrieNode::Decision { edges: vec![] };
+        }
+
+        // We must be a decision node.
+        let edges = match self {
+            &mut TrieNode::Decision { ref mut edges } => edges,
+            _ => panic!("insert on leaf node!"),
+        };
+
+        // Do we need to split?
+        let needs_split = edges
+            .iter()
+            .any(|edge| edge.range.contains(prio) && !edge.range.is_unit());
+
+        // If so, pass over all edges/subnodes and split each.
+        if needs_split {
+            let mut new_edges = vec![];
+            for edge in std::mem::take(edges) {
+                if !edge.range.contains(prio) || edge.range.is_unit() {
+                    new_edges.push(edge);
+                    continue;
+                }
+
+                let (lo_range, hi_range) = edge.range.split_at(prio);
+                let lo = edge.node.trim(lo_range);
+                let hi = edge.node.trim(hi_range);
+                if let Some((node, range)) = lo {
+                    new_edges.push(TrieEdge {
+                        range,
+                        symbol: edge.symbol.clone(),
+                        node,
+                    });
+                }
+                if let Some((node, range)) = hi {
+                    new_edges.push(TrieEdge {
+                        range,
+                        symbol: edge.symbol,
+                        node,
+                    });
+                }
+            }
+            *edges = new_edges;
+        }
+
+        // Now find or insert the appropriate edge.
+        let mut edge: Option<usize> = None;
+        let mut last_edge_with_op: Option<usize> = None;
+        let mut last_edge_with_op_prio: Option<Prio> = None;
+        for i in 0..(edges.len() + 1) {
+            if i == edges.len() || prio > edges[i].range.max {
+                // We've passed all edges with overlapping priority
+                // ranges. Maybe the last edge we saw with the op
+                // we're inserting can have its range expanded,
+                // however.
+                if last_edge_with_op.is_some() {
+                    // Move it to the end of the run of equal-unit-range ops.
+                    edges.swap(last_edge_with_op.unwrap(), i - 1);
+                    edge = Some(i - 1);
+                    edges[i - 1].range.max = prio;
+                    break;
+                }
+                edges.insert(
+                    i,
+                    TrieEdge {
+                        range: PrioRange {
+                            min: prio,
+                            max: prio,
+                        },
+                        symbol: op.clone(),
+                        node: TrieNode::Empty,
+                    },
+                );
+                edge = Some(i);
+                break;
+            }
+            if i == edges.len() {
+                break;
+            }
+            if edges[i].symbol == op {
+                last_edge_with_op = Some(i);
+                last_edge_with_op_prio = Some(edges[i].range.max);
+            }
+            if last_edge_with_op_prio.is_some()
+                && last_edge_with_op_prio.unwrap() < edges[i].range.max
+            {
+                last_edge_with_op = None;
+                last_edge_with_op_prio = None;
+            }
+            if edges[i].range.contains(prio) && edges[i].symbol == op {
+                edge = Some(i);
+                break;
+            }
+        }
+        let edge = edge.expect("Must have found an edge at least at last iter");
+        let edge = &mut edges[edge];
+
+        if is_last {
+            if !edge.node.is_empty() {
+                // If a leaf node already exists at an overlapping
+                // prio for this op, there are two competing rules, so
+                // we can't insert this one.
+                return false;
+            }
+            edge.node = TrieNode::Leaf { prio, output };
+            true
+        } else {
+            edge.node.insert(prio, input, output)
+        }
+    }
+
+    fn trim(&self, range: PrioRange) -> Option<(TrieNode, PrioRange)> {
+        match self {
+            &TrieNode::Empty => None,
+            &TrieNode::Leaf { prio, ref output } => {
+                if range.contains(prio) {
+                    Some((
+                        TrieNode::Leaf {
+                            prio,
+                            output: output.clone(),
+                        },
+                        PrioRange {
+                            min: prio,
+                            max: prio,
+                        },
+                    ))
+                } else {
+                    None
+                }
+            }
+            &TrieNode::Decision { ref edges } => {
+                let edges = edges
+                    .iter()
+                    .filter_map(|edge| {
+                        if !edge.range.overlaps(range) {
+                            None
+                        } else {
+                            let range = range.intersect(edge.range);
+                            if let Some((node, range)) = edge.node.trim(range) {
+                                Some(TrieEdge {
+                                    range,
+                                    symbol: edge.symbol.clone(),
+                                    node,
+                                })
+                            } else {
+                                None
+                            }
+                        }
+                    })
+                    .collect::<Vec<_>>();
+
+                if edges.is_empty() {
+                    None
+                } else {
+                    let range = edges
+                        .iter()
+                        .map(|edge| edge.range)
+                        .reduce(|a, b| a.union(b))
+                        .expect("reduce on non-empty vec must not return None");
+                    Some((TrieNode::Decision { edges }, range))
+                }
+            }
+        }
+    }
+
+    /// Get a pretty-printed version of this trie, for debugging.
+    pub fn pretty(&self) -> String {
+        let mut s = String::new();
+        pretty_rec(&mut s, self, "");
+        return s;
+
+        fn pretty_rec(s: &mut String, node: &TrieNode, indent: &str) {
+            match node {
+                TrieNode::Decision { edges } => {
+                    s.push_str(indent);
+                    s.push_str("TrieNode::Decision:\n");
+
+                    let new_indent = indent.to_owned() + "    ";
+                    for edge in edges {
+                        s.push_str(indent);
+                        s.push_str(&format!(
+                            "  edge: range = {:?}, symbol: {:?}\n",
+                            edge.range, edge.symbol
+                        ));
+                        pretty_rec(s, &edge.node, &new_indent);
+                    }
+                }
+                TrieNode::Empty | TrieNode::Leaf { .. } => {
+                    s.push_str(indent);
+                    s.push_str(&format!("{:?}\n", node));
+                }
+            }
+        }
+    }
+}
+
+/// Builder context for one function in generated code corresponding
+/// to one root input term.
+///
+/// A `TermFunctionBuilder` can correspond to the matching
+/// control-flow and operations that we execute either when evaluating
+/// *forward* on a term, trying to match left-hand sides against it
+/// and transforming it into another term; or *backward* on a term,
+/// trying to match another rule's left-hand side against an input to
+/// produce the term in question (when the term is used in the LHS of
+/// the calling term).
+#[derive(Debug)]
+struct TermFunctionBuilder {
+    trie: TrieNode,
+}
+
+impl TermFunctionBuilder {
+    fn new() -> Self {
+        TermFunctionBuilder {
+            trie: TrieNode::Empty,
+        }
+    }
+
+    fn add_rule(&mut self, prio: Prio, pattern_seq: PatternSequence, expr_seq: ExprSequence) {
+        let symbols = pattern_seq
+            .insts
+            .into_iter()
+            .map(|op| TrieSymbol::Match { op })
+            .chain(std::iter::once(TrieSymbol::EndOfMatch));
+        self.trie.insert(prio, symbols, expr_seq);
+    }
+}
+
+#[derive(Debug)]
+struct TermFunctionsBuilder<'a> {
+    typeenv: &'a TypeEnv,
+    termenv: &'a TermEnv,
+    builders_by_term: BTreeMap<TermId, TermFunctionBuilder>,
+}
+
+impl<'a> TermFunctionsBuilder<'a> {
+    fn new(typeenv: &'a TypeEnv, termenv: &'a TermEnv) -> Self {
+        log::trace!("typeenv: {:?}", typeenv);
+        log::trace!("termenv: {:?}", termenv);
+        Self {
+            builders_by_term: BTreeMap::new(),
+            typeenv,
+            termenv,
+        }
+    }
+
+    fn build(&mut self) {
+        for rule in 0..self.termenv.rules.len() {
+            let rule = RuleId(rule);
+            let prio = self.termenv.rules[rule.index()].prio.unwrap_or(0);
+
+            let (pattern, expr) = lower_rule(self.typeenv, self.termenv, rule);
+            let root_term = self.termenv.rules[rule.index()].lhs.root_term().unwrap();
+
+            log::trace!(
+                "build:\n- rule {:?}\n- pattern {:?}\n- expr {:?}",
+                self.termenv.rules[rule.index()],
+                pattern,
+                expr
+            );
+            self.builders_by_term
+                .entry(root_term)
+                .or_insert_with(|| TermFunctionBuilder::new())
+                .add_rule(prio, pattern.clone(), expr.clone());
+        }
+    }
+
+    fn finalize(self) -> BTreeMap<TermId, TrieNode> {
+        let functions_by_term = self
+            .builders_by_term
+            .into_iter()
+            .map(|(term, builder)| (term, builder.trie))
+            .collect::<BTreeMap<_, _>>();
+        functions_by_term
+    }
+}
diff --git a/cranelift/isle/isle_examples/construct-and-extract.isle b/cranelift/isle/isle_examples/construct-and-extract.isle
new file mode 100644
index 0000000000..a0951edd41
--- /dev/null
+++ b/cranelift/isle/isle_examples/construct-and-extract.isle
@@ -0,0 +1,17 @@
+(type i32 (primitive i32))
+
+(type B (enum (B (x i32) (y i32))))
+
+;; `isub` has a constructor and extractor.
+(decl isub (i32 i32) B)
+(rule (isub x y)
+      (B.B x y))
+(extractor (isub x y)
+           (B.B x y))
+
+;; `value_array_2` has both an external extractor and an external constructor.
+(type Value (primitive Value))
+(type ValueArray2 extern (enum))
+(decl value_array_2 (Value Value) ValueArray2)
+(extern extractor infallible value_array_2 unpack_value_array_2)
+(extern constructor value_array_2 pack_value_array_2)
diff --git a/cranelift/isle/isle_examples/error1.isle b/cranelift/isle/isle_examples/error1.isle
new file mode 100644
index 0000000000..5d304668f6
--- /dev/null
+++ b/cranelift/isle/isle_examples/error1.isle
@@ -0,0 +1,36 @@
+(type u32 (primitive u32))
+(type bool (primitive bool))
+(type A (enum (A1 (x u32))))
+
+(decl Ext1 (u32) A)
+(decl Ext2 (u32) A)
+(extern extractor Ext1 ext1)
+(extern extractor Ext2 ext2)
+
+(decl C (bool) A)
+(extern constructor C c)
+
+(decl Lower (A) A)
+
+(rule
+  (Lower
+    (and
+      a
+      (Ext1 x)
+      (Ext2 =q)))
+  (C y))
+
+(type R (enum (A (x u32))))
+
+(type Opcode (enum A B C))
+(type MachInst (enum D E F))
+(decl Lower2 (Opcode) MachInst)
+(rule
+  (Lower2 (Opcode.A))
+  (R.A (Opcode.A)))
+(rule
+  (Lower2 (Opcode.B))
+  (MachInst.E))
+(rule
+  (Lower2 (Opcode.C))
+  (MachInst.F))
diff --git a/cranelift/isle/isle_examples/let.isle b/cranelift/isle/isle_examples/let.isle
new file mode 100644
index 0000000000..c43d1ec668
--- /dev/null
+++ b/cranelift/isle/isle_examples/let.isle
@@ -0,0 +1,21 @@
+(type u32 (primitive u32))
+(type A (enum (Add (x u32) (y u32)) (Sub (x u32) (y u32))))
+(type B (enum (B (z u32))))
+
+(decl Sub (u32 u32) u32)
+(extern constructor Sub sub)
+
+(decl Add (u32 u32) u32)
+(extern constructor Add add)
+
+(decl Lower (A) B)
+
+(rule
+  (Lower (A.Add x y))
+  (let ((z u32 (Add x y)))
+    (B.B z)))
+
+(rule
+  (Lower (A.Sub x y))
+  (let ((z u32 (Sub x y)))
+    (B.B z)))
diff --git a/cranelift/isle/isle_examples/test.isle b/cranelift/isle/isle_examples/test.isle
new file mode 100644
index 0000000000..09d5ea92af
--- /dev/null
+++ b/cranelift/isle/isle_examples/test.isle
@@ -0,0 +1,21 @@
+(type u32 (primitive u32))
+(type A (enum (A1 (x u32)) (A2 (x u32))))
+(type B (enum (B1 (x u32)) (B2 (x u32))))
+
+(decl Input (A) u32)
+(extern extractor Input get_input) ;; fn get_input<C>(ctx: &mut C, ret: u32) -> Option<(A,)>
+
+(decl Lower (A) B)
+
+(rule
+  (Lower (A.A1 sub @ (Input (A.A2 42))))
+  (B.B2 sub))
+
+(decl Extractor (B) A)
+(extractor
+  (Extractor x)
+  (A.A2 x))
+
+(rule
+  (Lower (Extractor b))
+  (B.B1 b))
diff --git a/cranelift/isle/isle_examples/test2.isle b/cranelift/isle/isle_examples/test2.isle
new file mode 100644
index 0000000000..5c0977a702
--- /dev/null
+++ b/cranelift/isle/isle_examples/test2.isle
@@ -0,0 +1,24 @@
+(type u32 (primitive u32))
+(type A (enum
+  (A1 (x B) (y B))))
+(type B (enum
+  (B1 (x u32))
+  (B2 (x u32))))
+
+(decl A2B (A) B)
+
+(rule 1
+  (A2B (A.A1 _ (B.B1 x)))
+  (B.B1 x))
+
+(rule 0
+  (A2B (A.A1 (B.B1 x) _))
+  (B.B1 x))
+
+(rule 0
+  (A2B (A.A1 (B.B2 x) _))
+  (B.B1 x))
+
+(rule -1
+  (A2B (A.A1 _ _))
+  (B.B1 42))
diff --git a/cranelift/isle/isle_examples/test3.isle b/cranelift/isle/isle_examples/test3.isle
new file mode 100644
index 0000000000..e13c21d0f8
--- /dev/null
+++ b/cranelift/isle/isle_examples/test3.isle
@@ -0,0 +1,66 @@
+(type Opcode extern (enum
+  Iadd
+  Isub
+  Load
+  Store))
+
+(type Inst (primitive Inst))
+(type InstInput (primitive InstInput))
+(type Reg (primitive Reg))
+(type u32 (primitive u32))
+
+(decl Op (Opcode) Inst)
+(extern extractor infallible Op get_opcode)
+
+(decl InstInput (InstInput u32) Inst)
+(extern extractor infallible InstInput get_inst_input (out in))
+
+(decl Producer (Inst) InstInput)
+(extern extractor Producer get_input_producer)
+
+(decl UseInput (InstInput) Reg)
+(extern constructor UseInput put_input_in_reg)
+
+(type MachInst (enum
+  (Add (a Reg) (b Reg))
+  (Add3 (a Reg) (b Reg) (c Reg))
+  (Sub (a Reg) (b Reg))))
+
+(decl Lower (Inst) MachInst)
+
+;; Extractors that give syntax sugar for (Iadd ra rb), etc.
+;;
+;; Note that this is somewhat simplistic: it directly connects inputs to
+;; MachInst regs; really we'd want to return a VReg or InstInput that we can use
+;; another extractor to connect to another (producer) inst.
+;;
+;; Also, note that while it looks a little indirect, a verification effort could
+;; define equivalences across the `rule` LHS/RHS pairs, and the types ensure that
+;; we are dealing (at the semantic level) with pure value equivalences of
+;; "terms", not arbitrary side-effecting calls.
+
+(decl Iadd (InstInput InstInput) Inst)
+(decl Isub (InstInput InstInput) Inst)
+(extractor
+  (Iadd a b)
+  (and
+    (Op (Opcode.Iadd))
+    (InstInput a <0)
+    (InstInput b <1)))
+(extractor
+  (Isub a b)
+  (and
+    (Op (Opcode.Isub))
+    (InstInput a <0)
+    (InstInput b <1)))
+
+;; Now the nice syntax-sugar that "end-user" backend authors can write:
+(rule
+  (Lower (Iadd ra rb))
+  (MachInst.Add (UseInput ra) (UseInput rb)))
+(rule
+  (Lower (Iadd (Producer (Iadd ra rb)) rc))
+  (MachInst.Add3 (UseInput ra) (UseInput rb) (UseInput rc)))
+(rule
+  (Lower (Isub ra rb))
+  (MachInst.Sub (UseInput ra) (UseInput rb)))
\ No newline at end of file
diff --git a/cranelift/isle/isle_examples/test4.isle b/cranelift/isle/isle_examples/test4.isle
new file mode 100644
index 0000000000..2035167239
--- /dev/null
+++ b/cranelift/isle/isle_examples/test4.isle
@@ -0,0 +1,42 @@
+(type u32 (primitive u32))
+(type bool (primitive bool))
+(type A (enum (A1 (x u32))))
+
+(decl Ext1 (u32) A)
+(decl Ext2 (u32) A)
+(extern extractor Ext1 ext1)
+(extern extractor Ext2 ext2)
+
+(extern const $A u32)
+(extern const $B u32)
+
+(decl C (bool) A)
+(extern constructor C c)
+
+(decl Lower (A) A)
+
+(rule
+  (Lower
+    (and
+      a
+      (Ext1 x)
+      (Ext2 =x)))
+  (C #t))
+
+(type Opcode (enum A B C))
+(type MachInst (enum D E F))
+(decl Lower2 (Opcode) MachInst)
+(rule
+  (Lower2 (Opcode.A))
+  (MachInst.D))
+(rule
+  (Lower2 (Opcode.B))
+  (MachInst.E))
+(rule
+  (Lower2 (Opcode.C))
+  (MachInst.F))
+
+(decl F (Opcode) u32)
+(rule
+ (F _)
+ $B)
\ No newline at end of file
diff --git a/cranelift/isle/isle_examples/test_main.rs b/cranelift/isle/isle_examples/test_main.rs
new file mode 100644
index 0000000000..5bec55a798
--- /dev/null
+++ b/cranelift/isle/isle_examples/test_main.rs
@@ -0,0 +1,12 @@
+mod test;
+
+struct Context;
+impl test::Context for Context {
+    fn get_input(&mut self, x: u32) -> Option<(test::A,)> {
+        Some((test::A::A1 { x: x + 1 },))
+    }
+}
+
+fn main() {
+    test::constructor_Lower(&mut Context, &test::A::A1 { x: 42 });
+}
diff --git a/cranelift/isle/isle_examples/tutorial.isle b/cranelift/isle/isle_examples/tutorial.isle
new file mode 100644
index 0000000000..d040ceeedb
--- /dev/null
+++ b/cranelift/isle/isle_examples/tutorial.isle
@@ -0,0 +1,96 @@
+;;;; Type Definitions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Declare that we are using the `i32` primitive type from Rust.
+(type i32 (primitive i32))
+
+;; Our high-level, RISC-y input IR.
+(type HighLevelInst
+  (enum (Add (a Value) (b Value))
+        (Load (addr Value))
+        (Const (c i32))))
+
+;; A value in our high-level IR is a Rust `Copy` type. Values are either defined
+;; by an instruction, or are a basic block argument.
+(type Value (primitive Value))
+
+;; Our low-level, CISC-y machine instructions.
+(type LowLevelInst
+  (enum (Add (mode AddrMode))
+        (Load (offset i32) (addr Reg))
+        (Const (c i32))))
+
+;; Different kinds of addressing modes for operands to our low-level machine
+;; instructions.
+(type AddrMode
+  (enum
+    ;; Both operands in registers.
+    (RegReg (a Reg) (b Reg))
+    ;; The destination/first operand is a register; the second operand is in
+    ;; memory at `[b + offset]`.
+    (RegMem (a Reg) (b Reg) (offset i32))
+    ;; The destination/first operand is a register, second operand is an
+    ;; immediate.
+    (RegImm (a Reg) (imm i32))))
+
+;; The register type is a Rust `Copy` type.
+(type Reg (primitive Reg))
+
+;;;; Rules ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Declare our top-level lowering function. We will attach rules to this
+;; declaration for lowering various patterns of `HighLevelInst` inputs.
+(decl lower (HighLevelInst) LowLevelInst)
+
+;; Simple rule for lowering constants.
+(rule (lower (HighLevelInst.Const c))
+      (LowLevelInst.Const c))
+
+;; Declare an external constructor that puts a high-level `Value` into a
+;; low-level `Reg`.
+(decl put_in_reg (Value) Reg)
+(extern constructor put_in_reg put_in_reg)
+
+;; Simple rule for lowering adds.
+(rule (lower (HighLevelInst.Add a b))
+      (LowLevelInst.Add
+        (AddrMode.RegReg (put_in_reg a) (put_in_reg b))))
+
+;; Simple rule for lowering loads.
+(rule (lower (HighLevelInst.Load addr))
+      (LowLevelInst.Load 0 (put_in_reg addr)))
+
+;; Declare an external extractor for extracting the instruction that defined a
+;; given operand value.
+(decl inst_result (HighLevelInst) Value)
+(extern extractor inst_result inst_result)
+
+;; Rule to sink loads into adds.
+(rule (lower (HighLevelInst.Add a (inst_result (HighLevelInst.Load addr))))
+      (LowLevelInst.Add
+        (AddrMode.RegMem (put_in_reg a)
+                         (put_in_reg addr)
+                         0)))
+
+;; Rule to sink a load of a base address with a static offset into a single add.
+(rule (lower (HighLevelInst.Add
+               a
+               (inst_result (HighLevelInst.Load
+                              (inst_result (HighLevelInst.Add
+                                             base
+                                             (inst_result (HighLevelInst.Const offset))))))))
+      (LowLevelInst.Add
+        (AddrMode.RegMem (put_in_reg a)
+                         (put_in_reg base)
+                         offset)))
+
+;; Rule for sinking an immediate into an add.
+(rule (lower (HighLevelInst.Add a (inst_result (HighLevelInst.Const c))))
+      (LowLevelInst.Add
+        (AddrMode.RegImm (put_in_reg a) c)))
+
+;; Rule for lowering loads of a base address with a static offset.
+(rule (lower (HighLevelInst.Load
+               (inst_result (HighLevelInst.Add
+                              base
+                              (inst_result (HighLevelInst.Const offset))))))
+      (LowLevelInst.Load offset (put_in_reg base)))
diff --git a/cranelift/isle/islec/Cargo.toml b/cranelift/isle/islec/Cargo.toml
new file mode 100644
index 0000000000..3a6ab3d115
--- /dev/null
+++ b/cranelift/isle/islec/Cargo.toml
@@ -0,0 +1,14 @@
+[package]
+name = "islec"
+version = "0.1.0"
+authors = ["The Cranelift Project Developers"]
+edition = "2018"
+license = "Apache-2.0 WITH LLVM-exception"
+publish = false
+
+[dependencies]
+log = "0.4"
+isle = { version = "*", path = "../isle/" }
+env_logger = { version = "0.8", default-features = false }
+miette = { version = "3.0.0", features = ["fancy"] }
+structopt = "0.3.23"
diff --git a/cranelift/isle/islec/src/main.rs b/cranelift/isle/islec/src/main.rs
new file mode 100644
index 0000000000..c4e675d064
--- /dev/null
+++ b/cranelift/isle/islec/src/main.rs
@@ -0,0 +1,63 @@
+use isle::{compile, lexer, parser};
+use miette::{Context, IntoDiagnostic, Result};
+use std::{
+    fs,
+    io::{self, Write},
+    path::PathBuf,
+};
+use structopt::StructOpt;
+
+#[derive(StructOpt)]
+struct Opts {
+    /// The output file to write the generated Rust code to. `stdout` is used if
+    /// this is not given.
+    #[structopt(short, long, parse(from_os_str))]
+    output: Option<PathBuf>,
+
+    /// The input ISLE DSL source files.
+    #[structopt(parse(from_os_str))]
+    inputs: Vec<PathBuf>,
+}
+
+fn main() -> Result<()> {
+    let _ = env_logger::try_init();
+
+    let _ = miette::set_hook(Box::new(|_| {
+        Box::new(
+            miette::MietteHandlerOpts::new()
+                // `miette` mistakenly uses braille-optimized output for emacs's
+                // `M-x shell`.
+                .force_graphical(true)
+                .build(),
+        )
+    }));
+
+    let opts = Opts::from_args();
+
+    let lexer = lexer::Lexer::from_files(opts.inputs)?;
+    let defs = parser::parse(lexer)?;
+    let code = compile::compile(&defs)?;
+
+    let stdout = io::stdout();
+    let (mut output, output_name): (Box<dyn Write>, _) = match &opts.output {
+        Some(f) => {
+            let output = Box::new(
+                fs::File::create(f)
+                    .into_diagnostic()
+                    .with_context(|| format!("failed to create '{}'", f.display()))?,
+            );
+            (output, f.display().to_string())
+        }
+        None => {
+            let output = Box::new(stdout.lock());
+            (output, "<stdout>".to_string())
+        }
+    };
+
+    output
+        .write_all(code.as_bytes())
+        .into_diagnostic()
+        .with_context(|| format!("failed to write to '{}'", output_name))?;
+
+    Ok(())
+}
diff --git a/deny.toml b/deny.toml
index 1bfbbf46c8..bcfd4b41a2 100644
--- a/deny.toml
+++ b/deny.toml
@@ -35,4 +35,5 @@ skip = [
     { name = "wast" }, # old one pulled in by witx
     { name = "itertools" }, # 0.9 pulled in by criterion-plot
     { name = "quick-error" }, # transitive dependencies
+    { name = "textwrap" }, # `miette` and `clap` depend on different versions
 ]
diff --git a/scripts/publish.rs b/scripts/publish.rs
index e4aa342246..1b8083eb4a 100644
--- a/scripts/publish.rs
+++ b/scripts/publish.rs
@@ -26,6 +26,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[
     "peepmatic",
     "peepmatic-souper",
     // cranelift
+    "isle",
     "cranelift-entity",
     "wasmtime-types",
     "cranelift-bforest",