Merge pull request #1019 from alexcrichton/cranelift-merge-for-realz

Merge the Cranelift repository into Wasmtime

.github/ISSUE_TEMPLATE/blank-issue.md

@@ -0,0 +1,4 @@
+---
+name: Blank Issue
+about: Create a blank issue.
+---

.github/ISSUE_TEMPLATE/clif-bug-report.md

@@ -0,0 +1,16 @@
+---
+name: "Cranelift Bug report"
+about: "Report a bug or a crash in Cranelift."
+labels: 'bug'
+---
+
+Thanks for opening a bug report! Please answer the questions below
+if they're relevant and delete this text before submitting.
+
+- What are the steps to reproduce the issue? Can you include a CLIF test case,
+  ideally reduced with the `bugpoint` clif-util command?
+- What do you expect to happen? What does actually happen? Does it panic, and
+  if so, with which assertion?
+- Which Cranelift version / commit hash / branch are you using?
+- If relevant, can you include some extra information about your environment?
+  (Rust version, operating system, architecture...)

.github/ISSUE_TEMPLATE/improvement.md

@@ -0,0 +1,28 @@
+---
+name: "Improvement"
+about: "A feature request or code improvement."
+---
+
+<!-- Please try to describe precisely what you would like to do in
+Cranelift/Wasmtime and/or expect from it. You can answer the questions below if
+they're relevant and delete this text before submitting. Thanks for opening an
+issue! -->
+
+#### Feature
+
+<!-- What is the feature or code improvement you would like to do in
+Cranelift/Wasmtime? -->
+
+#### Benefit
+
+<!-- What is the value of adding this in Cranelift/Wasmtime? -->
+
+#### Implementation
+
+<!-- Do you have an implementation plan, and/or ideas for data structures or
+algorithms to use? -->
+
+#### Alternatives
+
+<!-- Have you considered alternative implementations? If so, how are they
+better or worse than your proposal? -->

.github/pull_request_template.md

@@ -0,0 +1,18 @@
+<!--
+
+Please ensure that the following steps are all taken care of before submitting
+the PR.
+
+- [ ] This has been discussed in issue #..., or if not, please tell us why
+  here.
+- [ ] A short description of what this does, why it is needed; if the
+  description becomes long, the matter should probably be discussed in an issue
+  first.
+- [ ] This PR contains test cases, if meaningful.
+- [ ] A reviewer from the core maintainer team has been assigned for this PR.
+  If you don't know who could review this, please indicate so. The list of
+  suggested reviewers on the right can help you.
+
+Please ensure all communication adheres to the [code of
+conduct](https://github.com/bytecodealliance/wasmtime/blob/master/CODE_OF_CONDUCT.md).
+-->

.github/workflows/main.yml

@@ -52,7 +52,8 @@ jobs:
     - uses: ./.github/actions/install-rust
       with:
         toolchain: nightly
-    - run: cargo doc --no-deps --all --exclude wasmtime-cli --exclude test-programs
+    - run: cargo doc --no-deps --all --exclude wasmtime-cli --exclude test-programs --exclude cranelift-codegen-meta
+    - run: cargo doc --package cranelift-codegen-meta --document-private-items
     - uses: actions/upload-artifact@v1
       with:
         name: doc-api
@@ -107,6 +108,16 @@ jobs:
             | shuf \
             | head -n 100 \
             | xargs cargo fuzz run differential --release --debug-assertions
+    - run: |
+        find fuzz/corpus/reader_parse_test -type f \
+            | shuf \
+            | head -n 100 \
+            | xargs cargo fuzz run reader_parse_test --release --debug-assertions
+    - run: |
+        find fuzz/corpus/translate_module -type f \
+            | shuf \
+            | head -n 100 \
+            | xargs cargo fuzz run translate_module --release --debug-assertions
 
   # Install wasm32-unknown-emscripten target, and ensure `crates/wasi-common`
   # compiles to Emscripten.
@@ -209,6 +220,19 @@ jobs:
       env:
         RUST_BACKTRACE: 1
 
+  # Verify that cranelift's code generation is deterministic
+  meta_determinist_check:
+    name: Meta deterministic check
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: true
+    - name: Install Rust
+      run: rustup update stable && rustup default stable
+    - run: cd cranelift/codegen && cargo build --features all-arch
+    - run: ci/ensure_deterministic_build.sh
+
   # Builds a Python wheel (package) for Windows/Mac/Linux. Note that we're
   # careful to create binary-compatible releases here to old releases of
   # Windows/Mac/Linux. This will also build wheels for Python 3.6, 3.7 and 3.8.

.gitignore

@@ -1,13 +1,16 @@
 *.bk
-*.swp
+*.pyc
 *.swo
+*.swp
 *.swx
-tags
-target
-.*.rustfmt
-cranelift.dbg*
-rusty-tags.*
 *~
+.*.rustfmt
+.mypy_cache
 \#*\#
+cranelift.dbg*
+docs/_build
 docs/book
 .vscode/
+rusty-tags.*
+tags
+target

CONTRIBUTING.md

@@ -1,13 +1,12 @@
-# Contributing to Wasmtime
+# Contributing to Wasmtime and/or Cranelift
 
-Wasmtime is a [Bytecode Alliance] project, and follows the Bytecode Alliance's [Code of Conduct] and [Organizational Code of Conduct].
+Wasmtime and Cranelift are [Bytecode Alliance] projects. They follow the
+Bytecode Alliance's [Code of Conduct] and [Organizational Code of Conduct].
 
-Wasmtime follows the same development style as Cranelift, so check out
+For more information about contributing to these projects you can consult the
-[Cranelift's CONTRIBUTING.md]. Of course, for Wasmtime-specific issues, please
+[online documentation] which should cover all sorts of topics.
-use the [Wasmtime issue tracker].
 
 [Bytecode Alliance]: https://bytecodealliance.org/
 [Code of Conduct]: CODE_OF_CONDUCT.md
 [Organizational Code of Conduct]: ORG_CODE_OF_CONDUCT.md
-[Cranelift's CONTRIBUTING.md]: https://github.com/bytecodealliance/cranelift/blob/master/CONTRIBUTING.md
+[online documentation]: https://bytecodealliance.github.io/wasmtime/contributing.html
-[Wasmtime issue tracker]: https://github.com/bytecodealliance/wasmtime/issues/new

Cargo.lock

@@ -6,6 +6,15 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
 
+[[package]]
+name = "ahash"
+version = "0.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3"
+dependencies = [
+ "const-random",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.8"
@@ -68,6 +77,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "autocfg"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
+
 [[package]]
 name = "autocfg"
 version = "1.0.0"
@@ -305,6 +320,26 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "const-random"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a"
+dependencies = [
+ "const-random-macro",
+ "proc-macro-hack",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a"
+dependencies = [
+ "getrandom",
+ "proc-macro-hack",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -324,8 +359,6 @@ dependencies = [
 [[package]]
 name = "cranelift-bforest"
 version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45a9c21f8042b9857bda93f6c1910b9f9f24100187a3d3d52f214a34e3dc5818"
 dependencies = [
  "cranelift-entity",
 ]
@@ -333,8 +366,6 @@ dependencies = [
 [[package]]
 name = "cranelift-codegen"
 version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7853f77a6e4a33c67a69c40f5e1bb982bd2dc5c4a22e17e67b65bbccf9b33b2e"
 dependencies = [
  "byteorder",
  "cranelift-bforest",
@@ -342,6 +373,7 @@ dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
  "gimli",
+ "hashbrown",
  "log",
  "serde",
  "smallvec",
@@ -352,8 +384,6 @@ dependencies = [
 [[package]]
 name = "cranelift-codegen-meta"
 version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "084cd6d5fb0d1da28acd72c199471bfb09acc703ec8f3bf07b1699584272a3b9"
 dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
@@ -362,14 +392,10 @@ dependencies = [
 [[package]]
 name = "cranelift-codegen-shared"
 version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "701b599783305a58c25027a4d73f2d6b599b2d8ef3f26677275f480b4d51e05d"
 
 [[package]]
 name = "cranelift-entity"
 version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b88e792b28e1ebbc0187b72ba5ba880dad083abe9231a99d19604d10c9e73f38"
 dependencies = [
  "serde",
 ]
@@ -377,10 +403,9 @@ dependencies = [
 [[package]]
 name = "cranelift-frontend"
 version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "518344698fa6c976d853319218415fdfb4f1bc6b42d0b2e2df652e55dff1f778"
 dependencies = [
  "cranelift-codegen",
+ "hashbrown",
  "log",
  "smallvec",
  "target-lexicon",
@@ -389,27 +414,34 @@ dependencies = [
 [[package]]
 name = "cranelift-native"
 version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32daf082da21c0c05d93394ff4842c2ab7c4991b1f3186a1d952f8ac660edd0b"
 dependencies = [
  "cranelift-codegen",
  "raw-cpuid",
  "target-lexicon",
 ]
 
+[[package]]
+name = "cranelift-reader"
+version = "0.59.0"
+dependencies = [
+ "cranelift-codegen",
+ "target-lexicon",
+]
+
 [[package]]
 name = "cranelift-wasm"
 version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2aa816f554a3ef739a5d17ca3081a1f8983f04c944ea8ff60fb8d9dd8cd2d7b"
 dependencies = [
  "cranelift-codegen",
  "cranelift-entity",
  "cranelift-frontend",
+ "hashbrown",
  "log",
  "serde",
+ "target-lexicon",
  "thiserror",
  "wasmparser 0.51.2",
+ "wat",
 ]
 
 [[package]]
@@ -438,7 +470,7 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.0",
  "cfg-if",
  "crossbeam-utils",
  "lazy_static",
@@ -463,7 +495,7 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.0",
  "cfg-if",
  "lazy_static",
 ]
@@ -776,6 +808,16 @@ dependencies = [
  "scroll",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead"
+dependencies = [
+ "ahash",
+ "autocfg 0.1.7",
+]
+
 [[package]]
 name = "heck"
 version = "0.3.1"
@@ -815,7 +857,7 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.0",
 ]
 
 [[package]]
@@ -1053,7 +1095,7 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.0",
  "num-traits",
 ]
 
@@ -1063,7 +1105,7 @@ version = "0.1.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.0",
  "num-traits",
 ]
 
@@ -1073,7 +1115,7 @@ version = "0.1.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dfb0800a0291891dd9f4fe7bd9c19384f98f7fbe0cd0f39a2c6b88b9868bbc00"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.0",
  "num-integer",
  "num-traits",
 ]
@@ -1084,7 +1126,7 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.0",
  "num-integer",
  "num-traits",
 ]
@@ -1095,7 +1137,7 @@ version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.0",
 ]
 
 [[package]]
@@ -2080,7 +2122,11 @@ name = "wasmtime-fuzz"
 version = "0.12.0"
 dependencies = [
  "arbitrary 0.2.0",
+ "cranelift-codegen",
+ "cranelift-reader",
+ "cranelift-wasm",
  "libfuzzer-sys",
+ "target-lexicon",
  "wasmtime",
  "wasmtime-fuzzing",
 ]

ci/ensure_deterministic_build.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# This script makes sure that the meta crate deterministically generate files
+# with a high probability.
+# The current directory must be set to the repository's root.
+
+set -e
+
+BUILD_SCRIPT=$(find -wholename "./target/debug/build/cranelift-codegen-*/build-script-build")
+
+# First, run the script to generate a reference comparison.
+rm -rf /tmp/reference
+mkdir /tmp/reference
+OUT_DIR=/tmp/reference TARGET=x86_64 $BUILD_SCRIPT
+
+# To make sure the build script doesn't depend on the current directory, we'll
+# change the current working directory on every iteration. Make this easy to
+# reproduce this locally by first copying the target/ directory into an initial
+# temporary directory (and not move and lose the local clone's content).
+rm -rf /tmp/src0
+mkdir /tmp/src0
+
+echo Copying target directory...
+cp -r ./target /tmp/src0/target
+cd /tmp/src0
+echo "Done, starting loop."
+
+# Then, repeatedly make sure that the output is the same.
+for i in {1..20}
+do
+    # Move to a different directory, as explained above.
+    rm -rf /tmp/src$i
+    mkdir /tmp/src$i
+    mv ./* /tmp/src$i
+    cd /tmp/src$i
+
+    rm -rf /tmp/try
+    mkdir /tmp/try
+    OUT_DIR=/tmp/try TARGET=x86_64 $BUILD_SCRIPT
+    diff -qr /tmp/reference /tmp/try
+done

clippy.toml

@@ -1 +1 @@
-doc-valid-idents = ["WebAssembly"]
+doc-valid-idents = [ "WebAssembly", "NaN", "SetCC" ]

cranelift/Cargo.toml

@@ -0,0 +1,47 @@
+[package]
+name = "cranelift-tools"
+authors = ["The Cranelift Project Developers"]
+version = "0.59.0"
+description = "Binaries for testing the Cranelift libraries"
+license = "Apache-2.0 WITH LLVM-exception"
+documentation = "https://cranelift.readthedocs.io/"
+repository = "https://github.com/bytecodealliance/cranelift"
+publish = false
+edition = "2018"
+
+[[bin]]
+name = "clif-util"
+path = "src/clif-util.rs"
+
+[dependencies]
+cfg-if = "0.1"
+cranelift-codegen = { path = "codegen", version = "0.59.0" }
+cranelift-entity = { path = "entity", version = "0.59.0" }
+cranelift-reader = { path = "reader", version = "0.59.0" }
+cranelift-frontend = { path = "frontend", version = "0.59.0" }
+cranelift-serde = { path = "serde", version = "0.59.0", optional = true }
+cranelift-wasm = { path = "wasm", version = "0.59.0", optional = true }
+cranelift-native = { path = "native", version = "0.59.0" }
+cranelift-filetests = { path = "filetests", version = "0.59.0" }
+cranelift-module = { path = "module", version = "0.59.0" }
+cranelift-faerie = { path = "faerie", version = "0.59.0" }
+cranelift-object = { path = "object", version = "0.59.0" }
+cranelift-simplejit = { path = "simplejit", version = "0.59.0" }
+cranelift-preopt = { path = "preopt", version = "0.59.0" }
+cranelift = { path = "umbrella", version = "0.59.0" }
+filecheck = "0.4.0"
+clap = "2.32.0"
+serde = "1.0.8"
+term = "0.6.1"
+capstone = { version = "0.6.0", optional = true }
+wat = { version = "1.0.7", optional = true }
+target-lexicon = "0.10"
+pretty_env_logger = "0.3.0"
+file-per-thread-logger = "0.1.2"
+indicatif = "0.13.0"
+walkdir = "2.2"
+
+[features]
+default = ["disas", "wasm", "cranelift-codegen/all-arch"]
+disas = ["capstone"]
+wasm = ["wat", "cranelift-wasm"]

cranelift/README.md

@@ -0,0 +1,183 @@
+Cranelift Code Generator
+========================
+
+**A [Bytecode Alliance][BA] project**
+
+Cranelift is a low-level retargetable code generator. It translates a
+[target-independent intermediate
+representation](https://cranelift.readthedocs.io/en/latest/ir.html)
+into executable machine code.
+
+[BA]: https://bytecodealliance.org/
+[![Documentation Status](https://readthedocs.org/projects/cranelift/badge/?version=latest)](https://cranelift.readthedocs.io/en/latest/?badge=latest)
+[![Build Status](https://github.com/bytecodealliance/cranelift/workflows/CI/badge.svg)](https://github.com/bytecodealliance/cranelift/actions)
+[![Fuzzit Status](https://app.fuzzit.dev/badge?org_id=bytecodealliance)](https://app.fuzzit.dev/orgs/bytecodealliance/dashboard)
+[![Chat](https://img.shields.io/badge/chat-zulip-brightgreen.svg)](https://bytecodealliance.zulipchat.com/#narrow/stream/217117-cranelift/topic/general)
+![Minimum rustc 1.37](https://img.shields.io/badge/rustc-1.37+-green.svg)
+
+For more information, see [the
+documentation](https://cranelift.readthedocs.io/en/latest/?badge=latest).
+
+For an example of how to use the JIT, see the [SimpleJIT Demo], which
+implements a toy language.
+
+[SimpleJIT Demo]: https://github.com/bytecodealliance/simplejit-demo
+
+For an example of how to use Cranelift to run WebAssembly code, see
+[Wasmtime], which implements a standalone, embeddable, VM using Cranelift.
+
+[Wasmtime]: https://github.com/bytecodealliance/wasmtime
+
+Status
+------
+
+Cranelift currently supports enough functionality to run a wide variety
+of programs, including all the functionality needed to execute
+WebAssembly MVP functions, although it needs to be used within an
+external WebAssembly embedding to be part of a complete WebAssembly
+implementation.
+
+The x86-64 backend is currently the most complete and stable; other
+architectures are in various stages of development. Cranelift currently
+supports both the System V AMD64 ABI calling convention used on many
+platforms and the Windows x64 calling convention. The performance
+of code produced by Cranelift is not yet impressive, though we have plans
+to fix that.
+
+The core codegen crates have minimal dependencies, support no\_std mode
+(see below), and do not require any host floating-point support, and
+do not use callstack recursion.
+
+Cranelift does not yet perform mitigations for Spectre or related
+security issues, though it may do so in the future. It does not
+currently make any security-relevant instruction timing guarantees. It
+has seen a fair amount of testing and fuzzing, although more work is
+needed before it would be ready for a production use case.
+
+Cranelift's APIs are not yet stable.
+
+Cranelift currently requires Rust 1.37 or later to build.
+
+Contributing
+------------
+
+If you're interested in contributing to Cranelift: thank you! We have a
+[contributing guide](CONTRIBUTING.md) which will help you getting involved in
+the Cranelift project.
+
+Planned uses
+------------
+
+Cranelift is designed to be a code generator for WebAssembly, but it is
+general enough to be useful elsewhere too. The initial planned uses that
+affected its design are:
+
+ - [WebAssembly compiler for the SpiderMonkey engine in
+    Firefox](spidermonkey.md#phase-1-webassembly).
+ - [Backend for the IonMonkey JavaScript JIT compiler in
+    Firefox](spidermonkey.md#phase-2-ionmonkey).
+ - [Debug build backend for the Rust compiler](rustc.md).
+ - [Wasmtime non-Web wasm engine](https://github.com/bytecodealliance/wasmtime).
+
+Building Cranelift
+------------------
+
+Cranelift uses a [conventional Cargo build
+process](https://doc.rust-lang.org/cargo/guide/working-on-an-existing-project.html).
+
+Cranelift consists of a collection of crates, and uses a [Cargo
+Workspace](https://doc.rust-lang.org/book/ch14-03-cargo-workspaces.html),
+so for some cargo commands, such as `cargo test`, the `--all` is needed
+to tell cargo to visit all of the crates.
+
+`test-all.sh` at the top level is a script which runs all the cargo
+tests and also performs code format, lint, and documentation checks.
+
+<details>
+<summary>Building with no_std</summary>
+
+The following crates support \`no\_std\`, although they do depend on liballoc:
+ - cranelift-entity
+ - cranelift-bforest
+ - cranelift-codegen
+ - cranelift-frontend
+ - cranelift-native
+ - cranelift-wasm
+ - cranelift-module
+ - cranelift-preopt
+ - cranelift
+
+To use no\_std mode, disable the std feature and enable the core
+feature. This currently requires nightly rust.
+
+For example, to build \`cranelift-codegen\`:
+
+``` {.sourceCode .sh}
+cd cranelift-codegen
+cargo build --no-default-features --features core
+```
+
+Or, when using cranelift-codegen as a dependency (in Cargo.toml):
+
+``` {.sourceCode .}
+[dependency.cranelift-codegen]
+...
+default-features = false
+features = ["core"]
+```
+
+no\_std support is currently "best effort". We won't try to break it,
+and we'll accept patches fixing problems, however we don't expect all
+developers to build and test no\_std when submitting patches.
+Accordingly, the ./test-all.sh script does not test no\_std.
+
+There is a separate ./test-no\_std.sh script that tests the no\_std
+support in packages which support it.
+
+It's important to note that cranelift still needs liballoc to compile.
+Thus, whatever environment is used must implement an allocator.
+
+Also, to allow the use of HashMaps with no\_std, an external crate
+called hashmap\_core is pulled in (via the core feature). This is mostly
+the same as std::collections::HashMap, except that it doesn't have DOS
+protection. Just something to think about.
+
+</details>
+
+<details>
+<summary>Log configuration</summary>
+
+Cranelift uses the `log` crate to log messages at various levels. It doesn't
+specify any maximal logging level, so embedders can choose what it should be;
+however, this can have an impact of Cranelift's code size. You can use `log`
+features to reduce the maximum logging level. For instance if you want to limit
+the level of logging to `warn` messages and above in release mode:
+
+```
+[dependency.log]
+...
+features = ["release_max_level_warn"]
+```
+</details>
+
+<details>
+<summary>Building the documentation</summary>
+
+Cranelift's documentation is [published online](https://cranelift.readthedocs.io/).
+
+To build the documentation locally, you need the [Sphinx documentation
+generator](http://www.sphinx-doc.org/) as well as Python 3::
+
+    $ pip install sphinx sphinx-autobuild sphinx_rtd_theme
+    $ cd cranelift/docs
+    $ make html
+    $ open _build/html/index.html
+
+</details>
+
+Editor Support
+--------------
+
+Editor support for working with Cranelift IR (clif) files:
+
+ - Vim: https://github.com/bytecodealliance/cranelift.vim

cranelift/bforest/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+authors = ["The Cranelift Project Developers"]
+name = "cranelift-bforest"
+version = "0.59.0"
+description = "A forest of B+-trees"
+license = "Apache-2.0 WITH LLVM-exception"
+documentation = "https://cranelift.readthedocs.io/"
+repository = "https://github.com/bytecodealliance/cranelift"
+categories = ["no-std"]
+readme = "README.md"
+keywords = ["btree", "forest", "set", "map"]
+edition = "2018"
+
+[dependencies]
+cranelift-entity = { path = "../entity", version = "0.59.0", default-features = false }
+
+[badges]
+maintenance = { status = "experimental" }
+travis-ci = { repository = "bytecodealliance/cranelift" }

cranelift/bforest/LICENSE

@@ -0,0 +1,220 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+--- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+

cranelift/bforest/README.md

@@ -0,0 +1,12 @@
+This crate contains array-based data structures used by the core Cranelift code
+generator which represent a set of small ordered sets or maps.
+
+**These are not general purpose data structures that are somehow magically faster that the
+standard library's `BTreeSet` and `BTreeMap` types.**
+
+The tradeoffs are different:
+
+- Keys and values are expected to be small and copyable. We optimize for 32-bit types.
+- A comparator object is used to compare keys, allowing smaller "context free" keys.
+- Empty trees have a very small 32-bit footprint.
+- All the trees in a forest can be cleared in constant time.

cranelift/bforest/src/lib.rs

@@ -0,0 +1,199 @@
+//! A forest of B+-trees.
+//!
+//! This crate provides a data structures representing a set of small ordered sets or maps.
+//! It is implemented as a forest of B+-trees all allocating nodes out of the same pool.
+//!
+//! **These are not general purpose data structures that are somehow magically faster that the
+//! standard library's `BTreeSet` and `BTreeMap` types.**
+//!
+//! The tradeoffs are different:
+//!
+//! - Keys and values are expected to be small and copyable. We optimize for 32-bit types.
+//! - A comparator object is used to compare keys, allowing smaller "context free" keys.
+//! - Empty trees have a very small 32-bit footprint.
+//! - All the trees in a forest can be cleared in constant time.
+
+#![deny(missing_docs, trivial_numeric_casts, unused_extern_crates)]
+#![warn(unused_import_braces)]
+#![cfg_attr(feature = "clippy", plugin(clippy(conf_file = "../../clippy.toml")))]
+#![cfg_attr(feature = "cargo-clippy", allow(clippy::new_without_default))]
+#![cfg_attr(
+    feature = "cargo-clippy",
+    warn(
+        clippy::float_arithmetic,
+        clippy::mut_mut,
+        clippy::nonminimal_bool,
+        clippy::option_map_unwrap_or,
+        clippy::option_map_unwrap_or_else,
+        clippy::print_stdout,
+        clippy::unicode_not_nfc,
+        clippy::use_self
+    )
+)]
+#![no_std]
+
+#[cfg(test)]
+extern crate alloc;
+
+#[macro_use]
+extern crate cranelift_entity as entity;
+use crate::entity::packed_option;
+
+use core::borrow::BorrowMut;
+use core::cmp::Ordering;
+
+mod map;
+mod node;
+mod path;
+mod pool;
+mod set;
+
+pub use self::map::{Map, MapCursor, MapForest, MapIter};
+pub use self::set::{Set, SetCursor, SetForest, SetIter};
+
+use self::node::NodeData;
+use self::path::Path;
+use self::pool::NodePool;
+
+/// The maximum branching factor of an inner node in a B+-tree.
+/// The minimum number of outgoing edges is `INNER_SIZE/2`.
+const INNER_SIZE: usize = 8;
+
+/// Given the worst case branching factor of `INNER_SIZE/2` = 4, this is the
+/// worst case path length from the root node to a leaf node in a tree with 2^32
+/// entries. We would run out of node references before we hit `MAX_PATH`.
+const MAX_PATH: usize = 16;
+
+/// Key comparator.
+///
+/// Keys don't need to implement `Ord`. They are compared using a comparator object which
+/// provides a context for comparison.
+pub trait Comparator<K>
+where
+    K: Copy,
+{
+    /// Compare keys `a` and `b`.
+    ///
+    /// This relation must provide a total ordering or the key space.
+    fn cmp(&self, a: K, b: K) -> Ordering;
+
+    /// Binary search for `k` in an ordered slice.
+    ///
+    /// Assume that `s` is already sorted according to this ordering, search for the key `k`.
+    ///
+    /// Returns `Ok(idx)` if `k` was found in the slice or `Err(idx)` with the position where it
+    /// should be inserted to preserve the ordering.
+    fn search(&self, k: K, s: &[K]) -> Result<usize, usize> {
+        s.binary_search_by(|x| self.cmp(*x, k))
+    }
+}
+
+/// Trivial comparator that doesn't actually provide any context.
+impl<K> Comparator<K> for ()
+where
+    K: Copy + Ord,
+{
+    fn cmp(&self, a: K, b: K) -> Ordering {
+        a.cmp(&b)
+    }
+}
+
+/// Family of types shared by the map and set forest implementations.
+trait Forest {
+    /// The key type is present for both sets and maps.
+    type Key: Copy;
+
+    /// The value type is `()` for sets.
+    type Value: Copy;
+
+    /// An array of keys for the leaf nodes.
+    type LeafKeys: Copy + BorrowMut<[Self::Key]>;
+
+    /// An array of values for the leaf nodes.
+    type LeafValues: Copy + BorrowMut<[Self::Value]>;
+
+    /// Splat a single key into a whole array.
+    fn splat_key(key: Self::Key) -> Self::LeafKeys;
+
+    /// Splat a single value inst a whole array
+    fn splat_value(value: Self::Value) -> Self::LeafValues;
+}
+
+/// A reference to a B+-tree node.
+#[derive(Clone, Copy, PartialEq, Eq)]
+struct Node(u32);
+entity_impl!(Node, "node");
+
+/// Empty type to be used as the "value" in B-trees representing sets.
+#[derive(Clone, Copy)]
+struct SetValue();
+
+/// Insert `x` into `s` at position `i`, pushing out the last element.
+fn slice_insert<T: Copy>(s: &mut [T], i: usize, x: T) {
+    for j in (i + 1..s.len()).rev() {
+        s[j] = s[j - 1];
+    }
+    s[i] = x;
+}
+
+/// Shift elements in `s` to the left by `n` positions.
+fn slice_shift<T: Copy>(s: &mut [T], n: usize) {
+    for j in 0..s.len() - n {
+        s[j] = s[j + n];
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::entity::EntityRef;
+
+    /// An opaque reference to a basic block in a function.
+    #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+    pub struct Block(u32);
+    entity_impl!(Block, "block");
+
+    #[test]
+    fn comparator() {
+        let block1 = Block::new(1);
+        let block2 = Block::new(2);
+        let block3 = Block::new(3);
+        let block4 = Block::new(4);
+        let vals = [block1, block2, block4];
+        let comp = ();
+        assert_eq!(comp.search(block1, &vals), Ok(0));
+        assert_eq!(comp.search(block3, &vals), Err(2));
+        assert_eq!(comp.search(block4, &vals), Ok(2));
+    }
+
+    #[test]
+    fn slice_insertion() {
+        let mut a = ['a', 'b', 'c', 'd'];
+
+        slice_insert(&mut a[0..1], 0, 'e');
+        assert_eq!(a, ['e', 'b', 'c', 'd']);
+
+        slice_insert(&mut a, 0, 'a');
+        assert_eq!(a, ['a', 'e', 'b', 'c']);
+
+        slice_insert(&mut a, 3, 'g');
+        assert_eq!(a, ['a', 'e', 'b', 'g']);
+
+        slice_insert(&mut a, 1, 'h');
+        assert_eq!(a, ['a', 'h', 'e', 'b']);
+    }
+
+    #[test]
+    fn slice_shifting() {
+        let mut a = ['a', 'b', 'c', 'd'];
+
+        slice_shift(&mut a[0..1], 1);
+        assert_eq!(a, ['a', 'b', 'c', 'd']);
+
+        slice_shift(&mut a[1..], 1);
+        assert_eq!(a, ['a', 'c', 'd', 'd']);
+
+        slice_shift(&mut a, 2);
+        assert_eq!(a, ['d', 'd', 'd', 'd']);
+    }
+}

cranelift/bforest/src/map.rs

@@ -0,0 +1,923 @@
+//! Forest of maps.
+
+use super::{Comparator, Forest, Node, NodeData, NodePool, Path, INNER_SIZE};
+use crate::packed_option::PackedOption;
+#[cfg(test)]
+use alloc::string::String;
+#[cfg(test)]
+use core::fmt;
+use core::marker::PhantomData;
+
+/// Tag type defining forest types for a map.
+struct MapTypes<K, V>(PhantomData<(K, V)>);
+
+impl<K, V> Forest for MapTypes<K, V>
+where
+    K: Copy,
+    V: Copy,
+{
+    type Key = K;
+    type Value = V;
+    type LeafKeys = [K; INNER_SIZE - 1];
+    type LeafValues = [V; INNER_SIZE - 1];
+
+    fn splat_key(key: Self::Key) -> Self::LeafKeys {
+        [key; INNER_SIZE - 1]
+    }
+
+    fn splat_value(value: Self::Value) -> Self::LeafValues {
+        [value; INNER_SIZE - 1]
+    }
+}
+
+/// Memory pool for a forest of `Map` instances.
+pub struct MapForest<K, V>
+where
+    K: Copy,
+    V: Copy,
+{
+    nodes: NodePool<MapTypes<K, V>>,
+}
+
+impl<K, V> MapForest<K, V>
+where
+    K: Copy,
+    V: Copy,
+{
+    /// Create a new empty forest.
+    pub fn new() -> Self {
+        Self {
+            nodes: NodePool::new(),
+        }
+    }
+
+    /// Clear all maps in the forest.
+    ///
+    /// All `Map` instances belong to this forest are invalidated and should no longer be used.
+    pub fn clear(&mut self) {
+        self.nodes.clear();
+    }
+}
+
+/// B-tree mapping from `K` to `V`.
+///
+/// This is not a general-purpose replacement for `BTreeMap`. See the [module
+/// documentation](index.html) for more information about design tradeoffs.
+///
+/// Maps can be cloned, but that operation should only be used as part of cloning the whole forest
+/// they belong to. *Cloning a map does not allocate new memory for the clone*. It creates an alias
+/// of the same memory.
+#[derive(Clone)]
+pub struct Map<K, V>
+where
+    K: Copy,
+    V: Copy,
+{
+    root: PackedOption<Node>,
+    unused: PhantomData<(K, V)>,
+}
+
+impl<K, V> Map<K, V>
+where
+    K: Copy,
+    V: Copy,
+{
+    /// Make an empty map.
+    pub fn new() -> Self {
+        Self {
+            root: None.into(),
+            unused: PhantomData,
+        }
+    }
+
+    /// Is this an empty map?
+    pub fn is_empty(&self) -> bool {
+        self.root.is_none()
+    }
+
+    /// Get the value stored for `key`.
+    pub fn get<C: Comparator<K>>(&self, key: K, forest: &MapForest<K, V>, comp: &C) -> Option<V> {
+        self.root
+            .expand()
+            .and_then(|root| Path::default().find(key, root, &forest.nodes, comp))
+    }
+
+    /// Look up the value stored for `key`.
+    ///
+    /// If it exists, return the stored key-value pair.
+    ///
+    /// Otherwise, return the last key-value pair with a key that is less than or equal to `key`.
+    ///
+    /// If no stored keys are less than or equal to `key`, return `None`.
+    pub fn get_or_less<C: Comparator<K>>(
+        &self,
+        key: K,
+        forest: &MapForest<K, V>,
+        comp: &C,
+    ) -> Option<(K, V)> {
+        self.root.expand().and_then(|root| {
+            let mut path = Path::default();
+            match path.find(key, root, &forest.nodes, comp) {
+                Some(v) => Some((key, v)),
+                None => path.prev(root, &forest.nodes),
+            }
+        })
+    }
+
+    /// Insert `key, value` into the map and return the old value stored for `key`, if any.
+    pub fn insert<C: Comparator<K>>(
+        &mut self,
+        key: K,
+        value: V,
+        forest: &mut MapForest<K, V>,
+        comp: &C,
+    ) -> Option<V> {
+        self.cursor(forest, comp).insert(key, value)
+    }
+
+    /// Remove `key` from the map and return the removed value for `key`, if any.
+    pub fn remove<C: Comparator<K>>(
+        &mut self,
+        key: K,
+        forest: &mut MapForest<K, V>,
+        comp: &C,
+    ) -> Option<V> {
+        let mut c = self.cursor(forest, comp);
+        if c.goto(key).is_some() {
+            c.remove()
+        } else {
+            None
+        }
+    }
+
+    /// Remove all entries.
+    pub fn clear(&mut self, forest: &mut MapForest<K, V>) {
+        if let Some(root) = self.root.take() {
+            forest.nodes.free_tree(root);
+        }
+    }
+
+    /// Retains only the elements specified by the predicate.
+    ///
+    /// Remove all key-value pairs where the predicate returns false.
+    ///
+    /// The predicate is allowed to update the values stored in the map.
+    pub fn retain<F>(&mut self, forest: &mut MapForest<K, V>, mut predicate: F)
+    where
+        F: FnMut(K, &mut V) -> bool,
+    {
+        let mut path = Path::default();
+        if let Some(root) = self.root.expand() {
+            path.first(root, &forest.nodes);
+        }
+        while let Some((node, entry)) = path.leaf_pos() {
+            let keep = {
+                let (ks, vs) = forest.nodes[node].unwrap_leaf_mut();
+                predicate(ks[entry], &mut vs[entry])
+            };
+            if keep {
+                path.next(&forest.nodes);
+            } else {
+                self.root = path.remove(&mut forest.nodes).into();
+            }
+        }
+    }
+
+    /// Create a cursor for navigating this map. The cursor is initially positioned off the end of
+    /// the map.
+    pub fn cursor<'a, C: Comparator<K>>(
+        &'a mut self,
+        forest: &'a mut MapForest<K, V>,
+        comp: &'a C,
+    ) -> MapCursor<'a, K, V, C> {
+        MapCursor::new(self, forest, comp)
+    }
+
+    /// Create an iterator traversing this map. The iterator type is `(K, V)`.
+    pub fn iter<'a>(&'a self, forest: &'a MapForest<K, V>) -> MapIter<'a, K, V> {
+        MapIter {
+            root: self.root,
+            pool: &forest.nodes,
+            path: Path::default(),
+        }
+    }
+}
+
+impl<K, V> Default for Map<K, V>
+where
+    K: Copy,
+    V: Copy,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+impl<K, V> Map<K, V>
+where
+    K: Copy + fmt::Display,
+    V: Copy,
+{
+    /// Verify consistency.
+    fn verify<C: Comparator<K>>(&self, forest: &MapForest<K, V>, comp: &C)
+    where
+        NodeData<MapTypes<K, V>>: fmt::Display,
+    {
+        if let Some(root) = self.root.expand() {
+            forest.nodes.verify_tree(root, comp);
+        }
+    }
+
+    /// Get a text version of the path to `key`.
+    fn tpath<C: Comparator<K>>(&self, key: K, forest: &MapForest<K, V>, comp: &C) -> String {
+        use alloc::string::ToString;
+        match self.root.expand() {
+            None => "map(empty)".to_string(),
+            Some(root) => {
+                let mut path = Path::default();
+                path.find(key, root, &forest.nodes, comp);
+                path.to_string()
+            }
+        }
+    }
+}
+
+/// A position in a `Map` used to navigate and modify the ordered map.
+///
+/// A cursor always points at a key-value pair in the map, or "off the end" which is a position
+/// after the last entry in the map.
+pub struct MapCursor<'a, K, V, C>
+where
+    K: 'a + Copy,
+    V: 'a + Copy,
+    C: 'a + Comparator<K>,
+{
+    root: &'a mut PackedOption<Node>,
+    pool: &'a mut NodePool<MapTypes<K, V>>,
+    comp: &'a C,
+    path: Path<MapTypes<K, V>>,
+}
+
+impl<'a, K, V, C> MapCursor<'a, K, V, C>
+where
+    K: Copy,
+    V: Copy,
+    C: Comparator<K>,
+{
+    /// Create a cursor with a default (off-the-end) location.
+    fn new(container: &'a mut Map<K, V>, forest: &'a mut MapForest<K, V>, comp: &'a C) -> Self {
+        Self {
+            root: &mut container.root,
+            pool: &mut forest.nodes,
+            comp,
+            path: Path::default(),
+        }
+    }
+
+    /// Is this cursor pointing to an empty map?
+    pub fn is_empty(&self) -> bool {
+        self.root.is_none()
+    }
+
+    /// Move cursor to the next key-value pair and return it.
+    ///
+    /// If the cursor reaches the end, return `None` and leave the cursor at the off-the-end
+    /// position.
+    #[cfg_attr(feature = "cargo-clippy", allow(clippy::should_implement_trait))]
+    pub fn next(&mut self) -> Option<(K, V)> {
+        self.path.next(self.pool)
+    }
+
+    /// Move cursor to the previous key-value pair and return it.
+    ///
+    /// If the cursor is already pointing at the first entry, leave it there and return `None`.
+    pub fn prev(&mut self) -> Option<(K, V)> {
+        self.root
+            .expand()
+            .and_then(|root| self.path.prev(root, self.pool))
+    }
+
+    /// Get the current key, or `None` if the cursor is at the end.
+    pub fn key(&self) -> Option<K> {
+        self.path
+            .leaf_pos()
+            .and_then(|(node, entry)| self.pool[node].unwrap_leaf().0.get(entry).cloned())
+    }
+
+    /// Get the current value, or `None` if the cursor is at the end.
+    pub fn value(&self) -> Option<V> {
+        self.path
+            .leaf_pos()
+            .and_then(|(node, entry)| self.pool[node].unwrap_leaf().1.get(entry).cloned())
+    }
+
+    /// Get a mutable reference to the current value, or `None` if the cursor is at the end.
+    pub fn value_mut(&mut self) -> Option<&mut V> {
+        self.path
+            .leaf_pos()
+            .and_then(move |(node, entry)| self.pool[node].unwrap_leaf_mut().1.get_mut(entry))
+    }
+
+    /// Move this cursor to `key`.
+    ///
+    /// If `key` is in the map, place the cursor at `key` and return the corresponding value.
+    ///
+    /// If `key` is not in the set, place the cursor at the next larger element (or the end) and
+    /// return `None`.
+    pub fn goto(&mut self, elem: K) -> Option<V> {
+        self.root.expand().and_then(|root| {
+            let v = self.path.find(elem, root, self.pool, self.comp);
+            if v.is_none() {
+                self.path.normalize(self.pool);
+            }
+            v
+        })
+    }
+
+    /// Move this cursor to the first element.
+    pub fn goto_first(&mut self) -> Option<V> {
+        self.root.map(|root| self.path.first(root, self.pool).1)
+    }
+
+    /// Insert `(key, value))` into the map and leave the cursor at the inserted pair.
+    ///
+    /// If the map did not contain `key`, return `None`.
+    ///
+    /// If `key` is already present, replace the existing with `value` and return the old value.
+    pub fn insert(&mut self, key: K, value: V) -> Option<V> {
+        match self.root.expand() {
+            None => {
+                let root = self.pool.alloc_node(NodeData::leaf(key, value));
+                *self.root = root.into();
+                self.path.set_root_node(root);
+                None
+            }
+            Some(root) => {
+                // TODO: Optimize the case where `self.path` is already at the correct insert pos.
+                let old = self.path.find(key, root, self.pool, self.comp);
+                if old.is_some() {
+                    *self.path.value_mut(self.pool) = value;
+                } else {
+                    *self.root = self.path.insert(key, value, self.pool).into();
+                }
+                old
+            }
+        }
+    }
+
+    /// Remove the current entry (if any) and return the mapped value.
+    /// This advances the cursor to the next entry after the removed one.
+    pub fn remove(&mut self) -> Option<V> {
+        let value = self.value();
+        if value.is_some() {
+            *self.root = self.path.remove(self.pool).into();
+        }
+        value
+    }
+}
+
+/// An iterator visiting the key-value pairs of a `Map`.
+pub struct MapIter<'a, K, V>
+where
+    K: 'a + Copy,
+    V: 'a + Copy,
+{
+    root: PackedOption<Node>,
+    pool: &'a NodePool<MapTypes<K, V>>,
+    path: Path<MapTypes<K, V>>,
+}
+
+impl<'a, K, V> Iterator for MapIter<'a, K, V>
+where
+    K: 'a + Copy,
+    V: 'a + Copy,
+{
+    type Item = (K, V);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // We use `self.root` to indicate if we need to go to the first element. Reset to `None`
+        // once we've returned the first element. This also works for an empty tree since the
+        // `path.next()` call returns `None` when the path is empty. This also fuses the iterator.
+        match self.root.take() {
+            Some(root) => Some(self.path.first(root, self.pool)),
+            None => self.path.next(self.pool),
+        }
+    }
+}
+
+#[cfg(test)]
+impl<'a, K, V, C> MapCursor<'a, K, V, C>
+where
+    K: Copy + fmt::Display,
+    V: Copy + fmt::Display,
+    C: Comparator<K>,
+{
+    fn verify(&self) {
+        self.path.verify(self.pool);
+        self.root.map(|root| self.pool.verify_tree(root, self.comp));
+    }
+
+    /// Get a text version of the path to the current position.
+    fn tpath(&self) -> String {
+        use alloc::string::ToString;
+        self.path.to_string()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::NodeData;
+    use super::*;
+    use alloc::vec::Vec;
+    use core::mem;
+
+    #[test]
+    fn node_size() {
+        // check that nodes are cache line sized when keys and values are 32 bits.
+        type F = MapTypes<u32, u32>;
+        assert_eq!(mem::size_of::<NodeData<F>>(), 64);
+    }
+
+    #[test]
+    fn empty() {
+        let mut f = MapForest::<u32, f32>::new();
+        f.clear();
+
+        let mut m = Map::<u32, f32>::new();
+        assert!(m.is_empty());
+        m.clear(&mut f);
+
+        assert_eq!(m.get(7, &f, &()), None);
+        assert_eq!(m.iter(&f).next(), None);
+        assert_eq!(m.get_or_less(7, &f, &()), None);
+        m.retain(&mut f, |_, _| unreachable!());
+
+        let mut c = m.cursor(&mut f, &());
+        assert!(c.is_empty());
+        assert_eq!(c.key(), None);
+        assert_eq!(c.value(), None);
+        assert_eq!(c.next(), None);
+        assert_eq!(c.prev(), None);
+        c.verify();
+        assert_eq!(c.tpath(), "<empty path>");
+        assert_eq!(c.goto_first(), None);
+        assert_eq!(c.tpath(), "<empty path>");
+    }
+
+    #[test]
+    fn inserting() {
+        let f = &mut MapForest::<u32, f32>::new();
+        let mut m = Map::<u32, f32>::new();
+
+        // The first seven values stay in a single leaf node.
+        assert_eq!(m.insert(50, 5.0, f, &()), None);
+        assert_eq!(m.insert(50, 5.5, f, &()), Some(5.0));
+        assert_eq!(m.insert(20, 2.0, f, &()), None);
+        assert_eq!(m.insert(80, 8.0, f, &()), None);
+        assert_eq!(m.insert(40, 4.0, f, &()), None);
+        assert_eq!(m.insert(60, 6.0, f, &()), None);
+        assert_eq!(m.insert(90, 9.0, f, &()), None);
+        assert_eq!(m.insert(200, 20.0, f, &()), None);
+
+        m.verify(f, &());
+
+        assert_eq!(
+            m.iter(f).collect::<Vec<_>>(),
+            [
+                (20, 2.0),
+                (40, 4.0),
+                (50, 5.5),
+                (60, 6.0),
+                (80, 8.0),
+                (90, 9.0),
+                (200, 20.0),
+            ]
+        );
+
+        assert_eq!(m.get(0, f, &()), None);
+        assert_eq!(m.get(20, f, &()), Some(2.0));
+        assert_eq!(m.get(30, f, &()), None);
+        assert_eq!(m.get(40, f, &()), Some(4.0));
+        assert_eq!(m.get(50, f, &()), Some(5.5));
+        assert_eq!(m.get(60, f, &()), Some(6.0));
+        assert_eq!(m.get(70, f, &()), None);
+        assert_eq!(m.get(80, f, &()), Some(8.0));
+        assert_eq!(m.get(100, f, &()), None);
+
+        assert_eq!(m.get_or_less(0, f, &()), None);
+        assert_eq!(m.get_or_less(20, f, &()), Some((20, 2.0)));
+        assert_eq!(m.get_or_less(30, f, &()), Some((20, 2.0)));
+        assert_eq!(m.get_or_less(40, f, &()), Some((40, 4.0)));
+        assert_eq!(m.get_or_less(200, f, &()), Some((200, 20.0)));
+        assert_eq!(m.get_or_less(201, f, &()), Some((200, 20.0)));
+
+        {
+            let mut c = m.cursor(f, &());
+            assert_eq!(c.prev(), Some((200, 20.0)));
+            assert_eq!(c.prev(), Some((90, 9.0)));
+            assert_eq!(c.prev(), Some((80, 8.0)));
+            assert_eq!(c.prev(), Some((60, 6.0)));
+            assert_eq!(c.prev(), Some((50, 5.5)));
+            assert_eq!(c.prev(), Some((40, 4.0)));
+            assert_eq!(c.prev(), Some((20, 2.0)));
+            assert_eq!(c.prev(), None);
+        }
+
+        // Test some removals where the node stays healthy.
+        assert_eq!(m.tpath(50, f, &()), "node0[2]");
+        assert_eq!(m.tpath(80, f, &()), "node0[4]");
+        assert_eq!(m.tpath(200, f, &()), "node0[6]");
+
+        assert_eq!(m.remove(80, f, &()), Some(8.0));
+        assert_eq!(m.tpath(50, f, &()), "node0[2]");
+        assert_eq!(m.tpath(80, f, &()), "node0[4]");
+        assert_eq!(m.tpath(200, f, &()), "node0[5]");
+        assert_eq!(m.remove(80, f, &()), None);
+        m.verify(f, &());
+
+        assert_eq!(m.remove(20, f, &()), Some(2.0));
+        assert_eq!(m.tpath(50, f, &()), "node0[1]");
+        assert_eq!(m.tpath(80, f, &()), "node0[3]");
+        assert_eq!(m.tpath(200, f, &()), "node0[4]");
+        assert_eq!(m.remove(20, f, &()), None);
+        m.verify(f, &());
+
+        // [ 40 50 60 90 200 ]
+
+        {
+            let mut c = m.cursor(f, &());
+            assert_eq!(c.goto_first(), Some(4.0));
+            assert_eq!(c.key(), Some(40));
+            assert_eq!(c.value(), Some(4.0));
+            assert_eq!(c.next(), Some((50, 5.5)));
+            assert_eq!(c.next(), Some((60, 6.0)));
+            assert_eq!(c.next(), Some((90, 9.0)));
+            assert_eq!(c.next(), Some((200, 20.0)));
+            c.verify();
+            assert_eq!(c.next(), None);
+            c.verify();
+        }
+
+        // Removals from the root leaf node beyond underflow.
+        assert_eq!(m.remove(200, f, &()), Some(20.0));
+        assert_eq!(m.remove(40, f, &()), Some(4.0));
+        assert_eq!(m.remove(60, f, &()), Some(6.0));
+        m.verify(f, &());
+        assert_eq!(m.remove(50, f, &()), Some(5.5));
+        m.verify(f, &());
+        assert_eq!(m.remove(90, f, &()), Some(9.0));
+        m.verify(f, &());
+        assert!(m.is_empty());
+    }
+
+    #[test]
+    fn split_level0_leaf() {
+        // Various ways of splitting a full leaf node at level 0.
+        let f = &mut MapForest::<u32, f32>::new();
+
+        fn full_leaf(f: &mut MapForest<u32, f32>) -> Map<u32, f32> {
+            let mut m = Map::new();
+            for n in 1..8 {
+                m.insert(n * 10, n as f32 * 1.1, f, &());
+            }
+            m
+        }
+
+        // Insert at front of leaf.
+        let mut m = full_leaf(f);
+        m.insert(5, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(5, f, &()), Some(4.2));
+
+        // Retain even entries, with altered values.
+        m.retain(f, |k, v| {
+            *v = (k / 10) as f32;
+            (k % 20) == 0
+        });
+        assert_eq!(
+            m.iter(f).collect::<Vec<_>>(),
+            [(20, 2.0), (40, 4.0), (60, 6.0)]
+        );
+
+        // Insert at back of leaf.
+        let mut m = full_leaf(f);
+        m.insert(80, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(80, f, &()), Some(4.2));
+
+        // Insert before middle (40).
+        let mut m = full_leaf(f);
+        m.insert(35, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(35, f, &()), Some(4.2));
+
+        // Insert after middle (40).
+        let mut m = full_leaf(f);
+        m.insert(45, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(45, f, &()), Some(4.2));
+
+        m.clear(f);
+        assert!(m.is_empty());
+    }
+
+    #[test]
+    fn split_level1_leaf() {
+        // Various ways of splitting a full leaf node at level 1.
+        let f = &mut MapForest::<u32, f32>::new();
+
+        // Return a map whose root node is a full inner node, and the leaf nodes are all full
+        // containing:
+        //
+        // 110, 120, ..., 170
+        // 210, 220, ..., 270
+        // ...
+        // 810, 820, ..., 870
+        fn full(f: &mut MapForest<u32, f32>) -> Map<u32, f32> {
+            let mut m = Map::new();
+
+            // Start by inserting elements in order.
+            // This should leave 8 leaf nodes with 4 elements in each.
+            for row in 1..9 {
+                for col in 1..5 {
+                    m.insert(row * 100 + col * 10, row as f32 + col as f32 * 0.1, f, &());
+                }
+            }
+
+            // Then top up the leaf nodes without splitting them.
+            for row in 1..9 {
+                for col in 5..8 {
+                    m.insert(row * 100 + col * 10, row as f32 + col as f32 * 0.1, f, &());
+                }
+            }
+
+            m
+        }
+
+        let mut m = full(f);
+        // Verify geometry. Get get node2 as the root and leaves node0, 1, 3, ...
+        m.verify(f, &());
+        assert_eq!(m.tpath(110, f, &()), "node2[0]--node0[0]");
+        assert_eq!(m.tpath(140, f, &()), "node2[0]--node0[3]");
+        assert_eq!(m.tpath(210, f, &()), "node2[1]--node1[0]");
+        assert_eq!(m.tpath(270, f, &()), "node2[1]--node1[6]");
+        assert_eq!(m.tpath(310, f, &()), "node2[2]--node3[0]");
+        assert_eq!(m.tpath(810, f, &()), "node2[7]--node8[0]");
+        assert_eq!(m.tpath(870, f, &()), "node2[7]--node8[6]");
+
+        {
+            let mut c = m.cursor(f, &());
+            assert_eq!(c.goto_first(), Some(1.1));
+            assert_eq!(c.key(), Some(110));
+        }
+
+        // Front of first leaf.
+        m.insert(0, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(0, f, &()), Some(4.2));
+
+        // First leaf split 4-4 after appending to LHS.
+        f.clear();
+        m = full(f);
+        m.insert(135, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(135, f, &()), Some(4.2));
+
+        // First leaf split 4-4 after prepending to RHS.
+        f.clear();
+        m = full(f);
+        m.insert(145, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(145, f, &()), Some(4.2));
+
+        // First leaf split 4-4 after appending to RHS.
+        f.clear();
+        m = full(f);
+        m.insert(175, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(175, f, &()), Some(4.2));
+
+        // Left-middle leaf split, ins LHS.
+        f.clear();
+        m = full(f);
+        m.insert(435, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(435, f, &()), Some(4.2));
+
+        // Left-middle leaf split, ins RHS.
+        f.clear();
+        m = full(f);
+        m.insert(445, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(445, f, &()), Some(4.2));
+
+        // Right-middle leaf split, ins LHS.
+        f.clear();
+        m = full(f);
+        m.insert(535, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(535, f, &()), Some(4.2));
+
+        // Right-middle leaf split, ins RHS.
+        f.clear();
+        m = full(f);
+        m.insert(545, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(545, f, &()), Some(4.2));
+
+        // Last leaf split, ins LHS.
+        f.clear();
+        m = full(f);
+        m.insert(835, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(835, f, &()), Some(4.2));
+
+        // Last leaf split, ins RHS.
+        f.clear();
+        m = full(f);
+        m.insert(845, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(845, f, &()), Some(4.2));
+
+        // Front of last leaf.
+        f.clear();
+        m = full(f);
+        m.insert(805, 4.2, f, &());
+        m.verify(f, &());
+        assert_eq!(m.get(805, f, &()), Some(4.2));
+
+        m.clear(f);
+        m.verify(f, &());
+    }
+
+    // Make a tree with two barely healthy leaf nodes:
+    // [ 10 20 30 40 ] [ 50 60 70 80 ]
+    fn two_leaf(f: &mut MapForest<u32, f32>) -> Map<u32, f32> {
+        f.clear();
+        let mut m = Map::new();
+        for n in 1..9 {
+            m.insert(n * 10, n as f32, f, &());
+        }
+        m
+    }
+
+    #[test]
+    fn remove_level1() {
+        let f = &mut MapForest::<u32, f32>::new();
+        let mut m = two_leaf(f);
+
+        // Verify geometry.
+        m.verify(f, &());
+        assert_eq!(m.tpath(10, f, &()), "node2[0]--node0[0]");
+        assert_eq!(m.tpath(40, f, &()), "node2[0]--node0[3]");
+        assert_eq!(m.tpath(49, f, &()), "node2[0]--node0[4]");
+        assert_eq!(m.tpath(50, f, &()), "node2[1]--node1[0]");
+        assert_eq!(m.tpath(80, f, &()), "node2[1]--node1[3]");
+
+        // Remove the front entry from a node that stays healthy.
+        assert_eq!(m.insert(55, 5.5, f, &()), None);
+        assert_eq!(m.remove(50, f, &()), Some(5.0));
+        m.verify(f, &());
+        assert_eq!(m.tpath(49, f, &()), "node2[0]--node0[4]");
+        assert_eq!(m.tpath(50, f, &()), "node2[0]--node0[4]");
+        assert_eq!(m.tpath(55, f, &()), "node2[1]--node1[0]");
+
+        // Remove the front entry from the first leaf node: No critical key to update.
+        assert_eq!(m.insert(15, 1.5, f, &()), None);
+        assert_eq!(m.remove(10, f, &()), Some(1.0));
+        m.verify(f, &());
+
+        // [ 15 20 30 40 ] [ 55 60 70 80 ]
+
+        // Remove the front entry from a right-most node that underflows.
+        // No rebalancing for the right-most node. Still need critical key update.
+        assert_eq!(m.remove(55, f, &()), Some(5.5));
+        m.verify(f, &());
+        assert_eq!(m.tpath(55, f, &()), "node2[0]--node0[4]");
+        assert_eq!(m.tpath(60, f, &()), "node2[1]--node1[0]");
+
+        // [ 15 20 30 40 ] [ 60 70 80 ]
+
+        // Replenish the right leaf.
+        assert_eq!(m.insert(90, 9.0, f, &()), None);
+        assert_eq!(m.insert(100, 10.0, f, &()), None);
+        m.verify(f, &());
+        assert_eq!(m.tpath(55, f, &()), "node2[0]--node0[4]");
+        assert_eq!(m.tpath(60, f, &()), "node2[1]--node1[0]");
+
+        // [ 15 20 30 40 ] [ 60 70 80 90 100 ]
+
+        // Removing one entry from the left leaf should trigger a rebalancing from the right
+        // sibling.
+        assert_eq!(m.remove(20, f, &()), Some(2.0));
+        m.verify(f, &());
+
+        // [ 15 30 40 60 ] [ 70 80 90 100 ]
+        // Check that the critical key was updated correctly.
+        assert_eq!(m.tpath(50, f, &()), "node2[0]--node0[3]");
+        assert_eq!(m.tpath(60, f, &()), "node2[0]--node0[3]");
+        assert_eq!(m.tpath(70, f, &()), "node2[1]--node1[0]");
+
+        // Remove front entry from the left-most leaf node, underflowing.
+        // This should cause two leaf nodes to be merged and the root node to go away.
+        assert_eq!(m.remove(15, f, &()), Some(1.5));
+        m.verify(f, &());
+    }
+
+    #[test]
+    fn remove_level1_rightmost() {
+        let f = &mut MapForest::<u32, f32>::new();
+        let mut m = two_leaf(f);
+
+        // [ 10 20 30 40 ] [ 50 60 70 80 ]
+
+        // Remove entries from the right leaf. This doesn't trigger a rebalancing.
+        assert_eq!(m.remove(60, f, &()), Some(6.0));
+        assert_eq!(m.remove(80, f, &()), Some(8.0));
+        assert_eq!(m.remove(50, f, &()), Some(5.0));
+        m.verify(f, &());
+
+        // [ 10 20 30 40 ] [ 70 ]
+        assert_eq!(m.tpath(50, f, &()), "node2[0]--node0[4]");
+        assert_eq!(m.tpath(70, f, &()), "node2[1]--node1[0]");
+
+        // Removing the last entry from the right leaf should cause a collapse.
+        assert_eq!(m.remove(70, f, &()), Some(7.0));
+        m.verify(f, &());
+    }
+
+    // Make a 3-level tree with barely healthy nodes.
+    // 1 root, 8 inner nodes, 7*4+5=33 leaf nodes, 4 entries each.
+    fn level3_sparse(f: &mut MapForest<u32, f32>) -> Map<u32, f32> {
+        f.clear();
+        let mut m = Map::new();
+        for n in 1..133 {
+            m.insert(n * 10, n as f32, f, &());
+        }
+        m
+    }
+
+    #[test]
+    fn level3_removes() {
+        let f = &mut MapForest::<u32, f32>::new();
+        let mut m = level3_sparse(f);
+        m.verify(f, &());
+
+        // Check geometry.
+        // Root: node11
+        // [ node2 170 node10 330 node16 490 node21 650 node26 810 node31 970 node36 1130 node41 ]
+        // L1: node11
+        assert_eq!(m.tpath(0, f, &()), "node11[0]--node2[0]--node0[0]");
+        assert_eq!(m.tpath(10000, f, &()), "node11[7]--node41[4]--node40[4]");
+
+        // 650 is a critical key in the middle of the root.
+        assert_eq!(m.tpath(640, f, &()), "node11[3]--node21[3]--node19[3]");
+        assert_eq!(m.tpath(650, f, &()), "node11[4]--node26[0]--node20[0]");
+
+        // Deleting 640 triggers a rebalance from node19 to node 20, cascading to n21 -> n26.
+        assert_eq!(m.remove(640, f, &()), Some(64.0));
+        m.verify(f, &());
+        assert_eq!(m.tpath(650, f, &()), "node11[3]--node26[3]--node20[3]");
+
+        // 1130 is in the first leaf of the last L1 node. Deleting it triggers a rebalance node35
+        // -> node37, but no rebalance above where there is no right sibling.
+        assert_eq!(m.tpath(1130, f, &()), "node11[6]--node41[0]--node35[0]");
+        assert_eq!(m.tpath(1140, f, &()), "node11[6]--node41[0]--node35[1]");
+        assert_eq!(m.remove(1130, f, &()), Some(113.0));
+        m.verify(f, &());
+        assert_eq!(m.tpath(1140, f, &()), "node11[6]--node41[0]--node37[0]");
+    }
+
+    #[test]
+    fn insert_many() {
+        let f = &mut MapForest::<u32, f32>::new();
+        let mut m = Map::<u32, f32>::new();
+
+        let mm = 4096;
+        let mut x = 0;
+
+        for n in 0..mm {
+            assert_eq!(m.insert(x, n as f32, f, &()), None);
+            m.verify(f, &());
+
+            x = (x + n + 1) % mm;
+        }
+
+        x = 0;
+        for n in 0..mm {
+            assert_eq!(m.get(x, f, &()), Some(n as f32));
+            x = (x + n + 1) % mm;
+        }
+
+        x = 0;
+        for n in 0..mm {
+            assert_eq!(m.remove(x, f, &()), Some(n as f32));
+            m.verify(f, &());
+
+            x = (x + n + 1) % mm;
+        }
+
+        assert!(m.is_empty());
+    }
+}

cranelift/bforest/src/node.rs

@@ -0,0 +1,806 @@
+//! B+-tree nodes.
+
+use super::{slice_insert, slice_shift, Forest, Node, SetValue, INNER_SIZE};
+use core::borrow::{Borrow, BorrowMut};
+use core::fmt;
+
+/// B+-tree node.
+///
+/// A B+-tree has different node types for inner nodes and leaf nodes. Inner nodes contain M node
+/// references and M-1 keys while leaf nodes contain N keys and values. Values for M and N are
+/// chosen such that a node is exactly 64 bytes (a cache line) when keys and values are 32 bits
+/// each.
+///
+/// An inner node contains at least M/2 node references unless it is the right-most node at its
+/// level. A leaf node contains at least N/2 keys unless it is the right-most leaf.
+#[allow(dead_code)] // workaround for https://github.com/rust-lang/rust/issues/64362
+pub(super) enum NodeData<F: Forest> {
+    Inner {
+        /// The number of keys in this node.
+        /// The number of node references is always one more.
+        size: u8,
+
+        /// Keys discriminating sub-trees.
+        ///
+        /// The key in `keys[i]` is greater than all keys in `tree[i]` and less than or equal to
+        /// all keys in `tree[i+1]`.
+        keys: [F::Key; INNER_SIZE - 1],
+
+        /// Sub-trees.
+        tree: [Node; INNER_SIZE],
+    },
+    Leaf {
+        /// Number of key-value pairs in this node.
+        size: u8,
+
+        // Key array.
+        keys: F::LeafKeys,
+
+        // Value array.
+        vals: F::LeafValues,
+    },
+    /// An unused node on the free list.
+    Free { next: Option<Node> },
+}
+
+// Implement `Clone` and `Copy` manually, because deriving them would also require `Forest` to
+// implement `Clone`.
+impl<F: Forest> Copy for NodeData<F> {}
+impl<F: Forest> Clone for NodeData<F> {
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<F: Forest> NodeData<F> {
+    /// Is this a free/unused node?
+    pub fn is_free(&self) -> bool {
+        match *self {
+            Self::Free { .. } => true,
+            _ => false,
+        }
+    }
+
+    /// Get the number of entries in this node.
+    ///
+    /// This is the number of outgoing edges in an inner node, or the number of key-value pairs in
+    /// a leaf node.
+    pub fn entries(&self) -> usize {
+        match *self {
+            Self::Inner { size, .. } => usize::from(size) + 1,
+            Self::Leaf { size, .. } => usize::from(size),
+            Self::Free { .. } => panic!("freed node"),
+        }
+    }
+
+    /// Create an inner node with a single key and two sub-trees.
+    pub fn inner(left: Node, key: F::Key, right: Node) -> Self {
+        // Splat the key and right node to the whole array.
+        // Saves us from inventing a default/reserved value.
+        let mut tree = [right; INNER_SIZE];
+        tree[0] = left;
+        Self::Inner {
+            size: 1,
+            keys: [key; INNER_SIZE - 1],
+            tree,
+        }
+    }
+
+    /// Create a leaf node with a single key-value pair.
+    pub fn leaf(key: F::Key, value: F::Value) -> Self {
+        Self::Leaf {
+            size: 1,
+            keys: F::splat_key(key),
+            vals: F::splat_value(value),
+        }
+    }
+
+    /// Unwrap an inner node into two slices (keys, trees).
+    pub fn unwrap_inner(&self) -> (&[F::Key], &[Node]) {
+        match *self {
+            Self::Inner {
+                size,
+                ref keys,
+                ref tree,
+            } => {
+                let size = usize::from(size);
+                // TODO: We could probably use `get_unchecked()` here since `size` is always in
+                // range.
+                (&keys[0..size], &tree[0..=size])
+            }
+            _ => panic!("Expected inner node"),
+        }
+    }
+
+    /// Unwrap a leaf node into two slices (keys, values) of the same length.
+    pub fn unwrap_leaf(&self) -> (&[F::Key], &[F::Value]) {
+        match *self {
+            Self::Leaf {
+                size,
+                ref keys,
+                ref vals,
+            } => {
+                let size = usize::from(size);
+                let keys = keys.borrow();
+                let vals = vals.borrow();
+                // TODO: We could probably use `get_unchecked()` here since `size` is always in
+                // range.
+                (&keys[0..size], &vals[0..size])
+            }
+            _ => panic!("Expected leaf node"),
+        }
+    }
+
+    /// Unwrap a mutable leaf node into two slices (keys, values) of the same length.
+    pub fn unwrap_leaf_mut(&mut self) -> (&mut [F::Key], &mut [F::Value]) {
+        match *self {
+            Self::Leaf {
+                size,
+                ref mut keys,
+                ref mut vals,
+            } => {
+                let size = usize::from(size);
+                let keys = keys.borrow_mut();
+                let vals = vals.borrow_mut();
+                // TODO: We could probably use `get_unchecked_mut()` here since `size` is always in
+                // range.
+                (&mut keys[0..size], &mut vals[0..size])
+            }
+            _ => panic!("Expected leaf node"),
+        }
+    }
+
+    /// Get the critical key for a leaf node.
+    /// This is simply the first key.
+    pub fn leaf_crit_key(&self) -> F::Key {
+        match *self {
+            Self::Leaf { size, ref keys, .. } => {
+                debug_assert!(size > 0, "Empty leaf node");
+                keys.borrow()[0]
+            }
+            _ => panic!("Expected leaf node"),
+        }
+    }
+
+    /// Try to insert `(key, node)` at key-position `index` in an inner node.
+    /// This means that `key` is inserted at `keys[i]` and `node` is inserted at `tree[i + 1]`.
+    /// If the node is full, this leaves the node unchanged and returns false.
+    pub fn try_inner_insert(&mut self, index: usize, key: F::Key, node: Node) -> bool {
+        match *self {
+            Self::Inner {
+                ref mut size,
+                ref mut keys,
+                ref mut tree,
+            } => {
+                let sz = usize::from(*size);
+                debug_assert!(sz <= keys.len());
+                debug_assert!(index <= sz, "Can't insert at {} with {} keys", index, sz);
+
+                if let Some(ks) = keys.get_mut(0..=sz) {
+                    *size = (sz + 1) as u8;
+                    slice_insert(ks, index, key);
+                    slice_insert(&mut tree[1..=sz + 1], index, node);
+                    true
+                } else {
+                    false
+                }
+            }
+            _ => panic!("Expected inner node"),
+        }
+    }
+
+    /// Try to insert `key, value` at `index` in a leaf node, but fail and return false if the node
+    /// is full.
+    pub fn try_leaf_insert(&mut self, index: usize, key: F::Key, value: F::Value) -> bool {
+        match *self {
+            Self::Leaf {
+                ref mut size,
+                ref mut keys,
+                ref mut vals,
+            } => {
+                let sz = usize::from(*size);
+                let keys = keys.borrow_mut();
+                let vals = vals.borrow_mut();
+                debug_assert!(sz <= keys.len());
+                debug_assert!(index <= sz);
+
+                if let Some(ks) = keys.get_mut(0..=sz) {
+                    *size = (sz + 1) as u8;
+                    slice_insert(ks, index, key);
+                    slice_insert(&mut vals[0..=sz], index, value);
+                    true
+                } else {
+                    false
+                }
+            }
+            _ => panic!("Expected leaf node"),
+        }
+    }
+
+    /// Split off the second half of this node.
+    /// It is assumed that this a completely full inner or leaf node.
+    ///
+    /// The `insert_index` parameter is the position where an insertion was tried and failed. The
+    /// node will be split in half with a bias towards an even split after the insertion is retried.
+    pub fn split(&mut self, insert_index: usize) -> SplitOff<F> {
+        match *self {
+            Self::Inner {
+                ref mut size,
+                ref keys,
+                ref tree,
+            } => {
+                debug_assert_eq!(usize::from(*size), keys.len(), "Node not full");
+
+                // Number of tree entries in the lhs node.
+                let l_ents = split_pos(tree.len(), insert_index + 1);
+                let r_ents = tree.len() - l_ents;
+
+                // With INNER_SIZE=8, we get l_ents=4 and:
+                //
+                // self: [ n0 k0 n1 k1 n2 k2 n3 k3 n4 k4 n5 k5 n6 k6 n7 ]
+                // lhs:  [ n0 k0 n1 k1 n2 k2 n3 ]
+                // crit_key = k3 (not present in either node)
+                // rhs:  [ n4 k4 n5 k5 n6 k6 n7 ]
+
+                // 1. Truncate the LHS.
+                *size = (l_ents - 1) as u8;
+
+                // 2. Copy second half to `rhs_data`.
+                let mut r_keys = *keys;
+                r_keys[0..r_ents - 1].copy_from_slice(&keys[l_ents..]);
+
+                let mut r_tree = *tree;
+                r_tree[0..r_ents].copy_from_slice(&tree[l_ents..]);
+
+                SplitOff {
+                    lhs_entries: l_ents,
+                    rhs_entries: r_ents,
+                    crit_key: keys[l_ents - 1],
+                    rhs_data: Self::Inner {
+                        size: (r_ents - 1) as u8,
+                        keys: r_keys,
+                        tree: r_tree,
+                    },
+                }
+            }
+            Self::Leaf {
+                ref mut size,
+                ref keys,
+                ref vals,
+            } => {
+                let o_keys = keys.borrow();
+                let o_vals = vals.borrow();
+                debug_assert_eq!(usize::from(*size), o_keys.len(), "Node not full");
+
+                let l_size = split_pos(o_keys.len(), insert_index);
+                let r_size = o_keys.len() - l_size;
+
+                // 1. Truncate the LHS node at `l_size`.
+                *size = l_size as u8;
+
+                // 2. Copy second half to `rhs_data`.
+                let mut r_keys = *keys;
+                r_keys.borrow_mut()[0..r_size].copy_from_slice(&o_keys[l_size..]);
+
+                let mut r_vals = *vals;
+                r_vals.borrow_mut()[0..r_size].copy_from_slice(&o_vals[l_size..]);
+
+                SplitOff {
+                    lhs_entries: l_size,
+                    rhs_entries: r_size,
+                    crit_key: o_keys[l_size],
+                    rhs_data: Self::Leaf {
+                        size: r_size as u8,
+                        keys: r_keys,
+                        vals: r_vals,
+                    },
+                }
+            }
+            _ => panic!("Expected leaf node"),
+        }
+    }
+
+    /// Remove the sub-tree at `index` from this inner node.
+    ///
+    /// Note that `index` refers to a sub-tree entry and not a key entry as it does for
+    /// `try_inner_insert()`. It is possible to remove the first sub-tree (which can't be inserted
+    /// by `try_inner_insert()`).
+    ///
+    /// Return an indication of the node's health (i.e. below half capacity).
+    pub fn inner_remove(&mut self, index: usize) -> Removed {
+        match *self {
+            Self::Inner {
+                ref mut size,
+                ref mut keys,
+                ref mut tree,
+            } => {
+                let ents = usize::from(*size) + 1;
+                debug_assert!(ents <= tree.len());
+                debug_assert!(index < ents);
+                // Leave an invalid 0xff size when node becomes empty.
+                *size = ents.wrapping_sub(2) as u8;
+                if ents > 1 {
+                    slice_shift(&mut keys[index.saturating_sub(1)..ents - 1], 1);
+                }
+                slice_shift(&mut tree[index..ents], 1);
+                Removed::new(index, ents - 1, tree.len())
+            }
+            _ => panic!("Expected inner node"),
+        }
+    }
+
+    /// Remove the key-value pair at `index` from this leaf node.
+    ///
+    /// Return an indication of the node's health (i.e. below half capacity).
+    pub fn leaf_remove(&mut self, index: usize) -> Removed {
+        match *self {
+            Self::Leaf {
+                ref mut size,
+                ref mut keys,
+                ref mut vals,
+            } => {
+                let sz = usize::from(*size);
+                let keys = keys.borrow_mut();
+                let vals = vals.borrow_mut();
+                *size -= 1;
+                slice_shift(&mut keys[index..sz], 1);
+                slice_shift(&mut vals[index..sz], 1);
+                Removed::new(index, sz - 1, keys.len())
+            }
+            _ => panic!("Expected leaf node"),
+        }
+    }
+
+    /// Balance this node with its right sibling.
+    ///
+    /// It is assumed that the current node has underflowed. Look at the right sibling node and do
+    /// one of two things:
+    ///
+    /// 1. Move all entries to the right node, leaving this node empty, or
+    /// 2. Distribute entries evenly between the two nodes.
+    ///
+    /// In the first case, `None` is returned. In the second case, the new critical key for the
+    /// right sibling node is returned.
+    pub fn balance(&mut self, crit_key: F::Key, rhs: &mut Self) -> Option<F::Key> {
+        match (self, rhs) {
+            (
+                &mut Self::Inner {
+                    size: ref mut l_size,
+                    keys: ref mut l_keys,
+                    tree: ref mut l_tree,
+                },
+                &mut Self::Inner {
+                    size: ref mut r_size,
+                    keys: ref mut r_keys,
+                    tree: ref mut r_tree,
+                },
+            ) => {
+                let l_ents = usize::from(*l_size) + 1;
+                let r_ents = usize::from(*r_size) + 1;
+                let ents = l_ents + r_ents;
+
+                if ents <= r_tree.len() {
+                    // All entries will fit in the RHS node.
+                    // We'll leave the LHS node empty, but first use it as a scratch space.
+                    *l_size = 0;
+                    // Insert `crit_key` between the two nodes.
+                    l_keys[l_ents - 1] = crit_key;
+                    l_keys[l_ents..ents - 1].copy_from_slice(&r_keys[0..r_ents - 1]);
+                    r_keys[0..ents - 1].copy_from_slice(&l_keys[0..ents - 1]);
+                    l_tree[l_ents..ents].copy_from_slice(&r_tree[0..r_ents]);
+                    r_tree[0..ents].copy_from_slice(&l_tree[0..ents]);
+                    *r_size = (ents - 1) as u8;
+                    None
+                } else {
+                    // The entries don't all fit in one node. Distribute some from RHS -> LHS.
+                    // Split evenly with a bias to putting one entry in LHS.
+                    let r_goal = ents / 2;
+                    let l_goal = ents - r_goal;
+                    debug_assert!(l_goal > l_ents, "Node must be underflowed");
+
+                    l_keys[l_ents - 1] = crit_key;
+                    l_keys[l_ents..l_goal - 1].copy_from_slice(&r_keys[0..l_goal - 1 - l_ents]);
+                    l_tree[l_ents..l_goal].copy_from_slice(&r_tree[0..l_goal - l_ents]);
+                    *l_size = (l_goal - 1) as u8;
+
+                    let new_crit = r_keys[r_ents - r_goal - 1];
+                    slice_shift(&mut r_keys[0..r_ents - 1], r_ents - r_goal);
+                    slice_shift(&mut r_tree[0..r_ents], r_ents - r_goal);
+                    *r_size = (r_goal - 1) as u8;
+
+                    Some(new_crit)
+                }
+            }
+            (
+                &mut Self::Leaf {
+                    size: ref mut l_size,
+                    keys: ref mut l_keys,
+                    vals: ref mut l_vals,
+                },
+                &mut Self::Leaf {
+                    size: ref mut r_size,
+                    keys: ref mut r_keys,
+                    vals: ref mut r_vals,
+                },
+            ) => {
+                let l_ents = usize::from(*l_size);
+                let l_keys = l_keys.borrow_mut();
+                let l_vals = l_vals.borrow_mut();
+                let r_ents = usize::from(*r_size);
+                let r_keys = r_keys.borrow_mut();
+                let r_vals = r_vals.borrow_mut();
+                let ents = l_ents + r_ents;
+
+                if ents <= r_vals.len() {
+                    // We can fit all entries in the RHS node.
+                    // We'll leave the LHS node empty, but first use it as a scratch space.
+                    *l_size = 0;
+                    l_keys[l_ents..ents].copy_from_slice(&r_keys[0..r_ents]);
+                    r_keys[0..ents].copy_from_slice(&l_keys[0..ents]);
+                    l_vals[l_ents..ents].copy_from_slice(&r_vals[0..r_ents]);
+                    r_vals[0..ents].copy_from_slice(&l_vals[0..ents]);
+                    *r_size = ents as u8;
+                    None
+                } else {
+                    // The entries don't all fit in one node. Distribute some from RHS -> LHS.
+                    // Split evenly with a bias to putting one entry in LHS.
+                    let r_goal = ents / 2;
+                    let l_goal = ents - r_goal;
+                    debug_assert!(l_goal > l_ents, "Node must be underflowed");
+
+                    l_keys[l_ents..l_goal].copy_from_slice(&r_keys[0..l_goal - l_ents]);
+                    l_vals[l_ents..l_goal].copy_from_slice(&r_vals[0..l_goal - l_ents]);
+                    *l_size = l_goal as u8;
+
+                    slice_shift(&mut r_keys[0..r_ents], r_ents - r_goal);
+                    slice_shift(&mut r_vals[0..r_ents], r_ents - r_goal);
+                    *r_size = r_goal as u8;
+
+                    Some(r_keys[0])
+                }
+            }
+            _ => panic!("Mismatched nodes"),
+        }
+    }
+}
+
+/// Find the right split position for halving a full node with `len` entries to recover from a
+/// failed insertion at `ins`.
+///
+/// If `len` is even, we should split straight down the middle regardless of `len`.
+///
+/// If `len` is odd, we should split the node such that the two halves are the same size after the
+/// insertion is retried.
+fn split_pos(len: usize, ins: usize) -> usize {
+    // Anticipate `len` being a compile time constant, so this all folds away when `len` is even.
+    if ins <= len / 2 {
+        len / 2
+    } else {
+        (len + 1) / 2
+    }
+}
+
+/// The result of splitting off the second half of a node.
+pub(super) struct SplitOff<F: Forest> {
+    /// The number of entries left in the original node which becomes the left-hand-side of the
+    /// pair. This is the number of outgoing node edges for an inner node, and the number of
+    /// key-value pairs for a leaf node.
+    pub lhs_entries: usize,
+
+    /// The number of entries in the new RHS node.
+    pub rhs_entries: usize,
+
+    /// The critical key separating the LHS and RHS nodes. All keys in the LHS sub-tree are less
+    /// than the critical key, and all entries in the RHS sub-tree are greater or equal to the
+    /// critical key.
+    pub crit_key: F::Key,
+
+    /// The RHS node data containing the elements that were removed from the original node (now the
+    /// LHS).
+    pub rhs_data: NodeData<F>,
+}
+
+/// The result of removing an entry from a node.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum Removed {
+    /// An entry was removed, and the node is still in good shape.
+    Healthy,
+
+    /// The node is in good shape after removing the rightmost element.
+    Rightmost,
+
+    /// The node has too few entries now, and it should be balanced with a sibling node.
+    Underflow,
+
+    /// The last entry was removed. For an inner node, this means that the `keys` array is empty
+    /// and there is just a single sub-tree left.
+    Empty,
+}
+
+impl Removed {
+    /// Create a `Removed` status from a size and capacity.
+    fn new(removed: usize, new_size: usize, capacity: usize) -> Self {
+        if 2 * new_size >= capacity {
+            if removed == new_size {
+                Self::Rightmost
+            } else {
+                Self::Healthy
+            }
+        } else if new_size > 0 {
+            Self::Underflow
+        } else {
+            Self::Empty
+        }
+    }
+}
+
+// Display ": value" or nothing at all for `()`.
+pub(super) trait ValDisp {
+    fn valfmt(&self, f: &mut fmt::Formatter) -> fmt::Result;
+}
+
+impl ValDisp for SetValue {
+    fn valfmt(&self, _: &mut fmt::Formatter) -> fmt::Result {
+        Ok(())
+    }
+}
+
+impl<T: fmt::Display> ValDisp for T {
+    fn valfmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, ":{}", self)
+    }
+}
+
+impl<F> fmt::Display for NodeData<F>
+where
+    F: Forest,
+    F::Key: fmt::Display,
+    F::Value: ValDisp,
+{
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            Self::Inner { size, keys, tree } => {
+                write!(f, "[ {}", tree[0])?;
+                for i in 0..usize::from(size) {
+                    write!(f, " {} {}", keys[i], tree[i + 1])?;
+                }
+                write!(f, " ]")
+            }
+            Self::Leaf { size, keys, vals } => {
+                let keys = keys.borrow();
+                let vals = vals.borrow();
+                write!(f, "[")?;
+                for i in 0..usize::from(size) {
+                    write!(f, " {}", keys[i])?;
+                    vals[i].valfmt(f)?;
+                }
+                write!(f, " ]")
+            }
+            Self::Free { next: Some(n) } => write!(f, "[ free -> {} ]", n),
+            Self::Free { next: None } => write!(f, "[ free ]"),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use alloc::string::ToString;
+    use core::mem;
+
+    // Forest impl for a set implementation.
+    struct TF();
+
+    impl Forest for TF {
+        type Key = char;
+        type Value = SetValue;
+        type LeafKeys = [char; 15];
+        type LeafValues = [SetValue; 15];
+
+        fn splat_key(key: Self::Key) -> Self::LeafKeys {
+            [key; 15]
+        }
+
+        fn splat_value(value: Self::Value) -> Self::LeafValues {
+            [value; 15]
+        }
+    }
+
+    #[test]
+    fn inner() {
+        let n1 = Node(1);
+        let n2 = Node(2);
+        let n3 = Node(3);
+        let n4 = Node(4);
+        let mut inner = NodeData::<TF>::inner(n1, 'c', n4);
+        assert_eq!(mem::size_of_val(&inner), 64);
+        assert_eq!(inner.to_string(), "[ node1 c node4 ]");
+
+        assert!(inner.try_inner_insert(0, 'a', n2));
+        assert_eq!(inner.to_string(), "[ node1 a node2 c node4 ]");
+
+        assert!(inner.try_inner_insert(1, 'b', n3));
+        assert_eq!(inner.to_string(), "[ node1 a node2 b node3 c node4 ]");
+
+        for i in 3..7 {
+            assert!(inner.try_inner_insert(
+                usize::from(i),
+                ('a' as u8 + i) as char,
+                Node(i as u32 + 2),
+            ));
+        }
+        assert_eq!(
+            inner.to_string(),
+            "[ node1 a node2 b node3 c node4 d node5 e node6 f node7 g node8 ]"
+        );
+
+        // Now the node is full and insertion should fail anywhere.
+        assert!(!inner.try_inner_insert(0, 'x', n3));
+        assert!(!inner.try_inner_insert(4, 'x', n3));
+        assert!(!inner.try_inner_insert(7, 'x', n3));
+
+        // Splitting should be independent of the hint because we have an even number of node
+        // references.
+        let saved = inner.clone();
+        let sp = inner.split(1);
+        assert_eq!(sp.lhs_entries, 4);
+        assert_eq!(sp.rhs_entries, 4);
+        assert_eq!(sp.crit_key, 'd');
+        // The critical key is not present in either of the resulting nodes.
+        assert_eq!(inner.to_string(), "[ node1 a node2 b node3 c node4 ]");
+        assert_eq!(sp.rhs_data.to_string(), "[ node5 e node6 f node7 g node8 ]");
+
+        assert_eq!(inner.inner_remove(0), Removed::Underflow);
+        assert_eq!(inner.to_string(), "[ node2 b node3 c node4 ]");
+
+        assert_eq!(inner.inner_remove(1), Removed::Underflow);
+        assert_eq!(inner.to_string(), "[ node2 c node4 ]");
+
+        assert_eq!(inner.inner_remove(1), Removed::Underflow);
+        assert_eq!(inner.to_string(), "[ node2 ]");
+
+        assert_eq!(inner.inner_remove(0), Removed::Empty);
+
+        inner = saved;
+        let sp = inner.split(6);
+        assert_eq!(sp.lhs_entries, 4);
+        assert_eq!(sp.rhs_entries, 4);
+        assert_eq!(sp.crit_key, 'd');
+        assert_eq!(inner.to_string(), "[ node1 a node2 b node3 c node4 ]");
+        assert_eq!(sp.rhs_data.to_string(), "[ node5 e node6 f node7 g node8 ]");
+    }
+
+    #[test]
+    fn leaf() {
+        let mut leaf = NodeData::<TF>::leaf('d', SetValue());
+        assert_eq!(leaf.to_string(), "[ d ]");
+
+        assert!(leaf.try_leaf_insert(0, 'a', SetValue()));
+        assert_eq!(leaf.to_string(), "[ a d ]");
+        assert!(leaf.try_leaf_insert(1, 'b', SetValue()));
+        assert!(leaf.try_leaf_insert(2, 'c', SetValue()));
+        assert_eq!(leaf.to_string(), "[ a b c d ]");
+        for i in 4..15 {
+            assert!(leaf.try_leaf_insert(usize::from(i), ('a' as u8 + i) as char, SetValue()));
+        }
+        assert_eq!(leaf.to_string(), "[ a b c d e f g h i j k l m n o ]");
+
+        // Now the node is full and insertion should fail anywhere.
+        assert!(!leaf.try_leaf_insert(0, 'x', SetValue()));
+        assert!(!leaf.try_leaf_insert(8, 'x', SetValue()));
+        assert!(!leaf.try_leaf_insert(15, 'x', SetValue()));
+
+        // The index given to `split` is not the split position, it's a hint for balancing the node.
+        let saved = leaf.clone();
+        let sp = leaf.split(12);
+        assert_eq!(sp.lhs_entries, 8);
+        assert_eq!(sp.rhs_entries, 7);
+        assert_eq!(sp.crit_key, 'i');
+        assert_eq!(leaf.to_string(), "[ a b c d e f g h ]");
+        assert_eq!(sp.rhs_data.to_string(), "[ i j k l m n o ]");
+
+        assert!(leaf.try_leaf_insert(8, 'i', SetValue()));
+        assert_eq!(leaf.leaf_remove(2), Removed::Healthy);
+        assert_eq!(leaf.to_string(), "[ a b d e f g h i ]");
+        assert_eq!(leaf.leaf_remove(7), Removed::Underflow);
+        assert_eq!(leaf.to_string(), "[ a b d e f g h ]");
+
+        leaf = saved;
+        let sp = leaf.split(7);
+        assert_eq!(sp.lhs_entries, 7);
+        assert_eq!(sp.rhs_entries, 8);
+        assert_eq!(sp.crit_key, 'h');
+        assert_eq!(leaf.to_string(), "[ a b c d e f g ]");
+        assert_eq!(sp.rhs_data.to_string(), "[ h i j k l m n o ]");
+    }
+
+    #[test]
+    fn optimal_split_pos() {
+        // An even split is easy.
+        assert_eq!(split_pos(8, 0), 4);
+        assert_eq!(split_pos(8, 8), 4);
+
+        // Easy cases for odd splits.
+        assert_eq!(split_pos(7, 0), 3);
+        assert_eq!(split_pos(7, 7), 4);
+
+        // If the insertion point is the same as the split position, we
+        // will append to the lhs node.
+        assert_eq!(split_pos(7, 3), 3);
+        assert_eq!(split_pos(7, 4), 4);
+    }
+
+    #[test]
+    fn inner_balance() {
+        let n1 = Node(1);
+        let n2 = Node(2);
+        let n3 = Node(3);
+        let mut lhs = NodeData::<TF>::inner(n1, 'a', n2);
+        assert!(lhs.try_inner_insert(1, 'b', n3));
+        assert_eq!(lhs.to_string(), "[ node1 a node2 b node3 ]");
+
+        let n11 = Node(11);
+        let n12 = Node(12);
+        let mut rhs = NodeData::<TF>::inner(n11, 'p', n12);
+
+        for i in 1..4 {
+            assert!(rhs.try_inner_insert(
+                usize::from(i),
+                ('p' as u8 + i) as char,
+                Node(i as u32 + 12),
+            ));
+        }
+        assert_eq!(
+            rhs.to_string(),
+            "[ node11 p node12 q node13 r node14 s node15 ]"
+        );
+
+        // 3+5 elements fit in RHS.
+        assert_eq!(lhs.balance('o', &mut rhs), None);
+        assert_eq!(
+            rhs.to_string(),
+            "[ node1 a node2 b node3 o node11 p node12 q node13 r node14 s node15 ]"
+        );
+
+        // 2+8 elements are redistributed.
+        lhs = NodeData::<TF>::inner(Node(20), 'x', Node(21));
+        assert_eq!(lhs.balance('y', &mut rhs), Some('o'));
+        assert_eq!(
+            lhs.to_string(),
+            "[ node20 x node21 y node1 a node2 b node3 ]"
+        );
+        assert_eq!(
+            rhs.to_string(),
+            "[ node11 p node12 q node13 r node14 s node15 ]"
+        );
+    }
+
+    #[test]
+    fn leaf_balance() {
+        let mut lhs = NodeData::<TF>::leaf('a', SetValue());
+        for i in 1..6 {
+            assert!(lhs.try_leaf_insert(usize::from(i), ('a' as u8 + i) as char, SetValue()));
+        }
+        assert_eq!(lhs.to_string(), "[ a b c d e f ]");
+
+        let mut rhs = NodeData::<TF>::leaf('0', SetValue());
+        for i in 1..8 {
+            assert!(rhs.try_leaf_insert(usize::from(i), ('0' as u8 + i) as char, SetValue()));
+        }
+        assert_eq!(rhs.to_string(), "[ 0 1 2 3 4 5 6 7 ]");
+
+        // 6+8 elements all fits in rhs.
+        assert_eq!(lhs.balance('0', &mut rhs), None);
+        assert_eq!(rhs.to_string(), "[ a b c d e f 0 1 2 3 4 5 6 7 ]");
+
+        assert!(lhs.try_leaf_insert(0, 'x', SetValue()));
+        assert!(lhs.try_leaf_insert(1, 'y', SetValue()));
+        assert!(lhs.try_leaf_insert(2, 'z', SetValue()));
+        assert_eq!(lhs.to_string(), "[ x y z ]");
+
+        // 3+14 elements need redistribution.
+        assert_eq!(lhs.balance('a', &mut rhs), Some('0'));
+        assert_eq!(lhs.to_string(), "[ x y z a b c d e f ]");
+        assert_eq!(rhs.to_string(), "[ 0 1 2 3 4 5 6 7 ]");
+    }
+}

cranelift/bforest/src/path.rs

@@ -0,0 +1,836 @@
+//! A path from the root of a B+-tree to a leaf node.
+
+use super::node::Removed;
+use super::{slice_insert, slice_shift, Comparator, Forest, Node, NodeData, NodePool, MAX_PATH};
+use core::borrow::Borrow;
+use core::marker::PhantomData;
+
+#[cfg(test)]
+use core::fmt;
+
+pub(super) struct Path<F: Forest> {
+    /// Number of path entries including the root and leaf nodes.
+    size: usize,
+
+    /// Path of node references from the root to a leaf node.
+    node: [Node; MAX_PATH],
+
+    /// Entry number in each node.
+    entry: [u8; MAX_PATH],
+
+    unused: PhantomData<F>,
+}
+
+impl<F: Forest> Default for Path<F> {
+    fn default() -> Self {
+        Self {
+            size: 0,
+            node: [Node(0); MAX_PATH],
+            entry: [0; MAX_PATH],
+            unused: PhantomData,
+        }
+    }
+}
+
+impl<F: Forest> Path<F> {
+    /// Reset path by searching for `key` starting from `root`.
+    ///
+    /// If `key` is in the tree, returns the corresponding value and leaved the path pointing at
+    /// the entry. Otherwise returns `None` and:
+    ///
+    /// - A key smaller than all stored keys returns a path to the first entry of the first leaf.
+    /// - A key larger than all stored keys returns a path to one beyond the last element of the
+    ///   last leaf.
+    /// - A key between the stored keys of adjacent leaf nodes returns a path to one beyond the
+    ///   last entry of the first of the leaf nodes.
+    ///
+    pub fn find(
+        &mut self,
+        key: F::Key,
+        root: Node,
+        pool: &NodePool<F>,
+        comp: &dyn Comparator<F::Key>,
+    ) -> Option<F::Value> {
+        let mut node = root;
+        for level in 0.. {
+            self.size = level + 1;
+            self.node[level] = node;
+            match pool[node] {
+                NodeData::Inner { size, keys, tree } => {
+                    // Invariant: `tree[i]` contains keys smaller than
+                    // `keys[i]`, greater or equal to `keys[i-1]`.
+                    let i = match comp.search(key, &keys[0..size.into()]) {
+                        // We hit an existing key, so follow the >= branch.
+                        Ok(i) => i + 1,
+                        // Key is less than `keys[i]`, so follow the < branch.
+                        Err(i) => i,
+                    };
+                    self.entry[level] = i as u8;
+                    node = tree[i];
+                }
+                NodeData::Leaf { size, keys, vals } => {
+                    // For a leaf we want either the found key or an insert position.
+                    return match comp.search(key, &keys.borrow()[0..size.into()]) {
+                        Ok(i) => {
+                            self.entry[level] = i as u8;
+                            Some(vals.borrow()[i])
+                        }
+                        Err(i) => {
+                            self.entry[level] = i as u8;
+                            None
+                        }
+                    };
+                }
+                NodeData::Free { .. } => panic!("Free {} reached from {}", node, root),
+            }
+        }
+        unreachable!();
+    }
+
+    /// Move path to the first entry of the tree starting at `root` and return it.
+    pub fn first(&mut self, root: Node, pool: &NodePool<F>) -> (F::Key, F::Value) {
+        let mut node = root;
+        for level in 0.. {
+            self.size = level + 1;
+            self.node[level] = node;
+            self.entry[level] = 0;
+            match pool[node] {
+                NodeData::Inner { tree, .. } => node = tree[0],
+                NodeData::Leaf { keys, vals, .. } => return (keys.borrow()[0], vals.borrow()[0]),
+                NodeData::Free { .. } => panic!("Free {} reached from {}", node, root),
+            }
+        }
+        unreachable!();
+    }
+
+    /// Move this path to the next key-value pair and return it.
+    pub fn next(&mut self, pool: &NodePool<F>) -> Option<(F::Key, F::Value)> {
+        match self.leaf_pos() {
+            None => return None,
+            Some((node, entry)) => {
+                let (keys, vals) = pool[node].unwrap_leaf();
+                if entry + 1 < keys.len() {
+                    self.entry[self.size - 1] += 1;
+                    return Some((keys[entry + 1], vals[entry + 1]));
+                }
+            }
+        }
+
+        // The current leaf node is exhausted. Move to the next one.
+        let leaf_level = self.size - 1;
+        self.next_node(leaf_level, pool).map(|node| {
+            let (keys, vals) = pool[node].unwrap_leaf();
+            (keys[0], vals[0])
+        })
+    }
+
+    /// Move this path to the previous key-value pair and return it.
+    ///
+    /// If the path is at the off-the-end position, go to the last key-value pair.
+    ///
+    /// If the path is already at the first key-value pair, leave it there and return `None`.
+    pub fn prev(&mut self, root: Node, pool: &NodePool<F>) -> Option<(F::Key, F::Value)> {
+        // We use `size == 0` as a generic off-the-end position.
+        if self.size == 0 {
+            self.goto_subtree_last(0, root, pool);
+            let (node, entry) = self.leaf_pos().unwrap();
+            let (keys, vals) = pool[node].unwrap_leaf();
+            return Some((keys[entry], vals[entry]));
+        }
+
+        match self.leaf_pos() {
+            None => return None,
+            Some((node, entry)) => {
+                if entry > 0 {
+                    self.entry[self.size - 1] -= 1;
+                    let (keys, vals) = pool[node].unwrap_leaf();
+                    return Some((keys[entry - 1], vals[entry - 1]));
+                }
+            }
+        }
+
+        // The current leaf node is exhausted. Move to the previous one.
+        self.prev_leaf(pool).map(|node| {
+            let (keys, vals) = pool[node].unwrap_leaf();
+            let e = self.leaf_entry();
+            (keys[e], vals[e])
+        })
+    }
+
+    /// Move path to the first entry of the next node at level, if one exists.
+    ///
+    /// Returns the new node if it exists.
+    ///
+    /// Reset the path to `size = 0` and return `None` if there is no next node.
+    fn next_node(&mut self, level: usize, pool: &NodePool<F>) -> Option<Node> {
+        match self.right_sibling_branch_level(level, pool) {
+            None => {
+                self.size = 0;
+                None
+            }
+            Some(bl) => {
+                let (_, bnodes) = pool[self.node[bl]].unwrap_inner();
+                self.entry[bl] += 1;
+                let mut node = bnodes[usize::from(self.entry[bl])];
+
+                for l in bl + 1..level {
+                    self.node[l] = node;
+                    self.entry[l] = 0;
+                    node = pool[node].unwrap_inner().1[0];
+                }
+
+                self.node[level] = node;
+                self.entry[level] = 0;
+                Some(node)
+            }
+        }
+    }
+
+    /// Move the path to the last entry of the previous leaf node, if one exists.
+    ///
+    /// Returns the new leaf node if it exists.
+    ///
+    /// Leave the path unchanged and returns `None` if we are already at the first leaf node.
+    fn prev_leaf(&mut self, pool: &NodePool<F>) -> Option<Node> {
+        self.left_sibling_branch_level(self.size - 1).map(|bl| {
+            let entry = self.entry[bl] - 1;
+            self.entry[bl] = entry;
+            let (_, bnodes) = pool[self.node[bl]].unwrap_inner();
+            self.goto_subtree_last(bl + 1, bnodes[usize::from(entry)], pool)
+        })
+    }
+
+    /// Move this path to the last position for the sub-tree at `level, root`.
+    fn goto_subtree_last(&mut self, level: usize, root: Node, pool: &NodePool<F>) -> Node {
+        let mut node = root;
+        for l in level.. {
+            self.node[l] = node;
+            match pool[node] {
+                NodeData::Inner { size, ref tree, .. } => {
+                    self.entry[l] = size;
+                    node = tree[usize::from(size)];
+                }
+                NodeData::Leaf { size, .. } => {
+                    self.entry[l] = size - 1;
+                    self.size = l + 1;
+                    break;
+                }
+                NodeData::Free { .. } => panic!("Free {} reached from {}", node, root),
+            }
+        }
+        node
+    }
+
+    /// Set the root node and point the path at the first entry of the node.
+    pub fn set_root_node(&mut self, root: Node) {
+        self.size = 1;
+        self.node[0] = root;
+        self.entry[0] = 0;
+    }
+
+    /// Get the current leaf node and entry, if any.
+    pub fn leaf_pos(&self) -> Option<(Node, usize)> {
+        let i = self.size.wrapping_sub(1);
+        self.node.get(i).map(|&n| (n, self.entry[i].into()))
+    }
+
+    /// Get the current leaf node.
+    fn leaf_node(&self) -> Node {
+        self.node[self.size - 1]
+    }
+
+    /// Get the current entry in the leaf node.
+    fn leaf_entry(&self) -> usize {
+        self.entry[self.size - 1].into()
+    }
+
+    /// Is this path pointing to the first entry in the tree?
+    /// This corresponds to the smallest key.
+    fn at_first_entry(&self) -> bool {
+        self.entry[0..self.size].iter().all(|&i| i == 0)
+    }
+
+    /// Get a mutable reference to the current value.
+    /// This assumes that there is a current value.
+    pub fn value_mut<'a>(&self, pool: &'a mut NodePool<F>) -> &'a mut F::Value {
+        &mut pool[self.leaf_node()].unwrap_leaf_mut().1[self.leaf_entry()]
+    }
+
+    /// Insert the key-value pair at the current position.
+    /// The current position must be the correct insertion location for the key.
+    /// This function does not check for duplicate keys. Use `find` or similar for that.
+    /// Returns the new root node.
+    pub fn insert(&mut self, key: F::Key, value: F::Value, pool: &mut NodePool<F>) -> Node {
+        if !self.try_leaf_insert(key, value, pool) {
+            self.split_and_insert(key, value, pool);
+        }
+        self.node[0]
+    }
+
+    /// Try to insert `key, value` at the current position, but fail and return false if the leaf
+    /// node is full.
+    fn try_leaf_insert(&self, key: F::Key, value: F::Value, pool: &mut NodePool<F>) -> bool {
+        let index = self.leaf_entry();
+
+        // The case `index == 0` should only ever happen when there are no earlier leaf nodes,
+        // otherwise we should have appended to the previous leaf node instead. This invariant
+        // means that we don't need to update keys stored in inner nodes here.
+        debug_assert!(index > 0 || self.at_first_entry());
+
+        pool[self.leaf_node()].try_leaf_insert(index, key, value)
+    }
+
+    /// Split the current leaf node and then insert `key, value`.
+    /// This should only be used if `try_leaf_insert()` fails.
+    fn split_and_insert(&mut self, mut key: F::Key, value: F::Value, pool: &mut NodePool<F>) {
+        let orig_root = self.node[0];
+
+        // Loop invariant: We need to split the node at `level` and then retry a failed insertion.
+        // The items to insert are either `(key, ins_node)` or `(key, value)`.
+        let mut ins_node = None;
+        let mut split;
+        for level in (0..self.size).rev() {
+            // Split the current node.
+            let mut node = self.node[level];
+            let mut entry = self.entry[level].into();
+            split = pool[node].split(entry);
+            let rhs_node = pool.alloc_node(split.rhs_data);
+
+            // Should the path be moved to the new RHS node?
+            // Prefer the smaller node if we're right in the middle.
+            // Prefer to append to LHS all other things being equal.
+            //
+            // When inserting into an inner node (`ins_node.is_some()`), we must point to a valid
+            // entry in the current node since the new entry is inserted *after* the insert
+            // location.
+            if entry > split.lhs_entries
+                || (entry == split.lhs_entries
+                    && (split.lhs_entries > split.rhs_entries || ins_node.is_some()))
+            {
+                node = rhs_node;
+                entry -= split.lhs_entries;
+                self.node[level] = node;
+                self.entry[level] = entry as u8;
+            }
+
+            // Now that we have a not-full node, it must be possible to insert.
+            match ins_node {
+                None => {
+                    let inserted = pool[node].try_leaf_insert(entry, key, value);
+                    debug_assert!(inserted);
+                    // If we inserted at the front of the new rhs_node leaf, we need to propagate
+                    // the inserted key as the critical key instead of the previous front key.
+                    if entry == 0 && node == rhs_node {
+                        split.crit_key = key;
+                    }
+                }
+                Some(n) => {
+                    let inserted = pool[node].try_inner_insert(entry, key, n);
+                    debug_assert!(inserted);
+                    // The lower level was moved to the new RHS node, so make sure that is
+                    // reflected here.
+                    if n == self.node[level + 1] {
+                        self.entry[level] += 1;
+                    }
+                }
+            }
+
+            // We are now done with the current level, but `rhs_node` must be inserted in the inner
+            // node above us. If we're already at level 0, the root node needs to be split.
+            key = split.crit_key;
+            ins_node = Some(rhs_node);
+            if level > 0 {
+                let pnode = &mut pool[self.node[level - 1]];
+                let pentry = self.entry[level - 1].into();
+                if pnode.try_inner_insert(pentry, key, rhs_node) {
+                    // If this level level was moved to the new RHS node, update parent entry.
+                    if node == rhs_node {
+                        self.entry[level - 1] += 1;
+                    }
+                    return;
+                }
+            }
+        }
+
+        // If we get here we have split the original root node and need to add an extra level.
+        let rhs_node = ins_node.expect("empty path");
+        let root = pool.alloc_node(NodeData::inner(orig_root, key, rhs_node));
+        let entry = if self.node[0] == rhs_node { 1 } else { 0 };
+        self.size += 1;
+        slice_insert(&mut self.node[0..self.size], 0, root);
+        slice_insert(&mut self.entry[0..self.size], 0, entry);
+    }
+
+    /// Remove the key-value pair at the current position and advance the path to the next
+    /// key-value pair, leaving the path in a normalized state.
+    ///
+    /// Return the new root node.
+    pub fn remove(&mut self, pool: &mut NodePool<F>) -> Option<Node> {
+        let e = self.leaf_entry();
+        match pool[self.leaf_node()].leaf_remove(e) {
+            Removed::Healthy => {
+                if e == 0 {
+                    self.update_crit_key(pool)
+                }
+                Some(self.node[0])
+            }
+            status => self.balance_nodes(status, pool),
+        }
+    }
+
+    /// Get the critical key for the current node at `level`.
+    ///
+    /// The critical key is less than or equal to all keys in the sub-tree at `level` and greater
+    /// than all keys to the left of the current node at `level`.
+    ///
+    /// The left-most node at any level does not have a critical key.
+    fn current_crit_key(&self, level: usize, pool: &NodePool<F>) -> Option<F::Key> {
+        // Find the level containing the critical key for the current node.
+        self.left_sibling_branch_level(level).map(|bl| {
+            let (keys, _) = pool[self.node[bl]].unwrap_inner();
+            keys[usize::from(self.entry[bl]) - 1]
+        })
+    }
+
+    /// Update the critical key after removing the front entry of the leaf node.
+    fn update_crit_key(&mut self, pool: &mut NodePool<F>) {
+        // Find the inner level containing the critical key for the current leaf node.
+        let crit_level = match self.left_sibling_branch_level(self.size - 1) {
+            None => return,
+            Some(l) => l,
+        };
+        let crit_kidx = self.entry[crit_level] - 1;
+
+        // Extract the new critical key from the leaf node.
+        let crit_key = pool[self.leaf_node()].leaf_crit_key();
+        let crit_node = self.node[crit_level];
+
+        match pool[crit_node] {
+            NodeData::Inner {
+                size, ref mut keys, ..
+            } => {
+                debug_assert!(crit_kidx < size);
+                keys[usize::from(crit_kidx)] = crit_key;
+            }
+            _ => panic!("Expected inner node"),
+        }
+    }
+
+    /// Given that the current leaf node is in an unhealthy (underflowed or even empty) status,
+    /// balance it with sibling nodes.
+    ///
+    /// Return the new root node.
+    fn balance_nodes(&mut self, status: Removed, pool: &mut NodePool<F>) -> Option<Node> {
+        // The current leaf node is not in a healthy state, and its critical key may have changed
+        // too.
+        //
+        // Start by dealing with a changed critical key for the leaf level.
+        if status != Removed::Empty && self.leaf_entry() == 0 {
+            self.update_crit_key(pool);
+        }
+
+        let leaf_level = self.size - 1;
+        if self.heal_level(status, leaf_level, pool) {
+            // Tree has become empty.
+            self.size = 0;
+            return None;
+        }
+
+        // Discard the root node if it has shrunk to a single sub-tree.
+        let mut ns = 0;
+        while let NodeData::Inner {
+            size: 0, ref tree, ..
+        } = pool[self.node[ns]]
+        {
+            ns += 1;
+            self.node[ns] = tree[0];
+        }
+
+        if ns > 0 {
+            for l in 0..ns {
+                pool.free_node(self.node[l]);
+            }
+
+            // Shift the whole array instead of just 0..size because `self.size` may be cleared
+            // here if the path is pointing off-the-end.
+            slice_shift(&mut self.node, ns);
+            slice_shift(&mut self.entry, ns);
+
+            if self.size > 0 {
+                self.size -= ns;
+            }
+        }
+
+        // Return the root node, even when `size=0` indicating that we're at the off-the-end
+        // position.
+        Some(self.node[0])
+    }
+
+    /// After removing an entry from the node at `level`, check its health and rebalance as needed.
+    ///
+    /// Leave the path up to and including `level` in a normalized state where all entries are in
+    /// bounds.
+    ///
+    /// Returns true if the tree becomes empty.
+    fn heal_level(&mut self, status: Removed, level: usize, pool: &mut NodePool<F>) -> bool {
+        match status {
+            Removed::Healthy => {}
+            Removed::Rightmost => {
+                // The rightmost entry was removed from the current node, so move the path so it
+                // points at the first entry of the next node at this level.
+                debug_assert_eq!(
+                    usize::from(self.entry[level]),
+                    pool[self.node[level]].entries()
+                );
+                self.next_node(level, pool);
+            }
+            Removed::Underflow => self.underflowed_node(level, pool),
+            Removed::Empty => return self.empty_node(level, pool),
+        }
+        false
+    }
+
+    /// The current node at `level` has underflowed, meaning that it is below half capacity but
+    /// not completely empty.
+    ///
+    /// Handle this by balancing entries with the right sibling node.
+    ///
+    /// Leave the path up to and including `level` in a valid state that points to the same entry.
+    fn underflowed_node(&mut self, level: usize, pool: &mut NodePool<F>) {
+        // Look for a right sibling node at this level. If none exists, we allow the underflowed
+        // node to persist as the right-most node at its level.
+        if let Some((crit_key, rhs_node)) = self.right_sibling(level, pool) {
+            // New critical key for the updated right sibling node.
+            let new_ck: Option<F::Key>;
+            let empty;
+            // Make a COPY of the sibling node to avoid fighting the borrow checker.
+            let mut rhs = pool[rhs_node];
+            match pool[self.node[level]].balance(crit_key, &mut rhs) {
+                None => {
+                    // Everything got moved to the RHS node.
+                    new_ck = self.current_crit_key(level, pool);
+                    empty = true;
+                }
+                Some(key) => {
+                    // Entries moved from RHS node.
+                    new_ck = Some(key);
+                    empty = false;
+                }
+            }
+            // Put back the updated RHS node data.
+            pool[rhs_node] = rhs;
+            // Update the critical key for the RHS node unless it has become a left-most
+            // node.
+            if let Some(ck) = new_ck {
+                self.update_right_crit_key(level, ck, pool);
+            }
+            if empty {
+                let empty_tree = self.empty_node(level, pool);
+                debug_assert!(!empty_tree);
+            }
+
+            // Any Removed::Rightmost state must have been cleared above by merging nodes. If the
+            // current entry[level] was one off the end of the node, it will now point at a proper
+            // entry.
+            debug_assert!(usize::from(self.entry[level]) < pool[self.node[level]].entries());
+        } else if usize::from(self.entry[level]) >= pool[self.node[level]].entries() {
+            // There's no right sibling at this level, so the node can't be rebalanced.
+            // Check if we are in an off-the-end position.
+            self.size = 0;
+        }
+    }
+
+    /// The current node at `level` has become empty.
+    ///
+    /// Remove the node from its parent node and leave the path in a normalized state. This means
+    /// that the path at this level will go through the right sibling of this node.
+    ///
+    /// If the current node has no right sibling, set `self.size = 0`.
+    ///
+    /// Returns true if the tree becomes empty.
+    fn empty_node(&mut self, level: usize, pool: &mut NodePool<F>) -> bool {
+        pool.free_node(self.node[level]);
+        if level == 0 {
+            // We just deleted the root node, so the tree is now empty.
+            return true;
+        }
+
+        // Get the right sibling node before recursively removing nodes.
+        let rhs_node = self.right_sibling(level, pool).map(|(_, n)| n);
+
+        // Remove the current sub-tree from the parent node.
+        let pl = level - 1;
+        let pe = self.entry[pl].into();
+        let status = pool[self.node[pl]].inner_remove(pe);
+        self.heal_level(status, pl, pool);
+
+        // Finally update the path at this level.
+        match rhs_node {
+            // We'll leave `self.entry[level]` unchanged. It can be non-zero after moving node
+            // entries to the right sibling node.
+            Some(rhs) => self.node[level] = rhs,
+            // We have no right sibling, so we must have deleted the right-most
+            // entry. The path should be moved to the "off-the-end" position.
+            None => self.size = 0,
+        }
+        false
+    }
+
+    /// Find the level where the right sibling to the current node at `level` branches off.
+    ///
+    /// This will be an inner node with two adjacent sub-trees: In one the current node at level is
+    /// a right-most node, in the other, the right sibling is a left-most node.
+    ///
+    /// Returns `None` if the current node is a right-most node so no right sibling exists.
+    fn right_sibling_branch_level(&self, level: usize, pool: &NodePool<F>) -> Option<usize> {
+        (0..level).rposition(|l| match pool[self.node[l]] {
+            NodeData::Inner { size, .. } => self.entry[l] < size,
+            _ => panic!("Expected inner node"),
+        })
+    }
+
+    /// Find the level where the left sibling to the current node at `level` branches off.
+    fn left_sibling_branch_level(&self, level: usize) -> Option<usize> {
+        self.entry[0..level].iter().rposition(|&e| e != 0)
+    }
+
+    /// Get the right sibling node to the current node at `level`.
+    /// Also return the critical key between the current node and the right sibling.
+    fn right_sibling(&self, level: usize, pool: &NodePool<F>) -> Option<(F::Key, Node)> {
+        // Find the critical level: The deepest level where two sibling subtrees contain the
+        // current node and its right sibling.
+        self.right_sibling_branch_level(level, pool).map(|bl| {
+            // Extract the critical key and the `bl+1` node.
+            let be = usize::from(self.entry[bl]);
+            let crit_key;
+            let mut node;
+            {
+                let (keys, tree) = pool[self.node[bl]].unwrap_inner();
+                crit_key = keys[be];
+                node = tree[be + 1];
+            }
+
+            // Follow left-most links back down to `level`.
+            for _ in bl + 1..level {
+                node = pool[node].unwrap_inner().1[0];
+            }
+
+            (crit_key, node)
+        })
+    }
+
+    /// Update the critical key for the right sibling node at `level`.
+    fn update_right_crit_key(&self, level: usize, crit_key: F::Key, pool: &mut NodePool<F>) {
+        let bl = self
+            .right_sibling_branch_level(level, pool)
+            .expect("No right sibling exists");
+        match pool[self.node[bl]] {
+            NodeData::Inner { ref mut keys, .. } => {
+                keys[usize::from(self.entry[bl])] = crit_key;
+            }
+            _ => panic!("Expected inner node"),
+        }
+    }
+
+    /// Normalize the path position such that it is either pointing at a real entry or `size=0`
+    /// indicating "off-the-end".
+    pub fn normalize(&mut self, pool: &mut NodePool<F>) {
+        if let Some((leaf, entry)) = self.leaf_pos() {
+            if entry >= pool[leaf].entries() {
+                let leaf_level = self.size - 1;
+                self.next_node(leaf_level, pool);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+impl<F: Forest> Path<F> {
+    /// Check the internal consistency of this path.
+    pub fn verify(&self, pool: &NodePool<F>) {
+        for level in 0..self.size {
+            match pool[self.node[level]] {
+                NodeData::Inner { size, tree, .. } => {
+                    assert!(
+                        level < self.size - 1,
+                        "Expected leaf node at level {}",
+                        level
+                    );
+                    assert!(
+                        self.entry[level] <= size,
+                        "OOB inner entry {}/{} at level {}",
+                        self.entry[level],
+                        size,
+                        level
+                    );
+                    assert_eq!(
+                        self.node[level + 1],
+                        tree[usize::from(self.entry[level])],
+                        "Node mismatch at level {}",
+                        level
+                    );
+                }
+                NodeData::Leaf { size, .. } => {
+                    assert_eq!(level, self.size - 1, "Expected inner node");
+                    assert!(
+                        self.entry[level] <= size,
+                        "OOB leaf entry {}/{}",
+                        self.entry[level],
+                        size,
+                    );
+                }
+                NodeData::Free { .. } => {
+                    panic!("Free {} in path", self.node[level]);
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+impl<F: Forest> fmt::Display for Path<F> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        if self.size == 0 {
+            write!(f, "<empty path>")
+        } else {
+            write!(f, "{}[{}]", self.node[0], self.entry[0])?;
+            for i in 1..self.size {
+                write!(f, "--{}[{}]", self.node[i], self.entry[i])?;
+            }
+            Ok(())
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::{Forest, NodeData, NodePool};
+    use super::*;
+    use core::cmp::Ordering;
+
+    struct TC();
+
+    impl Comparator<i32> for TC {
+        fn cmp(&self, a: i32, b: i32) -> Ordering {
+            a.cmp(&b)
+        }
+    }
+
+    struct TF();
+
+    impl Forest for TF {
+        type Key = i32;
+        type Value = char;
+        type LeafKeys = [i32; 7];
+        type LeafValues = [char; 7];
+
+        fn splat_key(key: Self::Key) -> Self::LeafKeys {
+            [key; 7]
+        }
+
+        fn splat_value(value: Self::Value) -> Self::LeafValues {
+            [value; 7]
+        }
+    }
+
+    #[test]
+    fn search_single_leaf() {
+        // Testing Path::new() for trees with a single leaf node.
+        let mut pool = NodePool::<TF>::new();
+        let root = pool.alloc_node(NodeData::leaf(10, 'a'));
+        let mut p = Path::default();
+        let comp = TC();
+
+        // Search for key less than stored key.
+        assert_eq!(p.find(5, root, &pool, &comp), None);
+        assert_eq!(p.size, 1);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 0);
+
+        // Search for stored key.
+        assert_eq!(p.find(10, root, &pool, &comp), Some('a'));
+        assert_eq!(p.size, 1);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 0);
+
+        // Search for key greater than stored key.
+        assert_eq!(p.find(15, root, &pool, &comp), None);
+        assert_eq!(p.size, 1);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 1);
+
+        // Modify leaf node to contain two values.
+        match pool[root] {
+            NodeData::Leaf {
+                ref mut size,
+                ref mut keys,
+                ref mut vals,
+            } => {
+                *size = 2;
+                keys[1] = 20;
+                vals[1] = 'b';
+            }
+            _ => unreachable!(),
+        }
+
+        // Search for key between stored keys.
+        assert_eq!(p.find(15, root, &pool, &comp), None);
+        assert_eq!(p.size, 1);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 1);
+
+        // Search for key greater than stored keys.
+        assert_eq!(p.find(25, root, &pool, &comp), None);
+        assert_eq!(p.size, 1);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 2);
+    }
+
+    #[test]
+    fn search_single_inner() {
+        // Testing Path::new() for trees with a single inner node and two leaves.
+        let mut pool = NodePool::<TF>::new();
+        let leaf1 = pool.alloc_node(NodeData::leaf(10, 'a'));
+        let leaf2 = pool.alloc_node(NodeData::leaf(20, 'b'));
+        let root = pool.alloc_node(NodeData::inner(leaf1, 20, leaf2));
+        let mut p = Path::default();
+        let comp = TC();
+
+        // Search for key less than stored keys.
+        assert_eq!(p.find(5, root, &pool, &comp), None);
+        assert_eq!(p.size, 2);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 0);
+        assert_eq!(p.node[1], leaf1);
+        assert_eq!(p.entry[1], 0);
+
+        assert_eq!(p.find(10, root, &pool, &comp), Some('a'));
+        assert_eq!(p.size, 2);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 0);
+        assert_eq!(p.node[1], leaf1);
+        assert_eq!(p.entry[1], 0);
+
+        // Midway between the two leaf nodes.
+        assert_eq!(p.find(15, root, &pool, &comp), None);
+        assert_eq!(p.size, 2);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 0);
+        assert_eq!(p.node[1], leaf1);
+        assert_eq!(p.entry[1], 1);
+
+        assert_eq!(p.find(20, root, &pool, &comp), Some('b'));
+        assert_eq!(p.size, 2);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 1);
+        assert_eq!(p.node[1], leaf2);
+        assert_eq!(p.entry[1], 0);
+
+        assert_eq!(p.find(25, root, &pool, &comp), None);
+        assert_eq!(p.size, 2);
+        assert_eq!(p.node[0], root);
+        assert_eq!(p.entry[0], 1);
+        assert_eq!(p.node[1], leaf2);
+        assert_eq!(p.entry[1], 1);
+    }
+}

cranelift/bforest/src/pool.rs

@@ -0,0 +1,220 @@
+//! B+-tree node pool.
+
+#[cfg(test)]
+use super::Comparator;
+use super::{Forest, Node, NodeData};
+use crate::entity::PrimaryMap;
+#[cfg(test)]
+use core::fmt;
+use core::ops::{Index, IndexMut};
+
+/// A pool of nodes, including a free list.
+pub(super) struct NodePool<F: Forest> {
+    nodes: PrimaryMap<Node, NodeData<F>>,
+    freelist: Option<Node>,
+}
+
+impl<F: Forest> NodePool<F> {
+    /// Allocate a new empty pool of nodes.
+    pub fn new() -> Self {
+        Self {
+            nodes: PrimaryMap::new(),
+            freelist: None,
+        }
+    }
+
+    /// Free all nodes.
+    pub fn clear(&mut self) {
+        self.nodes.clear();
+        self.freelist = None;
+    }
+
+    /// Allocate a new node containing `data`.
+    pub fn alloc_node(&mut self, data: NodeData<F>) -> Node {
+        debug_assert!(!data.is_free(), "can't allocate free node");
+        match self.freelist {
+            Some(node) => {
+                // Remove this node from the free list.
+                match self.nodes[node] {
+                    NodeData::Free { next } => self.freelist = next,
+                    _ => panic!("Invalid {} on free list", node),
+                }
+                self.nodes[node] = data;
+                node
+            }
+            None => {
+                // The free list is empty. Allocate a new node.
+                self.nodes.push(data)
+            }
+        }
+    }
+
+    /// Free a node.
+    pub fn free_node(&mut self, node: Node) {
+        // Quick check for a double free.
+        debug_assert!(!self.nodes[node].is_free(), "{} is already free", node);
+        self.nodes[node] = NodeData::Free {
+            next: self.freelist,
+        };
+        self.freelist = Some(node);
+    }
+
+    /// Free the entire tree rooted at `node`.
+    pub fn free_tree(&mut self, node: Node) {
+        if let NodeData::Inner { size, tree, .. } = self[node] {
+            // Note that we have to capture `tree` by value to avoid borrow checker trouble.
+            #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_range_loop))]
+            for i in 0..usize::from(size + 1) {
+                // Recursively free sub-trees. This recursion can never be deeper than `MAX_PATH`,
+                // and since most trees have less than a handful of nodes, it is worthwhile to
+                // avoid the heap allocation for an iterative tree traversal.
+                self.free_tree(tree[i]);
+            }
+        }
+        self.free_node(node);
+    }
+}
+
+#[cfg(test)]
+impl<F: Forest> NodePool<F> {
+    /// Verify the consistency of the tree rooted at `node`.
+    pub fn verify_tree<C: Comparator<F::Key>>(&self, node: Node, comp: &C)
+    where
+        NodeData<F>: fmt::Display,
+        F::Key: fmt::Display,
+    {
+        use crate::entity::EntitySet;
+        use alloc::vec::Vec;
+        use core::borrow::Borrow;
+        use core::cmp::Ordering;
+
+        // The root node can't be an inner node with just a single sub-tree. It should have been
+        // pruned.
+        if let NodeData::Inner { size, .. } = self[node] {
+            assert!(size > 0, "Root must have more than one sub-tree");
+        }
+
+        let mut done = match self[node] {
+            NodeData::Inner { size, .. } | NodeData::Leaf { size, .. } => {
+                EntitySet::with_capacity(size.into())
+            }
+            _ => EntitySet::new(),
+        };
+
+        let mut todo = Vec::new();
+
+        // Todo-list entries are:
+        // 1. Optional LHS key which must be <= all node entries.
+        // 2. The node reference.
+        // 3. Optional RHS key which must be > all node entries.
+        todo.push((None, node, None));
+
+        while let Some((lkey, node, rkey)) = todo.pop() {
+            assert!(done.insert(node), "Node appears more than once in tree");
+            let mut lower = lkey;
+
+            match self[node] {
+                NodeData::Inner { size, keys, tree } => {
+                    let size = size as usize;
+                    let capacity = tree.len();
+                    let keys = &keys[0..size];
+
+                    // Verify occupancy.
+                    // Right-most nodes can be small, but others must be at least half full.
+                    assert!(
+                        rkey.is_none() || (size + 1) * 2 >= capacity,
+                        "Only {}/{} entries in {}:{}, upper={}",
+                        size + 1,
+                        capacity,
+                        node,
+                        self[node],
+                        rkey.unwrap()
+                    );
+
+                    // Queue up the sub-trees, checking for duplicates.
+                    for i in 0..size + 1 {
+                        // Get an upper bound for node[i].
+                        let upper = keys.get(i).cloned().or(rkey);
+
+                        // Check that keys are strictly monotonic.
+                        if let (Some(a), Some(b)) = (lower, upper) {
+                            assert_eq!(
+                                comp.cmp(a, b),
+                                Ordering::Less,
+                                "Key order {} < {} failed in {}: {}",
+                                a,
+                                b,
+                                node,
+                                self[node]
+                            );
+                        }
+
+                        // Queue up the sub-tree.
+                        todo.push((lower, tree[i], upper));
+
+                        // Set a lower bound for the next tree.
+                        lower = upper;
+                    }
+                }
+                NodeData::Leaf { size, keys, .. } => {
+                    let size = size as usize;
+                    let capacity = keys.borrow().len();
+                    let keys = &keys.borrow()[0..size];
+
+                    // Verify occupancy.
+                    // Right-most nodes can be small, but others must be at least half full.
+                    assert!(size > 0, "Leaf {} is empty", node);
+                    assert!(
+                        rkey.is_none() || size * 2 >= capacity,
+                        "Only {}/{} entries in {}:{}, upper={}",
+                        size,
+                        capacity,
+                        node,
+                        self[node],
+                        rkey.unwrap()
+                    );
+
+                    for i in 0..size + 1 {
+                        let upper = keys.get(i).cloned().or(rkey);
+
+                        // Check that keys are strictly monotonic.
+                        if let (Some(a), Some(b)) = (lower, upper) {
+                            let wanted = if i == 0 {
+                                Ordering::Equal
+                            } else {
+                                Ordering::Less
+                            };
+                            assert_eq!(
+                                comp.cmp(a, b),
+                                wanted,
+                                "Key order for {} - {} failed in {}: {}",
+                                a,
+                                b,
+                                node,
+                                self[node]
+                            );
+                        }
+
+                        // Set a lower bound for the next key.
+                        lower = upper;
+                    }
+                }
+                NodeData::Free { .. } => panic!("Free {} reached", node),
+            }
+        }
+    }
+}
+
+impl<F: Forest> Index<Node> for NodePool<F> {
+    type Output = NodeData<F>;
+
+    fn index(&self, index: Node) -> &Self::Output {
+        self.nodes.index(index)
+    }
+}
+
+impl<F: Forest> IndexMut<Node> for NodePool<F> {
+    fn index_mut(&mut self, index: Node) -> &mut Self::Output {
+        self.nodes.index_mut(index)
+    }
+}

cranelift/bforest/src/set.rs

@@ -0,0 +1,598 @@
+//! Forest of sets.
+
+use super::{Comparator, Forest, Node, NodeData, NodePool, Path, SetValue, INNER_SIZE};
+use crate::packed_option::PackedOption;
+#[cfg(test)]
+use alloc::string::String;
+#[cfg(test)]
+use core::fmt;
+use core::marker::PhantomData;
+
+/// Tag type defining forest types for a set.
+struct SetTypes<K>(PhantomData<K>);
+
+impl<K> Forest for SetTypes<K>
+where
+    K: Copy,
+{
+    type Key = K;
+    type Value = SetValue;
+    type LeafKeys = [K; 2 * INNER_SIZE - 1];
+    type LeafValues = [SetValue; 2 * INNER_SIZE - 1];
+
+    fn splat_key(key: Self::Key) -> Self::LeafKeys {
+        [key; 2 * INNER_SIZE - 1]
+    }
+
+    fn splat_value(value: Self::Value) -> Self::LeafValues {
+        [value; 2 * INNER_SIZE - 1]
+    }
+}
+
+/// Memory pool for a forest of `Set` instances.
+pub struct SetForest<K>
+where
+    K: Copy,
+{
+    nodes: NodePool<SetTypes<K>>,
+}
+
+impl<K> SetForest<K>
+where
+    K: Copy,
+{
+    /// Create a new empty forest.
+    pub fn new() -> Self {
+        Self {
+            nodes: NodePool::new(),
+        }
+    }
+
+    /// Clear all sets in the forest.
+    ///
+    /// All `Set` instances belong to this forest are invalidated and should no longer be used.
+    pub fn clear(&mut self) {
+        self.nodes.clear();
+    }
+}
+
+/// B-tree representing an ordered set of `K`s using `C` for comparing elements.
+///
+/// This is not a general-purpose replacement for `BTreeSet`. See the [module
+/// documentation](index.html) for more information about design tradeoffs.
+///
+/// Sets can be cloned, but that operation should only be used as part of cloning the whole forest
+/// they belong to. *Cloning a set does not allocate new memory for the clone*. It creates an alias
+/// of the same memory.
+#[derive(Clone)]
+pub struct Set<K>
+where
+    K: Copy,
+{
+    root: PackedOption<Node>,
+    unused: PhantomData<K>,
+}
+
+impl<K> Set<K>
+where
+    K: Copy,
+{
+    /// Make an empty set.
+    pub fn new() -> Self {
+        Self {
+            root: None.into(),
+            unused: PhantomData,
+        }
+    }
+
+    /// Is this an empty set?
+    pub fn is_empty(&self) -> bool {
+        self.root.is_none()
+    }
+
+    /// Does the set contain `key`?.
+    pub fn contains<C: Comparator<K>>(&self, key: K, forest: &SetForest<K>, comp: &C) -> bool {
+        self.root
+            .expand()
+            .and_then(|root| Path::default().find(key, root, &forest.nodes, comp))
+            .is_some()
+    }
+
+    /// Try to insert `key` into the set.
+    ///
+    /// If the set did not contain `key`, insert it and return true.
+    ///
+    /// If `key` is already present, don't change the set and return false.
+    pub fn insert<C: Comparator<K>>(
+        &mut self,
+        key: K,
+        forest: &mut SetForest<K>,
+        comp: &C,
+    ) -> bool {
+        self.cursor(forest, comp).insert(key)
+    }
+
+    /// Remove `key` from the set and return true.
+    ///
+    /// If `key` was not present in the set, return false.
+    pub fn remove<C: Comparator<K>>(
+        &mut self,
+        key: K,
+        forest: &mut SetForest<K>,
+        comp: &C,
+    ) -> bool {
+        let mut c = self.cursor(forest, comp);
+        if c.goto(key) {
+            c.remove();
+            true
+        } else {
+            false
+        }
+    }
+
+    /// Remove all entries.
+    pub fn clear(&mut self, forest: &mut SetForest<K>) {
+        if let Some(root) = self.root.take() {
+            forest.nodes.free_tree(root);
+        }
+    }
+
+    /// Retains only the elements specified by the predicate.
+    ///
+    /// Remove all elements where the predicate returns false.
+    pub fn retain<F>(&mut self, forest: &mut SetForest<K>, mut predicate: F)
+    where
+        F: FnMut(K) -> bool,
+    {
+        let mut path = Path::default();
+        if let Some(root) = self.root.expand() {
+            path.first(root, &forest.nodes);
+        }
+        while let Some((node, entry)) = path.leaf_pos() {
+            if predicate(forest.nodes[node].unwrap_leaf().0[entry]) {
+                path.next(&forest.nodes);
+            } else {
+                self.root = path.remove(&mut forest.nodes).into();
+            }
+        }
+    }
+
+    /// Create a cursor for navigating this set. The cursor is initially positioned off the end of
+    /// the set.
+    pub fn cursor<'a, C: Comparator<K>>(
+        &'a mut self,
+        forest: &'a mut SetForest<K>,
+        comp: &'a C,
+    ) -> SetCursor<'a, K, C> {
+        SetCursor::new(self, forest, comp)
+    }
+
+    /// Create an iterator traversing this set. The iterator type is `K`.
+    pub fn iter<'a>(&'a self, forest: &'a SetForest<K>) -> SetIter<'a, K> {
+        SetIter {
+            root: self.root,
+            pool: &forest.nodes,
+            path: Path::default(),
+        }
+    }
+}
+
+impl<K> Default for Set<K>
+where
+    K: Copy,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// A position in a `Set` used to navigate and modify the ordered set.
+///
+/// A cursor always points at an element in the set, or "off the end" which is a position after the
+/// last element in the set.
+pub struct SetCursor<'a, K, C>
+where
+    K: 'a + Copy,
+    C: 'a + Comparator<K>,
+{
+    root: &'a mut PackedOption<Node>,
+    pool: &'a mut NodePool<SetTypes<K>>,
+    comp: &'a C,
+    path: Path<SetTypes<K>>,
+}
+
+impl<'a, K, C> SetCursor<'a, K, C>
+where
+    K: Copy,
+    C: Comparator<K>,
+{
+    /// Create a cursor with a default (invalid) location.
+    fn new(container: &'a mut Set<K>, forest: &'a mut SetForest<K>, comp: &'a C) -> Self {
+        Self {
+            root: &mut container.root,
+            pool: &mut forest.nodes,
+            comp,
+            path: Path::default(),
+        }
+    }
+
+    /// Is this cursor pointing to an empty set?
+    pub fn is_empty(&self) -> bool {
+        self.root.is_none()
+    }
+
+    /// Move cursor to the next element and return it.
+    ///
+    /// If the cursor reaches the end, return `None` and leave the cursor at the off-the-end
+    /// position.
+    #[cfg_attr(feature = "cargo-clippy", allow(clippy::should_implement_trait))]
+    pub fn next(&mut self) -> Option<K> {
+        self.path.next(self.pool).map(|(k, _)| k)
+    }
+
+    /// Move cursor to the previous element and return it.
+    ///
+    /// If the cursor is already pointing at the first element, leave it there and return `None`.
+    pub fn prev(&mut self) -> Option<K> {
+        self.root
+            .expand()
+            .and_then(|root| self.path.prev(root, self.pool).map(|(k, _)| k))
+    }
+
+    /// Get the current element, or `None` if the cursor is at the end.
+    pub fn elem(&self) -> Option<K> {
+        self.path
+            .leaf_pos()
+            .and_then(|(node, entry)| self.pool[node].unwrap_leaf().0.get(entry).cloned())
+    }
+
+    /// Move this cursor to `elem`.
+    ///
+    /// If `elem` is in the set, place the cursor at `elem` and return true.
+    ///
+    /// If `elem` is not in the set, place the cursor at the next larger element (or the end) and
+    /// return false.
+    pub fn goto(&mut self, elem: K) -> bool {
+        match self.root.expand() {
+            None => false,
+            Some(root) => {
+                if self.path.find(elem, root, self.pool, self.comp).is_some() {
+                    true
+                } else {
+                    self.path.normalize(self.pool);
+                    false
+                }
+            }
+        }
+    }
+
+    /// Move this cursor to the first element.
+    pub fn goto_first(&mut self) -> Option<K> {
+        self.root.map(|root| self.path.first(root, self.pool).0)
+    }
+
+    /// Try to insert `elem` into the set and leave the cursor at the inserted element.
+    ///
+    /// If the set did not contain `elem`, insert it and return true.
+    ///
+    /// If `elem` is already present, don't change the set, place the cursor at `goto(elem)`, and
+    /// return false.
+    pub fn insert(&mut self, elem: K) -> bool {
+        match self.root.expand() {
+            None => {
+                let root = self.pool.alloc_node(NodeData::leaf(elem, SetValue()));
+                *self.root = root.into();
+                self.path.set_root_node(root);
+                true
+            }
+            Some(root) => {
+                // TODO: Optimize the case where `self.path` is already at the correct insert pos.
+                if self.path.find(elem, root, self.pool, self.comp).is_none() {
+                    *self.root = self.path.insert(elem, SetValue(), self.pool).into();
+                    true
+                } else {
+                    false
+                }
+            }
+        }
+    }
+
+    /// Remove the current element (if any) and return it.
+    /// This advances the cursor to the next element after the removed one.
+    pub fn remove(&mut self) -> Option<K> {
+        let elem = self.elem();
+        if elem.is_some() {
+            *self.root = self.path.remove(self.pool).into();
+        }
+        elem
+    }
+}
+
+#[cfg(test)]
+impl<'a, K, C> SetCursor<'a, K, C>
+where
+    K: Copy + fmt::Display,
+    C: Comparator<K>,
+{
+    fn verify(&self) {
+        self.path.verify(self.pool);
+        self.root.map(|root| self.pool.verify_tree(root, self.comp));
+    }
+
+    /// Get a text version of the path to the current position.
+    fn tpath(&self) -> String {
+        use alloc::string::ToString;
+        self.path.to_string()
+    }
+}
+
+/// An iterator visiting the elements of a `Set`.
+pub struct SetIter<'a, K>
+where
+    K: 'a + Copy,
+{
+    root: PackedOption<Node>,
+    pool: &'a NodePool<SetTypes<K>>,
+    path: Path<SetTypes<K>>,
+}
+
+impl<'a, K> Iterator for SetIter<'a, K>
+where
+    K: 'a + Copy,
+{
+    type Item = K;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // We use `self.root` to indicate if we need to go to the first element. Reset to `None`
+        // once we've returned the first element. This also works for an empty tree since the
+        // `path.next()` call returns `None` when the path is empty. This also fuses the iterator.
+        match self.root.take() {
+            Some(root) => Some(self.path.first(root, self.pool).0),
+            None => self.path.next(self.pool).map(|(k, _)| k),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::NodeData;
+    use super::*;
+    use alloc::vec::Vec;
+    use core::mem;
+
+    #[test]
+    fn node_size() {
+        // check that nodes are cache line sized when keys are 32 bits.
+        type F = SetTypes<u32>;
+        assert_eq!(mem::size_of::<NodeData<F>>(), 64);
+    }
+
+    #[test]
+    fn empty() {
+        let mut f = SetForest::<u32>::new();
+        f.clear();
+
+        let mut s = Set::<u32>::new();
+        assert!(s.is_empty());
+        s.clear(&mut f);
+        assert!(!s.contains(7, &f, &()));
+
+        // Iterator for an empty set.
+        assert_eq!(s.iter(&f).next(), None);
+
+        s.retain(&mut f, |_| unreachable!());
+
+        let mut c = SetCursor::new(&mut s, &mut f, &());
+        c.verify();
+        assert_eq!(c.elem(), None);
+
+        assert_eq!(c.goto_first(), None);
+        assert_eq!(c.tpath(), "<empty path>");
+    }
+
+    #[test]
+    fn simple_cursor() {
+        let mut f = SetForest::<u32>::new();
+        let mut s = Set::<u32>::new();
+        let mut c = SetCursor::new(&mut s, &mut f, &());
+
+        assert!(c.insert(50));
+        c.verify();
+        assert_eq!(c.elem(), Some(50));
+
+        assert!(c.insert(100));
+        c.verify();
+        assert_eq!(c.elem(), Some(100));
+
+        assert!(c.insert(10));
+        c.verify();
+        assert_eq!(c.elem(), Some(10));
+
+        // Basic movement.
+        assert_eq!(c.next(), Some(50));
+        assert_eq!(c.next(), Some(100));
+        assert_eq!(c.next(), None);
+        assert_eq!(c.next(), None);
+        assert_eq!(c.prev(), Some(100));
+        assert_eq!(c.prev(), Some(50));
+        assert_eq!(c.prev(), Some(10));
+        assert_eq!(c.prev(), None);
+        assert_eq!(c.prev(), None);
+
+        assert!(c.goto(50));
+        assert_eq!(c.elem(), Some(50));
+        assert_eq!(c.remove(), Some(50));
+        c.verify();
+
+        assert_eq!(c.elem(), Some(100));
+        assert_eq!(c.remove(), Some(100));
+        c.verify();
+        assert_eq!(c.elem(), None);
+        assert_eq!(c.remove(), None);
+        c.verify();
+    }
+
+    #[test]
+    fn two_level_sparse_tree() {
+        let mut f = SetForest::<u32>::new();
+        let mut s = Set::<u32>::new();
+        let mut c = SetCursor::new(&mut s, &mut f, &());
+
+        // Insert enough elements that we get a two-level tree.
+        // Each leaf node holds 8 elements
+        assert!(c.is_empty());
+        for i in 0..50 {
+            assert!(c.insert(i));
+            assert_eq!(c.elem(), Some(i));
+        }
+        assert!(!c.is_empty());
+
+        assert_eq!(c.goto_first(), Some(0));
+        assert_eq!(c.tpath(), "node2[0]--node0[0]");
+
+        assert_eq!(c.prev(), None);
+        for i in 1..50 {
+            assert_eq!(c.next(), Some(i));
+        }
+        assert_eq!(c.next(), None);
+        for i in (0..50).rev() {
+            assert_eq!(c.prev(), Some(i));
+        }
+        assert_eq!(c.prev(), None);
+
+        assert!(c.goto(25));
+        for i in 25..50 {
+            assert_eq!(c.remove(), Some(i));
+            assert!(!c.is_empty());
+            c.verify();
+        }
+
+        for i in (0..25).rev() {
+            assert!(!c.is_empty());
+            assert_eq!(c.elem(), None);
+            assert_eq!(c.prev(), Some(i));
+            assert_eq!(c.remove(), Some(i));
+            c.verify();
+        }
+        assert_eq!(c.elem(), None);
+        assert!(c.is_empty());
+    }
+
+    #[test]
+    fn three_level_sparse_tree() {
+        let mut f = SetForest::<u32>::new();
+        let mut s = Set::<u32>::new();
+        let mut c = SetCursor::new(&mut s, &mut f, &());
+
+        // Insert enough elements that we get a 3-level tree.
+        // Each leaf node holds 8 elements when filled up sequentially.
+        // Inner nodes hold 8 node pointers.
+        assert!(c.is_empty());
+        for i in 0..150 {
+            assert!(c.insert(i));
+            assert_eq!(c.elem(), Some(i));
+        }
+        assert!(!c.is_empty());
+
+        assert!(c.goto(0));
+        assert_eq!(c.tpath(), "node11[0]--node2[0]--node0[0]");
+
+        assert_eq!(c.prev(), None);
+        for i in 1..150 {
+            assert_eq!(c.next(), Some(i));
+        }
+        assert_eq!(c.next(), None);
+        for i in (0..150).rev() {
+            assert_eq!(c.prev(), Some(i));
+        }
+        assert_eq!(c.prev(), None);
+
+        assert!(c.goto(125));
+        for i in 125..150 {
+            assert_eq!(c.remove(), Some(i));
+            assert!(!c.is_empty());
+            c.verify();
+        }
+
+        for i in (0..125).rev() {
+            assert!(!c.is_empty());
+            assert_eq!(c.elem(), None);
+            assert_eq!(c.prev(), Some(i));
+            assert_eq!(c.remove(), Some(i));
+            c.verify();
+        }
+        assert_eq!(c.elem(), None);
+        assert!(c.is_empty());
+    }
+
+    // Generate a densely populated 4-level tree.
+    //
+    // Level 1: 1 root
+    // Level 2: 8 inner
+    // Level 3: 64 inner
+    // Level 4: 512 leafs, up to 7680 elements
+    //
+    // A 3-level tree can hold at most 960 elements.
+    fn dense4l(f: &mut SetForest<i32>) -> Set<i32> {
+        f.clear();
+        let mut s = Set::new();
+
+        // Insert 400 elements in 7 passes over the range to avoid the half-full leaf node pattern
+        // that comes from sequential insertion. This will generate a normal leaf layer.
+        for n in 0..4000 {
+            assert!(s.insert((n * 7) % 4000, f, &()));
+        }
+        s
+    }
+
+    #[test]
+    fn four_level() {
+        let mut f = SetForest::<i32>::new();
+        let mut s = dense4l(&mut f);
+
+        assert_eq!(
+            s.iter(&f).collect::<Vec<_>>()[0..10],
+            [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+        );
+
+        let mut c = s.cursor(&mut f, &());
+
+        c.verify();
+
+        // Peel off a whole sub-tree of the root by deleting from the front.
+        // The 900 element is near the front of the second sub-tree.
+        assert!(c.goto(900));
+        assert_eq!(c.tpath(), "node48[1]--node47[0]--node26[0]--node20[4]");
+        assert!(c.goto(0));
+        for i in 0..900 {
+            assert!(!c.is_empty());
+            assert_eq!(c.remove(), Some(i));
+        }
+        c.verify();
+        assert_eq!(c.elem(), Some(900));
+
+        // Delete backwards from somewhere in the middle.
+        assert!(c.goto(3000));
+        for i in (2000..3000).rev() {
+            assert_eq!(c.prev(), Some(i));
+            assert_eq!(c.remove(), Some(i));
+            assert_eq!(c.elem(), Some(3000));
+        }
+        c.verify();
+
+        // Remove everything in a scattered manner, triggering many collapsing patterns.
+        for i in 0..4000 {
+            if c.goto((i * 7) % 4000) {
+                c.remove();
+            }
+        }
+        assert!(c.is_empty());
+    }
+
+    #[test]
+    fn four_level_clear() {
+        let mut f = SetForest::<i32>::new();
+        let mut s = dense4l(&mut f);
+        s.clear(&mut f);
+    }
+}

cranelift/codegen/Cargo.toml

@@ -0,0 +1,74 @@
+[package]
+authors = ["The Cranelift Project Developers"]
+name = "cranelift-codegen"
+version = "0.59.0"
+description = "Low-level code generator library"
+license = "Apache-2.0 WITH LLVM-exception"
+documentation = "https://cranelift.readthedocs.io/"
+repository = "https://github.com/bytecodealliance/cranelift"
+categories = ["no-std"]
+readme = "README.md"
+keywords = ["compile", "compiler", "jit"]
+build = "build.rs"
+edition = "2018"
+
+[dependencies]
+cranelift-codegen-shared = { path = "./shared", version = "0.59.0" }
+cranelift-entity = { path = "../entity", version = "0.59.0" }
+cranelift-bforest = { path = "../bforest", version = "0.59.0" }
+hashbrown = { version = "0.6", optional = true }
+target-lexicon = "0.10"
+log = { version = "0.4.6", default-features = false }
+serde = { version = "1.0.94", features = ["derive"], optional = true }
+gimli = { version = "0.20.0", default-features = false, features = ["write"], optional = true }
+smallvec = { version = "1.0.0" }
+thiserror = "1.0.4"
+byteorder = { version = "1.3.2", default-features = false }
+# It is a goal of the cranelift-codegen crate to have minimal external dependencies.
+# Please don't add any unless they are essential to the task of creating binary
+# machine code. Integration tests that need external dependencies can be
+# accomodated in `tests`.
+
+[build-dependencies]
+cranelift-codegen-meta = { path = "meta", version = "0.59.0" }
+
+[features]
+default = ["std", "unwind"]
+
+# The "std" feature enables use of libstd. The "core" feature enables use
+# of some minimal std-like replacement libraries. At least one of these two
+# features need to be enabled.
+std = []
+
+# The "core" features enables use of "hashbrown" since core doesn't have
+# a HashMap implementation, and a workaround for Cargo #4866.
+core = ["hashbrown"]
+
+# This enables some additional functions useful for writing tests, but which
+# can significantly increase the size of the library.
+testing_hooks = []
+
+# This enables unwind info generation functionality.
+unwind = ["gimli"]
+
+# ISA targets for which we should build.
+# If no ISA targets are explicitly enabled, the ISA target for the host machine is enabled.
+x86 = []
+arm32 = []
+arm64 = []
+riscv = []
+
+# Option to enable all architectures.
+all-arch = [
+    "x86",
+    "arm32",
+    "arm64",
+    "riscv"
+]
+
+# For dependent crates that want to serialize some parts of cranelift
+enable-serde = ["serde"]
+
+[badges]
+maintenance = { status = "experimental" }
+travis-ci = { repository = "bytecodealliance/cranelift" }

cranelift/codegen/LICENSE

@@ -0,0 +1,220 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+--- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+

cranelift/codegen/README.md

@@ -0,0 +1,2 @@
+This crate contains the core Cranelift code generator. It translates code from an
+intermediate representation into executable machine code.

cranelift/codegen/build.rs

@@ -0,0 +1,74 @@
+// Build script.
+//
+// This program is run by Cargo when building cranelift-codegen. It is used to generate Rust code from
+// the language definitions in the cranelift-codegen/meta directory.
+//
+// Environment:
+//
+// OUT_DIR
+//     Directory where generated files should be placed.
+//
+// TARGET
+//     Target triple provided by Cargo.
+//
+// The build script expects to be run from the directory where this build.rs file lives. The
+// current directory is used to find the sources.
+
+use cranelift_codegen_meta as meta;
+
+use std::env;
+use std::process;
+use std::time::Instant;
+
+fn main() {
+    let start_time = Instant::now();
+
+    let out_dir = env::var("OUT_DIR").expect("The OUT_DIR environment variable must be set");
+    let target_triple = env::var("TARGET").expect("The TARGET environment variable must be set");
+
+    // Configure isa targets cfg.
+    let isa_targets = meta::isa::Isa::all()
+        .iter()
+        .cloned()
+        .filter(|isa| {
+            let env_key = format!("CARGO_FEATURE_{}", isa.to_string().to_uppercase());
+            env::var(env_key).is_ok()
+        })
+        .collect::<Vec<_>>();
+
+    let isas = if isa_targets.is_empty() {
+        // Try to match native target.
+        let target_name = target_triple.split('-').next().unwrap();
+        let isa = meta::isa_from_arch(&target_name).expect("error when identifying target");
+        println!("cargo:rustc-cfg=feature=\"{}\"", isa);
+        vec![isa]
+    } else {
+        isa_targets
+    };
+
+    let cur_dir = env::current_dir().expect("Can't access current working directory");
+    let crate_dir = cur_dir.as_path();
+
+    // Make sure we rebuild if this build script changes (will not happen with
+    // if the path to this file contains non-UTF-8 bytes).
+    println!(
+        "cargo:rerun-if-changed={}",
+        crate_dir.join("build.rs").to_str().unwrap()
+    );
+
+    if let Err(err) = meta::generate(&isas, &out_dir) {
+        eprintln!("Error: {}", err);
+        process::exit(1);
+    }
+
+    if env::var("CRANELIFT_VERBOSE").is_ok() {
+        for isa in &isas {
+            println!("cargo:warning=Includes support for {} ISA", isa.to_string());
+        }
+        println!(
+            "cargo:warning=Build step took {:?}.",
+            Instant::now() - start_time
+        );
+        println!("cargo:warning=Generated files are in {}", out_dir);
+    }
+}

cranelift/codegen/meta/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "cranelift-codegen-meta"
+authors = ["The Cranelift Project Developers"]
+version = "0.59.0"
+description = "Metaprogram for cranelift-codegen code generator library"
+license = "Apache-2.0 WITH LLVM-exception"
+repository = "https://github.com/bytecodealliance/cranelift"
+readme = "README.md"
+edition = "2018"
+
+[dependencies]
+cranelift-codegen-shared = { path = "../shared", version = "0.59.0" }
+cranelift-entity = { path = "../../entity", version = "0.59.0" }
+
+[badges]
+maintenance = { status = "experimental" }
+travis-ci = { repository = "bytecodealliance/cranelift" }
+
+[package.metadata.docs.rs]
+rustdoc-args = [ "--document-private-items" ]

cranelift/codegen/meta/LICENSE

@@ -0,0 +1,220 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+--- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+

cranelift/codegen/meta/README.md

@@ -0,0 +1,2 @@
+This crate contains the metaprogram used by cranelift-codegen. It's not
+useful on its own.

cranelift/codegen/meta/src/cdsl/ast.rs

@@ -0,0 +1,753 @@
+use crate::cdsl::instructions::{InstSpec, Instruction, InstructionPredicate};
+use crate::cdsl::operands::{OperandKind, OperandKindFields};
+use crate::cdsl::types::ValueType;
+use crate::cdsl::typevar::{TypeSetBuilder, TypeVar};
+
+use cranelift_entity::{entity_impl, PrimaryMap, SparseMap, SparseMapValue};
+
+use std::fmt;
+use std::iter::IntoIterator;
+
+pub(crate) enum Expr {
+    Var(VarIndex),
+    Literal(Literal),
+}
+
+impl Expr {
+    pub fn maybe_literal(&self) -> Option<&Literal> {
+        match &self {
+            Expr::Literal(lit) => Some(lit),
+            _ => None,
+        }
+    }
+
+    pub fn maybe_var(&self) -> Option<VarIndex> {
+        if let Expr::Var(var) = &self {
+            Some(*var)
+        } else {
+            None
+        }
+    }
+
+    pub fn unwrap_var(&self) -> VarIndex {
+        self.maybe_var()
+            .expect("tried to unwrap a non-Var content in Expr::unwrap_var")
+    }
+
+    pub fn to_rust_code(&self, var_pool: &VarPool) -> String {
+        match self {
+            Expr::Var(var_index) => var_pool.get(*var_index).to_rust_code(),
+            Expr::Literal(literal) => literal.to_rust_code(),
+        }
+    }
+}
+
+/// An AST definition associates a set of variables with the values produced by an expression.
+pub(crate) struct Def {
+    pub apply: Apply,
+    pub defined_vars: Vec<VarIndex>,
+}
+
+impl Def {
+    pub fn to_comment_string(&self, var_pool: &VarPool) -> String {
+        let results = self
+            .defined_vars
+            .iter()
+            .map(|&x| var_pool.get(x).name.as_str())
+            .collect::<Vec<_>>();
+
+        let results = if results.len() == 1 {
+            results[0].to_string()
+        } else {
+            format!("({})", results.join(", "))
+        };
+
+        format!("{} := {}", results, self.apply.to_comment_string(var_pool))
+    }
+}
+
+pub(crate) struct DefPool {
+    pool: PrimaryMap<DefIndex, Def>,
+}
+
+impl DefPool {
+    pub fn new() -> Self {
+        Self {
+            pool: PrimaryMap::new(),
+        }
+    }
+    pub fn get(&self, index: DefIndex) -> &Def {
+        self.pool.get(index).unwrap()
+    }
+    pub fn next_index(&self) -> DefIndex {
+        self.pool.next_key()
+    }
+    pub fn create_inst(&mut self, apply: Apply, defined_vars: Vec<VarIndex>) -> DefIndex {
+        self.pool.push(Def {
+            apply,
+            defined_vars,
+        })
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub(crate) struct DefIndex(u32);
+entity_impl!(DefIndex);
+
+/// A definition which would lead to generate a block creation.
+#[derive(Clone)]
+pub(crate) struct Block {
+    /// Instruction index after which the block entry is set.
+    pub location: DefIndex,
+    /// Variable holding the new created block.
+    pub name: VarIndex,
+}
+
+pub(crate) struct BlockPool {
+    pool: SparseMap<DefIndex, Block>,
+}
+
+impl SparseMapValue<DefIndex> for Block {
+    fn key(&self) -> DefIndex {
+        self.location
+    }
+}
+
+impl BlockPool {
+    pub fn new() -> Self {
+        Self {
+            pool: SparseMap::new(),
+        }
+    }
+    pub fn get(&self, index: DefIndex) -> Option<&Block> {
+        self.pool.get(index)
+    }
+    pub fn create_block(&mut self, name: VarIndex, location: DefIndex) {
+        if self.pool.contains_key(location) {
+            panic!("Attempt to insert 2 blocks after the same instruction")
+        }
+        self.pool.insert(Block { location, name });
+    }
+    pub fn is_empty(&self) -> bool {
+        self.pool.is_empty()
+    }
+}
+
+// Implement IntoIterator such that we can iterate over blocks which are in the block pool.
+impl<'a> IntoIterator for &'a BlockPool {
+    type Item = <&'a SparseMap<DefIndex, Block> as IntoIterator>::Item;
+    type IntoIter = <&'a SparseMap<DefIndex, Block> as IntoIterator>::IntoIter;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.pool.into_iter()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) enum Literal {
+    /// A value of an enumerated immediate operand.
+    ///
+    /// Some immediate operand kinds like `intcc` and `floatcc` have an enumerated range of values
+    /// corresponding to a Rust enum type. An `Enumerator` object is an AST leaf node representing one
+    /// of the values.
+    Enumerator {
+        rust_type: &'static str,
+        value: &'static str,
+    },
+
+    /// A bitwise value of an immediate operand, used for bitwise exact floating point constants.
+    Bits { rust_type: &'static str, value: u64 },
+
+    /// A value of an integer immediate operand.
+    Int(i64),
+
+    /// A empty list of variable set of arguments.
+    EmptyVarArgs,
+}
+
+impl Literal {
+    pub fn enumerator_for(kind: &OperandKind, value: &'static str) -> Self {
+        let value = match &kind.fields {
+            OperandKindFields::ImmEnum(values) => values.get(value).unwrap_or_else(|| {
+                panic!(
+                    "nonexistent value '{}' in enumeration '{}'",
+                    value, kind.rust_type
+                )
+            }),
+            _ => panic!("enumerator is for enum values"),
+        };
+        Literal::Enumerator {
+            rust_type: kind.rust_type,
+            value,
+        }
+    }
+
+    pub fn bits(kind: &OperandKind, bits: u64) -> Self {
+        match kind.fields {
+            OperandKindFields::ImmValue => {}
+            _ => panic!("bits_of is for immediate scalar types"),
+        }
+        Literal::Bits {
+            rust_type: kind.rust_type,
+            value: bits,
+        }
+    }
+
+    pub fn constant(kind: &OperandKind, value: i64) -> Self {
+        match kind.fields {
+            OperandKindFields::ImmValue => {}
+            _ => panic!("constant is for immediate scalar types"),
+        }
+        Literal::Int(value)
+    }
+
+    pub fn empty_vararg() -> Self {
+        Literal::EmptyVarArgs
+    }
+
+    pub fn to_rust_code(&self) -> String {
+        match self {
+            Literal::Enumerator { rust_type, value } => format!("{}::{}", rust_type, value),
+            Literal::Bits { rust_type, value } => format!("{}::with_bits({:#x})", rust_type, value),
+            Literal::Int(val) => val.to_string(),
+            Literal::EmptyVarArgs => "&[]".into(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum PatternPosition {
+    Source,
+    Destination,
+}
+
+/// A free variable.
+///
+/// When variables are used in `XForms` with source and destination patterns, they are classified
+/// as follows:
+///
+/// Input values: Uses in the source pattern with no preceding def. These may appear as inputs in
+/// the destination pattern too, but no new inputs can be introduced.
+///
+/// Output values: Variables that are defined in both the source and destination pattern.  These
+/// values may have uses outside the source pattern, and the destination pattern must compute the
+/// same value.
+///
+/// Intermediate values: Values that are defined in the source pattern, but not in the destination
+/// pattern. These may have uses outside the source pattern, so the defining instruction can't be
+/// deleted immediately.
+///
+/// Temporary values are defined only in the destination pattern.
+pub(crate) struct Var {
+    pub name: String,
+
+    /// The `Def` defining this variable in a source pattern.
+    pub src_def: Option<DefIndex>,
+
+    /// The `Def` defining this variable in a destination pattern.
+    pub dst_def: Option<DefIndex>,
+
+    /// TypeVar representing the type of this variable.
+    type_var: Option<TypeVar>,
+
+    /// Is this the original type variable, or has it be redefined with set_typevar?
+    is_original_type_var: bool,
+}
+
+impl Var {
+    fn new(name: String) -> Self {
+        Self {
+            name,
+            src_def: None,
+            dst_def: None,
+            type_var: None,
+            is_original_type_var: false,
+        }
+    }
+
+    /// Is this an input value to the src pattern?
+    pub fn is_input(&self) -> bool {
+        self.src_def.is_none() && self.dst_def.is_none()
+    }
+
+    /// Is this an output value, defined in both src and dst patterns?
+    pub fn is_output(&self) -> bool {
+        self.src_def.is_some() && self.dst_def.is_some()
+    }
+
+    /// Is this an intermediate value, defined only in the src pattern?
+    pub fn is_intermediate(&self) -> bool {
+        self.src_def.is_some() && self.dst_def.is_none()
+    }
+
+    /// Is this a temp value, defined only in the dst pattern?
+    pub fn is_temp(&self) -> bool {
+        self.src_def.is_none() && self.dst_def.is_some()
+    }
+
+    /// Get the def of this variable according to the position.
+    pub fn get_def(&self, position: PatternPosition) -> Option<DefIndex> {
+        match position {
+            PatternPosition::Source => self.src_def,
+            PatternPosition::Destination => self.dst_def,
+        }
+    }
+
+    pub fn set_def(&mut self, position: PatternPosition, def: DefIndex) {
+        assert!(
+            self.get_def(position).is_none(),
+            format!("redefinition of variable {}", self.name)
+        );
+        match position {
+            PatternPosition::Source => {
+                self.src_def = Some(def);
+            }
+            PatternPosition::Destination => {
+                self.dst_def = Some(def);
+            }
+        }
+    }
+
+    /// Get the type variable representing the type of this variable.
+    pub fn get_or_create_typevar(&mut self) -> TypeVar {
+        match &self.type_var {
+            Some(tv) => tv.clone(),
+            None => {
+                // Create a new type var in which we allow all types.
+                let tv = TypeVar::new(
+                    format!("typeof_{}", self.name),
+                    format!("Type of the pattern variable {:?}", self),
+                    TypeSetBuilder::all(),
+                );
+                self.type_var = Some(tv.clone());
+                self.is_original_type_var = true;
+                tv
+            }
+        }
+    }
+    pub fn get_typevar(&self) -> Option<TypeVar> {
+        self.type_var.clone()
+    }
+    pub fn set_typevar(&mut self, tv: TypeVar) {
+        self.is_original_type_var = if let Some(previous_tv) = &self.type_var {
+            *previous_tv == tv
+        } else {
+            false
+        };
+        self.type_var = Some(tv);
+    }
+
+    /// Check if this variable has a free type variable. If not, the type of this variable is
+    /// computed from the type of another variable.
+    pub fn has_free_typevar(&self) -> bool {
+        match &self.type_var {
+            Some(tv) => tv.base.is_none() && self.is_original_type_var,
+            None => false,
+        }
+    }
+
+    pub fn to_rust_code(&self) -> String {
+        self.name.clone()
+    }
+    fn rust_type(&self) -> String {
+        self.type_var.as_ref().unwrap().to_rust_code()
+    }
+}
+
+impl fmt::Debug for Var {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        fmt.write_fmt(format_args!(
+            "Var({}{}{})",
+            self.name,
+            if self.src_def.is_some() { ", src" } else { "" },
+            if self.dst_def.is_some() { ", dst" } else { "" }
+        ))
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub(crate) struct VarIndex(u32);
+entity_impl!(VarIndex);
+
+pub(crate) struct VarPool {
+    pool: PrimaryMap<VarIndex, Var>,
+}
+
+impl VarPool {
+    pub fn new() -> Self {
+        Self {
+            pool: PrimaryMap::new(),
+        }
+    }
+    pub fn get(&self, index: VarIndex) -> &Var {
+        self.pool.get(index).unwrap()
+    }
+    pub fn get_mut(&mut self, index: VarIndex) -> &mut Var {
+        self.pool.get_mut(index).unwrap()
+    }
+    pub fn create(&mut self, name: impl Into<String>) -> VarIndex {
+        self.pool.push(Var::new(name.into()))
+    }
+}
+
+/// Contains constants created in the AST that must be inserted into the true [ConstantPool] when
+/// the legalizer code is generated. The constant data is named in the order it is inserted;
+/// inserting data using [insert] will avoid duplicates.
+///
+/// [ConstantPool]: ../../../cranelift_codegen/ir/constant/struct.ConstantPool.html
+/// [insert]: ConstPool::insert
+pub(crate) struct ConstPool {
+    pool: Vec<Vec<u8>>,
+}
+
+impl ConstPool {
+    /// Create an empty constant pool.
+    pub fn new() -> Self {
+        Self { pool: vec![] }
+    }
+
+    /// Create a name for a constant from its position in the pool.
+    fn create_name(position: usize) -> String {
+        format!("const{}", position)
+    }
+
+    /// Insert constant data into the pool, returning the name of the variable used to reference it.
+    /// This method will search for data that matches the new data and return the existing constant
+    /// name to avoid duplicates.
+    pub fn insert(&mut self, data: Vec<u8>) -> String {
+        let possible_position = self.pool.iter().position(|d| d == &data);
+        let position = if let Some(found_position) = possible_position {
+            found_position
+        } else {
+            let new_position = self.pool.len();
+            self.pool.push(data);
+            new_position
+        };
+        ConstPool::create_name(position)
+    }
+
+    /// Iterate over the name/value pairs in the pool.
+    pub fn iter(&self) -> impl Iterator<Item = (String, &Vec<u8>)> {
+        self.pool
+            .iter()
+            .enumerate()
+            .map(|(i, v)| (ConstPool::create_name(i), v))
+    }
+}
+
+/// Apply an instruction to arguments.
+///
+/// An `Apply` AST expression is created by using function call syntax on instructions. This
+/// applies to both bound and unbound polymorphic instructions.
+pub(crate) struct Apply {
+    pub inst: Instruction,
+    pub args: Vec<Expr>,
+    pub value_types: Vec<ValueType>,
+}
+
+impl Apply {
+    pub fn new(target: InstSpec, args: Vec<Expr>) -> Self {
+        let (inst, value_types) = match target {
+            InstSpec::Inst(inst) => (inst, Vec::new()),
+            InstSpec::Bound(bound_inst) => (bound_inst.inst, bound_inst.value_types),
+        };
+
+        // Apply should only operate on concrete value types, not "any".
+        let value_types = value_types
+            .into_iter()
+            .map(|vt| vt.expect("shouldn't be Any"))
+            .collect();
+
+        // Basic check on number of arguments.
+        assert!(
+            inst.operands_in.len() == args.len(),
+            format!("incorrect number of arguments in instruction {}", inst.name)
+        );
+
+        // Check that the kinds of Literals arguments match the expected operand.
+        for &imm_index in &inst.imm_opnums {
+            let arg = &args[imm_index];
+            if let Some(literal) = arg.maybe_literal() {
+                let op = &inst.operands_in[imm_index];
+                match &op.kind.fields {
+                    OperandKindFields::ImmEnum(values) => {
+                        if let Literal::Enumerator { value, .. } = literal {
+                            assert!(
+                                values.iter().any(|(_key, v)| v == value),
+                                "Nonexistent enum value '{}' passed to field of kind '{}' -- \
+                                 did you use the right enum?",
+                                value,
+                                op.kind.rust_type
+                            );
+                        } else {
+                            panic!(
+                                "Passed non-enum field value {:?} to field of kind {}",
+                                literal, op.kind.rust_type
+                            );
+                        }
+                    }
+                    OperandKindFields::ImmValue => match &literal {
+                        Literal::Enumerator { value, .. } => panic!(
+                            "Expected immediate value in immediate field of kind '{}', \
+                             obtained enum value '{}'",
+                            op.kind.rust_type, value
+                        ),
+                        Literal::Bits { .. } | Literal::Int(_) | Literal::EmptyVarArgs => {}
+                    },
+                    _ => {
+                        panic!(
+                            "Literal passed to non-literal field of kind {}",
+                            op.kind.rust_type
+                        );
+                    }
+                }
+            }
+        }
+
+        Self {
+            inst,
+            args,
+            value_types,
+        }
+    }
+
+    fn to_comment_string(&self, var_pool: &VarPool) -> String {
+        let args = self
+            .args
+            .iter()
+            .map(|arg| arg.to_rust_code(var_pool))
+            .collect::<Vec<_>>()
+            .join(", ");
+
+        let mut inst_and_bound_types = vec![self.inst.name.to_string()];
+        inst_and_bound_types.extend(self.value_types.iter().map(|vt| vt.to_string()));
+        let inst_name = inst_and_bound_types.join(".");
+
+        format!("{}({})", inst_name, args)
+    }
+
+    pub fn inst_predicate(&self, var_pool: &VarPool) -> InstructionPredicate {
+        let mut pred = InstructionPredicate::new();
+        for (format_field, &op_num) in self
+            .inst
+            .format
+            .imm_fields
+            .iter()
+            .zip(self.inst.imm_opnums.iter())
+        {
+            let arg = &self.args[op_num];
+            if arg.maybe_var().is_some() {
+                // Ignore free variables for now.
+                continue;
+            }
+            pred = pred.and(InstructionPredicate::new_is_field_equal_ast(
+                &*self.inst.format,
+                format_field,
+                arg.to_rust_code(var_pool),
+            ));
+        }
+
+        // Add checks for any bound secondary type variables.  We can't check the controlling type
+        // variable this way since it may not appear as the type of an operand.
+        if self.value_types.len() > 1 {
+            let poly = self
+                .inst
+                .polymorphic_info
+                .as_ref()
+                .expect("must have polymorphic info if it has bounded types");
+            for (bound_type, type_var) in
+                self.value_types[1..].iter().zip(poly.other_typevars.iter())
+            {
+                pred = pred.and(InstructionPredicate::new_typevar_check(
+                    &self.inst, type_var, bound_type,
+                ));
+            }
+        }
+
+        pred
+    }
+
+    /// Same as `inst_predicate()`, but also check the controlling type variable.
+    pub fn inst_predicate_with_ctrl_typevar(&self, var_pool: &VarPool) -> InstructionPredicate {
+        let mut pred = self.inst_predicate(var_pool);
+
+        if !self.value_types.is_empty() {
+            let bound_type = &self.value_types[0];
+            let poly = self.inst.polymorphic_info.as_ref().unwrap();
+            let type_check = if poly.use_typevar_operand {
+                InstructionPredicate::new_typevar_check(&self.inst, &poly.ctrl_typevar, bound_type)
+            } else {
+                InstructionPredicate::new_ctrl_typevar_check(&bound_type)
+            };
+            pred = pred.and(type_check);
+        }
+
+        pred
+    }
+
+    pub fn rust_builder(&self, defined_vars: &[VarIndex], var_pool: &VarPool) -> String {
+        let mut args = self
+            .args
+            .iter()
+            .map(|expr| expr.to_rust_code(var_pool))
+            .collect::<Vec<_>>()
+            .join(", ");
+
+        // Do we need to pass an explicit type argument?
+        if let Some(poly) = &self.inst.polymorphic_info {
+            if !poly.use_typevar_operand {
+                args = format!("{}, {}", var_pool.get(defined_vars[0]).rust_type(), args);
+            }
+        }
+
+        format!("{}({})", self.inst.snake_name(), args)
+    }
+}
+
+// Simple helpers for legalize actions construction.
+
+pub(crate) enum DummyExpr {
+    Var(DummyVar),
+    Literal(Literal),
+    Constant(DummyConstant),
+    Apply(InstSpec, Vec<DummyExpr>),
+    Block(DummyVar),
+}
+
+#[derive(Clone)]
+pub(crate) struct DummyVar {
+    pub name: String,
+}
+
+impl Into<DummyExpr> for DummyVar {
+    fn into(self) -> DummyExpr {
+        DummyExpr::Var(self)
+    }
+}
+impl Into<DummyExpr> for Literal {
+    fn into(self) -> DummyExpr {
+        DummyExpr::Literal(self)
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct DummyConstant(pub(crate) Vec<u8>);
+
+pub(crate) fn constant(data: Vec<u8>) -> DummyConstant {
+    DummyConstant(data)
+}
+
+impl Into<DummyExpr> for DummyConstant {
+    fn into(self) -> DummyExpr {
+        DummyExpr::Constant(self)
+    }
+}
+
+pub(crate) fn var(name: &str) -> DummyVar {
+    DummyVar {
+        name: name.to_owned(),
+    }
+}
+
+pub(crate) struct DummyDef {
+    pub expr: DummyExpr,
+    pub defined_vars: Vec<DummyVar>,
+}
+
+pub(crate) struct ExprBuilder {
+    expr: DummyExpr,
+}
+
+impl ExprBuilder {
+    pub fn apply(inst: InstSpec, args: Vec<DummyExpr>) -> Self {
+        let expr = DummyExpr::Apply(inst, args);
+        Self { expr }
+    }
+
+    pub fn assign_to(self, defined_vars: Vec<DummyVar>) -> DummyDef {
+        DummyDef {
+            expr: self.expr,
+            defined_vars,
+        }
+    }
+
+    pub fn block(name: DummyVar) -> Self {
+        let expr = DummyExpr::Block(name);
+        Self { expr }
+    }
+}
+
+macro_rules! def_rhs {
+    // inst(a, b, c)
+    ($inst:ident($($src:expr),*)) => {
+        ExprBuilder::apply($inst.into(), vec![$($src.clone().into()),*])
+    };
+
+    // inst.type(a, b, c)
+    ($inst:ident.$type:ident($($src:expr),*)) => {
+        ExprBuilder::apply($inst.bind($type).into(), vec![$($src.clone().into()),*])
+    };
+}
+
+// Helper macro to define legalization recipes.
+macro_rules! def {
+    // x = ...
+    ($dest:ident = $($tt:tt)*) => {
+        def_rhs!($($tt)*).assign_to(vec![$dest.clone()])
+    };
+
+    // (x, y, ...) = ...
+    (($($dest:ident),*) = $($tt:tt)*) => {
+        def_rhs!($($tt)*).assign_to(vec![$($dest.clone()),*])
+    };
+
+    // An instruction with no results.
+    ($($tt:tt)*) => {
+        def_rhs!($($tt)*).assign_to(Vec::new())
+    }
+}
+
+// Helper macro to define legalization recipes.
+macro_rules! block {
+    // a basic block definition, splitting the current block in 2.
+    ($block: ident) => {
+        ExprBuilder::block($block).assign_to(Vec::new())
+    };
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::cdsl::ast::ConstPool;
+
+    #[test]
+    fn const_pool_returns_var_names() {
+        let mut c = ConstPool::new();
+        assert_eq!(c.insert([0, 1, 2].to_vec()), "const0");
+        assert_eq!(c.insert([1, 2, 3].to_vec()), "const1");
+    }
+
+    #[test]
+    fn const_pool_avoids_duplicates() {
+        let data = [0, 1, 2].to_vec();
+        let mut c = ConstPool::new();
+        assert_eq!(c.pool.len(), 0);
+
+        assert_eq!(c.insert(data.clone()), "const0");
+        assert_eq!(c.pool.len(), 1);
+
+        assert_eq!(c.insert(data), "const0");
+        assert_eq!(c.pool.len(), 1);
+    }
+
+    #[test]
+    fn const_pool_iterates() {
+        let mut c = ConstPool::new();
+        c.insert([0, 1, 2].to_vec());
+        c.insert([3, 4, 5].to_vec());
+
+        let mut iter = c.iter();
+        assert_eq!(iter.next(), Some(("const0".to_owned(), &vec![0, 1, 2])));
+        assert_eq!(iter.next(), Some(("const1".to_owned(), &vec![3, 4, 5])));
+        assert_eq!(iter.next(), None);
+    }
+}

cranelift/codegen/meta/src/cdsl/cpu_modes.rs

@@ -0,0 +1,88 @@
+use std::collections::{hash_map, HashMap, HashSet};
+use std::iter::FromIterator;
+
+use crate::cdsl::encodings::Encoding;
+use crate::cdsl::types::{LaneType, ValueType};
+use crate::cdsl::xform::{TransformGroup, TransformGroupIndex};
+
+pub(crate) struct CpuMode {
+    pub name: &'static str,
+    default_legalize: Option<TransformGroupIndex>,
+    monomorphic_legalize: Option<TransformGroupIndex>,
+    typed_legalize: HashMap<ValueType, TransformGroupIndex>,
+    pub encodings: Vec<Encoding>,
+}
+
+impl CpuMode {
+    pub fn new(name: &'static str) -> Self {
+        Self {
+            name,
+            default_legalize: None,
+            monomorphic_legalize: None,
+            typed_legalize: HashMap::new(),
+            encodings: Vec::new(),
+        }
+    }
+
+    pub fn set_encodings(&mut self, encodings: Vec<Encoding>) {
+        assert!(self.encodings.is_empty(), "clobbering encodings");
+        self.encodings = encodings;
+    }
+
+    pub fn legalize_monomorphic(&mut self, group: &TransformGroup) {
+        assert!(self.monomorphic_legalize.is_none());
+        self.monomorphic_legalize = Some(group.id);
+    }
+    pub fn legalize_default(&mut self, group: &TransformGroup) {
+        assert!(self.default_legalize.is_none());
+        self.default_legalize = Some(group.id);
+    }
+    pub fn legalize_value_type(&mut self, lane_type: impl Into<ValueType>, group: &TransformGroup) {
+        assert!(self
+            .typed_legalize
+            .insert(lane_type.into(), group.id)
+            .is_none());
+    }
+    pub fn legalize_type(&mut self, lane_type: impl Into<LaneType>, group: &TransformGroup) {
+        assert!(self
+            .typed_legalize
+            .insert(lane_type.into().into(), group.id)
+            .is_none());
+    }
+
+    pub fn get_default_legalize_code(&self) -> TransformGroupIndex {
+        self.default_legalize
+            .expect("a finished CpuMode must have a default legalize code")
+    }
+    pub fn get_legalize_code_for(&self, typ: &Option<ValueType>) -> TransformGroupIndex {
+        match typ {
+            Some(typ) => self
+                .typed_legalize
+                .get(typ)
+                .copied()
+                .unwrap_or_else(|| self.get_default_legalize_code()),
+            None => self
+                .monomorphic_legalize
+                .unwrap_or_else(|| self.get_default_legalize_code()),
+        }
+    }
+    pub fn get_legalized_types(&self) -> hash_map::Keys<ValueType, TransformGroupIndex> {
+        self.typed_legalize.keys()
+    }
+
+    /// Returns a deterministically ordered, deduplicated list of TransformGroupIndex for the directly
+    /// reachable set of TransformGroup this TargetIsa uses.
+    pub fn direct_transform_groups(&self) -> Vec<TransformGroupIndex> {
+        let mut set = HashSet::new();
+        if let Some(i) = &self.default_legalize {
+            set.insert(*i);
+        }
+        if let Some(i) = &self.monomorphic_legalize {
+            set.insert(*i);
+        }
+        set.extend(self.typed_legalize.values().cloned());
+        let mut ret = Vec::from_iter(set);
+        ret.sort();
+        ret
+    }
+}

cranelift/codegen/meta/src/cdsl/encodings.rs

@@ -0,0 +1,179 @@
+use crate::cdsl::instructions::{
+    InstSpec, Instruction, InstructionPredicate, InstructionPredicateNode,
+    InstructionPredicateNumber, InstructionPredicateRegistry, ValueTypeOrAny,
+};
+use crate::cdsl::recipes::{EncodingRecipeNumber, Recipes};
+use crate::cdsl::settings::SettingPredicateNumber;
+use crate::cdsl::types::ValueType;
+use std::rc::Rc;
+use std::string::ToString;
+
+/// Encoding for a concrete instruction.
+///
+/// An `Encoding` object ties an instruction opcode with concrete type variables together with an
+/// encoding recipe and encoding encbits.
+///
+/// The concrete instruction can be in three different forms:
+///
+/// 1. A naked opcode: `trap` for non-polymorphic instructions.
+/// 2. With bound type variables: `iadd.i32` for polymorphic instructions.
+/// 3. With operands providing constraints: `icmp.i32(intcc.eq, x, y)`.
+///
+/// If the instruction is polymorphic, all type variables must be provided.
+pub(crate) struct EncodingContent {
+    /// The `Instruction` or `BoundInstruction` being encoded.
+    inst: InstSpec,
+
+    /// The `EncodingRecipe` to use.
+    pub recipe: EncodingRecipeNumber,
+
+    /// Additional encoding bits to be interpreted by `recipe`.
+    pub encbits: u16,
+
+    /// An instruction predicate that must be true to allow selecting this encoding.
+    pub inst_predicate: Option<InstructionPredicateNumber>,
+
+    /// An ISA predicate that must be true to allow selecting this encoding.
+    pub isa_predicate: Option<SettingPredicateNumber>,
+
+    /// The value type this encoding has been bound to, for encodings of polymorphic instructions.
+    pub bound_type: Option<ValueType>,
+}
+
+impl EncodingContent {
+    pub fn inst(&self) -> &Instruction {
+        self.inst.inst()
+    }
+    pub fn to_rust_comment(&self, recipes: &Recipes) -> String {
+        format!("[{}#{:02x}]", recipes[self.recipe].name, self.encbits)
+    }
+}
+
+pub(crate) type Encoding = Rc<EncodingContent>;
+
+pub(crate) struct EncodingBuilder {
+    inst: InstSpec,
+    recipe: EncodingRecipeNumber,
+    encbits: u16,
+    inst_predicate: Option<InstructionPredicate>,
+    isa_predicate: Option<SettingPredicateNumber>,
+    bound_type: Option<ValueType>,
+}
+
+impl EncodingBuilder {
+    pub fn new(inst: InstSpec, recipe: EncodingRecipeNumber, encbits: u16) -> Self {
+        let (inst_predicate, bound_type) = match &inst {
+            InstSpec::Bound(inst) => {
+                let other_typevars = &inst.inst.polymorphic_info.as_ref().unwrap().other_typevars;
+
+                assert_eq!(
+                    inst.value_types.len(),
+                    other_typevars.len() + 1,
+                    "partially bound polymorphic instruction"
+                );
+
+                // Add secondary type variables to the instruction predicate.
+                let value_types = &inst.value_types;
+                let mut inst_predicate: Option<InstructionPredicate> = None;
+                for (typevar, value_type) in other_typevars.iter().zip(value_types.iter().skip(1)) {
+                    let value_type = match value_type {
+                        ValueTypeOrAny::Any => continue,
+                        ValueTypeOrAny::ValueType(vt) => vt,
+                    };
+                    let type_predicate =
+                        InstructionPredicate::new_typevar_check(&inst.inst, typevar, value_type);
+                    inst_predicate = Some(type_predicate.into());
+                }
+
+                // Add immediate value predicates
+                for (immediate_value, immediate_operand) in inst
+                    .immediate_values
+                    .iter()
+                    .zip(inst.inst.operands_in.iter().filter(|o| o.is_immediate()))
+                {
+                    let immediate_predicate = InstructionPredicate::new_is_field_equal(
+                        &inst.inst.format,
+                        immediate_operand.kind.rust_field_name,
+                        immediate_value.to_string(),
+                    );
+                    inst_predicate = if let Some(type_predicate) = inst_predicate {
+                        Some(type_predicate.and(immediate_predicate))
+                    } else {
+                        Some(immediate_predicate.into())
+                    }
+                }
+
+                let ctrl_type = value_types[0]
+                    .clone()
+                    .expect("Controlling type shouldn't be Any");
+                (inst_predicate, Some(ctrl_type))
+            }
+
+            InstSpec::Inst(inst) => {
+                assert!(
+                    inst.polymorphic_info.is_none(),
+                    "unbound polymorphic instruction"
+                );
+                (None, None)
+            }
+        };
+
+        Self {
+            inst,
+            recipe,
+            encbits,
+            inst_predicate,
+            isa_predicate: None,
+            bound_type,
+        }
+    }
+
+    pub fn inst_predicate(mut self, inst_predicate: InstructionPredicateNode) -> Self {
+        let inst_predicate = Some(match self.inst_predicate {
+            Some(node) => node.and(inst_predicate),
+            None => inst_predicate.into(),
+        });
+        self.inst_predicate = inst_predicate;
+        self
+    }
+
+    pub fn isa_predicate(mut self, isa_predicate: SettingPredicateNumber) -> Self {
+        assert!(self.isa_predicate.is_none());
+        self.isa_predicate = Some(isa_predicate);
+        self
+    }
+
+    pub fn build(
+        self,
+        recipes: &Recipes,
+        inst_pred_reg: &mut InstructionPredicateRegistry,
+    ) -> Encoding {
+        let inst_predicate = self.inst_predicate.map(|pred| inst_pred_reg.insert(pred));
+
+        let inst = self.inst.inst();
+        assert!(
+            Rc::ptr_eq(&inst.format, &recipes[self.recipe].format),
+            format!(
+                "Inst {} and recipe {} must have the same format!",
+                inst.name, recipes[self.recipe].name
+            )
+        );
+
+        assert_eq!(
+            inst.is_branch && !inst.is_indirect_branch,
+            recipes[self.recipe].branch_range.is_some(),
+            "Inst {}'s is_branch contradicts recipe {} branch_range!",
+            inst.name,
+            recipes[self.recipe].name
+        );
+
+        Rc::new(EncodingContent {
+            inst: self.inst,
+            recipe: self.recipe,
+            encbits: self.encbits,
+            inst_predicate,
+            isa_predicate: self.isa_predicate,
+            bound_type: self.bound_type,
+        })
+    }
+}

cranelift/codegen/meta/src/cdsl/formats.rs

@@ -0,0 +1,171 @@
+use crate::cdsl::operands::OperandKind;
+use std::fmt;
+use std::rc::Rc;
+
+/// An immediate field in an instruction format.
+///
+/// This corresponds to a single member of a variant of the `InstructionData`
+/// data type.
+#[derive(Debug)]
+pub(crate) struct FormatField {
+    /// Immediate operand kind.
+    pub kind: OperandKind,
+
+    /// Member name in InstructionData variant.
+    pub member: &'static str,
+}
+
+/// Every instruction opcode has a corresponding instruction format which determines the number of
+/// operands and their kinds. Instruction formats are identified structurally, i.e., the format of
+/// an instruction is derived from the kinds of operands used in its declaration.
+///
+/// The instruction format stores two separate lists of operands: Immediates and values. Immediate
+/// operands (including entity references) are represented as explicit members in the
+/// `InstructionData` variants. The value operands are stored differently, depending on how many
+/// there are.  Beyond a certain point, instruction formats switch to an external value list for
+/// storing value arguments. Value lists can hold an arbitrary number of values.
+///
+/// All instruction formats must be predefined in the meta shared/formats.rs module.
+#[derive(Debug)]
+pub(crate) struct InstructionFormat {
+    /// Instruction format name in CamelCase. This is used as a Rust variant name in both the
+    /// `InstructionData` and `InstructionFormat` enums.
+    pub name: &'static str,
+
+    pub num_value_operands: usize,
+
+    pub has_value_list: bool,
+
+    pub imm_fields: Vec<FormatField>,
+
+    /// Index of the value input operand that is used to infer the controlling type variable. By
+    /// default, this is `0`, the first `value` operand. The index is relative to the values only,
+    /// ignoring immediate operands.
+    pub typevar_operand: Option<usize>,
+}
+
+/// A tuple serving as a key to deduplicate InstructionFormat.
+#[derive(Hash, PartialEq, Eq)]
+pub(crate) struct FormatStructure {
+    pub num_value_operands: usize,
+    pub has_value_list: bool,
+    /// Tuples of (Rust field name / Rust type) for each immediate field.
+    pub imm_field_names: Vec<(&'static str, &'static str)>,
+}
+
+impl fmt::Display for InstructionFormat {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+        let imm_args = self
+            .imm_fields
+            .iter()
+            .map(|field| format!("{}: {}", field.member, field.kind.rust_type))
+            .collect::<Vec<_>>()
+            .join(", ");
+        fmt.write_fmt(format_args!(
+            "{}(imms=({}), vals={})",
+            self.name, imm_args, self.num_value_operands
+        ))?;
+        Ok(())
+    }
+}
+
+impl InstructionFormat {
+    pub fn imm_by_name(&self, name: &'static str) -> &FormatField {
+        self.imm_fields
+            .iter()
+            .find(|&field| field.member == name)
+            .unwrap_or_else(|| {
+                panic!(
+                    "unexpected immediate field named {} in instruction format {}",
+                    name, self.name
+                )
+            })
+    }
+
+    /// Returns a tuple that uniquely identifies the structure.
+    pub fn structure(&self) -> FormatStructure {
+        FormatStructure {
+            num_value_operands: self.num_value_operands,
+            has_value_list: self.has_value_list,
+            imm_field_names: self
+                .imm_fields
+                .iter()
+                .map(|field| (field.kind.rust_field_name, field.kind.rust_type))
+                .collect::<Vec<_>>(),
+        }
+    }
+}
+
+pub(crate) struct InstructionFormatBuilder {
+    name: &'static str,
+    num_value_operands: usize,
+    has_value_list: bool,
+    imm_fields: Vec<FormatField>,
+    typevar_operand: Option<usize>,
+}
+
+impl InstructionFormatBuilder {
+    pub fn new(name: &'static str) -> Self {
+        Self {
+            name,
+            num_value_operands: 0,
+            has_value_list: false,
+            imm_fields: Vec::new(),
+            typevar_operand: None,
+        }
+    }
+
+    pub fn value(mut self) -> Self {
+        self.num_value_operands += 1;
+        self
+    }
+
+    pub fn varargs(mut self) -> Self {
+        self.has_value_list = true;
+        self
+    }
+
+    pub fn imm(mut self, operand_kind: &OperandKind) -> Self {
+        let field = FormatField {
+            kind: operand_kind.clone(),
+            member: operand_kind.rust_field_name,
+        };
+        self.imm_fields.push(field);
+        self
+    }
+
+    pub fn imm_with_name(mut self, member: &'static str, operand_kind: &OperandKind) -> Self {
+        let field = FormatField {
+            kind: operand_kind.clone(),
+            member,
+        };
+        self.imm_fields.push(field);
+        self
+    }
+
+    pub fn typevar_operand(mut self, operand_index: usize) -> Self {
+        assert!(self.typevar_operand.is_none());
+        assert!(self.has_value_list || operand_index < self.num_value_operands);
+        self.typevar_operand = Some(operand_index);
+        self
+    }
+
+    pub fn build(self) -> Rc<InstructionFormat> {
+        let typevar_operand = if self.typevar_operand.is_some() {
+            self.typevar_operand
+        } else if self.has_value_list || self.num_value_operands > 0 {
+            // Default to the first value operand, if there's one.
+            Some(0)
+        } else {
+            None
+        };
+
+        Rc::new(InstructionFormat {
+            name: self.name,
+            num_value_operands: self.num_value_operands,
+            has_value_list: self.has_value_list,
+            imm_fields: self.imm_fields,
+            typevar_operand,
+        })
+    }
+}

cranelift/codegen/meta/src/cdsl/isa.rs

@@ -0,0 +1,99 @@
+use std::collections::HashSet;
+use std::iter::FromIterator;
+
+use crate::cdsl::cpu_modes::CpuMode;
+use crate::cdsl::instructions::{InstructionGroup, InstructionPredicateMap};
+use crate::cdsl::recipes::Recipes;
+use crate::cdsl::regs::IsaRegs;
+use crate::cdsl::settings::SettingGroup;
+use crate::cdsl::xform::{TransformGroupIndex, TransformGroups};
+
+pub(crate) struct TargetIsa {
+    pub name: &'static str,
+    pub instructions: InstructionGroup,
+    pub settings: SettingGroup,
+    pub regs: IsaRegs,
+    pub recipes: Recipes,
+    pub cpu_modes: Vec<CpuMode>,
+    pub encodings_predicates: InstructionPredicateMap,
+
+    /// TransformGroupIndex are global to all the ISAs, while we want to have indices into the
+    /// local array of transform groups that are directly used. We use this map to get this
+    /// information.
+    pub local_transform_groups: Vec<TransformGroupIndex>,
+}
+
+impl TargetIsa {
+    pub fn new(
+        name: &'static str,
+        instructions: InstructionGroup,
+        settings: SettingGroup,
+        regs: IsaRegs,
+        recipes: Recipes,
+        cpu_modes: Vec<CpuMode>,
+        encodings_predicates: InstructionPredicateMap,
+    ) -> Self {
+        // Compute the local TransformGroup index.
+        let mut local_transform_groups = Vec::new();
+        for cpu_mode in &cpu_modes {
+            let transform_groups = cpu_mode.direct_transform_groups();
+            for group_index in transform_groups {
+                // find() is fine here: the number of transform group is < 5 as of June 2019.
+                if local_transform_groups
+                    .iter()
+                    .find(|&val| group_index == *val)
+                    .is_none()
+                {
+                    local_transform_groups.push(group_index);
+                }
+            }
+        }
+
+        Self {
+            name,
+            instructions,
+            settings,
+            regs,
+            recipes,
+            cpu_modes,
+            encodings_predicates,
+            local_transform_groups,
+        }
+    }
+
+    /// Returns a deterministically ordered, deduplicated list of TransformGroupIndex for the
+    /// transitive set of TransformGroup this TargetIsa uses.
+    pub fn transitive_transform_groups(
+        &self,
+        all_groups: &TransformGroups,
+    ) -> Vec<TransformGroupIndex> {
+        let mut set = HashSet::new();
+
+        for &root in self.local_transform_groups.iter() {
+            set.insert(root);
+            let mut base = root;
+            // Follow the chain of chain_with.
+            while let Some(chain_with) = &all_groups.get(base).chain_with {
+                set.insert(*chain_with);
+                base = *chain_with;
+            }
+        }
+
+        let mut vec = Vec::from_iter(set);
+        vec.sort();
+        vec
+    }
+
+    /// Returns a deterministically ordered, deduplicated list of TransformGroupIndex for the directly
+    /// reachable set of TransformGroup this TargetIsa uses.
+    pub fn direct_transform_groups(&self) -> &Vec<TransformGroupIndex> {
+        &self.local_transform_groups
+    }
+
+    pub fn translate_group_index(&self, group_index: TransformGroupIndex) -> usize {
+        self.local_transform_groups
+            .iter()
+            .position(|&val| val == group_index)
+            .expect("TransformGroup unused by this TargetIsa!")
+    }
+}

cranelift/codegen/meta/src/cdsl/mod.rs

@@ -0,0 +1,89 @@
+//! Cranelift DSL classes.
+//!
+//! This module defines the classes that are used to define Cranelift
+//! instructions and other entities.
+
+#[macro_use]
+pub mod ast;
+pub mod cpu_modes;
+pub mod encodings;
+pub mod formats;
+pub mod instructions;
+pub mod isa;
+pub mod operands;
+pub mod recipes;
+pub mod regs;
+pub mod settings;
+pub mod type_inference;
+pub mod types;
+pub mod typevar;
+pub mod xform;
+
+/// A macro that converts boolean settings into predicates to look more natural.
+#[macro_export]
+macro_rules! predicate {
+    ($a:ident && $($b:tt)*) => {
+        PredicateNode::And(Box::new($a.into()), Box::new(predicate!($($b)*)))
+    };
+    (!$a:ident && $($b:tt)*) => {
+        PredicateNode::And(
+            Box::new(PredicateNode::Not(Box::new($a.into()))),
+            Box::new(predicate!($($b)*))
+        )
+    };
+    (!$a:ident) => {
+        PredicateNode::Not(Box::new($a.into()))
+    };
+    ($a:ident) => {
+        $a.into()
+    };
+}
+
+/// A macro that joins boolean settings into a list (e.g. `preset!(feature_a && feature_b)`).
+#[macro_export]
+macro_rules! preset {
+    () => {
+        vec![]
+    };
+    ($($x:ident)&&*) => {
+        {
+            let mut v = Vec::new();
+            $(
+                v.push($x.into());
+            )*
+            v
+        }
+    };
+}
+
+/// Convert the string `s` to CamelCase.
+pub fn camel_case(s: &str) -> String {
+    let mut output_chars = String::with_capacity(s.len());
+
+    let mut capitalize = true;
+    for curr_char in s.chars() {
+        if curr_char == '_' {
+            capitalize = true;
+        } else {
+            if capitalize {
+                output_chars.extend(curr_char.to_uppercase());
+            } else {
+                output_chars.push(curr_char);
+            }
+            capitalize = false;
+        }
+    }
+
+    output_chars
+}
+
+#[cfg(test)]
+mod tests {
+    use super::camel_case;
+
+    #[test]
+    fn camel_case_works() {
+        assert_eq!(camel_case("x"), "X");
+        assert_eq!(camel_case("camel_case"), "CamelCase");
+    }
+}

cranelift/codegen/meta/src/cdsl/operands.rs

@@ -0,0 +1,173 @@
+use std::collections::HashMap;
+
+use crate::cdsl::typevar::TypeVar;
+
+/// An instruction operand can be an *immediate*, an *SSA value*, or an *entity reference*. The
+/// type of the operand is one of:
+///
+/// 1. A `ValueType` instance indicates an SSA value operand with a concrete type.
+///
+/// 2. A `TypeVar` instance indicates an SSA value operand, and the instruction is polymorphic over
+///    the possible concrete types that the type variable can assume.
+///
+/// 3. An `ImmediateKind` instance indicates an immediate operand whose value is encoded in the
+///    instruction itself rather than being passed as an SSA value.
+///
+/// 4. An `EntityRefKind` instance indicates an operand that references another entity in the
+///    function, typically something declared in the function preamble.
+#[derive(Clone, Debug)]
+pub(crate) struct Operand {
+    /// Name of the operand variable, as it appears in function parameters, legalizations, etc.
+    pub name: &'static str,
+
+    /// Type of the operand.
+    pub kind: OperandKind,
+
+    doc: Option<&'static str>,
+}
+
+impl Operand {
+    pub fn new(name: &'static str, kind: impl Into<OperandKind>) -> Self {
+        Self {
+            name,
+            doc: None,
+            kind: kind.into(),
+        }
+    }
+    pub fn with_doc(mut self, doc: &'static str) -> Self {
+        self.doc = Some(doc);
+        self
+    }
+
+    pub fn doc(&self) -> Option<&str> {
+        if let Some(doc) = &self.doc {
+            return Some(doc);
+        }
+        match &self.kind.fields {
+            OperandKindFields::TypeVar(tvar) => Some(&tvar.doc),
+            _ => self.kind.doc(),
+        }
+    }
+
+    pub fn is_value(&self) -> bool {
+        match self.kind.fields {
+            OperandKindFields::TypeVar(_) => true,
+            _ => false,
+        }
+    }
+
+    pub fn type_var(&self) -> Option<&TypeVar> {
+        match &self.kind.fields {
+            OperandKindFields::TypeVar(typevar) => Some(typevar),
+            _ => None,
+        }
+    }
+
+    pub fn is_varargs(&self) -> bool {
+        match self.kind.fields {
+            OperandKindFields::VariableArgs => true,
+            _ => false,
+        }
+    }
+
+    /// Returns true if the operand has an immediate kind or is an EntityRef.
+    pub fn is_immediate_or_entityref(&self) -> bool {
+        match self.kind.fields {
+            OperandKindFields::ImmEnum(_)
+            | OperandKindFields::ImmValue
+            | OperandKindFields::EntityRef => true,
+            _ => false,
+        }
+    }
+
+    /// Returns true if the operand has an immediate kind.
+    pub fn is_immediate(&self) -> bool {
+        match self.kind.fields {
+            OperandKindFields::ImmEnum(_) | OperandKindFields::ImmValue => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_cpu_flags(&self) -> bool {
+        match &self.kind.fields {
+            OperandKindFields::TypeVar(type_var)
+                if type_var.name == "iflags" || type_var.name == "fflags" =>
+            {
+                true
+            }
+            _ => false,
+        }
+    }
+}
+
+pub type EnumValues = HashMap<&'static str, &'static str>;
+
+#[derive(Clone, Debug)]
+pub(crate) enum OperandKindFields {
+    EntityRef,
+    VariableArgs,
+    ImmValue,
+    ImmEnum(EnumValues),
+    TypeVar(TypeVar),
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct OperandKind {
+    /// String representation of the Rust type mapping to this OperandKind.
+    pub rust_type: &'static str,
+
+    /// Name of this OperandKind in the format's member field.
+    pub rust_field_name: &'static str,
+
+    /// Type-specific fields for this OperandKind.
+    pub fields: OperandKindFields,
+
+    doc: Option<&'static str>,
+}
+
+impl OperandKind {
+    pub fn new(
+        rust_field_name: &'static str,
+        rust_type: &'static str,
+        fields: OperandKindFields,
+    ) -> Self {
+        Self {
+            rust_field_name,
+            rust_type,
+            fields,
+            doc: None,
+        }
+    }
+    pub fn with_doc(mut self, doc: &'static str) -> Self {
+        assert!(self.doc.is_none());
+        self.doc = Some(doc);
+        self
+    }
+    fn doc(&self) -> Option<&str> {
+        if let Some(doc) = &self.doc {
+            return Some(doc);
+        }
+        match &self.fields {
+            OperandKindFields::TypeVar(type_var) => Some(&type_var.doc),
+            OperandKindFields::ImmEnum(_)
+            | OperandKindFields::ImmValue
+            | OperandKindFields::EntityRef
+            | OperandKindFields::VariableArgs => None,
+        }
+    }
+}
+
+impl Into<OperandKind> for &TypeVar {
+    fn into(self) -> OperandKind {
+        OperandKind::new(
+            "value",
+            "ir::Value",
+            OperandKindFields::TypeVar(self.into()),
+        )
+    }
+}
+impl Into<OperandKind> for &OperandKind {
+    fn into(self) -> OperandKind {
+        self.clone()
+    }
+}

cranelift/codegen/meta/src/cdsl/recipes.rs

@@ -0,0 +1,298 @@
+use std::rc::Rc;
+
+use cranelift_entity::{entity_impl, PrimaryMap};
+
+use crate::cdsl::formats::InstructionFormat;
+use crate::cdsl::instructions::InstructionPredicate;
+use crate::cdsl::regs::RegClassIndex;
+use crate::cdsl::settings::SettingPredicateNumber;
+
+/// A specific register in a register class.
+///
+/// A register is identified by the top-level register class it belongs to and
+/// its first register unit.
+///
+/// Specific registers are used to describe constraints on instructions where
+/// some operands must use a fixed register.
+///
+/// Register instances can be created with the constructor, or accessed as
+/// attributes on the register class: `GPR.rcx`.
+#[derive(Copy, Clone, Hash, PartialEq, Eq)]
+pub(crate) struct Register {
+    pub regclass: RegClassIndex,
+    pub unit: u8,
+}
+
+impl Register {
+    pub fn new(regclass: RegClassIndex, unit: u8) -> Self {
+        Self { regclass, unit }
+    }
+}
+
+/// An operand that must be in a stack slot.
+///
+/// A `Stack` object can be used to indicate an operand constraint for a value
+/// operand that must live in a stack slot.
+#[derive(Copy, Clone, Hash, PartialEq)]
+pub(crate) struct Stack {
+    pub regclass: RegClassIndex,
+}
+
+impl Stack {
+    pub fn new(regclass: RegClassIndex) -> Self {
+        Self { regclass }
+    }
+    pub fn stack_base_mask(self) -> &'static str {
+        // TODO: Make this configurable instead of just using the SP.
+        "StackBaseMask(1)"
+    }
+}
+
+#[derive(Clone, Hash, PartialEq)]
+pub(crate) struct BranchRange {
+    pub inst_size: u64,
+    pub range: u64,
+}
+
+#[derive(Copy, Clone, Hash, PartialEq)]
+pub(crate) enum OperandConstraint {
+    RegClass(RegClassIndex),
+    FixedReg(Register),
+    TiedInput(usize),
+    Stack(Stack),
+}
+
+impl Into<OperandConstraint> for RegClassIndex {
+    fn into(self) -> OperandConstraint {
+        OperandConstraint::RegClass(self)
+    }
+}
+
+impl Into<OperandConstraint> for Register {
+    fn into(self) -> OperandConstraint {
+        OperandConstraint::FixedReg(self)
+    }
+}
+
+impl Into<OperandConstraint> for usize {
+    fn into(self) -> OperandConstraint {
+        OperandConstraint::TiedInput(self)
+    }
+}
+
+impl Into<OperandConstraint> for Stack {
+    fn into(self) -> OperandConstraint {
+        OperandConstraint::Stack(self)
+    }
+}
+
+/// A recipe for encoding instructions with a given format.
+///
+/// Many different instructions can be encoded by the same recipe, but they
+/// must all have the same instruction format.
+///
+/// The `operands_in` and `operands_out` arguments are tuples specifying the register
+/// allocation constraints for the value operands and results respectively. The
+/// possible constraints for an operand are:
+///
+/// - A `RegClass` specifying the set of allowed registers.
+/// - A `Register` specifying a fixed-register operand.
+/// - An integer indicating that this result is tied to a value operand, so
+///   they must use the same register.
+/// - A `Stack` specifying a value in a stack slot.
+///
+/// The `branch_range` argument must be provided for recipes that can encode
+/// branch instructions. It is an `(origin, bits)` tuple describing the exact
+/// range that can be encoded in a branch instruction.
+#[derive(Clone)]
+pub(crate) struct EncodingRecipe {
+    /// Short mnemonic name for this recipe.
+    pub name: String,
+
+    /// Associated instruction format.
+    pub format: Rc<InstructionFormat>,
+
+    /// Base number of bytes in the binary encoded instruction.
+    pub base_size: u64,
+
+    /// Tuple of register constraints for value operands.
+    pub operands_in: Vec<OperandConstraint>,
+
+    /// Tuple of register constraints for results.
+    pub operands_out: Vec<OperandConstraint>,
+
+    /// Function name to use when computing actual size.
+    pub compute_size: &'static str,
+
+    /// `(origin, bits)` range for branches.
+    pub branch_range: Option<BranchRange>,
+
+    /// This instruction clobbers `iflags` and `fflags`; true by default.
+    pub clobbers_flags: bool,
+
+    /// Instruction predicate.
+    pub inst_predicate: Option<InstructionPredicate>,
+
+    /// ISA predicate.
+    pub isa_predicate: Option<SettingPredicateNumber>,
+
+    /// Rust code for binary emission.
+    pub emit: Option<String>,
+}
+
+// Implement PartialEq ourselves: take all the fields into account but the name.
+impl PartialEq for EncodingRecipe {
+    fn eq(&self, other: &Self) -> bool {
+        Rc::ptr_eq(&self.format, &other.format)
+            && self.base_size == other.base_size
+            && self.operands_in == other.operands_in
+            && self.operands_out == other.operands_out
+            && self.compute_size == other.compute_size
+            && self.branch_range == other.branch_range
+            && self.clobbers_flags == other.clobbers_flags
+            && self.inst_predicate == other.inst_predicate
+            && self.isa_predicate == other.isa_predicate
+            && self.emit == other.emit
+    }
+}
+
+// To allow using it in a hashmap.
+impl Eq for EncodingRecipe {}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub(crate) struct EncodingRecipeNumber(u32);
+entity_impl!(EncodingRecipeNumber);
+
+pub(crate) type Recipes = PrimaryMap<EncodingRecipeNumber, EncodingRecipe>;
+
+#[derive(Clone)]
+pub(crate) struct EncodingRecipeBuilder {
+    pub name: String,
+    format: Rc<InstructionFormat>,
+    pub base_size: u64,
+    pub operands_in: Option<Vec<OperandConstraint>>,
+    pub operands_out: Option<Vec<OperandConstraint>>,
+    pub compute_size: Option<&'static str>,
+    pub branch_range: Option<BranchRange>,
+    pub emit: Option<String>,
+    clobbers_flags: Option<bool>,
+    inst_predicate: Option<InstructionPredicate>,
+    isa_predicate: Option<SettingPredicateNumber>,
+}
+
+impl EncodingRecipeBuilder {
+    pub fn new(name: impl Into<String>, format: &Rc<InstructionFormat>, base_size: u64) -> Self {
+        Self {
+            name: name.into(),
+            format: format.clone(),
+            base_size,
+            operands_in: None,
+            operands_out: None,
+            compute_size: None,
+            branch_range: None,
+            emit: None,
+            clobbers_flags: None,
+            inst_predicate: None,
+            isa_predicate: None,
+        }
+    }
+
+    // Setters.
+    pub fn operands_in(mut self, constraints: Vec<impl Into<OperandConstraint>>) -> Self {
+        assert!(self.operands_in.is_none());
+        self.operands_in = Some(
+            constraints
+                .into_iter()
+                .map(|constr| constr.into())
+                .collect(),
+        );
+        self
+    }
+    pub fn operands_out(mut self, constraints: Vec<impl Into<OperandConstraint>>) -> Self {
+        assert!(self.operands_out.is_none());
+        self.operands_out = Some(
+            constraints
+                .into_iter()
+                .map(|constr| constr.into())
+                .collect(),
+        );
+        self
+    }
+    pub fn clobbers_flags(mut self, flag: bool) -> Self {
+        assert!(self.clobbers_flags.is_none());
+        self.clobbers_flags = Some(flag);
+        self
+    }
+    pub fn emit(mut self, code: impl Into<String>) -> Self {
+        assert!(self.emit.is_none());
+        self.emit = Some(code.into());
+        self
+    }
+    pub fn branch_range(mut self, range: (u64, u64)) -> Self {
+        assert!(self.branch_range.is_none());
+        self.branch_range = Some(BranchRange {
+            inst_size: range.0,
+            range: range.1,
+        });
+        self
+    }
+    pub fn isa_predicate(mut self, pred: SettingPredicateNumber) -> Self {
+        assert!(self.isa_predicate.is_none());
+        self.isa_predicate = Some(pred);
+        self
+    }
+    pub fn inst_predicate(mut self, inst_predicate: impl Into<InstructionPredicate>) -> Self {
+        assert!(self.inst_predicate.is_none());
+        self.inst_predicate = Some(inst_predicate.into());
+        self
+    }
+    pub fn compute_size(mut self, compute_size: &'static str) -> Self {
+        assert!(self.compute_size.is_none());
+        self.compute_size = Some(compute_size);
+        self
+    }
+
+    pub fn build(self) -> EncodingRecipe {
+        let operands_in = self.operands_in.unwrap_or_default();
+        let operands_out = self.operands_out.unwrap_or_default();
+
+        // The number of input constraints must match the number of format input operands.
+        if !self.format.has_value_list {
+            assert!(
+                operands_in.len() == self.format.num_value_operands,
+                format!(
+                    "missing operand constraints for recipe {} (format {})",
+                    self.name, self.format.name
+                )
+            );
+        }
+
+        // Ensure tied inputs actually refer to existing inputs.
+        for constraint in operands_in.iter().chain(operands_out.iter()) {
+            if let OperandConstraint::TiedInput(n) = *constraint {
+                assert!(n < operands_in.len());
+            }
+        }
+
+        let compute_size = match self.compute_size {
+            Some(compute_size) => compute_size,
+            None => "base_size",
+        };
+
+        let clobbers_flags = self.clobbers_flags.unwrap_or(true);
+
+        EncodingRecipe {
+            name: self.name,
+            format: self.format,
+            base_size: self.base_size,
+            operands_in,
+            operands_out,
+            compute_size,
+            branch_range: self.branch_range,
+            clobbers_flags,
+            inst_predicate: self.inst_predicate,
+            isa_predicate: self.isa_predicate,
+            emit: self.emit,
+        }
+    }
+}

cranelift/codegen/meta/src/cdsl/regs.rs

@@ -0,0 +1,412 @@
+use cranelift_codegen_shared::constants;
+use cranelift_entity::{entity_impl, EntityRef, PrimaryMap};
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub(crate) struct RegBankIndex(u32);
+entity_impl!(RegBankIndex);
+
+pub(crate) struct RegBank {
+    pub name: &'static str,
+    pub first_unit: u8,
+    pub units: u8,
+    pub names: Vec<&'static str>,
+    pub prefix: &'static str,
+    pub pressure_tracking: bool,
+    pub pinned_reg: Option<u16>,
+    pub toprcs: Vec<RegClassIndex>,
+    pub classes: Vec<RegClassIndex>,
+}
+
+impl RegBank {
+    pub fn new(
+        name: &'static str,
+        first_unit: u8,
+        units: u8,
+        names: Vec<&'static str>,
+        prefix: &'static str,
+        pressure_tracking: bool,
+        pinned_reg: Option<u16>,
+    ) -> Self {
+        RegBank {
+            name,
+            first_unit,
+            units,
+            names,
+            prefix,
+            pressure_tracking,
+            pinned_reg,
+            toprcs: Vec::new(),
+            classes: Vec::new(),
+        }
+    }
+
+    fn unit_by_name(&self, name: &'static str) -> u8 {
+        let unit = if let Some(found) = self.names.iter().position(|&reg_name| reg_name == name) {
+            found
+        } else {
+            // Try to match without the bank prefix.
+            assert!(name.starts_with(self.prefix));
+            let name_without_prefix = &name[self.prefix.len()..];
+            if let Some(found) = self
+                .names
+                .iter()
+                .position(|&reg_name| reg_name == name_without_prefix)
+            {
+                found
+            } else {
+                // Ultimate try: try to parse a number and use this in the array, eg r15 on x86.
+                if let Ok(as_num) = name_without_prefix.parse::<u8>() {
+                    assert!(
+                        (as_num - self.first_unit) < self.units,
+                        "trying to get {}, but bank only has {} registers!",
+                        name,
+                        self.units
+                    );
+                    (as_num - self.first_unit) as usize
+                } else {
+                    panic!("invalid register name {}", name);
+                }
+            }
+        };
+        self.first_unit + (unit as u8)
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
+pub(crate) struct RegClassIndex(u32);
+entity_impl!(RegClassIndex);
+
+pub(crate) struct RegClass {
+    pub name: &'static str,
+    pub index: RegClassIndex,
+    pub width: u8,
+    pub bank: RegBankIndex,
+    pub toprc: RegClassIndex,
+    pub count: u8,
+    pub start: u8,
+    pub subclasses: Vec<RegClassIndex>,
+}
+
+impl RegClass {
+    pub fn new(
+        name: &'static str,
+        index: RegClassIndex,
+        width: u8,
+        bank: RegBankIndex,
+        toprc: RegClassIndex,
+        count: u8,
+        start: u8,
+    ) -> Self {
+        Self {
+            name,
+            index,
+            width,
+            bank,
+            toprc,
+            count,
+            start,
+            subclasses: Vec::new(),
+        }
+    }
+
+    /// Compute a bit-mask of subclasses, including self.
+    pub fn subclass_mask(&self) -> u64 {
+        let mut m = 1 << self.index.index();
+        for rc in self.subclasses.iter() {
+            m |= 1 << rc.index();
+        }
+        m
+    }
+
+    /// Compute a bit-mask of the register units allocated by this register class.
+    pub fn mask(&self, bank_first_unit: u8) -> Vec<u32> {
+        let mut u = (self.start + bank_first_unit) as usize;
+        let mut out_mask = vec![0, 0, 0];
+        for _ in 0..self.count {
+            out_mask[u / 32] |= 1 << (u % 32);
+            u += self.width as usize;
+        }
+        out_mask
+    }
+}
+
+pub(crate) enum RegClassProto {
+    TopLevel(RegBankIndex),
+    SubClass(RegClassIndex),
+}
+
+pub(crate) struct RegClassBuilder {
+    pub name: &'static str,
+    pub width: u8,
+    pub count: u8,
+    pub start: u8,
+    pub proto: RegClassProto,
+}
+
+impl RegClassBuilder {
+    pub fn new_toplevel(name: &'static str, bank: RegBankIndex) -> Self {
+        Self {
+            name,
+            width: 1,
+            count: 0,
+            start: 0,
+            proto: RegClassProto::TopLevel(bank),
+        }
+    }
+    pub fn subclass_of(
+        name: &'static str,
+        parent_index: RegClassIndex,
+        start: u8,
+        stop: u8,
+    ) -> Self {
+        assert!(stop >= start);
+        Self {
+            name,
+            width: 0,
+            count: stop - start,
+            start,
+            proto: RegClassProto::SubClass(parent_index),
+        }
+    }
+    pub fn count(mut self, count: u8) -> Self {
+        self.count = count;
+        self
+    }
+    pub fn width(mut self, width: u8) -> Self {
+        match self.proto {
+            RegClassProto::TopLevel(_) => self.width = width,
+            RegClassProto::SubClass(_) => panic!("Subclasses inherit their parent's width."),
+        }
+        self
+    }
+}
+
+pub(crate) struct RegBankBuilder {
+    pub name: &'static str,
+    pub units: u8,
+    pub names: Vec<&'static str>,
+    pub prefix: &'static str,
+    pub pressure_tracking: Option<bool>,
+    pub pinned_reg: Option<u16>,
+}
+
+impl RegBankBuilder {
+    pub fn new(name: &'static str, prefix: &'static str) -> Self {
+        Self {
+            name,
+            units: 0,
+            names: vec![],
+            prefix,
+            pressure_tracking: None,
+            pinned_reg: None,
+        }
+    }
+    pub fn units(mut self, units: u8) -> Self {
+        self.units = units;
+        self
+    }
+    pub fn names(mut self, names: Vec<&'static str>) -> Self {
+        self.names = names;
+        self
+    }
+    pub fn track_pressure(mut self, track: bool) -> Self {
+        self.pressure_tracking = Some(track);
+        self
+    }
+    pub fn pinned_reg(mut self, unit: u16) -> Self {
+        assert!(unit < u16::from(self.units));
+        self.pinned_reg = Some(unit);
+        self
+    }
+}
+
+pub(crate) struct IsaRegsBuilder {
+    pub banks: PrimaryMap<RegBankIndex, RegBank>,
+    pub classes: PrimaryMap<RegClassIndex, RegClass>,
+}
+
+impl IsaRegsBuilder {
+    pub fn new() -> Self {
+        Self {
+            banks: PrimaryMap::new(),
+            classes: PrimaryMap::new(),
+        }
+    }
+
+    pub fn add_bank(&mut self, builder: RegBankBuilder) -> RegBankIndex {
+        let first_unit = if self.banks.is_empty() {
+            0
+        } else {
+            let last = &self.banks.last().unwrap();
+            let first_available_unit = (last.first_unit + last.units) as i8;
+            let units = builder.units;
+            let align = if units.is_power_of_two() {
+                units
+            } else {
+                units.next_power_of_two()
+            } as i8;
+            (first_available_unit + align - 1) & -align
+        } as u8;
+
+        self.banks.push(RegBank::new(
+            builder.name,
+            first_unit,
+            builder.units,
+            builder.names,
+            builder.prefix,
+            builder
+                .pressure_tracking
+                .expect("Pressure tracking must be explicitly set"),
+            builder.pinned_reg,
+        ))
+    }
+
+    pub fn add_class(&mut self, builder: RegClassBuilder) -> RegClassIndex {
+        let class_index = self.classes.next_key();
+
+        // Finish delayed construction of RegClass.
+        let (bank, toprc, start, width) = match builder.proto {
+            RegClassProto::TopLevel(bank_index) => {
+                self.banks
+                    .get_mut(bank_index)
+                    .unwrap()
+                    .toprcs
+                    .push(class_index);
+                (bank_index, class_index, builder.start, builder.width)
+            }
+            RegClassProto::SubClass(parent_class_index) => {
+                assert!(builder.width == 0);
+                let (bank, toprc, start, width) = {
+                    let parent = self.classes.get(parent_class_index).unwrap();
+                    (parent.bank, parent.toprc, parent.start, parent.width)
+                };
+                for reg_class in self.classes.values_mut() {
+                    if reg_class.toprc == toprc {
+                        reg_class.subclasses.push(class_index);
+                    }
+                }
+                let subclass_start = start + builder.start * width;
+                (bank, toprc, subclass_start, width)
+            }
+        };
+
+        let reg_bank_units = self.banks.get(bank).unwrap().units;
+        assert!(start < reg_bank_units);
+
+        let count = if builder.count != 0 {
+            builder.count
+        } else {
+            reg_bank_units / width
+        };
+
+        let reg_class = RegClass::new(builder.name, class_index, width, bank, toprc, count, start);
+        self.classes.push(reg_class);
+
+        let reg_bank = self.banks.get_mut(bank).unwrap();
+        reg_bank.classes.push(class_index);
+
+        class_index
+    }
+
+    /// Checks that the set of register classes satisfies:
+    ///
+    /// 1. Closed under intersection: The intersection of any two register
+    ///    classes in the set is either empty or identical to a member of the
+    ///    set.
+    /// 2. There are no identical classes under different names.
+    /// 3. Classes are sorted topologically such that all subclasses have a
+    ///    higher index that the superclass.
+    pub fn build(self) -> IsaRegs {
+        for reg_bank in self.banks.values() {
+            for i1 in reg_bank.classes.iter() {
+                for i2 in reg_bank.classes.iter() {
+                    if i1 >= i2 {
+                        continue;
+                    }
+
+                    let rc1 = self.classes.get(*i1).unwrap();
+                    let rc2 = self.classes.get(*i2).unwrap();
+
+                    let rc1_mask = rc1.mask(0);
+                    let rc2_mask = rc2.mask(0);
+
+                    assert!(
+                        rc1.width != rc2.width || rc1_mask != rc2_mask,
+                        "no duplicates"
+                    );
+                    if rc1.width != rc2.width {
+                        continue;
+                    }
+
+                    let mut intersect = Vec::new();
+                    for (a, b) in rc1_mask.iter().zip(rc2_mask.iter()) {
+                        intersect.push(a & b);
+                    }
+                    if intersect == vec![0; intersect.len()] {
+                        continue;
+                    }
+
+                    // Classes must be topologically ordered, so the intersection can't be the
+                    // superclass.
+                    assert!(intersect != rc1_mask);
+
+                    // If the intersection is the second one, then it must be a subclass.
+                    if intersect == rc2_mask {
+                        assert!(self
+                            .classes
+                            .get(*i1)
+                            .unwrap()
+                            .subclasses
+                            .iter()
+                            .any(|x| *x == *i2));
+                    }
+                }
+            }
+        }
+
+        assert!(
+            self.classes.len() <= constants::MAX_NUM_REG_CLASSES,
+            "Too many register classes"
+        );
+
+        let num_toplevel = self
+            .classes
+            .values()
+            .filter(|x| x.toprc == x.index && self.banks.get(x.bank).unwrap().pressure_tracking)
+            .count();
+
+        assert!(
+            num_toplevel <= constants::MAX_TRACKED_TOP_RCS,
+            "Too many top-level register classes"
+        );
+
+        IsaRegs::new(self.banks, self.classes)
+    }
+}
+
+pub(crate) struct IsaRegs {
+    pub banks: PrimaryMap<RegBankIndex, RegBank>,
+    pub classes: PrimaryMap<RegClassIndex, RegClass>,
+}
+
+impl IsaRegs {
+    fn new(
+        banks: PrimaryMap<RegBankIndex, RegBank>,
+        classes: PrimaryMap<RegClassIndex, RegClass>,
+    ) -> Self {
+        Self { banks, classes }
+    }
+
+    pub fn class_by_name(&self, name: &str) -> RegClassIndex {
+        self.classes
+            .values()
+            .find(|&class| class.name == name)
+            .unwrap_or_else(|| panic!("register class {} not found", name))
+            .index
+    }
+
+    pub fn regunit_by_name(&self, class_index: RegClassIndex, name: &'static str) -> u8 {
+        let bank_index = self.classes.get(class_index).unwrap().bank;
+        self.banks.get(bank_index).unwrap().unit_by_name(name)
+    }
+}

cranelift/codegen/meta/src/cdsl/settings.rs

@@ -0,0 +1,407 @@
+use std::iter;
+
+#[derive(Clone, Copy, Hash, PartialEq, Eq)]
+pub(crate) struct BoolSettingIndex(usize);
+
+#[derive(Hash, PartialEq, Eq)]
+pub(crate) struct BoolSetting {
+    pub default: bool,
+    pub bit_offset: u8,
+    pub predicate_number: u8,
+}
+
+#[derive(Hash, PartialEq, Eq)]
+pub(crate) enum SpecificSetting {
+    Bool(BoolSetting),
+    Enum(Vec<&'static str>),
+    Num(u8),
+}
+
+#[derive(Hash, PartialEq, Eq)]
+pub(crate) struct Setting {
+    pub name: &'static str,
+    pub comment: &'static str,
+    pub specific: SpecificSetting,
+    pub byte_offset: u8,
+}
+
+impl Setting {
+    pub fn default_byte(&self) -> u8 {
+        match self.specific {
+            SpecificSetting::Bool(BoolSetting {
+                default,
+                bit_offset,
+                ..
+            }) => {
+                if default {
+                    1 << bit_offset
+                } else {
+                    0
+                }
+            }
+            SpecificSetting::Enum(_) => 0,
+            SpecificSetting::Num(default) => default,
+        }
+    }
+
+    fn byte_for_value(&self, v: bool) -> u8 {
+        match self.specific {
+            SpecificSetting::Bool(BoolSetting { bit_offset, .. }) => {
+                if v {
+                    1 << bit_offset
+                } else {
+                    0
+                }
+            }
+            _ => panic!("byte_for_value shouldn't be used for non-boolean settings."),
+        }
+    }
+
+    fn byte_mask(&self) -> u8 {
+        match self.specific {
+            SpecificSetting::Bool(BoolSetting { bit_offset, .. }) => 1 << bit_offset,
+            _ => panic!("byte_for_value shouldn't be used for non-boolean settings."),
+        }
+    }
+}
+
+#[derive(Hash, PartialEq, Eq)]
+pub(crate) struct PresetIndex(usize);
+
+#[derive(Hash, PartialEq, Eq)]
+pub(crate) enum PresetType {
+    BoolSetting(BoolSettingIndex),
+    OtherPreset(PresetIndex),
+}
+
+impl Into<PresetType> for BoolSettingIndex {
+    fn into(self) -> PresetType {
+        PresetType::BoolSetting(self)
+    }
+}
+impl Into<PresetType> for PresetIndex {
+    fn into(self) -> PresetType {
+        PresetType::OtherPreset(self)
+    }
+}
+
+#[derive(Hash, PartialEq, Eq)]
+pub(crate) struct Preset {
+    pub name: &'static str,
+    values: Vec<BoolSettingIndex>,
+}
+
+impl Preset {
+    pub fn layout(&self, group: &SettingGroup) -> Vec<(u8, u8)> {
+        let mut layout: Vec<(u8, u8)> = iter::repeat((0, 0))
+            .take(group.settings_size as usize)
+            .collect();
+        for bool_index in &self.values {
+            let setting = &group.settings[bool_index.0];
+            let mask = setting.byte_mask();
+            let val = setting.byte_for_value(true);
+            assert!((val & !mask) == 0);
+            let (ref mut l_mask, ref mut l_val) =
+                *layout.get_mut(setting.byte_offset as usize).unwrap();
+            *l_mask |= mask;
+            *l_val = (*l_val & !mask) | val;
+        }
+        layout
+    }
+}
+
+pub(crate) struct SettingGroup {
+    pub name: &'static str,
+    pub settings: Vec<Setting>,
+    pub bool_start_byte_offset: u8,
+    pub settings_size: u8,
+    pub presets: Vec<Preset>,
+    pub predicates: Vec<Predicate>,
+}
+
+impl SettingGroup {
+    fn num_bool_settings(&self) -> u8 {
+        self.settings
+            .iter()
+            .filter(|s| {
+                if let SpecificSetting::Bool(_) = s.specific {
+                    true
+                } else {
+                    false
+                }
+            })
+            .count() as u8
+    }
+
+    pub fn byte_size(&self) -> u8 {
+        let num_predicates = self.num_bool_settings() + (self.predicates.len() as u8);
+        self.bool_start_byte_offset + (num_predicates + 7) / 8
+    }
+
+    pub fn get_bool(&self, name: &'static str) -> (BoolSettingIndex, &Self) {
+        for (i, s) in self.settings.iter().enumerate() {
+            if let SpecificSetting::Bool(_) = s.specific {
+                if s.name == name {
+                    return (BoolSettingIndex(i), self);
+                }
+            }
+        }
+        panic!("Should have found bool setting by name.");
+    }
+
+    pub fn predicate_by_name(&self, name: &'static str) -> SettingPredicateNumber {
+        self.predicates
+            .iter()
+            .find(|pred| pred.name == name)
+            .unwrap_or_else(|| panic!("unknown predicate {}", name))
+            .number
+    }
+}
+
+/// This is the basic information needed to track the specific parts of a setting when building
+/// them.
+pub(crate) enum ProtoSpecificSetting {
+    Bool(bool),
+    Enum(Vec<&'static str>),
+    Num(u8),
+}
+
+/// This is the information provided during building for a setting.
+struct ProtoSetting {
+    name: &'static str,
+    comment: &'static str,
+    specific: ProtoSpecificSetting,
+}
+
+#[derive(Hash, PartialEq, Eq)]
+pub(crate) enum PredicateNode {
+    OwnedBool(BoolSettingIndex),
+    SharedBool(&'static str, &'static str),
+    Not(Box<PredicateNode>),
+    And(Box<PredicateNode>, Box<PredicateNode>),
+}
+
+impl Into<PredicateNode> for BoolSettingIndex {
+    fn into(self) -> PredicateNode {
+        PredicateNode::OwnedBool(self)
+    }
+}
+impl<'a> Into<PredicateNode> for (BoolSettingIndex, &'a SettingGroup) {
+    fn into(self) -> PredicateNode {
+        let (index, group) = (self.0, self.1);
+        let setting = &group.settings[index.0];
+        PredicateNode::SharedBool(group.name, setting.name)
+    }
+}
+
+impl PredicateNode {
+    fn render(&self, group: &SettingGroup) -> String {
+        match *self {
+            PredicateNode::OwnedBool(bool_setting_index) => format!(
+                "{}.{}()",
+                group.name, group.settings[bool_setting_index.0].name
+            ),
+            PredicateNode::SharedBool(ref group_name, ref bool_name) => {
+                format!("{}.{}()", group_name, bool_name)
+            }
+            PredicateNode::And(ref lhs, ref rhs) => {
+                format!("{} && {}", lhs.render(group), rhs.render(group))
+            }
+            PredicateNode::Not(ref node) => format!("!({})", node.render(group)),
+        }
+    }
+}
+
+struct ProtoPredicate {
+    pub name: &'static str,
+    node: PredicateNode,
+}
+
+pub(crate) type SettingPredicateNumber = u8;
+
+pub(crate) struct Predicate {
+    pub name: &'static str,
+    node: PredicateNode,
+    pub number: SettingPredicateNumber,
+}
+
+impl Predicate {
+    pub fn render(&self, group: &SettingGroup) -> String {
+        self.node.render(group)
+    }
+}
+
+pub(crate) struct SettingGroupBuilder {
+    name: &'static str,
+    settings: Vec<ProtoSetting>,
+    presets: Vec<Preset>,
+    predicates: Vec<ProtoPredicate>,
+}
+
+impl SettingGroupBuilder {
+    pub fn new(name: &'static str) -> Self {
+        Self {
+            name,
+            settings: Vec::new(),
+            presets: Vec::new(),
+            predicates: Vec::new(),
+        }
+    }
+
+    fn add_setting(
+        &mut self,
+        name: &'static str,
+        comment: &'static str,
+        specific: ProtoSpecificSetting,
+    ) {
+        self.settings.push(ProtoSetting {
+            name,
+            comment,
+            specific,
+        })
+    }
+
+    pub fn add_bool(
+        &mut self,
+        name: &'static str,
+        comment: &'static str,
+        default: bool,
+    ) -> BoolSettingIndex {
+        assert!(
+            self.predicates.is_empty(),
+            "predicates must be added after the boolean settings"
+        );
+        self.add_setting(name, comment, ProtoSpecificSetting::Bool(default));
+        BoolSettingIndex(self.settings.len() - 1)
+    }
+
+    pub fn add_enum(
+        &mut self,
+        name: &'static str,
+        comment: &'static str,
+        values: Vec<&'static str>,
+    ) {
+        self.add_setting(name, comment, ProtoSpecificSetting::Enum(values));
+    }
+
+    pub fn add_num(&mut self, name: &'static str, comment: &'static str, default: u8) {
+        self.add_setting(name, comment, ProtoSpecificSetting::Num(default));
+    }
+
+    pub fn add_predicate(&mut self, name: &'static str, node: PredicateNode) {
+        self.predicates.push(ProtoPredicate { name, node });
+    }
+
+    pub fn add_preset(&mut self, name: &'static str, args: Vec<PresetType>) -> PresetIndex {
+        let mut values = Vec::new();
+        for arg in args {
+            match arg {
+                PresetType::OtherPreset(index) => {
+                    values.extend(self.presets[index.0].values.iter());
+                }
+                PresetType::BoolSetting(index) => values.push(index),
+            }
+        }
+        self.presets.push(Preset { name, values });
+        PresetIndex(self.presets.len() - 1)
+    }
+
+    /// Compute the layout of the byte vector used to represent this settings
+    /// group.
+    ///
+    /// The byte vector contains the following entries in order:
+    ///
+    /// 1. Byte-sized settings like `NumSetting` and `EnumSetting`.
+    /// 2. `BoolSetting` settings.
+    /// 3. Precomputed named predicates.
+    /// 4. Other numbered predicates, including parent predicates that need to be accessible by
+    ///    number.
+    ///
+    /// Set `self.settings_size` to the length of the byte vector prefix that
+    /// contains the settings. All bytes after that are computed, not
+    /// configured.
+    ///
+    /// Set `self.boolean_offset` to the beginning of the numbered predicates,
+    /// 2. in the list above.
+    ///
+    /// Assign `byte_offset` and `bit_offset` fields in all settings.
+    pub fn build(self) -> SettingGroup {
+        let mut group = SettingGroup {
+            name: self.name,
+            settings: Vec::new(),
+            bool_start_byte_offset: 0,
+            settings_size: 0,
+            presets: Vec::new(),
+            predicates: Vec::new(),
+        };
+
+        let mut byte_offset = 0;
+
+        // Assign the non-boolean settings first.
+        for s in &self.settings {
+            let specific = match s.specific {
+                ProtoSpecificSetting::Bool(..) => continue,
+                ProtoSpecificSetting::Enum(ref values) => SpecificSetting::Enum(values.clone()),
+                ProtoSpecificSetting::Num(default) => SpecificSetting::Num(default),
+            };
+
+            group.settings.push(Setting {
+                name: s.name,
+                comment: s.comment,
+                byte_offset,
+                specific,
+            });
+
+            byte_offset += 1;
+        }
+
+        group.bool_start_byte_offset = byte_offset;
+
+        let mut predicate_number = 0;
+
+        // Then the boolean settings.
+        for s in &self.settings {
+            let default = match s.specific {
+                ProtoSpecificSetting::Bool(default) => default,
+                ProtoSpecificSetting::Enum(_) | ProtoSpecificSetting::Num(_) => continue,
+            };
+            group.settings.push(Setting {
+                name: s.name,
+                comment: s.comment,
+                byte_offset: byte_offset + predicate_number / 8,
+                specific: SpecificSetting::Bool(BoolSetting {
+                    default,
+                    bit_offset: predicate_number % 8,
+                    predicate_number,
+                }),
+            });
+            predicate_number += 1;
+        }
+
+        assert!(
+            group.predicates.is_empty(),
+            "settings_size is the byte size before adding predicates"
+        );
+        group.settings_size = group.byte_size();
+
+        // Sort predicates by name to ensure the same order as the Python code.
+        let mut predicates = self.predicates;
+        predicates.sort_by_key(|predicate| predicate.name);
+
+        group
+            .predicates
+            .extend(predicates.into_iter().map(|predicate| {
+                let number = predicate_number;
+                predicate_number += 1;
+                Predicate {
+                    name: predicate.name,
+                    node: predicate.node,
+                    number,
+                }
+            }));
+
+        group.presets.extend(self.presets);
+
+        group
+    }
+}

cranelift/codegen/meta/src/cdsl/type_inference.rs

@@ -0,0 +1,660 @@
+use crate::cdsl::ast::{Def, DefIndex, DefPool, Var, VarIndex, VarPool};
+use crate::cdsl::typevar::{DerivedFunc, TypeSet, TypeVar};
+
+use std::collections::{HashMap, HashSet};
+use std::iter::FromIterator;
+
+#[derive(Debug, Hash, PartialEq, Eq)]
+pub(crate) enum Constraint {
+    /// Constraint specifying that a type var tv1 must be wider than or equal to type var tv2 at
+    /// runtime. This requires that:
+    /// 1) They have the same number of lanes
+    /// 2) In a lane tv1 has at least as many bits as tv2.
+    WiderOrEq(TypeVar, TypeVar),
+
+    /// Constraint specifying that two derived type vars must have the same runtime type.
+    Eq(TypeVar, TypeVar),
+
+    /// Constraint specifying that a type var must belong to some typeset.
+    InTypeset(TypeVar, TypeSet),
+}
+
+impl Constraint {
+    fn translate_with<F: Fn(&TypeVar) -> TypeVar>(&self, func: F) -> Constraint {
+        match self {
+            Constraint::WiderOrEq(lhs, rhs) => {
+                let lhs = func(&lhs);
+                let rhs = func(&rhs);
+                Constraint::WiderOrEq(lhs, rhs)
+            }
+            Constraint::Eq(lhs, rhs) => {
+                let lhs = func(&lhs);
+                let rhs = func(&rhs);
+                Constraint::Eq(lhs, rhs)
+            }
+            Constraint::InTypeset(tv, ts) => {
+                let tv = func(&tv);
+                Constraint::InTypeset(tv, ts.clone())
+            }
+        }
+    }
+
+    /// Creates a new constraint by replacing type vars by their hashmap equivalent.
+    fn translate_with_map(
+        &self,
+        original_to_own_typevar: &HashMap<&TypeVar, TypeVar>,
+    ) -> Constraint {
+        self.translate_with(|tv| substitute(original_to_own_typevar, tv))
+    }
+
+    /// Creates a new constraint by replacing type vars by their canonical equivalent.
+    fn translate_with_env(&self, type_env: &TypeEnvironment) -> Constraint {
+        self.translate_with(|tv| type_env.get_equivalent(tv))
+    }
+
+    fn is_trivial(&self) -> bool {
+        match self {
+            Constraint::WiderOrEq(lhs, rhs) => {
+                // Trivially true.
+                if lhs == rhs {
+                    return true;
+                }
+
+                let ts1 = lhs.get_typeset();
+                let ts2 = rhs.get_typeset();
+
+                // Trivially true.
+                if ts1.is_wider_or_equal(&ts2) {
+                    return true;
+                }
+
+                // Trivially false.
+                if ts1.is_narrower(&ts2) {
+                    return true;
+                }
+
+                // Trivially false.
+                if (&ts1.lanes & &ts2.lanes).is_empty() {
+                    return true;
+                }
+
+                self.is_concrete()
+            }
+            Constraint::Eq(lhs, rhs) => lhs == rhs || self.is_concrete(),
+            Constraint::InTypeset(_, _) => {
+                // The way InTypeset are made, they would always be trivial if we were applying the
+                // same logic as the Python code did, so ignore this.
+                self.is_concrete()
+            }
+        }
+    }
+
+    /// Returns true iff all the referenced type vars are singletons.
+    fn is_concrete(&self) -> bool {
+        match self {
+            Constraint::WiderOrEq(lhs, rhs) => {
+                lhs.singleton_type().is_some() && rhs.singleton_type().is_some()
+            }
+            Constraint::Eq(lhs, rhs) => {
+                lhs.singleton_type().is_some() && rhs.singleton_type().is_some()
+            }
+            Constraint::InTypeset(tv, _) => tv.singleton_type().is_some(),
+        }
+    }
+
+    fn typevar_args(&self) -> Vec<&TypeVar> {
+        match self {
+            Constraint::WiderOrEq(lhs, rhs) => vec![lhs, rhs],
+            Constraint::Eq(lhs, rhs) => vec![lhs, rhs],
+            Constraint::InTypeset(tv, _) => vec![tv],
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum TypeEnvRank {
+    Singleton = 5,
+    Input = 4,
+    Intermediate = 3,
+    Output = 2,
+    Temp = 1,
+    Internal = 0,
+}
+
+/// Class encapsulating the necessary bookkeeping for type inference.
+pub(crate) struct TypeEnvironment {
+    vars: HashSet<VarIndex>,
+    ranks: HashMap<TypeVar, TypeEnvRank>,
+    equivalency_map: HashMap<TypeVar, TypeVar>,
+    pub constraints: Vec<Constraint>,
+}
+
+impl TypeEnvironment {
+    fn new() -> Self {
+        TypeEnvironment {
+            vars: HashSet::new(),
+            ranks: HashMap::new(),
+            equivalency_map: HashMap::new(),
+            constraints: Vec::new(),
+        }
+    }
+
+    fn register(&mut self, var_index: VarIndex, var: &mut Var) {
+        self.vars.insert(var_index);
+        let rank = if var.is_input() {
+            TypeEnvRank::Input
+        } else if var.is_intermediate() {
+            TypeEnvRank::Intermediate
+        } else if var.is_output() {
+            TypeEnvRank::Output
+        } else {
+            assert!(var.is_temp());
+            TypeEnvRank::Temp
+        };
+        self.ranks.insert(var.get_or_create_typevar(), rank);
+    }
+
+    fn add_constraint(&mut self, constraint: Constraint) {
+        if self.constraints.iter().any(|item| *item == constraint) {
+            return;
+        }
+
+        // Check extra conditions for InTypeset constraints.
+        if let Constraint::InTypeset(tv, _) = &constraint {
+            assert!(
+                tv.base.is_none(),
+                "type variable is {:?}, while expecting none",
+                tv
+            );
+            assert!(
+                tv.name.starts_with("typeof_"),
+                "Name \"{}\" should start with \"typeof_\"",
+                tv.name
+            );
+        }
+
+        self.constraints.push(constraint);
+    }
+
+    /// Returns the canonical representative of the equivalency class of the given argument, or
+    /// duplicates it if it's not there yet.
+    pub fn get_equivalent(&self, tv: &TypeVar) -> TypeVar {
+        let mut tv = tv;
+        while let Some(found) = self.equivalency_map.get(tv) {
+            tv = found;
+        }
+        match &tv.base {
+            Some(parent) => self
+                .get_equivalent(&parent.type_var)
+                .derived(parent.derived_func),
+            None => tv.clone(),
+        }
+    }
+
+    /// Get the rank of tv in the partial order:
+    /// - TVs directly associated with a Var get their rank from the Var (see register()).
+    /// - Internally generated non-derived TVs implicitly get the lowest rank (0).
+    /// - Derived variables get their rank from their free typevar.
+    /// - Singletons have the highest rank.
+    /// - TVs associated with vars in a source pattern have a higher rank than TVs associated with
+    /// temporary vars.
+    fn rank(&self, tv: &TypeVar) -> u8 {
+        let actual_tv = match tv.base {
+            Some(_) => tv.free_typevar(),
+            None => Some(tv.clone()),
+        };
+
+        let rank = match actual_tv {
+            Some(actual_tv) => match self.ranks.get(&actual_tv) {
+                Some(rank) => Some(*rank),
+                None => {
+                    assert!(
+                        !actual_tv.name.starts_with("typeof_"),
+                        format!("variable {} should be explicitly ranked", actual_tv.name)
+                    );
+                    None
+                }
+            },
+            None => None,
+        };
+
+        let rank = match rank {
+            Some(rank) => rank,
+            None => {
+                if tv.singleton_type().is_some() {
+                    TypeEnvRank::Singleton
+                } else {
+                    TypeEnvRank::Internal
+                }
+            }
+        };
+
+        rank as u8
+    }
+
+    /// Record the fact that the free tv1 is part of the same equivalence class as tv2. The
+    /// canonical representative of the merged class is tv2's canonical representative.
+    fn record_equivalent(&mut self, tv1: TypeVar, tv2: TypeVar) {
+        assert!(tv1.base.is_none());
+        assert!(self.get_equivalent(&tv1) == tv1);
+        if let Some(tv2_base) = &tv2.base {
+            // Ensure there are no cycles.
+            assert!(self.get_equivalent(&tv2_base.type_var) != tv1);
+        }
+        self.equivalency_map.insert(tv1, tv2);
+    }
+
+    /// Get the free typevars in the current type environment.
+    pub fn free_typevars(&self, var_pool: &mut VarPool) -> Vec<TypeVar> {
+        let mut typevars = Vec::new();
+        typevars.extend(self.equivalency_map.keys().cloned());
+        typevars.extend(
+            self.vars
+                .iter()
+                .map(|&var_index| var_pool.get_mut(var_index).get_or_create_typevar()),
+        );
+
+        let set: HashSet<TypeVar> = HashSet::from_iter(
+            typevars
+                .iter()
+                .map(|tv| self.get_equivalent(tv).free_typevar())
+                .filter(|opt_tv| {
+                    // Filter out singleton types.
+                    opt_tv.is_some()
+                })
+                .map(|tv| tv.unwrap()),
+        );
+        Vec::from_iter(set)
+    }
+
+    /// Normalize by collapsing any roots that don't correspond to a concrete type var AND have a
+    /// single type var derived from them or equivalent to them.
+    ///
+    /// e.g. if we have a root of the tree that looks like:
+    ///
+    ///   typeof_a   typeof_b
+    ///          \\  /
+    ///       typeof_x
+    ///           |
+    ///         half_width(1)
+    ///           |
+    ///           1
+    ///
+    /// we want to collapse the linear path between 1 and typeof_x. The resulting graph is:
+    ///
+    ///   typeof_a   typeof_b
+    ///          \\  /
+    ///       typeof_x
+    fn normalize(&mut self, var_pool: &mut VarPool) {
+        let source_tvs: HashSet<TypeVar> = HashSet::from_iter(
+            self.vars
+                .iter()
+                .map(|&var_index| var_pool.get_mut(var_index).get_or_create_typevar()),
+        );
+
+        let mut children: HashMap<TypeVar, HashSet<TypeVar>> = HashMap::new();
+
+        // Insert all the parents found by the derivation relationship.
+        for type_var in self.equivalency_map.values() {
+            if type_var.base.is_none() {
+                continue;
+            }
+
+            let parent_tv = type_var.free_typevar();
+            if parent_tv.is_none() {
+                // Ignore this type variable, it's a singleton.
+                continue;
+            }
+            let parent_tv = parent_tv.unwrap();
+
+            children
+                .entry(parent_tv)
+                .or_insert_with(HashSet::new)
+                .insert(type_var.clone());
+        }
+
+        // Insert all the explicit equivalency links.
+        for (equivalent_tv, canon_tv) in self.equivalency_map.iter() {
+            children
+                .entry(canon_tv.clone())
+                .or_insert_with(HashSet::new)
+                .insert(equivalent_tv.clone());
+        }
+
+        // Remove links that are straight paths up to typevar of variables.
+        for free_root in self.free_typevars(var_pool) {
+            let mut root = &free_root;
+            while !source_tvs.contains(&root)
+                && children.contains_key(&root)
+                && children.get(&root).unwrap().len() == 1
+            {
+                let child = children.get(&root).unwrap().iter().next().unwrap();
+                assert_eq!(self.equivalency_map[child], root.clone());
+                self.equivalency_map.remove(child);
+                root = child;
+            }
+        }
+    }
+
+    /// Extract a clean type environment from self, that only mentions type vars associated with
+    /// real variables.
+    fn extract(self, var_pool: &mut VarPool) -> TypeEnvironment {
+        let vars_tv: HashSet<TypeVar> = HashSet::from_iter(
+            self.vars
+                .iter()
+                .map(|&var_index| var_pool.get_mut(var_index).get_or_create_typevar()),
+        );
+
+        let mut new_equivalency_map: HashMap<TypeVar, TypeVar> = HashMap::new();
+        for tv in &vars_tv {
+            let canon_tv = self.get_equivalent(tv);
+            if *tv != canon_tv {
+                new_equivalency_map.insert(tv.clone(), canon_tv.clone());
+            }
+
+            // Sanity check: the translated type map should only refer to real variables.
+            assert!(vars_tv.contains(tv));
+            let canon_free_tv = canon_tv.free_typevar();
+            assert!(canon_free_tv.is_none() || vars_tv.contains(&canon_free_tv.unwrap()));
+        }
+
+        let mut new_constraints: HashSet<Constraint> = HashSet::new();
+        for constraint in &self.constraints {
+            let constraint = constraint.translate_with_env(&self);
+            if constraint.is_trivial() || new_constraints.contains(&constraint) {
+                continue;
+            }
+
+            // Sanity check: translated constraints should refer only to real variables.
+            for arg in constraint.typevar_args() {
+                let arg_free_tv = arg.free_typevar();
+                assert!(arg_free_tv.is_none() || vars_tv.contains(&arg_free_tv.unwrap()));
+            }
+
+            new_constraints.insert(constraint);
+        }
+
+        TypeEnvironment {
+            vars: self.vars,
+            ranks: self.ranks,
+            equivalency_map: new_equivalency_map,
+            constraints: Vec::from_iter(new_constraints),
+        }
+    }
+}
+
+/// Replaces an external type variable according to the following rules:
+/// - if a local copy is present in the map, return it.
+/// - or if it's derived, create a local derived one that recursively substitutes the parent.
+/// - or return itself.
+fn substitute(map: &HashMap<&TypeVar, TypeVar>, external_type_var: &TypeVar) -> TypeVar {
+    match map.get(&external_type_var) {
+        Some(own_type_var) => own_type_var.clone(),
+        None => match &external_type_var.base {
+            Some(parent) => {
+                let parent_substitute = substitute(map, &parent.type_var);
+                TypeVar::derived(&parent_substitute, parent.derived_func)
+            }
+            None => external_type_var.clone(),
+        },
+    }
+}
+
+/// Normalize a (potentially derived) typevar using the following rules:
+///
+/// - vector and width derived functions commute
+///     {HALF,DOUBLE}VECTOR({HALF,DOUBLE}WIDTH(base)) ->
+///     {HALF,DOUBLE}WIDTH({HALF,DOUBLE}VECTOR(base))
+///
+/// - half/double pairs collapse
+///     {HALF,DOUBLE}WIDTH({DOUBLE,HALF}WIDTH(base)) -> base
+///     {HALF,DOUBLE}VECTOR({DOUBLE,HALF}VECTOR(base)) -> base
+fn canonicalize_derivations(tv: TypeVar) -> TypeVar {
+    let base = match &tv.base {
+        Some(base) => base,
+        None => return tv,
+    };
+
+    let derived_func = base.derived_func;
+
+    if let Some(base_base) = &base.type_var.base {
+        let base_base_tv = &base_base.type_var;
+        match (derived_func, base_base.derived_func) {
+            (DerivedFunc::HalfWidth, DerivedFunc::DoubleWidth)
+            | (DerivedFunc::DoubleWidth, DerivedFunc::HalfWidth)
+            | (DerivedFunc::HalfVector, DerivedFunc::DoubleVector)
+            | (DerivedFunc::DoubleVector, DerivedFunc::HalfVector) => {
+                // Cancelling bijective transformations. This doesn't hide any overflow issues
+                // since derived type sets are checked upon derivaion, and base typesets are only
+                // allowed to shrink.
+                return canonicalize_derivations(base_base_tv.clone());
+            }
+            (DerivedFunc::HalfWidth, DerivedFunc::HalfVector)
+            | (DerivedFunc::HalfWidth, DerivedFunc::DoubleVector)
+            | (DerivedFunc::DoubleWidth, DerivedFunc::DoubleVector)
+            | (DerivedFunc::DoubleWidth, DerivedFunc::HalfVector) => {
+                // Arbitrarily put WIDTH derivations before VECTOR derivations, since they commute.
+                return canonicalize_derivations(
+                    base_base_tv
+                        .derived(derived_func)
+                        .derived(base_base.derived_func),
+                );
+            }
+            _ => {}
+        };
+    }
+
+    canonicalize_derivations(base.type_var.clone()).derived(derived_func)
+}
+
+/// Given typevars tv1 and tv2 (which could be derived from one another), constrain their typesets
+/// to be the same. When one is derived from the other, repeat the constrain process until
+/// a fixed point is reached.
+fn constrain_fixpoint(tv1: &TypeVar, tv2: &TypeVar) {
+    loop {
+        let old_tv1_ts = tv1.get_typeset().clone();
+        tv2.constrain_types(tv1.clone());
+        if tv1.get_typeset() == old_tv1_ts {
+            break;
+        }
+    }
+
+    let old_tv2_ts = tv2.get_typeset();
+    tv1.constrain_types(tv2.clone());
+    // The above loop should ensure that all reference cycles have been handled.
+    assert!(old_tv2_ts == tv2.get_typeset());
+}
+
+/// Unify tv1 and tv2 in the given type environment. tv1 must have a rank greater or equal to tv2's
+/// one, modulo commutations.
+fn unify(tv1: &TypeVar, tv2: &TypeVar, type_env: &mut TypeEnvironment) -> Result<(), String> {
+    let tv1 = canonicalize_derivations(type_env.get_equivalent(tv1));
+    let tv2 = canonicalize_derivations(type_env.get_equivalent(tv2));
+
+    if tv1 == tv2 {
+        // Already unified.
+        return Ok(());
+    }
+
+    if type_env.rank(&tv2) < type_env.rank(&tv1) {
+        // Make sure tv1 always has the smallest rank, since real variables have the higher rank
+        // and we want them to be the canonical representatives of their equivalency classes.
+        return unify(&tv2, &tv1, type_env);
+    }
+
+    constrain_fixpoint(&tv1, &tv2);
+
+    if tv1.get_typeset().size() == 0 || tv2.get_typeset().size() == 0 {
+        return Err(format!(
+            "Error: empty type created when unifying {} and {}",
+            tv1.name, tv2.name
+        ));
+    }
+
+    let base = match &tv1.base {
+        Some(base) => base,
+        None => {
+            type_env.record_equivalent(tv1, tv2);
+            return Ok(());
+        }
+    };
+
+    if let Some(inverse) = base.derived_func.inverse() {
+        return unify(&base.type_var, &tv2.derived(inverse), type_env);
+    }
+
+    type_env.add_constraint(Constraint::Eq(tv1, tv2));
+    Ok(())
+}
+
+/// Perform type inference on one Def in the current type environment and return an updated type
+/// environment or error.
+///
+/// At a high level this works by creating fresh copies of each formal type var in the Def's
+/// instruction's signature, and unifying the formal typevar with the corresponding actual typevar.
+fn infer_definition(
+    def: &Def,
+    var_pool: &mut VarPool,
+    type_env: TypeEnvironment,
+    last_type_index: &mut usize,
+) -> Result<TypeEnvironment, String> {
+    let apply = &def.apply;
+    let inst = &apply.inst;
+
+    let mut type_env = type_env;
+    let free_formal_tvs = inst.all_typevars();
+
+    let mut original_to_own_typevar: HashMap<&TypeVar, TypeVar> = HashMap::new();
+    for &tv in &free_formal_tvs {
+        assert!(original_to_own_typevar
+            .insert(
+                tv,
+                TypeVar::copy_from(tv, format!("own_{}", last_type_index))
+            )
+            .is_none());
+        *last_type_index += 1;
+    }
+
+    // Update the mapping with any explicity bound type vars:
+    for (i, value_type) in apply.value_types.iter().enumerate() {
+        let singleton = TypeVar::new_singleton(value_type.clone());
+        assert!(original_to_own_typevar
+            .insert(free_formal_tvs[i], singleton)
+            .is_some());
+    }
+
+    // Get fresh copies for each typevar in the signature (both free and derived).
+    let mut formal_tvs = Vec::new();
+    formal_tvs.extend(inst.value_results.iter().map(|&i| {
+        substitute(
+            &original_to_own_typevar,
+            inst.operands_out[i].type_var().unwrap(),
+        )
+    }));
+    formal_tvs.extend(inst.value_opnums.iter().map(|&i| {
+        substitute(
+            &original_to_own_typevar,
+            inst.operands_in[i].type_var().unwrap(),
+        )
+    }));
+
+    // Get the list of actual vars.
+    let mut actual_vars = Vec::new();
+    actual_vars.extend(inst.value_results.iter().map(|&i| def.defined_vars[i]));
+    actual_vars.extend(
+        inst.value_opnums
+            .iter()
+            .map(|&i| apply.args[i].unwrap_var()),
+    );
+
+    // Get the list of the actual TypeVars.
+    let mut actual_tvs = Vec::new();
+    for var_index in actual_vars {
+        let var = var_pool.get_mut(var_index);
+        type_env.register(var_index, var);
+        actual_tvs.push(var.get_or_create_typevar());
+    }
+
+    // Make sure we start unifying with the control type variable first, by putting it at the
+    // front of both vectors.
+    if let Some(poly) = &inst.polymorphic_info {
+        let own_ctrl_tv = &original_to_own_typevar[&poly.ctrl_typevar];
+        let ctrl_index = formal_tvs.iter().position(|tv| tv == own_ctrl_tv).unwrap();
+        if ctrl_index != 0 {
+            formal_tvs.swap(0, ctrl_index);
+            actual_tvs.swap(0, ctrl_index);
+        }
+    }
+
+    // Unify each actual type variable with the corresponding formal type variable.
+    for (actual_tv, formal_tv) in actual_tvs.iter().zip(&formal_tvs) {
+        if let Err(msg) = unify(actual_tv, formal_tv, &mut type_env) {
+            return Err(format!(
+                "fail ti on {} <: {}: {}",
+                actual_tv.name, formal_tv.name, msg
+            ));
+        }
+    }
+
+    // Add any instruction specific constraints.
+    for constraint in &inst.constraints {
+        type_env.add_constraint(constraint.translate_with_map(&original_to_own_typevar));
+    }
+
+    Ok(type_env)
+}
+
+/// Perform type inference on an transformation. Return an updated type environment or error.
+pub(crate) fn infer_transform(
+    src: DefIndex,
+    dst: &[DefIndex],
+    def_pool: &DefPool,
+    var_pool: &mut VarPool,
+) -> Result<TypeEnvironment, String> {
+    let mut type_env = TypeEnvironment::new();
+    let mut last_type_index = 0;
+
+    // Execute type inference on the source pattern.
+    type_env = infer_definition(def_pool.get(src), var_pool, type_env, &mut last_type_index)
+        .map_err(|err| format!("In src pattern: {}", err))?;
+
+    // Collect the type sets once after applying the source patterm; we'll compare the typesets
+    // after we've also considered the destination pattern, and will emit supplementary InTypeset
+    // checks if they don't match.
+    let src_typesets = type_env
+        .vars
+        .iter()
+        .map(|&var_index| {
+            let var = var_pool.get_mut(var_index);
+            let tv = type_env.get_equivalent(&var.get_or_create_typevar());
+            (var_index, tv.get_typeset())
+        })
+        .collect::<Vec<_>>();
+
+    // Execute type inference on the destination pattern.
+    for (i, &def_index) in dst.iter().enumerate() {
+        let def = def_pool.get(def_index);
+        type_env = infer_definition(def, var_pool, type_env, &mut last_type_index)
+            .map_err(|err| format!("line {}: {}", i, err))?;
+    }
+
+    for (var_index, src_typeset) in src_typesets {
+        let var = var_pool.get(var_index);
+        if !var.has_free_typevar() {
+            continue;
+        }
+        let tv = type_env.get_equivalent(&var.get_typevar().unwrap());
+        let new_typeset = tv.get_typeset();
+        assert!(
+            new_typeset.is_subset(&src_typeset),
+            "type sets can only get narrower"
+        );
+        if new_typeset != src_typeset {
+            type_env.add_constraint(Constraint::InTypeset(tv.clone(), new_typeset.clone()));
+        }
+    }
+
+    type_env.normalize(var_pool);
+
+    Ok(type_env.extract(var_pool))
+}

cranelift/codegen/meta/src/cdsl/types.rs

@@ -0,0 +1,571 @@
+//! Cranelift ValueType hierarchy
+
+use std::fmt;
+
+use crate::shared::types as shared_types;
+use cranelift_codegen_shared::constants;
+
+// Rust name prefix used for the `rust_name` method.
+static _RUST_NAME_PREFIX: &str = "ir::types::";
+
+// ValueType variants (i8, i32, ...) are provided in `shared::types.rs`.
+
+/// A concrete SSA value type.
+///
+/// All SSA values have a type that is described by an instance of `ValueType`
+/// or one of its subclasses.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub(crate) enum ValueType {
+    Lane(LaneType),
+    Reference(ReferenceType),
+    Special(SpecialType),
+    Vector(VectorType),
+}
+
+impl ValueType {
+    /// Iterate through all of the lane types.
+    pub fn all_lane_types() -> LaneTypeIterator {
+        LaneTypeIterator::new()
+    }
+
+    /// Iterate through all of the special types (neither lanes nor vectors).
+    pub fn all_special_types() -> SpecialTypeIterator {
+        SpecialTypeIterator::new()
+    }
+
+    pub fn all_reference_types() -> ReferenceTypeIterator {
+        ReferenceTypeIterator::new()
+    }
+
+    /// Return a string containing the documentation comment for this type.
+    pub fn doc(&self) -> String {
+        match *self {
+            ValueType::Lane(l) => l.doc(),
+            ValueType::Reference(r) => r.doc(),
+            ValueType::Special(s) => s.doc(),
+            ValueType::Vector(ref v) => v.doc(),
+        }
+    }
+
+    /// Return the number of bits in a lane.
+    pub fn lane_bits(&self) -> u64 {
+        match *self {
+            ValueType::Lane(l) => l.lane_bits(),
+            ValueType::Reference(r) => r.lane_bits(),
+            ValueType::Special(s) => s.lane_bits(),
+            ValueType::Vector(ref v) => v.lane_bits(),
+        }
+    }
+
+    /// Return the number of lanes.
+    pub fn lane_count(&self) -> u64 {
+        match *self {
+            ValueType::Vector(ref v) => v.lane_count(),
+            _ => 1,
+        }
+    }
+
+    /// Find the number of bytes that this type occupies in memory.
+    pub fn membytes(&self) -> u64 {
+        self.width() / 8
+    }
+
+    /// Find the unique number associated with this type.
+    pub fn number(&self) -> Option<u8> {
+        match *self {
+            ValueType::Lane(l) => Some(l.number()),
+            ValueType::Reference(r) => Some(r.number()),
+            ValueType::Special(s) => Some(s.number()),
+            ValueType::Vector(ref v) => Some(v.number()),
+        }
+    }
+
+    /// Return the name of this type for generated Rust source files.
+    pub fn rust_name(&self) -> String {
+        format!("{}{}", _RUST_NAME_PREFIX, self.to_string().to_uppercase())
+    }
+
+    /// Return true iff:
+    ///     1. self and other have equal number of lanes
+    ///     2. each lane in self has at least as many bits as a lane in other
+    pub fn _wider_or_equal(&self, rhs: &ValueType) -> bool {
+        (self.lane_count() == rhs.lane_count()) && (self.lane_bits() >= rhs.lane_bits())
+    }
+
+    /// Return the total number of bits of an instance of this type.
+    pub fn width(&self) -> u64 {
+        self.lane_count() * self.lane_bits()
+    }
+}
+
+impl fmt::Display for ValueType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            ValueType::Lane(l) => l.fmt(f),
+            ValueType::Reference(r) => r.fmt(f),
+            ValueType::Special(s) => s.fmt(f),
+            ValueType::Vector(ref v) => v.fmt(f),
+        }
+    }
+}
+
+/// Create a ValueType from a given lane type.
+impl From<LaneType> for ValueType {
+    fn from(lane: LaneType) -> Self {
+        ValueType::Lane(lane)
+    }
+}
+
+/// Create a ValueType from a given reference type.
+impl From<ReferenceType> for ValueType {
+    fn from(reference: ReferenceType) -> Self {
+        ValueType::Reference(reference)
+    }
+}
+
+/// Create a ValueType from a given special type.
+impl From<SpecialType> for ValueType {
+    fn from(spec: SpecialType) -> Self {
+        ValueType::Special(spec)
+    }
+}
+
+/// Create a ValueType from a given vector type.
+impl From<VectorType> for ValueType {
+    fn from(vector: VectorType) -> Self {
+        ValueType::Vector(vector)
+    }
+}
+
+/// A concrete scalar type that can appear as a vector lane too.
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+pub(crate) enum LaneType {
+    Bool(shared_types::Bool),
+    Float(shared_types::Float),
+    Int(shared_types::Int),
+}
+
+impl LaneType {
+    /// Return a string containing the documentation comment for this lane type.
+    pub fn doc(self) -> String {
+        match self {
+            LaneType::Bool(_) => format!("A boolean type with {} bits.", self.lane_bits()),
+            LaneType::Float(shared_types::Float::F32) => String::from(
+                "A 32-bit floating point type represented in the IEEE 754-2008
+                *binary32* interchange format. This corresponds to the :c:type:`float`
+                type in most C implementations.",
+            ),
+            LaneType::Float(shared_types::Float::F64) => String::from(
+                "A 64-bit floating point type represented in the IEEE 754-2008
+                *binary64* interchange format. This corresponds to the :c:type:`double`
+                type in most C implementations.",
+            ),
+            LaneType::Int(_) if self.lane_bits() < 32 => format!(
+                "An integer type with {} bits.
+                WARNING: arithmetic on {}bit integers is incomplete",
+                self.lane_bits(),
+                self.lane_bits()
+            ),
+            LaneType::Int(_) => format!("An integer type with {} bits.", self.lane_bits()),
+        }
+    }
+
+    /// Return the number of bits in a lane.
+    pub fn lane_bits(self) -> u64 {
+        match self {
+            LaneType::Bool(ref b) => *b as u64,
+            LaneType::Float(ref f) => *f as u64,
+            LaneType::Int(ref i) => *i as u64,
+        }
+    }
+
+    /// Find the unique number associated with this lane type.
+    pub fn number(self) -> u8 {
+        constants::LANE_BASE
+            + match self {
+                LaneType::Bool(shared_types::Bool::B1) => 0,
+                LaneType::Bool(shared_types::Bool::B8) => 1,
+                LaneType::Bool(shared_types::Bool::B16) => 2,
+                LaneType::Bool(shared_types::Bool::B32) => 3,
+                LaneType::Bool(shared_types::Bool::B64) => 4,
+                LaneType::Bool(shared_types::Bool::B128) => 5,
+                LaneType::Int(shared_types::Int::I8) => 6,
+                LaneType::Int(shared_types::Int::I16) => 7,
+                LaneType::Int(shared_types::Int::I32) => 8,
+                LaneType::Int(shared_types::Int::I64) => 9,
+                LaneType::Int(shared_types::Int::I128) => 10,
+                LaneType::Float(shared_types::Float::F32) => 11,
+                LaneType::Float(shared_types::Float::F64) => 12,
+            }
+    }
+
+    pub fn bool_from_bits(num_bits: u16) -> LaneType {
+        LaneType::Bool(match num_bits {
+            1 => shared_types::Bool::B1,
+            8 => shared_types::Bool::B8,
+            16 => shared_types::Bool::B16,
+            32 => shared_types::Bool::B32,
+            64 => shared_types::Bool::B64,
+            128 => shared_types::Bool::B128,
+            _ => unreachable!("unxpected num bits for bool"),
+        })
+    }
+
+    pub fn int_from_bits(num_bits: u16) -> LaneType {
+        LaneType::Int(match num_bits {
+            8 => shared_types::Int::I8,
+            16 => shared_types::Int::I16,
+            32 => shared_types::Int::I32,
+            64 => shared_types::Int::I64,
+            128 => shared_types::Int::I128,
+            _ => unreachable!("unxpected num bits for int"),
+        })
+    }
+
+    pub fn float_from_bits(num_bits: u16) -> LaneType {
+        LaneType::Float(match num_bits {
+            32 => shared_types::Float::F32,
+            64 => shared_types::Float::F64,
+            _ => unreachable!("unxpected num bits for float"),
+        })
+    }
+
+    pub fn by(self, lanes: u16) -> ValueType {
+        if lanes == 1 {
+            self.into()
+        } else {
+            ValueType::Vector(VectorType::new(self, lanes.into()))
+        }
+    }
+
+    pub fn is_float(self) -> bool {
+        match self {
+            LaneType::Float(_) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_int(self) -> bool {
+        match self {
+            LaneType::Int(_) => true,
+            _ => false,
+        }
+    }
+}
+
+impl fmt::Display for LaneType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            LaneType::Bool(_) => write!(f, "b{}", self.lane_bits()),
+            LaneType::Float(_) => write!(f, "f{}", self.lane_bits()),
+            LaneType::Int(_) => write!(f, "i{}", self.lane_bits()),
+        }
+    }
+}
+
+impl fmt::Debug for LaneType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let inner_msg = format!("bits={}", self.lane_bits());
+        write!(
+            f,
+            "{}",
+            match *self {
+                LaneType::Bool(_) => format!("BoolType({})", inner_msg),
+                LaneType::Float(_) => format!("FloatType({})", inner_msg),
+                LaneType::Int(_) => format!("IntType({})", inner_msg),
+            }
+        )
+    }
+}
+
+/// Create a LaneType from a given bool variant.
+impl From<shared_types::Bool> for LaneType {
+    fn from(b: shared_types::Bool) -> Self {
+        LaneType::Bool(b)
+    }
+}
+
+/// Create a LaneType from a given float variant.
+impl From<shared_types::Float> for LaneType {
+    fn from(f: shared_types::Float) -> Self {
+        LaneType::Float(f)
+    }
+}
+
+/// Create a LaneType from a given int variant.
+impl From<shared_types::Int> for LaneType {
+    fn from(i: shared_types::Int) -> Self {
+        LaneType::Int(i)
+    }
+}
+
+/// An iterator for different lane types.
+pub(crate) struct LaneTypeIterator {
+    bool_iter: shared_types::BoolIterator,
+    int_iter: shared_types::IntIterator,
+    float_iter: shared_types::FloatIterator,
+}
+
+impl LaneTypeIterator {
+    /// Create a new lane type iterator.
+    fn new() -> Self {
+        Self {
+            bool_iter: shared_types::BoolIterator::new(),
+            int_iter: shared_types::IntIterator::new(),
+            float_iter: shared_types::FloatIterator::new(),
+        }
+    }
+}
+
+impl Iterator for LaneTypeIterator {
+    type Item = LaneType;
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(b) = self.bool_iter.next() {
+            Some(LaneType::from(b))
+        } else if let Some(i) = self.int_iter.next() {
+            Some(LaneType::from(i))
+        } else if let Some(f) = self.float_iter.next() {
+            Some(LaneType::from(f))
+        } else {
+            None
+        }
+    }
+}
+
+/// A concrete SIMD vector type.
+///
+/// A vector type has a lane type which is an instance of `LaneType`,
+/// and a positive number of lanes.
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub(crate) struct VectorType {
+    base: LaneType,
+    lanes: u64,
+}
+
+impl VectorType {
+    /// Initialize a new integer type with `n` bits.
+    pub fn new(base: LaneType, lanes: u64) -> Self {
+        Self { base, lanes }
+    }
+
+    /// Return a string containing the documentation comment for this vector type.
+    pub fn doc(&self) -> String {
+        format!(
+            "A SIMD vector with {} lanes containing a `{}` each.",
+            self.lane_count(),
+            self.base
+        )
+    }
+
+    /// Return the number of bits in a lane.
+    pub fn lane_bits(&self) -> u64 {
+        self.base.lane_bits()
+    }
+
+    /// Return the number of lanes.
+    pub fn lane_count(&self) -> u64 {
+        self.lanes
+    }
+
+    /// Return the lane type.
+    pub fn lane_type(&self) -> LaneType {
+        self.base
+    }
+
+    /// Find the unique number associated with this vector type.
+    ///
+    /// Vector types are encoded with the lane type in the low 4 bits and
+    /// log2(lanes) in the high 4 bits, giving a range of 2-256 lanes.
+    pub fn number(&self) -> u8 {
+        let lanes_log_2: u32 = 63 - self.lane_count().leading_zeros();
+        let base_num = u32::from(self.base.number());
+        let num = (lanes_log_2 << 4) + base_num;
+        num as u8
+    }
+}
+
+impl fmt::Display for VectorType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}x{}", self.base, self.lane_count())
+    }
+}
+
+impl fmt::Debug for VectorType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "VectorType(base={}, lanes={})",
+            self.base,
+            self.lane_count()
+        )
+    }
+}
+
+/// A concrete scalar type that is neither a vector nor a lane type.
+///
+/// Special types cannot be used to form vectors.
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+pub(crate) enum SpecialType {
+    Flag(shared_types::Flag),
+}
+
+impl SpecialType {
+    /// Return a string containing the documentation comment for this special type.
+    pub fn doc(self) -> String {
+        match self {
+            SpecialType::Flag(shared_types::Flag::IFlags) => String::from(
+                "CPU flags representing the result of an integer comparison. These flags
+                can be tested with an :type:`intcc` condition code.",
+            ),
+            SpecialType::Flag(shared_types::Flag::FFlags) => String::from(
+                "CPU flags representing the result of a floating point comparison. These
+                flags can be tested with a :type:`floatcc` condition code.",
+            ),
+        }
+    }
+
+    /// Return the number of bits in a lane.
+    pub fn lane_bits(self) -> u64 {
+        match self {
+            SpecialType::Flag(_) => 0,
+        }
+    }
+
+    /// Find the unique number associated with this special type.
+    pub fn number(self) -> u8 {
+        match self {
+            SpecialType::Flag(shared_types::Flag::IFlags) => 1,
+            SpecialType::Flag(shared_types::Flag::FFlags) => 2,
+        }
+    }
+}
+
+impl fmt::Display for SpecialType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            SpecialType::Flag(shared_types::Flag::IFlags) => write!(f, "iflags"),
+            SpecialType::Flag(shared_types::Flag::FFlags) => write!(f, "fflags"),
+        }
+    }
+}
+
+impl fmt::Debug for SpecialType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "{}",
+            match *self {
+                SpecialType::Flag(_) => format!("FlagsType({})", self),
+            }
+        )
+    }
+}
+
+impl From<shared_types::Flag> for SpecialType {
+    fn from(f: shared_types::Flag) -> Self {
+        SpecialType::Flag(f)
+    }
+}
+
+pub(crate) struct SpecialTypeIterator {
+    flag_iter: shared_types::FlagIterator,
+}
+
+impl SpecialTypeIterator {
+    fn new() -> Self {
+        Self {
+            flag_iter: shared_types::FlagIterator::new(),
+        }
+    }
+}
+
+impl Iterator for SpecialTypeIterator {
+    type Item = SpecialType;
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(f) = self.flag_iter.next() {
+            Some(SpecialType::from(f))
+        } else {
+            None
+        }
+    }
+}
+
+/// Reference type is scalar type, but not lane type.
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+pub(crate) struct ReferenceType(pub shared_types::Reference);
+
+impl ReferenceType {
+    /// Return a string containing the documentation comment for this reference type.
+    pub fn doc(self) -> String {
+        format!("An opaque reference type with {} bits.", self.lane_bits())
+    }
+
+    /// Return the number of bits in a lane.
+    pub fn lane_bits(self) -> u64 {
+        match self.0 {
+            shared_types::Reference::R32 => 32,
+            shared_types::Reference::R64 => 64,
+        }
+    }
+
+    /// Find the unique number associated with this reference type.
+    pub fn number(self) -> u8 {
+        constants::REFERENCE_BASE
+            + match self {
+                ReferenceType(shared_types::Reference::R32) => 0,
+                ReferenceType(shared_types::Reference::R64) => 1,
+            }
+    }
+
+    pub fn ref_from_bits(num_bits: u16) -> ReferenceType {
+        ReferenceType(match num_bits {
+            32 => shared_types::Reference::R32,
+            64 => shared_types::Reference::R64,
+            _ => unreachable!("unexpected number of bits for a reference type"),
+        })
+    }
+}
+
+impl fmt::Display for ReferenceType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "r{}", self.lane_bits())
+    }
+}
+
+impl fmt::Debug for ReferenceType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "ReferenceType(bits={})", self.lane_bits())
+    }
+}
+
+/// Create a ReferenceType from a given reference variant.
+impl From<shared_types::Reference> for ReferenceType {
+    fn from(r: shared_types::Reference) -> Self {
+        ReferenceType(r)
+    }
+}
+
+/// An iterator for different reference types.
+pub(crate) struct ReferenceTypeIterator {
+    reference_iter: shared_types::ReferenceIterator,
+}
+
+impl ReferenceTypeIterator {
+    /// Create a new reference type iterator.
+    fn new() -> Self {
+        Self {
+            reference_iter: shared_types::ReferenceIterator::new(),
+        }
+    }
+}
+
+impl Iterator for ReferenceTypeIterator {
+    type Item = ReferenceType;
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(r) = self.reference_iter.next() {
+            Some(ReferenceType::from(r))
+        } else {
+            None
+        }
+    }
+}

cranelift/codegen/meta/src/cdsl/xform.rs

@@ -0,0 +1,484 @@
+use crate::cdsl::ast::{
+    Apply, BlockPool, ConstPool, DefIndex, DefPool, DummyDef, DummyExpr, Expr, PatternPosition,
+    VarIndex, VarPool,
+};
+use crate::cdsl::instructions::Instruction;
+use crate::cdsl::type_inference::{infer_transform, TypeEnvironment};
+use crate::cdsl::typevar::TypeVar;
+
+use cranelift_entity::{entity_impl, PrimaryMap};
+
+use std::collections::{HashMap, HashSet};
+use std::iter::FromIterator;
+
+/// An instruction transformation consists of a source and destination pattern.
+///
+/// Patterns are expressed in *register transfer language* as tuples of Def or Expr nodes. A
+/// pattern may optionally have a sequence of TypeConstraints, that additionally limit the set of
+/// cases when it applies.
+///
+/// The source pattern can contain only a single instruction.
+pub(crate) struct Transform {
+    pub src: DefIndex,
+    pub dst: Vec<DefIndex>,
+    pub var_pool: VarPool,
+    pub def_pool: DefPool,
+    pub block_pool: BlockPool,
+    pub const_pool: ConstPool,
+    pub type_env: TypeEnvironment,
+}
+
+type SymbolTable = HashMap<String, VarIndex>;
+
+impl Transform {
+    fn new(src: DummyDef, dst: Vec<DummyDef>) -> Self {
+        let mut var_pool = VarPool::new();
+        let mut def_pool = DefPool::new();
+        let mut block_pool = BlockPool::new();
+        let mut const_pool = ConstPool::new();
+
+        let mut input_vars: Vec<VarIndex> = Vec::new();
+        let mut defined_vars: Vec<VarIndex> = Vec::new();
+
+        // Maps variable names to our own Var copies.
+        let mut symbol_table: SymbolTable = SymbolTable::new();
+
+        // Rewrite variables in src and dst using our own copies.
+        let src = rewrite_def_list(
+            PatternPosition::Source,
+            vec![src],
+            &mut symbol_table,
+            &mut input_vars,
+            &mut defined_vars,
+            &mut var_pool,
+            &mut def_pool,
+            &mut block_pool,
+            &mut const_pool,
+        )[0];
+
+        let num_src_inputs = input_vars.len();
+
+        let dst = rewrite_def_list(
+            PatternPosition::Destination,
+            dst,
+            &mut symbol_table,
+            &mut input_vars,
+            &mut defined_vars,
+            &mut var_pool,
+            &mut def_pool,
+            &mut block_pool,
+            &mut const_pool,
+        );
+
+        // Sanity checks.
+        for &var_index in &input_vars {
+            assert!(
+                var_pool.get(var_index).is_input(),
+                format!("'{:?}' used as both input and def", var_pool.get(var_index))
+            );
+        }
+        assert!(
+            input_vars.len() == num_src_inputs,
+            format!(
+                "extra input vars in dst pattern: {:?}",
+                input_vars
+                    .iter()
+                    .map(|&i| var_pool.get(i))
+                    .skip(num_src_inputs)
+                    .collect::<Vec<_>>()
+            )
+        );
+
+        // Perform type inference and cleanup.
+        let type_env = infer_transform(src, &dst, &def_pool, &mut var_pool).unwrap();
+
+        // Sanity check: the set of inferred free type variables should be a subset of the type
+        // variables corresponding to Vars appearing in the source pattern.
+        {
+            let free_typevars: HashSet<TypeVar> =
+                HashSet::from_iter(type_env.free_typevars(&mut var_pool));
+            let src_tvs = HashSet::from_iter(
+                input_vars
+                    .clone()
+                    .iter()
+                    .chain(
+                        defined_vars
+                            .iter()
+                            .filter(|&&var_index| !var_pool.get(var_index).is_temp()),
+                    )
+                    .map(|&var_index| var_pool.get(var_index).get_typevar())
+                    .filter(|maybe_var| maybe_var.is_some())
+                    .map(|var| var.unwrap()),
+            );
+            if !free_typevars.is_subset(&src_tvs) {
+                let missing_tvs = (&free_typevars - &src_tvs)
+                    .iter()
+                    .map(|tv| tv.name.clone())
+                    .collect::<Vec<_>>()
+                    .join(", ");
+                panic!("Some free vars don't appear in src: {}", missing_tvs);
+            }
+        }
+
+        for &var_index in input_vars.iter().chain(defined_vars.iter()) {
+            let var = var_pool.get_mut(var_index);
+            let canon_tv = type_env.get_equivalent(&var.get_or_create_typevar());
+            var.set_typevar(canon_tv);
+        }
+
+        Self {
+            src,
+            dst,
+            var_pool,
+            def_pool,
+            block_pool,
+            const_pool,
+            type_env,
+        }
+    }
+
+    fn verify_legalize(&self) {
+        let def = self.def_pool.get(self.src);
+        for &var_index in def.defined_vars.iter() {
+            let defined_var = self.var_pool.get(var_index);
+            assert!(
+                defined_var.is_output(),
+                format!("{:?} not defined in the destination pattern", defined_var)
+            );
+        }
+    }
+}
+
+/// Inserts, if not present, a name in the `symbol_table`. Then returns its index in the variable
+/// pool `var_pool`. If the variable was not present in the symbol table, then add it to the list of
+/// `defined_vars`.
+fn var_index(
+    name: &str,
+    symbol_table: &mut SymbolTable,
+    defined_vars: &mut Vec<VarIndex>,
+    var_pool: &mut VarPool,
+) -> VarIndex {
+    let name = name.to_string();
+    match symbol_table.get(&name) {
+        Some(&existing_var) => existing_var,
+        None => {
+            // Materialize the variable.
+            let new_var = var_pool.create(name.clone());
+            symbol_table.insert(name, new_var);
+            defined_vars.push(new_var);
+            new_var
+        }
+    }
+}
+
+/// Given a list of symbols defined in a Def, rewrite them to local symbols. Yield the new locals.
+fn rewrite_defined_vars(
+    position: PatternPosition,
+    dummy_def: &DummyDef,
+    def_index: DefIndex,
+    symbol_table: &mut SymbolTable,
+    defined_vars: &mut Vec<VarIndex>,
+    var_pool: &mut VarPool,
+) -> Vec<VarIndex> {
+    let mut new_defined_vars = Vec::new();
+    for var in &dummy_def.defined_vars {
+        let own_var = var_index(&var.name, symbol_table, defined_vars, var_pool);
+        var_pool.get_mut(own_var).set_def(position, def_index);
+        new_defined_vars.push(own_var);
+    }
+    new_defined_vars
+}
+
+/// Find all uses of variables in `expr` and replace them with our own local symbols.
+fn rewrite_expr(
+    position: PatternPosition,
+    dummy_expr: DummyExpr,
+    symbol_table: &mut SymbolTable,
+    input_vars: &mut Vec<VarIndex>,
+    var_pool: &mut VarPool,
+    const_pool: &mut ConstPool,
+) -> Apply {
+    let (apply_target, dummy_args) = if let DummyExpr::Apply(apply_target, dummy_args) = dummy_expr
+    {
+        (apply_target, dummy_args)
+    } else {
+        panic!("we only rewrite apply expressions");
+    };
+
+    assert_eq!(
+        apply_target.inst().operands_in.len(),
+        dummy_args.len(),
+        "number of arguments in instruction {} is incorrect\nexpected: {:?}",
+        apply_target.inst().name,
+        apply_target
+            .inst()
+            .operands_in
+            .iter()
+            .map(|operand| format!("{}: {}", operand.name, operand.kind.rust_type))
+            .collect::<Vec<_>>(),
+    );
+
+    let mut args = Vec::new();
+    for (i, arg) in dummy_args.into_iter().enumerate() {
+        match arg {
+            DummyExpr::Var(var) => {
+                let own_var = var_index(&var.name, symbol_table, input_vars, var_pool);
+                let var = var_pool.get(own_var);
+                assert!(
+                    var.is_input() || var.get_def(position).is_some(),
+                    format!("{:?} used as both input and def", var)
+                );
+                args.push(Expr::Var(own_var));
+            }
+            DummyExpr::Literal(literal) => {
+                assert!(!apply_target.inst().operands_in[i].is_value());
+                args.push(Expr::Literal(literal));
+            }
+            DummyExpr::Constant(constant) => {
+                let const_name = const_pool.insert(constant.0);
+                // Here we abuse var_index by passing an empty, immediately-dropped vector to
+                // `defined_vars`; the reason for this is that unlike the `Var` case above,
+                // constants will create a variable that is not an input variable (it is tracked
+                // instead by ConstPool).
+                let const_var = var_index(&const_name, symbol_table, &mut vec![], var_pool);
+                args.push(Expr::Var(const_var));
+            }
+            DummyExpr::Apply(..) => {
+                panic!("Recursive apply is not allowed.");
+            }
+            DummyExpr::Block(_block) => {
+                panic!("Blocks are not valid arguments.");
+            }
+        }
+    }
+
+    Apply::new(apply_target, args)
+}
+
+#[allow(clippy::too_many_arguments)]
+fn rewrite_def_list(
+    position: PatternPosition,
+    dummy_defs: Vec<DummyDef>,
+    symbol_table: &mut SymbolTable,
+    input_vars: &mut Vec<VarIndex>,
+    defined_vars: &mut Vec<VarIndex>,
+    var_pool: &mut VarPool,
+    def_pool: &mut DefPool,
+    block_pool: &mut BlockPool,
+    const_pool: &mut ConstPool,
+) -> Vec<DefIndex> {
+    let mut new_defs = Vec::new();
+    // Register variable names of new blocks first as a block name can be used to jump forward. Thus
+    // the name has to be registered first to avoid misinterpreting it as an input-var.
+    for dummy_def in dummy_defs.iter() {
+        if let DummyExpr::Block(ref var) = dummy_def.expr {
+            var_index(&var.name, symbol_table, defined_vars, var_pool);
+        }
+    }
+
+    // Iterate over the definitions and blocks, to map variables names to inputs or outputs.
+    for dummy_def in dummy_defs {
+        let def_index = def_pool.next_index();
+
+        let new_defined_vars = rewrite_defined_vars(
+            position,
+            &dummy_def,
+            def_index,
+            symbol_table,
+            defined_vars,
+            var_pool,
+        );
+        if let DummyExpr::Block(var) = dummy_def.expr {
+            let var_index = *symbol_table
+                .get(&var.name)
+                .or_else(|| {
+                    panic!(
+                        "Block {} was not registered during the first visit",
+                        var.name
+                    )
+                })
+                .unwrap();
+            var_pool.get_mut(var_index).set_def(position, def_index);
+            block_pool.create_block(var_index, def_index);
+        } else {
+            let new_apply = rewrite_expr(
+                position,
+                dummy_def.expr,
+                symbol_table,
+                input_vars,
+                var_pool,
+                const_pool,
+            );
+
+            assert!(
+                def_pool.next_index() == def_index,
+                "shouldn't have created new defs in the meanwhile"
+            );
+            assert_eq!(
+                new_apply.inst.value_results.len(),
+                new_defined_vars.len(),
+                "number of Var results in instruction is incorrect"
+            );
+
+            new_defs.push(def_pool.create_inst(new_apply, new_defined_vars));
+        }
+    }
+    new_defs
+}
+
+/// A group of related transformations.
+pub(crate) struct TransformGroup {
+    pub name: &'static str,
+    pub doc: &'static str,
+    pub chain_with: Option<TransformGroupIndex>,
+    pub isa_name: Option<&'static str>,
+    pub id: TransformGroupIndex,
+
+    /// Maps Instruction camel_case names to custom legalization functions names.
+    pub custom_legalizes: HashMap<String, &'static str>,
+    pub transforms: Vec<Transform>,
+}
+
+impl TransformGroup {
+    pub fn rust_name(&self) -> String {
+        match self.isa_name {
+            Some(_) => {
+                // This is a function in the same module as the LEGALIZE_ACTIONS table referring to
+                // it.
+                self.name.to_string()
+            }
+            None => format!("crate::legalizer::{}", self.name),
+        }
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub(crate) struct TransformGroupIndex(u32);
+entity_impl!(TransformGroupIndex);
+
+pub(crate) struct TransformGroupBuilder {
+    name: &'static str,
+    doc: &'static str,
+    chain_with: Option<TransformGroupIndex>,
+    isa_name: Option<&'static str>,
+    pub custom_legalizes: HashMap<String, &'static str>,
+    pub transforms: Vec<Transform>,
+}
+
+impl TransformGroupBuilder {
+    pub fn new(name: &'static str, doc: &'static str) -> Self {
+        Self {
+            name,
+            doc,
+            chain_with: None,
+            isa_name: None,
+            custom_legalizes: HashMap::new(),
+            transforms: Vec::new(),
+        }
+    }
+
+    pub fn chain_with(mut self, next_id: TransformGroupIndex) -> Self {
+        assert!(self.chain_with.is_none());
+        self.chain_with = Some(next_id);
+        self
+    }
+
+    pub fn isa(mut self, isa_name: &'static str) -> Self {
+        assert!(self.isa_name.is_none());
+        self.isa_name = Some(isa_name);
+        self
+    }
+
+    /// Add a custom legalization action for `inst`.
+    ///
+    /// The `func_name` parameter is the fully qualified name of a Rust function which takes the
+    /// same arguments as the `isa::Legalize` actions.
+    ///
+    /// The custom function will be called to legalize `inst` and any return value is ignored.
+    pub fn custom_legalize(&mut self, inst: &Instruction, func_name: &'static str) {
+        assert!(
+            self.custom_legalizes
+                .insert(inst.camel_name.clone(), func_name)
+                .is_none(),
+            format!(
+                "custom legalization action for {} inserted twice",
+                inst.name
+            )
+        );
+    }
+
+    /// Add a legalization pattern to this group.
+    pub fn legalize(&mut self, src: DummyDef, dst: Vec<DummyDef>) {
+        let transform = Transform::new(src, dst);
+        transform.verify_legalize();
+        self.transforms.push(transform);
+    }
+
+    pub fn build_and_add_to(self, owner: &mut TransformGroups) -> TransformGroupIndex {
+        let next_id = owner.next_key();
+        owner.add(TransformGroup {
+            name: self.name,
+            doc: self.doc,
+            isa_name: self.isa_name,
+            id: next_id,
+            chain_with: self.chain_with,
+            custom_legalizes: self.custom_legalizes,
+            transforms: self.transforms,
+        })
+    }
+}
+
+pub(crate) struct TransformGroups {
+    groups: PrimaryMap<TransformGroupIndex, TransformGroup>,
+}
+
+impl TransformGroups {
+    pub fn new() -> Self {
+        Self {
+            groups: PrimaryMap::new(),
+        }
+    }
+    pub fn add(&mut self, new_group: TransformGroup) -> TransformGroupIndex {
+        for group in self.groups.values() {
+            assert!(
+                group.name != new_group.name,
+                format!("trying to insert {} for the second time", new_group.name)
+            );
+        }
+        self.groups.push(new_group)
+    }
+    pub fn get(&self, id: TransformGroupIndex) -> &TransformGroup {
+        &self.groups[id]
+    }
+    fn next_key(&self) -> TransformGroupIndex {
+        self.groups.next_key()
+    }
+    pub fn by_name(&self, name: &'static str) -> &TransformGroup {
+        for group in self.groups.values() {
+            if group.name == name {
+                return group;
+            }
+        }
+        panic!(format!("transform group with name {} not found", name));
+    }
+}
+
+#[test]
+#[should_panic]
+fn test_double_custom_legalization() {
+    use crate::cdsl::formats::InstructionFormatBuilder;
+    use crate::cdsl::instructions::{AllInstructions, InstructionBuilder, InstructionGroupBuilder};
+
+    let nullary = InstructionFormatBuilder::new("nullary").build();
+
+    let mut dummy_all = AllInstructions::new();
+    let mut inst_group = InstructionGroupBuilder::new(&mut dummy_all);
+    inst_group.push(InstructionBuilder::new("dummy", "doc", &nullary));
+
+    let inst_group = inst_group.build();
+    let dummy_inst = inst_group.by_name("dummy");
+
+    let mut transform_group = TransformGroupBuilder::new("test", "doc");
+    transform_group.custom_legalize(&dummy_inst, "custom 1");
+    transform_group.custom_legalize(&dummy_inst, "custom 2");
+}

cranelift/codegen/meta/src/default_map.rs

@@ -0,0 +1,20 @@
+//! Trait for extending `HashMap` with `get_or_default`.
+use std::collections::HashMap;
+use std::hash::Hash;
+
+pub(crate) trait MapWithDefault<K, V: Default> {
+    fn get_or_default(&mut self, k: K) -> &mut V;
+}
+
+impl<K: Eq + Hash, V: Default> MapWithDefault<K, V> for HashMap<K, V> {
+    fn get_or_default(&mut self, k: K) -> &mut V {
+        self.entry(k).or_insert_with(V::default)
+    }
+}
+
+#[test]
+fn test_default() {
+    let mut hash_map = HashMap::new();
+    hash_map.insert(42, "hello");
+    assert_eq!(*hash_map.get_or_default(43), "");
+}

cranelift/codegen/meta/src/error.rs

@@ -0,0 +1,48 @@
+//! Error returned during meta code-generation.
+use std::fmt;
+use std::io;
+
+/// An error that occurred when the cranelift_codegen_meta crate was generating
+/// source files for the cranelift_codegen crate.
+#[derive(Debug)]
+pub struct Error {
+    inner: Box<ErrorInner>,
+}
+
+impl Error {
+    /// Create a new error object with the given message.
+    pub fn with_msg<S: Into<String>>(msg: S) -> Error {
+        Error {
+            inner: Box::new(ErrorInner::Msg(msg.into())),
+        }
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.inner)
+    }
+}
+
+impl From<io::Error> for Error {
+    fn from(e: io::Error) -> Self {
+        Error {
+            inner: Box::new(ErrorInner::IoError(e)),
+        }
+    }
+}
+
+#[derive(Debug)]
+enum ErrorInner {
+    Msg(String),
+    IoError(io::Error),
+}
+
+impl fmt::Display for ErrorInner {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            ErrorInner::Msg(ref s) => write!(f, "{}", s),
+            ErrorInner::IoError(ref e) => write!(f, "{}", e),
+        }
+    }
+}

cranelift/codegen/meta/src/gen_binemit.rs

@@ -0,0 +1,224 @@
+//! Generate binary emission code for each ISA.
+
+use cranelift_entity::EntityRef;
+
+use crate::error;
+use crate::srcgen::Formatter;
+
+use crate::cdsl::recipes::{EncodingRecipe, OperandConstraint, Recipes};
+
+/// Generate code to handle a single recipe.
+///
+/// - Unpack the instruction data, knowing the format.
+/// - Determine register locations for operands with register constraints.
+/// - Determine stack slot locations for operands with stack constraints.
+/// - Call hand-written code for the actual emission.
+fn gen_recipe(recipe: &EncodingRecipe, fmt: &mut Formatter) {
+    let inst_format = &recipe.format;
+    let num_value_ops = inst_format.num_value_operands;
+
+    // TODO: Set want_args to true for only MultiAry instructions instead of all formats with value
+    // list.
+    let want_args = inst_format.has_value_list
+        || recipe.operands_in.iter().any(|c| match c {
+            OperandConstraint::RegClass(_) | OperandConstraint::Stack(_) => true,
+            OperandConstraint::FixedReg(_) | OperandConstraint::TiedInput(_) => false,
+        });
+    assert!(!want_args || num_value_ops > 0 || inst_format.has_value_list);
+
+    let want_outs = recipe.operands_out.iter().any(|c| match c {
+        OperandConstraint::RegClass(_) | OperandConstraint::Stack(_) => true,
+        OperandConstraint::FixedReg(_) | OperandConstraint::TiedInput(_) => false,
+    });
+
+    let is_regmove = ["RegMove", "RegSpill", "RegFill"].contains(&inst_format.name);
+
+    // Unpack the instruction data.
+    fmtln!(fmt, "if let InstructionData::{} {{", inst_format.name);
+    fmt.indent(|fmt| {
+        fmt.line("opcode,");
+        for f in &inst_format.imm_fields {
+            fmtln!(fmt, "{},", f.member);
+        }
+        if want_args {
+            if inst_format.has_value_list || num_value_ops > 1 {
+                fmt.line("ref args,");
+            } else {
+                fmt.line("arg,");
+            }
+        }
+        fmt.line("..");
+
+        fmt.outdented_line("} = *inst_data {");
+
+        // Pass recipe arguments in this order: inputs, imm_fields, outputs.
+        let mut args = String::new();
+
+        if want_args && !is_regmove {
+            if inst_format.has_value_list {
+                fmt.line("let args = args.as_slice(&func.dfg.value_lists);");
+            } else if num_value_ops == 1 {
+                fmt.line("let args = [arg];");
+            }
+            args += &unwrap_values(&recipe.operands_in, "in", "args", fmt);
+        }
+
+        for f in &inst_format.imm_fields {
+            args += &format!(", {}", f.member);
+        }
+
+        // Unwrap interesting output arguments.
+        if want_outs {
+            if recipe.operands_out.len() == 1 {
+                fmt.line("let results = [func.dfg.first_result(inst)];")
+            } else {
+                fmt.line("let results = func.dfg.inst_results(inst);");
+            }
+            args += &unwrap_values(&recipe.operands_out, "out", "results", fmt);
+        }
+
+        // Optimization: Only update the register diversion tracker for regmove instructions.
+        if is_regmove {
+            fmt.line("divert.apply(inst_data);")
+        }
+
+        match &recipe.emit {
+            Some(emit) => {
+                fmt.multi_line(emit);
+                fmt.line("return;");
+            }
+            None => {
+                fmtln!(
+                    fmt,
+                    "return recipe_{}(func, inst, sink, bits{});",
+                    recipe.name.to_lowercase(),
+                    args
+                );
+            }
+        }
+    });
+    fmt.line("}");
+}
+
+/// Emit code that unwraps values living in registers or stack slots.
+///
+/// :param args: Input or output constraints.
+/// :param prefix: Prefix to be used for the generated local variables.
+/// :param values: Name of slice containing the values to be unwrapped.
+/// :returns: Comma separated list of the generated variables
+fn unwrap_values(
+    args: &[OperandConstraint],
+    prefix: &str,
+    values_slice: &str,
+    fmt: &mut Formatter,
+) -> String {
+    let mut varlist = String::new();
+    for (i, cst) in args.iter().enumerate() {
+        match cst {
+            OperandConstraint::RegClass(_reg_class) => {
+                let v = format!("{}_reg{}", prefix, i);
+                varlist += &format!(", {}", v);
+                fmtln!(
+                    fmt,
+                    "let {} = divert.reg({}[{}], &func.locations);",
+                    v,
+                    values_slice,
+                    i
+                );
+            }
+            OperandConstraint::Stack(stack) => {
+                let v = format!("{}_stk{}", prefix, i);
+                varlist += &format!(", {}", v);
+                fmtln!(fmt, "let {} = StackRef::masked(", v);
+                fmt.indent(|fmt| {
+                    fmtln!(
+                        fmt,
+                        "divert.stack({}[{}], &func.locations),",
+                        values_slice,
+                        i
+                    );
+                    fmt.line(format!("{},", stack.stack_base_mask()));
+                    fmt.line("&func.stack_slots,");
+                });
+                fmt.line(").unwrap();");
+            }
+            _ => {}
+        }
+    }
+    varlist
+}
+
+fn gen_isa(isa_name: &str, recipes: &Recipes, fmt: &mut Formatter) {
+    fmt.doc_comment(format!(
+        "Emit binary machine code for `inst` for the {} ISA.",
+        isa_name
+    ));
+
+    if recipes.is_empty() {
+        fmt.line("pub fn emit_inst<CS: CodeSink + ?Sized>(");
+        fmt.indent(|fmt| {
+            fmt.line("func: &Function,");
+            fmt.line("inst: Inst,");
+            fmt.line("_divert: &mut RegDiversions,");
+            fmt.line("_sink: &mut CS,");
+            fmt.line("_isa: &dyn TargetIsa,");
+        });
+        fmt.line(") {");
+        fmt.indent(|fmt| {
+            // No encoding recipes: Emit a stub.
+            fmt.line("bad_encoding(func, inst)");
+        });
+        fmt.line("}");
+        return;
+    }
+
+    fmt.line("#[allow(unused_variables, unreachable_code)]");
+    fmt.line("pub fn emit_inst<CS: CodeSink + ?Sized>(");
+    fmt.indent(|fmt| {
+        fmt.line("func: &Function,");
+        fmt.line("inst: Inst,");
+        fmt.line("divert: &mut RegDiversions,");
+        fmt.line("sink: &mut CS,");
+        fmt.line("isa: &dyn TargetIsa,")
+    });
+
+    fmt.line(") {");
+    fmt.indent(|fmt| {
+        fmt.line("let encoding = func.encodings[inst];");
+        fmt.line("let bits = encoding.bits();");
+        fmt.line("let inst_data = &func.dfg[inst];");
+        fmt.line("match encoding.recipe() {");
+        fmt.indent(|fmt| {
+            for (i, recipe) in recipes.iter() {
+                fmt.comment(format!("Recipe {}", recipe.name));
+                fmtln!(fmt, "{} => {{", i.index());
+                fmt.indent(|fmt| {
+                    gen_recipe(recipe, fmt);
+                });
+                fmt.line("}");
+            }
+            fmt.line("_ => {},");
+        });
+        fmt.line("}");
+
+        // Allow for unencoded ghost instructions. The verifier will check details.
+        fmt.line("if encoding.is_legal() {");
+        fmt.indent(|fmt| {
+            fmt.line("bad_encoding(func, inst);");
+        });
+        fmt.line("}");
+    });
+    fmt.line("}");
+}
+
+pub(crate) fn generate(
+    isa_name: &str,
+    recipes: &Recipes,
+    binemit_filename: &str,
+    out_dir: &str,
+) -> Result<(), error::Error> {
+    let mut fmt = Formatter::new();
+    gen_isa(isa_name, recipes, &mut fmt);
+    fmt.update_file(binemit_filename, out_dir)?;
+    Ok(())
+}

cranelift/codegen/meta/src/gen_legalizer.rs

@@ -0,0 +1,727 @@
+//! Generate transformations to legalize instructions without encodings.
+use crate::cdsl::ast::{Def, DefPool, Expr, VarPool};
+use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::operands::Operand;
+use crate::cdsl::type_inference::Constraint;
+use crate::cdsl::typevar::{TypeSet, TypeVar};
+use crate::cdsl::xform::{Transform, TransformGroup, TransformGroups};
+
+use crate::error;
+use crate::gen_inst::gen_typesets_table;
+use crate::srcgen::Formatter;
+use crate::unique_table::UniqueTable;
+
+use std::collections::{HashMap, HashSet};
+use std::iter::FromIterator;
+
+/// Given a `Def` node, emit code that extracts all the instruction fields from
+/// `pos.func.dfg[iref]`.
+///
+/// Create local variables named after the `Var` instances in `node`.
+///
+/// Also create a local variable named `predicate` with the value of the evaluated instruction
+/// predicate, or `true` if the node has no predicate.
+fn unwrap_inst(transform: &Transform, fmt: &mut Formatter) -> bool {
+    let var_pool = &transform.var_pool;
+    let def_pool = &transform.def_pool;
+
+    let def = def_pool.get(transform.src);
+    let apply = &def.apply;
+    let inst = &apply.inst;
+    let iform = &inst.format;
+
+    fmt.comment(format!(
+        "Unwrap fields from instruction format {}",
+        def.to_comment_string(&transform.var_pool)
+    ));
+
+    // Extract the Var arguments.
+    let arg_names = apply
+        .args
+        .iter()
+        .enumerate()
+        .filter(|(arg_num, _)| {
+            // Variable args are specially handled after extracting args.
+            !inst.operands_in[*arg_num].is_varargs()
+        })
+        .map(|(arg_num, arg)| match &arg {
+            Expr::Var(var_index) => var_pool.get(*var_index).name.as_ref(),
+            Expr::Literal(_) => {
+                let n = inst.imm_opnums.iter().position(|&i| i == arg_num).unwrap();
+                iform.imm_fields[n].member
+            }
+        })
+        .collect::<Vec<_>>()
+        .join(", ");
+
+    // May we need "args" in the values consumed by predicates?
+    let emit_args = iform.num_value_operands >= 1 || iform.has_value_list;
+
+    // We need a tuple:
+    // - if there's at least one value operand, then we emit a variable for the value, and the
+    // value list as args.
+    // - otherwise, if there's the count of immediate operands added to the presence of a value list exceeds one.
+    let need_tuple = if iform.num_value_operands >= 1 {
+        true
+    } else {
+        let mut imm_and_varargs = inst
+            .operands_in
+            .iter()
+            .filter(|op| op.is_immediate_or_entityref())
+            .count();
+        if iform.has_value_list {
+            imm_and_varargs += 1;
+        }
+        imm_and_varargs > 1
+    };
+
+    let maybe_args = if emit_args { ", args" } else { "" };
+    let defined_values = format!("{}{}", arg_names, maybe_args);
+
+    let tuple_or_value = if need_tuple {
+        format!("({})", defined_values)
+    } else {
+        defined_values
+    };
+
+    fmtln!(
+        fmt,
+        "let {} = if let ir::InstructionData::{} {{",
+        tuple_or_value,
+        iform.name
+    );
+
+    fmt.indent(|fmt| {
+        // Fields are encoded directly.
+        for field in &iform.imm_fields {
+            fmtln!(fmt, "{},", field.member);
+        }
+
+        if iform.has_value_list || iform.num_value_operands > 1 {
+            fmt.line("ref args,");
+        } else if iform.num_value_operands == 1 {
+            fmt.line("arg,");
+        }
+
+        fmt.line("..");
+        fmt.outdented_line("} = pos.func.dfg[inst] {");
+
+        if iform.has_value_list {
+            fmt.line("let args = args.as_slice(&pos.func.dfg.value_lists);");
+        } else if iform.num_value_operands == 1 {
+            fmt.line("let args = [arg];")
+        }
+
+        // Generate the values for the tuple.
+        let emit_one_value =
+            |fmt: &mut Formatter, needs_comma: bool, op_num: usize, op: &Operand| {
+                let comma = if needs_comma { "," } else { "" };
+                if op.is_immediate_or_entityref() {
+                    let n = inst.imm_opnums.iter().position(|&i| i == op_num).unwrap();
+                    fmtln!(fmt, "{}{}", iform.imm_fields[n].member, comma);
+                } else if op.is_value() {
+                    let n = inst.value_opnums.iter().position(|&i| i == op_num).unwrap();
+                    fmtln!(fmt, "pos.func.dfg.resolve_aliases(args[{}]),", n);
+                } else {
+                    // This is a value list argument or a varargs.
+                    assert!(iform.has_value_list || op.is_varargs());
+                }
+            };
+
+        if need_tuple {
+            fmt.line("(");
+            fmt.indent(|fmt| {
+                for (op_num, op) in inst.operands_in.iter().enumerate() {
+                    let needs_comma = emit_args || op_num + 1 < inst.operands_in.len();
+                    emit_one_value(fmt, needs_comma, op_num, op);
+                }
+                if emit_args {
+                    fmt.line("args");
+                }
+            });
+            fmt.line(")");
+        } else {
+            // Only one of these can be true at the same time, otherwise we'd need a tuple.
+            emit_one_value(fmt, false, 0, &inst.operands_in[0]);
+            if emit_args {
+                fmt.line("args");
+            }
+        }
+
+        fmt.outdented_line("} else {");
+        fmt.line(r#"unreachable!("bad instruction format")"#);
+    });
+    fmtln!(fmt, "};");
+    fmt.empty_line();
+
+    assert_eq!(inst.operands_in.len(), apply.args.len());
+    for (i, op) in inst.operands_in.iter().enumerate() {
+        if op.is_varargs() {
+            let name = &var_pool
+                .get(apply.args[i].maybe_var().expect("vararg without name"))
+                .name;
+            let n = inst
+                .imm_opnums
+                .iter()
+                .chain(inst.value_opnums.iter())
+                .max()
+                .copied()
+                .unwrap_or(0);
+            fmtln!(fmt, "let {} = &Vec::from(&args[{}..]);", name, n);
+        }
+    }
+
+    for &op_num in &inst.value_opnums {
+        let arg = &apply.args[op_num];
+        if let Some(var_index) = arg.maybe_var() {
+            let var = var_pool.get(var_index);
+            if var.has_free_typevar() {
+                fmtln!(
+                    fmt,
+                    "let typeof_{} = pos.func.dfg.value_type({});",
+                    var.name,
+                    var.name
+                );
+            }
+        }
+    }
+
+    // If the definition creates results, detach the values and place them in locals.
+    let mut replace_inst = false;
+    if !def.defined_vars.is_empty() {
+        if def.defined_vars
+            == def_pool
+                .get(var_pool.get(def.defined_vars[0]).dst_def.unwrap())
+                .defined_vars
+        {
+            // Special case: The instruction replacing node defines the exact same values.
+            fmt.comment(format!(
+                "Results handled by {}.",
+                def_pool
+                    .get(var_pool.get(def.defined_vars[0]).dst_def.unwrap())
+                    .to_comment_string(var_pool)
+            ));
+
+            fmt.line("let r = pos.func.dfg.inst_results(inst);");
+            for (i, &var_index) in def.defined_vars.iter().enumerate() {
+                let var = var_pool.get(var_index);
+                fmtln!(fmt, "let {} = &r[{}];", var.name, i);
+                fmtln!(
+                    fmt,
+                    "let typeof_{} = pos.func.dfg.value_type(*{});",
+                    var.name,
+                    var.name
+                );
+            }
+
+            replace_inst = true;
+        } else {
+            // Boring case: Detach the result values, capture them in locals.
+            for &var_index in &def.defined_vars {
+                fmtln!(fmt, "let {};", var_pool.get(var_index).name);
+            }
+
+            fmt.line("{");
+            fmt.indent(|fmt| {
+                fmt.line("let r = pos.func.dfg.inst_results(inst);");
+                for i in 0..def.defined_vars.len() {
+                    let var = var_pool.get(def.defined_vars[i]);
+                    fmtln!(fmt, "{} = r[{}];", var.name, i);
+                }
+            });
+            fmt.line("}");
+
+            for &var_index in &def.defined_vars {
+                let var = var_pool.get(var_index);
+                if var.has_free_typevar() {
+                    fmtln!(
+                        fmt,
+                        "let typeof_{} = pos.func.dfg.value_type({});",
+                        var.name,
+                        var.name
+                    );
+                }
+            }
+        }
+    }
+    replace_inst
+}
+
+fn build_derived_expr(tv: &TypeVar) -> String {
+    let base = match &tv.base {
+        Some(base) => base,
+        None => {
+            assert!(tv.name.starts_with("typeof_"));
+            return format!("Some({})", tv.name);
+        }
+    };
+    let base_expr = build_derived_expr(&base.type_var);
+    format!(
+        "{}.map(|t: crate::ir::Type| t.{}())",
+        base_expr,
+        base.derived_func.name()
+    )
+}
+
+/// Emit rust code for the given check.
+///
+/// The emitted code is a statement redefining the `predicate` variable like this:
+///     let predicate = predicate && ...
+fn emit_runtime_typecheck<'a>(
+    constraint: &'a Constraint,
+    type_sets: &mut UniqueTable<'a, TypeSet>,
+    fmt: &mut Formatter,
+) {
+    match constraint {
+        Constraint::InTypeset(tv, ts) => {
+            let ts_index = type_sets.add(&ts);
+            fmt.comment(format!(
+                "{} must belong to {:?}",
+                tv.name,
+                type_sets.get(ts_index)
+            ));
+            fmtln!(
+                fmt,
+                "let predicate = predicate && TYPE_SETS[{}].contains({});",
+                ts_index,
+                tv.name
+            );
+        }
+        Constraint::Eq(tv1, tv2) => {
+            fmtln!(
+                fmt,
+                "let predicate = predicate && match ({}, {}) {{",
+                build_derived_expr(tv1),
+                build_derived_expr(tv2)
+            );
+            fmt.indent(|fmt| {
+                fmt.line("(Some(a), Some(b)) => a == b,");
+                fmt.comment("On overflow, constraint doesn\'t apply");
+                fmt.line("_ => false,");
+            });
+            fmtln!(fmt, "};");
+        }
+        Constraint::WiderOrEq(tv1, tv2) => {
+            fmtln!(
+                fmt,
+                "let predicate = predicate && match ({}, {}) {{",
+                build_derived_expr(tv1),
+                build_derived_expr(tv2)
+            );
+            fmt.indent(|fmt| {
+                fmt.line("(Some(a), Some(b)) => a.wider_or_equal(b),");
+                fmt.comment("On overflow, constraint doesn\'t apply");
+                fmt.line("_ => false,");
+            });
+            fmtln!(fmt, "};");
+        }
+    }
+}
+
+/// Determine if `node` represents one of the value splitting instructions: `isplit` or `vsplit.
+/// These instructions are lowered specially by the `legalize::split` module.
+fn is_value_split(def: &Def) -> bool {
+    let name = &def.apply.inst.name;
+    name == "isplit" || name == "vsplit"
+}
+
+fn emit_dst_inst(def: &Def, def_pool: &DefPool, var_pool: &VarPool, fmt: &mut Formatter) {
+    let defined_vars = {
+        let vars = def
+            .defined_vars
+            .iter()
+            .map(|&var_index| var_pool.get(var_index).name.as_ref())
+            .collect::<Vec<&str>>();
+        if vars.len() == 1 {
+            vars[0].to_string()
+        } else {
+            format!("({})", vars.join(", "))
+        }
+    };
+
+    if is_value_split(def) {
+        // Split instructions are not emitted with the builder, but by calling special functions in
+        // the `legalizer::split` module. These functions will eliminate concat-split patterns.
+        fmt.line("let curpos = pos.position();");
+        fmt.line("let srcloc = pos.srcloc();");
+        fmtln!(
+            fmt,
+            "let {} = split::{}(pos.func, cfg, curpos, srcloc, {});",
+            defined_vars,
+            def.apply.inst.snake_name(),
+            def.apply.args[0].to_rust_code(var_pool)
+        );
+        return;
+    }
+
+    if def.defined_vars.is_empty() {
+        // This node doesn't define any values, so just insert the new instruction.
+        fmtln!(
+            fmt,
+            "pos.ins().{};",
+            def.apply.rust_builder(&def.defined_vars, var_pool)
+        );
+        return;
+    }
+
+    if let Some(src_def0) = var_pool.get(def.defined_vars[0]).src_def {
+        if def.defined_vars == def_pool.get(src_def0).defined_vars {
+            // The replacement instruction defines the exact same values as the source pattern.
+            // Unwrapping would have left the results intact.  Replace the whole instruction.
+            fmtln!(
+                fmt,
+                "let {} = pos.func.dfg.replace(inst).{};",
+                defined_vars,
+                def.apply.rust_builder(&def.defined_vars, var_pool)
+            );
+
+            // We need to bump the cursor so following instructions are inserted *after* the
+            // replaced instruction.
+            fmt.line("if pos.current_inst() == Some(inst) {");
+            fmt.indent(|fmt| {
+                fmt.line("pos.next_inst();");
+            });
+            fmt.line("}");
+            return;
+        }
+    }
+
+    // Insert a new instruction.
+    let mut builder = format!("let {} = pos.ins()", defined_vars);
+
+    if def.defined_vars.len() == 1 && var_pool.get(def.defined_vars[0]).is_output() {
+        // Reuse the single source result value.
+        builder = format!(
+            "{}.with_result({})",
+            builder,
+            var_pool.get(def.defined_vars[0]).to_rust_code()
+        );
+    } else if def
+        .defined_vars
+        .iter()
+        .any(|&var_index| var_pool.get(var_index).is_output())
+    {
+        // There are more than one output values that can be reused.
+        let array = def
+            .defined_vars
+            .iter()
+            .map(|&var_index| {
+                let var = var_pool.get(var_index);
+                if var.is_output() {
+                    format!("Some({})", var.name)
+                } else {
+                    "None".into()
+                }
+            })
+            .collect::<Vec<_>>()
+            .join(", ");
+        builder = format!("{}.with_results([{}])", builder, array);
+    }
+
+    fmtln!(
+        fmt,
+        "{}.{};",
+        builder,
+        def.apply.rust_builder(&def.defined_vars, var_pool)
+    );
+}
+
+/// Emit code for `transform`, assuming that the opcode of transform's root instruction
+/// has already been matched.
+///
+/// `inst: Inst` is the variable to be replaced. It is pointed to by `pos: Cursor`.
+/// `dfg: DataFlowGraph` is available and mutable.
+fn gen_transform<'a>(
+    replace_inst: bool,
+    transform: &'a Transform,
+    type_sets: &mut UniqueTable<'a, TypeSet>,
+    fmt: &mut Formatter,
+) {
+    // Evaluate the instruction predicate if any.
+    let apply = &transform.def_pool.get(transform.src).apply;
+
+    let inst_predicate = apply
+        .inst_predicate_with_ctrl_typevar(&transform.var_pool)
+        .rust_predicate("pos.func");
+
+    let has_extra_constraints = !transform.type_env.constraints.is_empty();
+    if has_extra_constraints {
+        // Extra constraints rely on the predicate being a variable that we can rebind as we add
+        // more constraint predicates.
+        if let Some(pred) = &inst_predicate {
+            fmt.multi_line(&format!("let predicate = {};", pred));
+        } else {
+            fmt.line("let predicate = true;");
+        }
+    }
+
+    // Emit any runtime checks; these will rebind `predicate` emitted right above.
+    for constraint in &transform.type_env.constraints {
+        emit_runtime_typecheck(constraint, type_sets, fmt);
+    }
+
+    let do_expand = |fmt: &mut Formatter| {
+        // Emit any constants that must be created before use.
+        for (name, value) in transform.const_pool.iter() {
+            fmtln!(
+                fmt,
+                "let {} = pos.func.dfg.constants.insert(vec!{:?}.into());",
+                name,
+                value
+            );
+        }
+
+        // If we are adding some blocks, we need to recall the original block, such that we can
+        // recompute it.
+        if !transform.block_pool.is_empty() {
+            fmt.line("let orig_block = pos.current_block().unwrap();");
+        }
+
+        // If we're going to delete `inst`, we need to detach its results first so they can be
+        // reattached during pattern expansion.
+        if !replace_inst {
+            fmt.line("pos.func.dfg.clear_results(inst);");
+        }
+
+        // Emit new block creation.
+        for block in &transform.block_pool {
+            let var = transform.var_pool.get(block.name);
+            fmtln!(fmt, "let {} = pos.func.dfg.make_block();", var.name);
+        }
+
+        // Emit the destination pattern.
+        for &def_index in &transform.dst {
+            if let Some(block) = transform.block_pool.get(def_index) {
+                let var = transform.var_pool.get(block.name);
+                fmtln!(fmt, "pos.insert_block({});", var.name);
+            }
+            emit_dst_inst(
+                transform.def_pool.get(def_index),
+                &transform.def_pool,
+                &transform.var_pool,
+                fmt,
+            );
+        }
+
+        // Insert a new block after the last instruction, if needed.
+        let def_next_index = transform.def_pool.next_index();
+        if let Some(block) = transform.block_pool.get(def_next_index) {
+            let var = transform.var_pool.get(block.name);
+            fmtln!(fmt, "pos.insert_block({});", var.name);
+        }
+
+        // Delete the original instruction if we didn't have an opportunity to replace it.
+        if !replace_inst {
+            fmt.line("let removed = pos.remove_inst();");
+            fmt.line("debug_assert_eq!(removed, inst);");
+        }
+
+        if transform.block_pool.is_empty() {
+            if transform.def_pool.get(transform.src).apply.inst.is_branch {
+                // A branch might have been legalized into multiple branches, so we need to recompute
+                // the cfg.
+                fmt.line("cfg.recompute_block(pos.func, pos.current_block().unwrap());");
+            }
+        } else {
+            // Update CFG for the new blocks.
+            fmt.line("cfg.recompute_block(pos.func, orig_block);");
+            for block in &transform.block_pool {
+                let var = transform.var_pool.get(block.name);
+                fmtln!(fmt, "cfg.recompute_block(pos.func, {});", var.name);
+            }
+        }
+
+        fmt.line("return true;");
+    };
+
+    // Guard the actual expansion by `predicate`.
+    if has_extra_constraints {
+        fmt.line("if predicate {");
+        fmt.indent(|fmt| {
+            do_expand(fmt);
+        });
+        fmt.line("}");
+    } else if let Some(pred) = &inst_predicate {
+        fmt.multi_line(&format!("if {} {{", pred));
+        fmt.indent(|fmt| {
+            do_expand(fmt);
+        });
+        fmt.line("}");
+    } else {
+        // Unconditional transform (there was no predicate), just emit it.
+        do_expand(fmt);
+    }
+}
+
+fn gen_transform_group<'a>(
+    group: &'a TransformGroup,
+    transform_groups: &TransformGroups,
+    type_sets: &mut UniqueTable<'a, TypeSet>,
+    fmt: &mut Formatter,
+) {
+    fmt.doc_comment(group.doc);
+    fmt.line("#[allow(unused_variables,unused_assignments,unused_imports,non_snake_case)]");
+
+    // Function arguments.
+    fmtln!(fmt, "pub fn {}(", group.name);
+    fmt.indent(|fmt| {
+        fmt.line("inst: crate::ir::Inst,");
+        fmt.line("func: &mut crate::ir::Function,");
+        fmt.line("cfg: &mut crate::flowgraph::ControlFlowGraph,");
+        fmt.line("isa: &dyn crate::isa::TargetIsa,");
+    });
+    fmtln!(fmt, ") -> bool {");
+
+    // Function body.
+    fmt.indent(|fmt| {
+        fmt.line("use crate::ir::InstBuilder;");
+        fmt.line("use crate::cursor::{Cursor, FuncCursor};");
+        fmt.line("let mut pos = FuncCursor::new(func).at_inst(inst);");
+        fmt.line("pos.use_srcloc(inst);");
+
+        // Group the transforms by opcode so we can generate a big switch.
+        // Preserve ordering.
+        let mut inst_to_transforms = HashMap::new();
+        for transform in &group.transforms {
+            let def_index = transform.src;
+            let inst = &transform.def_pool.get(def_index).apply.inst;
+            inst_to_transforms
+                .entry(inst.camel_name.clone())
+                .or_insert_with(Vec::new)
+                .push(transform);
+        }
+
+        let mut sorted_inst_names = Vec::from_iter(inst_to_transforms.keys());
+        sorted_inst_names.sort();
+
+        fmt.line("{");
+        fmt.indent(|fmt| {
+            fmt.line("match pos.func.dfg[inst].opcode() {");
+            fmt.indent(|fmt| {
+                for camel_name in sorted_inst_names {
+                    fmtln!(fmt, "ir::Opcode::{} => {{", camel_name);
+                    fmt.indent(|fmt| {
+                        let transforms = inst_to_transforms.get(camel_name).unwrap();
+
+                        // Unwrap the source instruction, create local variables for the input variables.
+                        let replace_inst = unwrap_inst(&transforms[0], fmt);
+                        fmt.empty_line();
+
+                        for (i, transform) in transforms.iter().enumerate() {
+                            if i > 0 {
+                                fmt.empty_line();
+                            }
+                            gen_transform(replace_inst, transform, type_sets, fmt);
+                        }
+                    });
+                    fmtln!(fmt, "}");
+                    fmt.empty_line();
+                }
+
+                // Emit the custom transforms. The Rust compiler will complain about any overlap with
+                // the normal transforms.
+                let mut sorted_custom_legalizes = Vec::from_iter(&group.custom_legalizes);
+                sorted_custom_legalizes.sort();
+                for (inst_camel_name, func_name) in sorted_custom_legalizes {
+                    fmtln!(fmt, "ir::Opcode::{} => {{", inst_camel_name);
+                    fmt.indent(|fmt| {
+                        fmtln!(fmt, "{}(inst, func, cfg, isa);", func_name);
+                        fmt.line("return true;");
+                    });
+                    fmtln!(fmt, "}");
+                    fmt.empty_line();
+                }
+
+                // We'll assume there are uncovered opcodes.
+                fmt.line("_ => {},");
+            });
+            fmt.line("}");
+        });
+        fmt.line("}");
+
+        // If we fall through, nothing was expanded; call the chain if any.
+        match &group.chain_with {
+            Some(group_id) => fmtln!(
+                fmt,
+                "{}(inst, func, cfg, isa)",
+                transform_groups.get(*group_id).rust_name()
+            ),
+            None => fmt.line("false"),
+        };
+    });
+    fmtln!(fmt, "}");
+    fmt.empty_line();
+}
+
+/// Generate legalization functions for `isa` and add any shared `TransformGroup`s
+/// encountered to `shared_groups`.
+///
+/// Generate `TYPE_SETS` and `LEGALIZE_ACTIONS` tables.
+fn gen_isa(
+    isa: &TargetIsa,
+    transform_groups: &TransformGroups,
+    shared_group_names: &mut HashSet<&'static str>,
+    fmt: &mut Formatter,
+) {
+    let mut type_sets = UniqueTable::new();
+    for group_index in isa.transitive_transform_groups(transform_groups) {
+        let group = transform_groups.get(group_index);
+        match group.isa_name {
+            Some(isa_name) => {
+                assert!(
+                    isa_name == isa.name,
+                    "ISA-specific legalizations must be used by the same ISA"
+                );
+                gen_transform_group(group, transform_groups, &mut type_sets, fmt);
+            }
+            None => {
+                shared_group_names.insert(group.name);
+            }
+        }
+    }
+
+    gen_typesets_table(&type_sets, fmt);
+
+    let direct_groups = isa.direct_transform_groups();
+    fmtln!(
+        fmt,
+        "pub static LEGALIZE_ACTIONS: [isa::Legalize; {}] = [",
+        direct_groups.len()
+    );
+    fmt.indent(|fmt| {
+        for &group_index in direct_groups {
+            fmtln!(fmt, "{},", transform_groups.get(group_index).rust_name());
+        }
+    });
+    fmtln!(fmt, "];");
+}
+
+/// Generate the legalizer files.
+pub(crate) fn generate(
+    isas: &[TargetIsa],
+    transform_groups: &TransformGroups,
+    filename_prefix: &str,
+    out_dir: &str,
+) -> Result<(), error::Error> {
+    let mut shared_group_names = HashSet::new();
+
+    for isa in isas {
+        let mut fmt = Formatter::new();
+        gen_isa(isa, transform_groups, &mut shared_group_names, &mut fmt);
+        fmt.update_file(format!("{}-{}.rs", filename_prefix, isa.name), out_dir)?;
+    }
+
+    // Generate shared legalize groups.
+    let mut fmt = Formatter::new();
+    let mut type_sets = UniqueTable::new();
+    let mut sorted_shared_group_names = Vec::from_iter(shared_group_names);
+    sorted_shared_group_names.sort();
+    for group_name in &sorted_shared_group_names {
+        let group = transform_groups.by_name(group_name);
+        gen_transform_group(group, transform_groups, &mut type_sets, &mut fmt);
+    }
+    gen_typesets_table(&type_sets, &mut fmt);
+    fmt.update_file(format!("{}r.rs", filename_prefix), out_dir)?;
+
+    Ok(())
+}

cranelift/codegen/meta/src/gen_registers.rs

@@ -0,0 +1,142 @@
+//! Generate the ISA-specific registers.
+use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::regs::{RegBank, RegClass};
+use crate::error;
+use crate::srcgen::Formatter;
+use cranelift_entity::EntityRef;
+
+fn gen_regbank(fmt: &mut Formatter, reg_bank: &RegBank) {
+    let names = if !reg_bank.names.is_empty() {
+        format!(r#""{}""#, reg_bank.names.join(r#"", ""#))
+    } else {
+        "".to_string()
+    };
+    fmtln!(fmt, "RegBank {");
+    fmt.indent(|fmt| {
+        fmtln!(fmt, r#"name: "{}","#, reg_bank.name);
+        fmtln!(fmt, "first_unit: {},", reg_bank.first_unit);
+        fmtln!(fmt, "units: {},", reg_bank.units);
+        fmtln!(fmt, "names: &[{}],", names);
+        fmtln!(fmt, r#"prefix: "{}","#, reg_bank.prefix);
+        fmtln!(fmt, "first_toprc: {},", reg_bank.toprcs[0].index());
+        fmtln!(fmt, "num_toprcs: {},", reg_bank.toprcs.len());
+        fmtln!(
+            fmt,
+            "pressure_tracking: {},",
+            if reg_bank.pressure_tracking {
+                "true"
+            } else {
+                "false"
+            }
+        );
+    });
+    fmtln!(fmt, "},");
+}
+
+fn gen_regclass(isa: &TargetIsa, reg_class: &RegClass, fmt: &mut Formatter) {
+    let reg_bank = isa.regs.banks.get(reg_class.bank).unwrap();
+
+    let mask: Vec<String> = reg_class
+        .mask(reg_bank.first_unit)
+        .iter()
+        .map(|x| format!("0x{:08x}", x))
+        .collect();
+    let mask = mask.join(", ");
+
+    fmtln!(
+        fmt,
+        "pub static {}_DATA: RegClassData = RegClassData {{",
+        reg_class.name
+    );
+    fmt.indent(|fmt| {
+        fmtln!(fmt, r#"name: "{}","#, reg_class.name);
+        fmtln!(fmt, "index: {},", reg_class.index.index());
+        fmtln!(fmt, "width: {},", reg_class.width);
+        fmtln!(fmt, "bank: {},", reg_class.bank.index());
+        fmtln!(fmt, "toprc: {},", reg_class.toprc.index());
+        fmtln!(fmt, "first: {},", reg_bank.first_unit + reg_class.start);
+        fmtln!(fmt, "subclasses: {:#x},", reg_class.subclass_mask());
+        fmtln!(fmt, "mask: [{}],", mask);
+        fmtln!(fmt, "pinned_reg: {:?},", reg_bank.pinned_reg);
+        fmtln!(fmt, "info: &INFO,");
+    });
+    fmtln!(fmt, "};");
+
+    fmtln!(fmt, "#[allow(dead_code)]");
+    fmtln!(
+        fmt,
+        "pub static {}: RegClass = &{}_DATA;",
+        reg_class.name,
+        reg_class.name
+    );
+}
+
+fn gen_regbank_units(reg_bank: &RegBank, fmt: &mut Formatter) {
+    for unit in 0..reg_bank.units {
+        let v = unit + reg_bank.first_unit;
+        if (unit as usize) < reg_bank.names.len() {
+            fmtln!(fmt, "{} = {},", reg_bank.names[unit as usize], v);
+            continue;
+        }
+        fmtln!(fmt, "{}{} = {},", reg_bank.prefix, unit, v);
+    }
+}
+
+fn gen_isa(isa: &TargetIsa, fmt: &mut Formatter) {
+    // Emit RegInfo.
+    fmtln!(fmt, "pub static INFO: RegInfo = RegInfo {");
+
+    fmt.indent(|fmt| {
+        fmtln!(fmt, "banks: &[");
+        // Bank descriptors.
+        fmt.indent(|fmt| {
+            for reg_bank in isa.regs.banks.values() {
+                gen_regbank(fmt, &reg_bank);
+            }
+        });
+        fmtln!(fmt, "],");
+        // References to register classes.
+        fmtln!(fmt, "classes: &[");
+        fmt.indent(|fmt| {
+            for reg_class in isa.regs.classes.values() {
+                fmtln!(fmt, "&{}_DATA,", reg_class.name);
+            }
+        });
+        fmtln!(fmt, "],");
+    });
+    fmtln!(fmt, "};");
+
+    // Register class descriptors.
+    for rc in isa.regs.classes.values() {
+        gen_regclass(&isa, rc, fmt);
+    }
+
+    // Emit constants for all the register units.
+    fmtln!(fmt, "#[allow(dead_code, non_camel_case_types)]");
+    fmtln!(fmt, "#[derive(Clone, Copy)]");
+    fmtln!(fmt, "pub enum RU {");
+    fmt.indent(|fmt| {
+        for reg_bank in isa.regs.banks.values() {
+            gen_regbank_units(reg_bank, fmt);
+        }
+    });
+    fmtln!(fmt, "}");
+
+    // Emit Into conversion for the RU class.
+    fmtln!(fmt, "impl Into<RegUnit> for RU {");
+    fmt.indent(|fmt| {
+        fmtln!(fmt, "fn into(self) -> RegUnit {");
+        fmt.indent(|fmt| {
+            fmtln!(fmt, "self as RegUnit");
+        });
+        fmtln!(fmt, "}");
+    });
+    fmtln!(fmt, "}");
+}
+
+pub(crate) fn generate(isa: &TargetIsa, filename: &str, out_dir: &str) -> Result<(), error::Error> {
+    let mut fmt = Formatter::new();
+    gen_isa(&isa, &mut fmt);
+    fmt.update_file(filename, out_dir)?;
+    Ok(())
+}

cranelift/codegen/meta/src/gen_settings.rs

@@ -0,0 +1,447 @@
+//! Generate the ISA-specific settings.
+use std::collections::HashMap;
+
+use cranelift_codegen_shared::constant_hash::{generate_table, simple_hash};
+
+use crate::cdsl::camel_case;
+use crate::cdsl::settings::{
+    BoolSetting, Predicate, Preset, Setting, SettingGroup, SpecificSetting,
+};
+use crate::error;
+use crate::srcgen::{Formatter, Match};
+use crate::unique_table::UniqueSeqTable;
+
+pub(crate) enum ParentGroup {
+    None,
+    Shared,
+}
+
+/// Emits the constructor of the Flags structure.
+fn gen_constructor(group: &SettingGroup, parent: ParentGroup, fmt: &mut Formatter) {
+    let args = match parent {
+        ParentGroup::None => "builder: Builder",
+        ParentGroup::Shared => "shared: &settings::Flags, builder: Builder",
+    };
+    fmtln!(fmt, "impl Flags {");
+    fmt.indent(|fmt| {
+        fmt.doc_comment(format!("Create flags {} settings group.", group.name));
+        fmtln!(fmt, "#[allow(unused_variables)]");
+        fmtln!(fmt, "pub fn new({}) -> Self {{", args);
+        fmt.indent(|fmt| {
+            fmtln!(fmt, "let bvec = builder.state_for(\"{}\");", group.name);
+            fmtln!(
+                fmt,
+                "let mut {} = Self {{ bytes: [0; {}] }};",
+                group.name,
+                group.byte_size()
+            );
+            fmtln!(
+                fmt,
+                "debug_assert_eq!(bvec.len(), {});",
+                group.settings_size
+            );
+            fmtln!(
+                fmt,
+                "{}.bytes[0..{}].copy_from_slice(&bvec);",
+                group.name,
+                group.settings_size
+            );
+
+            // Now compute the predicates.
+            for p in &group.predicates {
+                fmt.comment(format!("Precompute #{}.", p.number));
+                fmtln!(fmt, "if {} {{", p.render(group));
+                fmt.indent(|fmt| {
+                    fmtln!(
+                        fmt,
+                        "{}.bytes[{}] |= 1 << {};",
+                        group.name,
+                        group.bool_start_byte_offset + p.number / 8,
+                        p.number % 8
+                    );
+                });
+                fmtln!(fmt, "}");
+            }
+
+            fmtln!(fmt, group.name);
+        });
+        fmtln!(fmt, "}");
+    });
+    fmtln!(fmt, "}");
+}
+
+/// Emit Display and FromStr implementations for enum settings.
+fn gen_to_and_from_str(name: &str, values: &[&'static str], fmt: &mut Formatter) {
+    fmtln!(fmt, "impl fmt::Display for {} {{", name);
+    fmt.indent(|fmt| {
+        fmtln!(
+            fmt,
+            "fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {"
+        );
+        fmt.indent(|fmt| {
+            fmtln!(fmt, "f.write_str(match *self {");
+            fmt.indent(|fmt| {
+                for v in values.iter() {
+                    fmtln!(fmt, "Self::{} => \"{}\",", camel_case(v), v);
+                }
+            });
+            fmtln!(fmt, "})");
+        });
+        fmtln!(fmt, "}");
+    });
+    fmtln!(fmt, "}");
+
+    fmtln!(fmt, "impl str::FromStr for {} {{", name);
+    fmt.indent(|fmt| {
+        fmtln!(fmt, "type Err = ();");
+        fmtln!(fmt, "fn from_str(s: &str) -> Result<Self, Self::Err> {");
+        fmt.indent(|fmt| {
+            fmtln!(fmt, "match s {");
+            fmt.indent(|fmt| {
+                for v in values.iter() {
+                    fmtln!(fmt, "\"{}\" => Ok(Self::{}),", v, camel_case(v));
+                }
+                fmtln!(fmt, "_ => Err(()),");
+            });
+            fmtln!(fmt, "}");
+        });
+        fmtln!(fmt, "}");
+    });
+    fmtln!(fmt, "}");
+}
+
+/// Emit real enum for the Enum settings.
+fn gen_enum_types(group: &SettingGroup, fmt: &mut Formatter) {
+    for setting in group.settings.iter() {
+        let values = match setting.specific {
+            SpecificSetting::Bool(_) | SpecificSetting::Num(_) => continue,
+            SpecificSetting::Enum(ref values) => values,
+        };
+        let name = camel_case(setting.name);
+
+        fmt.doc_comment(format!("Values for `{}.{}`.", group.name, setting.name));
+        fmtln!(fmt, "#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]");
+        fmtln!(fmt, "pub enum {} {{", name);
+        fmt.indent(|fmt| {
+            for v in values.iter() {
+                fmt.doc_comment(format!("`{}`.", v));
+                fmtln!(fmt, "{},", camel_case(v));
+            }
+        });
+        fmtln!(fmt, "}");
+
+        gen_to_and_from_str(&name, values, fmt);
+    }
+}
+
+/// Emit a getter function for `setting`.
+fn gen_getter(setting: &Setting, fmt: &mut Formatter) {
+    fmt.doc_comment(setting.comment);
+    match setting.specific {
+        SpecificSetting::Bool(BoolSetting {
+            predicate_number, ..
+        }) => {
+            fmtln!(fmt, "pub fn {}(&self) -> bool {{", setting.name);
+            fmt.indent(|fmt| {
+                fmtln!(fmt, "self.numbered_predicate({})", predicate_number);
+            });
+            fmtln!(fmt, "}");
+        }
+        SpecificSetting::Enum(ref values) => {
+            let ty = camel_case(setting.name);
+            fmtln!(fmt, "pub fn {}(&self) -> {} {{", setting.name, ty);
+            fmt.indent(|fmt| {
+                let mut m = Match::new(format!("self.bytes[{}]", setting.byte_offset));
+                for (i, v) in values.iter().enumerate() {
+                    m.arm_no_fields(format!("{}", i), format!("{}::{}", ty, camel_case(v)));
+                }
+                m.arm_no_fields("_", "panic!(\"Invalid enum value\")");
+                fmt.add_match(m);
+            });
+            fmtln!(fmt, "}");
+        }
+        SpecificSetting::Num(_) => {
+            fmtln!(fmt, "pub fn {}(&self) -> u8 {{", setting.name);
+            fmt.indent(|fmt| {
+                fmtln!(fmt, "self.bytes[{}]", setting.byte_offset);
+            });
+            fmtln!(fmt, "}");
+        }
+    }
+}
+
+fn gen_pred_getter(predicate: &Predicate, group: &SettingGroup, fmt: &mut Formatter) {
+    fmt.doc_comment(format!("Computed predicate `{}`.", predicate.render(group)));
+    fmtln!(fmt, "pub fn {}(&self) -> bool {{", predicate.name);
+    fmt.indent(|fmt| {
+        fmtln!(fmt, "self.numbered_predicate({})", predicate.number);
+    });
+    fmtln!(fmt, "}");
+}
+
+/// Emits getters for each setting value.
+fn gen_getters(group: &SettingGroup, fmt: &mut Formatter) {
+    fmt.doc_comment("User-defined settings.");
+    fmtln!(fmt, "#[allow(dead_code)]");
+    fmtln!(fmt, "impl Flags {");
+    fmt.indent(|fmt| {
+        fmt.doc_comment("Get a view of the boolean predicates.");
+        fmtln!(
+            fmt,
+            "pub fn predicate_view(&self) -> crate::settings::PredicateView {"
+        );
+        fmt.indent(|fmt| {
+            fmtln!(
+                fmt,
+                "crate::settings::PredicateView::new(&self.bytes[{}..])",
+                group.bool_start_byte_offset
+            );
+        });
+        fmtln!(fmt, "}");
+
+        if !group.settings.is_empty() {
+            fmt.doc_comment("Dynamic numbered predicate getter.");
+            fmtln!(fmt, "fn numbered_predicate(&self, p: usize) -> bool {");
+            fmt.indent(|fmt| {
+                fmtln!(
+                    fmt,
+                    "self.bytes[{} + p / 8] & (1 << (p % 8)) != 0",
+                    group.bool_start_byte_offset
+                );
+            });
+            fmtln!(fmt, "}");
+        }
+
+        for setting in &group.settings {
+            gen_getter(&setting, fmt);
+        }
+        for predicate in &group.predicates {
+            gen_pred_getter(&predicate, &group, fmt);
+        }
+    });
+    fmtln!(fmt, "}");
+}
+
+#[derive(Hash, PartialEq, Eq)]
+enum SettingOrPreset<'a> {
+    Setting(&'a Setting),
+    Preset(&'a Preset),
+}
+
+impl<'a> SettingOrPreset<'a> {
+    fn name(&self) -> &str {
+        match *self {
+            SettingOrPreset::Setting(s) => s.name,
+            SettingOrPreset::Preset(p) => p.name,
+        }
+    }
+}
+
+/// Emits DESCRIPTORS, ENUMERATORS, HASH_TABLE and PRESETS.
+fn gen_descriptors(group: &SettingGroup, fmt: &mut Formatter) {
+    let mut enum_table = UniqueSeqTable::new();
+
+    let mut descriptor_index_map: HashMap<SettingOrPreset, usize> = HashMap::new();
+
+    // Generate descriptors.
+    fmtln!(
+        fmt,
+        "static DESCRIPTORS: [detail::Descriptor; {}] = [",
+        group.settings.len() + group.presets.len()
+    );
+    fmt.indent(|fmt| {
+        for (idx, setting) in group.settings.iter().enumerate() {
+            fmtln!(fmt, "detail::Descriptor {");
+            fmt.indent(|fmt| {
+                fmtln!(fmt, "name: \"{}\",", setting.name);
+                fmtln!(fmt, "offset: {},", setting.byte_offset);
+                match setting.specific {
+                    SpecificSetting::Bool(BoolSetting { bit_offset, .. }) => {
+                        fmtln!(
+                            fmt,
+                            "detail: detail::Detail::Bool {{ bit: {} }},",
+                            bit_offset
+                        );
+                    }
+                    SpecificSetting::Enum(ref values) => {
+                        let offset = enum_table.add(values);
+                        fmtln!(
+                            fmt,
+                            "detail: detail::Detail::Enum {{ last: {}, enumerators: {} }},",
+                            values.len() - 1,
+                            offset
+                        );
+                    }
+                    SpecificSetting::Num(_) => {
+                        fmtln!(fmt, "detail: detail::Detail::Num,");
+                    }
+                }
+
+                descriptor_index_map.insert(SettingOrPreset::Setting(setting), idx);
+            });
+            fmtln!(fmt, "},");
+        }
+
+        for (idx, preset) in group.presets.iter().enumerate() {
+            fmtln!(fmt, "detail::Descriptor {");
+            fmt.indent(|fmt| {
+                fmtln!(fmt, "name: \"{}\",", preset.name);
+                fmtln!(fmt, "offset: {},", (idx as u8) * group.settings_size);
+                fmtln!(fmt, "detail: detail::Detail::Preset,");
+            });
+            fmtln!(fmt, "},");
+
+            let whole_idx = idx + group.settings.len();
+            descriptor_index_map.insert(SettingOrPreset::Preset(preset), whole_idx);
+        }
+    });
+    fmtln!(fmt, "];");
+
+    // Generate enumerators.
+    fmtln!(fmt, "static ENUMERATORS: [&str; {}] = [", enum_table.len());
+    fmt.indent(|fmt| {
+        for enum_val in enum_table.iter() {
+            fmtln!(fmt, "\"{}\",", enum_val);
+        }
+    });
+    fmtln!(fmt, "];");
+
+    // Generate hash table.
+    let mut hash_entries: Vec<SettingOrPreset> = Vec::new();
+    hash_entries.extend(group.settings.iter().map(|x| SettingOrPreset::Setting(x)));
+    hash_entries.extend(group.presets.iter().map(|x| SettingOrPreset::Preset(x)));
+
+    let hash_table = generate_table(hash_entries.iter(), hash_entries.len(), |entry| {
+        simple_hash(entry.name())
+    });
+    fmtln!(fmt, "static HASH_TABLE: [u16; {}] = [", hash_table.len());
+    fmt.indent(|fmt| {
+        for h in &hash_table {
+            match *h {
+                Some(setting_or_preset) => fmtln!(
+                    fmt,
+                    "{},",
+                    &descriptor_index_map
+                        .get(setting_or_preset)
+                        .unwrap()
+                        .to_string()
+                ),
+                None => fmtln!(fmt, "0xffff,"),
+            }
+        }
+    });
+    fmtln!(fmt, "];");
+
+    // Generate presets.
+    fmtln!(
+        fmt,
+        "static PRESETS: [(u8, u8); {}] = [",
+        group.presets.len() * (group.settings_size as usize)
+    );
+    fmt.indent(|fmt| {
+        for preset in &group.presets {
+            fmt.comment(preset.name);
+            for (mask, value) in preset.layout(&group) {
+                fmtln!(fmt, "(0b{:08b}, 0b{:08b}),", mask, value);
+            }
+        }
+    });
+    fmtln!(fmt, "];");
+}
+
+fn gen_template(group: &SettingGroup, fmt: &mut Formatter) {
+    let mut default_bytes: Vec<u8> = vec![0; group.settings_size as usize];
+    for setting in &group.settings {
+        *default_bytes.get_mut(setting.byte_offset as usize).unwrap() |= setting.default_byte();
+    }
+
+    let default_bytes: Vec<String> = default_bytes
+        .iter()
+        .map(|x| format!("{:#04x}", x))
+        .collect();
+    let default_bytes_str = default_bytes.join(", ");
+
+    fmtln!(
+        fmt,
+        "static TEMPLATE: detail::Template = detail::Template {"
+    );
+    fmt.indent(|fmt| {
+        fmtln!(fmt, "name: \"{}\",", group.name);
+        fmtln!(fmt, "descriptors: &DESCRIPTORS,");
+        fmtln!(fmt, "enumerators: &ENUMERATORS,");
+        fmtln!(fmt, "hash_table: &HASH_TABLE,");
+        fmtln!(fmt, "defaults: &[{}],", default_bytes_str);
+        fmtln!(fmt, "presets: &PRESETS,");
+    });
+    fmtln!(fmt, "};");
+
+    fmt.doc_comment(format!(
+        "Create a `settings::Builder` for the {} settings group.",
+        group.name
+    ));
+    fmtln!(fmt, "pub fn builder() -> Builder {");
+    fmt.indent(|fmt| {
+        fmtln!(fmt, "Builder::new(&TEMPLATE)");
+    });
+    fmtln!(fmt, "}");
+}
+
+fn gen_display(group: &SettingGroup, fmt: &mut Formatter) {
+    fmtln!(fmt, "impl fmt::Display for Flags {");
+    fmt.indent(|fmt| {
+        fmtln!(
+            fmt,
+            "fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {"
+        );
+        fmt.indent(|fmt| {
+            fmtln!(fmt, "writeln!(f, \"[{}]\")?;", group.name);
+            fmtln!(fmt, "for d in &DESCRIPTORS {");
+            fmt.indent(|fmt| {
+                fmtln!(fmt, "if !d.detail.is_preset() {");
+                fmt.indent(|fmt| {
+                    fmtln!(fmt, "write!(f, \"{} = \", d.name)?;");
+                    fmtln!(
+                        fmt,
+                        "TEMPLATE.format_toml_value(d.detail, self.bytes[d.offset as usize], f)?;",
+                    );
+                    fmtln!(fmt, "writeln!(f)?;");
+                });
+                fmtln!(fmt, "}");
+            });
+            fmtln!(fmt, "}");
+            fmtln!(fmt, "Ok(())");
+        });
+        fmtln!(fmt, "}")
+    });
+    fmtln!(fmt, "}");
+}
+
+fn gen_group(group: &SettingGroup, parent: ParentGroup, fmt: &mut Formatter) {
+    // Generate struct.
+    fmtln!(fmt, "#[derive(Clone)]");
+    fmt.doc_comment(format!("Flags group `{}`.", group.name));
+    fmtln!(fmt, "pub struct Flags {");
+    fmt.indent(|fmt| {
+        fmtln!(fmt, "bytes: [u8; {}],", group.byte_size());
+    });
+    fmtln!(fmt, "}");
+
+    gen_constructor(group, parent, fmt);
+    gen_enum_types(group, fmt);
+    gen_getters(group, fmt);
+    gen_descriptors(group, fmt);
+    gen_template(group, fmt);
+    gen_display(group, fmt);
+}
+
+pub(crate) fn generate(
+    settings: &SettingGroup,
+    parent_group: ParentGroup,
+    filename: &str,
+    out_dir: &str,
+) -> Result<(), error::Error> {
+    let mut fmt = Formatter::new();
+    gen_group(&settings, parent_group, &mut fmt);
+    fmt.update_file(filename, out_dir)?;
+    Ok(())
+}

cranelift/codegen/meta/src/gen_types.rs

@@ -0,0 +1,76 @@
+//! Generate sources with type info.
+//!
+//! This generates a `types.rs` file which is included in
+//! `cranelift-codegen/ir/types.rs`. The file provides constant definitions for the
+//! most commonly used types, including all of the scalar types.
+//!
+//! This ensures that the metaprogram and the generated program see the same
+//! type numbering.
+
+use crate::cdsl::types as cdsl_types;
+use crate::error;
+use crate::srcgen;
+
+/// Emit a constant definition of a single value type.
+fn emit_type(ty: &cdsl_types::ValueType, fmt: &mut srcgen::Formatter) -> Result<(), error::Error> {
+    let name = ty.to_string().to_uppercase();
+    let number = ty.number().ok_or_else(|| {
+        error::Error::with_msg(format!(
+            "Could not emit type `{}` which has no number.",
+            name
+        ))
+    })?;
+
+    fmt.doc_comment(&ty.doc());
+    fmtln!(fmt, "pub const {}: Type = Type({:#x});\n", name, number);
+
+    Ok(())
+}
+
+/// Emit definition for all vector types with `bits` total size.
+fn emit_vectors(bits: u64, fmt: &mut srcgen::Formatter) -> Result<(), error::Error> {
+    let vec_size: u64 = bits / 8;
+    for vec in cdsl_types::ValueType::all_lane_types()
+        .map(|ty| (ty, cdsl_types::ValueType::from(ty).membytes()))
+        .filter(|&(_, lane_size)| lane_size != 0 && lane_size < vec_size)
+        .map(|(ty, lane_size)| (ty, vec_size / lane_size))
+        .map(|(ty, lanes)| cdsl_types::VectorType::new(ty, lanes))
+    {
+        emit_type(&cdsl_types::ValueType::from(vec), fmt)?;
+    }
+
+    Ok(())
+}
+
+/// Emit types using the given formatter object.
+fn emit_types(fmt: &mut srcgen::Formatter) -> Result<(), error::Error> {
+    // Emit all of the special types, such as types for CPU flags.
+    for spec in cdsl_types::ValueType::all_special_types().map(cdsl_types::ValueType::from) {
+        emit_type(&spec, fmt)?;
+    }
+
+    // Emit all of the lane types, such integers, floats, and booleans.
+    for ty in cdsl_types::ValueType::all_lane_types().map(cdsl_types::ValueType::from) {
+        emit_type(&ty, fmt)?;
+    }
+
+    // Emit all reference types.
+    for ty in cdsl_types::ValueType::all_reference_types().map(cdsl_types::ValueType::from) {
+        emit_type(&ty, fmt)?;
+    }
+
+    // Emit vector definitions for common SIMD sizes.
+    for vec_size in &[64_u64, 128, 256, 512] {
+        emit_vectors(*vec_size, fmt)?;
+    }
+
+    Ok(())
+}
+
+/// Generate the types file.
+pub(crate) fn generate(filename: &str, out_dir: &str) -> Result<(), error::Error> {
+    let mut fmt = srcgen::Formatter::new();
+    emit_types(&mut fmt)?;
+    fmt.update_file(filename, out_dir)?;
+    Ok(())
+}

cranelift/codegen/meta/src/isa/arm32/mod.rs

@@ -0,0 +1,84 @@
+use crate::cdsl::cpu_modes::CpuMode;
+use crate::cdsl::instructions::{InstructionGroupBuilder, InstructionPredicateMap};
+use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::recipes::Recipes;
+use crate::cdsl::regs::{IsaRegs, IsaRegsBuilder, RegBankBuilder, RegClassBuilder};
+use crate::cdsl::settings::{SettingGroup, SettingGroupBuilder};
+
+use crate::shared::Definitions as SharedDefinitions;
+
+fn define_settings(_shared: &SettingGroup) -> SettingGroup {
+    let setting = SettingGroupBuilder::new("arm32");
+    setting.build()
+}
+
+fn define_regs() -> IsaRegs {
+    let mut regs = IsaRegsBuilder::new();
+
+    let builder = RegBankBuilder::new("FloatRegs", "s")
+        .units(64)
+        .track_pressure(true);
+    let float_regs = regs.add_bank(builder);
+
+    let builder = RegBankBuilder::new("IntRegs", "r")
+        .units(16)
+        .track_pressure(true);
+    let int_regs = regs.add_bank(builder);
+
+    let builder = RegBankBuilder::new("FlagRegs", "")
+        .units(1)
+        .names(vec!["nzcv"])
+        .track_pressure(false);
+    let flag_reg = regs.add_bank(builder);
+
+    let builder = RegClassBuilder::new_toplevel("S", float_regs).count(32);
+    regs.add_class(builder);
+
+    let builder = RegClassBuilder::new_toplevel("D", float_regs).width(2);
+    regs.add_class(builder);
+
+    let builder = RegClassBuilder::new_toplevel("Q", float_regs).width(4);
+    regs.add_class(builder);
+
+    let builder = RegClassBuilder::new_toplevel("GPR", int_regs);
+    regs.add_class(builder);
+
+    let builder = RegClassBuilder::new_toplevel("FLAG", flag_reg);
+    regs.add_class(builder);
+
+    regs.build()
+}
+
+pub(crate) fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
+    let settings = define_settings(&shared_defs.settings);
+    let regs = define_regs();
+
+    let inst_group = InstructionGroupBuilder::new(&mut shared_defs.all_instructions).build();
+
+    // CPU modes for 32-bit ARM and Thumb2.
+    let mut a32 = CpuMode::new("A32");
+    let mut t32 = CpuMode::new("T32");
+
+    // TODO refine these.
+    let narrow_flags = shared_defs.transform_groups.by_name("narrow_flags");
+    a32.legalize_default(narrow_flags);
+    t32.legalize_default(narrow_flags);
+
+    let cpu_modes = vec![a32, t32];
+
+    // TODO implement arm32 recipes.
+    let recipes = Recipes::new();
+
+    // TODO implement arm32 encodings and predicates.
+    let encodings_predicates = InstructionPredicateMap::new();
+
+    TargetIsa::new(
+        "arm32",
+        inst_group,
+        settings,
+        regs,
+        recipes,
+        cpu_modes,
+        encodings_predicates,
+    )
+}

cranelift/codegen/meta/src/isa/arm64/mod.rs

@@ -0,0 +1,77 @@
+use crate::cdsl::cpu_modes::CpuMode;
+use crate::cdsl::instructions::{InstructionGroupBuilder, InstructionPredicateMap};
+use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::recipes::Recipes;
+use crate::cdsl::regs::{IsaRegs, IsaRegsBuilder, RegBankBuilder, RegClassBuilder};
+use crate::cdsl::settings::{SettingGroup, SettingGroupBuilder};
+
+use crate::shared::Definitions as SharedDefinitions;
+
+fn define_settings(_shared: &SettingGroup) -> SettingGroup {
+    let setting = SettingGroupBuilder::new("arm64");
+    setting.build()
+}
+
+fn define_registers() -> IsaRegs {
+    let mut regs = IsaRegsBuilder::new();
+
+    // The `x31` regunit serves as the stack pointer / zero register depending on context. We
+    // reserve it and don't model the difference.
+    let builder = RegBankBuilder::new("IntRegs", "x")
+        .units(32)
+        .track_pressure(true);
+    let int_regs = regs.add_bank(builder);
+
+    let builder = RegBankBuilder::new("FloatRegs", "v")
+        .units(32)
+        .track_pressure(true);
+    let float_regs = regs.add_bank(builder);
+
+    let builder = RegBankBuilder::new("FlagRegs", "")
+        .units(1)
+        .names(vec!["nzcv"])
+        .track_pressure(false);
+    let flag_reg = regs.add_bank(builder);
+
+    let builder = RegClassBuilder::new_toplevel("GPR", int_regs);
+    regs.add_class(builder);
+
+    let builder = RegClassBuilder::new_toplevel("FPR", float_regs);
+    regs.add_class(builder);
+
+    let builder = RegClassBuilder::new_toplevel("FLAG", flag_reg);
+    regs.add_class(builder);
+
+    regs.build()
+}
+
+pub(crate) fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
+    let settings = define_settings(&shared_defs.settings);
+    let regs = define_registers();
+
+    let inst_group = InstructionGroupBuilder::new(&mut shared_defs.all_instructions).build();
+
+    let mut a64 = CpuMode::new("A64");
+
+    // TODO refine these.
+    let narrow_flags = shared_defs.transform_groups.by_name("narrow_flags");
+    a64.legalize_default(narrow_flags);
+
+    let cpu_modes = vec![a64];
+
+    // TODO implement arm64 recipes.
+    let recipes = Recipes::new();
+
+    // TODO implement arm64 encodings and predicates.
+    let encodings_predicates = InstructionPredicateMap::new();
+
+    TargetIsa::new(
+        "arm64",
+        inst_group,
+        settings,
+        regs,
+        recipes,
+        cpu_modes,
+        encodings_predicates,
+    )
+}

cranelift/codegen/meta/src/isa/mod.rs

@@ -0,0 +1,67 @@
+//! Define supported ISAs; includes ISA-specific instructions, encodings, registers, settings, etc.
+use crate::cdsl::isa::TargetIsa;
+use crate::shared::Definitions as SharedDefinitions;
+use std::fmt;
+
+mod arm32;
+mod arm64;
+mod riscv;
+mod x86;
+
+/// Represents known ISA target.
+#[derive(Copy, Clone)]
+pub enum Isa {
+    Riscv,
+    X86,
+    Arm32,
+    Arm64,
+}
+
+impl Isa {
+    /// Creates isa target using name.
+    pub fn from_name(name: &str) -> Option<Self> {
+        Isa::all()
+            .iter()
+            .cloned()
+            .find(|isa| isa.to_string() == name)
+    }
+
+    /// Creates isa target from arch.
+    pub fn from_arch(arch: &str) -> Option<Self> {
+        match arch {
+            "riscv" => Some(Isa::Riscv),
+            "aarch64" => Some(Isa::Arm64),
+            x if ["x86_64", "i386", "i586", "i686"].contains(&x) => Some(Isa::X86),
+            x if x.starts_with("arm") || arch.starts_with("thumb") => Some(Isa::Arm32),
+            _ => None,
+        }
+    }
+
+    /// Returns all supported isa targets.
+    pub fn all() -> &'static [Isa] {
+        &[Isa::Riscv, Isa::X86, Isa::Arm32, Isa::Arm64]
+    }
+}
+
+impl fmt::Display for Isa {
+    // These names should be kept in sync with the crate features.
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            Isa::Riscv => write!(f, "riscv"),
+            Isa::X86 => write!(f, "x86"),
+            Isa::Arm32 => write!(f, "arm32"),
+            Isa::Arm64 => write!(f, "arm64"),
+        }
+    }
+}
+
+pub(crate) fn define(isas: &[Isa], shared_defs: &mut SharedDefinitions) -> Vec<TargetIsa> {
+    isas.iter()
+        .map(|isa| match isa {
+            Isa::Riscv => riscv::define(shared_defs),
+            Isa::X86 => x86::define(shared_defs),
+            Isa::Arm32 => arm32::define(shared_defs),
+            Isa::Arm64 => arm64::define(shared_defs),
+        })
+        .collect()
+}

cranelift/codegen/meta/src/isa/riscv/encodings.rs

@@ -0,0 +1,431 @@
+use crate::cdsl::ast::{Apply, Expr, Literal, VarPool};
+use crate::cdsl::encodings::{Encoding, EncodingBuilder};
+use crate::cdsl::instructions::{
+    Bindable, BoundInstruction, InstSpec, InstructionPredicateNode, InstructionPredicateRegistry,
+};
+use crate::cdsl::recipes::{EncodingRecipeNumber, Recipes};
+use crate::cdsl::settings::SettingGroup;
+
+use crate::shared::types::Bool::B1;
+use crate::shared::types::Float::{F32, F64};
+use crate::shared::types::Int::{I16, I32, I64, I8};
+use crate::shared::types::Reference::{R32, R64};
+use crate::shared::Definitions as SharedDefinitions;
+
+use super::recipes::RecipeGroup;
+
+pub(crate) struct PerCpuModeEncodings<'defs> {
+    pub inst_pred_reg: InstructionPredicateRegistry,
+    pub enc32: Vec<Encoding>,
+    pub enc64: Vec<Encoding>,
+    recipes: &'defs Recipes,
+}
+
+impl<'defs> PerCpuModeEncodings<'defs> {
+    fn new(recipes: &'defs Recipes) -> Self {
+        Self {
+            inst_pred_reg: InstructionPredicateRegistry::new(),
+            enc32: Vec::new(),
+            enc64: Vec::new(),
+            recipes,
+        }
+    }
+    fn enc(
+        &self,
+        inst: impl Into<InstSpec>,
+        recipe: EncodingRecipeNumber,
+        bits: u16,
+    ) -> EncodingBuilder {
+        EncodingBuilder::new(inst.into(), recipe, bits)
+    }
+    fn add32(&mut self, encoding: EncodingBuilder) {
+        self.enc32
+            .push(encoding.build(self.recipes, &mut self.inst_pred_reg));
+    }
+    fn add64(&mut self, encoding: EncodingBuilder) {
+        self.enc64
+            .push(encoding.build(self.recipes, &mut self.inst_pred_reg));
+    }
+}
+
+// The low 7 bits of a RISC-V instruction is the base opcode. All 32-bit instructions have 11 as
+// the two low bits, with bits 6:2 determining the base opcode.
+//
+// Encbits for the 32-bit recipes are opcode[6:2] | (funct3 << 5) | ...
+// The functions below encode the encbits.
+
+fn load_bits(funct3: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    funct3 << 5
+}
+
+fn store_bits(funct3: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    0b01000 | (funct3 << 5)
+}
+
+fn branch_bits(funct3: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    0b11000 | (funct3 << 5)
+}
+
+fn jalr_bits() -> u16 {
+    // This was previously accepting an argument funct3 of 3 bits and used the following formula:
+    //0b11001 | (funct3 << 5)
+    0b11001
+}
+
+fn jal_bits() -> u16 {
+    0b11011
+}
+
+fn opimm_bits(funct3: u16, funct7: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    0b00100 | (funct3 << 5) | (funct7 << 8)
+}
+
+fn opimm32_bits(funct3: u16, funct7: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    0b00110 | (funct3 << 5) | (funct7 << 8)
+}
+
+fn op_bits(funct3: u16, funct7: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    assert!(funct7 <= 0b111_1111);
+    0b01100 | (funct3 << 5) | (funct7 << 8)
+}
+
+fn op32_bits(funct3: u16, funct7: u16) -> u16 {
+    assert!(funct3 <= 0b111);
+    assert!(funct7 <= 0b111_1111);
+    0b01110 | (funct3 << 5) | (funct7 << 8)
+}
+
+fn lui_bits() -> u16 {
+    0b01101
+}
+
+pub(crate) fn define<'defs>(
+    shared_defs: &'defs SharedDefinitions,
+    isa_settings: &SettingGroup,
+    recipes: &'defs RecipeGroup,
+) -> PerCpuModeEncodings<'defs> {
+    // Instructions shorthands.
+    let shared = &shared_defs.instructions;
+
+    let band = shared.by_name("band");
+    let band_imm = shared.by_name("band_imm");
+    let bor = shared.by_name("bor");
+    let bor_imm = shared.by_name("bor_imm");
+    let br_icmp = shared.by_name("br_icmp");
+    let brz = shared.by_name("brz");
+    let brnz = shared.by_name("brnz");
+    let bxor = shared.by_name("bxor");
+    let bxor_imm = shared.by_name("bxor_imm");
+    let call = shared.by_name("call");
+    let call_indirect = shared.by_name("call_indirect");
+    let copy = shared.by_name("copy");
+    let copy_nop = shared.by_name("copy_nop");
+    let copy_to_ssa = shared.by_name("copy_to_ssa");
+    let fill = shared.by_name("fill");
+    let fill_nop = shared.by_name("fill_nop");
+    let iadd = shared.by_name("iadd");
+    let iadd_imm = shared.by_name("iadd_imm");
+    let iconst = shared.by_name("iconst");
+    let icmp = shared.by_name("icmp");
+    let icmp_imm = shared.by_name("icmp_imm");
+    let imul = shared.by_name("imul");
+    let ishl = shared.by_name("ishl");
+    let ishl_imm = shared.by_name("ishl_imm");
+    let isub = shared.by_name("isub");
+    let jump = shared.by_name("jump");
+    let regmove = shared.by_name("regmove");
+    let spill = shared.by_name("spill");
+    let sshr = shared.by_name("sshr");
+    let sshr_imm = shared.by_name("sshr_imm");
+    let ushr = shared.by_name("ushr");
+    let ushr_imm = shared.by_name("ushr_imm");
+    let return_ = shared.by_name("return");
+
+    // Recipes shorthands, prefixed with r_.
+    let r_copytossa = recipes.by_name("copytossa");
+    let r_fillnull = recipes.by_name("fillnull");
+    let r_icall = recipes.by_name("Icall");
+    let r_icopy = recipes.by_name("Icopy");
+    let r_ii = recipes.by_name("Ii");
+    let r_iicmp = recipes.by_name("Iicmp");
+    let r_iret = recipes.by_name("Iret");
+    let r_irmov = recipes.by_name("Irmov");
+    let r_iz = recipes.by_name("Iz");
+    let r_gp_sp = recipes.by_name("GPsp");
+    let r_gp_fi = recipes.by_name("GPfi");
+    let r_r = recipes.by_name("R");
+    let r_ricmp = recipes.by_name("Ricmp");
+    let r_rshamt = recipes.by_name("Rshamt");
+    let r_sb = recipes.by_name("SB");
+    let r_sb_zero = recipes.by_name("SBzero");
+    let r_stacknull = recipes.by_name("stacknull");
+    let r_u = recipes.by_name("U");
+    let r_uj = recipes.by_name("UJ");
+    let r_uj_call = recipes.by_name("UJcall");
+
+    // Predicates shorthands.
+    let use_m = isa_settings.predicate_by_name("use_m");
+
+    // Definitions.
+    let mut e = PerCpuModeEncodings::new(&recipes.recipes);
+
+    // Basic arithmetic binary instructions are encoded in an R-type instruction.
+    for &(inst, inst_imm, f3, f7) in &[
+        (iadd, Some(iadd_imm), 0b000, 0b000_0000),
+        (isub, None, 0b000, 0b010_0000),
+        (bxor, Some(bxor_imm), 0b100, 0b000_0000),
+        (bor, Some(bor_imm), 0b110, 0b000_0000),
+        (band, Some(band_imm), 0b111, 0b000_0000),
+    ] {
+        e.add32(e.enc(inst.bind(I32), r_r, op_bits(f3, f7)));
+        e.add64(e.enc(inst.bind(I64), r_r, op_bits(f3, f7)));
+
+        // Immediate versions for add/xor/or/and.
+        if let Some(inst_imm) = inst_imm {
+            e.add32(e.enc(inst_imm.bind(I32), r_ii, opimm_bits(f3, 0)));
+            e.add64(e.enc(inst_imm.bind(I64), r_ii, opimm_bits(f3, 0)));
+        }
+    }
+
+    // 32-bit ops in RV64.
+    e.add64(e.enc(iadd.bind(I32), r_r, op32_bits(0b000, 0b000_0000)));
+    e.add64(e.enc(isub.bind(I32), r_r, op32_bits(0b000, 0b010_0000)));
+    // There are no andiw/oriw/xoriw variations.
+    e.add64(e.enc(iadd_imm.bind(I32), r_ii, opimm32_bits(0b000, 0)));
+
+    // Use iadd_imm with %x0 to materialize constants.
+    e.add32(e.enc(iconst.bind(I32), r_iz, opimm_bits(0b0, 0)));
+    e.add64(e.enc(iconst.bind(I32), r_iz, opimm_bits(0b0, 0)));
+    e.add64(e.enc(iconst.bind(I64), r_iz, opimm_bits(0b0, 0)));
+
+    // Dynamic shifts have the same masking semantics as the clif base instructions.
+    for &(inst, inst_imm, f3, f7) in &[
+        (ishl, ishl_imm, 0b1, 0b0),
+        (ushr, ushr_imm, 0b101, 0b0),
+        (sshr, sshr_imm, 0b101, 0b10_0000),
+    ] {
+        e.add32(e.enc(inst.bind(I32).bind(I32), r_r, op_bits(f3, f7)));
+        e.add64(e.enc(inst.bind(I64).bind(I64), r_r, op_bits(f3, f7)));
+        e.add64(e.enc(inst.bind(I32).bind(I32), r_r, op32_bits(f3, f7)));
+        // Allow i32 shift amounts in 64-bit shifts.
+        e.add64(e.enc(inst.bind(I64).bind(I32), r_r, op_bits(f3, f7)));
+        e.add64(e.enc(inst.bind(I32).bind(I64), r_r, op32_bits(f3, f7)));
+
+        // Immediate shifts.
+        e.add32(e.enc(inst_imm.bind(I32), r_rshamt, opimm_bits(f3, f7)));
+        e.add64(e.enc(inst_imm.bind(I64), r_rshamt, opimm_bits(f3, f7)));
+        e.add64(e.enc(inst_imm.bind(I32), r_rshamt, opimm32_bits(f3, f7)));
+    }
+
+    // Signed and unsigned integer 'less than'. There are no 'w' variants for comparing 32-bit
+    // numbers in RV64.
+    {
+        let mut var_pool = VarPool::new();
+
+        // Helper that creates an instruction predicate for an instruction in the icmp family.
+        let mut icmp_instp = |bound_inst: &BoundInstruction,
+                              intcc_field: &'static str|
+         -> InstructionPredicateNode {
+            let x = var_pool.create("x");
+            let y = var_pool.create("y");
+            let cc = Literal::enumerator_for(&shared_defs.imm.intcc, intcc_field);
+            Apply::new(
+                bound_inst.clone().into(),
+                vec![Expr::Literal(cc), Expr::Var(x), Expr::Var(y)],
+            )
+            .inst_predicate(&var_pool)
+            .unwrap()
+        };
+
+        let icmp_i32 = icmp.bind(I32);
+        let icmp_i64 = icmp.bind(I64);
+        e.add32(
+            e.enc(icmp_i32.clone(), r_ricmp, op_bits(0b010, 0b000_0000))
+                .inst_predicate(icmp_instp(&icmp_i32, "slt")),
+        );
+        e.add64(
+            e.enc(icmp_i64.clone(), r_ricmp, op_bits(0b010, 0b000_0000))
+                .inst_predicate(icmp_instp(&icmp_i64, "slt")),
+        );
+
+        e.add32(
+            e.enc(icmp_i32.clone(), r_ricmp, op_bits(0b011, 0b000_0000))
+                .inst_predicate(icmp_instp(&icmp_i32, "ult")),
+        );
+        e.add64(
+            e.enc(icmp_i64.clone(), r_ricmp, op_bits(0b011, 0b000_0000))
+                .inst_predicate(icmp_instp(&icmp_i64, "ult")),
+        );
+
+        // Immediate variants.
+        let icmp_i32 = icmp_imm.bind(I32);
+        let icmp_i64 = icmp_imm.bind(I64);
+        e.add32(
+            e.enc(icmp_i32.clone(), r_iicmp, opimm_bits(0b010, 0))
+                .inst_predicate(icmp_instp(&icmp_i32, "slt")),
+        );
+        e.add64(
+            e.enc(icmp_i64.clone(), r_iicmp, opimm_bits(0b010, 0))
+                .inst_predicate(icmp_instp(&icmp_i64, "slt")),
+        );
+
+        e.add32(
+            e.enc(icmp_i32.clone(), r_iicmp, opimm_bits(0b011, 0))
+                .inst_predicate(icmp_instp(&icmp_i32, "ult")),
+        );
+        e.add64(
+            e.enc(icmp_i64.clone(), r_iicmp, opimm_bits(0b011, 0))
+                .inst_predicate(icmp_instp(&icmp_i64, "ult")),
+        );
+    }
+
+    // Integer constants with the low 12 bits clear are materialized by lui.
+    e.add32(e.enc(iconst.bind(I32), r_u, lui_bits()));
+    e.add64(e.enc(iconst.bind(I32), r_u, lui_bits()));
+    e.add64(e.enc(iconst.bind(I64), r_u, lui_bits()));
+
+    // "M" Standard Extension for Integer Multiplication and Division.
+    // Gated by the `use_m` flag.
+    e.add32(
+        e.enc(imul.bind(I32), r_r, op_bits(0b000, 0b0000_0001))
+            .isa_predicate(use_m),
+    );
+    e.add64(
+        e.enc(imul.bind(I64), r_r, op_bits(0b000, 0b0000_0001))
+            .isa_predicate(use_m),
+    );
+    e.add64(
+        e.enc(imul.bind(I32), r_r, op32_bits(0b000, 0b0000_0001))
+            .isa_predicate(use_m),
+    );
+
+    // Control flow.
+
+    // Unconditional branches.
+    e.add32(e.enc(jump, r_uj, jal_bits()));
+    e.add64(e.enc(jump, r_uj, jal_bits()));
+    e.add32(e.enc(call, r_uj_call, jal_bits()));
+    e.add64(e.enc(call, r_uj_call, jal_bits()));
+
+    // Conditional branches.
+    {
+        let mut var_pool = VarPool::new();
+
+        // Helper that creates an instruction predicate for an instruction in the icmp family.
+        let mut br_icmp_instp = |bound_inst: &BoundInstruction,
+                                 intcc_field: &'static str|
+         -> InstructionPredicateNode {
+            let x = var_pool.create("x");
+            let y = var_pool.create("y");
+            let dest = var_pool.create("dest");
+            let args = var_pool.create("args");
+            let cc = Literal::enumerator_for(&shared_defs.imm.intcc, intcc_field);
+            Apply::new(
+                bound_inst.clone().into(),
+                vec![
+                    Expr::Literal(cc),
+                    Expr::Var(x),
+                    Expr::Var(y),
+                    Expr::Var(dest),
+                    Expr::Var(args),
+                ],
+            )
+            .inst_predicate(&var_pool)
+            .unwrap()
+        };
+
+        let br_icmp_i32 = br_icmp.bind(I32);
+        let br_icmp_i64 = br_icmp.bind(I64);
+        for &(cond, f3) in &[
+            ("eq", 0b000),
+            ("ne", 0b001),
+            ("slt", 0b100),
+            ("sge", 0b101),
+            ("ult", 0b110),
+            ("uge", 0b111),
+        ] {
+            e.add32(
+                e.enc(br_icmp_i32.clone(), r_sb, branch_bits(f3))
+                    .inst_predicate(br_icmp_instp(&br_icmp_i32, cond)),
+            );
+            e.add64(
+                e.enc(br_icmp_i64.clone(), r_sb, branch_bits(f3))
+                    .inst_predicate(br_icmp_instp(&br_icmp_i64, cond)),
+            );
+        }
+    }
+
+    for &(inst, f3) in &[(brz, 0b000), (brnz, 0b001)] {
+        e.add32(e.enc(inst.bind(I32), r_sb_zero, branch_bits(f3)));
+        e.add64(e.enc(inst.bind(I64), r_sb_zero, branch_bits(f3)));
+        e.add32(e.enc(inst.bind(B1), r_sb_zero, branch_bits(f3)));
+        e.add64(e.enc(inst.bind(B1), r_sb_zero, branch_bits(f3)));
+    }
+
+    // Returns are a special case of jalr_bits using %x1 to hold the return address.
+    // The return address is provided by a special-purpose `link` return value that
+    // is added by legalize_signature().
+    e.add32(e.enc(return_, r_iret, jalr_bits()));
+    e.add64(e.enc(return_, r_iret, jalr_bits()));
+    e.add32(e.enc(call_indirect.bind(I32), r_icall, jalr_bits()));
+    e.add64(e.enc(call_indirect.bind(I64), r_icall, jalr_bits()));
+
+    // Spill and fill.
+    e.add32(e.enc(spill.bind(I32), r_gp_sp, store_bits(0b010)));
+    e.add64(e.enc(spill.bind(I32), r_gp_sp, store_bits(0b010)));
+    e.add64(e.enc(spill.bind(I64), r_gp_sp, store_bits(0b011)));
+    e.add32(e.enc(fill.bind(I32), r_gp_fi, load_bits(0b010)));
+    e.add64(e.enc(fill.bind(I32), r_gp_fi, load_bits(0b010)));
+    e.add64(e.enc(fill.bind(I64), r_gp_fi, load_bits(0b011)));
+
+    // No-op fills, created by late-stage redundant-fill removal.
+    for &ty in &[I64, I32] {
+        e.add64(e.enc(fill_nop.bind(ty), r_fillnull, 0));
+        e.add32(e.enc(fill_nop.bind(ty), r_fillnull, 0));
+    }
+    e.add64(e.enc(fill_nop.bind(B1), r_fillnull, 0));
+    e.add32(e.enc(fill_nop.bind(B1), r_fillnull, 0));
+
+    // Register copies.
+    e.add32(e.enc(copy.bind(I32), r_icopy, opimm_bits(0b000, 0)));
+    e.add64(e.enc(copy.bind(I64), r_icopy, opimm_bits(0b000, 0)));
+    e.add64(e.enc(copy.bind(I32), r_icopy, opimm32_bits(0b000, 0)));
+
+    e.add32(e.enc(regmove.bind(I32), r_irmov, opimm_bits(0b000, 0)));
+    e.add64(e.enc(regmove.bind(I64), r_irmov, opimm_bits(0b000, 0)));
+    e.add64(e.enc(regmove.bind(I32), r_irmov, opimm32_bits(0b000, 0)));
+
+    e.add32(e.enc(copy.bind(B1), r_icopy, opimm_bits(0b000, 0)));
+    e.add64(e.enc(copy.bind(B1), r_icopy, opimm_bits(0b000, 0)));
+    e.add32(e.enc(regmove.bind(B1), r_irmov, opimm_bits(0b000, 0)));
+    e.add64(e.enc(regmove.bind(B1), r_irmov, opimm_bits(0b000, 0)));
+
+    // Stack-slot-to-the-same-stack-slot copy, which is guaranteed to turn
+    // into a no-op.
+    // The same encoding is generated for both the 64- and 32-bit architectures.
+    for &ty in &[I64, I32, I16, I8] {
+        e.add32(e.enc(copy_nop.bind(ty), r_stacknull, 0));
+        e.add64(e.enc(copy_nop.bind(ty), r_stacknull, 0));
+    }
+    for &ty in &[F64, F32] {
+        e.add32(e.enc(copy_nop.bind(ty), r_stacknull, 0));
+        e.add64(e.enc(copy_nop.bind(ty), r_stacknull, 0));
+    }
+
+    // Copy-to-SSA
+    e.add32(e.enc(copy_to_ssa.bind(I32), r_copytossa, opimm_bits(0b000, 0)));
+    e.add64(e.enc(copy_to_ssa.bind(I64), r_copytossa, opimm_bits(0b000, 0)));
+    e.add64(e.enc(copy_to_ssa.bind(I32), r_copytossa, opimm32_bits(0b000, 0)));
+    e.add32(e.enc(copy_to_ssa.bind(B1), r_copytossa, opimm_bits(0b000, 0)));
+    e.add64(e.enc(copy_to_ssa.bind(B1), r_copytossa, opimm_bits(0b000, 0)));
+    e.add32(e.enc(copy_to_ssa.bind(R32), r_copytossa, opimm_bits(0b000, 0)));
+    e.add64(e.enc(copy_to_ssa.bind(R64), r_copytossa, opimm_bits(0b000, 0)));
+
+    e
+}

cranelift/codegen/meta/src/isa/riscv/mod.rs

@@ -0,0 +1,134 @@
+use crate::cdsl::cpu_modes::CpuMode;
+use crate::cdsl::instructions::InstructionGroupBuilder;
+use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::regs::{IsaRegs, IsaRegsBuilder, RegBankBuilder, RegClassBuilder};
+use crate::cdsl::settings::{PredicateNode, SettingGroup, SettingGroupBuilder};
+
+use crate::shared::types::Float::{F32, F64};
+use crate::shared::types::Int::{I32, I64};
+use crate::shared::Definitions as SharedDefinitions;
+
+mod encodings;
+mod recipes;
+
+fn define_settings(shared: &SettingGroup) -> SettingGroup {
+    let mut setting = SettingGroupBuilder::new("riscv");
+
+    let supports_m = setting.add_bool(
+        "supports_m",
+        "CPU supports the 'M' extension (mul/div)",
+        false,
+    );
+    let supports_a = setting.add_bool(
+        "supports_a",
+        "CPU supports the 'A' extension (atomics)",
+        false,
+    );
+    let supports_f = setting.add_bool(
+        "supports_f",
+        "CPU supports the 'F' extension (float)",
+        false,
+    );
+    let supports_d = setting.add_bool(
+        "supports_d",
+        "CPU supports the 'D' extension (double)",
+        false,
+    );
+
+    let enable_m = setting.add_bool(
+        "enable_m",
+        "Enable the use of 'M' instructions if available",
+        true,
+    );
+
+    setting.add_bool(
+        "enable_e",
+        "Enable the 'RV32E' instruction set with only 16 registers",
+        false,
+    );
+
+    let shared_enable_atomics = shared.get_bool("enable_atomics");
+    let shared_enable_float = shared.get_bool("enable_float");
+    let shared_enable_simd = shared.get_bool("enable_simd");
+
+    setting.add_predicate("use_m", predicate!(supports_m && enable_m));
+    setting.add_predicate("use_a", predicate!(supports_a && shared_enable_atomics));
+    setting.add_predicate("use_f", predicate!(supports_f && shared_enable_float));
+    setting.add_predicate("use_d", predicate!(supports_d && shared_enable_float));
+    setting.add_predicate(
+        "full_float",
+        predicate!(shared_enable_simd && supports_f && supports_d),
+    );
+
+    setting.build()
+}
+
+fn define_registers() -> IsaRegs {
+    let mut regs = IsaRegsBuilder::new();
+
+    let builder = RegBankBuilder::new("IntRegs", "x")
+        .units(32)
+        .track_pressure(true);
+    let int_regs = regs.add_bank(builder);
+
+    let builder = RegBankBuilder::new("FloatRegs", "f")
+        .units(32)
+        .track_pressure(true);
+    let float_regs = regs.add_bank(builder);
+
+    let builder = RegClassBuilder::new_toplevel("GPR", int_regs);
+    regs.add_class(builder);
+
+    let builder = RegClassBuilder::new_toplevel("FPR", float_regs);
+    regs.add_class(builder);
+
+    regs.build()
+}
+
+pub(crate) fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
+    let settings = define_settings(&shared_defs.settings);
+    let regs = define_registers();
+
+    let inst_group = InstructionGroupBuilder::new(&mut shared_defs.all_instructions).build();
+
+    // CPU modes for 32-bit and 64-bit operation.
+    let mut rv_32 = CpuMode::new("RV32");
+    let mut rv_64 = CpuMode::new("RV64");
+
+    let expand = shared_defs.transform_groups.by_name("expand");
+    let narrow_no_flags = shared_defs.transform_groups.by_name("narrow_no_flags");
+
+    rv_32.legalize_monomorphic(expand);
+    rv_32.legalize_default(narrow_no_flags);
+    rv_32.legalize_type(I32, expand);
+    rv_32.legalize_type(F32, expand);
+    rv_32.legalize_type(F64, expand);
+
+    rv_64.legalize_monomorphic(expand);
+    rv_64.legalize_default(narrow_no_flags);
+    rv_64.legalize_type(I32, expand);
+    rv_64.legalize_type(I64, expand);
+    rv_64.legalize_type(F32, expand);
+    rv_64.legalize_type(F64, expand);
+
+    let recipes = recipes::define(shared_defs, &regs);
+
+    let encodings = encodings::define(shared_defs, &settings, &recipes);
+    rv_32.set_encodings(encodings.enc32);
+    rv_64.set_encodings(encodings.enc64);
+    let encodings_predicates = encodings.inst_pred_reg.extract();
+
+    let recipes = recipes.collect();
+
+    let cpu_modes = vec![rv_32, rv_64];
+
+    TargetIsa::new(
+        "riscv",
+        inst_group,
+        settings,
+        regs,
+        recipes,
+        cpu_modes,
+        encodings_predicates,
+    )
+}

cranelift/codegen/meta/src/isa/riscv/recipes.rs

@@ -0,0 +1,278 @@
+use std::collections::HashMap;
+
+use crate::cdsl::instructions::InstructionPredicate;
+use crate::cdsl::recipes::{EncodingRecipeBuilder, EncodingRecipeNumber, Recipes, Stack};
+use crate::cdsl::regs::IsaRegs;
+use crate::shared::Definitions as SharedDefinitions;
+
+/// An helper to create recipes and use them when defining the RISCV encodings.
+pub(crate) struct RecipeGroup {
+    /// The actualy list of recipes explicitly created in this file.
+    pub recipes: Recipes,
+
+    /// Provides fast lookup from a name to an encoding recipe.
+    name_to_recipe: HashMap<String, EncodingRecipeNumber>,
+}
+
+impl RecipeGroup {
+    fn new() -> Self {
+        Self {
+            recipes: Recipes::new(),
+            name_to_recipe: HashMap::new(),
+        }
+    }
+
+    fn push(&mut self, builder: EncodingRecipeBuilder) {
+        assert!(
+            self.name_to_recipe.get(&builder.name).is_none(),
+            format!("riscv recipe '{}' created twice", builder.name)
+        );
+        let name = builder.name.clone();
+        let number = self.recipes.push(builder.build());
+        self.name_to_recipe.insert(name, number);
+    }
+
+    pub fn by_name(&self, name: &str) -> EncodingRecipeNumber {
+        *self
+            .name_to_recipe
+            .get(name)
+            .unwrap_or_else(|| panic!("unknown riscv recipe name {}", name))
+    }
+
+    pub fn collect(self) -> Recipes {
+        self.recipes
+    }
+}
+
+pub(crate) fn define(shared_defs: &SharedDefinitions, regs: &IsaRegs) -> RecipeGroup {
+    let formats = &shared_defs.formats;
+
+    // Register classes shorthands.
+    let gpr = regs.class_by_name("GPR");
+
+    // Definitions.
+    let mut recipes = RecipeGroup::new();
+
+    // R-type 32-bit instructions: These are mostly binary arithmetic instructions.
+    // The encbits are `opcode[6:2] | (funct3 << 5) | (funct7 << 8)
+    recipes.push(
+        EncodingRecipeBuilder::new("R", &formats.binary, 4)
+            .operands_in(vec![gpr, gpr])
+            .operands_out(vec![gpr])
+            .emit("put_r(bits, in_reg0, in_reg1, out_reg0, sink);"),
+    );
+
+    // R-type with an immediate shift amount instead of rs2.
+    recipes.push(
+        EncodingRecipeBuilder::new("Rshamt", &formats.binary_imm, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .emit("put_rshamt(bits, in_reg0, imm.into(), out_reg0, sink);"),
+    );
+
+    // R-type encoding of an integer comparison.
+    recipes.push(
+        EncodingRecipeBuilder::new("Ricmp", &formats.int_compare, 4)
+            .operands_in(vec![gpr, gpr])
+            .operands_out(vec![gpr])
+            .emit("put_r(bits, in_reg0, in_reg1, out_reg0, sink);"),
+    );
+
+    recipes.push(
+        EncodingRecipeBuilder::new("Ii", &formats.binary_imm, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .inst_predicate(InstructionPredicate::new_is_signed_int(
+                &*formats.binary_imm,
+                "imm",
+                12,
+                0,
+            ))
+            .emit("put_i(bits, in_reg0, imm.into(), out_reg0, sink);"),
+    );
+
+    // I-type instruction with a hardcoded %x0 rs1.
+    recipes.push(
+        EncodingRecipeBuilder::new("Iz", &formats.unary_imm, 4)
+            .operands_out(vec![gpr])
+            .inst_predicate(InstructionPredicate::new_is_signed_int(
+                &formats.unary_imm,
+                "imm",
+                12,
+                0,
+            ))
+            .emit("put_i(bits, 0, imm.into(), out_reg0, sink);"),
+    );
+
+    // I-type encoding of an integer comparison.
+    recipes.push(
+        EncodingRecipeBuilder::new("Iicmp", &formats.int_compare_imm, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .inst_predicate(InstructionPredicate::new_is_signed_int(
+                &formats.int_compare_imm,
+                "imm",
+                12,
+                0,
+            ))
+            .emit("put_i(bits, in_reg0, imm.into(), out_reg0, sink);"),
+    );
+
+    // I-type encoding for `jalr` as a return instruction. We won't use the immediate offset.  The
+    // variable return values are not encoded.
+    recipes.push(
+        EncodingRecipeBuilder::new("Iret", &formats.multiary, 4).emit(
+            r#"
+                    // Return instructions are always a jalr to %x1.
+                    // The return address is provided as a special-purpose link argument.
+                    put_i(
+                        bits,
+                        1, // rs1 = %x1
+                        0, // no offset.
+                        0, // rd = %x0: no address written.
+                        sink,
+                    );
+                "#,
+        ),
+    );
+
+    // I-type encoding for `jalr` as a call_indirect.
+    recipes.push(
+        EncodingRecipeBuilder::new("Icall", &formats.call_indirect, 4)
+            .operands_in(vec![gpr])
+            .emit(
+                r#"
+                    // call_indirect instructions are jalr with rd=%x1.
+                    put_i(
+                        bits,
+                        in_reg0,
+                        0, // no offset.
+                        1, // rd = %x1: link register.
+                        sink,
+                    );
+                "#,
+            ),
+    );
+
+    // Copy of a GPR is implemented as addi x, 0.
+    recipes.push(
+        EncodingRecipeBuilder::new("Icopy", &formats.unary, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![gpr])
+            .emit("put_i(bits, in_reg0, 0, out_reg0, sink);"),
+    );
+
+    // Same for a GPR regmove.
+    recipes.push(
+        EncodingRecipeBuilder::new("Irmov", &formats.reg_move, 4)
+            .operands_in(vec![gpr])
+            .emit("put_i(bits, src, 0, dst, sink);"),
+    );
+
+    // Same for copy-to-SSA -- GPR regmove.
+    recipes.push(
+        EncodingRecipeBuilder::new("copytossa", &formats.copy_to_ssa, 4)
+            // No operands_in to mention, because a source register is specified directly.
+            .operands_out(vec![gpr])
+            .emit("put_i(bits, src, 0, out_reg0, sink);"),
+    );
+
+    // U-type instructions have a 20-bit immediate that targets bits 12-31.
+    recipes.push(
+        EncodingRecipeBuilder::new("U", &formats.unary_imm, 4)
+            .operands_out(vec![gpr])
+            .inst_predicate(InstructionPredicate::new_is_signed_int(
+                &formats.unary_imm,
+                "imm",
+                32,
+                12,
+            ))
+            .emit("put_u(bits, imm.into(), out_reg0, sink);"),
+    );
+
+    // UJ-type unconditional branch instructions.
+    recipes.push(
+        EncodingRecipeBuilder::new("UJ", &formats.jump, 4)
+            .branch_range((0, 21))
+            .emit(
+                r#"
+                    let dest = i64::from(func.offsets[destination]);
+                    let disp = dest - i64::from(sink.offset());
+                    put_uj(bits, disp, 0, sink);
+                "#,
+            ),
+    );
+
+    recipes.push(EncodingRecipeBuilder::new("UJcall", &formats.call, 4).emit(
+        r#"
+                    sink.reloc_external(Reloc::RiscvCall,
+                                        &func.dfg.ext_funcs[func_ref].name,
+                                        0);
+                    // rd=%x1 is the standard link register.
+                    put_uj(bits, 0, 1, sink);
+                "#,
+    ));
+
+    // SB-type branch instructions.
+    recipes.push(
+        EncodingRecipeBuilder::new("SB", &formats.branch_icmp, 4)
+            .operands_in(vec![gpr, gpr])
+            .branch_range((0, 13))
+            .emit(
+                r#"
+                    let dest = i64::from(func.offsets[destination]);
+                    let disp = dest - i64::from(sink.offset());
+                    put_sb(bits, disp, in_reg0, in_reg1, sink);
+                "#,
+            ),
+    );
+
+    // SB-type branch instruction with rs2 fixed to zero.
+    recipes.push(
+        EncodingRecipeBuilder::new("SBzero", &formats.branch, 4)
+            .operands_in(vec![gpr])
+            .branch_range((0, 13))
+            .emit(
+                r#"
+                    let dest = i64::from(func.offsets[destination]);
+                    let disp = dest - i64::from(sink.offset());
+                    put_sb(bits, disp, in_reg0, 0, sink);
+                "#,
+            ),
+    );
+
+    // Spill of a GPR.
+    recipes.push(
+        EncodingRecipeBuilder::new("GPsp", &formats.unary, 4)
+            .operands_in(vec![gpr])
+            .operands_out(vec![Stack::new(gpr)])
+            .emit("unimplemented!();"),
+    );
+
+    // Fill of a GPR.
+    recipes.push(
+        EncodingRecipeBuilder::new("GPfi", &formats.unary, 4)
+            .operands_in(vec![Stack::new(gpr)])
+            .operands_out(vec![gpr])
+            .emit("unimplemented!();"),
+    );
+
+    // Stack-slot to same stack-slot copy, which is guaranteed to turn into a no-op.
+    recipes.push(
+        EncodingRecipeBuilder::new("stacknull", &formats.unary, 0)
+            .operands_in(vec![Stack::new(gpr)])
+            .operands_out(vec![Stack::new(gpr)])
+            .emit(""),
+    );
+
+    // No-op fills, created by late-stage redundant-fill removal.
+    recipes.push(
+        EncodingRecipeBuilder::new("fillnull", &formats.unary, 0)
+            .operands_in(vec![Stack::new(gpr)])
+            .operands_out(vec![gpr])
+            .clobbers_flags(false)
+            .emit(""),
+    );
+
+    recipes
+}

cranelift/codegen/meta/src/isa/x86/instructions.rs

@@ -0,0 +1,582 @@
+#![allow(non_snake_case)]
+
+use crate::cdsl::instructions::{
+    AllInstructions, InstructionBuilder as Inst, InstructionGroup, InstructionGroupBuilder,
+};
+use crate::cdsl::operands::Operand;
+use crate::cdsl::types::ValueType;
+use crate::cdsl::typevar::{Interval, TypeSetBuilder, TypeVar};
+
+use crate::shared::entities::EntityRefs;
+use crate::shared::formats::Formats;
+use crate::shared::immediates::Immediates;
+use crate::shared::types;
+
+#[allow(clippy::many_single_char_names)]
+pub(crate) fn define(
+    mut all_instructions: &mut AllInstructions,
+    formats: &Formats,
+    immediates: &Immediates,
+    entities: &EntityRefs,
+) -> InstructionGroup {
+    let mut ig = InstructionGroupBuilder::new(&mut all_instructions);
+
+    let iflags: &TypeVar = &ValueType::Special(types::Flag::IFlags.into()).into();
+
+    let iWord = &TypeVar::new(
+        "iWord",
+        "A scalar integer machine word",
+        TypeSetBuilder::new().ints(32..64).build(),
+    );
+    let nlo = &Operand::new("nlo", iWord).with_doc("Low part of numerator");
+    let nhi = &Operand::new("nhi", iWord).with_doc("High part of numerator");
+    let d = &Operand::new("d", iWord).with_doc("Denominator");
+    let q = &Operand::new("q", iWord).with_doc("Quotient");
+    let r = &Operand::new("r", iWord).with_doc("Remainder");
+
+    ig.push(
+        Inst::new(
+            "x86_udivmodx",
+            r#"
+        Extended unsigned division.
+
+        Concatenate the bits in `nhi` and `nlo` to form the numerator.
+        Interpret the bits as an unsigned number and divide by the unsigned
+        denominator `d`. Trap when `d` is zero or if the quotient is larger
+        than the range of the output.
+
+        Return both quotient and remainder.
+        "#,
+            &formats.ternary,
+        )
+        .operands_in(vec![nlo, nhi, d])
+        .operands_out(vec![q, r])
+        .can_trap(true),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_sdivmodx",
+            r#"
+        Extended signed division.
+
+        Concatenate the bits in `nhi` and `nlo` to form the numerator.
+        Interpret the bits as a signed number and divide by the signed
+        denominator `d`. Trap when `d` is zero or if the quotient is outside
+        the range of the output.
+
+        Return both quotient and remainder.
+        "#,
+            &formats.ternary,
+        )
+        .operands_in(vec![nlo, nhi, d])
+        .operands_out(vec![q, r])
+        .can_trap(true),
+    );
+
+    let argL = &Operand::new("argL", iWord);
+    let argR = &Operand::new("argR", iWord);
+    let resLo = &Operand::new("resLo", iWord);
+    let resHi = &Operand::new("resHi", iWord);
+
+    ig.push(
+        Inst::new(
+            "x86_umulx",
+            r#"
+        Unsigned integer multiplication, producing a double-length result.
+
+        Polymorphic over all scalar integer types, but does not support vector
+        types.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![argL, argR])
+        .operands_out(vec![resLo, resHi]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_smulx",
+            r#"
+        Signed integer multiplication, producing a double-length result.
+
+        Polymorphic over all scalar integer types, but does not support vector
+        types.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![argL, argR])
+        .operands_out(vec![resLo, resHi]),
+    );
+
+    let Float = &TypeVar::new(
+        "Float",
+        "A scalar or vector floating point number",
+        TypeSetBuilder::new()
+            .floats(Interval::All)
+            .simd_lanes(Interval::All)
+            .build(),
+    );
+    let IntTo = &TypeVar::new(
+        "IntTo",
+        "An integer type with the same number of lanes",
+        TypeSetBuilder::new()
+            .ints(32..64)
+            .simd_lanes(Interval::All)
+            .build(),
+    );
+    let x = &Operand::new("x", Float);
+    let a = &Operand::new("a", IntTo);
+
+    ig.push(
+        Inst::new(
+            "x86_cvtt2si",
+            r#"
+        Convert with truncation floating point to signed integer.
+
+        The source floating point operand is converted to a signed integer by
+        rounding towards zero. If the result can't be represented in the output
+        type, returns the smallest signed value the output type can represent.
+
+        This instruction does not trap.
+        "#,
+            &formats.unary,
+        )
+        .operands_in(vec![x])
+        .operands_out(vec![a]),
+    );
+
+    let x = &Operand::new("x", Float);
+    let a = &Operand::new("a", Float);
+    let y = &Operand::new("y", Float);
+
+    ig.push(
+        Inst::new(
+            "x86_fmin",
+            r#"
+        Floating point minimum with x86 semantics.
+
+        This is equivalent to the C ternary operator `x < y ? x : y` which
+        differs from `fmin` when either operand is NaN or when comparing
+        +0.0 to -0.0.
+
+        When the two operands don't compare as LT, `y` is returned unchanged,
+        even if it is a signalling NaN.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_fmax",
+            r#"
+        Floating point maximum with x86 semantics.
+
+        This is equivalent to the C ternary operator `x > y ? x : y` which
+        differs from `fmax` when either operand is NaN or when comparing
+        +0.0 to -0.0.
+
+        When the two operands don't compare as GT, `y` is returned unchanged,
+        even if it is a signalling NaN.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    let x = &Operand::new("x", iWord);
+
+    ig.push(
+        Inst::new(
+            "x86_push",
+            r#"
+    Pushes a value onto the stack.
+
+    Decrements the stack pointer and stores the specified value on to the top.
+
+    This is polymorphic in i32 and i64. However, it is only implemented for i64
+    in 64-bit mode, and only for i32 in 32-bit mode.
+    "#,
+            &formats.unary,
+        )
+        .operands_in(vec![x])
+        .other_side_effects(true)
+        .can_store(true),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_pop",
+            r#"
+    Pops a value from the stack.
+
+    Loads a value from the top of the stack and then increments the stack
+    pointer.
+
+    This is polymorphic in i32 and i64. However, it is only implemented for i64
+    in 64-bit mode, and only for i32 in 32-bit mode.
+    "#,
+            &formats.nullary,
+        )
+        .operands_out(vec![x])
+        .other_side_effects(true)
+        .can_load(true),
+    );
+
+    let y = &Operand::new("y", iWord);
+    let rflags = &Operand::new("rflags", iflags);
+
+    ig.push(
+        Inst::new(
+            "x86_bsr",
+            r#"
+    Bit Scan Reverse -- returns the bit-index of the most significant 1
+    in the word. Result is undefined if the argument is zero. However, it
+    sets the Z flag depending on the argument, so it is at least easy to
+    detect and handle that case.
+
+    This is polymorphic in i32 and i64. It is implemented for both i64 and
+    i32 in 64-bit mode, and only for i32 in 32-bit mode.
+    "#,
+            &formats.unary,
+        )
+        .operands_in(vec![x])
+        .operands_out(vec![y, rflags]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_bsf",
+            r#"
+    Bit Scan Forwards -- returns the bit-index of the least significant 1
+    in the word. Is otherwise identical to 'bsr', just above.
+    "#,
+            &formats.unary,
+        )
+        .operands_in(vec![x])
+        .operands_out(vec![y, rflags]),
+    );
+
+    let uimm8 = &immediates.uimm8;
+    let TxN = &TypeVar::new(
+        "TxN",
+        "A SIMD vector type",
+        TypeSetBuilder::new()
+            .ints(Interval::All)
+            .floats(Interval::All)
+            .bools(Interval::All)
+            .simd_lanes(Interval::All)
+            .includes_scalars(false)
+            .build(),
+    );
+    let a = &Operand::new("a", TxN).with_doc("A vector value (i.e. held in an XMM register)");
+    let b = &Operand::new("b", TxN).with_doc("A vector value (i.e. held in an XMM register)");
+    let i = &Operand::new("i", uimm8,).with_doc( "An ordering operand controlling the copying of data from the source to the destination; see PSHUFD in Intel manual for details");
+
+    ig.push(
+        Inst::new(
+            "x86_pshufd",
+            r#"
+    Packed Shuffle Doublewords -- copies data from either memory or lanes in an extended
+    register and re-orders the data according to the passed immediate byte.
+    "#,
+            &formats.extract_lane,
+        )
+        .operands_in(vec![a, i]) // TODO allow copying from memory here (need more permissive type than TxN)
+        .operands_out(vec![a]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_pshufb",
+            r#"
+    Packed Shuffle Bytes -- re-orders data in an extended register using a shuffle
+    mask from either memory or another extended register
+    "#,
+            &formats.binary,
+        )
+        .operands_in(vec![a, b]) // TODO allow re-ordering from memory here (need more permissive type than TxN)
+        .operands_out(vec![a]),
+    );
+
+    let Idx = &Operand::new("Idx", uimm8).with_doc("Lane index");
+    let x = &Operand::new("x", TxN);
+    let a = &Operand::new("a", &TxN.lane_of());
+
+    ig.push(
+        Inst::new(
+            "x86_pextr",
+            r#"
+        Extract lane ``Idx`` from ``x``.
+        The lane index, ``Idx``, is an immediate value, not an SSA value. It
+        must indicate a valid lane index for the type of ``x``.
+        "#,
+            &formats.extract_lane,
+        )
+        .operands_in(vec![x, Idx])
+        .operands_out(vec![a]),
+    );
+
+    let IBxN = &TypeVar::new(
+        "IBxN",
+        "A SIMD vector type containing only booleans and integers",
+        TypeSetBuilder::new()
+            .ints(Interval::All)
+            .bools(Interval::All)
+            .simd_lanes(Interval::All)
+            .includes_scalars(false)
+            .build(),
+    );
+    let x = &Operand::new("x", IBxN);
+    let y = &Operand::new("y", &IBxN.lane_of()).with_doc("New lane value");
+    let a = &Operand::new("a", IBxN);
+
+    ig.push(
+        Inst::new(
+            "x86_pinsr",
+            r#"
+        Insert ``y`` into ``x`` at lane ``Idx``.
+        The lane index, ``Idx``, is an immediate value, not an SSA value. It
+        must indicate a valid lane index for the type of ``x``.
+        "#,
+            &formats.insert_lane,
+        )
+        .operands_in(vec![x, Idx, y])
+        .operands_out(vec![a]),
+    );
+
+    let FxN = &TypeVar::new(
+        "FxN",
+        "A SIMD vector type containing floats",
+        TypeSetBuilder::new()
+            .floats(Interval::All)
+            .simd_lanes(Interval::All)
+            .includes_scalars(false)
+            .build(),
+    );
+    let x = &Operand::new("x", FxN);
+    let y = &Operand::new("y", &FxN.lane_of()).with_doc("New lane value");
+    let a = &Operand::new("a", FxN);
+
+    ig.push(
+        Inst::new(
+            "x86_insertps",
+            r#"
+        Insert a lane of ``y`` into ``x`` at using ``Idx`` to encode both which lane the value is
+        extracted from and which it is inserted to. This is similar to x86_pinsr but inserts
+        floats, which are already stored in an XMM register.
+        "#,
+            &formats.insert_lane,
+        )
+        .operands_in(vec![x, Idx, y])
+        .operands_out(vec![a]),
+    );
+
+    let x = &Operand::new("x", FxN);
+    let y = &Operand::new("y", FxN);
+    let a = &Operand::new("a", FxN);
+
+    ig.push(
+        Inst::new(
+            "x86_movsd",
+            r#"
+        Move the low 64 bits of the float vector ``y`` to the low 64 bits of float vector ``x``
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_movlhps",
+            r#"
+        Move the low 64 bits of the float vector ``y`` to the high 64 bits of float vector ``x``
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    let IxN = &TypeVar::new(
+        "IxN",
+        "A SIMD vector type containing integers",
+        TypeSetBuilder::new()
+            .ints(Interval::All)
+            .simd_lanes(Interval::All)
+            .includes_scalars(false)
+            .build(),
+    );
+    let I64x2 = &TypeVar::new(
+        "I64x2",
+        "A SIMD vector type containing one large integer (the upper lane is concatenated with \
+         the lower lane to form the integer)",
+        TypeSetBuilder::new()
+            .ints(64..64)
+            .simd_lanes(2..2)
+            .includes_scalars(false)
+            .build(),
+    );
+
+    let x = &Operand::new("x", IxN).with_doc("Vector value to shift");
+    let y = &Operand::new("y", I64x2).with_doc("Number of bits to shift");
+    let a = &Operand::new("a", IxN);
+
+    ig.push(
+        Inst::new(
+            "x86_psll",
+            r#"
+        Shift Packed Data Left Logical -- This implements the behavior of the shared instruction
+        ``ishl`` but alters the shift operand to live in an XMM register as expected by the PSLL*
+        family of instructions.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_psrl",
+            r#"
+        Shift Packed Data Right Logical -- This implements the behavior of the shared instruction
+        ``ushr`` but alters the shift operand to live in an XMM register as expected by the PSRL*
+        family of instructions.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_psra",
+            r#"
+        Shift Packed Data Right Arithmetic -- This implements the behavior of the shared
+        instruction ``sshr`` but alters the shift operand to live in an XMM register as expected by
+        the PSRA* family of instructions.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    let x = &Operand::new("x", TxN);
+    let y = &Operand::new("y", TxN);
+    let f = &Operand::new("f", iflags);
+    ig.push(
+        Inst::new(
+            "x86_ptest",
+            r#"
+        Logical Compare -- PTEST will set the ZF flag if all bits in the result are 0 of the
+        bitwise AND of the first source operand (first operand) and the second source operand
+        (second operand). PTEST sets the CF flag if all bits in the result are 0 of the bitwise
+        AND of the second source operand (second operand) and the logical NOT of the destination
+        operand (first operand).
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![f]),
+    );
+
+    let x = &Operand::new("x", IxN);
+    let y = &Operand::new("y", IxN);
+    let a = &Operand::new("a", IxN);
+    ig.push(
+        Inst::new(
+            "x86_pmaxs",
+            r#"
+        Maximum of Packed Signed Integers -- Compare signed integers in the first and second
+        operand and return the maximum values.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_pmaxu",
+            r#"
+        Maximum of Packed Unsigned Integers -- Compare unsigned integers in the first and second
+        operand and return the maximum values.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_pmins",
+            r#"
+        Minimum of Packed Signed Integers -- Compare signed integers in the first and second
+        operand and return the minimum values.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    ig.push(
+        Inst::new(
+            "x86_pminu",
+            r#"
+        Minimum of Packed Unsigned Integers -- Compare unsigned integers in the first and second
+        operand and return the minimum values.
+        "#,
+            &formats.binary,
+        )
+        .operands_in(vec![x, y])
+        .operands_out(vec![a]),
+    );
+
+    let i64_t = &TypeVar::new(
+        "i64_t",
+        "A scalar 64bit integer",
+        TypeSetBuilder::new().ints(64..64).build(),
+    );
+
+    let GV = &Operand::new("GV", &entities.global_value);
+    let addr = &Operand::new("addr", i64_t);
+
+    ig.push(
+        Inst::new(
+            "x86_elf_tls_get_addr",
+            r#"
+        Elf tls get addr -- This implements the GD TLS model for ELF. The clobber output should
+        not be used.
+            "#,
+            &formats.unary_global_value,
+        )
+        .operands_in(vec![GV])
+        .operands_out(vec![addr]),
+    );
+    ig.push(
+        Inst::new(
+            "x86_macho_tls_get_addr",
+            r#"
+        Mach-O tls get addr -- This implements TLS access for Mach-O. The clobber output should
+        not be used.
+            "#,
+            &formats.unary_global_value,
+        )
+        .operands_in(vec![GV])
+        .operands_out(vec![addr]),
+    );
+
+    ig.build()
+}

cranelift/codegen/meta/src/isa/x86/legalize.rs

@@ -0,0 +1,650 @@
+use crate::cdsl::ast::{constant, var, ExprBuilder, Literal};
+use crate::cdsl::instructions::{vector, Bindable, InstructionGroup};
+use crate::cdsl::types::{LaneType, ValueType};
+use crate::cdsl::xform::TransformGroupBuilder;
+use crate::shared::types::Float::{F32, F64};
+use crate::shared::types::Int::{I16, I32, I64, I8};
+use crate::shared::Definitions as SharedDefinitions;
+
+#[allow(clippy::many_single_char_names)]
+pub(crate) fn define(shared: &mut SharedDefinitions, x86_instructions: &InstructionGroup) {
+    let mut group = TransformGroupBuilder::new(
+        "x86_expand",
+        r#"
+    Legalize instructions by expansion.
+
+    Use x86-specific instructions if needed."#,
+    )
+    .isa("x86")
+    .chain_with(shared.transform_groups.by_name("expand_flags").id);
+
+    // List of instructions.
+    let insts = &shared.instructions;
+    let band = insts.by_name("band");
+    let band_not = insts.by_name("band_not");
+    let bitcast = insts.by_name("bitcast");
+    let bitselect = insts.by_name("bitselect");
+    let bor = insts.by_name("bor");
+    let bnot = insts.by_name("bnot");
+    let bxor = insts.by_name("bxor");
+    let clz = insts.by_name("clz");
+    let ctz = insts.by_name("ctz");
+    let extractlane = insts.by_name("extractlane");
+    let fcmp = insts.by_name("fcmp");
+    let fcvt_from_uint = insts.by_name("fcvt_from_uint");
+    let fcvt_to_sint = insts.by_name("fcvt_to_sint");
+    let fcvt_to_uint = insts.by_name("fcvt_to_uint");
+    let fcvt_to_sint_sat = insts.by_name("fcvt_to_sint_sat");
+    let fcvt_to_uint_sat = insts.by_name("fcvt_to_uint_sat");
+    let fabs = insts.by_name("fabs");
+    let fmax = insts.by_name("fmax");
+    let fmin = insts.by_name("fmin");
+    let fneg = insts.by_name("fneg");
+    let iadd = insts.by_name("iadd");
+    let icmp = insts.by_name("icmp");
+    let iconst = insts.by_name("iconst");
+    let imax = insts.by_name("imax");
+    let imin = insts.by_name("imin");
+    let imul = insts.by_name("imul");
+    let ineg = insts.by_name("ineg");
+    let insertlane = insts.by_name("insertlane");
+    let ishl = insts.by_name("ishl");
+    let ishl_imm = insts.by_name("ishl_imm");
+    let isub = insts.by_name("isub");
+    let popcnt = insts.by_name("popcnt");
+    let raw_bitcast = insts.by_name("raw_bitcast");
+    let scalar_to_vector = insts.by_name("scalar_to_vector");
+    let sdiv = insts.by_name("sdiv");
+    let selectif = insts.by_name("selectif");
+    let smulhi = insts.by_name("smulhi");
+    let splat = insts.by_name("splat");
+    let shuffle = insts.by_name("shuffle");
+    let srem = insts.by_name("srem");
+    let sshr = insts.by_name("sshr");
+    let tls_value = insts.by_name("tls_value");
+    let trueif = insts.by_name("trueif");
+    let udiv = insts.by_name("udiv");
+    let umax = insts.by_name("umax");
+    let umin = insts.by_name("umin");
+    let umulhi = insts.by_name("umulhi");
+    let ushr_imm = insts.by_name("ushr_imm");
+    let urem = insts.by_name("urem");
+    let ushr = insts.by_name("ushr");
+    let vconst = insts.by_name("vconst");
+    let vall_true = insts.by_name("vall_true");
+    let vany_true = insts.by_name("vany_true");
+
+    let x86_bsf = x86_instructions.by_name("x86_bsf");
+    let x86_bsr = x86_instructions.by_name("x86_bsr");
+    let x86_pmaxs = x86_instructions.by_name("x86_pmaxs");
+    let x86_pmaxu = x86_instructions.by_name("x86_pmaxu");
+    let x86_pmins = x86_instructions.by_name("x86_pmins");
+    let x86_pminu = x86_instructions.by_name("x86_pminu");
+    let x86_pshufb = x86_instructions.by_name("x86_pshufb");
+    let x86_pshufd = x86_instructions.by_name("x86_pshufd");
+    let x86_psll = x86_instructions.by_name("x86_psll");
+    let x86_psra = x86_instructions.by_name("x86_psra");
+    let x86_psrl = x86_instructions.by_name("x86_psrl");
+    let x86_ptest = x86_instructions.by_name("x86_ptest");
+    let x86_umulx = x86_instructions.by_name("x86_umulx");
+    let x86_smulx = x86_instructions.by_name("x86_smulx");
+
+    let imm = &shared.imm;
+
+    // Division and remainder.
+    //
+    // The srem expansion requires custom code because srem INT_MIN, -1 is not
+    // allowed to trap. The other ops need to check avoid_div_traps.
+    group.custom_legalize(sdiv, "expand_sdivrem");
+    group.custom_legalize(srem, "expand_sdivrem");
+    group.custom_legalize(udiv, "expand_udivrem");
+    group.custom_legalize(urem, "expand_udivrem");
+
+    // Double length (widening) multiplication.
+    let a = var("a");
+    let x = var("x");
+    let y = var("y");
+    let a1 = var("a1");
+    let a2 = var("a2");
+    let res_lo = var("res_lo");
+    let res_hi = var("res_hi");
+
+    group.legalize(
+        def!(res_hi = umulhi(x, y)),
+        vec![def!((res_lo, res_hi) = x86_umulx(x, y))],
+    );
+
+    group.legalize(
+        def!(res_hi = smulhi(x, y)),
+        vec![def!((res_lo, res_hi) = x86_smulx(x, y))],
+    );
+
+    // Floating point condition codes.
+    //
+    // The 8 condition codes in `supported_floatccs` are directly supported by a
+    // `ucomiss` or `ucomisd` instruction. The remaining codes need legalization
+    // patterns.
+
+    let floatcc_eq = Literal::enumerator_for(&imm.floatcc, "eq");
+    let floatcc_ord = Literal::enumerator_for(&imm.floatcc, "ord");
+    let floatcc_ueq = Literal::enumerator_for(&imm.floatcc, "ueq");
+    let floatcc_ne = Literal::enumerator_for(&imm.floatcc, "ne");
+    let floatcc_uno = Literal::enumerator_for(&imm.floatcc, "uno");
+    let floatcc_one = Literal::enumerator_for(&imm.floatcc, "one");
+
+    // Equality needs an explicit `ord` test which checks the parity bit.
+    group.legalize(
+        def!(a = fcmp(floatcc_eq, x, y)),
+        vec![
+            def!(a1 = fcmp(floatcc_ord, x, y)),
+            def!(a2 = fcmp(floatcc_ueq, x, y)),
+            def!(a = band(a1, a2)),
+        ],
+    );
+    group.legalize(
+        def!(a = fcmp(floatcc_ne, x, y)),
+        vec![
+            def!(a1 = fcmp(floatcc_uno, x, y)),
+            def!(a2 = fcmp(floatcc_one, x, y)),
+            def!(a = bor(a1, a2)),
+        ],
+    );
+
+    let floatcc_lt = &Literal::enumerator_for(&imm.floatcc, "lt");
+    let floatcc_gt = &Literal::enumerator_for(&imm.floatcc, "gt");
+    let floatcc_le = &Literal::enumerator_for(&imm.floatcc, "le");
+    let floatcc_ge = &Literal::enumerator_for(&imm.floatcc, "ge");
+    let floatcc_ugt = &Literal::enumerator_for(&imm.floatcc, "ugt");
+    let floatcc_ult = &Literal::enumerator_for(&imm.floatcc, "ult");
+    let floatcc_uge = &Literal::enumerator_for(&imm.floatcc, "uge");
+    let floatcc_ule = &Literal::enumerator_for(&imm.floatcc, "ule");
+
+    // Inequalities that need to be reversed.
+    for &(cc, rev_cc) in &[
+        (floatcc_lt, floatcc_gt),
+        (floatcc_le, floatcc_ge),
+        (floatcc_ugt, floatcc_ult),
+        (floatcc_uge, floatcc_ule),
+    ] {
+        group.legalize(def!(a = fcmp(cc, x, y)), vec![def!(a = fcmp(rev_cc, y, x))]);
+    }
+
+    // We need to modify the CFG for min/max legalization.
+    group.custom_legalize(fmin, "expand_minmax");
+    group.custom_legalize(fmax, "expand_minmax");
+
+    // Conversions from unsigned need special handling.
+    group.custom_legalize(fcvt_from_uint, "expand_fcvt_from_uint");
+    // Conversions from float to int can trap and modify the control flow graph.
+    group.custom_legalize(fcvt_to_sint, "expand_fcvt_to_sint");
+    group.custom_legalize(fcvt_to_uint, "expand_fcvt_to_uint");
+    group.custom_legalize(fcvt_to_sint_sat, "expand_fcvt_to_sint_sat");
+    group.custom_legalize(fcvt_to_uint_sat, "expand_fcvt_to_uint_sat");
+
+    // Count leading and trailing zeroes, for baseline x86_64
+    let c_minus_one = var("c_minus_one");
+    let c_thirty_one = var("c_thirty_one");
+    let c_thirty_two = var("c_thirty_two");
+    let c_sixty_three = var("c_sixty_three");
+    let c_sixty_four = var("c_sixty_four");
+    let index1 = var("index1");
+    let r2flags = var("r2flags");
+    let index2 = var("index2");
+
+    let intcc_eq = Literal::enumerator_for(&imm.intcc, "eq");
+    let imm64_minus_one = Literal::constant(&imm.imm64, -1);
+    let imm64_63 = Literal::constant(&imm.imm64, 63);
+    group.legalize(
+        def!(a = clz.I64(x)),
+        vec![
+            def!(c_minus_one = iconst(imm64_minus_one)),
+            def!(c_sixty_three = iconst(imm64_63)),
+            def!((index1, r2flags) = x86_bsr(x)),
+            def!(index2 = selectif(intcc_eq, r2flags, c_minus_one, index1)),
+            def!(a = isub(c_sixty_three, index2)),
+        ],
+    );
+
+    let imm64_31 = Literal::constant(&imm.imm64, 31);
+    group.legalize(
+        def!(a = clz.I32(x)),
+        vec![
+            def!(c_minus_one = iconst(imm64_minus_one)),
+            def!(c_thirty_one = iconst(imm64_31)),
+            def!((index1, r2flags) = x86_bsr(x)),
+            def!(index2 = selectif(intcc_eq, r2flags, c_minus_one, index1)),
+            def!(a = isub(c_thirty_one, index2)),
+        ],
+    );
+
+    let imm64_64 = Literal::constant(&imm.imm64, 64);
+    group.legalize(
+        def!(a = ctz.I64(x)),
+        vec![
+            def!(c_sixty_four = iconst(imm64_64)),
+            def!((index1, r2flags) = x86_bsf(x)),
+            def!(a = selectif(intcc_eq, r2flags, c_sixty_four, index1)),
+        ],
+    );
+
+    let imm64_32 = Literal::constant(&imm.imm64, 32);
+    group.legalize(
+        def!(a = ctz.I32(x)),
+        vec![
+            def!(c_thirty_two = iconst(imm64_32)),
+            def!((index1, r2flags) = x86_bsf(x)),
+            def!(a = selectif(intcc_eq, r2flags, c_thirty_two, index1)),
+        ],
+    );
+
+    // Population count for baseline x86_64
+    let x = var("x");
+    let r = var("r");
+
+    let qv3 = var("qv3");
+    let qv4 = var("qv4");
+    let qv5 = var("qv5");
+    let qv6 = var("qv6");
+    let qv7 = var("qv7");
+    let qv8 = var("qv8");
+    let qv9 = var("qv9");
+    let qv10 = var("qv10");
+    let qv11 = var("qv11");
+    let qv12 = var("qv12");
+    let qv13 = var("qv13");
+    let qv14 = var("qv14");
+    let qv15 = var("qv15");
+    let qc77 = var("qc77");
+    #[allow(non_snake_case)]
+    let qc0F = var("qc0F");
+    let qc01 = var("qc01");
+
+    let imm64_1 = Literal::constant(&imm.imm64, 1);
+    let imm64_4 = Literal::constant(&imm.imm64, 4);
+    group.legalize(
+        def!(r = popcnt.I64(x)),
+        vec![
+            def!(qv3 = ushr_imm(x, imm64_1)),
+            def!(qc77 = iconst(Literal::constant(&imm.imm64, 0x7777_7777_7777_7777))),
+            def!(qv4 = band(qv3, qc77)),
+            def!(qv5 = isub(x, qv4)),
+            def!(qv6 = ushr_imm(qv4, imm64_1)),
+            def!(qv7 = band(qv6, qc77)),
+            def!(qv8 = isub(qv5, qv7)),
+            def!(qv9 = ushr_imm(qv7, imm64_1)),
+            def!(qv10 = band(qv9, qc77)),
+            def!(qv11 = isub(qv8, qv10)),
+            def!(qv12 = ushr_imm(qv11, imm64_4)),
+            def!(qv13 = iadd(qv11, qv12)),
+            def!(qc0F = iconst(Literal::constant(&imm.imm64, 0x0F0F_0F0F_0F0F_0F0F))),
+            def!(qv14 = band(qv13, qc0F)),
+            def!(qc01 = iconst(Literal::constant(&imm.imm64, 0x0101_0101_0101_0101))),
+            def!(qv15 = imul(qv14, qc01)),
+            def!(r = ushr_imm(qv15, Literal::constant(&imm.imm64, 56))),
+        ],
+    );
+
+    let lv3 = var("lv3");
+    let lv4 = var("lv4");
+    let lv5 = var("lv5");
+    let lv6 = var("lv6");
+    let lv7 = var("lv7");
+    let lv8 = var("lv8");
+    let lv9 = var("lv9");
+    let lv10 = var("lv10");
+    let lv11 = var("lv11");
+    let lv12 = var("lv12");
+    let lv13 = var("lv13");
+    let lv14 = var("lv14");
+    let lv15 = var("lv15");
+    let lc77 = var("lc77");
+    #[allow(non_snake_case)]
+    let lc0F = var("lc0F");
+    let lc01 = var("lc01");
+
+    group.legalize(
+        def!(r = popcnt.I32(x)),
+        vec![
+            def!(lv3 = ushr_imm(x, imm64_1)),
+            def!(lc77 = iconst(Literal::constant(&imm.imm64, 0x7777_7777))),
+            def!(lv4 = band(lv3, lc77)),
+            def!(lv5 = isub(x, lv4)),
+            def!(lv6 = ushr_imm(lv4, imm64_1)),
+            def!(lv7 = band(lv6, lc77)),
+            def!(lv8 = isub(lv5, lv7)),
+            def!(lv9 = ushr_imm(lv7, imm64_1)),
+            def!(lv10 = band(lv9, lc77)),
+            def!(lv11 = isub(lv8, lv10)),
+            def!(lv12 = ushr_imm(lv11, imm64_4)),
+            def!(lv13 = iadd(lv11, lv12)),
+            def!(lc0F = iconst(Literal::constant(&imm.imm64, 0x0F0F_0F0F))),
+            def!(lv14 = band(lv13, lc0F)),
+            def!(lc01 = iconst(Literal::constant(&imm.imm64, 0x0101_0101))),
+            def!(lv15 = imul(lv14, lc01)),
+            def!(r = ushr_imm(lv15, Literal::constant(&imm.imm64, 24))),
+        ],
+    );
+
+    group.custom_legalize(ineg, "convert_ineg");
+
+    group.custom_legalize(tls_value, "expand_tls_value");
+
+    group.build_and_add_to(&mut shared.transform_groups);
+
+    let mut narrow = TransformGroupBuilder::new(
+        "x86_narrow",
+        r#"
+    Legalize instructions by narrowing.
+
+    Use x86-specific instructions if needed."#,
+    )
+    .isa("x86")
+    .chain_with(shared.transform_groups.by_name("narrow_flags").id);
+
+    // SIMD
+    let uimm8_zero = Literal::constant(&imm.uimm8, 0x00);
+    let uimm8_one = Literal::constant(&imm.uimm8, 0x01);
+    let u128_zeroes = constant(vec![0x00; 16]);
+    let u128_ones = constant(vec![0xff; 16]);
+    let b = var("b");
+    let c = var("c");
+    let d = var("d");
+    let e = var("e");
+
+    // SIMD vector size: eventually multiple vector sizes may be supported but for now only SSE-sized vectors are available
+    let sse_vector_size: u64 = 128;
+    let allowed_simd_type = |t: &LaneType| t.lane_bits() >= 8 && t.lane_bits() < 128;
+
+    // SIMD splat: 8-bits
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 8) {
+        let splat_any8x16 = splat.bind(vector(ty, sse_vector_size));
+        narrow.legalize(
+            def!(y = splat_any8x16(x)),
+            vec![
+                def!(a = scalar_to_vector(x)), // move into the lowest 8 bits of an XMM register
+                def!(b = vconst(u128_zeroes)), // zero out a different XMM register; the shuffle mask
+                // for moving the lowest byte to all other byte lanes is 0x0
+                def!(y = x86_pshufb(a, b)), // PSHUFB takes two XMM operands, one of which is a
+                                            // shuffle mask (i.e. b)
+            ],
+        );
+    }
+
+    // SIMD splat: 16-bits
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 16) {
+        let splat_x16x8 = splat.bind(vector(ty, sse_vector_size));
+        let raw_bitcast_any16x8_to_i32x4 = raw_bitcast
+            .bind(vector(I32, sse_vector_size))
+            .bind(vector(ty, sse_vector_size));
+        let raw_bitcast_i32x4_to_any16x8 = raw_bitcast
+            .bind(vector(ty, sse_vector_size))
+            .bind(vector(I32, sse_vector_size));
+        narrow.legalize(
+            def!(y = splat_x16x8(x)),
+            vec![
+                def!(a = scalar_to_vector(x)), // move into the lowest 16 bits of an XMM register
+                def!(b = insertlane(a, uimm8_one, x)), // insert the value again but in the next lowest 16 bits
+                def!(c = raw_bitcast_any16x8_to_i32x4(b)), // no instruction emitted; pretend this is an I32x4 so we can use PSHUFD
+                def!(d = x86_pshufd(c, uimm8_zero)), // broadcast the bytes in the XMM register with PSHUFD
+                def!(y = raw_bitcast_i32x4_to_any16x8(d)), // no instruction emitted; pretend this is an X16x8 again
+            ],
+        );
+    }
+
+    // SIMD splat: 32-bits
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 32) {
+        let splat_any32x4 = splat.bind(vector(ty, sse_vector_size));
+        narrow.legalize(
+            def!(y = splat_any32x4(x)),
+            vec![
+                def!(a = scalar_to_vector(x)), // translate to an x86 MOV to get the value in an XMM register
+                def!(y = x86_pshufd(a, uimm8_zero)), // broadcast the bytes in the XMM register with PSHUF
+            ],
+        );
+    }
+
+    // SIMD splat: 64-bits
+    for ty in ValueType::all_lane_types().filter(|t| t.lane_bits() == 64) {
+        let splat_any64x2 = splat.bind(vector(ty, sse_vector_size));
+        narrow.legalize(
+            def!(y = splat_any64x2(x)),
+            vec![
+                def!(a = scalar_to_vector(x)), // move into the lowest 64 bits of an XMM register
+                def!(y = insertlane(a, uimm8_one, x)), // move into the highest 64 bits of the same XMM register
+            ],
+        );
+    }
+
+    // SIMD bnot
+    for ty in ValueType::all_lane_types().filter(allowed_simd_type) {
+        let bnot = bnot.bind(vector(ty, sse_vector_size));
+        narrow.legalize(
+            def!(y = bnot(x)),
+            vec![def!(a = vconst(u128_ones)), def!(y = bxor(a, x))],
+        );
+    }
+
+    // SIMD shift left (logical)
+    for ty in &[I16, I32, I64] {
+        let ishl = ishl.bind(vector(*ty, sse_vector_size));
+        let bitcast = bitcast.bind(vector(I64, sse_vector_size));
+        narrow.legalize(
+            def!(a = ishl(x, y)),
+            vec![def!(b = bitcast(y)), def!(a = x86_psll(x, b))],
+        );
+    }
+
+    // SIMD shift right (logical)
+    for ty in &[I16, I32, I64] {
+        let ushr = ushr.bind(vector(*ty, sse_vector_size));
+        let bitcast = bitcast.bind(vector(I64, sse_vector_size));
+        narrow.legalize(
+            def!(a = ushr(x, y)),
+            vec![def!(b = bitcast(y)), def!(a = x86_psrl(x, b))],
+        );
+    }
+
+    // SIMD shift left (arithmetic)
+    for ty in &[I16, I32, I64] {
+        let sshr = sshr.bind(vector(*ty, sse_vector_size));
+        let bitcast = bitcast.bind(vector(I64, sse_vector_size));
+        narrow.legalize(
+            def!(a = sshr(x, y)),
+            vec![def!(b = bitcast(y)), def!(a = x86_psra(x, b))],
+        );
+    }
+
+    // SIMD select
+    for ty in ValueType::all_lane_types().filter(allowed_simd_type) {
+        let bitselect = bitselect.bind(vector(ty, sse_vector_size)); // must bind both x/y and c
+        narrow.legalize(
+            def!(d = bitselect(c, x, y)),
+            vec![
+                def!(a = band(x, c)),
+                def!(b = band_not(y, c)),
+                def!(d = bor(a, b)),
+            ],
+        );
+    }
+
+    // SIMD vany_true
+    let ne = Literal::enumerator_for(&imm.intcc, "ne");
+    for ty in ValueType::all_lane_types().filter(allowed_simd_type) {
+        let vany_true = vany_true.bind(vector(ty, sse_vector_size));
+        narrow.legalize(
+            def!(y = vany_true(x)),
+            vec![def!(a = x86_ptest(x, x)), def!(y = trueif(ne, a))],
+        );
+    }
+
+    // SIMD vall_true
+    let eq = Literal::enumerator_for(&imm.intcc, "eq");
+    for ty in ValueType::all_lane_types().filter(allowed_simd_type) {
+        let vall_true = vall_true.bind(vector(ty, sse_vector_size));
+        if ty.is_int() {
+            // In the common case (Wasm's integer-only all_true), we do not require a bitcast.
+            narrow.legalize(
+                def!(y = vall_true(x)),
+                vec![
+                    def!(a = vconst(u128_zeroes)),
+                    def!(c = icmp(eq, x, a)),
+                    def!(d = x86_ptest(c, c)),
+                    def!(y = trueif(eq, d)),
+                ],
+            );
+        } else {
+            // However, to support other types we must bitcast them to an integer vector to use
+            // icmp.
+            let lane_type_as_int = LaneType::int_from_bits(ty.lane_bits() as u16);
+            let raw_bitcast_to_int = raw_bitcast.bind(vector(lane_type_as_int, sse_vector_size));
+            narrow.legalize(
+                def!(y = vall_true(x)),
+                vec![
+                    def!(a = vconst(u128_zeroes)),
+                    def!(b = raw_bitcast_to_int(x)),
+                    def!(c = icmp(eq, b, a)),
+                    def!(d = x86_ptest(c, c)),
+                    def!(y = trueif(eq, d)),
+                ],
+            );
+        }
+    }
+
+    // SIMD icmp ne
+    let ne = Literal::enumerator_for(&imm.intcc, "ne");
+    for ty in ValueType::all_lane_types().filter(|ty| allowed_simd_type(ty) && ty.is_int()) {
+        let icmp_ = icmp.bind(vector(ty, sse_vector_size));
+        narrow.legalize(
+            def!(c = icmp_(ne, a, b)),
+            vec![def!(x = icmp(eq, a, b)), def!(c = bnot(x))],
+        );
+    }
+
+    // SIMD icmp greater-/less-than
+    let sgt = Literal::enumerator_for(&imm.intcc, "sgt");
+    let ugt = Literal::enumerator_for(&imm.intcc, "ugt");
+    let sge = Literal::enumerator_for(&imm.intcc, "sge");
+    let uge = Literal::enumerator_for(&imm.intcc, "uge");
+    let slt = Literal::enumerator_for(&imm.intcc, "slt");
+    let ult = Literal::enumerator_for(&imm.intcc, "ult");
+    let sle = Literal::enumerator_for(&imm.intcc, "sle");
+    let ule = Literal::enumerator_for(&imm.intcc, "ule");
+    for ty in &[I8, I16, I32] {
+        // greater-than
+        let icmp_ = icmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(
+            def!(c = icmp_(ugt, a, b)),
+            vec![
+                def!(x = x86_pmaxu(a, b)),
+                def!(y = icmp(eq, x, b)),
+                def!(c = bnot(y)),
+            ],
+        );
+        let icmp_ = icmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(
+            def!(c = icmp_(sge, a, b)),
+            vec![def!(x = x86_pmins(a, b)), def!(c = icmp(eq, x, b))],
+        );
+        let icmp_ = icmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(
+            def!(c = icmp_(uge, a, b)),
+            vec![def!(x = x86_pminu(a, b)), def!(c = icmp(eq, x, b))],
+        );
+
+        // less-than
+        let icmp_ = icmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = icmp_(slt, a, b)), vec![def!(c = icmp(sgt, b, a))]);
+        let icmp_ = icmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = icmp_(ult, a, b)), vec![def!(c = icmp(ugt, b, a))]);
+        let icmp_ = icmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = icmp_(sle, a, b)), vec![def!(c = icmp(sge, b, a))]);
+        let icmp_ = icmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = icmp_(ule, a, b)), vec![def!(c = icmp(uge, b, a))]);
+    }
+
+    // SIMD integer min/max
+    for ty in &[I8, I16, I32] {
+        let imin = imin.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = imin(a, b)), vec![def!(c = x86_pmins(a, b))]);
+        let umin = umin.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = umin(a, b)), vec![def!(c = x86_pminu(a, b))]);
+        let imax = imax.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = imax(a, b)), vec![def!(c = x86_pmaxs(a, b))]);
+        let umax = umax.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = umax(a, b)), vec![def!(c = x86_pmaxu(a, b))]);
+    }
+
+    // SIMD fcmp greater-/less-than
+    let gt = Literal::enumerator_for(&imm.floatcc, "gt");
+    let lt = Literal::enumerator_for(&imm.floatcc, "lt");
+    let ge = Literal::enumerator_for(&imm.floatcc, "ge");
+    let le = Literal::enumerator_for(&imm.floatcc, "le");
+    let ugt = Literal::enumerator_for(&imm.floatcc, "ugt");
+    let ult = Literal::enumerator_for(&imm.floatcc, "ult");
+    let uge = Literal::enumerator_for(&imm.floatcc, "uge");
+    let ule = Literal::enumerator_for(&imm.floatcc, "ule");
+    for ty in &[F32, F64] {
+        let fcmp_ = fcmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = fcmp_(gt, a, b)), vec![def!(c = fcmp(lt, b, a))]);
+        let fcmp_ = fcmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = fcmp_(ge, a, b)), vec![def!(c = fcmp(le, b, a))]);
+        let fcmp_ = fcmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = fcmp_(ult, a, b)), vec![def!(c = fcmp(ugt, b, a))]);
+        let fcmp_ = fcmp.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(def!(c = fcmp_(ule, a, b)), vec![def!(c = fcmp(uge, b, a))]);
+    }
+
+    for ty in &[F32, F64] {
+        let fneg = fneg.bind(vector(*ty, sse_vector_size));
+        let lane_type_as_int = LaneType::int_from_bits(LaneType::from(*ty).lane_bits() as u16);
+        let uimm8_shift = Literal::constant(&imm.uimm8, lane_type_as_int.lane_bits() as i64 - 1);
+        let vconst = vconst.bind(vector(lane_type_as_int, sse_vector_size));
+        let bitcast_to_float = raw_bitcast.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(
+            def!(b = fneg(a)),
+            vec![
+                def!(c = vconst(u128_ones)),
+                def!(d = ishl_imm(c, uimm8_shift)), // Create a mask of all 0s except the MSB.
+                def!(e = bitcast_to_float(d)),      // Cast mask to the floating-point type.
+                def!(b = bxor(a, e)),               // Flip the MSB.
+            ],
+        );
+    }
+
+    // SIMD fabs
+    for ty in &[F32, F64] {
+        let fabs = fabs.bind(vector(*ty, sse_vector_size));
+        let lane_type_as_int = LaneType::int_from_bits(LaneType::from(*ty).lane_bits() as u16);
+        let vconst = vconst.bind(vector(lane_type_as_int, sse_vector_size));
+        let bitcast_to_float = raw_bitcast.bind(vector(*ty, sse_vector_size));
+        narrow.legalize(
+            def!(b = fabs(a)),
+            vec![
+                def!(c = vconst(u128_ones)),
+                def!(d = ushr_imm(c, uimm8_one)), // Create a mask of all 1s except the MSB.
+                def!(e = bitcast_to_float(d)),    // Cast mask to the floating-point type.
+                def!(b = band(a, e)),             // Unset the MSB.
+            ],
+        );
+    }
+
+    narrow.custom_legalize(shuffle, "convert_shuffle");
+    narrow.custom_legalize(extractlane, "convert_extractlane");
+    narrow.custom_legalize(insertlane, "convert_insertlane");
+    narrow.custom_legalize(ineg, "convert_ineg");
+
+    narrow.build_and_add_to(&mut shared.transform_groups);
+
+    let mut widen = TransformGroupBuilder::new(
+        "x86_widen",
+        r#"
+    Legalize instructions by widening.
+
+    Use x86-specific instructions if needed."#,
+    )
+    .isa("x86")
+    .chain_with(shared.transform_groups.by_name("widen").id);
+
+    widen.custom_legalize(ineg, "convert_ineg");
+    widen.build_and_add_to(&mut shared.transform_groups);
+}

cranelift/codegen/meta/src/isa/x86/mod.rs

@@ -0,0 +1,81 @@
+use crate::cdsl::cpu_modes::CpuMode;
+use crate::cdsl::isa::TargetIsa;
+use crate::cdsl::types::ReferenceType;
+
+use crate::shared::types::Bool::B1;
+use crate::shared::types::Float::{F32, F64};
+use crate::shared::types::Int::{I16, I32, I64, I8};
+use crate::shared::types::Reference::{R32, R64};
+use crate::shared::Definitions as SharedDefinitions;
+
+mod encodings;
+mod instructions;
+mod legalize;
+mod opcodes;
+mod recipes;
+mod registers;
+mod settings;
+
+pub(crate) fn define(shared_defs: &mut SharedDefinitions) -> TargetIsa {
+    let settings = settings::define(&shared_defs.settings);
+    let regs = registers::define();
+
+    let inst_group = instructions::define(
+        &mut shared_defs.all_instructions,
+        &shared_defs.formats,
+        &shared_defs.imm,
+        &shared_defs.entities,
+    );
+    legalize::define(shared_defs, &inst_group);
+
+    // CPU modes for 32-bit and 64-bit operations.
+    let mut x86_64 = CpuMode::new("I64");
+    let mut x86_32 = CpuMode::new("I32");
+
+    let expand_flags = shared_defs.transform_groups.by_name("expand_flags");
+    let x86_widen = shared_defs.transform_groups.by_name("x86_widen");
+    let x86_narrow = shared_defs.transform_groups.by_name("x86_narrow");
+    let x86_expand = shared_defs.transform_groups.by_name("x86_expand");
+
+    x86_32.legalize_monomorphic(expand_flags);
+    x86_32.legalize_default(x86_narrow);
+    x86_32.legalize_type(B1, expand_flags);
+    x86_32.legalize_type(I8, x86_widen);
+    x86_32.legalize_type(I16, x86_widen);
+    x86_32.legalize_type(I32, x86_expand);
+    x86_32.legalize_value_type(ReferenceType(R32), x86_expand);
+    x86_32.legalize_type(F32, x86_expand);
+    x86_32.legalize_type(F64, x86_expand);
+
+    x86_64.legalize_monomorphic(expand_flags);
+    x86_64.legalize_default(x86_narrow);
+    x86_64.legalize_type(B1, expand_flags);
+    x86_64.legalize_type(I8, x86_widen);
+    x86_64.legalize_type(I16, x86_widen);
+    x86_64.legalize_type(I32, x86_expand);
+    x86_64.legalize_type(I64, x86_expand);
+    x86_64.legalize_value_type(ReferenceType(R64), x86_expand);
+    x86_64.legalize_type(F32, x86_expand);
+    x86_64.legalize_type(F64, x86_expand);
+
+    let recipes = recipes::define(shared_defs, &settings, &regs);
+
+    let encodings = encodings::define(shared_defs, &settings, &inst_group, &recipes);
+    x86_32.set_encodings(encodings.enc32);
+    x86_64.set_encodings(encodings.enc64);
+    let encodings_predicates = encodings.inst_pred_reg.extract();
+
+    let recipes = encodings.recipes;
+
+    let cpu_modes = vec![x86_64, x86_32];
+
+    TargetIsa::new(
+        "x86",
+        inst_group,
+        settings,
+        regs,
+        recipes,
+        cpu_modes,
+        encodings_predicates,
+    )
+}

cranelift/codegen/meta/src/isa/x86/opcodes.rs

@@ -0,0 +1,604 @@
+//! Static, named definitions of instruction opcodes.
+
+/// Empty opcode for use as a default.
+pub static EMPTY: [u8; 0] = [];
+
+/// Add with carry flag r{16,32,64} to r/m of the same size.
+pub static ADC: [u8; 1] = [0x11];
+
+/// Add r{16,32,64} to r/m of the same size.
+pub static ADD: [u8; 1] = [0x01];
+
+/// Add imm{16,32} to r/m{16,32,64}, possibly sign-extended.
+pub static ADD_IMM: [u8; 1] = [0x81];
+
+/// Add sign-extended imm8 to r/m{16,32,64}.
+pub static ADD_IMM8_SIGN_EXTEND: [u8; 1] = [0x83];
+
+/// Add packed double-precision floating-point values from xmm2/mem to xmm1 and store result in  
+/// xmm1 (SSE2).
+pub static ADDPD: [u8; 3] = [0x66, 0x0f, 0x58];
+
+/// Add packed single-precision floating-point values from xmm2/mem to xmm1 and store result in  
+/// xmm1 (SSE).
+pub static ADDPS: [u8; 2] = [0x0f, 0x58];
+
+/// Add the low double-precision floating-point value from xmm2/mem to xmm1
+/// and store the result in xmm1.
+pub static ADDSD: [u8; 3] = [0xf2, 0x0f, 0x58];
+
+/// Add the low single-precision floating-point value from xmm2/mem to xmm1
+/// and store the result in xmm1.
+pub static ADDSS: [u8; 3] = [0xf3, 0x0f, 0x58];
+
+/// r/m{16,32,64} AND register of the same size (Intel docs have a typo).
+pub static AND: [u8; 1] = [0x21];
+
+/// imm{16,32} AND r/m{16,32,64}, possibly sign-extended.
+pub static AND_IMM: [u8; 1] = [0x81];
+
+/// r/m{16,32,64} AND sign-extended imm8.
+pub static AND_IMM8_SIGN_EXTEND: [u8; 1] = [0x83];
+
+/// Return the bitwise logical AND NOT of packed single-precision floating-point
+/// values in xmm1 and xmm2/mem.
+pub static ANDNPS: [u8; 2] = [0x0f, 0x55];
+
+/// Return the bitwise logical AND of packed single-precision floating-point values
+/// in xmm1 and xmm2/mem.
+pub static ANDPS: [u8; 2] = [0x0f, 0x54];
+
+/// Bit scan forward (stores index of first encountered 1 from the front).
+pub static BIT_SCAN_FORWARD: [u8; 2] = [0x0f, 0xbc];
+
+/// Bit scan reverse (stores index of first encountered 1 from the back).
+pub static BIT_SCAN_REVERSE: [u8; 2] = [0x0f, 0xbd];
+
+/// Call near, relative, displacement relative to next instruction (sign-extended).
+pub static CALL_RELATIVE: [u8; 1] = [0xe8];
+
+/// Move r/m{16,32,64} if overflow (OF=1).
+pub static CMOV_OVERFLOW: [u8; 2] = [0x0f, 0x40];
+
+/// Compare imm{16,32} with r/m{16,32,64} (sign-extended if 64).
+pub static CMP_IMM: [u8; 1] = [0x81];
+
+/// Compare imm8 with r/m{16,32,64}.
+pub static CMP_IMM8: [u8; 1] = [0x83];
+
+/// Compare r{16,32,64} with r/m of the same size.
+pub static CMP_REG: [u8; 1] = [0x39];
+
+/// Compare packed double-precision floating-point value in xmm2/m32 and xmm1 using bits 2:0 of
+/// imm8 as comparison predicate (SSE2).
+pub static CMPPD: [u8; 3] = [0x66, 0x0f, 0xc2];
+
+/// Compare packed single-precision floating-point value in xmm2/m32 and xmm1 using bits 2:0 of
+/// imm8 as comparison predicate (SSE).
+pub static CMPPS: [u8; 2] = [0x0f, 0xc2];
+
+/// Convert scalar double-precision floating-point value to scalar single-precision
+/// floating-point value.
+pub static CVTSD2SS: [u8; 3] = [0xf2, 0x0f, 0x5a];
+
+/// Convert doubleword integer to scalar double-precision floating-point value.
+pub static CVTSI2SD: [u8; 3] = [0xf2, 0x0f, 0x2a];
+
+/// Convert doubleword integer to scalar single-precision floating-point value.
+pub static CVTSI2SS: [u8; 3] = [0xf3, 0x0f, 0x2a];
+
+/// Convert scalar single-precision floating-point value to scalar double-precision
+/// float-point value.
+pub static CVTSS2SD: [u8; 3] = [0xf3, 0x0f, 0x5a];
+
+/// Convert with truncation scalar double-precision floating-point value to signed
+/// integer.
+pub static CVTTSD2SI: [u8; 3] = [0xf2, 0x0f, 0x2c];
+
+/// Convert with truncation scalar single-precision floating-point value to integer.
+pub static CVTTSS2SI: [u8; 3] = [0xf3, 0x0f, 0x2c];
+
+/// Unsigned divide for {16,32,64}-bit.
+pub static DIV: [u8; 1] = [0xf7];
+
+/// Divide packed double-precision floating-point values in xmm1 by packed double-precision
+/// floating-point values in xmm2/mem (SSE2).
+pub static DIVPD: [u8; 3] = [0x66, 0x0f, 0x5e];
+
+/// Divide packed single-precision floating-point values in xmm1 by packed single-precision
+/// floating-point values in xmm2/mem (SSE).
+pub static DIVPS: [u8; 2] = [0x0f, 0x5e];
+
+/// Divide low double-precision floating-point value in xmm1 by low double-precision
+/// floating-point value in xmm2/m64.
+pub static DIVSD: [u8; 3] = [0xf2, 0x0f, 0x5e];
+
+/// Divide low single-precision floating-point value in xmm1 by low single-precision
+/// floating-point value in xmm2/m32.
+pub static DIVSS: [u8; 3] = [0xf3, 0x0f, 0x5e];
+
+/// Signed divide for {16,32,64}-bit.
+pub static IDIV: [u8; 1] = [0xf7];
+
+/// Signed multiply for {16,32,64}-bit, generic registers.
+pub static IMUL: [u8; 2] = [0x0f, 0xaf];
+
+/// Signed multiply for {16,32,64}-bit, storing into RDX:RAX.
+pub static IMUL_RDX_RAX: [u8; 1] = [0xf7];
+
+/// Insert scalar single-precision floating-point value.
+pub static INSERTPS: [u8; 4] = [0x66, 0x0f, 0x3a, 0x21];
+
+/// Either:
+///  1. Jump near, absolute indirect, RIP = 64-bit offset from register or memory.
+///  2. Jump far, absolute indirect, address given in m16:64.
+pub static JUMP_ABSOLUTE: [u8; 1] = [0xff];
+
+/// Jump near, relative, RIP = RIP + 32-bit displacement sign extended to 64 bits.
+pub static JUMP_NEAR_RELATIVE: [u8; 1] = [0xe9];
+
+/// Jump near (rel32) if overflow (OF=1).
+pub static JUMP_NEAR_IF_OVERFLOW: [u8; 2] = [0x0f, 0x80];
+
+/// Jump short, relative, RIP = RIP + 8-bit displacement sign extended to 64 bits.
+pub static JUMP_SHORT: [u8; 1] = [0xeb];
+
+/// Jump short (rel8) if equal (ZF=1).
+pub static JUMP_SHORT_IF_EQUAL: [u8; 1] = [0x74];
+
+/// Jump short (rel8) if not equal (ZF=0).
+pub static JUMP_SHORT_IF_NOT_EQUAL: [u8; 1] = [0x75];
+
+/// Jump short (rel8) if overflow (OF=1).
+pub static JUMP_SHORT_IF_OVERFLOW: [u8; 1] = [0x70];
+
+/// Store effective address for m in register r{16,32,64}.
+pub static LEA: [u8; 1] = [0x8d];
+
+/// Count the number of leading zero bits.
+pub static LZCNT: [u8; 3] = [0xf3, 0x0f, 0xbd];
+
+/// Return the maximum packed double-precision floating-point values between xmm1 and xmm2/m128
+/// (SSE2).
+pub static MAXPD: [u8; 3] = [0x66, 0x0f, 0x5f];
+
+/// Return the maximum packed single-precision floating-point values between  xmm1 and xmm2/m128
+/// (SSE).
+pub static MAXPS: [u8; 2] = [0x0f, 0x5f];
+
+/// Return the maximum scalar double-precision floating-point value between
+/// xmm2/m64 and xmm1.
+pub static MAXSD: [u8; 3] = [0xf2, 0x0f, 0x5f];
+
+/// Return the maximum scalar single-precision floating-point value between
+/// xmm2/m32 and xmm1.
+pub static MAXSS: [u8; 3] = [0xf3, 0x0f, 0x5f];
+
+/// Return the minimum packed double-precision floating-point values between xmm1 and xmm2/m128
+/// (SSE2).
+pub static MINPD: [u8; 3] = [0x66, 0x0f, 0x5d];
+
+/// Return the minimum packed single-precision floating-point values between xmm1 and xmm2/m128
+/// (SSE).
+pub static MINPS: [u8; 2] = [0x0f, 0x5d];
+
+/// Return the minimum scalar double-precision floating-point value between
+/// xmm2/m64 and xmm1.
+pub static MINSD: [u8; 3] = [0xf2, 0x0f, 0x5d];
+
+/// Return the minimum scalar single-precision floating-point value between
+/// xmm2/m32 and xmm1.
+pub static MINSS: [u8; 3] = [0xf3, 0x0f, 0x5d];
+
+/// Move r8 to r/m8.
+pub static MOV_BYTE_STORE: [u8; 1] = [0x88];
+
+/// Move imm{16,32,64} to same-sized register.
+pub static MOV_IMM: [u8; 1] = [0xb8];
+
+/// Move imm{16,32} to r{16,32,64}, sign-extended if 64-bit target.
+pub static MOV_IMM_SIGNEXTEND: [u8; 1] = [0xc7];
+
+/// Move {r/m16, r/m32, r/m64} to same-sized register.
+pub static MOV_LOAD: [u8; 1] = [0x8b];
+
+/// Move r16 to r/m16.
+pub static MOV_STORE_16: [u8; 2] = [0x66, 0x89];
+
+/// Move {r16, r32, r64} to same-sized register or memory.
+pub static MOV_STORE: [u8; 1] = [0x89];
+
+/// Move aligned packed single-precision floating-point values from x/m to xmm (SSE).
+pub static MOVAPS_LOAD: [u8; 2] = [0x0f, 0x28];
+
+/// Move doubleword from r/m32 to xmm (SSE2). Quadword with REX prefix.
+pub static MOVD_LOAD_XMM: [u8; 3] = [0x66, 0x0f, 0x6e];
+
+/// Move doubleword from xmm to r/m32 (SSE2). Quadword with REX prefix.
+pub static MOVD_STORE_XMM: [u8; 3] = [0x66, 0x0f, 0x7e];
+
+/// Move packed single-precision floating-point values low to high (SSE).
+pub static MOVLHPS: [u8; 2] = [0x0f, 0x16];
+
+/// Move scalar double-precision floating-point value (from reg/mem to reg).
+pub static MOVSD_LOAD: [u8; 3] = [0xf2, 0x0f, 0x10];
+
+/// Move scalar double-precision floating-point value (from reg to reg/mem).
+pub static MOVSD_STORE: [u8; 3] = [0xf2, 0x0f, 0x11];
+
+/// Move scalar single-precision floating-point value (from reg to reg/mem).
+pub static MOVSS_STORE: [u8; 3] = [0xf3, 0x0f, 0x11];
+
+/// Move scalar single-precision floating-point-value (from reg/mem to reg).
+pub static MOVSS_LOAD: [u8; 3] = [0xf3, 0x0f, 0x10];
+
+/// Move byte to register with sign-extension.
+pub static MOVSX_BYTE: [u8; 2] = [0x0f, 0xbe];
+
+/// Move word to register with sign-extension.
+pub static MOVSX_WORD: [u8; 2] = [0x0f, 0xbf];
+
+/// Move doubleword to register with sign-extension.
+pub static MOVSXD: [u8; 1] = [0x63];
+
+/// Move unaligned packed single-precision floating-point from x/m to xmm (SSE).
+pub static MOVUPS_LOAD: [u8; 2] = [0x0f, 0x10];
+
+/// Move unaligned packed single-precision floating-point value from xmm to x/m (SSE).
+pub static MOVUPS_STORE: [u8; 2] = [0x0f, 0x11];
+
+/// Move byte to register with zero-extension.
+pub static MOVZX_BYTE: [u8; 2] = [0x0f, 0xb6];
+
+/// Move word to register with zero-extension.
+pub static MOVZX_WORD: [u8; 2] = [0x0f, 0xb7];
+
+/// Unsigned multiply for {16,32,64}-bit.
+pub static MUL: [u8; 1] = [0xf7];
+
+/// Multiply packed double-precision floating-point values from xmm2/mem to xmm1 and store result
+/// in xmm1 (SSE2).
+pub static MULPD: [u8; 3] = [0x66, 0x0f, 0x59];
+
+/// Multiply packed single-precision floating-point values from xmm2/mem to xmm1 and store result
+/// in xmm1 (SSE).
+pub static MULPS: [u8; 2] = [0x0f, 0x59];
+
+/// Multiply the low double-precision floating-point value in xmm2/m64 by the
+/// low double-precision floating-point value in xmm1.
+pub static MULSD: [u8; 3] = [0xf2, 0x0f, 0x59];
+
+/// Multiply the low single-precision floating-point value in xmm2/m32 by the
+/// low single-precision floating-point value in xmm1.
+pub static MULSS: [u8; 3] = [0xf3, 0x0f, 0x59];
+
+/// Reverse each bit of r/m{16,32,64}.
+pub static NOT: [u8; 1] = [0xf7];
+
+/// r{16,32,64} OR register of same size.
+pub static OR: [u8; 1] = [0x09];
+
+/// imm{16,32} OR r/m{16,32,64}, possibly sign-extended.
+pub static OR_IMM: [u8; 1] = [0x81];
+
+/// r/m{16,32,64} OR sign-extended imm8.
+pub static OR_IMM8_SIGN_EXTEND: [u8; 1] = [0x83];
+
+/// Return the bitwise logical OR of packed single-precision values in xmm and x/m (SSE).
+pub static ORPS: [u8; 2] = [0x0f, 0x56];
+
+/// Add packed byte integers from xmm2/m128 and xmm1 (SSE2).
+pub static PADDB: [u8; 3] = [0x66, 0x0f, 0xfc];
+
+/// Add packed doubleword integers from xmm2/m128 and xmm1 (SSE2).
+pub static PADDD: [u8; 3] = [0x66, 0x0f, 0xfe];
+
+/// Add packed quadword integers from xmm2/m128 and xmm1 (SSE2).
+pub static PADDQ: [u8; 3] = [0x66, 0x0f, 0xd4];
+
+/// Add packed word integers from xmm2/m128 and xmm1 (SSE2).
+pub static PADDW: [u8; 3] = [0x66, 0x0f, 0xfd];
+
+/// Add packed signed byte integers from xmm2/m128 and xmm1 saturate the results (SSE).
+pub static PADDSB: [u8; 3] = [0x66, 0x0f, 0xec];
+
+/// Add packed signed word integers from xmm2/m128 and xmm1 saturate the results (SSE).
+pub static PADDSW: [u8; 3] = [0x66, 0x0f, 0xed];
+
+/// Add packed unsigned byte integers from xmm2/m128 and xmm1 saturate the results (SSE).
+pub static PADDUSB: [u8; 3] = [0x66, 0x0f, 0xdc];
+
+/// Add packed unsigned word integers from xmm2/m128 and xmm1 saturate the results (SSE).
+pub static PADDUSW: [u8; 3] = [0x66, 0x0f, 0xdd];
+
+/// Bitwise AND of xmm2/m128 and xmm1 (SSE2).
+pub static PAND: [u8; 3] = [0x66, 0x0f, 0xdb];
+
+/// Bitwise AND NOT of xmm2/m128 and xmm1 (SSE2).
+pub static PANDN: [u8; 3] = [0x66, 0x0f, 0xdf];
+
+/// Average packed unsigned byte integers from xmm2/m128 and xmm1 with rounding (SSE2).
+pub static PAVGB: [u8; 3] = [0x66, 0x0f, 0xE0];
+
+/// Average packed unsigned word integers from xmm2/m128 and xmm1 with rounding (SSE2).
+pub static PAVGW: [u8; 3] = [0x66, 0x0f, 0xE3];
+
+/// Compare packed data for equal (SSE2).
+pub static PCMPEQB: [u8; 3] = [0x66, 0x0f, 0x74];
+
+/// Compare packed data for equal (SSE2).
+pub static PCMPEQD: [u8; 3] = [0x66, 0x0f, 0x76];
+
+/// Compare packed data for equal (SSE4.1).
+pub static PCMPEQQ: [u8; 4] = [0x66, 0x0f, 0x38, 0x29];
+
+/// Compare packed data for equal (SSE2).
+pub static PCMPEQW: [u8; 3] = [0x66, 0x0f, 0x75];
+
+/// Compare packed signed byte integers for greater than (SSE2).
+pub static PCMPGTB: [u8; 3] = [0x66, 0x0f, 0x64];
+
+/// Compare packed signed doubleword integers for greater than (SSE2).
+pub static PCMPGTD: [u8; 3] = [0x66, 0x0f, 0x66];
+
+/// Compare packed signed quadword integers for greater than (SSE4.2).
+pub static PCMPGTQ: [u8; 4] = [0x66, 0x0f, 0x38, 0x37];
+
+/// Compare packed signed word integers for greater than (SSE2).
+pub static PCMPGTW: [u8; 3] = [0x66, 0x0f, 0x65];
+
+/// Extract doubleword or quadword, depending on REX.W (SSE4.1).
+pub static PEXTR: [u8; 4] = [0x66, 0x0f, 0x3a, 0x16];
+
+/// Extract byte (SSE4.1).
+pub static PEXTRB: [u8; 4] = [0x66, 0x0f, 0x3a, 0x14];
+
+/// Extract word (SSE4.1). There is a 3-byte SSE2 variant that can also move to m/16.
+pub static PEXTRW: [u8; 4] = [0x66, 0x0f, 0x3a, 0x15];
+
+/// Insert doubleword or quadword, depending on REX.W (SSE4.1).
+pub static PINSR: [u8; 4] = [0x66, 0x0f, 0x3a, 0x22];
+
+/// Insert byte (SSE4.1).
+pub static PINSRB: [u8; 4] = [0x66, 0x0f, 0x3a, 0x20];
+
+/// Insert word (SSE2).
+pub static PINSRW: [u8; 3] = [0x66, 0x0f, 0xc4];
+
+/// Compare packed signed byte integers in xmm1 and xmm2/m128 and store packed maximum values in
+/// xmm1 (SSE4.1).
+pub static PMAXSB: [u8; 4] = [0x66, 0x0f, 0x38, 0x3c];
+
+/// Compare packed signed doubleword integers in xmm1 and xmm2/m128 and store packed maximum
+/// values in xmm1 (SSE4.1).
+pub static PMAXSD: [u8; 4] = [0x66, 0x0f, 0x38, 0x3d];
+
+/// Compare packed signed word integers in xmm1 and xmm2/m128 and store packed maximum values in
+/// xmm1 (SSE2).
+pub static PMAXSW: [u8; 3] = [0x66, 0x0f, 0xee];
+
+/// Compare packed unsigned byte integers in xmm1 and xmm2/m128 and store packed maximum values in
+/// xmm1 (SSE2).
+pub static PMAXUB: [u8; 3] = [0x66, 0x0f, 0xde];
+
+/// Compare packed unsigned doubleword integers in xmm1 and xmm2/m128 and store packed maximum
+/// values in xmm1 (SSE4.1).
+pub static PMAXUD: [u8; 4] = [0x66, 0x0f, 0x38, 0x3f];
+
+/// Compare packed unsigned word integers in xmm1 and xmm2/m128 and store packed maximum values in
+/// xmm1 (SSE4.1).
+pub static PMAXUW: [u8; 4] = [0x66, 0x0f, 0x38, 0x3e];
+
+/// Compare packed signed byte integers in xmm1 and xmm2/m128 and store packed minimum values in
+/// xmm1 (SSE4.1).
+pub static PMINSB: [u8; 4] = [0x66, 0x0f, 0x38, 0x38];
+
+/// Compare packed signed doubleword integers in xmm1 and xmm2/m128 and store packed minimum
+/// values in xmm1 (SSE4.1).
+pub static PMINSD: [u8; 4] = [0x66, 0x0f, 0x38, 0x39];
+
+/// Compare packed signed word integers in xmm1 and xmm2/m128 and store packed minimum values in
+/// xmm1 (SSE2).
+pub static PMINSW: [u8; 3] = [0x66, 0x0f, 0xea];
+
+/// Compare packed unsigned byte integers in xmm1 and xmm2/m128 and store packed minimum values in
+/// xmm1 (SSE2).
+pub static PMINUB: [u8; 3] = [0x66, 0x0f, 0xda];
+
+/// Compare packed unsigned doubleword integers in xmm1 and xmm2/m128 and store packed minimum
+/// values in xmm1 (SSE4.1).
+pub static PMINUD: [u8; 4] = [0x66, 0x0f, 0x38, 0x3b];
+
+/// Compare packed unsigned word integers in xmm1 and xmm2/m128 and store packed minimum values in
+/// xmm1 (SSE4.1).
+pub static PMINUW: [u8; 4] = [0x66, 0x0f, 0x38, 0x3a];
+
+/// Multiply the packed signed word integers in xmm1 and xmm2/m128, and store the low 16 bits of
+/// the results in xmm1 (SSE2).
+pub static PMULLW: [u8; 3] = [0x66, 0x0f, 0xd5];
+
+/// Multiply the packed doubleword signed integers in xmm1 and xmm2/m128 and store the low 32
+/// bits of each product in xmm1 (SSE4.1).
+pub static PMULLD: [u8; 4] = [0x66, 0x0f, 0x38, 0x40];
+
+/// Pop top of stack into r{16,32,64}; increment stack pointer.
+pub static POP_REG: [u8; 1] = [0x58];
+
+/// Returns the count of number of bits set to 1.
+pub static POPCNT: [u8; 3] = [0xf3, 0x0f, 0xb8];
+
+/// Bitwise OR of xmm2/m128 and xmm1 (SSE2).
+pub static POR: [u8; 3] = [0x66, 0x0f, 0xeb];
+
+/// Shuffle bytes in xmm1 according to contents of xmm2/m128 (SSE3).
+pub static PSHUFB: [u8; 4] = [0x66, 0x0f, 0x38, 0x00];
+
+/// Shuffle the doublewords in xmm2/m128 based on the encoding in imm8 and
+/// store the result in xmm1 (SSE2).
+pub static PSHUFD: [u8; 3] = [0x66, 0x0f, 0x70];
+
+/// Shift words in xmm1 by imm8; the direction and sign-bit behavior is controlled by the RRR
+/// digit used in the ModR/M byte (SSE2).
+pub static PS_W_IMM: [u8; 3] = [0x66, 0x0f, 0x71];
+
+/// Shift doublewords in xmm1 by imm8; the direction and sign-bit behavior is controlled by the RRR
+/// digit used in the ModR/M byte (SSE2).
+pub static PS_D_IMM: [u8; 3] = [0x66, 0x0f, 0x72];
+
+/// Shift quadwords in xmm1 by imm8; the direction and sign-bit behavior is controlled by the RRR
+/// digit used in the ModR/M byte (SSE2).
+pub static PS_Q_IMM: [u8; 3] = [0x66, 0x0f, 0x73];
+
+/// Shift words in xmm1 left by xmm2/m128 while shifting in 0s (SSE2).
+pub static PSLLW: [u8; 3] = [0x66, 0x0f, 0xf1];
+
+/// Shift doublewords in xmm1 left by xmm2/m128 while shifting in 0s (SSE2).
+pub static PSLLD: [u8; 3] = [0x66, 0x0f, 0xf2];
+
+/// Shift quadwords in xmm1 left by xmm2/m128 while shifting in 0s (SSE2).
+pub static PSLLQ: [u8; 3] = [0x66, 0x0f, 0xf3];
+
+/// Shift words in xmm1 right by xmm2/m128 while shifting in 0s (SSE2).
+pub static PSRLW: [u8; 3] = [0x66, 0x0f, 0xd1];
+
+/// Shift doublewords in xmm1 right by xmm2/m128 while shifting in 0s (SSE2).
+pub static PSRLD: [u8; 3] = [0x66, 0x0f, 0xd2];
+
+/// Shift quadwords in xmm1 right by xmm2/m128 while shifting in 0s (SSE2).
+pub static PSRLQ: [u8; 3] = [0x66, 0x0f, 0xd3];
+
+/// Shift words in xmm1 right by xmm2/m128 while shifting in sign bits (SSE2).
+pub static PSRAW: [u8; 3] = [0x66, 0x0f, 0xe1];
+
+/// Shift doublewords in xmm1 right by xmm2/m128 while shifting in sign bits (SSE2).
+pub static PSRAD: [u8; 3] = [0x66, 0x0f, 0xe2];
+
+/// Subtract packed byte integers in xmm2/m128 from packed byte integers in xmm1 (SSE2).
+pub static PSUBB: [u8; 3] = [0x66, 0x0f, 0xf8];
+
+/// Subtract packed word integers in xmm2/m128 from packed word integers in xmm1 (SSE2).
+pub static PSUBW: [u8; 3] = [0x66, 0x0f, 0xf9];
+
+/// Subtract packed doubleword integers in xmm2/m128 from doubleword byte integers in xmm1 (SSE2).
+pub static PSUBD: [u8; 3] = [0x66, 0x0f, 0xfa];
+
+/// Subtract packed quadword integers in xmm2/m128 from xmm1 (SSE2).
+pub static PSUBQ: [u8; 3] = [0x66, 0x0f, 0xfb];
+
+/// Subtract packed signed byte integers in xmm2/m128 from packed signed byte integers in xmm1
+/// and saturate results (SSE2).
+pub static PSUBSB: [u8; 3] = [0x66, 0x0f, 0xe8];
+
+/// Subtract packed signed word integers in xmm2/m128 from packed signed word integers in xmm1
+/// and saturate results (SSE2).
+pub static PSUBSW: [u8; 3] = [0x66, 0x0f, 0xe9];
+
+/// Subtract packed unsigned byte integers in xmm2/m128 from packed unsigned byte integers in xmm1
+/// and saturate results (SSE2).
+pub static PSUBUSB: [u8; 3] = [0x66, 0x0f, 0xd8];
+
+/// Subtract packed unsigned word integers in xmm2/m128 from packed unsigned word integers in xmm1
+/// and saturate results (SSE2).
+pub static PSUBUSW: [u8; 3] = [0x66, 0x0f, 0xd9];
+
+/// Set ZF if xmm2/m128 AND xmm1 result is all 0s; set CF if xmm2/m128 AND NOT xmm1 result is all
+/// 0s (SSE4.1).
+pub static PTEST: [u8; 4] = [0x66, 0x0f, 0x38, 0x17];
+
+/// Push r{16,32,64}.
+pub static PUSH_REG: [u8; 1] = [0x50];
+
+/// Logical exclusive OR (SSE2).
+pub static PXOR: [u8; 3] = [0x66, 0x0f, 0xef];
+
+/// Near return to calling procedure.
+pub static RET_NEAR: [u8; 1] = [0xc3];
+
+/// General rotation opcode. Kind of rotation depends on encoding.
+pub static ROTATE_CL: [u8; 1] = [0xd3];
+
+/// General rotation opcode. Kind of rotation depends on encoding.
+pub static ROTATE_IMM8: [u8; 1] = [0xc1];
+
+/// Round scalar doubl-precision floating-point values.
+pub static ROUNDSD: [u8; 4] = [0x66, 0x0f, 0x3a, 0x0b];
+
+/// Round scalar single-precision floating-point values.
+pub static ROUNDSS: [u8; 4] = [0x66, 0x0f, 0x3a, 0x0a];
+
+/// Subtract with borrow r{16,32,64} from r/m of the same size.
+pub static SBB: [u8; 1] = [0x19];
+
+/// Set byte if overflow (OF=1).
+pub static SET_BYTE_IF_OVERFLOW: [u8; 2] = [0x0f, 0x90];
+
+/// Compute the square root of the packed double-precision floating-point values and store the
+/// result in xmm1 (SSE2).
+pub static SQRTPD: [u8; 3] = [0x66, 0x0f, 0x51];
+
+/// Compute the square root of the packed double-precision floating-point values and store the
+/// result in xmm1 (SSE).
+pub static SQRTPS: [u8; 2] = [0x0f, 0x51];
+
+/// Compute square root of scalar double-precision floating-point value.
+pub static SQRTSD: [u8; 3] = [0xf2, 0x0f, 0x51];
+
+/// Compute square root of scalar single-precision value.
+pub static SQRTSS: [u8; 3] = [0xf3, 0x0f, 0x51];
+
+/// Subtract r{16,32,64} from r/m of same size.
+pub static SUB: [u8; 1] = [0x29];
+
+/// Subtract packed double-precision floating-point values in xmm2/mem from xmm1 and store result
+/// in xmm1 (SSE2).
+pub static SUBPD: [u8; 3] = [0x66, 0x0f, 0x5c];
+
+/// Subtract packed single-precision floating-point values in xmm2/mem from xmm1 and store result
+/// in xmm1 (SSE).
+pub static SUBPS: [u8; 2] = [0x0f, 0x5c];
+
+/// Subtract the low double-precision floating-point value in xmm2/m64 from xmm1
+/// and store the result in xmm1.
+pub static SUBSD: [u8; 3] = [0xf2, 0x0f, 0x5c];
+
+/// Subtract the low single-precision floating-point value in xmm2/m32 from xmm1
+/// and store the result in xmm1.
+pub static SUBSS: [u8; 3] = [0xf3, 0x0f, 0x5c];
+
+/// AND r8 with r/m8; set SF, ZF, PF according to result.
+pub static TEST_BYTE_REG: [u8; 1] = [0x84];
+
+/// AND {r16, r32, r64} with r/m of the same size; set SF, ZF, PF according to result.
+pub static TEST_REG: [u8; 1] = [0x85];
+
+/// Count the number of trailing zero bits.
+pub static TZCNT: [u8; 3] = [0xf3, 0x0f, 0xbc];
+
+/// Compare low double-precision floating-point values in xmm1 and xmm2/mem64
+/// and set the EFLAGS flags accordingly.
+pub static UCOMISD: [u8; 3] = [0x66, 0x0f, 0x2e];
+
+/// Compare low single-precision floating-point values in xmm1 and xmm2/mem32
+/// and set the EFLAGS flags accordingly.
+pub static UCOMISS: [u8; 2] = [0x0f, 0x2e];
+
+/// Raise invalid opcode instruction.
+pub static UNDEFINED2: [u8; 2] = [0x0f, 0x0b];
+
+/// imm{16,32} XOR r/m{16,32,64}, possibly sign-extended.
+pub static XOR_IMM: [u8; 1] = [0x81];
+
+/// r/m{16,32,64} XOR sign-extended imm8.
+pub static XOR_IMM8_SIGN_EXTEND: [u8; 1] = [0x83];
+
+/// r/m{16,32,64} XOR register of the same size.
+pub static XOR: [u8; 1] = [0x31];
+
+/// r/m8 XOR r8.
+pub static XORB: [u8; 1] = [0x30];
+
+/// Bitwise logical XOR of packed double-precision floating-point values.
+pub static XORPD: [u8; 3] = [0x66, 0x0f, 0x57];
+
+/// Bitwise logical XOR of packed single-precision floating-point values.
+pub static XORPS: [u8; 2] = [0x0f, 0x57];

cranelift/codegen/meta/src/isa/x86/registers.rs

@@ -0,0 +1,43 @@
+use crate::cdsl::regs::{IsaRegs, IsaRegsBuilder, RegBankBuilder, RegClassBuilder};
+
+pub(crate) fn define() -> IsaRegs {
+    let mut regs = IsaRegsBuilder::new();
+
+    let builder = RegBankBuilder::new("IntRegs", "r")
+        .units(16)
+        .names(vec!["rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi"])
+        .track_pressure(true)
+        .pinned_reg(15);
+    let int_regs = regs.add_bank(builder);
+
+    let builder = RegBankBuilder::new("FloatRegs", "xmm")
+        .units(16)
+        .track_pressure(true);
+    let float_regs = regs.add_bank(builder);
+
+    let builder = RegBankBuilder::new("FlagRegs", "")
+        .units(1)
+        .names(vec!["rflags"])
+        .track_pressure(false);
+    let flag_reg = regs.add_bank(builder);
+
+    let builder = RegClassBuilder::new_toplevel("GPR", int_regs);
+    let gpr = regs.add_class(builder);
+
+    let builder = RegClassBuilder::new_toplevel("FPR", float_regs);
+    let fpr = regs.add_class(builder);
+
+    let builder = RegClassBuilder::new_toplevel("FLAG", flag_reg);
+    regs.add_class(builder);
+
+    let builder = RegClassBuilder::subclass_of("GPR8", gpr, 0, 8);
+    let gpr8 = regs.add_class(builder);
+
+    let builder = RegClassBuilder::subclass_of("ABCD", gpr8, 0, 4);
+    regs.add_class(builder);
+
+    let builder = RegClassBuilder::subclass_of("FPR8", fpr, 0, 8);
+    regs.add_class(builder);
+
+    regs.build()
+}

cranelift/codegen/meta/src/isa/x86/settings.rs

@@ -0,0 +1,104 @@
+use crate::cdsl::settings::{PredicateNode, SettingGroup, SettingGroupBuilder};
+
+pub(crate) fn define(shared: &SettingGroup) -> SettingGroup {
+    let mut settings = SettingGroupBuilder::new("x86");
+
+    // CPUID.01H:ECX
+    let has_sse3 = settings.add_bool("has_sse3", "SSE3: CPUID.01H:ECX.SSE3[bit 0]", false);
+    let has_ssse3 = settings.add_bool("has_ssse3", "SSSE3: CPUID.01H:ECX.SSSE3[bit 9]", false);
+    let has_sse41 = settings.add_bool("has_sse41", "SSE4.1: CPUID.01H:ECX.SSE4_1[bit 19]", false);
+    let has_sse42 = settings.add_bool("has_sse42", "SSE4.2: CPUID.01H:ECX.SSE4_2[bit 20]", false);
+    let has_popcnt = settings.add_bool("has_popcnt", "POPCNT: CPUID.01H:ECX.POPCNT[bit 23]", false);
+    settings.add_bool("has_avx", "AVX: CPUID.01H:ECX.AVX[bit 28]", false);
+
+    // CPUID.(EAX=07H, ECX=0H):EBX
+    let has_bmi1 = settings.add_bool(
+        "has_bmi1",
+        "BMI1: CPUID.(EAX=07H, ECX=0H):EBX.BMI1[bit 3]",
+        false,
+    );
+    let has_bmi2 = settings.add_bool(
+        "has_bmi2",
+        "BMI2: CPUID.(EAX=07H, ECX=0H):EBX.BMI2[bit 8]",
+        false,
+    );
+
+    // CPUID.EAX=80000001H:ECX
+    let has_lzcnt = settings.add_bool(
+        "has_lzcnt",
+        "LZCNT: CPUID.EAX=80000001H:ECX.LZCNT[bit 5]",
+        false,
+    );
+
+    let shared_enable_simd = shared.get_bool("enable_simd");
+
+    settings.add_predicate("use_ssse3", predicate!(has_ssse3));
+    settings.add_predicate("use_sse41", predicate!(has_sse41));
+    settings.add_predicate("use_sse42", predicate!(has_sse41 && has_sse42));
+
+    settings.add_predicate(
+        "use_ssse3_simd",
+        predicate!(shared_enable_simd && has_ssse3),
+    );
+    settings.add_predicate(
+        "use_sse41_simd",
+        predicate!(shared_enable_simd && has_sse41),
+    );
+    settings.add_predicate(
+        "use_sse42_simd",
+        predicate!(shared_enable_simd && has_sse41 && has_sse42),
+    );
+
+    settings.add_predicate("use_popcnt", predicate!(has_popcnt && has_sse42));
+    settings.add_predicate("use_bmi1", predicate!(has_bmi1));
+    settings.add_predicate("use_lzcnt", predicate!(has_lzcnt));
+
+    // Some shared boolean values are used in x86 instruction predicates, so we need to group them
+    // in the same TargetIsa, for compabitibity with code generated by meta-python.
+    // TODO Once all the meta generation code has been migrated from Python to Rust, we can put it
+    // back in the shared SettingGroup, and use it in x86 instruction predicates.
+
+    let is_pic = shared.get_bool("is_pic");
+    let emit_all_ones_funcaddrs = shared.get_bool("emit_all_ones_funcaddrs");
+    settings.add_predicate("is_pic", predicate!(is_pic));
+    settings.add_predicate("not_is_pic", predicate!(!is_pic));
+    settings.add_predicate(
+        "all_ones_funcaddrs_and_not_is_pic",
+        predicate!(emit_all_ones_funcaddrs && !is_pic),
+    );
+    settings.add_predicate(
+        "not_all_ones_funcaddrs_and_not_is_pic",
+        predicate!(!emit_all_ones_funcaddrs && !is_pic),
+    );
+
+    // Presets corresponding to x86 CPUs.
+
+    settings.add_preset("baseline", preset!());
+    let nehalem = settings.add_preset(
+        "nehalem",
+        preset!(has_sse3 && has_ssse3 && has_sse41 && has_sse42 && has_popcnt),
+    );
+    let haswell = settings.add_preset(
+        "haswell",
+        preset!(nehalem && has_bmi1 && has_bmi2 && has_lzcnt),
+    );
+    let broadwell = settings.add_preset("broadwell", preset!(haswell));
+    let skylake = settings.add_preset("skylake", preset!(broadwell));
+    let cannonlake = settings.add_preset("cannonlake", preset!(skylake));
+    settings.add_preset("icelake", preset!(cannonlake));
+    settings.add_preset(
+        "znver1",
+        preset!(
+            has_sse3
+                && has_ssse3
+                && has_sse41
+                && has_sse42
+                && has_popcnt
+                && has_bmi1
+                && has_bmi2
+                && has_lzcnt
+        ),
+    );
+
+    settings.build()
+}

cranelift/codegen/meta/src/lib.rs

@@ -0,0 +1,84 @@
+//! This crate generates Rust sources for use by
+//! [`cranelift_codegen`](../cranelift_codegen/index.html).
+#[macro_use]
+mod cdsl;
+mod srcgen;
+
+pub mod error;
+pub mod isa;
+
+mod gen_binemit;
+mod gen_encodings;
+mod gen_inst;
+mod gen_legalizer;
+mod gen_registers;
+mod gen_settings;
+mod gen_types;
+
+mod default_map;
+mod shared;
+mod unique_table;
+
+/// Generate an ISA from an architecture string (e.g. "x86_64").
+pub fn isa_from_arch(arch: &str) -> Result<isa::Isa, String> {
+    isa::Isa::from_arch(arch).ok_or_else(|| format!("no supported isa found for arch `{}`", arch))
+}
+
+/// Generates all the Rust source files used in Cranelift from the meta-language.
+pub fn generate(isas: &[isa::Isa], out_dir: &str) -> Result<(), error::Error> {
+    // Create all the definitions:
+    // - common definitions.
+    let mut shared_defs = shared::define();
+
+    gen_settings::generate(
+        &shared_defs.settings,
+        gen_settings::ParentGroup::None,
+        "settings.rs",
+        &out_dir,
+    )?;
+    gen_types::generate("types.rs", &out_dir)?;
+
+    // - per ISA definitions.
+    let isas = isa::define(isas, &mut shared_defs);
+
+    // At this point, all definitions are done.
+    let all_formats = shared_defs.verify_instruction_formats();
+
+    // Generate all the code.
+    gen_inst::generate(
+        all_formats,
+        &shared_defs.all_instructions,
+        "opcodes.rs",
+        "inst_builder.rs",
+        &out_dir,
+    )?;
+
+    gen_legalizer::generate(&isas, &shared_defs.transform_groups, "legalize", &out_dir)?;
+
+    for isa in isas {
+        gen_registers::generate(&isa, &format!("registers-{}.rs", isa.name), &out_dir)?;
+
+        gen_settings::generate(
+            &isa.settings,
+            gen_settings::ParentGroup::Shared,
+            &format!("settings-{}.rs", isa.name),
+            &out_dir,
+        )?;
+
+        gen_encodings::generate(
+            &shared_defs,
+            &isa,
+            &format!("encoding-{}.rs", isa.name),
+            &out_dir,
+        )?;
+
+        gen_binemit::generate(
+            &isa.name,
+            &isa.recipes,
+            &format!("binemit-{}.rs", isa.name),
+            &out_dir,
+        )?;
+    }
+
+    Ok(())
+}

cranelift/codegen/meta/src/shared/entities.rs

@@ -0,0 +1,73 @@
+use crate::cdsl::operands::{OperandKind, OperandKindFields};
+
+/// Small helper to initialize an OperandBuilder with the right kind, for a given name and doc.
+fn new(format_field_name: &'static str, rust_type: &'static str, doc: &'static str) -> OperandKind {
+    OperandKind::new(format_field_name, rust_type, OperandKindFields::EntityRef).with_doc(doc)
+}
+
+pub(crate) struct EntityRefs {
+    /// A reference to a basic block in the same function.
+    /// This is primarliy used in control flow instructions.
+    pub(crate) block: OperandKind,
+
+    /// A reference to a stack slot declared in the function preamble.
+    pub(crate) stack_slot: OperandKind,
+
+    /// A reference to a global value.
+    pub(crate) global_value: OperandKind,
+
+    /// A reference to a function signature declared in the function preamble.
+    /// This is used to provide the call signature in a call_indirect instruction.
+    pub(crate) sig_ref: OperandKind,
+
+    /// A reference to an external function declared in the function preamble.
+    /// This is used to provide the callee and signature in a call instruction.
+    pub(crate) func_ref: OperandKind,
+
+    /// A reference to a jump table declared in the function preamble.
+    pub(crate) jump_table: OperandKind,
+
+    /// A reference to a heap declared in the function preamble.
+    pub(crate) heap: OperandKind,
+
+    /// A reference to a table declared in the function preamble.
+    pub(crate) table: OperandKind,
+
+    /// A variable-sized list of value operands. Use for Block and function call arguments.
+    pub(crate) varargs: OperandKind,
+}
+
+impl EntityRefs {
+    pub fn new() -> Self {
+        Self {
+            block: new(
+                "destination",
+                "ir::Block",
+                "a basic block in the same function.",
+            ),
+            stack_slot: new("stack_slot", "ir::StackSlot", "A stack slot"),
+
+            global_value: new("global_value", "ir::GlobalValue", "A global value."),
+
+            sig_ref: new("sig_ref", "ir::SigRef", "A function signature."),
+
+            func_ref: new("func_ref", "ir::FuncRef", "An external function."),
+
+            jump_table: new("table", "ir::JumpTable", "A jump table."),
+
+            heap: new("heap", "ir::Heap", "A heap."),
+
+            table: new("table", "ir::Table", "A table."),
+
+            varargs: OperandKind::new("", "&[Value]", OperandKindFields::VariableArgs).with_doc(
+                r#"
+                        A variable size list of `value` operands.
+
+                        Use this to represent arguments passed to a function call, arguments
+                        passed to a basic block, or a variable number of results
+                        returned from an instruction.
+                    "#,
+            ),
+        }
+    }
+}

cranelift/codegen/meta/src/shared/formats.rs

@@ -0,0 +1,303 @@
+use crate::cdsl::formats::{InstructionFormat, InstructionFormatBuilder as Builder};
+use crate::shared::{entities::EntityRefs, immediates::Immediates};
+use std::rc::Rc;
+
+pub(crate) struct Formats {
+    pub(crate) binary: Rc<InstructionFormat>,
+    pub(crate) binary_imm: Rc<InstructionFormat>,
+    pub(crate) branch: Rc<InstructionFormat>,
+    pub(crate) branch_float: Rc<InstructionFormat>,
+    pub(crate) branch_icmp: Rc<InstructionFormat>,
+    pub(crate) branch_int: Rc<InstructionFormat>,
+    pub(crate) branch_table: Rc<InstructionFormat>,
+    pub(crate) branch_table_base: Rc<InstructionFormat>,
+    pub(crate) branch_table_entry: Rc<InstructionFormat>,
+    pub(crate) call: Rc<InstructionFormat>,
+    pub(crate) call_indirect: Rc<InstructionFormat>,
+    pub(crate) cond_trap: Rc<InstructionFormat>,
+    pub(crate) copy_special: Rc<InstructionFormat>,
+    pub(crate) copy_to_ssa: Rc<InstructionFormat>,
+    pub(crate) extract_lane: Rc<InstructionFormat>,
+    pub(crate) float_compare: Rc<InstructionFormat>,
+    pub(crate) float_cond: Rc<InstructionFormat>,
+    pub(crate) float_cond_trap: Rc<InstructionFormat>,
+    pub(crate) func_addr: Rc<InstructionFormat>,
+    pub(crate) heap_addr: Rc<InstructionFormat>,
+    pub(crate) indirect_jump: Rc<InstructionFormat>,
+    pub(crate) insert_lane: Rc<InstructionFormat>,
+    pub(crate) int_compare: Rc<InstructionFormat>,
+    pub(crate) int_compare_imm: Rc<InstructionFormat>,
+    pub(crate) int_cond: Rc<InstructionFormat>,
+    pub(crate) int_cond_trap: Rc<InstructionFormat>,
+    pub(crate) int_select: Rc<InstructionFormat>,
+    pub(crate) jump: Rc<InstructionFormat>,
+    pub(crate) load: Rc<InstructionFormat>,
+    pub(crate) load_complex: Rc<InstructionFormat>,
+    pub(crate) multiary: Rc<InstructionFormat>,
+    pub(crate) nullary: Rc<InstructionFormat>,
+    pub(crate) reg_fill: Rc<InstructionFormat>,
+    pub(crate) reg_move: Rc<InstructionFormat>,
+    pub(crate) reg_spill: Rc<InstructionFormat>,
+    pub(crate) shuffle: Rc<InstructionFormat>,
+    pub(crate) stack_load: Rc<InstructionFormat>,
+    pub(crate) stack_store: Rc<InstructionFormat>,
+    pub(crate) store: Rc<InstructionFormat>,
+    pub(crate) store_complex: Rc<InstructionFormat>,
+    pub(crate) table_addr: Rc<InstructionFormat>,
+    pub(crate) ternary: Rc<InstructionFormat>,
+    pub(crate) trap: Rc<InstructionFormat>,
+    pub(crate) unary: Rc<InstructionFormat>,
+    pub(crate) unary_bool: Rc<InstructionFormat>,
+    pub(crate) unary_const: Rc<InstructionFormat>,
+    pub(crate) unary_global_value: Rc<InstructionFormat>,
+    pub(crate) unary_ieee32: Rc<InstructionFormat>,
+    pub(crate) unary_ieee64: Rc<InstructionFormat>,
+    pub(crate) unary_imm: Rc<InstructionFormat>,
+}
+
+impl Formats {
+    pub fn new(imm: &Immediates, entities: &EntityRefs) -> Self {
+        Self {
+            unary: Builder::new("Unary").value().build(),
+
+            unary_imm: Builder::new("UnaryImm").imm(&imm.imm64).build(),
+
+            unary_ieee32: Builder::new("UnaryIeee32").imm(&imm.ieee32).build(),
+
+            unary_ieee64: Builder::new("UnaryIeee64").imm(&imm.ieee64).build(),
+
+            unary_bool: Builder::new("UnaryBool").imm(&imm.boolean).build(),
+
+            unary_const: Builder::new("UnaryConst").imm(&imm.pool_constant).build(),
+
+            unary_global_value: Builder::new("UnaryGlobalValue")
+                .imm(&entities.global_value)
+                .build(),
+
+            binary: Builder::new("Binary").value().value().build(),
+
+            binary_imm: Builder::new("BinaryImm").value().imm(&imm.imm64).build(),
+
+            // The select instructions are controlled by the second VALUE operand.
+            // The first VALUE operand is the controlling flag which has a derived type.
+            // The fma instruction has the same constraint on all inputs.
+            ternary: Builder::new("Ternary")
+                .value()
+                .value()
+                .value()
+                .typevar_operand(1)
+                .build(),
+
+            // Catch-all for instructions with many outputs and inputs and no immediate
+            // operands.
+            multiary: Builder::new("MultiAry").varargs().build(),
+
+            nullary: Builder::new("NullAry").build(),
+
+            insert_lane: Builder::new("InsertLane")
+                .value()
+                .imm_with_name("lane", &imm.uimm8)
+                .value()
+                .build(),
+
+            extract_lane: Builder::new("ExtractLane")
+                .value()
+                .imm_with_name("lane", &imm.uimm8)
+                .build(),
+
+            shuffle: Builder::new("Shuffle")
+                .value()
+                .value()
+                .imm_with_name("mask", &imm.uimm128)
+                .build(),
+
+            int_compare: Builder::new("IntCompare")
+                .imm(&imm.intcc)
+                .value()
+                .value()
+                .build(),
+
+            int_compare_imm: Builder::new("IntCompareImm")
+                .imm(&imm.intcc)
+                .value()
+                .imm(&imm.imm64)
+                .build(),
+
+            int_cond: Builder::new("IntCond").imm(&imm.intcc).value().build(),
+
+            float_compare: Builder::new("FloatCompare")
+                .imm(&imm.floatcc)
+                .value()
+                .value()
+                .build(),
+
+            float_cond: Builder::new("FloatCond").imm(&imm.floatcc).value().build(),
+
+            int_select: Builder::new("IntSelect")
+                .imm(&imm.intcc)
+                .value()
+                .value()
+                .value()
+                .build(),
+
+            jump: Builder::new("Jump").imm(&entities.block).varargs().build(),
+
+            branch: Builder::new("Branch")
+                .value()
+                .imm(&entities.block)
+                .varargs()
+                .build(),
+
+            branch_int: Builder::new("BranchInt")
+                .imm(&imm.intcc)
+                .value()
+                .imm(&entities.block)
+                .varargs()
+                .build(),
+
+            branch_float: Builder::new("BranchFloat")
+                .imm(&imm.floatcc)
+                .value()
+                .imm(&entities.block)
+                .varargs()
+                .build(),
+
+            branch_icmp: Builder::new("BranchIcmp")
+                .imm(&imm.intcc)
+                .value()
+                .value()
+                .imm(&entities.block)
+                .varargs()
+                .build(),
+
+            branch_table: Builder::new("BranchTable")
+                .value()
+                .imm(&entities.block)
+                .imm(&entities.jump_table)
+                .build(),
+
+            branch_table_entry: Builder::new("BranchTableEntry")
+                .value()
+                .value()
+                .imm(&imm.uimm8)
+                .imm(&entities.jump_table)
+                .build(),
+
+            branch_table_base: Builder::new("BranchTableBase")
+                .imm(&entities.jump_table)
+                .build(),
+
+            indirect_jump: Builder::new("IndirectJump")
+                .value()
+                .imm(&entities.jump_table)
+                .build(),
+
+            call: Builder::new("Call")
+                .imm(&entities.func_ref)
+                .varargs()
+                .build(),
+
+            call_indirect: Builder::new("CallIndirect")
+                .imm(&entities.sig_ref)
+                .value()
+                .varargs()
+                .build(),
+
+            func_addr: Builder::new("FuncAddr").imm(&entities.func_ref).build(),
+
+            load: Builder::new("Load")
+                .imm(&imm.memflags)
+                .value()
+                .imm(&imm.offset32)
+                .build(),
+
+            load_complex: Builder::new("LoadComplex")
+                .imm(&imm.memflags)
+                .varargs()
+                .imm(&imm.offset32)
+                .build(),
+
+            store: Builder::new("Store")
+                .imm(&imm.memflags)
+                .value()
+                .value()
+                .imm(&imm.offset32)
+                .build(),
+
+            store_complex: Builder::new("StoreComplex")
+                .imm(&imm.memflags)
+                .value()
+                .varargs()
+                .imm(&imm.offset32)
+                .build(),
+
+            stack_load: Builder::new("StackLoad")
+                .imm(&entities.stack_slot)
+                .imm(&imm.offset32)
+                .build(),
+
+            stack_store: Builder::new("StackStore")
+                .value()
+                .imm(&entities.stack_slot)
+                .imm(&imm.offset32)
+                .build(),
+
+            // Accessing a WebAssembly heap.
+            heap_addr: Builder::new("HeapAddr")
+                .imm(&entities.heap)
+                .value()
+                .imm(&imm.uimm32)
+                .build(),
+
+            // Accessing a WebAssembly table.
+            table_addr: Builder::new("TableAddr")
+                .imm(&entities.table)
+                .value()
+                .imm(&imm.offset32)
+                .build(),
+
+            reg_move: Builder::new("RegMove")
+                .value()
+                .imm_with_name("src", &imm.regunit)
+                .imm_with_name("dst", &imm.regunit)
+                .build(),
+
+            copy_special: Builder::new("CopySpecial")
+                .imm_with_name("src", &imm.regunit)
+                .imm_with_name("dst", &imm.regunit)
+                .build(),
+
+            copy_to_ssa: Builder::new("CopyToSsa")
+                .imm_with_name("src", &imm.regunit)
+                .build(),
+
+            reg_spill: Builder::new("RegSpill")
+                .value()
+                .imm_with_name("src", &imm.regunit)
+                .imm_with_name("dst", &entities.stack_slot)
+                .build(),
+
+            reg_fill: Builder::new("RegFill")
+                .value()
+                .imm_with_name("src", &entities.stack_slot)
+                .imm_with_name("dst", &imm.regunit)
+                .build(),
+
+            trap: Builder::new("Trap").imm(&imm.trapcode).build(),
+
+            cond_trap: Builder::new("CondTrap").value().imm(&imm.trapcode).build(),
+
+            int_cond_trap: Builder::new("IntCondTrap")
+                .imm(&imm.intcc)
+                .value()
+                .imm(&imm.trapcode)
+                .build(),
+
+            float_cond_trap: Builder::new("FloatCondTrap")
+                .imm(&imm.floatcc)
+                .value()
+                .imm(&imm.trapcode)
+                .build(),
+        }
+    }
+}

cranelift/codegen/meta/src/shared/immediates.rs

@@ -0,0 +1,161 @@
+use crate::cdsl::operands::{EnumValues, OperandKind, OperandKindFields};
+
+use std::collections::HashMap;
+
+pub(crate) struct Immediates {
+    /// A 64-bit immediate integer operand.
+    ///
+    /// This type of immediate integer can interact with SSA values with any IntType type.
+    pub imm64: OperandKind,
+
+    /// An unsigned 8-bit immediate integer operand.
+    ///
+    /// This small operand is used to indicate lane indexes in SIMD vectors and immediate bit
+    /// counts on shift instructions.
+    pub uimm8: OperandKind,
+
+    /// An unsigned 32-bit immediate integer operand.
+    pub uimm32: OperandKind,
+
+    /// An unsigned 128-bit immediate integer operand.
+    ///
+    /// This operand is used to pass entire 128-bit vectors as immediates to instructions like
+    /// const.
+    pub uimm128: OperandKind,
+
+    /// A constant stored in the constant pool.
+    ///
+    /// This operand is used to pass constants to instructions like vconst while storing the
+    /// actual bytes in the constant pool.
+    pub pool_constant: OperandKind,
+
+    /// A 32-bit immediate signed offset.
+    ///
+    /// This is used to represent an immediate address offset in load/store instructions.
+    pub offset32: OperandKind,
+
+    /// A 32-bit immediate floating point operand.
+    ///
+    /// IEEE 754-2008 binary32 interchange format.
+    pub ieee32: OperandKind,
+
+    /// A 64-bit immediate floating point operand.
+    ///
+    /// IEEE 754-2008 binary64 interchange format.
+    pub ieee64: OperandKind,
+
+    /// An immediate boolean operand.
+    ///
+    /// This type of immediate boolean can interact with SSA values with any BoolType type.
+    pub boolean: OperandKind,
+
+    /// A condition code for comparing integer values.
+    ///
+    /// This enumerated operand kind is used for the `icmp` instruction and corresponds to the
+    /// condcodes::IntCC` Rust type.
+    pub intcc: OperandKind,
+
+    /// A condition code for comparing floating point values.
+    ///
+    /// This enumerated operand kind is used for the `fcmp` instruction and corresponds to the
+    /// `condcodes::FloatCC` Rust type.
+    pub floatcc: OperandKind,
+
+    /// Flags for memory operations like `load` and `store`.
+    pub memflags: OperandKind,
+
+    /// A register unit in the current target ISA.
+    pub regunit: OperandKind,
+
+    /// A trap code indicating the reason for trapping.
+    ///
+    /// The Rust enum type also has a `User(u16)` variant for user-provided trap codes.
+    pub trapcode: OperandKind,
+}
+
+fn new_imm(format_field_name: &'static str, rust_type: &'static str) -> OperandKind {
+    OperandKind::new(format_field_name, rust_type, OperandKindFields::ImmValue)
+}
+fn new_enum(
+    format_field_name: &'static str,
+    rust_type: &'static str,
+    values: EnumValues,
+) -> OperandKind {
+    OperandKind::new(
+        format_field_name,
+        rust_type,
+        OperandKindFields::ImmEnum(values),
+    )
+}
+
+impl Immediates {
+    pub fn new() -> Self {
+        Self {
+            imm64: new_imm("imm", "ir::immediates::Imm64").with_doc("A 64-bit immediate integer."),
+            uimm8: new_imm("imm", "ir::immediates::Uimm8")
+                .with_doc("An 8-bit immediate unsigned integer."),
+            uimm32: new_imm("imm", "ir::immediates::Uimm32")
+                .with_doc("A 32-bit immediate unsigned integer."),
+            uimm128: new_imm("imm", "ir::Immediate")
+                .with_doc("A 128-bit immediate unsigned integer."),
+            pool_constant: new_imm("constant_handle", "ir::Constant")
+                .with_doc("A constant stored in the constant pool."),
+            offset32: new_imm("offset", "ir::immediates::Offset32")
+                .with_doc("A 32-bit immediate signed offset."),
+            ieee32: new_imm("imm", "ir::immediates::Ieee32")
+                .with_doc("A 32-bit immediate floating point number."),
+            ieee64: new_imm("imm", "ir::immediates::Ieee64")
+                .with_doc("A 64-bit immediate floating point number."),
+            boolean: new_imm("imm", "bool").with_doc("An immediate boolean."),
+            intcc: {
+                let mut intcc_values = HashMap::new();
+                intcc_values.insert("eq", "Equal");
+                intcc_values.insert("ne", "NotEqual");
+                intcc_values.insert("sge", "SignedGreaterThanOrEqual");
+                intcc_values.insert("sgt", "SignedGreaterThan");
+                intcc_values.insert("sle", "SignedLessThanOrEqual");
+                intcc_values.insert("slt", "SignedLessThan");
+                intcc_values.insert("uge", "UnsignedGreaterThanOrEqual");
+                intcc_values.insert("ugt", "UnsignedGreaterThan");
+                intcc_values.insert("ule", "UnsignedLessThanOrEqual");
+                intcc_values.insert("ult", "UnsignedLessThan");
+                intcc_values.insert("of", "Overflow");
+                intcc_values.insert("nof", "NotOverflow");
+                new_enum("cond", "ir::condcodes::IntCC", intcc_values)
+                    .with_doc("An integer comparison condition code.")
+            },
+
+            floatcc: {
+                let mut floatcc_values = HashMap::new();
+                floatcc_values.insert("ord", "Ordered");
+                floatcc_values.insert("uno", "Unordered");
+                floatcc_values.insert("eq", "Equal");
+                floatcc_values.insert("ne", "NotEqual");
+                floatcc_values.insert("one", "OrderedNotEqual");
+                floatcc_values.insert("ueq", "UnorderedOrEqual");
+                floatcc_values.insert("lt", "LessThan");
+                floatcc_values.insert("le", "LessThanOrEqual");
+                floatcc_values.insert("gt", "GreaterThan");
+                floatcc_values.insert("ge", "GreaterThanOrEqual");
+                floatcc_values.insert("ult", "UnorderedOrLessThan");
+                floatcc_values.insert("ule", "UnorderedOrLessThanOrEqual");
+                floatcc_values.insert("ugt", "UnorderedOrGreaterThan");
+                floatcc_values.insert("uge", "UnorderedOrGreaterThanOrEqual");
+                new_enum("cond", "ir::condcodes::FloatCC", floatcc_values)
+                    .with_doc("A floating point comparison condition code")
+            },
+
+            memflags: new_imm("flags", "ir::MemFlags").with_doc("Memory operation flags"),
+            regunit: new_imm("regunit", "isa::RegUnit")
+                .with_doc("A register unit in the target ISA"),
+            trapcode: {
+                let mut trapcode_values = HashMap::new();
+                trapcode_values.insert("stk_ovf", "StackOverflow");
+                trapcode_values.insert("heap_oob", "HeapOutOfBounds");
+                trapcode_values.insert("int_ovf", "IntegerOverflow");
+                trapcode_values.insert("int_divz", "IntegerDivisionByZero");
+                new_enum("code", "ir::TrapCode", trapcode_values).with_doc("A trap reason code.")
+            },
+        }
+    }
+}

cranelift/codegen/meta/src/shared/mod.rs

@@ -0,0 +1,101 @@
+//! Shared definitions for the Cranelift intermediate language.
+
+pub mod entities;
+pub mod formats;
+pub mod immediates;
+pub mod instructions;
+pub mod legalize;
+pub mod settings;
+pub mod types;
+
+use crate::cdsl::formats::{FormatStructure, InstructionFormat};
+use crate::cdsl::instructions::{AllInstructions, InstructionGroup};
+use crate::cdsl::settings::SettingGroup;
+use crate::cdsl::xform::TransformGroups;
+
+use crate::shared::entities::EntityRefs;
+use crate::shared::formats::Formats;
+use crate::shared::immediates::Immediates;
+
+use std::collections::HashMap;
+use std::iter::FromIterator;
+use std::rc::Rc;
+
+pub(crate) struct Definitions {
+    pub settings: SettingGroup,
+    pub all_instructions: AllInstructions,
+    pub instructions: InstructionGroup,
+    pub imm: Immediates,
+    pub formats: Formats,
+    pub transform_groups: TransformGroups,
+    pub entities: EntityRefs,
+}
+
+pub(crate) fn define() -> Definitions {
+    let mut all_instructions = AllInstructions::new();
+
+    let immediates = Immediates::new();
+    let entities = EntityRefs::new();
+    let formats = Formats::new(&immediates, &entities);
+    let instructions =
+        instructions::define(&mut all_instructions, &formats, &immediates, &entities);
+    let transform_groups = legalize::define(&instructions, &immediates);
+
+    Definitions {
+        settings: settings::define(),
+        all_instructions,
+        instructions,
+        imm: immediates,
+        formats,
+        transform_groups,
+        entities,
+    }
+}
+
+impl Definitions {
+    /// Verifies certain properties of formats.
+    ///
+    /// - Formats must be uniquely named: if two formats have the same name, they must refer to the
+    /// same data. Otherwise, two format variants in the codegen crate would have the same name.
+    /// - Formats must be structurally different from each other. Otherwise, this would lead to
+    /// code duplicate in the codegen crate.
+    ///
+    /// Returns a list of all the instruction formats effectively used.
+    pub fn verify_instruction_formats(&self) -> Vec<&InstructionFormat> {
+        let mut format_names: HashMap<&'static str, &Rc<InstructionFormat>> = HashMap::new();
+
+        // A structure is: number of input value operands / whether there's varargs or not / names
+        // of immediate fields.
+        let mut format_structures: HashMap<FormatStructure, &InstructionFormat> = HashMap::new();
+
+        for inst in self.all_instructions.values() {
+            // Check name.
+            if let Some(existing_format) = format_names.get(&inst.format.name) {
+                assert!(
+                    Rc::ptr_eq(&existing_format, &inst.format),
+                    "formats must uniquely named; there's a\
+                     conflict on the name '{}', please make sure it is used only once.",
+                    existing_format.name
+                );
+            } else {
+                format_names.insert(inst.format.name, &inst.format);
+            }
+
+            // Check structure.
+            let key = inst.format.structure();
+            if let Some(existing_format) = format_structures.get(&key) {
+                assert_eq!(
+                    existing_format.name, inst.format.name,
+                    "duplicate instruction formats {} and {}; please remove one.",
+                    existing_format.name, inst.format.name
+                );
+            } else {
+                format_structures.insert(key, &inst.format);
+            }
+        }
+
+        let mut result = Vec::from_iter(format_structures.into_iter().map(|(_, v)| v));
+        result.sort_by_key(|format| format.name);
+        result
+    }
+}

cranelift/codegen/meta/src/shared/settings.rs

@@ -0,0 +1,235 @@
+use crate::cdsl::settings::{SettingGroup, SettingGroupBuilder};
+
+pub(crate) fn define() -> SettingGroup {
+    let mut settings = SettingGroupBuilder::new("shared");
+
+    settings.add_enum(
+        "opt_level",
+        r#"
+        Optimization level:
+
+        - none: Minimise compile time by disabling most optimizations.
+        - speed: Generate the fastest possible code
+        - speed_and_size: like "speed", but also perform transformations
+          aimed at reducing code size.
+        "#,
+        vec!["none", "speed", "speed_and_size"],
+    );
+
+    settings.add_bool(
+        "enable_verifier",
+        r#"
+        Run the Cranelift IR verifier at strategic times during compilation.
+
+        This makes compilation slower but catches many bugs. The verifier is always enabled by
+        default, which is useful during development.
+        "#,
+        true,
+    );
+
+    // Note that Cranelift doesn't currently need an is_pie flag, because PIE is
+    // just PIC where symbols can't be pre-empted, which can be expressed with the
+    // `colocated` flag on external functions and global values.
+    settings.add_bool(
+        "is_pic",
+        "Enable Position-Independent Code generation",
+        false,
+    );
+
+    settings.add_bool(
+        "use_colocated_libcalls",
+        r#"
+            Use colocated libcalls.
+
+            Generate code that assumes that libcalls can be declared "colocated",
+            meaning they will be defined along with the current function, such that
+            they can use more efficient addressing.
+            "#,
+        false,
+    );
+
+    settings.add_bool(
+        "avoid_div_traps",
+        r#"
+            Generate explicit checks around native division instructions to avoid
+            their trapping.
+
+            This is primarily used by SpiderMonkey which doesn't install a signal
+            handler for SIGFPE, but expects a SIGILL trap for division by zero.
+
+            On ISAs like ARM where the native division instructions don't trap,
+            this setting has no effect - explicit checks are always inserted.
+            "#,
+        false,
+    );
+
+    settings.add_bool(
+        "enable_float",
+        r#"
+            Enable the use of floating-point instructions
+
+            Disabling use of floating-point instructions is not yet implemented.
+            "#,
+        true,
+    );
+
+    settings.add_bool(
+        "enable_nan_canonicalization",
+        r#"
+            Enable NaN canonicalization
+
+            This replaces NaNs with a single canonical value, for users requiring
+            entirely deterministic WebAssembly computation. This is not required
+            by the WebAssembly spec, so it is not enabled by default.
+            "#,
+        false,
+    );
+
+    settings.add_bool(
+        "enable_pinned_reg",
+        r#"Enable the use of the pinned register.
+
+        This register is excluded from register allocation, and is completely under the control of
+        the end-user. It is possible to read it via the get_pinned_reg instruction, and to set it
+        with the set_pinned_reg instruction.
+        "#,
+        false,
+    );
+
+    settings.add_bool(
+        "use_pinned_reg_as_heap_base",
+        r#"Use the pinned register as the heap base.
+
+        Enabling this requires the enable_pinned_reg setting to be set to true. It enables a custom
+        legalization of the `heap_addr` instruction so it will use the pinned register as the heap
+        base, instead of fetching it from a global value.
+
+        Warning! Enabling this means that the pinned register *must* be maintained to contain the
+        heap base address at all times, during the lifetime of a function. Using the pinned
+        register for other purposes when this is set is very likely to cause crashes.
+        "#,
+        false,
+    );
+
+    settings.add_bool("enable_simd", "Enable the use of SIMD instructions.", false);
+
+    settings.add_bool(
+        "enable_atomics",
+        "Enable the use of atomic instructions",
+        true,
+    );
+
+    settings.add_bool(
+        "enable_safepoints",
+        r#"
+            Enable safepoint instruction insertions.
+
+            This will allow the emit_stackmaps() function to insert the safepoint
+            instruction on top of calls and interrupt traps in order to display the
+            live reference values at that point in the program.
+            "#,
+        false,
+    );
+
+    settings.add_enum(
+        "tls_model",
+        r#"
+            Defines the model used to perform TLS accesses.
+        "#,
+        vec!["none", "elf_gd", "macho", "coff"],
+    );
+
+    // Settings specific to the `baldrdash` calling convention.
+
+    settings.add_enum(
+        "libcall_call_conv",
+        r#"
+            Defines the calling convention to use for LibCalls call expansion,
+            since it may be different from the ISA default calling convention.
+
+            The default value is to use the same calling convention as the ISA
+            default calling convention.
+
+            This list should be kept in sync with the list of calling
+            conventions available in isa/call_conv.rs.
+        "#,
+        vec![
+            "isa_default",
+            "fast",
+            "cold",
+            "system_v",
+            "windows_fastcall",
+            "baldrdash_system_v",
+            "baldrdash_windows",
+            "probestack",
+        ],
+    );
+
+    settings.add_num(
+        "baldrdash_prologue_words",
+        r#"
+            Number of pointer-sized words pushed by the baldrdash prologue.
+
+            Functions with the `baldrdash` calling convention don't generate their
+            own prologue and epilogue. They depend on externally generated code
+            that pushes a fixed number of words in the prologue and restores them
+            in the epilogue.
+
+            This setting configures the number of pointer-sized words pushed on the
+            stack when the Cranelift-generated code is entered. This includes the
+            pushed return address on x86.
+            "#,
+        0,
+    );
+
+    // BaldrMonkey requires that not-yet-relocated function addresses be encoded
+    // as all-ones bitpatterns.
+    settings.add_bool(
+        "emit_all_ones_funcaddrs",
+        "Emit not-yet-relocated function addresses as all-ones bit patterns.",
+        false,
+    );
+
+    // Stack probing options.
+
+    settings.add_bool(
+        "enable_probestack",
+        r#"
+            Enable the use of stack probes, for calling conventions which support this
+            functionality.
+            "#,
+        true,
+    );
+
+    settings.add_bool(
+        "probestack_func_adjusts_sp",
+        r#"
+            Set this to true of the stack probe function modifies the stack pointer
+            itself.
+            "#,
+        false,
+    );
+
+    settings.add_num(
+        "probestack_size_log2",
+        r#"
+            The log2 of the size of the stack guard region.
+
+            Stack frames larger than this size will have stack overflow checked
+            by calling the probestack function.
+
+            The default is 12, which translates to a size of 4096.
+            "#,
+        12,
+    );
+
+    // Jump table options.
+
+    settings.add_bool(
+        "enable_jump_tables",
+        "Enable the use of jump tables in generated machine code.",
+        true,
+    );
+
+    settings.build()
+}

cranelift/codegen/meta/src/shared/types.rs

@@ -0,0 +1,236 @@
+//! This module predefines all the Cranelift scalar types.
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
+pub(crate) enum Bool {
+    /// 1-bit bool.
+    B1 = 1,
+    /// 8-bit bool.
+    B8 = 8,
+    /// 16-bit bool.
+    B16 = 16,
+    /// 32-bit bool.
+    B32 = 32,
+    /// 64-bit bool.
+    B64 = 64,
+    /// 128-bit bool.
+    B128 = 128,
+}
+
+/// This provides an iterator through all of the supported bool variants.
+pub(crate) struct BoolIterator {
+    index: u8,
+}
+
+impl BoolIterator {
+    pub fn new() -> Self {
+        Self { index: 0 }
+    }
+}
+
+impl Iterator for BoolIterator {
+    type Item = Bool;
+    fn next(&mut self) -> Option<Self::Item> {
+        let res = match self.index {
+            0 => Some(Bool::B1),
+            1 => Some(Bool::B8),
+            2 => Some(Bool::B16),
+            3 => Some(Bool::B32),
+            4 => Some(Bool::B64),
+            5 => Some(Bool::B128),
+            _ => return None,
+        };
+        self.index += 1;
+        res
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
+pub(crate) enum Int {
+    /// 8-bit int.
+    I8 = 8,
+    /// 16-bit int.
+    I16 = 16,
+    /// 32-bit int.
+    I32 = 32,
+    /// 64-bit int.
+    I64 = 64,
+    /// 128-bit int.
+    I128 = 128,
+}
+
+/// This provides an iterator through all of the supported int variants.
+pub(crate) struct IntIterator {
+    index: u8,
+}
+
+impl IntIterator {
+    pub fn new() -> Self {
+        Self { index: 0 }
+    }
+}
+
+impl Iterator for IntIterator {
+    type Item = Int;
+    fn next(&mut self) -> Option<Self::Item> {
+        let res = match self.index {
+            0 => Some(Int::I8),
+            1 => Some(Int::I16),
+            2 => Some(Int::I32),
+            3 => Some(Int::I64),
+            4 => Some(Int::I128),
+            _ => return None,
+        };
+        self.index += 1;
+        res
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
+pub(crate) enum Float {
+    F32 = 32,
+    F64 = 64,
+}
+
+/// Iterator through the variants of the Float enum.
+pub(crate) struct FloatIterator {
+    index: u8,
+}
+
+impl FloatIterator {
+    pub fn new() -> Self {
+        Self { index: 0 }
+    }
+}
+
+/// This provides an iterator through all of the supported float variants.
+impl Iterator for FloatIterator {
+    type Item = Float;
+    fn next(&mut self) -> Option<Self::Item> {
+        let res = match self.index {
+            0 => Some(Float::F32),
+            1 => Some(Float::F64),
+            _ => return None,
+        };
+        self.index += 1;
+        res
+    }
+}
+
+/// A type representing CPU flags.
+///
+/// Flags can't be stored in memory.
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
+pub(crate) enum Flag {
+    /// CPU flags from an integer comparison.
+    IFlags,
+    /// CPU flags from a floating point comparison.
+    FFlags,
+}
+
+/// Iterator through the variants of the Flag enum.
+pub(crate) struct FlagIterator {
+    index: u8,
+}
+
+impl FlagIterator {
+    pub fn new() -> Self {
+        Self { index: 0 }
+    }
+}
+
+impl Iterator for FlagIterator {
+    type Item = Flag;
+    fn next(&mut self) -> Option<Self::Item> {
+        let res = match self.index {
+            0 => Some(Flag::IFlags),
+            1 => Some(Flag::FFlags),
+            _ => return None,
+        };
+        self.index += 1;
+        res
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
+pub(crate) enum Reference {
+    /// 32-bit reference.
+    R32 = 32,
+    /// 64-bit reference.
+    R64 = 64,
+}
+
+/// This provides an iterator through all of the supported reference variants.
+pub(crate) struct ReferenceIterator {
+    index: u8,
+}
+
+impl ReferenceIterator {
+    pub fn new() -> Self {
+        Self { index: 0 }
+    }
+}
+
+impl Iterator for ReferenceIterator {
+    type Item = Reference;
+    fn next(&mut self) -> Option<Self::Item> {
+        let res = match self.index {
+            0 => Some(Reference::R32),
+            1 => Some(Reference::R64),
+            _ => return None,
+        };
+        self.index += 1;
+        res
+    }
+}
+
+#[cfg(test)]
+mod iter_tests {
+    use super::*;
+
+    #[test]
+    fn bool_iter_works() {
+        let mut bool_iter = BoolIterator::new();
+        assert_eq!(bool_iter.next(), Some(Bool::B1));
+        assert_eq!(bool_iter.next(), Some(Bool::B8));
+        assert_eq!(bool_iter.next(), Some(Bool::B16));
+        assert_eq!(bool_iter.next(), Some(Bool::B32));
+        assert_eq!(bool_iter.next(), Some(Bool::B64));
+        assert_eq!(bool_iter.next(), Some(Bool::B128));
+        assert_eq!(bool_iter.next(), None);
+    }
+
+    #[test]
+    fn int_iter_works() {
+        let mut int_iter = IntIterator::new();
+        assert_eq!(int_iter.next(), Some(Int::I8));
+        assert_eq!(int_iter.next(), Some(Int::I16));
+        assert_eq!(int_iter.next(), Some(Int::I32));
+        assert_eq!(int_iter.next(), Some(Int::I64));
+        assert_eq!(int_iter.next(), Some(Int::I128));
+        assert_eq!(int_iter.next(), None);
+    }
+
+    #[test]
+    fn float_iter_works() {
+        let mut float_iter = FloatIterator::new();
+        assert_eq!(float_iter.next(), Some(Float::F32));
+        assert_eq!(float_iter.next(), Some(Float::F64));
+        assert_eq!(float_iter.next(), None);
+    }
+
+    #[test]
+    fn flag_iter_works() {
+        let mut flag_iter = FlagIterator::new();
+        assert_eq!(flag_iter.next(), Some(Flag::IFlags));
+        assert_eq!(flag_iter.next(), Some(Flag::FFlags));
+        assert_eq!(flag_iter.next(), None);
+    }
+
+    #[test]
+    fn reference_iter_works() {
+        let mut reference_iter = ReferenceIterator::new();
+        assert_eq!(reference_iter.next(), Some(Reference::R32));
+        assert_eq!(reference_iter.next(), Some(Reference::R64));
+        assert_eq!(reference_iter.next(), None);
+    }
+}

cranelift/codegen/meta/src/srcgen.rs

@@ -0,0 +1,484 @@
+//! Source code generator.
+//!
+//! The `srcgen` module contains generic helper routines and classes for
+//! generating source code.
+
+#![macro_use]
+
+use std::cmp;
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs;
+use std::io::Write;
+use std::path;
+
+use crate::error;
+
+static SHIFTWIDTH: usize = 4;
+
+/// A macro that simplifies the usage of the Formatter by allowing format
+/// strings.
+macro_rules! fmtln {
+    ($fmt:ident, $fmtstring:expr, $($fmtargs:expr),*) => {
+        $fmt.line(format!($fmtstring, $($fmtargs),*));
+    };
+
+    ($fmt:ident, $arg:expr) => {
+        $fmt.line($arg);
+    };
+
+    ($_:tt, $($args:expr),+) => {
+        compile_error!("This macro requires at least two arguments: the Formatter instance and a format string.");
+    };
+
+    ($_:tt) => {
+        compile_error!("This macro requires at least two arguments: the Formatter instance and a format string.");
+    };
+}
+
+pub(crate) struct Formatter {
+    indent: usize,
+    lines: Vec<String>,
+}
+
+impl Formatter {
+    /// Source code formatter class. Used to collect source code to be written
+    /// to a file, and keep track of indentation.
+    pub fn new() -> Self {
+        Self {
+            indent: 0,
+            lines: Vec::new(),
+        }
+    }
+
+    /// Increase current indentation level by one.
+    pub fn indent_push(&mut self) {
+        self.indent += 1;
+    }
+
+    /// Decrease indentation by one level.
+    pub fn indent_pop(&mut self) {
+        assert!(self.indent > 0, "Already at top level indentation");
+        self.indent -= 1;
+    }
+
+    pub fn indent<T, F: FnOnce(&mut Formatter) -> T>(&mut self, f: F) -> T {
+        self.indent_push();
+        let ret = f(self);
+        self.indent_pop();
+        ret
+    }
+
+    /// Get the current whitespace indentation in the form of a String.
+    fn get_indent(&self) -> String {
+        if self.indent == 0 {
+            String::new()
+        } else {
+            format!("{:-1$}", " ", self.indent * SHIFTWIDTH)
+        }
+    }
+
+    /// Get a string containing whitespace outdented one level. Used for
+    /// lines of code that are inside a single indented block.
+    fn get_outdent(&mut self) -> String {
+        self.indent_pop();
+        let s = self.get_indent();
+        self.indent_push();
+        s
+    }
+
+    /// Add an indented line.
+    pub fn line(&mut self, contents: impl AsRef<str>) {
+        let indented_line = format!("{}{}\n", self.get_indent(), contents.as_ref());
+        self.lines.push(indented_line);
+    }
+
+    /// Pushes an empty line.
+    pub fn empty_line(&mut self) {
+        self.lines.push("\n".to_string());
+    }
+
+    /// Emit a line outdented one level.
+    pub fn outdented_line(&mut self, s: &str) {
+        let new_line = format!("{}{}\n", self.get_outdent(), s);
+        self.lines.push(new_line);
+    }
+
+    /// Write `self.lines` to a file.
+    pub fn update_file(
+        &self,
+        filename: impl AsRef<str>,
+        directory: &str,
+    ) -> Result<(), error::Error> {
+        #[cfg(target_family = "windows")]
+        let path_str = format!("{}\\{}", directory, filename.as_ref());
+        #[cfg(not(target_family = "windows"))]
+        let path_str = format!("{}/{}", directory, filename.as_ref());
+
+        let path = path::Path::new(&path_str);
+        let mut f = fs::File::create(path)?;
+
+        for l in self.lines.iter().map(|l| l.as_bytes()) {
+            f.write_all(l)?;
+        }
+
+        Ok(())
+    }
+
+    /// Add one or more lines after stripping common indentation.
+    pub fn multi_line(&mut self, s: &str) {
+        parse_multiline(s).into_iter().for_each(|l| self.line(&l));
+    }
+
+    /// Add a comment line.
+    pub fn comment(&mut self, s: impl AsRef<str>) {
+        fmtln!(self, "// {}", s.as_ref());
+    }
+
+    /// Add a (multi-line) documentation comment.
+    pub fn doc_comment(&mut self, contents: impl AsRef<str>) {
+        parse_multiline(contents.as_ref())
+            .iter()
+            .map(|l| {
+                if l.is_empty() {
+                    "///".into()
+                } else {
+                    format!("/// {}", l)
+                }
+            })
+            .for_each(|s| self.line(s.as_str()));
+    }
+
+    /// Add a match expression.
+    pub fn add_match(&mut self, m: Match) {
+        fmtln!(self, "match {} {{", m.expr);
+        self.indent(|fmt| {
+            for (&(ref fields, ref body), ref names) in m.arms.iter() {
+                // name { fields } | name { fields } => { body }
+                let conditions = names
+                    .iter()
+                    .map(|name| {
+                        if !fields.is_empty() {
+                            format!("{} {{ {} }}", name, fields.join(", "))
+                        } else {
+                            name.clone()
+                        }
+                    })
+                    .collect::<Vec<_>>()
+                    .join(" |\n")
+                    + " => {";
+
+                fmt.multi_line(&conditions);
+                fmt.indent(|fmt| {
+                    fmt.line(body);
+                });
+                fmt.line("}");
+            }
+
+            // Make sure to include the catch all clause last.
+            if let Some(body) = m.catch_all {
+                fmt.line("_ => {");
+                fmt.indent(|fmt| {
+                    fmt.line(body);
+                });
+                fmt.line("}");
+            }
+        });
+        self.line("}");
+    }
+}
+
+/// Compute the indentation of s, or None of an empty line.
+fn _indent(s: &str) -> Option<usize> {
+    if s.is_empty() {
+        None
+    } else {
+        let t = s.trim_start();
+        Some(s.len() - t.len())
+    }
+}
+
+/// Given a multi-line string, split it into a sequence of lines after
+/// stripping a common indentation. This is useful for strings defined with
+/// doc strings.
+fn parse_multiline(s: &str) -> Vec<String> {
+    // Convert tabs into spaces.
+    let expanded_tab = format!("{:-1$}", " ", SHIFTWIDTH);
+    let lines: Vec<String> = s.lines().map(|l| l.replace("\t", &expanded_tab)).collect();
+
+    // Determine minimum indentation, ignoring the first line and empty lines.
+    let indent = lines
+        .iter()
+        .skip(1)
+        .filter(|l| !l.trim().is_empty())
+        .map(|l| l.len() - l.trim_start().len())
+        .min();
+
+    // Strip off leading blank lines.
+    let mut lines_iter = lines.iter().skip_while(|l| l.is_empty());
+    let mut trimmed = Vec::with_capacity(lines.len());
+
+    // Remove indentation (first line is special)
+    if let Some(s) = lines_iter.next().map(|l| l.trim()).map(|l| l.to_string()) {
+        trimmed.push(s);
+    }
+
+    // Remove trailing whitespace from other lines.
+    let mut other_lines = if let Some(indent) = indent {
+        // Note that empty lines may have fewer than `indent` chars.
+        lines_iter
+            .map(|l| &l[cmp::min(indent, l.len())..])
+            .map(|l| l.trim_end())
+            .map(|l| l.to_string())
+            .collect::<Vec<_>>()
+    } else {
+        lines_iter
+            .map(|l| l.trim_end())
+            .map(|l| l.to_string())
+            .collect::<Vec<_>>()
+    };
+
+    trimmed.append(&mut other_lines);
+
+    // Strip off trailing blank lines.
+    while let Some(s) = trimmed.pop() {
+        if s.is_empty() {
+            continue;
+        } else {
+            trimmed.push(s);
+            break;
+        }
+    }
+
+    trimmed
+}
+
+/// Match formatting class.
+///
+/// Match objects collect all the information needed to emit a Rust `match`
+/// expression, automatically deduplicating overlapping identical arms.
+///
+/// Note that this class is ignorant of Rust types, and considers two fields
+/// with the same name to be equivalent. BTreeMap/BTreeSet are used to
+/// represent the arms in order to make the order deterministic.
+pub(crate) struct Match {
+    expr: String,
+    arms: BTreeMap<(Vec<String>, String), BTreeSet<String>>,
+    /// The clause for the placeholder pattern _.
+    catch_all: Option<String>,
+}
+
+impl Match {
+    /// Create a new match statement on `expr`.
+    pub fn new(expr: impl Into<String>) -> Self {
+        Self {
+            expr: expr.into(),
+            arms: BTreeMap::new(),
+            catch_all: None,
+        }
+    }
+
+    fn set_catch_all(&mut self, clause: String) {
+        assert!(self.catch_all.is_none());
+        self.catch_all = Some(clause);
+    }
+
+    /// Add an arm that reads fields to the Match statement.
+    pub fn arm<T: Into<String>, S: Into<String>>(&mut self, name: T, fields: Vec<S>, body: T) {
+        let name = name.into();
+        assert!(
+            name != "_",
+            "catch all clause can't extract fields, use arm_no_fields instead."
+        );
+
+        let body = body.into();
+        let fields = fields.into_iter().map(|x| x.into()).collect();
+        let match_arm = self
+            .arms
+            .entry((fields, body))
+            .or_insert_with(BTreeSet::new);
+        match_arm.insert(name);
+    }
+
+    /// Adds an arm that doesn't read anythings from the fields to the Match statement.
+    pub fn arm_no_fields(&mut self, name: impl Into<String>, body: impl Into<String>) {
+        let body = body.into();
+
+        let name = name.into();
+        if name == "_" {
+            self.set_catch_all(body);
+            return;
+        }
+
+        let match_arm = self
+            .arms
+            .entry((Vec::new(), body))
+            .or_insert_with(BTreeSet::new);
+        match_arm.insert(name);
+    }
+}
+
+#[cfg(test)]
+mod srcgen_tests {
+    use super::parse_multiline;
+    use super::Formatter;
+    use super::Match;
+
+    fn from_raw_string<S: Into<String>>(s: S) -> Vec<String> {
+        s.into()
+            .trim()
+            .split("\n")
+            .into_iter()
+            .map(|x| format!("{}\n", x))
+            .collect()
+    }
+
+    #[test]
+    fn adding_arms_works() {
+        let mut m = Match::new("x");
+        m.arm("Orange", vec!["a", "b"], "some body");
+        m.arm("Yellow", vec!["a", "b"], "some body");
+        m.arm("Green", vec!["a", "b"], "different body");
+        m.arm("Blue", vec!["x", "y"], "some body");
+        assert_eq!(m.arms.len(), 3);
+
+        let mut fmt = Formatter::new();
+        fmt.add_match(m);
+
+        let expected_lines = from_raw_string(
+            r#"
+match x {
+    Green { a, b } => {
+        different body
+    }
+    Orange { a, b } |
+    Yellow { a, b } => {
+        some body
+    }
+    Blue { x, y } => {
+        some body
+    }
+}
+        "#,
+        );
+        assert_eq!(fmt.lines, expected_lines);
+    }
+
+    #[test]
+    fn match_with_catchall_order() {
+        // The catchall placeholder must be placed after other clauses.
+        let mut m = Match::new("x");
+        m.arm("Orange", vec!["a", "b"], "some body");
+        m.arm("Green", vec!["a", "b"], "different body");
+        m.arm_no_fields("_", "unreachable!()");
+        assert_eq!(m.arms.len(), 2); // catchall is not counted
+
+        let mut fmt = Formatter::new();
+        fmt.add_match(m);
+
+        let expected_lines = from_raw_string(
+            r#"
+match x {
+    Green { a, b } => {
+        different body
+    }
+    Orange { a, b } => {
+        some body
+    }
+    _ => {
+        unreachable!()
+    }
+}
+        "#,
+        );
+        assert_eq!(fmt.lines, expected_lines);
+    }
+
+    #[test]
+    fn parse_multiline_works() {
+        let input = "\n    hello\n    world\n";
+        let expected = vec!["hello", "world"];
+        let output = parse_multiline(input);
+        assert_eq!(output, expected);
+    }
+
+    #[test]
+    fn formatter_basic_example_works() {
+        let mut fmt = Formatter::new();
+        fmt.line("Hello line 1");
+        fmt.indent_push();
+        fmt.comment("Nested comment");
+        fmt.indent_pop();
+        fmt.line("Back home again");
+        let expected_lines = vec![
+            "Hello line 1\n",
+            "    // Nested comment\n",
+            "Back home again\n",
+        ];
+        assert_eq!(fmt.lines, expected_lines);
+    }
+
+    #[test]
+    fn get_indent_works() {
+        let mut fmt = Formatter::new();
+        let expected_results = vec!["", "    ", "        ", ""];
+
+        let actual_results = Vec::with_capacity(4);
+        (0..3).for_each(|_| {
+            fmt.get_indent();
+            fmt.indent_push();
+        });
+        (0..3).for_each(|_| fmt.indent_pop());
+        fmt.get_indent();
+
+        actual_results
+            .into_iter()
+            .zip(expected_results.into_iter())
+            .for_each(|(actual, expected): (String, &str)| assert_eq!(&actual, expected));
+    }
+
+    #[test]
+    fn fmt_can_add_type_to_lines() {
+        let mut fmt = Formatter::new();
+        fmt.line(format!("pub const {}: Type = Type({:#x});", "example", 0,));
+        let expected_lines = vec!["pub const example: Type = Type(0x0);\n"];
+        assert_eq!(fmt.lines, expected_lines);
+    }
+
+    #[test]
+    fn fmt_can_add_indented_line() {
+        let mut fmt = Formatter::new();
+        fmt.line("hello");
+        fmt.indent_push();
+        fmt.line("world");
+        let expected_lines = vec!["hello\n", "    world\n"];
+        assert_eq!(fmt.lines, expected_lines);
+    }
+
+    #[test]
+    fn fmt_can_add_doc_comments() {
+        let mut fmt = Formatter::new();
+        fmt.doc_comment("documentation\nis\ngood");
+        let expected_lines = vec!["/// documentation\n", "/// is\n", "/// good\n"];
+        assert_eq!(fmt.lines, expected_lines);
+    }
+
+    #[test]
+    fn fmt_can_add_doc_comments_with_empty_lines() {
+        let mut fmt = Formatter::new();
+        fmt.doc_comment(
+            r#"documentation
+        can be really good.
+
+        If you stick to writing it.
+"#,
+        );
+        let expected_lines = from_raw_string(
+            r#"
+/// documentation
+/// can be really good.
+///
+/// If you stick to writing it."#,
+        );
+        assert_eq!(fmt.lines, expected_lines);
+    }
+}

cranelift/codegen/meta/src/unique_table.rs

@@ -0,0 +1,141 @@
+//! An index-accessed table implementation that avoids duplicate entries.
+use std::collections::HashMap;
+use std::hash::Hash;
+use std::slice;
+
+/// Collect items into the `table` list, removing duplicates.
+pub(crate) struct UniqueTable<'entries, T: Eq + Hash> {
+    table: Vec<&'entries T>,
+    map: HashMap<&'entries T, usize>,
+}
+
+impl<'entries, T: Eq + Hash> UniqueTable<'entries, T> {
+    pub fn new() -> Self {
+        Self {
+            table: Vec::new(),
+            map: HashMap::new(),
+        }
+    }
+
+    pub fn add(&mut self, entry: &'entries T) -> usize {
+        match self.map.get(&entry) {
+            None => {
+                let i = self.table.len();
+                self.table.push(entry);
+                self.map.insert(entry, i);
+                i
+            }
+            Some(&i) => i,
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.table.len()
+    }
+    pub fn get(&self, index: usize) -> &T {
+        self.table[index]
+    }
+    pub fn iter(&self) -> slice::Iter<&'entries T> {
+        self.table.iter()
+    }
+}
+
+/// A table of sequences which tries to avoid common subsequences.
+pub(crate) struct UniqueSeqTable<T: PartialEq + Clone> {
+    table: Vec<T>,
+}
+
+impl<T: PartialEq + Clone> UniqueSeqTable<T> {
+    pub fn new() -> Self {
+        Self { table: Vec::new() }
+    }
+    pub fn add(&mut self, values: &[T]) -> usize {
+        if values.is_empty() {
+            return 0;
+        }
+        if let Some(offset) = find_subsequence(values, &self.table) {
+            offset
+        } else {
+            let table_len = self.table.len();
+
+            // Try to put in common the last elements of the table if they're a prefix of the new
+            // sequence.
+            //
+            // We know there wasn't a full match, so the best prefix we can hope to find contains
+            // all the values but the last one.
+            let mut start_from = usize::min(table_len, values.len() - 1);
+            while start_from != 0 {
+                // Loop invariant: start_from <= table_len, so table_len - start_from >= 0.
+                if values[0..start_from] == self.table[table_len - start_from..table_len] {
+                    break;
+                }
+                start_from -= 1;
+            }
+
+            self.table
+                .extend(values[start_from..values.len()].iter().cloned());
+            table_len - start_from
+        }
+    }
+    pub fn len(&self) -> usize {
+        self.table.len()
+    }
+    pub fn iter(&self) -> slice::Iter<T> {
+        self.table.iter()
+    }
+}
+
+/// Try to find the subsequence `sub` in the `whole` sequence. Returns None if
+/// it's not been found, or Some(index) if it has been. Naive implementation
+/// until proven we need something better.
+fn find_subsequence<T: PartialEq>(sub: &[T], whole: &[T]) -> Option<usize> {
+    assert!(!sub.is_empty());
+    // We want i + sub.len() <= whole.len(), i.e. i < whole.len() + 1 - sub.len().
+    if whole.len() < sub.len() {
+        return None;
+    }
+    let max = whole.len() - sub.len();
+    for i in 0..=max {
+        if whole[i..i + sub.len()] == sub[..] {
+            return Some(i);
+        }
+    }
+    None
+}
+
+#[test]
+fn test_find_subsequence() {
+    assert_eq!(find_subsequence(&vec![1], &vec![4]), None);
+    assert_eq!(find_subsequence(&vec![1], &vec![1]), Some(0));
+    assert_eq!(find_subsequence(&vec![1, 2], &vec![1]), None);
+    assert_eq!(find_subsequence(&vec![1, 2], &vec![1, 2]), Some(0));
+    assert_eq!(find_subsequence(&vec![1, 2], &vec![1, 3]), None);
+    assert_eq!(find_subsequence(&vec![1, 2], &vec![0, 1, 2]), Some(1));
+    assert_eq!(find_subsequence(&vec![1, 2], &vec![0, 1, 3, 1]), None);
+    assert_eq!(find_subsequence(&vec![1, 2], &vec![0, 1, 3, 1, 2]), Some(3));
+    assert_eq!(
+        find_subsequence(&vec![1, 1, 3], &vec![1, 1, 1, 3, 3]),
+        Some(1)
+    );
+}
+
+#[test]
+fn test_optimal_add() {
+    let mut seq_table = UniqueSeqTable::new();
+    // [0, 1, 2, 3]
+    assert_eq!(seq_table.add(&vec![0, 1, 2, 3]), 0);
+    assert_eq!(seq_table.add(&vec![0, 1, 2, 3]), 0);
+    assert_eq!(seq_table.add(&vec![1, 2, 3]), 1);
+    assert_eq!(seq_table.add(&vec![2, 3]), 2);
+    assert_eq!(seq_table.len(), 4);
+    // [0, 1, 2, 3, 4]
+    assert_eq!(seq_table.add(&vec![2, 3, 4]), 2);
+    assert_eq!(seq_table.len(), 5);
+    // [0, 1, 2, 3, 4, 6, 5, 7]
+    assert_eq!(seq_table.add(&vec![4, 6, 5, 7]), 4);
+    assert_eq!(seq_table.len(), 8);
+    // [0, 1, 2, 3, 4, 6, 5, 7, 8, 2, 3, 4]
+    assert_eq!(seq_table.add(&vec![8, 2, 3, 4]), 8);
+    assert_eq!(seq_table.add(&vec![8]), 8);
+    assert_eq!(seq_table.len(), 12);
+}

cranelift/codegen/shared/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+authors = ["The Cranelift Project Developers"]
+name = "cranelift-codegen-shared"
+version = "0.59.0"
+description = "For code shared between cranelift-codegen-meta and cranelift-codegen"
+license = "Apache-2.0 WITH LLVM-exception"
+repository = "https://github.com/bytecodealliance/cranelift"
+readme = "README.md"
+edition = "2018"
+
+# Since this is a shared dependency of several packages, please strive to keep this dependency-free.

cranelift/codegen/shared/LICENSE

@@ -0,0 +1,220 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+--- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+

cranelift/codegen/shared/README.md

@@ -0,0 +1,2 @@
+This crate contains shared definitions for use in both `cranelift-codegen-meta` and `cranelift
+-codegen`.

cranelift/codegen/shared/src/condcodes.rs

@@ -0,0 +1,405 @@
+//! Condition codes for the Cranelift code generator.
+//!
+//! A condition code here is an enumerated type that determined how to compare two numbers. There
+//! are different rules for comparing integers and floating point numbers, so they use different
+//! condition codes.
+
+use core::fmt::{self, Display, Formatter};
+use core::str::FromStr;
+
+/// Common traits of condition codes.
+pub trait CondCode: Copy {
+    /// Get the inverse condition code of `self`.
+    ///
+    /// The inverse condition code produces the opposite result for all comparisons.
+    /// That is, `cmp CC, x, y` is true if and only if `cmp CC.inverse(), x, y` is false.
+    #[must_use]
+    fn inverse(self) -> Self;
+
+    /// Get the reversed condition code for `self`.
+    ///
+    /// The reversed condition code produces the same result as swapping `x` and `y` in the
+    /// comparison. That is, `cmp CC, x, y` is the same as `cmp CC.reverse(), y, x`.
+    #[must_use]
+    fn reverse(self) -> Self;
+}
+
+/// Condition code for comparing integers.
+///
+/// This condition code is used by the `icmp` instruction to compare integer values. There are
+/// separate codes for comparing the integers as signed or unsigned numbers where it makes a
+/// difference.
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
+pub enum IntCC {
+    /// `==`.
+    Equal,
+    /// `!=`.
+    NotEqual,
+    /// Signed `<`.
+    SignedLessThan,
+    /// Signed `>=`.
+    SignedGreaterThanOrEqual,
+    /// Signed `>`.
+    SignedGreaterThan,
+    /// Signed `<=`.
+    SignedLessThanOrEqual,
+    /// Unsigned `<`.
+    UnsignedLessThan,
+    /// Unsigned `>=`.
+    UnsignedGreaterThanOrEqual,
+    /// Unsigned `>`.
+    UnsignedGreaterThan,
+    /// Unsigned `<=`.
+    UnsignedLessThanOrEqual,
+    /// Signed Overflow.
+    Overflow,
+    /// Signed No Overflow.
+    NotOverflow,
+}
+
+impl CondCode for IntCC {
+    fn inverse(self) -> Self {
+        use self::IntCC::*;
+        match self {
+            Equal => NotEqual,
+            NotEqual => Equal,
+            SignedLessThan => SignedGreaterThanOrEqual,
+            SignedGreaterThanOrEqual => SignedLessThan,
+            SignedGreaterThan => SignedLessThanOrEqual,
+            SignedLessThanOrEqual => SignedGreaterThan,
+            UnsignedLessThan => UnsignedGreaterThanOrEqual,
+            UnsignedGreaterThanOrEqual => UnsignedLessThan,
+            UnsignedGreaterThan => UnsignedLessThanOrEqual,
+            UnsignedLessThanOrEqual => UnsignedGreaterThan,
+            Overflow => NotOverflow,
+            NotOverflow => Overflow,
+        }
+    }
+
+    fn reverse(self) -> Self {
+        use self::IntCC::*;
+        match self {
+            Equal => Equal,
+            NotEqual => NotEqual,
+            SignedGreaterThan => SignedLessThan,
+            SignedGreaterThanOrEqual => SignedLessThanOrEqual,
+            SignedLessThan => SignedGreaterThan,
+            SignedLessThanOrEqual => SignedGreaterThanOrEqual,
+            UnsignedGreaterThan => UnsignedLessThan,
+            UnsignedGreaterThanOrEqual => UnsignedLessThanOrEqual,
+            UnsignedLessThan => UnsignedGreaterThan,
+            UnsignedLessThanOrEqual => UnsignedGreaterThanOrEqual,
+            Overflow => Overflow,
+            NotOverflow => NotOverflow,
+        }
+    }
+}
+
+impl IntCC {
+    /// Get the corresponding IntCC with the equal component removed.
+    /// For conditions without a zero component, this is a no-op.
+    pub fn without_equal(self) -> Self {
+        use self::IntCC::*;
+        match self {
+            SignedGreaterThan | SignedGreaterThanOrEqual => SignedGreaterThan,
+            SignedLessThan | SignedLessThanOrEqual => SignedLessThan,
+            UnsignedGreaterThan | UnsignedGreaterThanOrEqual => UnsignedGreaterThan,
+            UnsignedLessThan | UnsignedLessThanOrEqual => UnsignedLessThan,
+            _ => self,
+        }
+    }
+
+    /// Get the corresponding IntCC with the signed component removed.
+    /// For conditions without a signed component, this is a no-op.
+    pub fn unsigned(self) -> Self {
+        use self::IntCC::*;
+        match self {
+            SignedGreaterThan | UnsignedGreaterThan => UnsignedGreaterThan,
+            SignedGreaterThanOrEqual | UnsignedGreaterThanOrEqual => UnsignedGreaterThanOrEqual,
+            SignedLessThan | UnsignedLessThan => UnsignedLessThan,
+            SignedLessThanOrEqual | UnsignedLessThanOrEqual => UnsignedLessThanOrEqual,
+            _ => self,
+        }
+    }
+
+    /// Get the corresponding string condition code for the IntCC object.
+    pub fn to_static_str(self) -> &'static str {
+        use self::IntCC::*;
+        match self {
+            Equal => "eq",
+            NotEqual => "ne",
+            SignedGreaterThan => "sgt",
+            SignedGreaterThanOrEqual => "sge",
+            SignedLessThan => "slt",
+            SignedLessThanOrEqual => "sle",
+            UnsignedGreaterThan => "ugt",
+            UnsignedGreaterThanOrEqual => "uge",
+            UnsignedLessThan => "ult",
+            UnsignedLessThanOrEqual => "ule",
+            Overflow => "of",
+            NotOverflow => "nof",
+        }
+    }
+}
+
+impl Display for IntCC {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        f.write_str(self.to_static_str())
+    }
+}
+
+impl FromStr for IntCC {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        use self::IntCC::*;
+        match s {
+            "eq" => Ok(Equal),
+            "ne" => Ok(NotEqual),
+            "sge" => Ok(SignedGreaterThanOrEqual),
+            "sgt" => Ok(SignedGreaterThan),
+            "sle" => Ok(SignedLessThanOrEqual),
+            "slt" => Ok(SignedLessThan),
+            "uge" => Ok(UnsignedGreaterThanOrEqual),
+            "ugt" => Ok(UnsignedGreaterThan),
+            "ule" => Ok(UnsignedLessThanOrEqual),
+            "ult" => Ok(UnsignedLessThan),
+            "of" => Ok(Overflow),
+            "nof" => Ok(NotOverflow),
+            _ => Err(()),
+        }
+    }
+}
+
+/// Condition code for comparing floating point numbers.
+///
+/// This condition code is used by the `fcmp` instruction to compare floating point values. Two
+/// IEEE floating point values relate in exactly one of four ways:
+///
+/// 1. `UN` - unordered when either value is NaN.
+/// 2. `EQ` - equal numerical value.
+/// 3. `LT` - `x` is less than `y`.
+/// 4. `GT` - `x` is greater than `y`.
+///
+/// Note that `0.0` and `-0.0` relate as `EQ` because they both represent the number 0.
+///
+/// The condition codes described here are used to produce a single boolean value from the
+/// comparison. The 14 condition codes here cover every possible combination of the relation above
+/// except the impossible `!UN & !EQ & !LT & !GT` and the always true `UN | EQ | LT | GT`.
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
+pub enum FloatCC {
+    /// EQ | LT | GT
+    Ordered,
+    /// UN
+    Unordered,
+
+    /// EQ
+    Equal,
+    /// The C '!=' operator is the inverse of '==': `NotEqual`.
+    /// UN | LT | GT
+    NotEqual,
+    /// LT | GT
+    OrderedNotEqual,
+    /// UN | EQ
+    UnorderedOrEqual,
+
+    /// LT
+    LessThan,
+    /// LT | EQ
+    LessThanOrEqual,
+    /// GT
+    GreaterThan,
+    /// GT | EQ
+    GreaterThanOrEqual,
+
+    /// UN | LT
+    UnorderedOrLessThan,
+    /// UN | LT | EQ
+    UnorderedOrLessThanOrEqual,
+    /// UN | GT
+    UnorderedOrGreaterThan,
+    /// UN | GT | EQ
+    UnorderedOrGreaterThanOrEqual,
+}
+
+impl CondCode for FloatCC {
+    fn inverse(self) -> Self {
+        use self::FloatCC::*;
+        match self {
+            Ordered => Unordered,
+            Unordered => Ordered,
+            Equal => NotEqual,
+            NotEqual => Equal,
+            OrderedNotEqual => UnorderedOrEqual,
+            UnorderedOrEqual => OrderedNotEqual,
+            LessThan => UnorderedOrGreaterThanOrEqual,
+            LessThanOrEqual => UnorderedOrGreaterThan,
+            GreaterThan => UnorderedOrLessThanOrEqual,
+            GreaterThanOrEqual => UnorderedOrLessThan,
+            UnorderedOrLessThan => GreaterThanOrEqual,
+            UnorderedOrLessThanOrEqual => GreaterThan,
+            UnorderedOrGreaterThan => LessThanOrEqual,
+            UnorderedOrGreaterThanOrEqual => LessThan,
+        }
+    }
+    fn reverse(self) -> Self {
+        use self::FloatCC::*;
+        match self {
+            Ordered => Ordered,
+            Unordered => Unordered,
+            Equal => Equal,
+            NotEqual => NotEqual,
+            OrderedNotEqual => OrderedNotEqual,
+            UnorderedOrEqual => UnorderedOrEqual,
+            LessThan => GreaterThan,
+            LessThanOrEqual => GreaterThanOrEqual,
+            GreaterThan => LessThan,
+            GreaterThanOrEqual => LessThanOrEqual,
+            UnorderedOrLessThan => UnorderedOrGreaterThan,
+            UnorderedOrLessThanOrEqual => UnorderedOrGreaterThanOrEqual,
+            UnorderedOrGreaterThan => UnorderedOrLessThan,
+            UnorderedOrGreaterThanOrEqual => UnorderedOrLessThanOrEqual,
+        }
+    }
+}
+
+impl Display for FloatCC {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        use self::FloatCC::*;
+        f.write_str(match *self {
+            Ordered => "ord",
+            Unordered => "uno",
+            Equal => "eq",
+            NotEqual => "ne",
+            OrderedNotEqual => "one",
+            UnorderedOrEqual => "ueq",
+            LessThan => "lt",
+            LessThanOrEqual => "le",
+            GreaterThan => "gt",
+            GreaterThanOrEqual => "ge",
+            UnorderedOrLessThan => "ult",
+            UnorderedOrLessThanOrEqual => "ule",
+            UnorderedOrGreaterThan => "ugt",
+            UnorderedOrGreaterThanOrEqual => "uge",
+        })
+    }
+}
+
+impl FromStr for FloatCC {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        use self::FloatCC::*;
+        match s {
+            "ord" => Ok(Ordered),
+            "uno" => Ok(Unordered),
+            "eq" => Ok(Equal),
+            "ne" => Ok(NotEqual),
+            "one" => Ok(OrderedNotEqual),
+            "ueq" => Ok(UnorderedOrEqual),
+            "lt" => Ok(LessThan),
+            "le" => Ok(LessThanOrEqual),
+            "gt" => Ok(GreaterThan),
+            "ge" => Ok(GreaterThanOrEqual),
+            "ult" => Ok(UnorderedOrLessThan),
+            "ule" => Ok(UnorderedOrLessThanOrEqual),
+            "ugt" => Ok(UnorderedOrGreaterThan),
+            "uge" => Ok(UnorderedOrGreaterThanOrEqual),
+            _ => Err(()),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::string::ToString;
+
+    static INT_ALL: [IntCC; 12] = [
+        IntCC::Equal,
+        IntCC::NotEqual,
+        IntCC::SignedLessThan,
+        IntCC::SignedGreaterThanOrEqual,
+        IntCC::SignedGreaterThan,
+        IntCC::SignedLessThanOrEqual,
+        IntCC::UnsignedLessThan,
+        IntCC::UnsignedGreaterThanOrEqual,
+        IntCC::UnsignedGreaterThan,
+        IntCC::UnsignedLessThanOrEqual,
+        IntCC::Overflow,
+        IntCC::NotOverflow,
+    ];
+
+    #[test]
+    fn int_inverse() {
+        for r in &INT_ALL {
+            let cc = *r;
+            let inv = cc.inverse();
+            assert!(cc != inv);
+            assert_eq!(inv.inverse(), cc);
+        }
+    }
+
+    #[test]
+    fn int_reverse() {
+        for r in &INT_ALL {
+            let cc = *r;
+            let rev = cc.reverse();
+            assert_eq!(rev.reverse(), cc);
+        }
+    }
+
+    #[test]
+    fn int_display() {
+        for r in &INT_ALL {
+            let cc = *r;
+            assert_eq!(cc.to_string().parse(), Ok(cc));
+        }
+        assert_eq!("bogus".parse::<IntCC>(), Err(()));
+    }
+
+    static FLOAT_ALL: [FloatCC; 14] = [
+        FloatCC::Ordered,
+        FloatCC::Unordered,
+        FloatCC::Equal,
+        FloatCC::NotEqual,
+        FloatCC::OrderedNotEqual,
+        FloatCC::UnorderedOrEqual,
+        FloatCC::LessThan,
+        FloatCC::LessThanOrEqual,
+        FloatCC::GreaterThan,
+        FloatCC::GreaterThanOrEqual,
+        FloatCC::UnorderedOrLessThan,
+        FloatCC::UnorderedOrLessThanOrEqual,
+        FloatCC::UnorderedOrGreaterThan,
+        FloatCC::UnorderedOrGreaterThanOrEqual,
+    ];
+
+    #[test]
+    fn float_inverse() {
+        for r in &FLOAT_ALL {
+            let cc = *r;
+            let inv = cc.inverse();
+            assert!(cc != inv);
+            assert_eq!(inv.inverse(), cc);
+        }
+    }
+
+    #[test]
+    fn float_reverse() {
+        for r in &FLOAT_ALL {
+            let cc = *r;
+            let rev = cc.reverse();
+            assert_eq!(rev.reverse(), cc);
+        }
+    }
+
+    #[test]
+    fn float_display() {
+        for r in &FLOAT_ALL {
+            let cc = *r;
+            assert_eq!(cc.to_string().parse(), Ok(cc));
+        }
+        assert_eq!("bogus".parse::<FloatCC>(), Err(()));
+    }
+}

cranelift/codegen/shared/src/constant_hash.rs

@@ -0,0 +1,81 @@
+//! Build support for precomputed constant hash tables.
+//!
+//! This module can generate constant hash tables using open addressing and quadratic probing.
+//!
+//! The hash tables are arrays that are guaranteed to:
+//!
+//! - Have a power-of-two size.
+//! - Contain at least one empty slot.
+//!
+//! This module provides build meta support for lookups in these tables, as well as the shared hash
+//! function used for probing.
+
+use std::iter;
+
+/// A primitive hash function for matching opcodes.
+pub fn simple_hash(s: &str) -> usize {
+    let mut h: u32 = 5381;
+    for c in s.chars() {
+        h = (h ^ c as u32).wrapping_add(h.rotate_right(6));
+    }
+    h as usize
+}
+
+/// Compute an open addressed, quadratically probed hash table containing
+/// `items`. The returned table is a list containing the elements of the
+/// iterable `items` and `None` in unused slots.
+#[allow(clippy::float_arithmetic)]
+pub fn generate_table<'cont, T, I: iter::Iterator<Item = &'cont T>, H: Fn(&T) -> usize>(
+    items: I,
+    num_items: usize,
+    hash_function: H,
+) -> Vec<Option<&'cont T>> {
+    let size = (1.20 * num_items as f64) as usize;
+
+    // Probing code's stop condition relies on the table having one vacant entry at least.
+    let size = if size.is_power_of_two() {
+        size * 2
+    } else {
+        size.next_power_of_two()
+    };
+
+    let mut table = vec![None; size];
+
+    for i in items {
+        let mut h = hash_function(&i) % size;
+        let mut s = 0;
+        while table[h].is_some() {
+            s += 1;
+            h = (h + s) % size;
+        }
+        table[h] = Some(i);
+    }
+
+    table
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{generate_table, simple_hash};
+
+    #[test]
+    fn basic() {
+        assert_eq!(simple_hash("Hello"), 0x2fa70c01);
+        assert_eq!(simple_hash("world"), 0x5b0c31d5);
+    }
+
+    #[test]
+    fn test_generate_table() {
+        let v = vec!["Hello".to_string(), "world".to_string()];
+        let table = generate_table(v.iter(), v.len(), |s| simple_hash(&s));
+        assert_eq!(
+            table,
+            vec![
+                None,
+                Some(&"Hello".to_string()),
+                Some(&"world".to_string()),
+                None
+            ]
+        );
+    }
+}

cranelift/codegen/shared/src/constants.rs

@@ -0,0 +1,30 @@
+//! This module contains constants that are shared between the codegen and the meta crate, so they
+//! are kept in sync.
+
+// Numbering scheme for value types:
+//
+// 0: Void
+// 0x01-0x6f: Special types
+// 0x70-0x7d: Lane types
+// 0x7e-0x7f: Reference types
+// 0x80-0xff: Vector types
+//
+// Vector types are encoded with the lane type in the low 4 bits and log2(lanes)
+// in the high 4 bits, giving a range of 2-256 lanes.
+
+/// Start of the lane types.
+pub const LANE_BASE: u8 = 0x70;
+
+/// Base for reference types.
+pub const REFERENCE_BASE: u8 = 0x7E;
+
+/// Start of the 2-lane vector types.
+pub const VECTOR_BASE: u8 = 0x80;
+
+// Some constants about register classes and types.
+
+/// Guaranteed maximum number of top-level register classes with pressure tracking in any ISA.
+pub const MAX_TRACKED_TOP_RCS: usize = 4;
+
+/// Guaranteed maximum number of register classes in any ISA.
+pub const MAX_NUM_REG_CLASSES: usize = 32;

cranelift/codegen/shared/src/isa/mod.rs

@@ -0,0 +1,3 @@
+//! Shared ISA-specific definitions.
+
+pub mod x86;

cranelift/codegen/shared/src/isa/x86/encoding_bits.rs

@@ -0,0 +1,419 @@
+//! Provides a named interface to the `u16` Encoding bits.
+
+use std::ops::RangeInclusive;
+
+/// Named interface to the `u16` Encoding bits, representing an opcode.
+///
+/// Cranelift requires each recipe to have a single encoding size in bytes.
+/// X86 opcodes are variable length, so we use separate recipes for different
+/// styles of opcodes and prefixes. The opcode format is indicated by the
+/// recipe name prefix.
+///
+/// VEX/XOP and EVEX prefixes are not yet supported.
+/// Encodings using any of these prefixes are represented by separate recipes.
+///
+/// The encoding bits are:
+///
+/// 0-7:   The opcode byte <op>.
+/// 8-9:   pp, mandatory prefix:
+///        00: none (Op*)
+///        01: 66   (Mp*)
+///        10: F3   (Mp*)
+///        11: F2   (Mp*)
+/// 10-11: mm, opcode map:
+///        00: <op>        (Op1/Mp1)
+///        01: 0F <op>     (Op2/Mp2)
+///        10: 0F 38 <op>  (Op3/Mp3)
+///        11: 0F 3A <op>  (Op3/Mp3)
+/// 12-14  rrr, opcode bits for the ModR/M byte for certain opcodes.
+/// 15:    REX.W bit (or VEX.W/E)
+#[derive(Copy, Clone, PartialEq)]
+pub struct EncodingBits(u16);
+const OPCODE: RangeInclusive<u16> = 0..=7;
+const OPCODE_PREFIX: RangeInclusive<u16> = 8..=11; // Includes pp and mm.
+const RRR: RangeInclusive<u16> = 12..=14;
+const REX_W: RangeInclusive<u16> = 15..=15;
+
+impl From<u16> for EncodingBits {
+    fn from(bits: u16) -> Self {
+        Self(bits)
+    }
+}
+
+impl EncodingBits {
+    /// Constructs a new EncodingBits from parts.
+    pub fn new(op_bytes: &[u8], rrr: u16, rex_w: u16) -> Self {
+        assert!(
+            !op_bytes.is_empty(),
+            "op_bytes must include at least one opcode byte"
+        );
+        let mut new = Self::from(0);
+        let last_byte = op_bytes[op_bytes.len() - 1];
+        new.write(OPCODE, last_byte as u16);
+        let prefix: u8 = OpcodePrefix::from_opcode(op_bytes).into();
+        new.write(OPCODE_PREFIX, prefix as u16);
+        new.write(RRR, rrr);
+        new.write(REX_W, rex_w);
+        new
+    }
+
+    /// Returns a copy of the EncodingBits with the RRR bits set.
+    #[inline]
+    pub fn with_rrr(mut self, rrr: u8) -> Self {
+        debug_assert_eq!(self.rrr(), 0);
+        self.write(RRR, rrr.into());
+        self
+    }
+
+    /// Returns a copy of the EncodingBits with the REX.W bit set.
+    #[inline]
+    pub fn with_rex_w(mut self) -> Self {
+        debug_assert_eq!(self.rex_w(), 0);
+        self.write(REX_W, 1);
+        self
+    }
+
+    /// Returns the raw bits.
+    #[inline]
+    pub fn bits(self) -> u16 {
+        self.0
+    }
+
+    /// Convenience method for writing bits to specific range.
+    #[inline]
+    fn write(&mut self, range: RangeInclusive<u16>, value: u16) {
+        assert!(ExactSizeIterator::len(&range) > 0);
+        let size = range.end() - range.start() + 1; // Calculate the number of bits in the range.
+        let mask = (1 << size) - 1; // Generate a bit mask.
+        debug_assert!(
+            value <= mask,
+            "The written value should have fewer than {} bits.",
+            size
+        );
+        let mask_complement = !(mask << *range.start()); // Create the bitwise complement for the clear mask.
+        self.0 &= mask_complement; // Clear the bits in `range`.
+        let value = (value & mask) << *range.start(); // Place the value in the correct location.
+        self.0 |= value; // Modify the bits in `range`.
+    }
+
+    /// Convenience method for reading bits from a specific range.
+    #[inline]
+    fn read(self, range: RangeInclusive<u16>) -> u8 {
+        assert!(ExactSizeIterator::len(&range) > 0);
+        let size = range.end() - range.start() + 1; // Calculate the number of bits in the range.
+        debug_assert!(size <= 8, "This structure expects ranges of at most 8 bits");
+        let mask = (1 << size) - 1; // Generate a bit mask.
+        ((self.0 >> *range.start()) & mask) as u8
+    }
+
+    /// Instruction opcode byte, without the prefix.
+    #[inline]
+    pub fn opcode_byte(self) -> u8 {
+        self.read(OPCODE)
+    }
+
+    /// Prefix kind for the instruction, as an enum.
+    #[inline]
+    pub fn prefix(self) -> OpcodePrefix {
+        OpcodePrefix::from(self.read(OPCODE_PREFIX))
+    }
+
+    /// Extracts the PP bits of the OpcodePrefix.
+    #[inline]
+    pub fn pp(self) -> u8 {
+        self.prefix().to_primitive() & 0x3
+    }
+
+    /// Extracts the MM bits of the OpcodePrefix.
+    #[inline]
+    pub fn mm(self) -> u8 {
+        (self.prefix().to_primitive() >> 2) & 0x3
+    }
+
+    /// Bits for the ModR/M byte for certain opcodes.
+    #[inline]
+    pub fn rrr(self) -> u8 {
+        self.read(RRR)
+    }
+
+    /// REX.W bit (or VEX.W/E).
+    #[inline]
+    pub fn rex_w(self) -> u8 {
+        self.read(REX_W)
+    }
+}
+
+/// Opcode prefix representation.
+///
+/// The prefix type occupies four of the EncodingBits.
+#[allow(non_camel_case_types)]
+#[allow(missing_docs)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum OpcodePrefix {
+    Op1,
+    Mp1_66,
+    Mp1_f3,
+    Mp1_f2,
+    Op2_0f,
+    Mp2_66_0f,
+    Mp2_f3_0f,
+    Mp2_f2_0f,
+    Op3_0f_38,
+    Mp3_66_0f_38,
+    Mp3_f3_0f_38,
+    Mp3_f2_0f_38,
+    Op3_0f_3a,
+    Mp3_66_0f_3a,
+    Mp3_f3_0f_3a,
+    Mp3_f2_0f_3a,
+}
+
+impl From<u8> for OpcodePrefix {
+    fn from(n: u8) -> Self {
+        use OpcodePrefix::*;
+        match n {
+            0b0000 => Op1,
+            0b0001 => Mp1_66,
+            0b0010 => Mp1_f3,
+            0b0011 => Mp1_f2,
+            0b0100 => Op2_0f,
+            0b0101 => Mp2_66_0f,
+            0b0110 => Mp2_f3_0f,
+            0b0111 => Mp2_f2_0f,
+            0b1000 => Op3_0f_38,
+            0b1001 => Mp3_66_0f_38,
+            0b1010 => Mp3_f3_0f_38,
+            0b1011 => Mp3_f2_0f_38,
+            0b1100 => Op3_0f_3a,
+            0b1101 => Mp3_66_0f_3a,
+            0b1110 => Mp3_f3_0f_3a,
+            0b1111 => Mp3_f2_0f_3a,
+            _ => panic!("invalid opcode prefix"),
+        }
+    }
+}
+
+impl Into<u8> for OpcodePrefix {
+    fn into(self) -> u8 {
+        use OpcodePrefix::*;
+        match self {
+            Op1 => 0b0000,
+            Mp1_66 => 0b0001,
+            Mp1_f3 => 0b0010,
+            Mp1_f2 => 0b0011,
+            Op2_0f => 0b0100,
+            Mp2_66_0f => 0b0101,
+            Mp2_f3_0f => 0b0110,
+            Mp2_f2_0f => 0b0111,
+            Op3_0f_38 => 0b1000,
+            Mp3_66_0f_38 => 0b1001,
+            Mp3_f3_0f_38 => 0b1010,
+            Mp3_f2_0f_38 => 0b1011,
+            Op3_0f_3a => 0b1100,
+            Mp3_66_0f_3a => 0b1101,
+            Mp3_f3_0f_3a => 0b1110,
+            Mp3_f2_0f_3a => 0b1111,
+        }
+    }
+}
+
+impl OpcodePrefix {
+    /// Convert an opcode prefix to a `u8`; this is a convenience proxy for `Into<u8>`.
+    fn to_primitive(self) -> u8 {
+        self.into()
+    }
+
+    /// Extracts the OpcodePrefix from the opcode.
+    pub fn from_opcode(op_bytes: &[u8]) -> Self {
+        assert!(!op_bytes.is_empty(), "at least one opcode byte");
+
+        let prefix_bytes = &op_bytes[..op_bytes.len() - 1];
+        match prefix_bytes {
+            [] => Self::Op1,
+            [0x66] => Self::Mp1_66,
+            [0xf3] => Self::Mp1_f3,
+            [0xf2] => Self::Mp1_f2,
+            [0x0f] => Self::Op2_0f,
+            [0x66, 0x0f] => Self::Mp2_66_0f,
+            [0xf3, 0x0f] => Self::Mp2_f3_0f,
+            [0xf2, 0x0f] => Self::Mp2_f2_0f,
+            [0x0f, 0x38] => Self::Op3_0f_38,
+            [0x66, 0x0f, 0x38] => Self::Mp3_66_0f_38,
+            [0xf3, 0x0f, 0x38] => Self::Mp3_f3_0f_38,
+            [0xf2, 0x0f, 0x38] => Self::Mp3_f2_0f_38,
+            [0x0f, 0x3a] => Self::Op3_0f_3a,
+            [0x66, 0x0f, 0x3a] => Self::Mp3_66_0f_3a,
+            [0xf3, 0x0f, 0x3a] => Self::Mp3_f3_0f_3a,
+            [0xf2, 0x0f, 0x3a] => Self::Mp3_f2_0f_3a,
+            _ => {
+                panic!("unexpected opcode sequence: {:?}", op_bytes);
+            }
+        }
+    }
+
+    /// Returns the recipe name prefix.
+    ///
+    /// At the moment, each similar OpcodePrefix group is given its own Recipe.
+    /// In order to distinguish them, this string is prefixed.
+    pub fn recipe_name_prefix(self) -> &'static str {
+        use OpcodePrefix::*;
+        match self {
+            Op1 => "Op1",
+            Op2_0f => "Op2",
+            Op3_0f_38 | Op3_0f_3a => "Op3",
+            Mp1_66 | Mp1_f3 | Mp1_f2 => "Mp1",
+            Mp2_66_0f | Mp2_f3_0f | Mp2_f2_0f => "Mp2",
+            Mp3_66_0f_38 | Mp3_f3_0f_38 | Mp3_f2_0f_38 => "Mp3",
+            Mp3_66_0f_3a | Mp3_f3_0f_3a | Mp3_f2_0f_3a => "Mp3",
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Helper function for prefix_roundtrip() to avoid long lines.
+    fn test_roundtrip(p: OpcodePrefix) {
+        assert_eq!(p, OpcodePrefix::from(p.to_primitive()));
+    }
+
+    /// Tests that to/from each opcode matches.
+    #[test]
+    fn prefix_roundtrip() {
+        test_roundtrip(OpcodePrefix::Op1);
+        test_roundtrip(OpcodePrefix::Mp1_66);
+        test_roundtrip(OpcodePrefix::Mp1_f3);
+        test_roundtrip(OpcodePrefix::Mp1_f2);
+        test_roundtrip(OpcodePrefix::Op2_0f);
+        test_roundtrip(OpcodePrefix::Mp2_66_0f);
+        test_roundtrip(OpcodePrefix::Mp2_f3_0f);
+        test_roundtrip(OpcodePrefix::Mp2_f2_0f);
+        test_roundtrip(OpcodePrefix::Op3_0f_38);
+        test_roundtrip(OpcodePrefix::Mp3_66_0f_38);
+        test_roundtrip(OpcodePrefix::Mp3_f3_0f_38);
+        test_roundtrip(OpcodePrefix::Mp3_f2_0f_38);
+        test_roundtrip(OpcodePrefix::Op3_0f_3a);
+        test_roundtrip(OpcodePrefix::Mp3_66_0f_3a);
+        test_roundtrip(OpcodePrefix::Mp3_f3_0f_3a);
+        test_roundtrip(OpcodePrefix::Mp3_f2_0f_3a);
+    }
+
+    #[test]
+    fn prefix_to_name() {
+        assert_eq!(OpcodePrefix::Op1.recipe_name_prefix(), "Op1");
+        assert_eq!(OpcodePrefix::Op2_0f.recipe_name_prefix(), "Op2");
+        assert_eq!(OpcodePrefix::Op3_0f_38.recipe_name_prefix(), "Op3");
+        assert_eq!(OpcodePrefix::Mp1_66.recipe_name_prefix(), "Mp1");
+        assert_eq!(OpcodePrefix::Mp2_66_0f.recipe_name_prefix(), "Mp2");
+        assert_eq!(OpcodePrefix::Mp3_66_0f_3a.recipe_name_prefix(), "Mp3");
+    }
+
+    /// Tests that the opcode_byte is the lower of the EncodingBits.
+    #[test]
+    fn encodingbits_opcode_byte() {
+        let enc = EncodingBits::from(0x00ff);
+        assert_eq!(enc.opcode_byte(), 0xff);
+        assert_eq!(enc.prefix().to_primitive(), 0x0);
+        assert_eq!(enc.rrr(), 0x0);
+        assert_eq!(enc.rex_w(), 0x0);
+
+        let enc = EncodingBits::from(0x00cd);
+        assert_eq!(enc.opcode_byte(), 0xcd);
+    }
+
+    /// Tests that the OpcodePrefix is encoded correctly.
+    #[test]
+    fn encodingbits_prefix() {
+        let enc = EncodingBits::from(0x0c00);
+        assert_eq!(enc.opcode_byte(), 0x00);
+        assert_eq!(enc.prefix().to_primitive(), 0xc);
+        assert_eq!(enc.prefix(), OpcodePrefix::Op3_0f_3a);
+        assert_eq!(enc.rrr(), 0x0);
+        assert_eq!(enc.rex_w(), 0x0);
+    }
+
+    /// Tests that the PP bits are encoded correctly.
+    #[test]
+    fn encodingbits_pp() {
+        let enc = EncodingBits::from(0x0300);
+        assert_eq!(enc.opcode_byte(), 0x0);
+        assert_eq!(enc.pp(), 0x3);
+        assert_eq!(enc.mm(), 0x0);
+        assert_eq!(enc.rrr(), 0x0);
+        assert_eq!(enc.rex_w(), 0x0);
+    }
+
+    /// Tests that the MM bits are encoded correctly.
+    #[test]
+    fn encodingbits_mm() {
+        let enc = EncodingBits::from(0x0c00);
+        assert_eq!(enc.opcode_byte(), 0x0);
+        assert_eq!(enc.pp(), 0x00);
+        assert_eq!(enc.mm(), 0x3);
+        assert_eq!(enc.rrr(), 0x0);
+        assert_eq!(enc.rex_w(), 0x0);
+    }
+
+    /// Tests that the ModR/M bits are encoded correctly.
+    #[test]
+    fn encodingbits_rrr() {
+        let enc = EncodingBits::from(0x5000);
+        assert_eq!(enc.opcode_byte(), 0x0);
+        assert_eq!(enc.prefix().to_primitive(), 0x0);
+        assert_eq!(enc.rrr(), 0x5);
+        assert_eq!(enc.rex_w(), 0x0);
+    }
+
+    /// Tests that the REX.W bit is encoded correctly.
+    #[test]
+    fn encodingbits_rex_w() {
+        let enc = EncodingBits::from(0x8000);
+        assert_eq!(enc.opcode_byte(), 0x00);
+        assert_eq!(enc.prefix().to_primitive(), 0x0);
+        assert_eq!(enc.rrr(), 0x0);
+        assert_eq!(enc.rex_w(), 0x1);
+    }
+
+    /// Tests setting and unsetting a bit using EncodingBits::write.
+    #[test]
+    fn encodingbits_flip() {
+        let mut bits = EncodingBits::from(0);
+        let range = 2..=2;
+
+        bits.write(range.clone(), 1);
+        assert_eq!(bits.bits(), 0b100);
+
+        bits.write(range, 0);
+        assert_eq!(bits.bits(), 0b000);
+    }
+
+    /// Tests a round-trip of EncodingBits from/to a u16 (hardcoded endianness).
+    #[test]
+    fn encodingbits_roundtrip() {
+        let bits: u16 = 0x1234;
+        assert_eq!(EncodingBits::from(bits).bits(), bits);
+    }
+
+    #[test]
+    // I purposely want to divide the bits using the ranges defined above.
+    #[allow(clippy::inconsistent_digit_grouping)]
+    fn encodingbits_construction() {
+        assert_eq!(
+            EncodingBits::new(&[0x66, 0x40], 5, 1).bits(),
+            0b1_101_0001_01000000 // 1 = rex_w, 101 = rrr, 0001 = prefix, 01000000 = opcode
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn encodingbits_panics_at_write_to_invalid_range() {
+        EncodingBits::from(0).write(1..=0, 42);
+    }
+
+    #[test]
+    #[should_panic]
+    fn encodingbits_panics_at_read_to_invalid_range() {
+        EncodingBits::from(0).read(1..=0);
+    }
+}

cranelift/codegen/shared/src/isa/x86/mod.rs

@@ -0,0 +1,4 @@
+//! Shared x86-specific definitions.
+
+mod encoding_bits;
+pub use encoding_bits::*;

cranelift/codegen/shared/src/lib.rs

@@ -0,0 +1,29 @@
+//! This library contains code that is common to both the `cranelift-codegen` and
+//! `cranelift-codegen-meta` libraries.
+
+#![deny(missing_docs, trivial_numeric_casts, unused_extern_crates)]
+#![warn(unused_import_braces)]
+#![cfg_attr(feature = "std", deny(unstable_features))]
+#![cfg_attr(feature = "clippy", plugin(clippy(conf_file = "../../clippy.toml")))]
+#![cfg_attr(feature = "cargo-clippy", allow(clippy::new_without_default))]
+#![cfg_attr(
+    feature = "cargo-clippy",
+    warn(
+        clippy::float_arithmetic,
+        clippy::mut_mut,
+        clippy::nonminimal_bool,
+        clippy::option_map_unwrap_or,
+        clippy::option_map_unwrap_or_else,
+        clippy::print_stdout,
+        clippy::unicode_not_nfc,
+        clippy::use_self
+    )
+)]
+
+pub mod condcodes;
+pub mod constant_hash;
+pub mod constants;
+pub mod isa;
+
+/// Version number of this crate.
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");

cranelift/codegen/src/abi.rs

@@ -0,0 +1,230 @@
+//! Common helper code for ABI lowering.
+//!
+//! This module provides functions and data structures that are useful for implementing the
+//! `TargetIsa::legalize_signature()` method.
+
+use crate::ir::{AbiParam, ArgumentExtension, ArgumentLoc, Type};
+use alloc::borrow::Cow;
+use alloc::vec::Vec;
+use core::cmp::Ordering;
+
+/// Legalization action to perform on a single argument or return value when converting a
+/// signature.
+///
+/// An argument may go through a sequence of legalization steps before it reaches the final
+/// `Assign` action.
+#[derive(Clone, Copy, Debug)]
+pub enum ArgAction {
+    /// Assign the argument to the given location.
+    Assign(ArgumentLoc),
+
+    /// Convert the argument, then call again.
+    ///
+    /// This action can split an integer type into two smaller integer arguments, or it can split a
+    /// SIMD vector into halves.
+    Convert(ValueConversion),
+}
+
+impl From<ArgumentLoc> for ArgAction {
+    fn from(x: ArgumentLoc) -> Self {
+        Self::Assign(x)
+    }
+}
+
+impl From<ValueConversion> for ArgAction {
+    fn from(x: ValueConversion) -> Self {
+        Self::Convert(x)
+    }
+}
+
+/// Legalization action to be applied to a value that is being passed to or from a legalized ABI.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ValueConversion {
+    /// Split an integer types into low and high parts, using `isplit`.
+    IntSplit,
+
+    /// Split a vector type into halves with identical lane types, using `vsplit`.
+    VectorSplit,
+
+    /// Bit-cast to an integer type of the same size.
+    IntBits,
+
+    /// Sign-extend integer value to the required type.
+    Sext(Type),
+
+    /// Unsigned zero-extend value to the required type.
+    Uext(Type),
+}
+
+impl ValueConversion {
+    /// Apply this conversion to a type, return the converted type.
+    pub fn apply(self, ty: Type) -> Type {
+        match self {
+            Self::IntSplit => ty.half_width().expect("Integer type too small to split"),
+            Self::VectorSplit => ty.half_vector().expect("Not a vector"),
+            Self::IntBits => Type::int(ty.bits()).expect("Bad integer size"),
+            Self::Sext(nty) | Self::Uext(nty) => nty,
+        }
+    }
+
+    /// Is this a split conversion that results in two arguments?
+    pub fn is_split(self) -> bool {
+        match self {
+            Self::IntSplit | Self::VectorSplit => true,
+            _ => false,
+        }
+    }
+}
+
+/// Common trait for assigning arguments to registers or stack locations.
+///
+/// This will be implemented by individual ISAs.
+pub trait ArgAssigner {
+    /// Pick an assignment action for function argument (or return value) `arg`.
+    fn assign(&mut self, arg: &AbiParam) -> ArgAction;
+}
+
+/// Legalize the arguments in `args` using the given argument assigner.
+///
+/// This function can be used for both arguments and return values.
+pub fn legalize_args<AA: ArgAssigner>(args: &[AbiParam], aa: &mut AA) -> Option<Vec<AbiParam>> {
+    let mut args = Cow::Borrowed(args);
+
+    // Iterate over the arguments.
+    // We may need to mutate the vector in place, so don't use a normal iterator, and clone the
+    // argument to avoid holding a reference.
+    let mut argno = 0;
+    while let Some(arg) = args.get(argno).cloned() {
+        // Leave the pre-assigned arguments alone.
+        // We'll assume that they don't interfere with our assignments.
+        if arg.location.is_assigned() {
+            argno += 1;
+            continue;
+        }
+
+        match aa.assign(&arg) {
+            // Assign argument to a location and move on to the next one.
+            ArgAction::Assign(loc) => {
+                args.to_mut()[argno].location = loc;
+                argno += 1;
+            }
+            // Split this argument into two smaller ones. Then revisit both.
+            ArgAction::Convert(conv) => {
+                let value_type = conv.apply(arg.value_type);
+                let new_arg = AbiParam { value_type, ..arg };
+                args.to_mut()[argno].value_type = value_type;
+                if conv.is_split() {
+                    args.to_mut().insert(argno + 1, new_arg);
+                }
+            }
+        }
+    }
+
+    match args {
+        Cow::Borrowed(_) => None,
+        Cow::Owned(a) => Some(a),
+    }
+}
+
+/// Determine the right action to take when passing a `have` value type to a call signature where
+/// the next argument is `arg` which has a different value type.
+///
+/// The signature legalization process in `legalize_args` above can replace a single argument value
+/// with multiple arguments of smaller types. It can also change the type of an integer argument to
+/// a larger integer type, requiring the smaller value to be sign- or zero-extended.
+///
+/// The legalizer needs to repair the values at all ABI boundaries:
+///
+/// - Incoming function arguments to the entry block.
+/// - Function arguments passed to a call.
+/// - Return values from a call.
+/// - Return values passed to a return instruction.
+///
+/// The `legalize_abi_value` function helps the legalizer with the process. When the legalizer
+/// needs to pass a pre-legalized `have` argument, but the ABI argument `arg` has a different value
+/// type, `legalize_abi_value(have, arg)` tells the legalizer how to create the needed value type
+/// for the argument.
+///
+/// It may be necessary to call `legalize_abi_value` more than once for a given argument before the
+/// desired argument type appears. This will happen when a vector or integer type needs to be split
+/// more than once, for example.
+pub fn legalize_abi_value(have: Type, arg: &AbiParam) -> ValueConversion {
+    let have_bits = have.bits();
+    let arg_bits = arg.value_type.bits();
+
+    match have_bits.cmp(&arg_bits) {
+        // We have fewer bits than the ABI argument.
+        Ordering::Less => {
+            debug_assert!(
+                have.is_int() && arg.value_type.is_int(),
+                "Can only extend integer values"
+            );
+            match arg.extension {
+                ArgumentExtension::Uext => ValueConversion::Uext(arg.value_type),
+                ArgumentExtension::Sext => ValueConversion::Sext(arg.value_type),
+                _ => panic!("No argument extension specified"),
+            }
+        }
+        // We have the same number of bits as the argument.
+        Ordering::Equal => {
+            // This must be an integer vector that is split and then extended.
+            debug_assert!(arg.value_type.is_int());
+            debug_assert!(have.is_vector(), "expected vector type, got {}", have);
+            ValueConversion::VectorSplit
+        }
+        // We have more bits than the argument.
+        Ordering::Greater => {
+            if have.is_vector() {
+                ValueConversion::VectorSplit
+            } else if have.is_float() {
+                // Convert a float to int so it can be split the next time.
+                // ARM would do this to pass an `f64` in two registers.
+                ValueConversion::IntBits
+            } else {
+                ValueConversion::IntSplit
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::ir::types;
+    use crate::ir::AbiParam;
+
+    #[test]
+    fn legalize() {
+        let mut arg = AbiParam::new(types::I32);
+
+        assert_eq!(
+            legalize_abi_value(types::I64X2, &arg),
+            ValueConversion::VectorSplit
+        );
+        assert_eq!(
+            legalize_abi_value(types::I64, &arg),
+            ValueConversion::IntSplit
+        );
+
+        // Vector of integers is broken down, then sign-extended.
+        arg.extension = ArgumentExtension::Sext;
+        assert_eq!(
+            legalize_abi_value(types::I16X4, &arg),
+            ValueConversion::VectorSplit
+        );
+        assert_eq!(
+            legalize_abi_value(types::I16.by(2).unwrap(), &arg),
+            ValueConversion::VectorSplit
+        );
+        assert_eq!(
+            legalize_abi_value(types::I16, &arg),
+            ValueConversion::Sext(types::I32)
+        );
+
+        // 64-bit float is split as an integer.
+        assert_eq!(
+            legalize_abi_value(types::F64, &arg),
+            ValueConversion::IntBits
+        );
+    }
+}

cranelift/codegen/src/binemit/memorysink.rs

@@ -0,0 +1,205 @@
+//! Code sink that writes binary machine code into contiguous memory.
+//!
+//! The `CodeSink` trait is the most general way of extracting binary machine code from Cranelift,
+//! and it is implemented by things like the `test binemit` file test driver to generate
+//! hexadecimal machine code. The `CodeSink` has some undesirable performance properties because of
+//! the dual abstraction: `TargetIsa` is a trait object implemented by each supported ISA, so it
+//! can't have any generic functions that could be specialized for each `CodeSink` implementation.
+//! This results in many virtual function callbacks (one per `put*` call) when
+//! `TargetIsa::emit_inst()` is used.
+//!
+//! The `MemoryCodeSink` type fixes the performance problem because it is a type known to
+//! `TargetIsa` so it can specialize its machine code generation for the type. The trade-off is
+//! that a `MemoryCodeSink` will always write binary machine code to raw memory. It forwards any
+//! relocations to a `RelocSink` trait object. Relocations are less frequent than the
+//! `CodeSink::put*` methods, so the performance impact of the virtual callbacks is less severe.
+use super::{Addend, CodeInfo, CodeOffset, CodeSink, Reloc};
+use crate::binemit::stackmap::Stackmap;
+use crate::ir::entities::Value;
+use crate::ir::{ConstantOffset, ExternalName, Function, JumpTable, SourceLoc, TrapCode};
+use crate::isa::TargetIsa;
+use core::ptr::write_unaligned;
+
+/// A `CodeSink` that writes binary machine code directly into memory.
+///
+/// A `MemoryCodeSink` object should be used when emitting a Cranelift IR function into executable
+/// memory. It writes machine code directly to a raw pointer without any bounds checking, so make
+/// sure to allocate enough memory for the whole function. The number of bytes required is returned
+/// by the `Context::compile()` function.
+///
+/// Any relocations in the function are forwarded to the `RelocSink` trait object.
+///
+/// Note that `MemoryCodeSink` writes multi-byte values in the native byte order of the host. This
+/// is not the right thing to do for cross compilation.
+pub struct MemoryCodeSink<'a> {
+    /// Pointer to start of sink's preallocated memory.
+    data: *mut u8,
+    /// Offset is isize because its major consumer needs it in that form.
+    offset: isize,
+    relocs: &'a mut dyn RelocSink,
+    traps: &'a mut dyn TrapSink,
+    stackmaps: &'a mut dyn StackmapSink,
+    /// Information about the generated code and read-only data.
+    pub info: CodeInfo,
+}
+
+impl<'a> MemoryCodeSink<'a> {
+    /// Create a new memory code sink that writes a function to the memory pointed to by `data`.
+    ///
+    /// # Safety
+    ///
+    /// This function is unsafe since `MemoryCodeSink` does not perform bounds checking on the
+    /// memory buffer, and it can't guarantee that the `data` pointer is valid.
+    pub unsafe fn new(
+        data: *mut u8,
+        relocs: &'a mut dyn RelocSink,
+        traps: &'a mut dyn TrapSink,
+        stackmaps: &'a mut dyn StackmapSink,
+    ) -> Self {
+        Self {
+            data,
+            offset: 0,
+            info: CodeInfo {
+                code_size: 0,
+                jumptables_size: 0,
+                rodata_size: 0,
+                total_size: 0,
+            },
+            relocs,
+            traps,
+            stackmaps,
+        }
+    }
+}
+
+/// A trait for receiving relocations for code that is emitted directly into memory.
+pub trait RelocSink {
+    /// Add a relocation referencing an block at the current offset.
+    fn reloc_block(&mut self, _: CodeOffset, _: Reloc, _: CodeOffset);
+
+    /// Add a relocation referencing an external symbol at the current offset.
+    fn reloc_external(&mut self, _: CodeOffset, _: Reloc, _: &ExternalName, _: Addend);
+
+    /// Add a relocation referencing a constant.
+    fn reloc_constant(&mut self, _: CodeOffset, _: Reloc, _: ConstantOffset);
+
+    /// Add a relocation referencing a jump table.
+    fn reloc_jt(&mut self, _: CodeOffset, _: Reloc, _: JumpTable);
+}
+
+/// A trait for receiving trap codes and offsets.
+///
+/// If you don't need information about possible traps, you can use the
+/// [`NullTrapSink`](NullTrapSink) implementation.
+pub trait TrapSink {
+    /// Add trap information for a specific offset.
+    fn trap(&mut self, _: CodeOffset, _: SourceLoc, _: TrapCode);
+}
+
+impl<'a> MemoryCodeSink<'a> {
+    fn write<T>(&mut self, x: T) {
+        unsafe {
+            #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))]
+            write_unaligned(self.data.offset(self.offset) as *mut T, x);
+            self.offset += core::mem::size_of::<T>() as isize;
+        }
+    }
+}
+
+impl<'a> CodeSink for MemoryCodeSink<'a> {
+    fn offset(&self) -> CodeOffset {
+        self.offset as CodeOffset
+    }
+
+    fn put1(&mut self, x: u8) {
+        self.write(x);
+    }
+
+    fn put2(&mut self, x: u16) {
+        self.write(x);
+    }
+
+    fn put4(&mut self, x: u32) {
+        self.write(x);
+    }
+
+    fn put8(&mut self, x: u64) {
+        self.write(x);
+    }
+
+    fn reloc_block(&mut self, rel: Reloc, block_offset: CodeOffset) {
+        let ofs = self.offset();
+        self.relocs.reloc_block(ofs, rel, block_offset);
+    }
+
+    fn reloc_external(&mut self, rel: Reloc, name: &ExternalName, addend: Addend) {
+        let ofs = self.offset();
+        self.relocs.reloc_external(ofs, rel, name, addend);
+    }
+
+    fn reloc_constant(&mut self, rel: Reloc, constant_offset: ConstantOffset) {
+        let ofs = self.offset();
+        self.relocs.reloc_constant(ofs, rel, constant_offset);
+    }
+
+    fn reloc_jt(&mut self, rel: Reloc, jt: JumpTable) {
+        let ofs = self.offset();
+        self.relocs.reloc_jt(ofs, rel, jt);
+    }
+
+    fn trap(&mut self, code: TrapCode, srcloc: SourceLoc) {
+        let ofs = self.offset();
+        self.traps.trap(ofs, srcloc, code);
+    }
+
+    fn begin_jumptables(&mut self) {
+        self.info.code_size = self.offset();
+    }
+
+    fn begin_rodata(&mut self) {
+        self.info.jumptables_size = self.offset() - self.info.code_size;
+    }
+
+    fn end_codegen(&mut self) {
+        self.info.rodata_size = self.offset() - (self.info.jumptables_size + self.info.code_size);
+        self.info.total_size = self.offset();
+    }
+
+    fn add_stackmap(&mut self, val_list: &[Value], func: &Function, isa: &dyn TargetIsa) {
+        let ofs = self.offset();
+        let stackmap = Stackmap::from_values(&val_list, func, isa);
+        self.stackmaps.add_stackmap(ofs, stackmap);
+    }
+}
+
+/// A `RelocSink` implementation that does nothing, which is convenient when
+/// compiling code that does not relocate anything.
+pub struct NullRelocSink {}
+
+impl RelocSink for NullRelocSink {
+    fn reloc_block(&mut self, _: u32, _: Reloc, _: u32) {}
+    fn reloc_external(&mut self, _: u32, _: Reloc, _: &ExternalName, _: i64) {}
+    fn reloc_constant(&mut self, _: CodeOffset, _: Reloc, _: ConstantOffset) {}
+    fn reloc_jt(&mut self, _: u32, _: Reloc, _: JumpTable) {}
+}
+
+/// A `TrapSink` implementation that does nothing, which is convenient when
+/// compiling code that does not rely on trapping semantics.
+pub struct NullTrapSink {}
+
+impl TrapSink for NullTrapSink {
+    fn trap(&mut self, _offset: CodeOffset, _srcloc: SourceLoc, _code: TrapCode) {}
+}
+
+/// A trait for emitting stackmaps.
+pub trait StackmapSink {
+    /// Output a bitmap of the stack representing the live reference variables at this code offset.
+    fn add_stackmap(&mut self, _: CodeOffset, _: Stackmap);
+}
+
+/// Placeholder StackmapSink that does nothing.
+pub struct NullStackmapSink {}
+
+impl StackmapSink for NullStackmapSink {
+    fn add_stackmap(&mut self, _: CodeOffset, _: Stackmap) {}
+}

cranelift/codegen/src/binemit/mod.rs

@@ -0,0 +1,246 @@
+//! Binary machine code emission.
+//!
+//! The `binemit` module contains code for translating Cranelift's intermediate representation into
+//! binary machine code.
+
+mod memorysink;
+mod relaxation;
+mod shrink;
+mod stackmap;
+
+pub use self::memorysink::{
+    MemoryCodeSink, NullRelocSink, NullStackmapSink, NullTrapSink, RelocSink, StackmapSink,
+    TrapSink,
+};
+pub use self::relaxation::relax_branches;
+pub use self::shrink::shrink_instructions;
+pub use self::stackmap::Stackmap;
+use crate::ir::entities::Value;
+use crate::ir::{ConstantOffset, ExternalName, Function, Inst, JumpTable, SourceLoc, TrapCode};
+use crate::isa::TargetIsa;
+pub use crate::regalloc::RegDiversions;
+use core::fmt;
+#[cfg(feature = "enable-serde")]
+use serde::{Deserialize, Serialize};
+
+/// Offset in bytes from the beginning of the function.
+///
+/// Cranelift can be used as a cross compiler, so we don't want to use a type like `usize` which
+/// depends on the *host* platform, not the *target* platform.
+pub type CodeOffset = u32;
+
+/// Addend to add to the symbol value.
+pub type Addend = i64;
+
+/// Relocation kinds for every ISA
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub enum Reloc {
+    /// absolute 4-byte
+    Abs4,
+    /// absolute 8-byte
+    Abs8,
+    /// x86 PC-relative 4-byte
+    X86PCRel4,
+    /// x86 PC-relative 4-byte offset to trailing rodata
+    X86PCRelRodata4,
+    /// x86 call to PC-relative 4-byte
+    X86CallPCRel4,
+    /// x86 call to PLT-relative 4-byte
+    X86CallPLTRel4,
+    /// x86 GOT PC-relative 4-byte
+    X86GOTPCRel4,
+    /// Arm32 call target
+    Arm32Call,
+    /// Arm64 call target
+    Arm64Call,
+    /// RISC-V call target
+    RiscvCall,
+
+    /// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol.
+    ElfX86_64TlsGd,
+
+    /// Mach-O x86_64 32 bit signed PC relative offset to a `__thread_vars` entry.
+    MachOX86_64Tlv,
+}
+
+impl fmt::Display for Reloc {
+    /// Display trait implementation drops the arch, since its used in contexts where the arch is
+    /// already unambiguous, e.g. clif syntax with isa specified. In other contexts, use Debug.
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            Self::Abs4 => write!(f, "Abs4"),
+            Self::Abs8 => write!(f, "Abs8"),
+            Self::X86PCRel4 => write!(f, "PCRel4"),
+            Self::X86PCRelRodata4 => write!(f, "PCRelRodata4"),
+            Self::X86CallPCRel4 => write!(f, "CallPCRel4"),
+            Self::X86CallPLTRel4 => write!(f, "CallPLTRel4"),
+            Self::X86GOTPCRel4 => write!(f, "GOTPCRel4"),
+            Self::Arm32Call | Self::Arm64Call | Self::RiscvCall => write!(f, "Call"),
+
+            Self::ElfX86_64TlsGd => write!(f, "ElfX86_64TlsGd"),
+            Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"),
+        }
+    }
+}
+
+/// Container for information about a vector of compiled code and its supporting read-only data.
+///
+/// The code starts at offset 0 and is followed optionally by relocatable jump tables and copyable
+/// (raw binary) read-only data.  Any padding between sections is always part of the section that
+/// precedes the boundary between the sections.
+#[derive(PartialEq)]
+pub struct CodeInfo {
+    /// Number of bytes of machine code (the code starts at offset 0).
+    pub code_size: CodeOffset,
+
+    /// Number of bytes of jumptables.
+    pub jumptables_size: CodeOffset,
+
+    /// Number of bytes of rodata.
+    pub rodata_size: CodeOffset,
+
+    /// Number of bytes in total.
+    pub total_size: CodeOffset,
+}
+
+impl CodeInfo {
+    /// Offset of any relocatable jump tables, or equal to rodata if there are no jump tables.
+    pub fn jumptables(&self) -> CodeOffset {
+        self.code_size
+    }
+
+    /// Offset of any copyable read-only data, or equal to total_size if there are no rodata.
+    pub fn rodata(&self) -> CodeOffset {
+        self.code_size + self.jumptables_size
+    }
+}
+
+/// Abstract interface for adding bytes to the code segment.
+///
+/// A `CodeSink` will receive all of the machine code for a function. It also accepts relocations
+/// which are locations in the code section that need to be fixed up when linking.
+pub trait CodeSink {
+    /// Get the current position.
+    fn offset(&self) -> CodeOffset;
+
+    /// Add 1 byte to the code section.
+    fn put1(&mut self, _: u8);
+
+    /// Add 2 bytes to the code section.
+    fn put2(&mut self, _: u16);
+
+    /// Add 4 bytes to the code section.
+    fn put4(&mut self, _: u32);
+
+    /// Add 8 bytes to the code section.
+    fn put8(&mut self, _: u64);
+
+    /// Add a relocation referencing an block at the current offset.
+    fn reloc_block(&mut self, _: Reloc, _: CodeOffset);
+
+    /// Add a relocation referencing an external symbol plus the addend at the current offset.
+    fn reloc_external(&mut self, _: Reloc, _: &ExternalName, _: Addend);
+
+    /// Add a relocation referencing a constant.
+    fn reloc_constant(&mut self, _: Reloc, _: ConstantOffset);
+
+    /// Add a relocation referencing a jump table.
+    fn reloc_jt(&mut self, _: Reloc, _: JumpTable);
+
+    /// Add trap information for the current offset.
+    fn trap(&mut self, _: TrapCode, _: SourceLoc);
+
+    /// Machine code output is complete, jump table data may follow.
+    fn begin_jumptables(&mut self);
+
+    /// Jump table output is complete, raw read-only data may follow.
+    fn begin_rodata(&mut self);
+
+    /// Read-only data output is complete, we're done.
+    fn end_codegen(&mut self);
+
+    /// Add a stackmap at the current code offset.
+    fn add_stackmap(&mut self, _: &[Value], _: &Function, _: &dyn TargetIsa);
+}
+
+/// Type of the frame unwind information.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum FrameUnwindKind {
+    /// Windows fastcall unwinding (as in .pdata).
+    Fastcall,
+    /// FDE entry for libunwind (similar to .eh_frame format).
+    Libunwind,
+}
+
+/// Offset in frame unwind information buffer.
+pub type FrameUnwindOffset = usize;
+
+/// Sink for frame unwind information.
+pub trait FrameUnwindSink {
+    /// Get the current position.
+    fn len(&self) -> FrameUnwindOffset;
+
+    /// Add bytes to the code section.
+    fn bytes(&mut self, _: &[u8]);
+
+    /// Reserves bytes in the buffer.
+    fn reserve(&mut self, _len: usize) {}
+
+    /// Add a relocation entry.
+    fn reloc(&mut self, _: Reloc, _: FrameUnwindOffset);
+
+    /// Specified offset to main structure.
+    fn set_entry_offset(&mut self, _: FrameUnwindOffset);
+}
+
+/// Report a bad encoding error.
+#[cold]
+pub fn bad_encoding(func: &Function, inst: Inst) -> ! {
+    panic!(
+        "Bad encoding {} for {}",
+        func.encodings[inst],
+        func.dfg.display_inst(inst, None)
+    );
+}
+
+/// Emit a function to `sink`, given an instruction emitter function.
+///
+/// This function is called from the `TargetIsa::emit_function()` implementations with the
+/// appropriate instruction emitter.
+pub fn emit_function<CS, EI>(func: &Function, emit_inst: EI, sink: &mut CS, isa: &dyn TargetIsa)
+where
+    CS: CodeSink,
+    EI: Fn(&Function, Inst, &mut RegDiversions, &mut CS, &dyn TargetIsa),
+{
+    let mut divert = RegDiversions::new();
+    for block in func.layout.blocks() {
+        divert.at_block(&func.entry_diversions, block);
+        debug_assert_eq!(func.offsets[block], sink.offset());
+        for inst in func.layout.block_insts(block) {
+            emit_inst(func, inst, &mut divert, sink, isa);
+        }
+    }
+
+    sink.begin_jumptables();
+
+    // Output jump tables.
+    for (jt, jt_data) in func.jump_tables.iter() {
+        let jt_offset = func.jt_offsets[jt];
+        for block in jt_data.iter() {
+            let rel_offset: i32 = func.offsets[*block] as i32 - jt_offset as i32;
+            sink.put4(rel_offset as u32)
+        }
+    }
+
+    sink.begin_rodata();
+
+    // Output constants.
+    for (_, constant_data) in func.dfg.constants.iter() {
+        for byte in constant_data.iter() {
+            sink.put1(*byte)
+        }
+    }
+
+    sink.end_codegen();
+}

cranelift/codegen/src/binemit/relaxation.rs

@@ -0,0 +1,393 @@
+//! Branch relaxation and offset computation.
+//!
+//! # block header offsets
+//!
+//! Before we can generate binary machine code for branch instructions, we need to know the final
+//! offsets of all the block headers in the function. This information is encoded in the
+//! `func.offsets` table.
+//!
+//! # Branch relaxation
+//!
+//! Branch relaxation is the process of ensuring that all branches in the function have enough
+//! range to encode their destination. It is common to have multiple branch encodings in an ISA.
+//! For example, x86 branches can have either an 8-bit or a 32-bit displacement.
+//!
+//! On RISC architectures, it can happen that conditional branches have a shorter range than
+//! unconditional branches:
+//!
+//! ```clif
+//!     brz v1, block17
+//! ```
+//!
+//! can be transformed into:
+//!
+//! ```clif
+//!     brnz v1, block23
+//!     jump block17
+//! block23:
+//! ```
+
+use crate::binemit::{CodeInfo, CodeOffset};
+use crate::cursor::{Cursor, FuncCursor};
+use crate::dominator_tree::DominatorTree;
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::{Block, Function, Inst, InstructionData, Opcode, Value, ValueList};
+use crate::isa::{EncInfo, TargetIsa};
+use crate::iterators::IteratorExtras;
+use crate::regalloc::RegDiversions;
+use crate::timing;
+use crate::CodegenResult;
+use core::convert::TryFrom;
+use log::debug;
+
+/// Relax branches and compute the final layout of block headers in `func`.
+///
+/// Fill in the `func.offsets` table so the function is ready for binary emission.
+pub fn relax_branches(
+    func: &mut Function,
+    _cfg: &mut ControlFlowGraph,
+    _domtree: &mut DominatorTree,
+    isa: &dyn TargetIsa,
+) -> CodegenResult<CodeInfo> {
+    let _tt = timing::relax_branches();
+
+    let encinfo = isa.encoding_info();
+
+    // Clear all offsets so we can recognize blocks that haven't been visited yet.
+    func.offsets.clear();
+    func.offsets.resize(func.dfg.num_blocks());
+
+    // Start by removing redundant jumps.
+    fold_redundant_jumps(func, _cfg, _domtree);
+
+    // Convert jumps to fallthrough instructions where possible.
+    fallthroughs(func);
+
+    let mut offset = 0;
+    let mut divert = RegDiversions::new();
+
+    // First, compute initial offsets for every block.
+    {
+        let mut cur = FuncCursor::new(func);
+        while let Some(block) = cur.next_block() {
+            divert.at_block(&cur.func.entry_diversions, block);
+            cur.func.offsets[block] = offset;
+            while let Some(inst) = cur.next_inst() {
+                divert.apply(&cur.func.dfg[inst]);
+                let enc = cur.func.encodings[inst];
+                offset += encinfo.byte_size(enc, inst, &divert, &cur.func);
+            }
+        }
+    }
+
+    // Then, run the relaxation algorithm until it converges.
+    let mut go_again = true;
+    while go_again {
+        go_again = false;
+        offset = 0;
+
+        // Visit all instructions in layout order.
+        let mut cur = FuncCursor::new(func);
+        while let Some(block) = cur.next_block() {
+            divert.at_block(&cur.func.entry_diversions, block);
+
+            // Record the offset for `block` and make sure we iterate until offsets are stable.
+            if cur.func.offsets[block] != offset {
+                cur.func.offsets[block] = offset;
+                go_again = true;
+            }
+
+            while let Some(inst) = cur.next_inst() {
+                divert.apply(&cur.func.dfg[inst]);
+
+                let enc = cur.func.encodings[inst];
+
+                // See if this is a branch has a range and a destination, and if the target is in
+                // range.
+                if let Some(range) = encinfo.branch_range(enc) {
+                    if let Some(dest) = cur.func.dfg[inst].branch_destination() {
+                        let dest_offset = cur.func.offsets[dest];
+                        if !range.contains(offset, dest_offset) {
+                            offset +=
+                                relax_branch(&mut cur, &divert, offset, dest_offset, &encinfo, isa);
+                            continue;
+                        }
+                    }
+                }
+
+                offset += encinfo.byte_size(enc, inst, &divert, &cur.func);
+            }
+        }
+    }
+
+    let code_size = offset;
+    let jumptables = offset;
+
+    for (jt, jt_data) in func.jump_tables.iter() {
+        func.jt_offsets[jt] = offset;
+        // TODO: this should be computed based on the min size needed to hold the furthest branch.
+        offset += jt_data.len() as u32 * 4;
+    }
+
+    let jumptables_size = offset - jumptables;
+    let rodata = offset;
+
+    for constant in func.dfg.constants.entries_mut() {
+        constant.set_offset(offset);
+        offset +=
+            u32::try_from(constant.len()).expect("Constants must have a length that fits in a u32")
+    }
+
+    let rodata_size = offset - rodata;
+
+    Ok(CodeInfo {
+        code_size,
+        jumptables_size,
+        rodata_size,
+        total_size: offset,
+    })
+}
+
+/// Folds an instruction if it is a redundant jump.
+/// Returns whether folding was performed (which invalidates the CFG).
+fn try_fold_redundant_jump(
+    func: &mut Function,
+    cfg: &mut ControlFlowGraph,
+    block: Block,
+    first_inst: Inst,
+) -> bool {
+    let first_dest = match func.dfg[first_inst].branch_destination() {
+        Some(block) => block, // The instruction was a single-target branch.
+        None => {
+            return false; // The instruction was either multi-target or not a branch.
+        }
+    };
+
+    // For the moment, only attempt to fold a branch to an block that is parameterless.
+    // These blocks are mainly produced by critical edge splitting.
+    //
+    // TODO: Allow folding blocks that define SSA values and function as phi nodes.
+    if func.dfg.num_block_params(first_dest) != 0 {
+        return false;
+    }
+
+    // Look at the first instruction of the first branch's destination.
+    // If it is an unconditional branch, maybe the second jump can be bypassed.
+    let second_inst = func.layout.first_inst(first_dest).expect("Instructions");
+    if func.dfg[second_inst].opcode() != Opcode::Jump {
+        return false;
+    }
+
+    // Now we need to fix up first_inst's block parameters to match second_inst's,
+    // without changing the branch-specific arguments.
+    //
+    // The intermediary block is allowed to reference any SSA value that dominates it,
+    // but that SSA value may not necessarily also dominate the instruction that's
+    // being patched.
+
+    // Get the arguments and parameters passed by the first branch.
+    let num_fixed = func.dfg[first_inst]
+        .opcode()
+        .constraints()
+        .num_fixed_value_arguments();
+    let (first_args, first_params) = func.dfg[first_inst]
+        .arguments(&func.dfg.value_lists)
+        .split_at(num_fixed);
+
+    // Get the parameters passed by the second jump.
+    let num_fixed = func.dfg[second_inst]
+        .opcode()
+        .constraints()
+        .num_fixed_value_arguments();
+    let (_, second_params) = func.dfg[second_inst]
+        .arguments(&func.dfg.value_lists)
+        .split_at(num_fixed);
+    let mut second_params = second_params.to_vec(); // Clone for rewriting below.
+
+    // For each parameter passed by the second jump, if any of those parameters
+    // was a block parameter, rewrite it to refer to the value that the first jump
+    // passed in its parameters. Otherwise, make sure it dominates first_inst.
+    //
+    // For example: if we `block0: jump block1(v1)` to `block1(v2): jump block2(v2)`,
+    // we want to rewrite the original jump to `jump block2(v1)`.
+    let block_params: &[Value] = func.dfg.block_params(first_dest);
+    debug_assert!(block_params.len() == first_params.len());
+
+    for value in second_params.iter_mut() {
+        if let Some((n, _)) = block_params.iter().enumerate().find(|(_, &p)| p == *value) {
+            // This value was the Nth parameter passed to the second_inst's block.
+            // Rewrite it as the Nth parameter passed by first_inst.
+            *value = first_params[n];
+        }
+    }
+
+    // Build a value list of first_args (unchanged) followed by second_params (rewritten).
+    let arguments_vec: alloc::vec::Vec<_> = first_args
+        .iter()
+        .chain(second_params.iter())
+        .copied()
+        .collect();
+    let value_list = ValueList::from_slice(&arguments_vec, &mut func.dfg.value_lists);
+
+    func.dfg[first_inst].take_value_list(); // Drop the current list.
+    func.dfg[first_inst].put_value_list(value_list); // Put the new list.
+
+    // Bypass the second jump.
+    // This can disconnect the Block containing `second_inst`, to be cleaned up later.
+    let second_dest = func.dfg[second_inst].branch_destination().expect("Dest");
+    func.change_branch_destination(first_inst, second_dest);
+    cfg.recompute_block(func, block);
+
+    // The previously-intermediary Block may now be unreachable. Update CFG.
+    if cfg.pred_iter(first_dest).count() == 0 {
+        // Remove all instructions from that block.
+        while let Some(inst) = func.layout.first_inst(first_dest) {
+            func.layout.remove_inst(inst);
+        }
+
+        // Remove the block...
+        cfg.recompute_block(func, first_dest); // ...from predecessor lists.
+        func.layout.remove_block(first_dest); // ...from the layout.
+    }
+
+    true
+}
+
+/// Redirects `jump` instructions that point to other `jump` instructions to the final destination.
+/// This transformation may orphan some blocks.
+fn fold_redundant_jumps(
+    func: &mut Function,
+    cfg: &mut ControlFlowGraph,
+    domtree: &mut DominatorTree,
+) {
+    let mut folded = false;
+
+    // Postorder iteration guarantees that a chain of jumps is visited from
+    // the end of the chain to the start of the chain.
+    for &block in domtree.cfg_postorder() {
+        // Only proceed if the first terminator instruction is a single-target branch.
+        let first_inst = func
+            .layout
+            .last_inst(block)
+            .expect("Block has no terminator");
+        folded |= try_fold_redundant_jump(func, cfg, block, first_inst);
+
+        // Also try the previous instruction.
+        if let Some(prev_inst) = func.layout.prev_inst(first_inst) {
+            folded |= try_fold_redundant_jump(func, cfg, block, prev_inst);
+        }
+    }
+
+    // Folding jumps invalidates the dominator tree.
+    if folded {
+        domtree.compute(func, cfg);
+    }
+}
+
+/// Convert `jump` instructions to `fallthrough` instructions where possible and verify that any
+/// existing `fallthrough` instructions are correct.
+fn fallthroughs(func: &mut Function) {
+    for (block, succ) in func.layout.blocks().adjacent_pairs() {
+        let term = func
+            .layout
+            .last_inst(block)
+            .expect("block has no terminator.");
+        if let InstructionData::Jump {
+            ref mut opcode,
+            destination,
+            ..
+        } = func.dfg[term]
+        {
+            match *opcode {
+                Opcode::Fallthrough => {
+                    // Somebody used a fall-through instruction before the branch relaxation pass.
+                    // Make sure it is correct, i.e. the destination is the layout successor.
+                    debug_assert_eq!(destination, succ, "Illegal fall-through in {}", block)
+                }
+                Opcode::Jump => {
+                    // If this is a jump to the successor block, change it to a fall-through.
+                    if destination == succ {
+                        *opcode = Opcode::Fallthrough;
+                        func.encodings[term] = Default::default();
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+}
+
+/// Relax the branch instruction at `cur` so it can cover the range `offset - dest_offset`.
+///
+/// Return the size of the replacement instructions up to and including the location where `cur` is
+/// left.
+fn relax_branch(
+    cur: &mut FuncCursor,
+    divert: &RegDiversions,
+    offset: CodeOffset,
+    dest_offset: CodeOffset,
+    encinfo: &EncInfo,
+    isa: &dyn TargetIsa,
+) -> CodeOffset {
+    let inst = cur.current_inst().unwrap();
+    debug!(
+        "Relaxing [{}] {} for {:#x}-{:#x} range",
+        encinfo.display(cur.func.encodings[inst]),
+        cur.func.dfg.display_inst(inst, isa),
+        offset,
+        dest_offset
+    );
+
+    // Pick the smallest encoding that can handle the branch range.
+    let dfg = &cur.func.dfg;
+    let ctrl_type = dfg.ctrl_typevar(inst);
+    if let Some(enc) = isa
+        .legal_encodings(cur.func, &dfg[inst], ctrl_type)
+        .filter(|&enc| {
+            let range = encinfo.branch_range(enc).expect("Branch with no range");
+            if !range.contains(offset, dest_offset) {
+                debug!("  trying [{}]: out of range", encinfo.display(enc));
+                false
+            } else if encinfo.operand_constraints(enc)
+                != encinfo.operand_constraints(cur.func.encodings[inst])
+            {
+                // Conservatively give up if the encoding has different constraints
+                // than the original, so that we don't risk picking a new encoding
+                // which the existing operands don't satisfy. We can't check for
+                // validity directly because we don't have a RegDiversions active so
+                // we don't know which registers are actually in use.
+                debug!("  trying [{}]: constraints differ", encinfo.display(enc));
+                false
+            } else {
+                debug!("  trying [{}]: OK", encinfo.display(enc));
+                true
+            }
+        })
+        .min_by_key(|&enc| encinfo.byte_size(enc, inst, &divert, &cur.func))
+    {
+        debug_assert!(enc != cur.func.encodings[inst]);
+        cur.func.encodings[inst] = enc;
+        return encinfo.byte_size(enc, inst, &divert, &cur.func);
+    }
+
+    // Note: On some RISC ISAs, conditional branches have shorter range than unconditional
+    // branches, so one way of extending the range of a conditional branch is to invert its
+    // condition and make it branch over an unconditional jump which has the larger range.
+    //
+    // Splitting the block is problematic this late because there may be register diversions in
+    // effect across the conditional branch, and they can't survive the control flow edge to a new
+    // block. We have two options for handling that:
+    //
+    // 1. Set a flag on the new block that indicates it wants the preserve the register diversions of
+    //    its layout predecessor, or
+    // 2. Use an encoding macro for the branch-over-jump pattern so we don't need to split the block.
+    //
+    // It seems that 1. would allow us to share code among RISC ISAs that need this.
+    //
+    // We can't allow register diversions to survive from the layout predecessor because the layout
+    // predecessor could contain kill points for some values that are live in this block, and
+    // diversions are not automatically cancelled when the live range of a value ends.
+
+    // This assumes solution 2. above:
+    panic!("No branch in range for {:#x}-{:#x}", offset, dest_offset);
+}

cranelift/codegen/src/binemit/shrink.rs

@@ -0,0 +1,73 @@
+//! Instruction shrinking.
+//!
+//! Sometimes there are multiple valid encodings for a given instruction. Cranelift often initially
+//! chooses the largest one, because this typically provides the register allocator the most
+//! flexibility. However, once register allocation is done, this is no longer important, and we
+//! can switch to smaller encodings when possible.
+
+use crate::ir::instructions::InstructionData;
+use crate::ir::Function;
+use crate::isa::TargetIsa;
+use crate::regalloc::RegDiversions;
+use crate::timing;
+use log::debug;
+
+/// Pick the smallest valid encodings for instructions.
+pub fn shrink_instructions(func: &mut Function, isa: &dyn TargetIsa) {
+    let _tt = timing::shrink_instructions();
+
+    let encinfo = isa.encoding_info();
+    let mut divert = RegDiversions::new();
+
+    for block in func.layout.blocks() {
+        // Load diversions from predecessors.
+        divert.at_block(&func.entry_diversions, block);
+
+        for inst in func.layout.block_insts(block) {
+            let enc = func.encodings[inst];
+            if enc.is_legal() {
+                // regmove/regfill/regspill are special instructions with register immediates
+                // that represented as normal operands, so the normal predicates below don't
+                // handle them correctly.
+                //
+                // Also, they need to be presented to the `RegDiversions` to update the
+                // location tracking.
+                //
+                // TODO: Eventually, we want the register allocator to avoid leaving these special
+                // instructions behind, but for now, just temporarily avoid trying to shrink them.
+                let inst_data = &func.dfg[inst];
+                match inst_data {
+                    InstructionData::RegMove { .. }
+                    | InstructionData::RegFill { .. }
+                    | InstructionData::RegSpill { .. } => {
+                        divert.apply(inst_data);
+                        continue;
+                    }
+                    _ => (),
+                }
+
+                let ctrl_type = func.dfg.ctrl_typevar(inst);
+
+                // Pick the last encoding with constraints that are satisfied.
+                let best_enc = isa
+                    .legal_encodings(func, &func.dfg[inst], ctrl_type)
+                    .filter(|e| encinfo.constraints[e.recipe()].satisfied(inst, &divert, &func))
+                    .min_by_key(|e| encinfo.byte_size(*e, inst, &divert, &func))
+                    .unwrap();
+
+                if best_enc != enc {
+                    func.encodings[inst] = best_enc;
+
+                    debug!(
+                        "Shrunk [{}] to [{}] in {}, reducing the size from {} to {}",
+                        encinfo.display(enc),
+                        encinfo.display(best_enc),
+                        func.dfg.display_inst(inst, isa),
+                        encinfo.byte_size(enc, inst, &divert, &func),
+                        encinfo.byte_size(best_enc, inst, &divert, &func)
+                    );
+                }
+            }
+        }
+    }
+}

cranelift/codegen/src/binemit/stackmap.rs

@@ -0,0 +1,150 @@
+use crate::bitset::BitSet;
+use crate::ir;
+use crate::isa::TargetIsa;
+use alloc::vec::Vec;
+
+type Num = u32;
+const NUM_BITS: usize = core::mem::size_of::<Num>() * 8;
+
+/// A stack map is a bitmap with one bit per machine word on the stack. Stack
+/// maps are created at `safepoint` instructions and record all live reference
+/// values that are on the stack. All slot kinds, except `OutgoingArg` are
+/// captured in a stack map. The `OutgoingArg`'s will be captured in the callee
+/// function as `IncomingArg`'s.
+///
+/// The first value in the bitmap is of the lowest addressed slot on the stack.
+/// As all stacks in Isa's supported by Cranelift grow down, this means that
+/// first value is of the top of the stack and values proceed down the stack.
+#[derive(Clone, Debug)]
+pub struct Stackmap {
+    bitmap: Vec<BitSet<Num>>,
+    mapped_words: u32,
+}
+
+impl Stackmap {
+    /// Create a stackmap based on where references are located on a function's stack.
+    pub fn from_values(
+        args: &[ir::entities::Value],
+        func: &ir::Function,
+        isa: &dyn TargetIsa,
+    ) -> Self {
+        let loc = &func.locations;
+        let mut live_ref_in_stack_slot = crate::HashSet::new();
+        // References can be in registers, and live registers values are pushed onto the stack before calls and traps.
+        // TODO: Implement register maps. If a register containing a reference is spilled and reused after a safepoint,
+        // it could contain a stale reference value if the garbage collector relocated the value.
+        for val in args {
+            if let Some(value_loc) = loc.get(*val) {
+                match *value_loc {
+                    ir::ValueLoc::Stack(stack_slot) => {
+                        live_ref_in_stack_slot.insert(stack_slot);
+                    }
+                    _ => {}
+                }
+            }
+        }
+
+        let stack = &func.stack_slots;
+        let info = func.stack_slots.layout_info.unwrap();
+
+        // Refer to the doc comment for `Stackmap` above to understand the
+        // bitmap representation used here.
+        let map_size = (info.frame_size + info.inbound_args_size) as usize;
+        let word_size = isa.pointer_bytes() as usize;
+        let num_words = map_size / word_size;
+
+        let mut vec = alloc::vec::Vec::with_capacity(num_words);
+        vec.resize(num_words, false);
+
+        for (ss, ssd) in stack.iter() {
+            if !live_ref_in_stack_slot.contains(&ss)
+                || ssd.kind == ir::stackslot::StackSlotKind::OutgoingArg
+            {
+                continue;
+            }
+
+            debug_assert!(ssd.size as usize == word_size);
+            let bytes_from_bottom = info.frame_size as i32 + ssd.offset.unwrap();
+            let words_from_bottom = (bytes_from_bottom as usize) / word_size;
+            vec[words_from_bottom] = true;
+        }
+
+        Self::from_slice(&vec)
+    }
+
+    /// Create a vec of Bitsets from a slice of bools.
+    pub fn from_slice(vec: &[bool]) -> Self {
+        let len = vec.len();
+        let num_word = len / NUM_BITS + (len % NUM_BITS != 0) as usize;
+        let mut bitmap = Vec::with_capacity(num_word);
+
+        for segment in vec.chunks(NUM_BITS) {
+            let mut curr_word = 0;
+            for (i, set) in segment.iter().enumerate() {
+                if *set {
+                    curr_word |= 1 << i;
+                }
+            }
+            bitmap.push(BitSet(curr_word));
+        }
+        Self {
+            mapped_words: len as u32,
+            bitmap,
+        }
+    }
+
+    /// Returns a specified bit.
+    pub fn get_bit(&self, bit_index: usize) -> bool {
+        assert!(bit_index < NUM_BITS * self.bitmap.len());
+        let word_index = bit_index / NUM_BITS;
+        let word_offset = (bit_index % NUM_BITS) as u8;
+        self.bitmap[word_index].contains(word_offset)
+    }
+
+    /// Returns the raw bitmap that represents this stack map.
+    pub fn as_slice(&self) -> &[BitSet<u32>] {
+        &self.bitmap
+    }
+
+    /// Returns the number of words represented by this stack map.
+    pub fn mapped_words(&self) -> u32 {
+        self.mapped_words
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn stackmaps() {
+        let vec: Vec<bool> = Vec::new();
+        assert!(Stackmap::from_slice(&vec).bitmap.is_empty());
+
+        let mut vec: [bool; NUM_BITS] = Default::default();
+        let set_true_idx = [5, 7, 24, 31];
+
+        for &idx in &set_true_idx {
+            vec[idx] = true;
+        }
+
+        let mut vec = vec.to_vec();
+        assert_eq!(
+            vec![BitSet::<Num>(2164261024)],
+            Stackmap::from_slice(&vec).bitmap
+        );
+
+        vec.push(false);
+        vec.push(true);
+        let res = Stackmap::from_slice(&vec);
+        assert_eq!(
+            vec![BitSet::<Num>(2164261024), BitSet::<Num>(2)],
+            res.bitmap
+        );
+
+        assert!(res.get_bit(5));
+        assert!(res.get_bit(31));
+        assert!(res.get_bit(33));
+        assert!(!res.get_bit(1));
+    }
+}

cranelift/codegen/src/bitset.rs

@@ -0,0 +1,161 @@
+//! Small Bitset
+//!
+//! This module defines a struct `BitSet<T>` encapsulating a bitset built over the type T.
+//! T is intended to be a primitive unsigned type. Currently it can be any type between u8 and u32
+//!
+//! If you would like to add support for larger bitsets in the future, you need to change the trait
+//! bound Into<u32> and the u32 in the implementation of `max_bits()`.
+use core::convert::{From, Into};
+use core::mem::size_of;
+use core::ops::{Add, BitOr, Shl, Sub};
+
+/// A small bitset built on a single primitive integer type
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct BitSet<T>(pub T);
+
+impl<T> BitSet<T>
+where
+    T: Into<u32>
+        + From<u8>
+        + BitOr<T, Output = T>
+        + Shl<u8, Output = T>
+        + Sub<T, Output = T>
+        + Add<T, Output = T>
+        + PartialEq
+        + Copy,
+{
+    /// Maximum number of bits supported by this BitSet instance
+    pub fn bits() -> usize {
+        size_of::<T>() * 8
+    }
+
+    /// Maximum number of bits supported by any bitset instance atm.
+    pub fn max_bits() -> usize {
+        size_of::<u32>() * 8
+    }
+
+    /// Check if this BitSet contains the number num
+    pub fn contains(&self, num: u8) -> bool {
+        debug_assert!((num as usize) < Self::bits());
+        debug_assert!((num as usize) < Self::max_bits());
+        self.0.into() & (1 << num) != 0
+    }
+
+    /// Return the smallest number contained in the bitset or None if empty
+    pub fn min(&self) -> Option<u8> {
+        if self.0.into() == 0 {
+            None
+        } else {
+            Some(self.0.into().trailing_zeros() as u8)
+        }
+    }
+
+    /// Return the largest number contained in the bitset or None if empty
+    pub fn max(&self) -> Option<u8> {
+        if self.0.into() == 0 {
+            None
+        } else {
+            let leading_zeroes = self.0.into().leading_zeros() as usize;
+            Some((Self::max_bits() - leading_zeroes - 1) as u8)
+        }
+    }
+
+    /// Construct a BitSet with the half-open range [lo,hi) filled in
+    pub fn from_range(lo: u8, hi: u8) -> Self {
+        debug_assert!(lo <= hi);
+        debug_assert!((hi as usize) <= Self::bits());
+        let one: T = T::from(1);
+        // I can't just do (one << hi) - one here as the shift may overflow
+        let hi_rng = if hi >= 1 {
+            (one << (hi - 1)) + ((one << (hi - 1)) - one)
+        } else {
+            T::from(0)
+        };
+
+        let lo_rng = (one << lo) - one;
+
+        Self(hi_rng - lo_rng)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn contains() {
+        let s = BitSet::<u8>(255);
+        for i in 0..7 {
+            assert!(s.contains(i));
+        }
+
+        let s1 = BitSet::<u8>(0);
+        for i in 0..7 {
+            assert!(!s1.contains(i));
+        }
+
+        let s2 = BitSet::<u8>(127);
+        for i in 0..6 {
+            assert!(s2.contains(i));
+        }
+        assert!(!s2.contains(7));
+
+        let s3 = BitSet::<u8>(2 | 4 | 64);
+        assert!(!s3.contains(0) && !s3.contains(3) && !s3.contains(4));
+        assert!(!s3.contains(5) && !s3.contains(7));
+        assert!(s3.contains(1) && s3.contains(2) && s3.contains(6));
+
+        let s4 = BitSet::<u16>(4 | 8 | 256 | 1024);
+        assert!(
+            !s4.contains(0)
+                && !s4.contains(1)
+                && !s4.contains(4)
+                && !s4.contains(5)
+                && !s4.contains(6)
+                && !s4.contains(7)
+                && !s4.contains(9)
+                && !s4.contains(11)
+        );
+        assert!(s4.contains(2) && s4.contains(3) && s4.contains(8) && s4.contains(10));
+    }
+
+    #[test]
+    fn minmax() {
+        let s = BitSet::<u8>(255);
+        assert_eq!(s.min(), Some(0));
+        assert_eq!(s.max(), Some(7));
+        assert!(s.min() == Some(0) && s.max() == Some(7));
+        let s1 = BitSet::<u8>(0);
+        assert!(s1.min() == None && s1.max() == None);
+        let s2 = BitSet::<u8>(127);
+        assert!(s2.min() == Some(0) && s2.max() == Some(6));
+        let s3 = BitSet::<u8>(2 | 4 | 64);
+        assert!(s3.min() == Some(1) && s3.max() == Some(6));
+        let s4 = BitSet::<u16>(4 | 8 | 256 | 1024);
+        assert!(s4.min() == Some(2) && s4.max() == Some(10));
+    }
+
+    #[test]
+    fn from_range() {
+        let s = BitSet::<u8>::from_range(5, 5);
+        assert!(s.0 == 0);
+
+        let s = BitSet::<u8>::from_range(0, 8);
+        assert!(s.0 == 255);
+
+        let s = BitSet::<u16>::from_range(0, 8);
+        assert!(s.0 == 255u16);
+
+        let s = BitSet::<u16>::from_range(0, 16);
+        assert!(s.0 == 65535u16);
+
+        let s = BitSet::<u8>::from_range(5, 6);
+        assert!(s.0 == 32u8);
+
+        let s = BitSet::<u8>::from_range(3, 7);
+        assert!(s.0 == 8 | 16 | 32 | 64);
+
+        let s = BitSet::<u16>::from_range(5, 11);
+        assert!(s.0 == 32 | 64 | 128 | 256 | 512 | 1024);
+    }
+}

cranelift/codegen/src/cfg_printer.rs

@@ -0,0 +1,83 @@
+//! The `CFGPrinter` utility.
+
+use alloc::vec::Vec;
+use core::fmt::{Display, Formatter, Result, Write};
+
+use crate::entity::SecondaryMap;
+use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
+use crate::ir::Function;
+use crate::write::{FuncWriter, PlainWriter};
+
+/// A utility for pretty-printing the CFG of a `Function`.
+pub struct CFGPrinter<'a> {
+    func: &'a Function,
+    cfg: ControlFlowGraph,
+}
+
+/// A utility for pretty-printing the CFG of a `Function`.
+impl<'a> CFGPrinter<'a> {
+    /// Create a new CFGPrinter.
+    pub fn new(func: &'a Function) -> Self {
+        Self {
+            func,
+            cfg: ControlFlowGraph::with_function(func),
+        }
+    }
+
+    /// Write the CFG for this function to `w`.
+    pub fn write(&self, w: &mut dyn Write) -> Result {
+        self.header(w)?;
+        self.block_nodes(w)?;
+        self.cfg_connections(w)?;
+        writeln!(w, "}}")
+    }
+
+    fn header(&self, w: &mut dyn Write) -> Result {
+        writeln!(w, "digraph \"{}\" {{", self.func.name)?;
+        if let Some(entry) = self.func.layout.entry_block() {
+            writeln!(w, "    {{rank=min; {}}}", entry)?;
+        }
+        Ok(())
+    }
+
+    fn block_nodes(&self, w: &mut dyn Write) -> Result {
+        let mut aliases = SecondaryMap::<_, Vec<_>>::new();
+        for v in self.func.dfg.values() {
+            // VADFS returns the immediate target of an alias
+            if let Some(k) = self.func.dfg.value_alias_dest_for_serialization(v) {
+                aliases[k].push(v);
+            }
+        }
+
+        for block in &self.func.layout {
+            write!(w, "    {} [shape=record, label=\"{{", block)?;
+            crate::write::write_block_header(w, self.func, None, block, 4)?;
+            // Add all outgoing branch instructions to the label.
+            for inst in self.func.layout.block_insts(block) {
+                write!(w, " | <{}>", inst)?;
+                PlainWriter.write_instruction(w, self.func, &aliases, None, inst, 0)?;
+            }
+            writeln!(w, "}}\"]")?
+        }
+        Ok(())
+    }
+
+    fn cfg_connections(&self, w: &mut dyn Write) -> Result {
+        for block in &self.func.layout {
+            for BlockPredecessor {
+                block: parent,
+                inst,
+            } in self.cfg.pred_iter(block)
+            {
+                writeln!(w, "    {}:{} -> {}", parent, inst, block)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+impl<'a> Display for CFGPrinter<'a> {
+    fn fmt(&self, f: &mut Formatter) -> Result {
+        self.write(f)
+    }
+}

cranelift/codegen/src/constant_hash.rs

@@ -0,0 +1,62 @@
+//! Runtime support for precomputed constant hash tables.
+//!
+//! The shared module with the same name can generate constant hash tables using open addressing
+//! and quadratic probing.
+//!
+//! The hash tables are arrays that are guaranteed to:
+//!
+//! - Have a power-of-two size.
+//! - Contain at least one empty slot.
+//!
+//! This module provides runtime support for lookups in these tables.
+
+// Re-export entities from constant_hash for simplicity of use.
+pub use cranelift_codegen_shared::constant_hash::*;
+
+/// Trait that must be implemented by the entries in a constant hash table.
+pub trait Table<K: Copy + Eq> {
+    /// Get the number of entries in this table which must be a power of two.
+    fn len(&self) -> usize;
+
+    /// Get the key corresponding to the entry at `idx`, or `None` if the entry is empty.
+    /// The `idx` must be in range.
+    fn key(&self, idx: usize) -> Option<K>;
+}
+
+/// Look for `key` in `table`.
+///
+/// The provided `hash` value must have been computed from `key` using the same hash function that
+/// was used to construct the table.
+///
+/// Returns `Ok(idx)` with the table index containing the found entry, or `Err(idx)` with the empty
+/// sentinel entry if no entry could be found.
+pub fn probe<K: Copy + Eq, T: Table<K> + ?Sized>(
+    table: &T,
+    key: K,
+    hash: usize,
+) -> Result<usize, usize> {
+    debug_assert!(table.len().is_power_of_two());
+    let mask = table.len() - 1;
+
+    let mut idx = hash;
+    let mut step = 0;
+
+    loop {
+        idx &= mask;
+
+        match table.key(idx) {
+            None => return Err(idx),
+            Some(k) if k == key => return Ok(idx),
+            _ => {}
+        }
+
+        // Quadratic probing.
+        step += 1;
+
+        // When `table.len()` is a power of two, it can be proven that `idx` will visit all
+        // entries. This means that this loop will always terminate if the hash table has even
+        // one unused entry.
+        debug_assert!(step < table.len());
+        idx += step;
+    }
+}

cranelift/codegen/src/context.rs

@@ -0,0 +1,393 @@
+//! Cranelift compilation context and main entry point.
+//!
+//! When compiling many small functions, it is important to avoid repeatedly allocating and
+//! deallocating the data structures needed for compilation. The `Context` struct is used to hold
+//! on to memory allocations between function compilations.
+//!
+//! The context does not hold a `TargetIsa` instance which has to be provided as an argument
+//! instead. This is because an ISA instance is immutable and can be used by multiple compilation
+//! contexts concurrently. Typically, you would have one context per compilation thread and only a
+//! single ISA instance.
+
+use crate::binemit::{
+    relax_branches, shrink_instructions, CodeInfo, FrameUnwindKind, FrameUnwindSink,
+    MemoryCodeSink, RelocSink, StackmapSink, TrapSink,
+};
+use crate::dce::do_dce;
+use crate::dominator_tree::DominatorTree;
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::Function;
+use crate::isa::TargetIsa;
+use crate::legalize_function;
+use crate::licm::do_licm;
+use crate::loop_analysis::LoopAnalysis;
+use crate::nan_canonicalization::do_nan_canonicalization;
+use crate::postopt::do_postopt;
+use crate::redundant_reload_remover::RedundantReloadRemover;
+use crate::regalloc;
+use crate::result::CodegenResult;
+use crate::settings::{FlagsOrIsa, OptLevel};
+use crate::simple_gvn::do_simple_gvn;
+use crate::simple_preopt::do_preopt;
+use crate::timing;
+use crate::unreachable_code::eliminate_unreachable_code;
+use crate::value_label::{build_value_labels_ranges, ComparableSourceLoc, ValueLabelsRanges};
+use crate::verifier::{verify_context, verify_locations, VerifierErrors, VerifierResult};
+use alloc::vec::Vec;
+use log::debug;
+
+/// Persistent data structures and compilation pipeline.
+pub struct Context {
+    /// The function we're compiling.
+    pub func: Function,
+
+    /// The control flow graph of `func`.
+    pub cfg: ControlFlowGraph,
+
+    /// Dominator tree for `func`.
+    pub domtree: DominatorTree,
+
+    /// Register allocation context.
+    pub regalloc: regalloc::Context,
+
+    /// Loop analysis of `func`.
+    pub loop_analysis: LoopAnalysis,
+
+    /// Redundant-reload remover context.
+    pub redundant_reload_remover: RedundantReloadRemover,
+}
+
+impl Context {
+    /// Allocate a new compilation context.
+    ///
+    /// The returned instance should be reused for compiling multiple functions in order to avoid
+    /// needless allocator thrashing.
+    pub fn new() -> Self {
+        Self::for_function(Function::new())
+    }
+
+    /// Allocate a new compilation context with an existing Function.
+    ///
+    /// The returned instance should be reused for compiling multiple functions in order to avoid
+    /// needless allocator thrashing.
+    pub fn for_function(func: Function) -> Self {
+        Self {
+            func,
+            cfg: ControlFlowGraph::new(),
+            domtree: DominatorTree::new(),
+            regalloc: regalloc::Context::new(),
+            loop_analysis: LoopAnalysis::new(),
+            redundant_reload_remover: RedundantReloadRemover::new(),
+        }
+    }
+
+    /// Clear all data structures in this context.
+    pub fn clear(&mut self) {
+        self.func.clear();
+        self.cfg.clear();
+        self.domtree.clear();
+        self.regalloc.clear();
+        self.loop_analysis.clear();
+        self.redundant_reload_remover.clear();
+    }
+
+    /// Compile the function, and emit machine code into a `Vec<u8>`.
+    ///
+    /// Run the function through all the passes necessary to generate code for the target ISA
+    /// represented by `isa`, as well as the final step of emitting machine code into a
+    /// `Vec<u8>`. The machine code is not relocated. Instead, any relocations are emitted
+    /// into `relocs`.
+    ///
+    /// This function calls `compile` and `emit_to_memory`, taking care to resize `mem` as
+    /// needed, so it provides a safe interface.
+    ///
+    /// Returns information about the function's code and read-only data.
+    pub fn compile_and_emit(
+        &mut self,
+        isa: &dyn TargetIsa,
+        mem: &mut Vec<u8>,
+        relocs: &mut dyn RelocSink,
+        traps: &mut dyn TrapSink,
+        stackmaps: &mut dyn StackmapSink,
+    ) -> CodegenResult<CodeInfo> {
+        let info = self.compile(isa)?;
+        let old_len = mem.len();
+        mem.resize(old_len + info.total_size as usize, 0);
+        let new_info = unsafe {
+            self.emit_to_memory(isa, mem.as_mut_ptr().add(old_len), relocs, traps, stackmaps)
+        };
+        debug_assert!(new_info == info);
+        Ok(info)
+    }
+
+    /// Compile the function.
+    ///
+    /// Run the function through all the passes necessary to generate code for the target ISA
+    /// represented by `isa`. This does not include the final step of emitting machine code into a
+    /// code sink.
+    ///
+    /// Returns information about the function's code and read-only data.
+    pub fn compile(&mut self, isa: &dyn TargetIsa) -> CodegenResult<CodeInfo> {
+        let _tt = timing::compile();
+        self.verify_if(isa)?;
+        debug!("Compiling:\n{}", self.func.display(isa));
+
+        let opt_level = isa.flags().opt_level();
+
+        self.compute_cfg();
+        if opt_level != OptLevel::None {
+            self.preopt(isa)?;
+        }
+        if isa.flags().enable_nan_canonicalization() {
+            self.canonicalize_nans(isa)?;
+        }
+        self.legalize(isa)?;
+        if opt_level != OptLevel::None {
+            self.postopt(isa)?;
+            self.compute_domtree();
+            self.compute_loop_analysis();
+            self.licm(isa)?;
+            self.simple_gvn(isa)?;
+        }
+        self.compute_domtree();
+        self.eliminate_unreachable_code(isa)?;
+        if opt_level != OptLevel::None {
+            self.dce(isa)?;
+        }
+        self.regalloc(isa)?;
+        self.prologue_epilogue(isa)?;
+        if opt_level == OptLevel::Speed || opt_level == OptLevel::SpeedAndSize {
+            self.redundant_reload_remover(isa)?;
+        }
+        if opt_level == OptLevel::SpeedAndSize {
+            self.shrink_instructions(isa)?;
+        }
+        let result = self.relax_branches(isa);
+
+        debug!("Compiled:\n{}", self.func.display(isa));
+        result
+    }
+
+    /// Emit machine code directly into raw memory.
+    ///
+    /// Write all of the function's machine code to the memory at `mem`. The size of the machine
+    /// code is returned by `compile` above.
+    ///
+    /// The machine code is not relocated. Instead, any relocations are emitted into `relocs`.
+    ///
+    /// # Safety
+    ///
+    /// This function is unsafe since it does not perform bounds checking on the memory buffer,
+    /// and it can't guarantee that the `mem` pointer is valid.
+    ///
+    /// Returns information about the emitted code and data.
+    pub unsafe fn emit_to_memory(
+        &self,
+        isa: &dyn TargetIsa,
+        mem: *mut u8,
+        relocs: &mut dyn RelocSink,
+        traps: &mut dyn TrapSink,
+        stackmaps: &mut dyn StackmapSink,
+    ) -> CodeInfo {
+        let _tt = timing::binemit();
+        let mut sink = MemoryCodeSink::new(mem, relocs, traps, stackmaps);
+        isa.emit_function_to_memory(&self.func, &mut sink);
+        sink.info
+    }
+
+    /// Emit unwind information.
+    ///
+    /// Requires that the function layout be calculated (see `relax_branches`).
+    ///
+    /// Only some calling conventions (e.g. Windows fastcall) will have unwind information.
+    /// This is a no-op if the function has no unwind information.
+    pub fn emit_unwind_info(
+        &self,
+        isa: &dyn TargetIsa,
+        kind: FrameUnwindKind,
+        sink: &mut dyn FrameUnwindSink,
+    ) {
+        isa.emit_unwind_info(&self.func, kind, sink);
+    }
+
+    /// Run the verifier on the function.
+    ///
+    /// Also check that the dominator tree and control flow graph are consistent with the function.
+    pub fn verify<'a, FOI: Into<FlagsOrIsa<'a>>>(&self, fisa: FOI) -> VerifierResult<()> {
+        let mut errors = VerifierErrors::default();
+        let _ = verify_context(&self.func, &self.cfg, &self.domtree, fisa, &mut errors);
+
+        if errors.is_empty() {
+            Ok(())
+        } else {
+            Err(errors)
+        }
+    }
+
+    /// Run the verifier only if the `enable_verifier` setting is true.
+    pub fn verify_if<'a, FOI: Into<FlagsOrIsa<'a>>>(&self, fisa: FOI) -> CodegenResult<()> {
+        let fisa = fisa.into();
+        if fisa.flags.enable_verifier() {
+            self.verify(fisa)?;
+        }
+        Ok(())
+    }
+
+    /// Run the locations verifier on the function.
+    pub fn verify_locations(&self, isa: &dyn TargetIsa) -> VerifierResult<()> {
+        let mut errors = VerifierErrors::default();
+        let _ = verify_locations(isa, &self.func, &self.cfg, None, &mut errors);
+
+        if errors.is_empty() {
+            Ok(())
+        } else {
+            Err(errors)
+        }
+    }
+
+    /// Run the locations verifier only if the `enable_verifier` setting is true.
+    pub fn verify_locations_if(&self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        if isa.flags().enable_verifier() {
+            self.verify_locations(isa)?;
+        }
+        Ok(())
+    }
+
+    /// Perform dead-code elimination on the function.
+    pub fn dce<'a, FOI: Into<FlagsOrIsa<'a>>>(&mut self, fisa: FOI) -> CodegenResult<()> {
+        do_dce(&mut self.func, &mut self.domtree);
+        self.verify_if(fisa)?;
+        Ok(())
+    }
+
+    /// Perform pre-legalization rewrites on the function.
+    pub fn preopt(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        do_preopt(&mut self.func, &mut self.cfg, isa);
+        self.verify_if(isa)?;
+        Ok(())
+    }
+
+    /// Perform NaN canonicalizing rewrites on the function.
+    pub fn canonicalize_nans(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        do_nan_canonicalization(&mut self.func);
+        self.verify_if(isa)
+    }
+
+    /// Run the legalizer for `isa` on the function.
+    pub fn legalize(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        // Legalization invalidates the domtree and loop_analysis by mutating the CFG.
+        // TODO: Avoid doing this when legalization doesn't actually mutate the CFG.
+        self.domtree.clear();
+        self.loop_analysis.clear();
+        legalize_function(&mut self.func, &mut self.cfg, isa);
+        debug!("Legalized:\n{}", self.func.display(isa));
+        self.verify_if(isa)
+    }
+
+    /// Perform post-legalization rewrites on the function.
+    pub fn postopt(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        do_postopt(&mut self.func, isa);
+        self.verify_if(isa)?;
+        Ok(())
+    }
+
+    /// Compute the control flow graph.
+    pub fn compute_cfg(&mut self) {
+        self.cfg.compute(&self.func)
+    }
+
+    /// Compute dominator tree.
+    pub fn compute_domtree(&mut self) {
+        self.domtree.compute(&self.func, &self.cfg)
+    }
+
+    /// Compute the loop analysis.
+    pub fn compute_loop_analysis(&mut self) {
+        self.loop_analysis
+            .compute(&self.func, &self.cfg, &self.domtree)
+    }
+
+    /// Compute the control flow graph and dominator tree.
+    pub fn flowgraph(&mut self) {
+        self.compute_cfg();
+        self.compute_domtree()
+    }
+
+    /// Perform simple GVN on the function.
+    pub fn simple_gvn<'a, FOI: Into<FlagsOrIsa<'a>>>(&mut self, fisa: FOI) -> CodegenResult<()> {
+        do_simple_gvn(&mut self.func, &mut self.domtree);
+        self.verify_if(fisa)
+    }
+
+    /// Perform LICM on the function.
+    pub fn licm(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        do_licm(
+            isa,
+            &mut self.func,
+            &mut self.cfg,
+            &mut self.domtree,
+            &mut self.loop_analysis,
+        );
+        self.verify_if(isa)
+    }
+
+    /// Perform unreachable code elimination.
+    pub fn eliminate_unreachable_code<'a, FOI>(&mut self, fisa: FOI) -> CodegenResult<()>
+    where
+        FOI: Into<FlagsOrIsa<'a>>,
+    {
+        eliminate_unreachable_code(&mut self.func, &mut self.cfg, &self.domtree);
+        self.verify_if(fisa)
+    }
+
+    /// Run the register allocator.
+    pub fn regalloc(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        self.regalloc
+            .run(isa, &mut self.func, &mut self.cfg, &mut self.domtree)
+    }
+
+    /// Insert prologue and epilogues after computing the stack frame layout.
+    pub fn prologue_epilogue(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        isa.prologue_epilogue(&mut self.func)?;
+        self.verify_if(isa)?;
+        self.verify_locations_if(isa)?;
+        Ok(())
+    }
+
+    /// Do redundant-reload removal after allocation of both registers and stack slots.
+    pub fn redundant_reload_remover(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        self.redundant_reload_remover
+            .run(isa, &mut self.func, &self.cfg);
+        self.verify_if(isa)?;
+        Ok(())
+    }
+
+    /// Run the instruction shrinking pass.
+    pub fn shrink_instructions(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        shrink_instructions(&mut self.func, isa);
+        self.verify_if(isa)?;
+        self.verify_locations_if(isa)?;
+        Ok(())
+    }
+
+    /// Run the branch relaxation pass and return information about the function's code and
+    /// read-only data.
+    pub fn relax_branches(&mut self, isa: &dyn TargetIsa) -> CodegenResult<CodeInfo> {
+        let info = relax_branches(&mut self.func, &mut self.cfg, &mut self.domtree, isa)?;
+        self.verify_if(isa)?;
+        self.verify_locations_if(isa)?;
+        Ok(info)
+    }
+
+    /// Builds ranges and location for specified value labels.
+    pub fn build_value_labels_ranges(
+        &self,
+        isa: &dyn TargetIsa,
+    ) -> CodegenResult<ValueLabelsRanges> {
+        Ok(build_value_labels_ranges::<ComparableSourceLoc>(
+            &self.func,
+            &self.regalloc,
+            isa,
+        ))
+    }
+}

cranelift/codegen/src/cursor.rs

@@ -0,0 +1,810 @@
+//! Cursor library.
+//!
+//! This module defines cursor data types that can be used for inserting instructions.
+
+use crate::ir;
+use crate::isa::TargetIsa;
+
+/// The possible positions of a cursor.
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum CursorPosition {
+    /// Cursor is not pointing anywhere. No instructions can be inserted.
+    Nowhere,
+    /// Cursor is pointing at an existing instruction.
+    /// New instructions will be inserted *before* the current instruction.
+    At(ir::Inst),
+    /// Cursor is before the beginning of an block. No instructions can be inserted. Calling
+    /// `next_inst()` will move to the first instruction in the block.
+    Before(ir::Block),
+    /// Cursor is pointing after the end of an block.
+    /// New instructions will be appended to the block.
+    After(ir::Block),
+}
+
+/// All cursor types implement the `Cursor` which provides common navigation operations.
+pub trait Cursor {
+    /// Get the current cursor position.
+    fn position(&self) -> CursorPosition;
+
+    /// Set the current position.
+    fn set_position(&mut self, pos: CursorPosition);
+
+    /// Get the source location that should be assigned to new instructions.
+    fn srcloc(&self) -> ir::SourceLoc;
+
+    /// Set the source location that should be assigned to new instructions.
+    fn set_srcloc(&mut self, srcloc: ir::SourceLoc);
+
+    /// Borrow a reference to the function layout that this cursor is navigating.
+    fn layout(&self) -> &ir::Layout;
+
+    /// Borrow a mutable reference to the function layout that this cursor is navigating.
+    fn layout_mut(&mut self) -> &mut ir::Layout;
+
+    /// Exchange this cursor for one with a set source location.
+    ///
+    /// This is intended to be used as a builder method:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block, SourceLoc};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function, srcloc: SourceLoc) {
+    ///     let mut pos = FuncCursor::new(func).with_srcloc(srcloc);
+    ///
+    ///     // Use `pos`...
+    /// }
+    /// ```
+    fn with_srcloc(mut self, srcloc: ir::SourceLoc) -> Self
+    where
+        Self: Sized,
+    {
+        self.set_srcloc(srcloc);
+        self
+    }
+
+    /// Rebuild this cursor positioned at `pos`.
+    fn at_position(mut self, pos: CursorPosition) -> Self
+    where
+        Self: Sized,
+    {
+        self.set_position(pos);
+        self
+    }
+
+    /// Rebuild this cursor positioned at `inst`.
+    ///
+    /// This is intended to be used as a builder method:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block, Inst};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function, inst: Inst) {
+    ///     let mut pos = FuncCursor::new(func).at_inst(inst);
+    ///
+    ///     // Use `pos`...
+    /// }
+    /// ```
+    fn at_inst(mut self, inst: ir::Inst) -> Self
+    where
+        Self: Sized,
+    {
+        self.goto_inst(inst);
+        self
+    }
+
+    /// Rebuild this cursor positioned at the first insertion point for `block`.
+    /// This differs from `at_first_inst` in that it doesn't assume that any
+    /// instructions have been inserted into `block` yet.
+    ///
+    /// This is intended to be used as a builder method:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block, Inst};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function, block: Block) {
+    ///     let mut pos = FuncCursor::new(func).at_first_insertion_point(block);
+    ///
+    ///     // Use `pos`...
+    /// }
+    /// ```
+    fn at_first_insertion_point(mut self, block: ir::Block) -> Self
+    where
+        Self: Sized,
+    {
+        self.goto_first_insertion_point(block);
+        self
+    }
+
+    /// Rebuild this cursor positioned at the first instruction in `block`.
+    ///
+    /// This is intended to be used as a builder method:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block, Inst};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function, block: Block) {
+    ///     let mut pos = FuncCursor::new(func).at_first_inst(block);
+    ///
+    ///     // Use `pos`...
+    /// }
+    /// ```
+    fn at_first_inst(mut self, block: ir::Block) -> Self
+    where
+        Self: Sized,
+    {
+        self.goto_first_inst(block);
+        self
+    }
+
+    /// Rebuild this cursor positioned at the last instruction in `block`.
+    ///
+    /// This is intended to be used as a builder method:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block, Inst};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function, block: Block) {
+    ///     let mut pos = FuncCursor::new(func).at_last_inst(block);
+    ///
+    ///     // Use `pos`...
+    /// }
+    /// ```
+    fn at_last_inst(mut self, block: ir::Block) -> Self
+    where
+        Self: Sized,
+    {
+        self.goto_last_inst(block);
+        self
+    }
+
+    /// Rebuild this cursor positioned after `inst`.
+    ///
+    /// This is intended to be used as a builder method:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block, Inst};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function, inst: Inst) {
+    ///     let mut pos = FuncCursor::new(func).after_inst(inst);
+    ///
+    ///     // Use `pos`...
+    /// }
+    /// ```
+    fn after_inst(mut self, inst: ir::Inst) -> Self
+    where
+        Self: Sized,
+    {
+        self.goto_after_inst(inst);
+        self
+    }
+
+    /// Rebuild this cursor positioned at the top of `block`.
+    ///
+    /// This is intended to be used as a builder method:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block, Inst};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function, block: Block) {
+    ///     let mut pos = FuncCursor::new(func).at_top(block);
+    ///
+    ///     // Use `pos`...
+    /// }
+    /// ```
+    fn at_top(mut self, block: ir::Block) -> Self
+    where
+        Self: Sized,
+    {
+        self.goto_top(block);
+        self
+    }
+
+    /// Rebuild this cursor positioned at the bottom of `block`.
+    ///
+    /// This is intended to be used as a builder method:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block, Inst};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function, block: Block) {
+    ///     let mut pos = FuncCursor::new(func).at_bottom(block);
+    ///
+    ///     // Use `pos`...
+    /// }
+    /// ```
+    fn at_bottom(mut self, block: ir::Block) -> Self
+    where
+        Self: Sized,
+    {
+        self.goto_bottom(block);
+        self
+    }
+
+    /// Get the block corresponding to the current position.
+    fn current_block(&self) -> Option<ir::Block> {
+        use self::CursorPosition::*;
+        match self.position() {
+            Nowhere => None,
+            At(inst) => self.layout().inst_block(inst),
+            Before(block) | After(block) => Some(block),
+        }
+    }
+
+    /// Get the instruction corresponding to the current position, if any.
+    fn current_inst(&self) -> Option<ir::Inst> {
+        use self::CursorPosition::*;
+        match self.position() {
+            At(inst) => Some(inst),
+            _ => None,
+        }
+    }
+
+    /// Go to the position after a specific instruction, which must be inserted
+    /// in the layout. New instructions will be inserted after `inst`.
+    fn goto_after_inst(&mut self, inst: ir::Inst) {
+        debug_assert!(self.layout().inst_block(inst).is_some());
+        let new_pos = if let Some(next) = self.layout().next_inst(inst) {
+            CursorPosition::At(next)
+        } else {
+            CursorPosition::After(
+                self.layout()
+                    .inst_block(inst)
+                    .expect("current instruction removed?"),
+            )
+        };
+        self.set_position(new_pos);
+    }
+
+    /// Go to a specific instruction which must be inserted in the layout.
+    /// New instructions will be inserted before `inst`.
+    fn goto_inst(&mut self, inst: ir::Inst) {
+        debug_assert!(self.layout().inst_block(inst).is_some());
+        self.set_position(CursorPosition::At(inst));
+    }
+
+    /// Go to the position for inserting instructions at the beginning of `block`,
+    /// which unlike `goto_first_inst` doesn't assume that any instructions have
+    /// been inserted into `block` yet.
+    fn goto_first_insertion_point(&mut self, block: ir::Block) {
+        if let Some(inst) = self.layout().first_inst(block) {
+            self.goto_inst(inst);
+        } else {
+            self.goto_bottom(block);
+        }
+    }
+
+    /// Go to the first instruction in `block`.
+    fn goto_first_inst(&mut self, block: ir::Block) {
+        let inst = self.layout().first_inst(block).expect("Empty block");
+        self.goto_inst(inst);
+    }
+
+    /// Go to the last instruction in `block`.
+    fn goto_last_inst(&mut self, block: ir::Block) {
+        let inst = self.layout().last_inst(block).expect("Empty block");
+        self.goto_inst(inst);
+    }
+
+    /// Go to the top of `block` which must be inserted into the layout.
+    /// At this position, instructions cannot be inserted, but `next_inst()` will move to the first
+    /// instruction in `block`.
+    fn goto_top(&mut self, block: ir::Block) {
+        debug_assert!(self.layout().is_block_inserted(block));
+        self.set_position(CursorPosition::Before(block));
+    }
+
+    /// Go to the bottom of `block` which must be inserted into the layout.
+    /// At this position, inserted instructions will be appended to `block`.
+    fn goto_bottom(&mut self, block: ir::Block) {
+        debug_assert!(self.layout().is_block_inserted(block));
+        self.set_position(CursorPosition::After(block));
+    }
+
+    /// Go to the top of the next block in layout order and return it.
+    ///
+    /// - If the cursor wasn't pointing at anything, go to the top of the first block in the
+    ///   function.
+    /// - If there are no more blocks, leave the cursor pointing at nothing and return `None`.
+    ///
+    /// # Examples
+    ///
+    /// The `next_block()` method is intended for iterating over the blocks in layout order:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function) {
+    ///     let mut cursor = FuncCursor::new(func);
+    ///     while let Some(block) = cursor.next_block() {
+    ///         // Edit block.
+    ///     }
+    /// }
+    /// ```
+    fn next_block(&mut self) -> Option<ir::Block> {
+        let next = if let Some(block) = self.current_block() {
+            self.layout().next_block(block)
+        } else {
+            self.layout().entry_block()
+        };
+        self.set_position(match next {
+            Some(block) => CursorPosition::Before(block),
+            None => CursorPosition::Nowhere,
+        });
+        next
+    }
+
+    /// Go to the bottom of the previous block in layout order and return it.
+    ///
+    /// - If the cursor wasn't pointing at anything, go to the bottom of the last block in the
+    ///   function.
+    /// - If there are no more blocks, leave the cursor pointing at nothing and return `None`.
+    ///
+    /// # Examples
+    ///
+    /// The `prev_block()` method is intended for iterating over the blocks in backwards layout order:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function) {
+    ///     let mut cursor = FuncCursor::new(func);
+    ///     while let Some(block) = cursor.prev_block() {
+    ///         // Edit block.
+    ///     }
+    /// }
+    /// ```
+    fn prev_block(&mut self) -> Option<ir::Block> {
+        let prev = if let Some(block) = self.current_block() {
+            self.layout().prev_block(block)
+        } else {
+            self.layout().last_block()
+        };
+        self.set_position(match prev {
+            Some(block) => CursorPosition::After(block),
+            None => CursorPosition::Nowhere,
+        });
+        prev
+    }
+
+    /// Move to the next instruction in the same block and return it.
+    ///
+    /// - If the cursor was positioned before an block, go to the first instruction in that block.
+    /// - If there are no more instructions in the block, go to the `After(block)` position and return
+    ///   `None`.
+    /// - If the cursor wasn't pointing anywhere, keep doing that.
+    ///
+    /// This method will never move the cursor to a different block.
+    ///
+    /// # Examples
+    ///
+    /// The `next_inst()` method is intended for iterating over the instructions in an block like
+    /// this:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_block(func: &mut Function, block: Block) {
+    ///     let mut cursor = FuncCursor::new(func).at_top(block);
+    ///     while let Some(inst) = cursor.next_inst() {
+    ///         // Edit instructions...
+    ///     }
+    /// }
+    /// ```
+    /// The loop body can insert and remove instructions via the cursor.
+    ///
+    /// Iterating over all the instructions in a function looks like this:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_func(func: &mut Function) {
+    ///     let mut cursor = FuncCursor::new(func);
+    ///     while let Some(block) = cursor.next_block() {
+    ///         while let Some(inst) = cursor.next_inst() {
+    ///             // Edit instructions...
+    ///         }
+    ///     }
+    /// }
+    /// ```
+    fn next_inst(&mut self) -> Option<ir::Inst> {
+        use self::CursorPosition::*;
+        match self.position() {
+            Nowhere | After(..) => None,
+            At(inst) => {
+                if let Some(next) = self.layout().next_inst(inst) {
+                    self.set_position(At(next));
+                    Some(next)
+                } else {
+                    let pos = After(
+                        self.layout()
+                            .inst_block(inst)
+                            .expect("current instruction removed?"),
+                    );
+                    self.set_position(pos);
+                    None
+                }
+            }
+            Before(block) => {
+                if let Some(next) = self.layout().first_inst(block) {
+                    self.set_position(At(next));
+                    Some(next)
+                } else {
+                    self.set_position(After(block));
+                    None
+                }
+            }
+        }
+    }
+
+    /// Move to the previous instruction in the same block and return it.
+    ///
+    /// - If the cursor was positioned after an block, go to the last instruction in that block.
+    /// - If there are no more instructions in the block, go to the `Before(block)` position and return
+    ///   `None`.
+    /// - If the cursor wasn't pointing anywhere, keep doing that.
+    ///
+    /// This method will never move the cursor to a different block.
+    ///
+    /// # Examples
+    ///
+    /// The `prev_inst()` method is intended for iterating backwards over the instructions in an
+    /// block like this:
+    ///
+    /// ```
+    /// # use cranelift_codegen::ir::{Function, Block};
+    /// # use cranelift_codegen::cursor::{Cursor, FuncCursor};
+    /// fn edit_block(func: &mut Function, block: Block) {
+    ///     let mut cursor = FuncCursor::new(func).at_bottom(block);
+    ///     while let Some(inst) = cursor.prev_inst() {
+    ///         // Edit instructions...
+    ///     }
+    /// }
+    /// ```
+    fn prev_inst(&mut self) -> Option<ir::Inst> {
+        use self::CursorPosition::*;
+        match self.position() {
+            Nowhere | Before(..) => None,
+            At(inst) => {
+                if let Some(prev) = self.layout().prev_inst(inst) {
+                    self.set_position(At(prev));
+                    Some(prev)
+                } else {
+                    let pos = Before(
+                        self.layout()
+                            .inst_block(inst)
+                            .expect("current instruction removed?"),
+                    );
+                    self.set_position(pos);
+                    None
+                }
+            }
+            After(block) => {
+                if let Some(prev) = self.layout().last_inst(block) {
+                    self.set_position(At(prev));
+                    Some(prev)
+                } else {
+                    self.set_position(Before(block));
+                    None
+                }
+            }
+        }
+    }
+
+    /// Insert an instruction at the current position.
+    ///
+    /// - If pointing at an instruction, the new instruction is inserted before the current
+    ///   instruction.
+    /// - If pointing at the bottom of an block, the new instruction is appended to the block.
+    /// - Otherwise panic.
+    ///
+    /// In either case, the cursor is not moved, such that repeated calls to `insert_inst()` causes
+    /// instructions to appear in insertion order in the block.
+    fn insert_inst(&mut self, inst: ir::Inst) {
+        use self::CursorPosition::*;
+        match self.position() {
+            Nowhere | Before(..) => panic!("Invalid insert_inst position"),
+            At(cur) => self.layout_mut().insert_inst(inst, cur),
+            After(block) => self.layout_mut().append_inst(inst, block),
+        }
+    }
+
+    /// Remove the instruction under the cursor.
+    ///
+    /// The cursor is left pointing at the position following the current instruction.
+    ///
+    /// Return the instruction that was removed.
+    fn remove_inst(&mut self) -> ir::Inst {
+        let inst = self.current_inst().expect("No instruction to remove");
+        self.next_inst();
+        self.layout_mut().remove_inst(inst);
+        inst
+    }
+
+    /// Remove the instruction under the cursor.
+    ///
+    /// The cursor is left pointing at the position preceding the current instruction.
+    ///
+    /// Return the instruction that was removed.
+    fn remove_inst_and_step_back(&mut self) -> ir::Inst {
+        let inst = self.current_inst().expect("No instruction to remove");
+        self.prev_inst();
+        self.layout_mut().remove_inst(inst);
+        inst
+    }
+
+    /// Insert an block at the current position and switch to it.
+    ///
+    /// As far as possible, this method behaves as if the block header were an instruction inserted
+    /// at the current position.
+    ///
+    /// - If the cursor is pointing at an existing instruction, *the current block is split in two*
+    ///   and the current instruction becomes the first instruction in the inserted block.
+    /// - If the cursor points at the bottom of an block, the new block is inserted after the current
+    ///   one, and moved to the bottom of the new block where instructions can be appended.
+    /// - If the cursor points to the top of an block, the new block is inserted above the current one.
+    /// - If the cursor is not pointing at anything, the new block is placed last in the layout.
+    ///
+    /// This means that it is always valid to call this method, and it always leaves the cursor in
+    /// a state that will insert instructions into the new block.
+    fn insert_block(&mut self, new_block: ir::Block) {
+        use self::CursorPosition::*;
+        match self.position() {
+            At(inst) => {
+                self.layout_mut().split_block(new_block, inst);
+                // All other cases move to `After(block)`, but in this case we'll stay `At(inst)`.
+                return;
+            }
+            Nowhere => self.layout_mut().append_block(new_block),
+            Before(block) => self.layout_mut().insert_block(new_block, block),
+            After(block) => self.layout_mut().insert_block_after(new_block, block),
+        }
+        // For everything but `At(inst)` we end up appending to the new block.
+        self.set_position(After(new_block));
+    }
+}
+
+/// Function cursor.
+///
+/// A `FuncCursor` holds a mutable reference to a whole `ir::Function` while keeping a position
+/// too. The function can be re-borrowed by accessing the public `cur.func` member.
+///
+/// This cursor is for use before legalization. The inserted instructions are not given an
+/// encoding.
+pub struct FuncCursor<'f> {
+    pos: CursorPosition,
+    srcloc: ir::SourceLoc,
+
+    /// The referenced function.
+    pub func: &'f mut ir::Function,
+}
+
+impl<'f> FuncCursor<'f> {
+    /// Create a new `FuncCursor` pointing nowhere.
+    pub fn new(func: &'f mut ir::Function) -> Self {
+        Self {
+            pos: CursorPosition::Nowhere,
+            srcloc: Default::default(),
+            func,
+        }
+    }
+
+    /// Use the source location of `inst` for future instructions.
+    pub fn use_srcloc(&mut self, inst: ir::Inst) {
+        self.srcloc = self.func.srclocs[inst];
+    }
+
+    /// Create an instruction builder that inserts an instruction at the current position.
+    pub fn ins(&mut self) -> ir::InsertBuilder<&mut FuncCursor<'f>> {
+        ir::InsertBuilder::new(self)
+    }
+}
+
+impl<'f> Cursor for FuncCursor<'f> {
+    fn position(&self) -> CursorPosition {
+        self.pos
+    }
+
+    fn set_position(&mut self, pos: CursorPosition) {
+        self.pos = pos
+    }
+
+    fn srcloc(&self) -> ir::SourceLoc {
+        self.srcloc
+    }
+
+    fn set_srcloc(&mut self, srcloc: ir::SourceLoc) {
+        self.srcloc = srcloc;
+    }
+
+    fn layout(&self) -> &ir::Layout {
+        &self.func.layout
+    }
+
+    fn layout_mut(&mut self) -> &mut ir::Layout {
+        &mut self.func.layout
+    }
+}
+
+impl<'c, 'f> ir::InstInserterBase<'c> for &'c mut FuncCursor<'f> {
+    fn data_flow_graph(&self) -> &ir::DataFlowGraph {
+        &self.func.dfg
+    }
+
+    fn data_flow_graph_mut(&mut self) -> &mut ir::DataFlowGraph {
+        &mut self.func.dfg
+    }
+
+    fn insert_built_inst(self, inst: ir::Inst, _: ir::Type) -> &'c mut ir::DataFlowGraph {
+        // TODO: Remove this assertion once #796 is fixed.
+        #[cfg(debug_assertions)]
+        {
+            if let CursorPosition::At(_) = self.position() {
+                if let Some(curr) = self.current_inst() {
+                    if let Some(prev) = self.layout().prev_inst(curr) {
+                        let prev_op = self.data_flow_graph()[prev].opcode();
+                        let inst_op = self.data_flow_graph()[inst].opcode();
+                        let curr_op = self.data_flow_graph()[curr].opcode();
+                        if prev_op.is_branch()
+                            && !prev_op.is_terminator()
+                            && !inst_op.is_terminator()
+                        {
+                            panic!(
+                                "Inserting instruction {} after {}, and before {}",
+                                inst_op, prev_op, curr_op
+                            )
+                        }
+                    };
+                };
+            };
+        }
+        self.insert_inst(inst);
+        if !self.srcloc.is_default() {
+            self.func.srclocs[inst] = self.srcloc;
+        }
+        &mut self.func.dfg
+    }
+}
+
+/// Encoding cursor.
+///
+/// An `EncCursor` can be used to insert instructions that are immediately assigned an encoding.
+/// The cursor holds a mutable reference to the whole function which can be re-borrowed from the
+/// public `pos.func` member.
+pub struct EncCursor<'f> {
+    pos: CursorPosition,
+    srcloc: ir::SourceLoc,
+    built_inst: Option<ir::Inst>,
+
+    /// The referenced function.
+    pub func: &'f mut ir::Function,
+
+    /// The target ISA that will be used to encode instructions.
+    pub isa: &'f dyn TargetIsa,
+}
+
+impl<'f> EncCursor<'f> {
+    /// Create a new `EncCursor` pointing nowhere.
+    pub fn new(func: &'f mut ir::Function, isa: &'f dyn TargetIsa) -> Self {
+        Self {
+            pos: CursorPosition::Nowhere,
+            srcloc: Default::default(),
+            built_inst: None,
+            func,
+            isa,
+        }
+    }
+
+    /// Use the source location of `inst` for future instructions.
+    pub fn use_srcloc(&mut self, inst: ir::Inst) {
+        self.srcloc = self.func.srclocs[inst];
+    }
+
+    /// Create an instruction builder that will insert an encoded instruction at the current
+    /// position.
+    ///
+    /// The builder will panic if it is used to insert an instruction that can't be encoded for
+    /// `self.isa`.
+    pub fn ins(&mut self) -> ir::InsertBuilder<&mut EncCursor<'f>> {
+        ir::InsertBuilder::new(self)
+    }
+
+    /// Get the last built instruction.
+    ///
+    /// This returns the last instruction that was built using the `ins()` method on this cursor.
+    /// Panics if no instruction was built.
+    pub fn built_inst(&self) -> ir::Inst {
+        self.built_inst.expect("No instruction was inserted")
+    }
+
+    /// Return an object that can display `inst`.
+    ///
+    /// This is a convenience wrapper for the DFG equivalent.
+    pub fn display_inst(&self, inst: ir::Inst) -> ir::dfg::DisplayInst {
+        self.func.dfg.display_inst(inst, self.isa)
+    }
+}
+
+impl<'f> Cursor for EncCursor<'f> {
+    fn position(&self) -> CursorPosition {
+        self.pos
+    }
+
+    fn set_position(&mut self, pos: CursorPosition) {
+        self.pos = pos
+    }
+
+    fn srcloc(&self) -> ir::SourceLoc {
+        self.srcloc
+    }
+
+    fn set_srcloc(&mut self, srcloc: ir::SourceLoc) {
+        self.srcloc = srcloc;
+    }
+
+    fn layout(&self) -> &ir::Layout {
+        &self.func.layout
+    }
+
+    fn layout_mut(&mut self) -> &mut ir::Layout {
+        &mut self.func.layout
+    }
+}
+
+impl<'c, 'f> ir::InstInserterBase<'c> for &'c mut EncCursor<'f> {
+    fn data_flow_graph(&self) -> &ir::DataFlowGraph {
+        &self.func.dfg
+    }
+
+    fn data_flow_graph_mut(&mut self) -> &mut ir::DataFlowGraph {
+        &mut self.func.dfg
+    }
+
+    fn insert_built_inst(
+        self,
+        inst: ir::Inst,
+        ctrl_typevar: ir::Type,
+    ) -> &'c mut ir::DataFlowGraph {
+        // TODO: Remove this assertion once #796 is fixed.
+        #[cfg(debug_assertions)]
+        {
+            if let CursorPosition::At(_) = self.position() {
+                if let Some(curr) = self.current_inst() {
+                    if let Some(prev) = self.layout().prev_inst(curr) {
+                        let prev_op = self.data_flow_graph()[prev].opcode();
+                        let inst_op = self.data_flow_graph()[inst].opcode();
+                        if prev_op.is_branch()
+                            && !prev_op.is_terminator()
+                            && !inst_op.is_terminator()
+                        {
+                            panic!(
+                                "Inserting instruction {} after {} and before {}",
+                                self.display_inst(inst),
+                                self.display_inst(prev),
+                                self.display_inst(curr)
+                            )
+                        }
+                    };
+                };
+            };
+        }
+        // Insert the instruction and remember the reference.
+        self.insert_inst(inst);
+        self.built_inst = Some(inst);
+
+        if !self.srcloc.is_default() {
+            self.func.srclocs[inst] = self.srcloc;
+        }
+        // Assign an encoding.
+        // XXX Is there a way to describe this error to the user?
+        #[cfg_attr(feature = "cargo-clippy", allow(clippy::match_wild_err_arm))]
+        match self
+            .isa
+            .encode(&self.func, &self.func.dfg[inst], ctrl_typevar)
+        {
+            Ok(e) => self.func.encodings[inst] = e,
+            Err(_) => panic!("can't encode {}", self.display_inst(inst)),
+        }
+
+        &mut self.func.dfg
+    }
+}

cranelift/codegen/src/dbg.rs

@@ -0,0 +1,28 @@
+//! Debug tracing helpers.
+use core::fmt;
+
+/// Prefix added to the log file names, just before the thread name or id.
+pub static LOG_FILENAME_PREFIX: &str = "cranelift.dbg.";
+
+/// Helper for printing lists.
+pub struct DisplayList<'a, T>(pub &'a [T])
+where
+    T: 'a + fmt::Display;
+
+impl<'a, T> fmt::Display for DisplayList<'a, T>
+where
+    T: 'a + fmt::Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self.0.split_first() {
+            None => write!(f, "[]"),
+            Some((first, rest)) => {
+                write!(f, "[{}", first)?;
+                for x in rest {
+                    write!(f, ", {}", x)?;
+                }
+                write!(f, "]")
+            }
+        }
+    }
+}

cranelift/codegen/src/dce.rs

@@ -0,0 +1,69 @@
+//! A Dead-Code Elimination (DCE) pass.
+//!
+//! Dead code here means instructions that have no side effects and have no
+//! result values used by other instructions.
+
+use crate::cursor::{Cursor, FuncCursor};
+use crate::dominator_tree::DominatorTree;
+use crate::entity::EntityRef;
+use crate::ir::instructions::InstructionData;
+use crate::ir::{DataFlowGraph, Function, Inst, Opcode};
+use crate::timing;
+
+/// Test whether the given opcode is unsafe to even consider for DCE.
+fn trivially_unsafe_for_dce(opcode: Opcode) -> bool {
+    opcode.is_call()
+        || opcode.is_branch()
+        || opcode.is_terminator()
+        || opcode.is_return()
+        || opcode.can_trap()
+        || opcode.other_side_effects()
+        || opcode.can_store()
+}
+
+/// Preserve instructions with used result values.
+fn any_inst_results_used(inst: Inst, live: &[bool], dfg: &DataFlowGraph) -> bool {
+    dfg.inst_results(inst).iter().any(|v| live[v.index()])
+}
+
+/// Load instructions without the `notrap` flag are defined to trap when
+/// operating on inaccessible memory, so we can't DCE them even if the
+/// loaded value is unused.
+fn is_load_with_defined_trapping(opcode: Opcode, data: &InstructionData) -> bool {
+    if !opcode.can_load() {
+        return false;
+    }
+    match *data {
+        InstructionData::StackLoad { .. } => false,
+        InstructionData::Load { flags, .. } => !flags.notrap(),
+        _ => true,
+    }
+}
+
+/// Perform DCE on `func`.
+pub fn do_dce(func: &mut Function, domtree: &mut DominatorTree) {
+    let _tt = timing::dce();
+    debug_assert!(domtree.is_valid());
+
+    let mut live = vec![false; func.dfg.num_values()];
+    for &block in domtree.cfg_postorder() {
+        let mut pos = FuncCursor::new(func).at_bottom(block);
+        while let Some(inst) = pos.prev_inst() {
+            {
+                let data = &pos.func.dfg[inst];
+                let opcode = data.opcode();
+                if trivially_unsafe_for_dce(opcode)
+                    || is_load_with_defined_trapping(opcode, &data)
+                    || any_inst_results_used(inst, &live, &pos.func.dfg)
+                {
+                    for arg in pos.func.dfg.inst_args(inst) {
+                        let v = pos.func.dfg.resolve_aliases(*arg);
+                        live[v.index()] = true;
+                    }
+                    continue;
+                }
+            }
+            pos.remove_inst();
+        }
+    }
+}