diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3885cd9f09..9ce263c5cb 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -274,6 +274,7 @@ jobs:
             --exclude lightbeam \
             --exclude wasmtime-lightbeam \
             --exclude wasmtime-wasi-nn \
+            --exclude wasmtime-wasi-crypto \
             --exclude peepmatic \
             --exclude peepmatic-automata \
             --exclude peepmatic-fuzzing \
@@ -292,6 +293,14 @@ jobs:
       env:
         RUST_BACKTRACE: 1
 
+    # Test debug (DWARF) related functionality on new backend.
+    - run: |
+        sudo apt-get update && sudo apt-get install -y gdb
+        cargo test --features experimental_x64 test_debug_dwarf -- --ignored --test-threads 1 --test debug::
+      if: matrix.os == 'ubuntu-latest'
+      env:
+        RUST_BACKTRACE: 1
+
     # Build and test lightbeam. Note that
     # Lightbeam tests fail right now, but we don't want to block on that.
     - run: cargo build --package lightbeam
@@ -343,6 +352,21 @@ jobs:
         env:
           RUST_BACKTRACE: 1
 
+  # Build and test the wasi-crypto module.
+  test_wasi_crypto:
+    name: Test wasi-crypto module
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - run: rustup target add wasm32-wasi
+    - name: Install Rust
+      run: rustup update stable && rustup default stable
+    - run: ./ci/run-wasi-crypto-example.sh
+      env:
+        RUST_BACKTRACE: 1
+
   # Verify that cranelift's code generation is deterministic
   meta_determinist_check:
     name: Meta deterministic check
@@ -451,6 +475,7 @@ jobs:
             --exclude lightbeam \
             --exclude wasmtime-lightbeam \
             --exclude wasmtime-wasi-nn \
+            --exclude wasmtime-wasi-crypto \
             --exclude peepmatic \
             --exclude peepmatic-automata \
             --exclude peepmatic-fuzzing \
diff --git a/.gitmodules b/.gitmodules
index 0eb1df4e0f..ee264b99c4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -10,3 +10,6 @@
 [submodule "crates/wasi-nn/spec"]
 	path = crates/wasi-nn/spec
 	url = https://github.com/WebAssembly/wasi-nn
+[submodule "crates/wasi-crypto/spec"]
+	path = crates/wasi-crypto/spec
+	url = https://github.com/WebAssembly/wasi-crypto.git
diff --git a/Cargo.lock b/Cargo.lock
index 824da0733f..bcb3cd5862 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,9 +2,9 @@
 # It is not intended for manual editing.
 [[package]]
 name = "addr2line"
-version = "0.14.0"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423"
+checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7"
 dependencies = [
  "gimli",
 ]
@@ -15,6 +15,60 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
 
+[[package]]
+name = "aead"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "aes"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561"
+dependencies = [
+ "aes-soft",
+ "aesni",
+ "cipher",
+]
+
+[[package]]
+name = "aes-gcm"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da"
+dependencies = [
+ "aead",
+ "aes",
+ "cipher",
+ "ctr",
+ "ghash",
+ "subtle",
+]
+
+[[package]]
+name = "aes-soft"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072"
+dependencies = [
+ "cipher",
+ "opaque-debug",
+]
+
+[[package]]
+name = "aesni"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce"
+dependencies = [
+ "cipher",
+ "opaque-debug",
+]
+
 [[package]]
 name = "ahash"
 version = "0.4.7"
@@ -50,9 +104,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.35"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c0df63cb2955042487fad3aefd2c6e3ae7389ac5dc1beb28921de0b69f779d4"
+checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1"
 
 [[package]]
 name = "arbitrary"
@@ -86,6 +140,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "autocfg"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
+
 [[package]]
 name = "autocfg"
 version = "1.0.1"
@@ -94,9 +154,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "backtrace"
-version = "0.3.55"
+version = "0.3.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598"
+checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
 dependencies = [
  "addr2line",
  "cfg-if 1.0.0",
@@ -167,6 +227,17 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 
+[[package]]
+name = "bitvec"
+version = "0.18.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d2838fdd79e8776dbe07a106c784b0f8dda571a21b2750a092cc4cbaa653c8e"
+dependencies = [
+ "funty",
+ "radium",
+ "wyz",
+]
+
 [[package]]
 name = "blake2b_simd"
 version = "0.5.11"
@@ -189,32 +260,32 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.4.0"
+version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
+checksum = "f07aa6688c702439a1be0307b6a94dffe1168569e45b9500c1372bc580740d59"
 
 [[package]]
 name = "byteorder"
-version = "1.3.4"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
+checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
 
 [[package]]
 name = "cap-fs-ext"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce7f38ec6d0dd8a67d9553f95a61ee41fce23c9947bceac7bfe5b3711f9ab2fe"
+checksum = "0341ab6f42e202d53cac0823df52f1e3bc07f5f0407f03439a74de4902442c24"
 dependencies = [
  "cap-primitives",
  "cap-std",
- "unsafe-io 0.2.0",
+ "unsafe-io",
 ]
 
 [[package]]
 name = "cap-primitives"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0363d39135d2cd4694e669f858246839648e11de632c5107ad3ee8ae9d12249c"
+checksum = "0a639a9262b4f722e8cdec2b198ac62388ffc9b347cd0e84536e02419447f98d"
 dependencies = [
  "errno",
  "fs-set-times",
@@ -223,7 +294,7 @@ dependencies = [
  "maybe-owned",
  "once_cell",
  "posish",
- "unsafe-io 0.2.0",
+ "unsafe-io",
  "winapi",
  "winapi-util",
  "winx",
@@ -231,28 +302,28 @@ dependencies = [
 
 [[package]]
 name = "cap-rand"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ff901c15baead18c2abaae7345dbb2e2a84229c56310cefd303b0b6f2f82bd2"
+checksum = "703eebf88df08a21d6f689e0781babb9b03e446b65d8da9d7663306bf8d7499a"
 dependencies = [
- "rand 0.8.1",
+ "rand 0.8.3",
 ]
 
 [[package]]
 name = "cap-std"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e03b5ca2d4181980f2546074ad15a069f0f5860519e0d83e492c78b411d7c39f"
+checksum = "710da2154f2853ac49014a72a11f73cea87420d304fa0e5324266d350cfaad00"
 dependencies = [
  "cap-primitives",
- "unsafe-io 0.2.0",
+ "unsafe-io",
 ]
 
 [[package]]
 name = "cap-time-ext"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f51194e7bd1b6e77f10c367b1c7b21856a49142e70b4cd3d4da17b51c25ca76"
+checksum = "a366f04445d4d818ea2eb60a09ca5c81e35ac04fd612f82488171604e22598ee"
 dependencies = [
  "cap-primitives",
  "once_cell",
@@ -310,6 +381,29 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "chacha20"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed8738f14471a99f0e316c327e68fc82a3611cc2895fcb604b89eedaf8f39d95"
+dependencies = [
+ "cipher",
+ "zeroize",
+]
+
+[[package]]
+name = "chacha20poly1305"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af1fc18e6d90c40164bf6c317476f2a98f04661e310e79830366b7e914c58a8e"
+dependencies = [
+ "aead",
+ "chacha20",
+ "cipher",
+ "poly1305",
+ "zeroize",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.19"
@@ -323,6 +417,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "cipher"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "clang-sys"
 version = "1.0.3"
@@ -360,9 +463,9 @@ dependencies = [
 
 [[package]]
 name = "console"
-version = "0.13.0"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a50aab2529019abfabfa93f1e6c41ef392f91fbf179b347a7e96abb524884a08"
+checksum = "7cc80946b3480f421c2f17ed1cb841753a371c7c5104f51d507e13f532c856aa"
 dependencies = [
  "encode_unicode",
  "lazy_static",
@@ -371,14 +474,19 @@ dependencies = [
  "terminal_size",
  "unicode-width",
  "winapi",
- "winapi-util",
 ]
 
 [[package]]
-name = "const_fn"
-version = "0.4.4"
+name = "const-oid"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826"
+checksum = "c5d82796b70971fbb603900a5edc797a4d9be0f9ec1257f83a1dba0aa374e3e9"
+
+[[package]]
+name = "const_fn"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6"
 
 [[package]]
 name = "constant_time_eq"
@@ -402,6 +510,12 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
 
+[[package]]
+name = "cpuid-bool"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba"
+
 [[package]]
 name = "cranelift"
 version = "0.69.0"
@@ -439,7 +553,7 @@ dependencies = [
  "souper-ir",
  "target-lexicon",
  "thiserror",
- "wast 29.0.0",
+ "wast 32.0.0",
 ]
 
 [[package]]
@@ -544,7 +658,6 @@ name = "cranelift-native"
 version = "0.69.0"
 dependencies = [
  "cranelift-codegen",
- "raw-cpuid",
  "target-lexicon",
 ]
 
@@ -636,7 +749,7 @@ dependencies = [
  "cranelift-entity",
  "cranelift-frontend",
  "hashbrown",
- "itertools",
+ "itertools 0.10.0",
  "log",
  "serde",
  "smallvec",
@@ -696,11 +809,43 @@ version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
  "cfg-if 1.0.0",
  "lazy_static",
 ]
 
+[[package]]
+name = "crypto-mac"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6"
+dependencies = [
+ "generic-array",
+ "subtle",
+]
+
+[[package]]
+name = "ctr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f"
+dependencies = [
+ "cipher",
+]
+
+[[package]]
+name = "curve25519-dalek"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f627126b946c25a4638eec0ea634fc52506dea98db118aae985118ce7c3d723f"
+dependencies = [
+ "byteorder",
+ "digest",
+ "rand_core 0.5.1",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "cvt"
 version = "0.1.1"
@@ -710,6 +855,26 @@ dependencies = [
  "cfg-if 0.1.10",
 ]
 
+[[package]]
+name = "der"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51f59c66c30bb7445c8320a5f9233e437e3572368099f25532a59054328899b4"
+dependencies = [
+ "const-oid",
+]
+
+[[package]]
+name = "derivative"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "derive_arbitrary"
 version = "0.4.7"
@@ -734,9 +899,9 @@ dependencies = [
 
 [[package]]
 name = "derive_utils"
-version = "0.11.0"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64196eb9f551916167225134f1e8a90f0b5774331d3c900d6328fd94bafe3544"
+checksum = "532b4c15dccee12c7044f1fcad956e98410860b22231e44a3b827464797ca7bf"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -795,18 +960,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a"
 dependencies = [
  "libc",
- "redox_users",
+ "redox_users 0.3.5",
  "winapi",
 ]
 
 [[package]]
 name = "dirs-sys-next"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99de365f605554ae33f115102a02057d4fc18b01f3284d6870be0938743cfe7d"
+checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
 dependencies = [
  "libc",
- "redox_users",
+ "redox_users 0.4.0",
  "winapi",
 ]
 
@@ -818,9 +983,9 @@ checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
 
 [[package]]
 name = "dynasm"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62a59fbab09460c1569eeea9b5e4cf62f13f5198b1c2ba0e5196dd7fdd17cd42"
+checksum = "3d7d1242462849390bb2ad38aeed769499f1afc7383affa2ab0c1baa894c0200"
 dependencies = [
  "bitflags",
  "byteorder",
@@ -833,13 +998,47 @@ dependencies = [
 
 [[package]]
 name = "dynasmrt"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85bec3edae2841d37b1c3dc7f3fd403c9061f26e9ffeeee97a3ea909b1bb2ef1"
+checksum = "c1dd4d1d5ca12258cef339a57a7643e8b233a42dea9bb849630ddd9dd7726aa9"
 dependencies = [
  "byteorder",
  "dynasm",
- "memmap",
+ "memmap2",
+]
+
+[[package]]
+name = "ecdsa"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41fbdb4ff710acb4db8ca29f93b897529ea6d6a45626d5183b47e012aa6ae7e4"
+dependencies = [
+ "elliptic-curve",
+ "hmac",
+ "signature",
+]
+
+[[package]]
+name = "ed25519"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37c66a534cbb46ab4ea03477eae19d5c22c01da8258030280b7bd9d8433fb6ef"
+dependencies = [
+ "signature",
+]
+
+[[package]]
+name = "ed25519-dalek"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d"
+dependencies = [
+ "curve25519-dalek",
+ "ed25519",
+ "rand 0.7.3",
+ "serde",
+ "sha2",
+ "zeroize",
 ]
 
 [[package]]
@@ -848,6 +1047,23 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
+[[package]]
+name = "elliptic-curve"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "592b1c857559479c056b73a3053c717108a70e4dce320ad28c79c63f5c2e62ba"
+dependencies = [
+ "bitvec",
+ "digest",
+ "ff",
+ "generic-array",
+ "group",
+ "pkcs8",
+ "rand_core 0.5.1",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "encode_unicode"
 version = "0.3.6"
@@ -874,7 +1090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e"
 dependencies = [
  "atty",
- "humantime 2.0.1",
+ "humantime 2.1.0",
  "log",
  "regex",
  "termcolor",
@@ -919,6 +1135,17 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
 
+[[package]]
+name = "ff"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01646e077d4ebda82b73f1bca002ea1e91561a77df2431a9e79729bcc31950ef"
+dependencies = [
+ "bitvec",
+ "rand_core 0.5.1",
+ "subtle",
+]
+
 [[package]]
 name = "file-per-thread-logger"
 version = "0.1.4"
@@ -941,13 +1168,13 @@ dependencies = [
 
 [[package]]
 name = "filetime"
-version = "0.2.13"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe"
+checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
- "redox_syscall 0.1.57",
+ "redox_syscall 0.2.4",
  "winapi",
 ]
 
@@ -973,25 +1200,18 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d79238883cf0307100b90aba4a755d8051a3182305dfe7f649a1e9dc0517006f"
 
+[[package]]
+name = "funty"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"
+
 [[package]]
 name = "gcc"
 version = "0.3.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
 
-[[package]]
-name = "generator"
-version = "0.6.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc"
-dependencies = [
- "cc",
- "libc",
- "log",
- "rustc_version 0.2.3",
- "winapi",
-]
-
 [[package]]
 name = "generic-array"
 version = "0.14.4"
@@ -1004,24 +1224,34 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "libc",
  "wasi 0.9.0+wasi-snapshot-preview1",
 ]
 
 [[package]]
 name = "getrandom"
-version = "0.2.0"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee8025cf36f917e6a52cce185b7c7177689b838b7ec138364e50cc2277a56cf4"
+checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "libc",
- "wasi 0.9.0+wasi-snapshot-preview1",
+ "wasi 0.10.2+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "ghash"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375"
+dependencies = [
+ "opaque-debug",
+ "polyval",
 ]
 
 [[package]]
@@ -1041,6 +1271,17 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
+[[package]]
+name = "group"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc11f9f5fbf1943b48ae7c2bf6846e7d827a512d1be4f23af708f5ca5d01dde1"
+dependencies = [
+ "ff",
+ "rand_core 0.5.1",
+ "subtle",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.9.1"
@@ -1052,22 +1293,42 @@ dependencies = [
 
 [[package]]
 name = "heck"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
+checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac"
 dependencies = [
  "unicode-segmentation",
 ]
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.17"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8"
+checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
 dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "hkdf"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f"
+dependencies = [
+ "digest",
+ "hmac",
+]
+
+[[package]]
+name = "hmac"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15"
+dependencies = [
+ "crypto-mac",
+ "digest",
+]
+
 [[package]]
 name = "humantime"
 version = "1.3.0"
@@ -1079,9 +1340,9 @@ dependencies = [
 
 [[package]]
 name = "humantime"
-version = "2.0.1"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "id-arena"
@@ -1095,7 +1356,7 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
  "hashbrown",
  "serde",
 ]
@@ -1112,6 +1373,15 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "instant"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.3.0"
@@ -1120,9 +1390,9 @@ checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135"
 
 [[package]]
 name = "iter-enum"
-version = "0.2.5"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86a94bc12a53bf84b705acee29eb8697a5ea7b4587d836152499e5db0a6d52b9"
+checksum = "cad34f24d3b48ceffdff38af2df5ce1b7d1d9cc113e503d8e86fe8cdb889c871"
 dependencies = [
  "derive_utils",
  "quote",
@@ -1138,6 +1408,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "0.4.7"
@@ -1162,11 +1441,25 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "k256"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf02ecc966e1b7e8db1c81ac8f321ba24d1cfab5b634961fab10111f015858e1"
+dependencies = [
+ "cfg-if 1.0.0",
+ "ecdsa",
+ "elliptic-curve",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+dependencies = [
+ "spin",
+]
 
 [[package]]
 name = "lazycell"
@@ -1182,9 +1475,9 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a"
 
 [[package]]
 name = "libc"
-version = "0.2.82"
+version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
+checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff"
 
 [[package]]
 name = "libfuzzer-sys"
@@ -1198,14 +1491,20 @@ dependencies = [
 
 [[package]]
 name = "libloading"
-version = "0.6.6"
+version = "0.6.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9367bdfa836b7e3cf895867f7a570283444da90562980ec2263d6e1569b16bc"
+checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883"
 dependencies = [
  "cfg-if 1.0.0",
  "winapi",
 ]
 
+[[package]]
+name = "libm"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a"
+
 [[package]]
 name = "lightbeam"
 version = "0.22.0"
@@ -1218,7 +1517,7 @@ dependencies = [
  "dynasm",
  "dynasmrt",
  "iter-enum",
- "itertools",
+ "itertools 0.10.0",
  "lazy_static",
  "memoffset",
  "more-asserts",
@@ -1231,25 +1530,21 @@ dependencies = [
 ]
 
 [[package]]
-name = "log"
-version = "0.4.11"
+name = "lock_api"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
+checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
 dependencies = [
- "cfg-if 0.1.10",
+ "scopeguard",
 ]
 
 [[package]]
-name = "loom"
-version = "0.3.6"
+name = "log"
+version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0e8460f2f2121162705187214720353c517b97bdfb3494c0b1e33d83ebe4bed"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
 dependencies = [
- "cfg-if 0.1.10",
- "generator",
- "scoped-tls",
- "serde",
- "serde_json",
+ "cfg-if 1.0.0",
 ]
 
 [[package]]
@@ -1292,13 +1587,22 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "memmap2"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e73be3b7d04a0123e933fea1d50d126cc7196bbc0362c0ce426694f777194eee"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "memoffset"
 version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
 ]
 
 [[package]]
@@ -1314,7 +1618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
 dependencies = [
  "adler",
- "autocfg",
+ "autocfg 1.0.1",
 ]
 
 [[package]]
@@ -1339,18 +1643,59 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
  "num-integer",
  "num-traits",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e9a41747ae4633fce5adffb4d2e81ffc5e89593cb19917f8fb2cc5ff76507bf"
+dependencies = [
+ "autocfg 1.0.1",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-bigint-dig"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d51546d704f52ef14b3c962b5776e53d5b862e5790e40a350d366c209bd7f7a"
+dependencies = [
+ "autocfg 0.1.7",
+ "byteorder",
+ "lazy_static",
+ "libm",
+ "num-integer",
+ "num-iter",
+ "num-traits",
+ "rand 0.7.3",
+ "serde",
+ "smallvec",
+ "zeroize",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
+ "num-traits",
+]
+
+[[package]]
+name = "num-iter"
+version = "0.1.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59"
+dependencies = [
+ "autocfg 1.0.1",
+ "num-integer",
  "num-traits",
 ]
 
@@ -1360,8 +1705,8 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
 dependencies = [
- "autocfg",
- "num-bigint",
+ "autocfg 1.0.1",
+ "num-bigint 0.2.6",
  "num-integer",
  "num-traits",
 ]
@@ -1372,7 +1717,7 @@ version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
 ]
 
 [[package]]
@@ -1393,9 +1738,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
 
 [[package]]
 name = "object"
-version = "0.22.0"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397"
+checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
 dependencies = [
  "crc32fast",
  "indexmap",
@@ -1415,9 +1760,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openvino"
-version = "0.1.5"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87a74f90f07f134153e3ad2ffa724a3ebda92cdc6e099f7fe7d9185cf960f028"
+checksum = "43eeb44285b7ce8e2012b92bec32968622e1dad452e812e6edea9e001e5e9410"
 dependencies = [
  "openvino-sys",
  "thiserror",
@@ -1425,9 +1770,9 @@ dependencies = [
 
 [[package]]
 name = "openvino-sys"
-version = "0.1.5"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e72a2e5bd353bd3cf39b2663767e0ae0325a7588c47fd496cbf9a09237ef7ca8"
+checksum = "8fb64bef270a1ff665b0b2e28ebfa213e6205a007ce88223d020730225d6008f"
 dependencies = [
  "bindgen",
  "cmake",
@@ -1443,12 +1788,48 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "p256"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ca0196a204bb3f33305ba4a48b38f6e6e621cba8603a4e0650e6532e0949de4"
+dependencies = [
+ "ecdsa",
+ "elliptic-curve",
+ "sha2",
+]
+
 [[package]]
 name = "parity-wasm"
 version = "0.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865"
 
+[[package]]
+name = "parking_lot"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
+dependencies = [
+ "instant",
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272"
+dependencies = [
+ "cfg-if 1.0.0",
+ "instant",
+ "libc",
+ "redox_syscall 0.1.57",
+ "smallvec",
+ "winapi",
+]
+
 [[package]]
 name = "peeking_take_while"
 version = "0.1.2"
@@ -1466,7 +1847,7 @@ dependencies = [
  "peepmatic-test-operator",
  "peepmatic-traits",
  "serde",
- "wast 29.0.0",
+ "wast 32.0.0",
  "z3",
 ]
 
@@ -1492,9 +1873,9 @@ dependencies = [
  "peepmatic-test",
  "peepmatic-test-operator",
  "peepmatic-traits",
- "rand 0.7.3",
+ "rand 0.8.3",
  "serde",
- "wast 29.0.0",
+ "wast 32.0.0",
 ]
 
 [[package]]
@@ -1519,7 +1900,7 @@ dependencies = [
  "serde",
  "serde_test",
  "thiserror",
- "wast 29.0.0",
+ "wast 32.0.0",
 ]
 
 [[package]]
@@ -1531,7 +1912,7 @@ dependencies = [
  "peepmatic",
  "peepmatic-test-operator",
  "souper-ir",
- "wast 29.0.0",
+ "wast 32.0.0",
 ]
 
 [[package]]
@@ -1552,13 +1933,24 @@ version = "0.69.0"
 dependencies = [
  "peepmatic-traits",
  "serde",
- "wast 29.0.0",
+ "wast 32.0.0",
 ]
 
 [[package]]
 name = "peepmatic-traits"
 version = "0.69.0"
 
+[[package]]
+name = "pem"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c220d01f863d13d96ca82359d1e81e64a7c6bf0637bcde7b2349630addf0c6"
+dependencies = [
+ "base64",
+ "once_cell",
+ "regex",
+]
+
 [[package]]
 name = "pest"
 version = "2.1.3"
@@ -1570,9 +1962,41 @@ dependencies = [
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.0"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c"
+checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827"
+
+[[package]]
+name = "pkcs8"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4839a901843f3942576e65857f0ebf2e190ef7024d3c62a94099ba3f819ad1d"
+dependencies = [
+ "der",
+ "subtle-encoding",
+ "zeroize",
+]
+
+[[package]]
+name = "poly1305"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b7456bc1ad2d4cf82b3a016be4c2ac48daf11bf990c1603ebd447fe6f30fca8"
+dependencies = [
+ "cpuid-bool 0.2.0",
+ "universal-hash",
+]
+
+[[package]]
+name = "polyval"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd"
+dependencies = [
+ "cpuid-bool 0.2.0",
+ "opaque-debug",
+ "universal-hash",
+]
 
 [[package]]
 name = "posish"
@@ -1593,6 +2017,34 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
 
+[[package]]
+name = "pqcrypto"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d3874384bf37d988b83f806d632e2f7fca69a8cd0338efaa64e8e7664573052"
+dependencies = [
+ "pqcrypto-kyber",
+ "pqcrypto-traits",
+]
+
+[[package]]
+name = "pqcrypto-kyber"
+version = "0.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33550a5b6e0844d1b2363f67e15e4ca64586bb4fb2363a83af762e6c2d092bff"
+dependencies = [
+ "cc",
+ "glob",
+ "libc",
+ "pqcrypto-traits",
+]
+
+[[package]]
+name = "pqcrypto-traits"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4e1563eff60a9ae869cacee0a33fa5c4ba27861fec6e3e23de95eb0ae805e4b"
+
 [[package]]
 name = "pretty_env_logger"
 version = "0.4.0"
@@ -1673,44 +2125,48 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
 [[package]]
 name = "quickcheck"
-version = "0.9.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f"
+checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6"
 dependencies = [
- "env_logger 0.7.1",
+ "env_logger 0.8.2",
  "log",
- "rand 0.7.3",
- "rand_core 0.5.1",
+ "rand 0.8.3",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
+checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
 dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "radium"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64de9a0c5361e034f1aefc9f71a86871ec870e766fe31a009734a989b329286a"
+
 [[package]]
 name = "rand"
 version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
 dependencies = [
- "getrandom 0.1.15",
+ "getrandom 0.1.16",
  "libc",
  "rand_chacha 0.2.2",
  "rand_core 0.5.1",
  "rand_hc 0.2.0",
- "rand_pcg",
 ]
 
 [[package]]
 name = "rand"
-version = "0.8.1"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c24fcd450d3fa2b592732565aa4f17a27a61c65ece4726353e000939b0edee34"
+checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
 dependencies = [
  "libc",
  "rand_chacha 0.3.0",
@@ -1744,7 +2200,7 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
 dependencies = [
- "getrandom 0.1.15",
+ "getrandom 0.1.16",
 ]
 
 [[package]]
@@ -1753,7 +2209,7 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5"
 dependencies = [
- "getrandom 0.2.0",
+ "getrandom 0.2.2",
 ]
 
 [[package]]
@@ -1774,15 +2230,6 @@ dependencies = [
  "rand_core 0.6.1",
 ]
 
-[[package]]
-name = "rand_pcg"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
-dependencies = [
- "rand_core 0.5.1",
-]
-
 [[package]]
 name = "rand_xorshift"
 version = "0.2.0"
@@ -1793,15 +2240,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "raw-cpuid"
-version = "8.1.2"
+name = "rawbytes"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fdf7d9dbd43f3d81d94a49c1c3df73cc2b3827995147e6cf7f89d4ec5483e73"
-dependencies = [
- "bitflags",
- "cc",
- "rustc_version 0.2.3",
-]
+checksum = "a26d81f4c222fd11ad63bf56cbda89d1810aecf1a720a423ff7eb2020475d8bb"
 
 [[package]]
 name = "rayon"
@@ -1809,7 +2251,7 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
  "crossbeam-deque",
  "either",
  "rayon-core",
@@ -1849,11 +2291,21 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d"
 dependencies = [
- "getrandom 0.1.15",
+ "getrandom 0.1.16",
  "redox_syscall 0.1.57",
  "rust-argon2",
 ]
 
+[[package]]
+name = "redox_users"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
+dependencies = [
+ "getrandom 0.2.2",
+ "redox_syscall 0.2.4",
+]
+
 [[package]]
 name = "regalloc"
 version = "0.0.31"
@@ -1868,9 +2320,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.4.2"
+version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c"
+checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1890,9 +2342,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.21"
+version = "0.6.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189"
+checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
 
 [[package]]
 name = "region"
@@ -1915,6 +2367,40 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "rsa"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3648b669b10afeab18972c105e284a7b953a669b0be3514c27f9b17acab2f9cd"
+dependencies = [
+ "byteorder",
+ "digest",
+ "lazy_static",
+ "num-bigint-dig",
+ "num-integer",
+ "num-iter",
+ "num-traits",
+ "pem",
+ "rand 0.7.3",
+ "sha2",
+ "simple_asn1 0.4.1",
+ "subtle",
+ "thiserror",
+ "zeroize",
+]
+
+[[package]]
+name = "rsa-export"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "358de25c89a5a71597ebc85f7ad222e2c67ed553e0ce31170104c3a77296a01c"
+dependencies = [
+ "num-bigint-dig",
+ "pem",
+ "rsa",
+ "simple_asn1 0.5.1",
+]
+
 [[package]]
 name = "run-examples"
 version = "0.19.0"
@@ -1947,22 +2433,13 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
-[[package]]
-name = "rustc_version"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-dependencies = [
- "semver 0.9.0",
-]
-
 [[package]]
 name = "rustc_version"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee"
 dependencies = [
- "semver 0.11.0",
+ "semver",
 ]
 
 [[package]]
@@ -1992,12 +2469,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "scoped-tls"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
-
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -2015,39 +2486,24 @@ dependencies = [
 
 [[package]]
 name = "scroll_derive"
-version = "0.10.4"
+version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b12bd20b94c7cdfda8c7ba9b92ad0d9a56e3fa018c25fca83b51aa664c9b4c0d"
+checksum = "aaaae8f38bb311444cfb7f1979af0bc9240d95795f75f9ceddf6a59b79ceffa0"
 dependencies = [
  "proc-macro2",
  "quote",
  "syn",
 ]
 
-[[package]]
-name = "semver"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
-dependencies = [
- "semver-parser 0.7.0",
-]
-
 [[package]]
 name = "semver"
 version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6"
 dependencies = [
- "semver-parser 0.10.2",
+ "semver-parser",
 ]
 
-[[package]]
-name = "semver-parser"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-
 [[package]]
 name = "semver-parser"
 version = "0.10.2"
@@ -2059,18 +2515,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.118"
+version = "1.0.123"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800"
+checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.118"
+version = "1.0.123"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df"
+checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2079,9 +2535,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.60"
+version = "1.0.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779"
+checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a"
 dependencies = [
  "itoa",
  "ryu",
@@ -2090,9 +2546,9 @@ dependencies = [
 
 [[package]]
 name = "serde_test"
-version = "1.0.118"
+version = "1.0.123"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7f3f8714511d29f60be0ea965bc784df1b6903da5bbac801df36b1bbc7b4880"
+checksum = "38145a8510bdf71d9a8cceeb57664049538446e77f24648328bdbcf22dc7e169"
 dependencies = [
  "serde",
 ]
@@ -2105,19 +2561,18 @@ checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8"
 dependencies = [
  "block-buffer",
  "cfg-if 1.0.0",
- "cpuid-bool",
+ "cpuid-bool 0.1.2",
  "digest",
  "opaque-debug",
 ]
 
 [[package]]
 name = "sharded-slab"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127"
+checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3"
 dependencies = [
  "lazy_static",
- "loom",
 ]
 
 [[package]]
@@ -2135,6 +2590,51 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
 
+[[package]]
+name = "shuffling-allocator"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ee9977fa98489d9006f4ab26fc5cbe2a139985baed09d2ec08dee6e506fc496"
+dependencies = [
+ "cfg-if 1.0.0",
+ "libc",
+ "rand 0.8.3",
+ "winapi",
+]
+
+[[package]]
+name = "signature"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29f060a7d147e33490ec10da418795238fd7545bba241504d6b31a409f2e6210"
+dependencies = [
+ "digest",
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "simple_asn1"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b"
+dependencies = [
+ "chrono",
+ "num-bigint 0.2.6",
+ "num-traits",
+]
+
+[[package]]
+name = "simple_asn1"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db8d597fce66eb0f19dd129b9956e4054cba21aeaf97d4116595027b670fac50"
+dependencies = [
+ "chrono",
+ "num-bigint 0.3.1",
+ "num-traits",
+ "thiserror",
+]
+
 [[package]]
 name = "smallvec"
 version = "1.6.1"
@@ -2150,6 +2650,12 @@ dependencies = [
  "id-arena",
 ]
 
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.0"
@@ -2187,10 +2693,25 @@ dependencies = [
 ]
 
 [[package]]
-name = "syn"
-version = "1.0.54"
+name = "subtle"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44"
+checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
+
+[[package]]
+name = "subtle-encoding"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dcb1ed7b8330c5eed5441052651dd7a12c75e2ed88f2ec024ae1fa3a5e59945"
+dependencies = [
+ "zeroize",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2198,18 +2719,30 @@ dependencies = [
 ]
 
 [[package]]
-name = "system-interface"
-version = "0.5.4"
+name = "synstructure"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a02b6a59a0980c8704e45b8498f07020248dc8a6734a8b3d37649800fb80a87f"
+checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "unicode-xid",
+]
+
+[[package]]
+name = "system-interface"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae234a0c346c613b2bae30491bf786d9edbf005c63fa267424da0d55dfdd2cc7"
 dependencies = [
  "atty",
  "bitflags",
  "cap-fs-ext",
  "cap-std",
  "posish",
- "rustc_version 0.3.3",
- "unsafe-io 0.3.0",
+ "rustc_version",
+ "unsafe-io",
  "winapi",
  "winx",
 ]
@@ -2228,7 +2761,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
- "rand 0.8.1",
+ "rand 0.8.3",
  "redox_syscall 0.2.4",
  "remove_dir_all",
  "winapi",
@@ -2255,9 +2788,9 @@ dependencies = [
 
 [[package]]
 name = "terminal_size"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bd2d183bd3fac5f5fe38ddbeb4dc9aec4a39a9d7d59e7491d900302da01cbe1"
+checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406"
 dependencies = [
  "libc",
  "winapi",
@@ -2292,18 +2825,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.22"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e"
+checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.22"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56"
+checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2312,29 +2845,28 @@ dependencies = [
 
 [[package]]
 name = "thread_local"
-version = "1.0.1"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
+checksum = "d8208a331e1cb318dd5bd76951d2b8fc48ca38a69f5f4e4af1b6a9f8c6236915"
 dependencies = [
- "lazy_static",
+ "once_cell",
 ]
 
 [[package]]
 name = "time"
-version = "0.1.44"
+version = "0.1.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
+checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
 dependencies = [
  "libc",
- "wasi 0.10.0+wasi-snapshot-preview1",
  "winapi",
 ]
 
 [[package]]
 name = "toml"
-version = "0.5.7"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75cf45bb0bef80604d001caaec0d09da99611b3c0fd39d3080468875cdb65645"
+checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
 dependencies = [
  "serde",
 ]
@@ -2460,6 +2992,16 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
 
+[[package]]
+name = "universal-hash"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402"
+dependencies = [
+ "generic-array",
+ "subtle",
+]
+
 [[package]]
 name = "unsafe-any"
 version = "0.4.2"
@@ -2469,22 +3011,13 @@ dependencies = [
  "traitobject",
 ]
 
-[[package]]
-name = "unsafe-io"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80ba583908d065c07f1eaf5eb389b9454f8bc735d182c8b93538e274568509b0"
-dependencies = [
- "rustc_version 0.3.3",
-]
-
 [[package]]
 name = "unsafe-io"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd35327a2b46b3350186b75a3a337d4517e5ca08118c4e54175d49d7832578d8"
 dependencies = [
- "rustc_version 0.3.3",
+ "rustc_version",
 ]
 
 [[package]]
@@ -2527,9 +3060,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
 
 [[package]]
 name = "wasi"
-version = "0.10.0+wasi-snapshot-preview1"
+version = "0.10.2+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
 
 [[package]]
 name = "wasi-cap-std-sync"
@@ -2546,7 +3079,7 @@ dependencies = [
  "libc",
  "system-interface",
  "tracing",
- "unsafe-io 0.3.0",
+ "unsafe-io",
  "wasi-common",
  "winapi",
 ]
@@ -2567,22 +3100,52 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "wasi-crypto"
+version = "0.1.4"
+dependencies = [
+ "aes-gcm",
+ "anyhow",
+ "bincode",
+ "byteorder",
+ "chacha20poly1305",
+ "curve25519-dalek",
+ "derivative",
+ "ed25519-dalek",
+ "hkdf",
+ "hmac",
+ "k256",
+ "p256",
+ "parking_lot",
+ "pqcrypto",
+ "rand_core 0.5.1",
+ "rsa",
+ "rsa-export",
+ "serde",
+ "sha2",
+ "subtle",
+ "thiserror",
+ "xoodyak",
+ "zeroize",
+]
+
 [[package]]
 name = "wasm-encoder"
-version = "0.2.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed89eaf99e08b84f96e477a16588a07dd3b51dc5f07291c3706782f62a10a5e1"
+checksum = "c75fa62cf1464aa6655479ae454202a159cc82b7b4d66e8f174409669c0654c5"
 dependencies = [
  "leb128",
 ]
 
 [[package]]
 name = "wasm-smith"
-version = "0.3.0"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "509904d9c4c4659ac238a3f27c3656dd6d3931697eddd4b0f32e335769c298d0"
+checksum = "5af0d70f17515b1bc412b7727b01304ba2484bc416da72f325d6bf4ab58f4213"
 dependencies = [
  "arbitrary",
+ "indexmap",
  "leb128",
  "wasm-encoder",
 ]
@@ -2613,15 +3176,15 @@ dependencies = [
 
 [[package]]
 name = "wasmparser"
-version = "0.71.0"
+version = "0.73.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89a30c99437829ede826802bfcf28500cf58df00e66cb9114df98813bc145ff1"
+checksum = "b8526ab131cbc49495a483c98954913ae7b83551adacab5e294cf77992e70ee7"
 
 [[package]]
 name = "wasmprinter"
-version = "0.2.18"
+version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0515db67c610037f3c53ec36976edfd1eb01bac6b1226914b17ce609480e729f"
+checksum = "edf67b8f2b3b49a5ca4f8ab6879c1c70057a6b2c14474d6126be912051f2392e"
 dependencies = [
  "anyhow",
  "wasmparser",
@@ -2661,6 +3224,7 @@ name = "wasmtime-bench-api"
 version = "0.19.0"
 dependencies = [
  "anyhow",
+ "shuffling-allocator",
  "wasi-common",
  "wasmtime",
  "wasmtime-wasi",
@@ -2722,7 +3286,7 @@ dependencies = [
  "env_logger 0.8.2",
  "file-per-thread-logger",
  "filecheck",
- "humantime 2.0.1",
+ "humantime 2.1.0",
  "libc",
  "log",
  "more-asserts",
@@ -2746,8 +3310,10 @@ dependencies = [
  "wasmtime-obj",
  "wasmtime-runtime",
  "wasmtime-wasi",
+ "wasmtime-wasi-crypto",
  "wasmtime-wasi-nn",
  "wasmtime-wast",
+ "wast 32.0.0",
  "wat",
 ]
 
@@ -2759,6 +3325,7 @@ dependencies = [
  "cranelift-entity",
  "cranelift-frontend",
  "cranelift-wasm",
+ "wasmparser",
  "wasmtime-environ",
 ]
 
@@ -2950,6 +3517,17 @@ dependencies = [
  "wiggle",
 ]
 
+[[package]]
+name = "wasmtime-wasi-crypto"
+version = "0.22.0"
+dependencies = [
+ "anyhow",
+ "wasi-crypto",
+ "wasmtime",
+ "wasmtime-wiggle",
+ "wiggle",
+]
+
 [[package]]
 name = "wasmtime-wasi-nn"
 version = "0.22.0"
@@ -2972,7 +3550,7 @@ version = "0.22.0"
 dependencies = [
  "anyhow",
  "wasmtime",
- "wast 29.0.0",
+ "wast 32.0.0",
 ]
 
 [[package]]
@@ -3008,29 +3586,20 @@ dependencies = [
 
 [[package]]
 name = "wast"
-version = "29.0.0"
+version = "32.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf2268937131d63c3d833242bf5e075406f9ed868b4265f3280e15dac29ac18"
-dependencies = [
- "leb128",
-]
-
-[[package]]
-name = "wast"
-version = "30.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b79907b22f740634810e882d8d1d9d0f9563095a8ab94e786e370242bff5cd2"
+checksum = "c24a3ee360d01d60ed0a0f960ab76a6acce64348cdb0bf8699c2a866fad57c7c"
 dependencies = [
  "leb128",
 ]
 
 [[package]]
 name = "wat"
-version = "1.0.31"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8279a02835bf12e61ed2b3c3cbc6ecf9918762fd97e036917c11a09ec20ca44"
+checksum = "5e8f7f34773fa6318e8897283abf7941c1f250faae4e1a52f82df09c3bad7cce"
 dependencies = [
- "wast 30.0.0",
+ "wast 32.0.0",
 ]
 
 [[package]]
@@ -3154,6 +3723,22 @@ dependencies = [
  "wast 22.0.0",
 ]
 
+[[package]]
+name = "wyz"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"
+
+[[package]]
+name = "xoodyak"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f9c85605c3a376cec858899f7d284f453359743adaeddf09c7d6ef18474a481"
+dependencies = [
+ "rawbytes",
+ "zeroize",
+]
+
 [[package]]
 name = "z3"
 version = "0.7.1"
@@ -3175,19 +3760,40 @@ dependencies = [
 ]
 
 [[package]]
-name = "zstd"
-version = "0.5.4+zstd.1.4.7"
+name = "zeroize"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69996ebdb1ba8b1517f61387a883857818a66c8a295f487b1ffd8fd9d2c82910"
+checksum = "81a974bcdd357f0dca4d41677db03436324d45a4c9ed2d0b873a5a360ce41c36"
+dependencies = [
+ "zeroize_derive",
+]
+
+[[package]]
+name = "zeroize_derive"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3f369ddb18862aba61aa49bf31e74d29f0f162dec753063200e1dc084345d16"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "synstructure",
+]
+
+[[package]]
+name = "zstd"
+version = "0.6.0+zstd.1.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4e44664feba7f2f1a9f300c1f6157f2d1bfc3c15c6f3cf4beabf3f5abe9c237"
 dependencies = [
  "zstd-safe",
 ]
 
 [[package]]
 name = "zstd-safe"
-version = "2.0.6+zstd.1.4.7"
+version = "3.0.0+zstd.1.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98aa931fb69ecee256d44589d19754e61851ae4769bf963b385119b1cc37a49e"
+checksum = "d9447afcd795693ad59918c7bbffe42fdd6e467d708f3537e3dc14dc598c573f"
 dependencies = [
  "libc",
  "zstd-sys",
@@ -3195,12 +3801,12 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "1.4.18+zstd.1.4.7"
+version = "1.4.19+zstd.1.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81"
+checksum = "ec24a9273d24437afb8e71b16f3d9a5d569193cccdb7896213b59f552f387674"
 dependencies = [
  "cc",
  "glob",
- "itertools",
+ "itertools 0.9.0",
  "libc",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 8a8e5faf87..1cd4a7477c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -30,22 +30,23 @@ wasmtime-jit = { path = "crates/jit", version = "0.22.0" }
 wasmtime-obj = { path = "crates/obj", version = "0.22.0" }
 wasmtime-wast = { path = "crates/wast", version = "0.22.0" }
 wasmtime-wasi = { path = "crates/wasi", version = "0.22.0" }
+wasmtime-wasi-crypto = { path = "crates/wasi-crypto", version = "0.22.0", optional = true }
 wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "0.22.0", optional = true }
 wasi-common = { path = "crates/wasi-common", version = "0.22.0" }
 wasi-cap-std-sync = { path = "crates/wasi-common/cap-std-sync", version = "0.22.0" }
 structopt = { version = "0.3.5", features = ["color", "suggestions"] }
-object = { version = "0.22.0", default-features = false, features = ["write"] }
+object = { version = "0.23.0", default-features = false, features = ["write"] }
 anyhow = "1.0.19"
 target-lexicon = { version = "0.11.0", default-features = false }
 pretty_env_logger = "0.4.0"
 file-per-thread-logger = "0.1.1"
-wat = "1.0.30"
+wat = "1.0.33"
 libc = "0.2.60"
 log = "0.4.8"
 rayon = "1.2.1"
 humantime = "2.0.0"
-wasmparser = "0.71.0"
-cap-std = "0.11.0"
+wasmparser = "0.73.0"
+cap-std = "0.12"
 
 [dev-dependencies]
 env_logger = "0.8.1"
@@ -56,6 +57,7 @@ test-programs = { path = "crates/test-programs" }
 wasmtime-fuzzing = { path = "crates/fuzzing" }
 wasmtime-runtime = { path = "crates/runtime" }
 tracing-subscriber = "0.2.0"
+wast = "32.0.0"
 
 [build-dependencies]
 anyhow = "1.0.19"
@@ -88,6 +90,7 @@ default = ["jitdump", "wasmtime/wat", "wasmtime/parallel-compilation"]
 lightbeam = ["wasmtime/lightbeam"]
 jitdump = ["wasmtime/jitdump"]
 vtune = ["wasmtime/vtune"]
+wasi-crypto = ["wasmtime-wasi-crypto"]
 wasi-nn = ["wasmtime-wasi-nn"]
 
 # Try the experimental, work-in-progress new x86_64 backend. This is not stable
diff --git a/build.rs b/build.rs
index 35257cdb7b..9827fd2b0b 100644
--- a/build.rs
+++ b/build.rs
@@ -176,10 +176,52 @@ fn write_testsuite_tests(
 
 /// For experimental_x64 backend features that are not supported yet, mark tests as panicking, so
 /// they stop "passing" once the features are properly implemented.
-///
-/// TODO(#2470): removed all tests from this set as we are disabling x64 SIMD tests unconditionally
-/// instead until we resolve a nondeterminism bug. Restore when fixed.
-fn experimental_x64_should_panic(_testsuite: &str, _testname: &str, _strategy: &str) -> bool {
+fn experimental_x64_should_panic(testsuite: &str, testname: &str, strategy: &str) -> bool {
+    if !cfg!(feature = "experimental_x64") || strategy != "Cranelift" {
+        return false;
+    }
+
+    match (testsuite, testname) {
+        ("simd", "simd_address") => return false,
+        ("simd", "simd_align") => return false,
+        ("simd", "simd_bitwise") => return false,
+        ("simd", "simd_bit_shift") => return false,
+        ("simd", "simd_boolean") => return false,
+        ("simd", "simd_const") => return false,
+        ("simd", "simd_i8x16_arith") => return false,
+        ("simd", "simd_i8x16_arith2") => return false,
+        ("simd", "simd_i8x16_cmp") => return false,
+        ("simd", "simd_i8x16_sat_arith") => return false,
+        ("simd", "simd_i16x8_arith") => return false,
+        ("simd", "simd_i16x8_arith2") => return false,
+        ("simd", "simd_i16x8_cmp") => return false,
+        ("simd", "simd_i16x8_sat_arith") => return false,
+        ("simd", "simd_i32x4_arith") => return false,
+        ("simd", "simd_i32x4_arith2") => return false,
+        ("simd", "simd_i32x4_cmp") => return false,
+        ("simd", "simd_i32x4_dot_i16x8") => return false,
+        ("simd", "simd_i64x2_arith") => return false,
+        ("simd", "simd_f32x4") => return false,
+        ("simd", "simd_f32x4_arith") => return false,
+        ("simd", "simd_f32x4_cmp") => return false,
+        ("simd", "simd_f32x4_pmin_pmax") => return false,
+        ("simd", "simd_f64x2") => return false,
+        ("simd", "simd_f64x2_arith") => return false,
+        ("simd", "simd_f64x2_cmp") => return false,
+        ("simd", "simd_f64x2_pmin_pmax") => return false,
+        ("simd", "simd_lane") => return false,
+        ("simd", "simd_load") => return false,
+        ("simd", "simd_load_extend") => return false,
+        ("simd", "simd_load_splat") => return false,
+        ("simd", "simd_load_zero") => return false,
+        ("simd", "simd_splat") => return false,
+        ("simd", "simd_store") => return false,
+        ("simd", "simd_conversions") => return false,
+        ("simd", "simd_f32x4_rounding") => return false,
+        ("simd", "simd_f64x2_rounding") => return false,
+        ("simd", _) => return true,
+        _ => {}
+    }
     false
 }
 
@@ -201,14 +243,12 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
                 return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "x86_64";
             }
 
-            // Ignore all x64 SIMD tests for now (#2470).
-            ("simd", _) if cfg!(feature = "experimental_x64") => {
-                return env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "x86_64";
-            }
+            // Waiting for an update to the spec testsuite to not use old
+            // instruction names.
+            ("simd", "simd_boolean") | ("simd", "simd_lane") => return true,
 
             // These are only implemented on aarch64 and x64.
-            ("simd", "simd_boolean")
-            | ("simd", "simd_f32x4_pmin_pmax")
+            ("simd", "simd_f32x4_pmin_pmax")
             | ("simd", "simd_f64x2_pmin_pmax")
             | ("simd", "simd_f32x4_rounding")
             | ("simd", "simd_f64x2_rounding")
diff --git a/ci/run-experimental-x64-ci.sh b/ci/run-experimental-x64-ci.sh
index 8b41831b15..6b11352d47 100755
--- a/ci/run-experimental-x64-ci.sh
+++ b/ci/run-experimental-x64-ci.sh
@@ -14,6 +14,7 @@ cargo $CARGO_VERSION \
             --all \
             --exclude wasmtime-lightbeam \
             --exclude wasmtime-wasi-nn \
+            --exclude wasmtime-wasi-crypto \
             --exclude peepmatic \
             --exclude peepmatic-automata \
             --exclude peepmatic-fuzzing \
diff --git a/ci/run-wasi-crypto-example.sh b/ci/run-wasi-crypto-example.sh
new file mode 100755
index 0000000000..d2582c71b1
--- /dev/null
+++ b/ci/run-wasi-crypto-example.sh
@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+RUST_BINDINGS="crates/wasi-crypto/spec/implementations/bindings/rust"
+pushd "$RUST_BINDINGS"
+cargo build --release --target=wasm32-wasi
+popd
+
+cargo run --features wasi-crypto -- run "$RUST_BINDINGS/target/wasm32-wasi/release/wasi-crypto-guest.wasm"
diff --git a/cranelift/README.md b/cranelift/README.md
index 52d4e0cf8f..b083122b37 100644
--- a/cranelift/README.md
+++ b/cranelift/README.md
@@ -19,7 +19,7 @@ For more information, see [the documentation](docs/index.md).
 For an example of how to use the JIT, see the [JIT Demo], which
 implements a toy language.
 
-[JIT Demo]: https://github.com/bytecodealliance/simplejit-demo
+[JIT Demo]: https://github.com/bytecodealliance/cranelift-jit-demo
 
 For an example of how to use Cranelift to run WebAssembly code, see
 [Wasmtime], which implements a standalone, embeddable, VM using Cranelift.
diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml
index d8a8baf384..e7a618ba46 100644
--- a/cranelift/codegen/Cargo.toml
+++ b/cranelift/codegen/Cargo.toml
@@ -30,7 +30,7 @@ peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, versi
 peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.69.0" }
 regalloc = { version = "0.0.31" }
 souper-ir = { version = "2.1.0", optional = true }
-wast = { version = "29.0.0", optional = true }
+wast = { version = "32.0.0", optional = true }
 # It is a goal of the cranelift-codegen crate to have minimal external dependencies.
 # Please don't add any unless they are essential to the task of creating binary
 # machine code. Integration tests that need external dependencies can be
@@ -74,7 +74,7 @@ all-arch = [
 ]
 
 # For dependent crates that want to serialize some parts of cranelift
-enable-serde = ["serde"]
+enable-serde = ["serde", "regalloc/enable-serde"]
 
 # Allow snapshotting regalloc test cases. Useful only to report bad register
 # allocation failures, or for regalloc.rs developers.
diff --git a/cranelift/codegen/meta/src/gen_settings.rs b/cranelift/codegen/meta/src/gen_settings.rs
index 2ed5941b80..a70ddccfe1 100644
--- a/cranelift/codegen/meta/src/gen_settings.rs
+++ b/cranelift/codegen/meta/src/gen_settings.rs
@@ -418,7 +418,7 @@ fn gen_display(group: &SettingGroup, fmt: &mut Formatter) {
 
 fn gen_group(group: &SettingGroup, parent: ParentGroup, fmt: &mut Formatter) {
     // Generate struct.
-    fmtln!(fmt, "#[derive(Clone)]");
+    fmtln!(fmt, "#[derive(Clone, Hash)]");
     fmt.doc_comment(format!("Flags group `{}`.", group.name));
     fmtln!(fmt, "pub struct Flags {");
     fmt.indent(|fmt| {
diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs
index ef092fb818..b831f9966a 100644
--- a/cranelift/codegen/src/context.rs
+++ b/cranelift/codegen/src/context.rs
@@ -473,6 +473,7 @@ impl Context {
         Ok(build_value_labels_ranges::<ComparableSourceLoc>(
             &self.func,
             &self.regalloc,
+            self.mach_compile_result.as_ref(),
             isa,
         ))
     }
diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs
index c5e827db3d..c78dde81de 100644
--- a/cranelift/codegen/src/ir/mod.rs
+++ b/cranelift/codegen/src/ir/mod.rs
@@ -58,6 +58,7 @@ pub use crate::ir::table::TableData;
 pub use crate::ir::trapcode::TrapCode;
 pub use crate::ir::types::Type;
 pub use crate::ir::valueloc::{ArgumentLoc, ValueLoc};
+pub use crate::value_label::LabelValueLoc;
 pub use cranelift_codegen_shared::condcodes;
 
 use crate::binemit;
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs
index d5d88e7770..8b371cb159 100644
--- a/cranelift/codegen/src/isa/aarch64/abi.rs
+++ b/cranelift/codegen/src/isa/aarch64/abi.rs
@@ -4,6 +4,8 @@ use crate::ir;
 use crate::ir::types;
 use crate::ir::types::*;
 use crate::ir::MemFlags;
+use crate::ir::Opcode;
+use crate::ir::{ExternalName, LibCall};
 use crate::isa;
 use crate::isa::aarch64::{inst::EmitState, inst::*};
 use crate::machinst::*;
@@ -76,41 +78,41 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
         match &param.purpose {
             &ir::ArgumentPurpose::VMContext => {
                 // This is SpiderMonkey's `WasmTlsReg`.
-                Some(ABIArg::Reg(
-                    ValueRegs::one(xreg(BALDRDASH_TLS_REG).to_real_reg()),
-                    ir::types::I64,
-                    param.extension,
-                    param.purpose,
-                ))
+                Some(ABIArg::Reg {
+                    regs: ValueRegs::one(xreg(BALDRDASH_TLS_REG).to_real_reg()),
+                    ty: ir::types::I64,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::SignatureId => {
                 // This is SpiderMonkey's `WasmTableCallSigReg`.
-                Some(ABIArg::Reg(
-                    ValueRegs::one(xreg(BALDRDASH_SIG_REG).to_real_reg()),
-                    ir::types::I64,
-                    param.extension,
-                    param.purpose,
-                ))
+                Some(ABIArg::Reg {
+                    regs: ValueRegs::one(xreg(BALDRDASH_SIG_REG).to_real_reg()),
+                    ty: ir::types::I64,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::CalleeTLS => {
                 // This is SpiderMonkey's callee TLS slot in the extended frame of Wasm's ABI-2020.
                 assert!(call_conv == isa::CallConv::Baldrdash2020);
-                Some(ABIArg::Stack(
-                    BALDRDASH_CALLEE_TLS_OFFSET,
-                    ir::types::I64,
-                    ir::ArgumentExtension::None,
-                    param.purpose,
-                ))
+                Some(ABIArg::Stack {
+                    offset: BALDRDASH_CALLEE_TLS_OFFSET,
+                    ty: ir::types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::CallerTLS => {
                 // This is SpiderMonkey's caller TLS slot in the extended frame of Wasm's ABI-2020.
                 assert!(call_conv == isa::CallConv::Baldrdash2020);
-                Some(ABIArg::Stack(
-                    BALDRDASH_CALLER_TLS_OFFSET,
-                    ir::types::I64,
-                    ir::ArgumentExtension::None,
-                    param.purpose,
-                ))
+                Some(ABIArg::Stack {
+                    offset: BALDRDASH_CALLER_TLS_OFFSET,
+                    ty: ir::types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: param.purpose,
+                })
             }
             _ => None,
         }
@@ -208,7 +210,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
                 | &ir::ArgumentPurpose::StackLimit
                 | &ir::ArgumentPurpose::SignatureId
                 | &ir::ArgumentPurpose::CallerTLS
-                | &ir::ArgumentPurpose::CalleeTLS => {}
+                | &ir::ArgumentPurpose::CalleeTLS
+                | &ir::ArgumentPurpose::StructReturn
+                | &ir::ArgumentPurpose::StructArgument(_) => {}
                 _ => panic!(
                     "Unsupported argument purpose {:?} in signature: {:?}",
                     param.purpose, params
@@ -233,18 +237,28 @@ impl ABIMachineSpec for AArch64MachineDeps {
             if let Some(param) = try_fill_baldrdash_reg(call_conv, param) {
                 assert!(rc == RegClass::I64);
                 ret.push(param);
+            } else if let ir::ArgumentPurpose::StructArgument(size) = param.purpose {
+                let offset = next_stack as i64;
+                let size = size as u64;
+                assert!(size % 8 == 0, "StructArgument size is not properly aligned");
+                next_stack += size;
+                ret.push(ABIArg::StructArg {
+                    offset,
+                    size,
+                    purpose: param.purpose,
+                });
             } else if *next_reg < max_per_class_reg_vals && remaining_reg_vals > 0 {
                 let reg = match rc {
                     RegClass::I64 => xreg(*next_reg),
                     RegClass::V128 => vreg(*next_reg),
                     _ => unreachable!(),
                 };
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(reg.to_real_reg()),
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(reg.to_real_reg()),
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
                 *next_reg += 1;
                 remaining_reg_vals -= 1;
             } else {
@@ -255,12 +269,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
                 // Align.
                 debug_assert!(size.is_power_of_two());
                 next_stack = (next_stack + size - 1) & !(size - 1);
-                ret.push(ABIArg::Stack(
-                    next_stack as i64,
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
+                ret.push(ABIArg::Stack {
+                    offset: next_stack as i64,
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
                 next_stack += size;
             }
         }
@@ -272,19 +286,19 @@ impl ABIMachineSpec for AArch64MachineDeps {
         let extra_arg = if add_ret_area_ptr {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
             if next_xreg < max_per_class_reg_vals && remaining_reg_vals > 0 {
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(xreg(next_xreg).to_real_reg()),
-                    I64,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(xreg(next_xreg).to_real_reg()),
+                    ty: I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
             } else {
-                ret.push(ABIArg::Stack(
-                    next_stack as i64,
-                    I64,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Stack {
+                    offset: next_stack as i64,
+                    ty: I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
                 next_stack += 8;
             }
             Some(ret.len() - 1)
@@ -708,6 +722,34 @@ impl ABIMachineSpec for AArch64MachineDeps {
         insts
     }
 
+    fn gen_memcpy(
+        call_conv: isa::CallConv,
+        dst: Reg,
+        src: Reg,
+        size: usize,
+    ) -> SmallVec<[Self::I; 8]> {
+        // Baldrdash should not use struct args.
+        assert!(!call_conv.extends_baldrdash());
+        let mut insts = SmallVec::new();
+        let arg0 = writable_xreg(0);
+        let arg1 = writable_xreg(1);
+        let arg2 = writable_xreg(2);
+        insts.push(Inst::gen_move(arg0, dst, I64));
+        insts.push(Inst::gen_move(arg1, src, I64));
+        insts.extend(Inst::load_constant(arg2, size as u64).into_iter());
+        insts.push(Inst::Call {
+            info: Box::new(CallInfo {
+                dest: ExternalName::LibCall(LibCall::Memcpy),
+                uses: vec![arg0.to_reg(), arg1.to_reg(), arg2.to_reg()],
+                defs: Self::get_regs_clobbered_by_call(call_conv),
+                opcode: Opcode::Call,
+                caller_callconv: call_conv,
+                callee_callconv: call_conv,
+            }),
+        });
+        insts
+    }
+
     fn get_number_of_spillslots_for_value(rc: RegClass, ty: Type) -> u32 {
         // We allocate in terms of 8-byte slots.
         match (rc, ty) {
diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs
index 738495714a..1a55029b32 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/args.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs
@@ -601,6 +601,14 @@ impl ScalarSize {
         }
     }
 
+    /// Convert from an integer operand size.
+    pub fn from_operand_size(size: OperandSize) -> ScalarSize {
+        match size {
+            OperandSize::Size32 => ScalarSize::Size32,
+            OperandSize::Size64 => ScalarSize::Size64,
+        }
+    }
+
     /// Convert from a type into the smallest size that fits.
     pub fn from_ty(ty: Type) -> ScalarSize {
         Self::from_bits(ty_bits(ty))
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 432bbc19dd..599a8edacd 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -1463,12 +1463,18 @@ impl MachInstEmit for Inst {
                         debug_assert!(size == VectorSize::Size32x4 || size == VectorSize::Size64x2);
                         (0b0, 0b11000, enc_size | 0b10)
                     }
+                    VecMisc2::Cnt => {
+                        debug_assert!(size == VectorSize::Size8x8 || size == VectorSize::Size8x16);
+                        (0b0, 0b00101, enc_size)
+                    }
                 };
                 sink.put4(enc_vec_rr_misc((q << 1) | u, size, bits_12_16, rd, rn));
             }
             &Inst::VecLanes { op, rd, rn, size } => {
                 let (q, size) = match size {
+                    VectorSize::Size8x8 => (0b0, 0b00),
                     VectorSize::Size8x16 => (0b1, 0b00),
+                    VectorSize::Size16x4 => (0b0, 0b01),
                     VectorSize::Size16x8 => (0b1, 0b01),
                     VectorSize::Size32x4 => (0b1, 0b10),
                     _ => unreachable!(),
@@ -2365,6 +2371,9 @@ impl MachInstEmit for Inst {
                     sink.bind_label(jump_around_label);
                 }
             }
+            &Inst::ValueLabelMarker { .. } => {
+                // Nothing; this is only used to compute debug info.
+            }
         }
 
         let end_off = sink.cur_offset();
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
index f01fbf43f0..63232d58a4 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
@@ -3792,6 +3792,28 @@ fn test_aarch64_binemit() {
         "frintp v12.2d, v17.2d",
     ));
 
+    insns.push((
+        Inst::VecMisc {
+            op: VecMisc2::Cnt,
+            rd: writable_vreg(23),
+            rn: vreg(5),
+            size: VectorSize::Size8x8,
+        },
+        "B758200E",
+        "cnt v23.8b, v5.8b",
+    ));
+
+    insns.push((
+        Inst::VecLanes {
+            op: VecLanesOp::Uminv,
+            rd: writable_vreg(0),
+            rn: vreg(31),
+            size: VectorSize::Size8x8,
+        },
+        "E0AB312E",
+        "uminv b0, v31.8b",
+    ));
+
     insns.push((
         Inst::VecLanes {
             op: VecLanesOp::Uminv,
@@ -3836,6 +3858,17 @@ fn test_aarch64_binemit() {
         "addv b2, v29.16b",
     ));
 
+    insns.push((
+        Inst::VecLanes {
+            op: VecLanesOp::Addv,
+            rd: writable_vreg(15),
+            rn: vreg(7),
+            size: VectorSize::Size16x4,
+        },
+        "EFB8710E",
+        "addv h15, v7.4h",
+    ));
+
     insns.push((
         Inst::VecLanes {
             op: VecLanesOp::Addv,
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 38b6e29ce4..aff9b36e0a 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -7,7 +7,7 @@ use crate::binemit::CodeOffset;
 use crate::ir::types::{
     B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64,
 };
-use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type};
+use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type, ValueLabel};
 use crate::isa::CallConv;
 use crate::machinst::*;
 use crate::{settings, CodegenError, CodegenResult};
@@ -331,6 +331,8 @@ pub enum VecMisc2 {
     Frintm,
     /// Floating point round to integral, rounding towards plus infinity
     Frintp,
+    /// Population count per byte
+    Cnt,
 }
 
 /// A Vector narrowing operation with two registers.
@@ -1208,6 +1210,12 @@ pub enum Inst {
         /// The needed space before the next deadline.
         needed_space: CodeOffset,
     },
+
+    /// A definition of a value label.
+    ValueLabelMarker {
+        reg: Reg,
+        label: ValueLabel,
+    },
 }
 
 fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize {
@@ -2015,6 +2023,9 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
             memarg_regs(mem, collector);
         }
         &Inst::VirtualSPOffsetAdj { .. } => {}
+        &Inst::ValueLabelMarker { reg, .. } => {
+            collector.add_use(reg);
+        }
         &Inst::EmitIsland { .. } => {}
     }
 }
@@ -2765,6 +2776,9 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
         }
         &mut Inst::VirtualSPOffsetAdj { .. } => {}
         &mut Inst::EmitIsland { .. } => {}
+        &mut Inst::ValueLabelMarker { ref mut reg, .. } => {
+            map_use(mapper, reg);
+        }
     }
 }
 
@@ -2893,11 +2907,10 @@ impl MachInst for Inst {
         }
     }
 
-    fn gen_zero_len_nop() -> Inst {
-        Inst::Nop0
-    }
-
     fn gen_nop(preferred_size: usize) -> Inst {
+        if preferred_size == 0 {
+            return Inst::Nop0;
+        }
         // We can't give a NOP (or any insn) < 4 bytes.
         assert!(preferred_size >= 4);
         Inst::Nop4
@@ -2960,6 +2973,17 @@ impl MachInst for Inst {
     fn ref_type_regclass(_: &settings::Flags) -> RegClass {
         RegClass::I64
     }
+
+    fn gen_value_label_marker(label: ValueLabel, reg: Reg) -> Self {
+        Inst::ValueLabelMarker { label, reg }
+    }
+
+    fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> {
+        match self {
+            Inst::ValueLabelMarker { label, reg } => Some((*label, *reg)),
+            _ => None,
+        }
+    }
 }
 
 //=============================================================================
@@ -3752,6 +3776,7 @@ impl Inst {
                     VecMisc2::Frintz => ("frintz", size),
                     VecMisc2::Frintm => ("frintm", size),
                     VecMisc2::Frintp => ("frintp", size),
+                    VecMisc2::Cnt => ("cnt", size),
                 };
 
                 let rd_size = if is_shll { size.widen() } else { size };
@@ -4068,6 +4093,10 @@ impl Inst {
                 format!("virtual_sp_offset_adjust {}", offset)
             }
             &Inst::EmitIsland { needed_space } => format!("emit_island {}", needed_space),
+
+            &Inst::ValueLabelMarker { label, reg } => {
+                format!("value_label {:?}, {}", label, reg.show_rru(mb_rru))
+            }
         }
     }
 }
diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs
index 37c5e79c8d..0f37bb6123 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower.rs
@@ -1231,7 +1231,7 @@ impl LowerBackend for AArch64Backend {
     type MInst = Inst;
 
     fn lower<C: LowerCtx<I = Inst>>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> {
-        lower_inst::lower_insn_to_regs(ctx, ir_inst)
+        lower_inst::lower_insn_to_regs(ctx, ir_inst, &self.flags)
     }
 
     fn lower_branch_group<C: LowerCtx<I = Inst>>(
diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs
index 1c4e3d7e99..93c2385098 100644
--- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs
@@ -7,6 +7,7 @@ use crate::ir::Inst as IRInst;
 use crate::ir::{InstructionData, Opcode, TrapCode};
 use crate::machinst::lower::*;
 use crate::machinst::*;
+use crate::settings::Flags;
 use crate::{CodegenError, CodegenResult};
 
 use crate::isa::aarch64::abi::*;
@@ -24,6 +25,7 @@ use super::lower::*;
 pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
     ctx: &mut C,
     insn: IRInst,
+    flags: &Flags,
 ) -> CodegenResult<()> {
     let op = ctx.data(insn).opcode();
     let inputs = insn_inputs(ctx, insn);
@@ -960,143 +962,57 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Popcnt => {
-            // Lower popcount using the following algorithm:
-            //
-            //   x -= (x >> 1) & 0x5555555555555555
-            //   x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333)
-            //   x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f
-            //   x += x << 8
-            //   x += x << 16
-            //   x += x << 32
-            //   x >> 56
-            let ty = ty.unwrap();
             let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            // FIXME(#1537): zero-extend 8/16/32-bit operands only to 32 bits,
-            // and fix the sequence below to work properly for this.
-            let narrow_mode = NarrowValueMode::ZeroExtend64;
-            let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-            let tmp = ctx.alloc_tmp(I64).only_reg().unwrap();
+            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
+            let ty = ty.unwrap();
+            let size = ScalarSize::from_operand_size(OperandSize::from_ty(ty));
+            let tmp = ctx.alloc_tmp(I8X16).only_reg().unwrap();
 
-            // If this is a 32-bit Popcnt, use Lsr32 to clear the top 32 bits of the register, then
-            // the rest of the code is identical to the 64-bit version.
-            // lsr [wx]d, [wx]n, #1
-            ctx.emit(Inst::AluRRImmShift {
-                alu_op: choose_32_64(ty, ALUOp::Lsr32, ALUOp::Lsr64),
-                rd: rd,
+            // fmov tmp, rn
+            // cnt tmp.8b, tmp.8b
+            // addp tmp.8b, tmp.8b, tmp.8b / addv tmp, tmp.8b / (no instruction for 8-bit inputs)
+            // umov rd, tmp.b[0]
+
+            ctx.emit(Inst::MovToFpu {
+                rd: tmp,
                 rn: rn,
-                immshift: ImmShift::maybe_from_u64(1).unwrap(),
+                size,
             });
-
-            // and xd, xd, #0x5555555555555555
-            ctx.emit(Inst::AluRRImmLogic {
-                alu_op: ALUOp::And64,
-                rd: rd,
-                rn: rd.to_reg(),
-                imml: ImmLogic::maybe_from_u64(0x5555555555555555, I64).unwrap(),
-            });
-
-            // sub xd, xn, xd
-            ctx.emit(Inst::AluRRR {
-                alu_op: ALUOp::Sub64,
-                rd: rd,
-                rn: rn,
-                rm: rd.to_reg(),
-            });
-
-            // and xt, xd, #0x3333333333333333
-            ctx.emit(Inst::AluRRImmLogic {
-                alu_op: ALUOp::And64,
-                rd: tmp,
-                rn: rd.to_reg(),
-                imml: ImmLogic::maybe_from_u64(0x3333333333333333, I64).unwrap(),
-            });
-
-            // lsr xd, xd, #2
-            ctx.emit(Inst::AluRRImmShift {
-                alu_op: ALUOp::Lsr64,
-                rd: rd,
-                rn: rd.to_reg(),
-                immshift: ImmShift::maybe_from_u64(2).unwrap(),
-            });
-
-            // and xd, xd, #0x3333333333333333
-            ctx.emit(Inst::AluRRImmLogic {
-                alu_op: ALUOp::And64,
-                rd: rd,
-                rn: rd.to_reg(),
-                imml: ImmLogic::maybe_from_u64(0x3333333333333333, I64).unwrap(),
-            });
-
-            // add xt, xd, xt
-            ctx.emit(Inst::AluRRR {
-                alu_op: ALUOp::Add64,
-                rd: tmp,
-                rn: rd.to_reg(),
-                rm: tmp.to_reg(),
-            });
-
-            // add xt, xt, xt LSR #4
-            ctx.emit(Inst::AluRRRShift {
-                alu_op: ALUOp::Add64,
+            ctx.emit(Inst::VecMisc {
+                op: VecMisc2::Cnt,
                 rd: tmp,
                 rn: tmp.to_reg(),
-                rm: tmp.to_reg(),
-                shiftop: ShiftOpAndAmt::new(
-                    ShiftOp::LSR,
-                    ShiftOpShiftImm::maybe_from_shift(4).unwrap(),
-                ),
+                size: VectorSize::Size8x8,
             });
 
-            // and xt, xt, #0x0f0f0f0f0f0f0f0f
-            ctx.emit(Inst::AluRRImmLogic {
-                alu_op: ALUOp::And64,
-                rd: tmp,
-                rn: tmp.to_reg(),
-                imml: ImmLogic::maybe_from_u64(0x0f0f0f0f0f0f0f0f, I64).unwrap(),
-            });
+            match ScalarSize::from_ty(ty) {
+                ScalarSize::Size8 => {}
+                ScalarSize::Size16 => {
+                    // ADDP is usually cheaper than ADDV.
+                    ctx.emit(Inst::VecRRR {
+                        alu_op: VecALUOp::Addp,
+                        rd: tmp,
+                        rn: tmp.to_reg(),
+                        rm: tmp.to_reg(),
+                        size: VectorSize::Size8x8,
+                    });
+                }
+                ScalarSize::Size32 | ScalarSize::Size64 => {
+                    ctx.emit(Inst::VecLanes {
+                        op: VecLanesOp::Addv,
+                        rd: tmp,
+                        rn: tmp.to_reg(),
+                        size: VectorSize::Size8x8,
+                    });
+                }
+                sz => panic!("Unexpected scalar FP operand size: {:?}", sz),
+            }
 
-            // add xt, xt, xt, LSL #8
-            ctx.emit(Inst::AluRRRShift {
-                alu_op: ALUOp::Add64,
-                rd: tmp,
+            ctx.emit(Inst::MovFromVec {
+                rd,
                 rn: tmp.to_reg(),
-                rm: tmp.to_reg(),
-                shiftop: ShiftOpAndAmt::new(
-                    ShiftOp::LSL,
-                    ShiftOpShiftImm::maybe_from_shift(8).unwrap(),
-                ),
-            });
-
-            // add xt, xt, xt, LSL #16
-            ctx.emit(Inst::AluRRRShift {
-                alu_op: ALUOp::Add64,
-                rd: tmp,
-                rn: tmp.to_reg(),
-                rm: tmp.to_reg(),
-                shiftop: ShiftOpAndAmt::new(
-                    ShiftOp::LSL,
-                    ShiftOpShiftImm::maybe_from_shift(16).unwrap(),
-                ),
-            });
-
-            // add xt, xt, xt, LSL #32
-            ctx.emit(Inst::AluRRRShift {
-                alu_op: ALUOp::Add64,
-                rd: tmp,
-                rn: tmp.to_reg(),
-                rm: tmp.to_reg(),
-                shiftop: ShiftOpAndAmt::new(
-                    ShiftOp::LSL,
-                    ShiftOpShiftImm::maybe_from_shift(32).unwrap(),
-                ),
-            });
-
-            // lsr xd, xt, #56
-            ctx.emit(Inst::AluRRImmShift {
-                alu_op: ALUOp::Lsr64,
-                rd: rd,
-                rn: tmp.to_reg(),
-                immshift: ImmShift::maybe_from_u64(56).unwrap(),
+                idx: 0,
+                size: VectorSize::Size8x16,
             });
         }
 
@@ -1803,7 +1719,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert!(inputs.len() == sig.params.len());
                     assert!(outputs.len() == sig.returns.len());
                     (
-                        AArch64ABICaller::from_func(sig, &extname, dist, caller_conv)?,
+                        AArch64ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?,
                         &inputs[..],
                     )
                 }
@@ -1813,7 +1729,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert!(inputs.len() - 1 == sig.params.len());
                     assert!(outputs.len() == sig.returns.len());
                     (
-                        AArch64ABICaller::from_ptr(sig, ptr, op, caller_conv)?,
+                        AArch64ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?,
                         &inputs[1..],
                     )
                 }
@@ -1822,8 +1738,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
             abi.emit_stack_pre_adjust(ctx);
             assert!(inputs.len() == abi.num_args());
-            for (i, input) in inputs.iter().enumerate() {
-                let arg_reg = put_input_in_reg(ctx, *input, NarrowValueMode::None);
+            for i in abi.get_copy_to_arg_order() {
+                let input = inputs[i];
+                let arg_reg = put_input_in_reg(ctx, input, NarrowValueMode::None);
                 abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg));
             }
             abi.emit_call(ctx);
diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs
index c3c56632d3..af13cb70c0 100644
--- a/cranelift/codegen/src/isa/aarch64/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/mod.rs
@@ -8,6 +8,7 @@ use crate::result::CodegenResult;
 use crate::settings;
 
 use alloc::boxed::Box;
+use core::hash::{Hash, Hasher};
 
 use regalloc::{PrettyPrint, RealRegUniverse};
 use target_lexicon::{Aarch64Architecture, Architecture, Triple};
@@ -79,6 +80,7 @@ impl MachBackend for AArch64Backend {
             frame_size,
             disasm,
             unwind_info,
+            value_labels_ranges: None,
         })
     }
 
@@ -94,6 +96,10 @@ impl MachBackend for AArch64Backend {
         &self.flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.flags.hash(&mut hasher);
+    }
+
     fn reg_universe(&self) -> &RealRegUniverse {
         &self.reg_universe
     }
diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs
index 9e92a7b7aa..e1a64aeb76 100644
--- a/cranelift/codegen/src/isa/arm32/abi.rs
+++ b/cranelift/codegen/src/isa/arm32/abi.rs
@@ -81,12 +81,12 @@ impl ABIMachineSpec for Arm32MachineDeps {
             if next_rreg < max_reg_val {
                 let reg = rreg(next_rreg);
 
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(reg.to_real_reg()),
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(reg.to_real_reg()),
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
                 next_rreg += 1;
             } else {
                 // Arguments are stored on stack in reversed order.
@@ -101,12 +101,12 @@ impl ABIMachineSpec for Arm32MachineDeps {
         let extra_arg = if add_ret_area_ptr {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
             if next_rreg < max_reg_val {
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(rreg(next_rreg).to_real_reg()),
-                    I32,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(rreg(next_rreg).to_real_reg()),
+                    ty: I32,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
             } else {
                 stack_args.push((
                     I32,
@@ -124,12 +124,12 @@ impl ABIMachineSpec for Arm32MachineDeps {
         let max_stack = next_stack;
         for (ty, ext, purpose) in stack_args.into_iter().rev() {
             next_stack -= 4;
-            ret.push(ABIArg::Stack(
-                (max_stack - next_stack) as i64,
+            ret.push(ABIArg::Stack {
+                offset: (max_stack - next_stack) as i64,
                 ty,
-                ext,
+                extension: ext,
                 purpose,
-            ));
+            });
         }
         assert_eq!(next_stack, 0);
 
@@ -426,6 +426,15 @@ impl ABIMachineSpec for Arm32MachineDeps {
         insts
     }
 
+    fn gen_memcpy(
+        _call_conv: isa::CallConv,
+        _dst: Reg,
+        _src: Reg,
+        _size: usize,
+    ) -> SmallVec<[Self::I; 8]> {
+        unimplemented!("StructArgs not implemented for ARM32 yet");
+    }
+
     fn get_number_of_spillslots_for_value(rc: RegClass, _ty: Type) -> u32 {
         match rc {
             RegClass::I32 => 1,
diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs
index 309aa43102..811e8cf920 100644
--- a/cranelift/codegen/src/isa/arm32/inst/mod.rs
+++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs
@@ -831,11 +831,10 @@ impl MachInst for Inst {
         }
     }
 
-    fn gen_zero_len_nop() -> Inst {
-        Inst::Nop0
-    }
-
     fn gen_nop(preferred_size: usize) -> Inst {
+        if preferred_size == 0 {
+            return Inst::Nop0;
+        }
         assert!(preferred_size >= 2);
         Inst::Nop2
     }
diff --git a/cranelift/codegen/src/isa/arm32/lower.rs b/cranelift/codegen/src/isa/arm32/lower.rs
index 372c18b8e9..f2a35f9820 100644
--- a/cranelift/codegen/src/isa/arm32/lower.rs
+++ b/cranelift/codegen/src/isa/arm32/lower.rs
@@ -224,7 +224,7 @@ impl LowerBackend for Arm32Backend {
     type MInst = Inst;
 
     fn lower<C: LowerCtx<I = Inst>>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> {
-        lower_inst::lower_insn_to_regs(ctx, ir_inst)
+        lower_inst::lower_insn_to_regs(ctx, ir_inst, &self.flags)
     }
 
     fn lower_branch_group<C: LowerCtx<I = Inst>>(
diff --git a/cranelift/codegen/src/isa/arm32/lower_inst.rs b/cranelift/codegen/src/isa/arm32/lower_inst.rs
index dd453d772a..16fd528c56 100644
--- a/cranelift/codegen/src/isa/arm32/lower_inst.rs
+++ b/cranelift/codegen/src/isa/arm32/lower_inst.rs
@@ -5,6 +5,7 @@ use crate::ir::Inst as IRInst;
 use crate::ir::Opcode;
 use crate::machinst::lower::*;
 use crate::machinst::*;
+use crate::settings::Flags;
 use crate::CodegenResult;
 
 use crate::isa::arm32::abi::*;
@@ -18,6 +19,7 @@ use super::lower::*;
 pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
     ctx: &mut C,
     insn: IRInst,
+    flags: &Flags,
 ) -> CodegenResult<()> {
     let op = ctx.data(insn).opcode();
     let inputs: SmallVec<[InsnInput; 4]> = (0..ctx.num_inputs(insn))
@@ -502,7 +504,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert_eq!(inputs.len(), sig.params.len());
                     assert_eq!(outputs.len(), sig.returns.len());
                     (
-                        Arm32ABICaller::from_func(sig, &extname, dist, caller_conv)?,
+                        Arm32ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?,
                         &inputs[..],
                     )
                 }
@@ -512,7 +514,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert_eq!(inputs.len() - 1, sig.params.len());
                     assert_eq!(outputs.len(), sig.returns.len());
                     (
-                        Arm32ABICaller::from_ptr(sig, ptr, op, caller_conv)?,
+                        Arm32ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?,
                         &inputs[1..],
                     )
                 }
diff --git a/cranelift/codegen/src/isa/arm32/mod.rs b/cranelift/codegen/src/isa/arm32/mod.rs
index 4b9701fd1d..3976d74ba6 100644
--- a/cranelift/codegen/src/isa/arm32/mod.rs
+++ b/cranelift/codegen/src/isa/arm32/mod.rs
@@ -8,6 +8,7 @@ use crate::result::CodegenResult;
 use crate::settings;
 
 use alloc::boxed::Box;
+use core::hash::{Hash, Hasher};
 use regalloc::{PrettyPrint, RealRegUniverse};
 use target_lexicon::{Architecture, ArmArchitecture, Triple};
 
@@ -74,6 +75,7 @@ impl MachBackend for Arm32Backend {
             frame_size,
             disasm,
             unwind_info: None,
+            value_labels_ranges: None,
         })
     }
 
@@ -89,6 +91,10 @@ impl MachBackend for Arm32Backend {
         &self.flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.flags.hash(&mut hasher);
+    }
+
     fn reg_universe(&self) -> &RealRegUniverse {
         &self.reg_universe
     }
diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs
index 900dfd9ddd..bfc4e0d0d0 100644
--- a/cranelift/codegen/src/isa/mod.rs
+++ b/cranelift/codegen/src/isa/mod.rs
@@ -69,6 +69,7 @@ use alloc::boxed::Box;
 use core::any::Any;
 use core::fmt;
 use core::fmt::{Debug, Formatter};
+use core::hash::Hasher;
 use target_lexicon::{triple, Architecture, PointerWidth, Triple};
 use thiserror::Error;
 
@@ -265,6 +266,10 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
     /// Get the ISA-independent flags that were used to make this trait object.
     fn flags(&self) -> &settings::Flags;
 
+    /// Hashes all flags, both ISA-independent and ISA-specific, into the
+    /// specified hasher.
+    fn hash_all_flags(&self, hasher: &mut dyn Hasher);
+
     /// Get the default calling convention of this target.
     fn default_call_conv(&self) -> CallConv {
         CallConv::triple_default(self.triple())
@@ -325,6 +330,12 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
         Err(RegisterMappingError::UnsupportedArchitecture)
     }
 
+    #[cfg(feature = "unwind")]
+    /// Map a regalloc::Reg to its corresponding DWARF register.
+    fn map_regalloc_reg_to_dwarf(&self, _: ::regalloc::Reg) -> Result<u16, RegisterMappingError> {
+        Err(RegisterMappingError::UnsupportedArchitecture)
+    }
+
     /// Returns an iterator over legal encodings for the instruction.
     fn legal_encodings<'a>(
         &'a self,
diff --git a/cranelift/codegen/src/isa/riscv/mod.rs b/cranelift/codegen/src/isa/riscv/mod.rs
index e69a3a0e12..500451c72e 100644
--- a/cranelift/codegen/src/isa/riscv/mod.rs
+++ b/cranelift/codegen/src/isa/riscv/mod.rs
@@ -19,6 +19,7 @@ use alloc::borrow::Cow;
 use alloc::boxed::Box;
 use core::any::Any;
 use core::fmt;
+use core::hash::{Hash, Hasher};
 use target_lexicon::{PointerWidth, Triple};
 
 #[allow(dead_code)]
@@ -69,6 +70,11 @@ impl TargetIsa for Isa {
         &self.shared_flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.shared_flags.hash(&mut hasher);
+        self.isa_flags.hash(&mut hasher);
+    }
+
     fn register_info(&self) -> RegInfo {
         registers::INFO.clone()
     }
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index 74dca6c3ec..d4f7d5c60c 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -31,41 +31,41 @@ fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option<A
         match &param.purpose {
             &ir::ArgumentPurpose::VMContext => {
                 // This is SpiderMonkey's `WasmTlsReg`.
-                Some(ABIArg::Reg(
-                    ValueRegs::one(regs::r14().to_real_reg()),
-                    types::I64,
-                    param.extension,
-                    param.purpose,
-                ))
+                Some(ABIArg::Reg {
+                    regs: ValueRegs::one(regs::r14().to_real_reg()),
+                    ty: types::I64,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::SignatureId => {
                 // This is SpiderMonkey's `WasmTableCallSigReg`.
-                Some(ABIArg::Reg(
-                    ValueRegs::one(regs::r10().to_real_reg()),
-                    types::I64,
-                    param.extension,
-                    param.purpose,
-                ))
+                Some(ABIArg::Reg {
+                    regs: ValueRegs::one(regs::r10().to_real_reg()),
+                    ty: types::I64,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::CalleeTLS => {
                 // This is SpiderMonkey's callee TLS slot in the extended frame of Wasm's ABI-2020.
                 assert!(call_conv == isa::CallConv::Baldrdash2020);
-                Some(ABIArg::Stack(
-                    BALDRDASH_CALLEE_TLS_OFFSET,
-                    ir::types::I64,
-                    ir::ArgumentExtension::None,
-                    param.purpose,
-                ))
+                Some(ABIArg::Stack {
+                    offset: BALDRDASH_CALLEE_TLS_OFFSET,
+                    ty: ir::types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::CallerTLS => {
                 // This is SpiderMonkey's caller TLS slot in the extended frame of Wasm's ABI-2020.
                 assert!(call_conv == isa::CallConv::Baldrdash2020);
-                Some(ABIArg::Stack(
-                    BALDRDASH_CALLER_TLS_OFFSET,
-                    ir::types::I64,
-                    ir::ArgumentExtension::None,
-                    param.purpose,
-                ))
+                Some(ABIArg::Stack {
+                    offset: BALDRDASH_CALLER_TLS_OFFSET,
+                    ty: ir::types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: param.purpose,
+                })
             }
             _ => None,
         }
@@ -131,49 +131,84 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                 | &ir::ArgumentPurpose::StackLimit
                 | &ir::ArgumentPurpose::SignatureId
                 | &ir::ArgumentPurpose::CalleeTLS
-                | &ir::ArgumentPurpose::CallerTLS => {}
+                | &ir::ArgumentPurpose::CallerTLS
+                | &ir::ArgumentPurpose::StructReturn
+                | &ir::ArgumentPurpose::StructArgument(_) => {}
                 _ => panic!(
                     "Unsupported argument purpose {:?} in signature: {:?}",
                     param.purpose, params
                 ),
             }
 
-            let intreg = in_int_reg(param.value_type);
-            let vecreg = in_vec_reg(param.value_type);
-            debug_assert!(intreg || vecreg);
-            debug_assert!(!(intreg && vecreg));
-
-            let (next_reg, candidate) = if intreg {
-                let candidate = match args_or_rets {
-                    ArgsOrRets::Args => get_intreg_for_arg_systemv(&call_conv, next_gpr),
-                    ArgsOrRets::Rets => get_intreg_for_retval_systemv(&call_conv, next_gpr, i),
-                };
-                debug_assert!(candidate
-                    .map(|r| r.get_class() == RegClass::I64)
-                    .unwrap_or(true));
-                (&mut next_gpr, candidate)
-            } else {
-                let candidate = match args_or_rets {
-                    ArgsOrRets::Args => get_fltreg_for_arg_systemv(&call_conv, next_vreg),
-                    ArgsOrRets::Rets => get_fltreg_for_retval_systemv(&call_conv, next_vreg, i),
-                };
-                debug_assert!(candidate
-                    .map(|r| r.get_class() == RegClass::V128)
-                    .unwrap_or(true));
-                (&mut next_vreg, candidate)
-            };
-
             if let Some(param) = try_fill_baldrdash_reg(call_conv, param) {
-                assert!(intreg);
                 ret.push(param);
-            } else if let Some(reg) = candidate {
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(reg.to_real_reg()),
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
-                *next_reg += 1;
+                continue;
+            }
+
+            if let ir::ArgumentPurpose::StructArgument(size) = param.purpose {
+                let offset = next_stack as i64;
+                let size = size as u64;
+                assert!(size % 8 == 0, "StructArgument size is not properly aligned");
+                next_stack += size;
+                ret.push(ABIArg::StructArg {
+                    offset,
+                    size,
+                    purpose: param.purpose,
+                });
+                continue;
+            }
+
+            // Find regclass(es) of the register(s) used to store a value of this type.
+            let (rcs, _) = Inst::rc_for_type(param.value_type)?;
+            let intreg = rcs[0] == RegClass::I64;
+            let num_regs = rcs.len();
+            assert!(num_regs <= 2);
+            if num_regs == 2 {
+                assert_eq!(rcs[0], rcs[1]);
+            }
+
+            let mut regs: SmallVec<[RealReg; 2]> = smallvec![];
+            for j in 0..num_regs {
+                let nextreg = if intreg {
+                    match args_or_rets {
+                        ArgsOrRets::Args => get_intreg_for_arg_systemv(&call_conv, next_gpr + j),
+                        ArgsOrRets::Rets => {
+                            get_intreg_for_retval_systemv(&call_conv, next_gpr + j, i + j)
+                        }
+                    }
+                } else {
+                    match args_or_rets {
+                        ArgsOrRets::Args => get_fltreg_for_arg_systemv(&call_conv, next_vreg + j),
+                        ArgsOrRets::Rets => {
+                            get_fltreg_for_retval_systemv(&call_conv, next_vreg + j, i + j)
+                        }
+                    }
+                };
+                if let Some(reg) = nextreg {
+                    regs.push(reg.to_real_reg());
+                } else {
+                    regs.clear();
+                    break;
+                }
+            }
+
+            if regs.len() > 0 {
+                let regs = match num_regs {
+                    1 => ValueRegs::one(regs[0]),
+                    2 => ValueRegs::two(regs[0], regs[1]),
+                    _ => panic!("More than two registers unexpected"),
+                };
+                ret.push(ABIArg::Reg {
+                    regs,
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
+                if intreg {
+                    next_gpr += num_regs;
+                } else {
+                    next_vreg += num_regs;
+                }
             } else {
                 // Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte
                 // stack alignment happens separately after all args.)
@@ -182,12 +217,12 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                 // Align.
                 debug_assert!(size.is_power_of_two());
                 next_stack = (next_stack + size - 1) & !(size - 1);
-                ret.push(ABIArg::Stack(
-                    next_stack as i64,
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
+                ret.push(ABIArg::Stack {
+                    offset: next_stack as i64,
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
                 next_stack += size;
             }
         }
@@ -199,19 +234,19 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         let extra_arg = if add_ret_area_ptr {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
             if let Some(reg) = get_intreg_for_arg_systemv(&call_conv, next_gpr) {
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(reg.to_real_reg()),
-                    types::I64,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(reg.to_real_reg()),
+                    ty: types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
             } else {
-                ret.push(ABIArg::Stack(
-                    next_stack as i64,
-                    types::I64,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Stack {
+                    offset: next_stack as i64,
+                    ty: types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
                 next_stack += 8;
             }
             Some(ret.len() - 1)
@@ -421,6 +456,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         let stack_size = clobbered_size + fixed_frame_storage_size;
         // Align to 16 bytes.
         let stack_size = (stack_size + 15) & !15;
+        let clobbered_size = stack_size - fixed_frame_storage_size;
         // Adjust the stack pointer downward with one `sub rsp, IMM`
         // instruction.
         if stack_size > 0 {
@@ -547,6 +583,51 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         insts
     }
 
+    fn gen_memcpy(
+        call_conv: isa::CallConv,
+        dst: Reg,
+        src: Reg,
+        size: usize,
+    ) -> SmallVec<[Self::I; 8]> {
+        // Baldrdash should not use struct args.
+        assert!(!call_conv.extends_baldrdash());
+        let mut insts = SmallVec::new();
+        let arg0 = get_intreg_for_arg_systemv(&call_conv, 0).unwrap();
+        let arg1 = get_intreg_for_arg_systemv(&call_conv, 1).unwrap();
+        let arg2 = get_intreg_for_arg_systemv(&call_conv, 2).unwrap();
+        // We need a register to load the address of `memcpy()` below and we
+        // don't have a lowering context to allocate a temp here; so just use a
+        // register we know we are free to mutate as part of this sequence
+        // (because it is clobbered by the call as per the ABI anyway).
+        let memcpy_addr = get_intreg_for_arg_systemv(&call_conv, 3).unwrap();
+        insts.push(Inst::gen_move(Writable::from_reg(arg0), dst, I64));
+        insts.push(Inst::gen_move(Writable::from_reg(arg1), src, I64));
+        insts.extend(
+            Inst::gen_constant(
+                ValueRegs::one(Writable::from_reg(arg2)),
+                size as u128,
+                I64,
+                |_| panic!("tmp should not be needed"),
+            )
+            .into_iter(),
+        );
+        // We use an indirect call and a full LoadExtName because we do not have
+        // information about the libcall `RelocDistance` here, so we
+        // conservatively use the more flexible calling sequence.
+        insts.push(Inst::LoadExtName {
+            dst: Writable::from_reg(memcpy_addr),
+            name: Box::new(ExternalName::LibCall(LibCall::Memcpy)),
+            offset: 0,
+        });
+        insts.push(Inst::call_unknown(
+            RegMem::reg(memcpy_addr),
+            /* uses = */ vec![arg0, arg1, arg2],
+            /* defs = */ Self::get_regs_clobbered_by_call(call_conv),
+            Opcode::Call,
+        ));
+        insts
+    }
+
     fn get_number_of_spillslots_for_value(rc: RegClass, ty: Type) -> u32 {
         // We allocate in terms of 8-byte slots.
         match (rc, ty) {
@@ -658,31 +739,6 @@ impl From<StackAMode> for SyntheticAmode {
     }
 }
 
-fn in_int_reg(ty: types::Type) -> bool {
-    match ty {
-        types::I8
-        | types::I16
-        | types::I32
-        | types::I64
-        | types::B1
-        | types::B8
-        | types::B16
-        | types::B32
-        | types::B64
-        | types::R64 => true,
-        types::R32 => panic!("unexpected 32-bits refs on x64!"),
-        _ => false,
-    }
-}
-
-fn in_vec_reg(ty: types::Type) -> bool {
-    match ty {
-        types::F32 | types::F64 => true,
-        _ if ty.is_vector() => true,
-        _ => false,
-    }
-}
-
 fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> {
     match call_conv {
         CallConv::Fast
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 898134644f..0e99ff94fa 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -346,23 +346,35 @@ impl PrettyPrintSized for RegMem {
 #[derive(Copy, Clone, PartialEq)]
 pub enum AluRmiROpcode {
     Add,
+    Adc,
     Sub,
+    Sbb,
     And,
     Or,
     Xor,
     /// The signless, non-extending (N x N -> N, for N in {32,64}) variant.
     Mul,
+    /// 8-bit form of And. Handled separately as we don't have full 8-bit op
+    /// support (we just use wider instructions). Used only with some sequences
+    /// with SETcc.
+    And8,
+    /// 8-bit form of Or.
+    Or8,
 }
 
 impl fmt::Debug for AluRmiROpcode {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
         let name = match self {
             AluRmiROpcode::Add => "add",
+            AluRmiROpcode::Adc => "adc",
             AluRmiROpcode::Sub => "sub",
+            AluRmiROpcode::Sbb => "sbb",
             AluRmiROpcode::And => "and",
             AluRmiROpcode::Or => "or",
             AluRmiROpcode::Xor => "xor",
             AluRmiROpcode::Mul => "imul",
+            AluRmiROpcode::And8 => "and",
+            AluRmiROpcode::Or8 => "or",
         };
         write!(fmt, "{}", name)
     }
@@ -374,12 +386,28 @@ impl fmt::Display for AluRmiROpcode {
     }
 }
 
+impl AluRmiROpcode {
+    /// Is this a special-cased 8-bit ALU op?
+    pub fn is_8bit(self) -> bool {
+        match self {
+            AluRmiROpcode::And8 | AluRmiROpcode::Or8 => true,
+            _ => false,
+        }
+    }
+}
+
 #[derive(Clone, PartialEq)]
 pub enum UnaryRmROpcode {
     /// Bit-scan reverse.
     Bsr,
     /// Bit-scan forward.
     Bsf,
+    /// Counts leading zeroes (Leading Zero CouNT).
+    Lzcnt,
+    /// Counts trailing zeroes (Trailing Zero CouNT).
+    Tzcnt,
+    /// Counts the number of ones (POPulation CouNT).
+    Popcnt,
 }
 
 impl fmt::Debug for UnaryRmROpcode {
@@ -387,6 +415,9 @@ impl fmt::Debug for UnaryRmROpcode {
         match self {
             UnaryRmROpcode::Bsr => write!(fmt, "bsr"),
             UnaryRmROpcode::Bsf => write!(fmt, "bsf"),
+            UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"),
+            UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"),
+            UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"),
         }
     }
 }
@@ -1010,7 +1041,7 @@ impl fmt::Display for ExtMode {
 }
 
 /// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right.
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub enum ShiftKind {
     ShiftLeft,
     /// Inserts zeros in the most significant bits.
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 580d469b8d..4d6ae596eb 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -1,5 +1,6 @@
 use crate::binemit::{Addend, Reloc};
 use crate::ir::immediates::{Ieee32, Ieee64};
+use crate::ir::LibCall;
 use crate::ir::TrapCode;
 use crate::isa::x64::inst::args::*;
 use crate::isa::x64::inst::*;
@@ -83,6 +84,14 @@ impl RexFlags {
         self
     }
 
+    #[inline(always)]
+    fn always_emit_if_8bit_needed(&mut self, reg: u8) -> &mut Self {
+        if reg >= 4 && reg <= 7 {
+            self.always_emit();
+        }
+        self
+    }
+
     #[inline(always)]
     fn must_clear_w(&self) -> bool {
         (self.0 & 1) != 0
@@ -120,18 +129,20 @@ impl RexFlags {
 /// We may need to include one or more legacy prefix bytes before the REX prefix.  This enum
 /// covers only the small set of possibilities that we actually need.
 enum LegacyPrefixes {
-    /// No prefix bytes
+    /// No prefix bytes.
     None,
-    /// Operand Size Override -- here, denoting "16-bit operation"
+    /// Operand Size Override -- here, denoting "16-bit operation".
     _66,
-    /// The Lock prefix
+    /// The Lock prefix.
     _F0,
-    /// Operand size override and Lock
+    /// Operand size override and Lock.
     _66F0,
-    /// REPNE, but no specific meaning here -- is just an opcode extension
+    /// REPNE, but no specific meaning here -- is just an opcode extension.
     _F2,
-    /// REP/REPE, but no specific meaning here -- is just an opcode extension
+    /// REP/REPE, but no specific meaning here -- is just an opcode extension.
     _F3,
+    /// Operand size override and same effect as F3.
+    _66F3,
 }
 
 impl LegacyPrefixes {
@@ -148,6 +159,10 @@ impl LegacyPrefixes {
             }
             LegacyPrefixes::_F2 => sink.put1(0xF2),
             LegacyPrefixes::_F3 => sink.put1(0xF3),
+            LegacyPrefixes::_66F3 => {
+                sink.put1(0x66);
+                sink.put1(0xF3);
+            }
             LegacyPrefixes::None => (),
         }
     }
@@ -527,7 +542,7 @@ pub(crate) fn emit(
             src,
             dst: reg_g,
         } => {
-            let rex = if *is_64 {
+            let mut rex = if *is_64 {
                 RexFlags::set_w()
             } else {
                 RexFlags::clear_w()
@@ -581,17 +596,26 @@ pub(crate) fn emit(
                     }
                 }
             } else {
-                let (opcode_r, opcode_m, subopcode_i) = match op {
-                    AluRmiROpcode::Add => (0x01, 0x03, 0),
-                    AluRmiROpcode::Sub => (0x29, 0x2B, 5),
-                    AluRmiROpcode::And => (0x21, 0x23, 4),
-                    AluRmiROpcode::Or => (0x09, 0x0B, 1),
-                    AluRmiROpcode::Xor => (0x31, 0x33, 6),
+                let (opcode_r, opcode_m, subopcode_i, is_8bit) = match op {
+                    AluRmiROpcode::Add => (0x01, 0x03, 0, false),
+                    AluRmiROpcode::Adc => (0x11, 0x03, 0, false),
+                    AluRmiROpcode::Sub => (0x29, 0x2B, 5, false),
+                    AluRmiROpcode::Sbb => (0x19, 0x2B, 5, false),
+                    AluRmiROpcode::And => (0x21, 0x23, 4, false),
+                    AluRmiROpcode::Or => (0x09, 0x0B, 1, false),
+                    AluRmiROpcode::Xor => (0x31, 0x33, 6, false),
+                    AluRmiROpcode::And8 => (0x20, 0x22, 4, true),
+                    AluRmiROpcode::Or8 => (0x08, 0x0A, 1, true),
                     AluRmiROpcode::Mul => panic!("unreachable"),
                 };
+                assert!(!(is_8bit && *is_64));
 
                 match src {
                     RegMemImm::Reg { reg: reg_e } => {
+                        if is_8bit {
+                            rex.always_emit_if_8bit_needed(int_reg_enc(*reg_e));
+                            rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg()));
+                        }
                         // GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R
                         // duality). Do this too, so as to be able to compare generated machine
                         // code easily.
@@ -604,11 +628,12 @@ pub(crate) fn emit(
                             reg_g.to_reg(),
                             rex,
                         );
-                        // NB: if this is ever extended to handle byte size ops, be sure to retain
-                        // redundant REX prefixes.
                     }
 
                     RegMemImm::Mem { addr } => {
+                        if is_8bit {
+                            rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg()));
+                        }
                         // Here we revert to the "normal" G-E ordering.
                         let amode = addr.finalize(state, sink);
                         emit_std_reg_mem(
@@ -625,6 +650,7 @@ pub(crate) fn emit(
                     }
 
                     RegMemImm::Imm { simm32 } => {
+                        assert!(!is_8bit);
                         let use_imm8 = low8_will_sign_extend_to_32(*simm32);
                         let opcode = if use_imm8 { 0x83 } else { 0x81 };
                         // And also here we use the "normal" G-E ordering.
@@ -645,16 +671,31 @@ pub(crate) fn emit(
         }
 
         Inst::UnaryRmR { size, op, src, dst } => {
-            let (prefix, rex_flags) = match size {
-                2 => (LegacyPrefixes::_66, RexFlags::clear_w()),
-                4 => (LegacyPrefixes::None, RexFlags::clear_w()),
-                8 => (LegacyPrefixes::None, RexFlags::set_w()),
+            let rex_flags = match size {
+                2 | 4 => RexFlags::clear_w(),
+                8 => RexFlags::set_w(),
+                _ => unreachable!(),
+            };
+
+            use UnaryRmROpcode::*;
+            let prefix = match size {
+                2 => match op {
+                    Bsr | Bsf => LegacyPrefixes::_66,
+                    Lzcnt | Tzcnt | Popcnt => LegacyPrefixes::_66F3,
+                },
+                4 | 8 => match op {
+                    Bsr | Bsf => LegacyPrefixes::None,
+                    Lzcnt | Tzcnt | Popcnt => LegacyPrefixes::_F3,
+                },
                 _ => unreachable!(),
             };
 
             let (opcode, num_opcodes) = match op {
-                UnaryRmROpcode::Bsr => (0x0fbd, 2),
-                UnaryRmROpcode::Bsf => (0x0fbc, 2),
+                Bsr => (0x0fbd, 2),
+                Bsf => (0x0fbc, 2),
+                Lzcnt => (0x0fbd, 2),
+                Tzcnt => (0x0fbc, 2),
+                Popcnt => (0x0fb8, 2),
             };
 
             match src {
@@ -685,8 +726,13 @@ pub(crate) fn emit(
         }
 
         Inst::Not { size, src } => {
+            let src = int_reg_enc(src.to_reg());
             let (opcode, prefix, rex_flags) = match size {
-                1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()),
+                1 => (
+                    0xF6,
+                    LegacyPrefixes::None,
+                    *RexFlags::clear_w().always_emit_if_8bit_needed(src),
+                ),
                 2 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
                 4 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
                 8 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()),
@@ -694,13 +740,17 @@ pub(crate) fn emit(
             };
 
             let subopcode = 2;
-            let src = int_reg_enc(src.to_reg());
             emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags)
         }
 
         Inst::Neg { size, src } => {
+            let src = int_reg_enc(src.to_reg());
             let (opcode, prefix, rex_flags) = match size {
-                1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()),
+                1 => (
+                    0xF6,
+                    LegacyPrefixes::None,
+                    *RexFlags::clear_w().always_emit_if_8bit_needed(src),
+                ),
                 2 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
                 4 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
                 8 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()),
@@ -708,7 +758,6 @@ pub(crate) fn emit(
             };
 
             let subopcode = 3;
-            let src = int_reg_enc(src.to_reg());
             emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags)
         }
 
@@ -717,7 +766,7 @@ pub(crate) fn emit(
             signed,
             divisor,
         } => {
-            let (opcode, prefix, rex_flags) = match size {
+            let (opcode, prefix, mut rex_flags) = match size {
                 1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()),
                 2 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
                 4 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
@@ -732,6 +781,9 @@ pub(crate) fn emit(
             match divisor {
                 RegMem::Reg { reg } => {
                     let src = int_reg_enc(*reg);
+                    if *size == 1 {
+                        rex_flags.always_emit_if_8bit_needed(src);
+                    }
                     emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags)
                 }
                 RegMem::Mem { addr: src } => {
@@ -987,9 +1039,7 @@ pub(crate) fn emit(
                         ExtMode::BL | ExtMode::BQ => {
                             // A redundant REX prefix must be emitted for certain register inputs.
                             let enc_src = int_reg_enc(*src);
-                            if enc_src >= 4 && enc_src <= 7 {
-                                rex_flags.always_emit();
-                            };
+                            rex_flags.always_emit_if_8bit_needed(enc_src);
                         }
                         _ => {}
                     }
@@ -1084,9 +1134,7 @@ pub(crate) fn emit(
                         ExtMode::BL | ExtMode::BQ => {
                             // A redundant REX prefix must be emitted for certain register inputs.
                             let enc_src = int_reg_enc(*src);
-                            if enc_src >= 4 && enc_src <= 7 {
-                                rex_flags.always_emit();
-                            };
+                            rex_flags.always_emit_if_8bit_needed(enc_src);
                         }
                         _ => {}
                     }
@@ -1130,9 +1178,7 @@ pub(crate) fn emit(
                     let mut rex = RexFlags::clear_w();
 
                     let enc_src = int_reg_enc(*src);
-                    if enc_src >= 4 && enc_src <= 7 {
-                        rex.always_emit();
-                    };
+                    rex.always_emit_if_8bit_needed(enc_src);
 
                     // MOV r8, r/m8 is (REX.W==0) 88 /r
                     emit_std_reg_mem(
@@ -1215,7 +1261,11 @@ pub(crate) fn emit(
             match num_bits {
                 None => {
                     let (opcode, prefix, rex_flags) = match size {
-                        1 => (0xD2, LegacyPrefixes::None, RexFlags::clear_w()),
+                        1 => (
+                            0xD2,
+                            LegacyPrefixes::None,
+                            *RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst),
+                        ),
                         2 => (0xD3, LegacyPrefixes::_66, RexFlags::clear_w()),
                         4 => (0xD3, LegacyPrefixes::None, RexFlags::clear_w()),
                         8 => (0xD3, LegacyPrefixes::None, RexFlags::set_w()),
@@ -1231,7 +1281,11 @@ pub(crate) fn emit(
 
                 Some(num_bits) => {
                     let (opcode, prefix, rex_flags) = match size {
-                        1 => (0xC0, LegacyPrefixes::None, RexFlags::clear_w()),
+                        1 => (
+                            0xC0,
+                            LegacyPrefixes::None,
+                            *RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst),
+                        ),
                         2 => (0xC1, LegacyPrefixes::_66, RexFlags::clear_w()),
                         4 => (0xC1, LegacyPrefixes::None, RexFlags::clear_w()),
                         8 => (0xC1, LegacyPrefixes::None, RexFlags::set_w()),
@@ -1330,9 +1384,7 @@ pub(crate) fn emit(
                     let mut rex = RexFlags::clear_w();
                     // Here, a redundant REX prefix changes the meaning of the instruction.
                     let enc_g = int_reg_enc(*reg_g);
-                    if enc_g >= 4 && enc_g <= 7 {
-                        rex.always_emit();
-                    }
+                    rex.always_emit_if_8bit_needed(enc_g);
                     rex
                 }
                 _ => panic!("x64::Inst::Cmp_RMI_R::emit: unreachable"),
@@ -1343,9 +1395,7 @@ pub(crate) fn emit(
                     if *size == 1 {
                         // Check whether the E register forces the use of a redundant REX.
                         let enc_e = int_reg_enc(*reg_e);
-                        if enc_e >= 4 && enc_e <= 7 {
-                            rex.always_emit();
-                        }
+                        rex.always_emit_if_8bit_needed(enc_e);
                     }
 
                     // Use the swapped operands encoding for CMP, to stay consistent with the output of
@@ -2065,7 +2115,9 @@ pub(crate) fn emit(
                 SseOpcode::Pextrd => (LegacyPrefixes::_66, 0x0F3A16, 3),
                 SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2),
                 SseOpcode::Roundps => (LegacyPrefixes::_66, 0x0F3A08, 3),
+                SseOpcode::Roundss => (LegacyPrefixes::_66, 0x0F3A0A, 3),
                 SseOpcode::Roundpd => (LegacyPrefixes::_66, 0x0F3A09, 3),
+                SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3),
                 _ => unimplemented!("Opcode {:?} not implemented", op),
             };
             let rex = if *is64 {
@@ -2761,9 +2813,7 @@ pub(crate) fn emit(
                 types::I8 => {
                     let mut rex_flags = RexFlags::clear_w();
                     let enc_src = int_reg_enc(*src);
-                    if enc_src >= 4 && enc_src <= 7 {
-                        rex_flags.always_emit();
-                    };
+                    rex_flags.always_emit_if_8bit_needed(enc_src);
                     (LegacyPrefixes::_F0, rex_flags, 0x0FB0)
                 }
                 types::I16 => (LegacyPrefixes::_66F0, RexFlags::clear_w(), 0x0FB1),
@@ -2962,6 +3012,50 @@ pub(crate) fn emit(
         Inst::EpiloguePlaceholder => {
             // Generate no code.
         }
+
+        Inst::ElfTlsGetAddr { ref symbol } => {
+            // N.B.: Must be exactly this byte sequence; the linker requires it,
+            // because it must know how to rewrite the bytes.
+
+            // data16 lea gv@tlsgd(%rip),%rdi
+            sink.put1(0x66); // data16
+            sink.put1(0b01001000); // REX.W
+            sink.put1(0x8d); // LEA
+            sink.put1(0x3d); // ModRM byte
+            emit_reloc(sink, state, Reloc::ElfX86_64TlsGd, symbol, -4);
+            sink.put4(0); // offset
+
+            // data16 data16 callq __tls_get_addr-4
+            sink.put1(0x66); // data16
+            sink.put1(0x66); // data16
+            sink.put1(0b01001000); // REX.W
+            sink.put1(0xe8); // CALL
+            emit_reloc(
+                sink,
+                state,
+                Reloc::X86CallPLTRel4,
+                &ExternalName::LibCall(LibCall::ElfTlsGetAddr),
+                -4,
+            );
+            sink.put4(0); // offset
+        }
+
+        Inst::MachOTlsGetAddr { ref symbol } => {
+            // movq gv@tlv(%rip), %rdi
+            sink.put1(0x48); // REX.w
+            sink.put1(0x8b); // MOV
+            sink.put1(0x3d); // ModRM byte
+            emit_reloc(sink, state, Reloc::MachOX86_64Tlv, symbol, -4);
+            sink.put4(0); // offset
+
+            // callq *(%rdi)
+            sink.put1(0xff);
+            sink.put1(0x17);
+        }
+
+        Inst::ValueLabelMarker { .. } => {
+            // Nothing; this is only used to compute debug info.
+        }
     }
 
     state.clear_post_insn();
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
index c3489089b9..a3e1c9b658 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
@@ -1025,6 +1025,56 @@ fn test_x64_emit() {
         "4C09FA",
         "orq     %r15, %rdx",
     ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::And8, RegMemImm::reg(r15), w_rdx),
+        "4420FA",
+        "andb    %r15b, %dl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::And8, RegMemImm::reg(rax), w_rsi),
+        "4020C6",
+        "andb    %al, %sil",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::And8, RegMemImm::reg(rax), w_rbx),
+        "20C3",
+        "andb    %al, %bl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(
+            false,
+            AluRmiROpcode::And8,
+            RegMemImm::mem(Amode::imm_reg(0, rax)),
+            w_rbx,
+        ),
+        "2218",
+        "andb    0(%rax), %bl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::Or8, RegMemImm::reg(r15), w_rdx),
+        "4408FA",
+        "orb     %r15b, %dl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::Or8, RegMemImm::reg(rax), w_rsi),
+        "4008C6",
+        "orb     %al, %sil",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::Or8, RegMemImm::reg(rax), w_rbx),
+        "08C3",
+        "orb     %al, %bl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(
+            false,
+            AluRmiROpcode::Or8,
+            RegMemImm::mem(Amode::imm_reg(0, rax)),
+            w_rbx,
+        ),
+        "0A18",
+        "orb     0(%rax), %bl",
+    ));
     insns.push((
         Inst::alu_rmi_r(true, AluRmiROpcode::Xor, RegMemImm::reg(r15), w_rdx),
         "4C31FA",
@@ -1193,6 +1243,16 @@ fn test_x64_emit() {
         "66F7D7",
         "notw    %di",
     ));
+    insns.push((
+        Inst::not(1, Writable::from_reg(regs::rdi())),
+        "40F6D7",
+        "notb    %dil",
+    ));
+    insns.push((
+        Inst::not(1, Writable::from_reg(regs::rax())),
+        "F6D0",
+        "notb    %al",
+    ));
 
     // ========================================================
     // Neg
@@ -1216,6 +1276,16 @@ fn test_x64_emit() {
         "66F7DF",
         "negw    %di",
     ));
+    insns.push((
+        Inst::neg(1, Writable::from_reg(regs::rdi())),
+        "40F6DF",
+        "negb    %dil",
+    ));
+    insns.push((
+        Inst::neg(1, Writable::from_reg(regs::rax())),
+        "F6D8",
+        "negb    %al",
+    ));
 
     // ========================================================
     // Div
@@ -1239,6 +1309,16 @@ fn test_x64_emit() {
         "48F7F7",
         "div     %rdi",
     ));
+    insns.push((
+        Inst::div(1, false, RegMem::reg(regs::rax())),
+        "F6F0",
+        "div     %al",
+    ));
+    insns.push((
+        Inst::div(1, false, RegMem::reg(regs::rsi())),
+        "40F6F6",
+        "div     %sil",
+    ));
 
     // ========================================================
     // MulHi
@@ -2352,9 +2432,14 @@ fn test_x64_emit() {
     ));
     insns.push((
         Inst::shift_r(1, ShiftKind::RotateRight, None, w_rsi),
-        "D2CE",
+        "40D2CE",
         "rorb    %cl, %sil",
     ));
+    insns.push((
+        Inst::shift_r(1, ShiftKind::RotateRight, None, w_rax),
+        "D2C8",
+        "rorb    %cl, %al",
+    ));
     insns.push((
         Inst::shift_r(1, ShiftKind::RotateRight, Some(5), w_r15),
         "41C0CF05",
@@ -3819,6 +3904,28 @@ fn test_x64_emit() {
     let trap_code = TrapCode::UnreachableCodeReached;
     insns.push((Inst::Ud2 { trap_code }, "0F0B", "ud2 unreachable"));
 
+    insns.push((
+        Inst::ElfTlsGetAddr {
+            symbol: ExternalName::User {
+                namespace: 0,
+                index: 0,
+            },
+        },
+        "66488D3D00000000666648E800000000",
+        "elf_tls_get_addr User { namespace: 0, index: 0 }",
+    ));
+
+    insns.push((
+        Inst::MachOTlsGetAddr {
+            symbol: ExternalName::User {
+                namespace: 0,
+                index: 0,
+            },
+        },
+        "488B3D00000000FF17",
+        "macho_tls_get_addr User { namespace: 0, index: 0 }",
+    ));
+
     // ========================================================
     // Actually run the tests!
     let mut flag_builder = settings::builder();
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 09c469498d..e6a6ec9486 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -1,8 +1,10 @@
 //! This module defines x86_64-specific machine instruction types.
 
 use crate::binemit::{CodeOffset, StackMap};
-use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type};
+use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type, ValueLabel};
+use crate::isa::x64::abi::X64ABIMachineSpec;
 use crate::isa::x64::settings as x64_settings;
+use crate::isa::CallConv;
 use crate::machinst::*;
 use crate::{settings, settings::Flags, CodegenError, CodegenResult};
 use alloc::boxed::Box;
@@ -474,6 +476,17 @@ pub enum Inst {
     /// reports its own `def`s/`use`s/`mod`s; this adds complexity (the instruction list is no
     /// longer flat) and requires knowledge about semantics and initial-value independence anyway.
     XmmUninitializedValue { dst: Writable<Reg> },
+
+    /// A call to the `ElfTlsGetAddr` libcall. Returns address
+    /// of TLS symbol in rax.
+    ElfTlsGetAddr { symbol: ExternalName },
+
+    /// A Mach-O TLS symbol access. Returns address of the TLS
+    /// symbol in rax.
+    MachOTlsGetAddr { symbol: ExternalName },
+
+    /// A definition of a value label.
+    ValueLabelMarker { reg: Reg, label: ValueLabel },
 }
 
 pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool {
@@ -532,7 +545,10 @@ impl Inst {
             | Inst::XmmCmpRmR { .. }
             | Inst::XmmLoadConst { .. }
             | Inst::XmmMinMaxSeq { .. }
-            | Inst::XmmUninitializedValue { .. } => None,
+            | Inst::XmmUninitializedValue { .. }
+            | Inst::ElfTlsGetAddr { .. }
+            | Inst::MachOTlsGetAddr { .. }
+            | Inst::ValueLabelMarker { .. } => None,
 
             // These use dynamic SSE opcodes.
             Inst::GprToXmm { op, .. }
@@ -550,7 +566,7 @@ impl Inst {
 
 impl Inst {
     pub(crate) fn nop(len: u8) -> Self {
-        debug_assert!(len <= 16);
+        debug_assert!(len <= 15);
         Self::Nop { len }
     }
 
@@ -1243,6 +1259,14 @@ impl PrettyPrint for Inst {
             (if is_64 { "q" } else { "l" }).to_string()
         }
 
+        fn suffix_lqb(is_64: bool, is_8: bool) -> String {
+            match (is_64, is_8) {
+                (_, true) => "b".to_string(),
+                (true, false) => "q".to_string(),
+                (false, false) => "l".to_string(),
+            }
+        }
+
         fn size_lq(is_64: bool) -> u8 {
             if is_64 {
                 8
@@ -1251,6 +1275,16 @@ impl PrettyPrint for Inst {
             }
         }
 
+        fn size_lqb(is_64: bool, is_8: bool) -> u8 {
+            if is_8 {
+                1
+            } else if is_64 {
+                8
+            } else {
+                4
+            }
+        }
+
         fn suffix_bwlq(size: u8) -> String {
             match size {
                 1 => "b".to_string(),
@@ -1271,9 +1305,9 @@ impl PrettyPrint for Inst {
                 dst,
             } => format!(
                 "{} {}, {}",
-                ljustify2(op.to_string(), suffix_lq(*is_64)),
-                src.show_rru_sized(mb_rru, size_lq(*is_64)),
-                show_ireg_sized(dst.to_reg(), mb_rru, size_lq(*is_64)),
+                ljustify2(op.to_string(), suffix_lqb(*is_64, op.is_8bit())),
+                src.show_rru_sized(mb_rru, size_lqb(*is_64, op.is_8bit())),
+                show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*is_64, op.is_8bit())),
             ),
 
             Inst::UnaryRmR { src, dst, op, size } => format!(
@@ -1762,6 +1796,18 @@ impl PrettyPrint for Inst {
             Inst::Hlt => "hlt".into(),
 
             Inst::Ud2 { trap_code } => format!("ud2 {}", trap_code),
+
+            Inst::ElfTlsGetAddr { ref symbol } => {
+                format!("elf_tls_get_addr {:?}", symbol)
+            }
+
+            Inst::MachOTlsGetAddr { ref symbol } => {
+                format!("macho_tls_get_addr {:?}", symbol)
+            }
+
+            Inst::ValueLabelMarker { label, reg } => {
+                format!("value_label {:?}, {}", label, reg.show_rru(mb_rru))
+            }
         }
     }
 }
@@ -1777,7 +1823,7 @@ impl fmt::Debug for Inst {
 fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
     // This is a bit subtle. If some register is in the modified set, then it may not be in either
     // the use or def sets. However, enforcing that directly is somewhat difficult. Instead,
-    // regalloc.rs will "fix" this for us by removing the the modified set from the use and def
+    // regalloc.rs will "fix" this for us by removing the modified set from the use and def
     // sets.
     match inst {
         Inst::AluRmiR { src, dst, .. } => {
@@ -1849,6 +1895,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
                 || *op == SseOpcode::Pextrw
                 || *op == SseOpcode::Pextrd
                 || *op == SseOpcode::Pshufd
+                || *op == SseOpcode::Roundss
+                || *op == SseOpcode::Roundsd
+                || *op == SseOpcode::Roundps
+                || *op == SseOpcode::Roundpd
             {
                 src.get_regs_as_uses(collector);
                 collector.add_def(*dst);
@@ -2021,6 +2071,22 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
         | Inst::Fence { .. } => {
             // No registers are used.
         }
+
+        Inst::ElfTlsGetAddr { .. } | Inst::MachOTlsGetAddr { .. } => {
+            // All caller-saves are clobbered.
+            //
+            // We use the SysV calling convention here because the
+            // pseudoinstruction (and relocation that it emits) is specific to
+            // ELF systems; other x86-64 targets with other conventions (i.e.,
+            // Windows) use different TLS strategies.
+            for reg in X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV) {
+                collector.add_def(reg);
+            }
+        }
+
+        Inst::ValueLabelMarker { reg, .. } => {
+            collector.add_use(*reg);
+        }
     }
 }
 
@@ -2065,6 +2131,17 @@ impl Amode {
             }
         }
     }
+
+    /// Offset the amode by a fixed offset.
+    pub(crate) fn offset(&self, offset: u32) -> Self {
+        let mut ret = self.clone();
+        match &mut ret {
+            &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
+            &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
+            _ => panic!("Cannot offset amode: {:?}", self),
+        }
+        ret
+    }
 }
 
 impl RegMemImm {
@@ -2163,6 +2240,10 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
                 || *op == SseOpcode::Pextrw
                 || *op == SseOpcode::Pextrd
                 || *op == SseOpcode::Pshufd
+                || *op == SseOpcode::Roundss
+                || *op == SseOpcode::Roundsd
+                || *op == SseOpcode::Roundps
+                || *op == SseOpcode::Roundpd
             {
                 src.map_uses(mapper);
                 map_def(mapper, dst);
@@ -2385,6 +2466,8 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             dst.map_uses(mapper);
         }
 
+        Inst::ValueLabelMarker { ref mut reg, .. } => map_use(mapper, reg),
+
         Inst::Ret
         | Inst::EpiloguePlaceholder
         | Inst::JmpKnown { .. }
@@ -2396,6 +2479,8 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
         | Inst::Ud2 { .. }
         | Inst::Hlt
         | Inst::AtomicRmwSeq { .. }
+        | Inst::ElfTlsGetAddr { .. }
+        | Inst::MachOTlsGetAddr { .. }
         | Inst::Fence { .. } => {
             // Instruction doesn't explicitly mention any regs, so it can't have any virtual
             // regs that we'd need to remap.  Hence no action required.
@@ -2473,6 +2558,25 @@ impl MachInst for Inst {
         }
     }
 
+    fn stack_op_info(&self) -> Option<MachInstStackOpInfo> {
+        match self {
+            Self::VirtualSPOffsetAdj { offset } => Some(MachInstStackOpInfo::NomSPAdj(*offset)),
+            Self::MovRM {
+                size: 8,
+                src,
+                dst: SyntheticAmode::NominalSPOffset { simm32 },
+            } => Some(MachInstStackOpInfo::StoreNomSPOff(*src, *simm32 as i64)),
+            Self::Mov64MR {
+                src: SyntheticAmode::NominalSPOffset { simm32 },
+                dst,
+            } => Some(MachInstStackOpInfo::LoadNomSPOff(
+                dst.to_reg(),
+                *simm32 as i64,
+            )),
+            _ => None,
+        }
+    }
+
     fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst {
         let rc_dst = dst_reg.to_reg().get_class();
         let rc_src = src_reg.get_class();
@@ -2497,12 +2601,8 @@ impl MachInst for Inst {
         }
     }
 
-    fn gen_zero_len_nop() -> Inst {
-        Inst::Nop { len: 0 }
-    }
-
     fn gen_nop(preferred_size: usize) -> Inst {
-        Inst::nop((preferred_size % 16) as u8)
+        Inst::nop(std::cmp::min(preferred_size, 15) as u8)
     }
 
     fn maybe_direct_reload(&self, _reg: VirtualReg, _slot: SpillSlot) -> Option<Inst> {
@@ -2548,77 +2648,88 @@ impl MachInst for Inst {
         ty: Type,
         mut alloc_tmp: F,
     ) -> SmallVec<[Self; 4]> {
-        // We don't support 128-bit constants.
-        assert!(value <= u64::MAX as u128);
         let mut ret = SmallVec::new();
-        let to_reg = to_regs
-            .only_reg()
-            .expect("multi-reg values not supported on x64");
-        if ty == types::F32 {
-            if value == 0 {
-                ret.push(Inst::xmm_rm_r(
-                    SseOpcode::Xorps,
-                    RegMem::reg(to_reg.to_reg()),
-                    to_reg,
-                ));
-            } else {
-                let tmp = alloc_tmp(types::I32);
-                ret.push(Inst::imm(OperandSize::Size32, value as u64, tmp));
-
-                ret.push(Inst::gpr_to_xmm(
-                    SseOpcode::Movd,
-                    RegMem::reg(tmp.to_reg()),
-                    OperandSize::Size32,
-                    to_reg,
-                ));
-            }
-        } else if ty == types::F64 {
-            if value == 0 {
-                ret.push(Inst::xmm_rm_r(
-                    SseOpcode::Xorpd,
-                    RegMem::reg(to_reg.to_reg()),
-                    to_reg,
-                ));
-            } else {
-                let tmp = alloc_tmp(types::I64);
-                ret.push(Inst::imm(OperandSize::Size64, value as u64, tmp));
-
-                ret.push(Inst::gpr_to_xmm(
-                    SseOpcode::Movq,
-                    RegMem::reg(tmp.to_reg()),
-                    OperandSize::Size64,
-                    to_reg,
-                ));
-            }
+        if ty == types::I128 {
+            ret.push(Inst::imm(
+                OperandSize::Size64,
+                value as u64,
+                to_regs.regs()[0],
+            ));
+            ret.push(Inst::imm(
+                OperandSize::Size64,
+                (value >> 64) as u64,
+                to_regs.regs()[1],
+            ));
         } else {
-            // Must be an integer type.
-            debug_assert!(
-                ty == types::B1
-                    || ty == types::I8
-                    || ty == types::B8
-                    || ty == types::I16
-                    || ty == types::B16
-                    || ty == types::I32
-                    || ty == types::B32
-                    || ty == types::I64
-                    || ty == types::B64
-                    || ty == types::R32
-                    || ty == types::R64
-            );
-            if value == 0 {
-                ret.push(Inst::alu_rmi_r(
-                    ty == types::I64,
-                    AluRmiROpcode::Xor,
-                    RegMemImm::reg(to_reg.to_reg()),
-                    to_reg,
-                ));
+            let to_reg = to_regs
+                .only_reg()
+                .expect("multi-reg values not supported on x64");
+            if ty == types::F32 {
+                if value == 0 {
+                    ret.push(Inst::xmm_rm_r(
+                        SseOpcode::Xorps,
+                        RegMem::reg(to_reg.to_reg()),
+                        to_reg,
+                    ));
+                } else {
+                    let tmp = alloc_tmp(types::I32);
+                    ret.push(Inst::imm(OperandSize::Size32, value as u64, tmp));
+
+                    ret.push(Inst::gpr_to_xmm(
+                        SseOpcode::Movd,
+                        RegMem::reg(tmp.to_reg()),
+                        OperandSize::Size32,
+                        to_reg,
+                    ));
+                }
+            } else if ty == types::F64 {
+                if value == 0 {
+                    ret.push(Inst::xmm_rm_r(
+                        SseOpcode::Xorpd,
+                        RegMem::reg(to_reg.to_reg()),
+                        to_reg,
+                    ));
+                } else {
+                    let tmp = alloc_tmp(types::I64);
+                    ret.push(Inst::imm(OperandSize::Size64, value as u64, tmp));
+
+                    ret.push(Inst::gpr_to_xmm(
+                        SseOpcode::Movq,
+                        RegMem::reg(tmp.to_reg()),
+                        OperandSize::Size64,
+                        to_reg,
+                    ));
+                }
             } else {
-                let value = value as u64;
-                ret.push(Inst::imm(
-                    OperandSize::from_bytes(ty.bytes()),
-                    value.into(),
-                    to_reg,
-                ));
+                // Must be an integer type.
+                debug_assert!(
+                    ty == types::B1
+                        || ty == types::I8
+                        || ty == types::B8
+                        || ty == types::I16
+                        || ty == types::B16
+                        || ty == types::I32
+                        || ty == types::B32
+                        || ty == types::I64
+                        || ty == types::B64
+                        || ty == types::R32
+                        || ty == types::R64
+                );
+                if value == 0 {
+                    ret.push(Inst::alu_rmi_r(
+                        ty == types::I64,
+                        AluRmiROpcode::Xor,
+                        RegMemImm::reg(to_reg.to_reg()),
+                        to_reg,
+                    ));
+                } else {
+                    let value = value as u64;
+                    ret.push(Inst::imm(
+                        OperandSize::from_bytes(ty.bytes()),
+                        value.into(),
+                        to_reg,
+                    ));
+                }
             }
         }
         ret
@@ -2636,6 +2747,17 @@ impl MachInst for Inst {
         RegClass::I64
     }
 
+    fn gen_value_label_marker(label: ValueLabel, reg: Reg) -> Self {
+        Inst::ValueLabelMarker { label, reg }
+    }
+
+    fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> {
+        match self {
+            Inst::ValueLabelMarker { label, reg } => Some((*label, *reg)),
+            _ => None,
+        }
+    }
+
     type LabelUse = LabelUse;
 }
 
diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs
index 0b9784d59d..1319ac2f94 100644
--- a/cranelift/codegen/src/isa/x64/lower.rs
+++ b/cranelift/codegen/src/isa/x64/lower.rs
@@ -8,17 +8,17 @@ use crate::ir::{
 use crate::isa::x64::abi::*;
 use crate::isa::x64::inst::args::*;
 use crate::isa::x64::inst::*;
-use crate::isa::{x64::X64Backend, CallConv};
+use crate::isa::{x64::settings as x64_settings, x64::X64Backend, CallConv};
 use crate::machinst::lower::*;
 use crate::machinst::*;
 use crate::result::CodegenResult;
-use crate::settings::Flags;
+use crate::settings::{Flags, TlsModel};
 use alloc::boxed::Box;
 use alloc::vec::Vec;
 use cranelift_codegen_shared::condcodes::CondCode;
 use log::trace;
 use regalloc::{Reg, RegClass, Writable};
-use smallvec::SmallVec;
+use smallvec::{smallvec, SmallVec};
 use std::convert::TryFrom;
 use target_lexicon::Triple;
 
@@ -28,6 +28,7 @@ use target_lexicon::Triple;
 fn is_int_or_ref_ty(ty: Type) -> bool {
     match ty {
         types::I8 | types::I16 | types::I32 | types::I64 | types::R64 => true,
+        types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true,
         types::R32 => panic!("shouldn't have 32-bits refs on x64"),
         _ => false,
     }
@@ -107,23 +108,26 @@ fn generate_constant<C: LowerCtx<I = Inst>>(ctx: &mut C, ty: Type, c: u64) -> Va
     non_writable_value_regs(cst_copy)
 }
 
-/// Put the given input into a register, and mark it as used (side-effect).
-fn put_input_in_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput) -> Reg {
+/// Put the given input into possibly multiple registers, and mark it as used (side-effect).
+fn put_input_in_regs<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput) -> ValueRegs<Reg> {
     let ty = ctx.input_ty(spec.insn, spec.input);
     let input = ctx.get_input_as_source_or_const(spec.insn, spec.input);
 
     if let Some(c) = input.constant {
         // Generate constants fresh at each use to minimize long-range register pressure.
         generate_constant(ctx, ty, c)
-            .only_reg()
-            .expect("multi-reg values not supported yet")
     } else {
         ctx.put_input_in_regs(spec.insn, spec.input)
-            .only_reg()
-            .expect("multi-reg values not supported yet")
     }
 }
 
+/// Put the given input into a register, and mark it as used (side-effect).
+fn put_input_in_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput) -> Reg {
+    put_input_in_regs(ctx, spec)
+        .only_reg()
+        .expect("Multi-register value not expected")
+}
+
 /// Determines whether a load operation (indicated by `src_insn`) can be merged
 /// into the current lowering point. If so, returns the address-base source (as
 /// an `InsnInput`) and an offset from that address from which to perform the
@@ -373,18 +377,120 @@ fn emit_extract_lane<C: LowerCtx<I = Inst>>(
 ///
 /// Note: make sure that there are no instructions modifying the flags between a call to this
 /// function and the use of the flags!
-fn emit_cmp<C: LowerCtx<I = Inst>>(ctx: &mut C, insn: IRInst) {
+///
+/// Takes the condition code that will be tested, and returns
+/// the condition code that should be used. This allows us to
+/// synthesize comparisons out of multiple instructions for
+/// special cases (e.g., 128-bit integers).
+fn emit_cmp<C: LowerCtx<I = Inst>>(ctx: &mut C, insn: IRInst, cc: IntCC) -> IntCC {
     let ty = ctx.input_ty(insn, 0);
 
     let inputs = [InsnInput { insn, input: 0 }, InsnInput { insn, input: 1 }];
 
-    // TODO Try to commute the operands (and invert the condition) if one is an immediate.
-    let lhs = put_input_in_reg(ctx, inputs[0]);
-    let rhs = input_to_reg_mem_imm(ctx, inputs[1]);
+    if ty == types::I128 {
+        // We need to compare both halves and combine the results appropriately.
+        let cmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+        let cmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+        let lhs = put_input_in_regs(ctx, inputs[0]);
+        let lhs_lo = lhs.regs()[0];
+        let lhs_hi = lhs.regs()[1];
+        let rhs = put_input_in_regs(ctx, inputs[1]);
+        let rhs_lo = RegMemImm::reg(rhs.regs()[0]);
+        let rhs_hi = RegMemImm::reg(rhs.regs()[1]);
+        match cc {
+            IntCC::Equal => {
+                ctx.emit(Inst::cmp_rmi_r(8, rhs_hi, lhs_hi));
+                ctx.emit(Inst::setcc(CC::Z, cmp1));
+                ctx.emit(Inst::cmp_rmi_r(8, rhs_lo, lhs_lo));
+                ctx.emit(Inst::setcc(CC::Z, cmp2));
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::And,
+                    RegMemImm::reg(cmp1.to_reg()),
+                    cmp2,
+                ));
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::And,
+                    RegMemImm::imm(1),
+                    cmp2,
+                ));
+                IntCC::NotEqual
+            }
+            IntCC::NotEqual => {
+                ctx.emit(Inst::cmp_rmi_r(8, rhs_hi, lhs_hi));
+                ctx.emit(Inst::setcc(CC::NZ, cmp1));
+                ctx.emit(Inst::cmp_rmi_r(8, rhs_lo, lhs_lo));
+                ctx.emit(Inst::setcc(CC::NZ, cmp2));
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::Or,
+                    RegMemImm::reg(cmp1.to_reg()),
+                    cmp2,
+                ));
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::And,
+                    RegMemImm::imm(1),
+                    cmp2,
+                ));
+                IntCC::NotEqual
+            }
+            IntCC::SignedLessThan
+            | IntCC::SignedLessThanOrEqual
+            | IntCC::SignedGreaterThan
+            | IntCC::SignedGreaterThanOrEqual
+            | IntCC::UnsignedLessThan
+            | IntCC::UnsignedLessThanOrEqual
+            | IntCC::UnsignedGreaterThan
+            | IntCC::UnsignedGreaterThanOrEqual => {
+                // Result = (lhs_hi <> rhs_hi) ||
+                //          (lhs_hi == rhs_hi && lhs_lo <> rhs_lo)
+                let cmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                ctx.emit(Inst::cmp_rmi_r(8, rhs_hi, lhs_hi));
+                ctx.emit(Inst::setcc(CC::from_intcc(cc.without_equal()), cmp1));
+                ctx.emit(Inst::setcc(CC::Z, cmp2));
+                ctx.emit(Inst::cmp_rmi_r(8, rhs_lo, lhs_lo));
+                ctx.emit(Inst::setcc(CC::from_intcc(cc.unsigned()), cmp3));
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::And,
+                    RegMemImm::reg(cmp2.to_reg()),
+                    cmp3,
+                ));
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::Or,
+                    RegMemImm::reg(cmp1.to_reg()),
+                    cmp3,
+                ));
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::And,
+                    RegMemImm::imm(1),
+                    cmp3,
+                ));
+                IntCC::NotEqual
+            }
+            _ => panic!("Unhandled IntCC in I128 comparison: {:?}", cc),
+        }
+    } else {
+        // TODO Try to commute the operands (and invert the condition) if one is an immediate.
+        let lhs = put_input_in_reg(ctx, inputs[0]);
+        // We force the RHS into a register, and disallow load-op fusion, because we
+        // do not have a transitive guarantee that this cmp-site will be the sole
+        // user of the value. Consider: the icmp might be the only user of a load,
+        // but there may be multiple users of the icmp (e.g.  select or bint
+        // instructions) that each invoke `emit_cmp()`. If we were to allow a load
+        // to sink to the *latest* one, but other sites did not permit sinking, then
+        // we would be missing the load for other cmp-sites.
+        let rhs = put_input_in_reg(ctx, inputs[1]);
 
-    // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives
-    // us dst - src at the machine instruction level, so invert operands.
-    ctx.emit(Inst::cmp_rmi_r(ty.bytes() as u8, rhs, lhs));
+        // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives
+        // us dst - src at the machine instruction level, so invert operands.
+        ctx.emit(Inst::cmp_rmi_r(ty.bytes() as u8, RegMemImm::reg(rhs), lhs));
+        cc
+    }
 }
 
 /// A specification for a fcmp emission.
@@ -465,8 +571,10 @@ fn emit_fcmp<C: LowerCtx<I = Inst>>(
         (inputs[0], inputs[1])
     };
     let lhs = put_input_in_reg(ctx, lhs_input);
-    let rhs = input_to_reg_mem(ctx, rhs_input);
-    ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs));
+    // See above in `emit_cmp()`. We must only use the reg/reg form of the
+    // comparison in order to avoid issues with merged loads.
+    let rhs = put_input_in_reg(ctx, rhs_input);
+    ctx.emit(Inst::xmm_cmp_rm_r(op, RegMem::reg(rhs), lhs));
 
     let cond_result = match cond_code {
         FloatCC::Equal => FcmpCondResult::AndConditions(CC::NP, CC::Z),
@@ -480,6 +588,458 @@ fn emit_fcmp<C: LowerCtx<I = Inst>>(
     cond_result
 }
 
+fn emit_bitrev<C: LowerCtx<I = Inst>>(ctx: &mut C, src: Reg, dst: Writable<Reg>, ty: Type) {
+    let bits = ty.bits();
+    let const_mask = if bits == 64 {
+        0xffff_ffff_ffff_ffff
+    } else {
+        (1u64 << bits) - 1
+    };
+    let tmp0 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+    let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+    let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+
+    ctx.emit(Inst::gen_move(tmp0, src, types::I64));
+
+    // Swap 1-bit units.
+    // tmp1 = src
+    ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64));
+    // tmp2 = 0b0101..
+    ctx.emit(Inst::imm(
+        OperandSize::Size64,
+        0x5555_5555_5555_5555 & const_mask,
+        tmp2,
+    ));
+    // tmp1 = src >> 1
+    ctx.emit(Inst::shift_r(
+        8,
+        ShiftKind::ShiftRightLogical,
+        Some(1),
+        tmp1,
+    ));
+    // tmp1 = (src >> 1) & 0b0101..
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::And,
+        RegMemImm::reg(tmp2.to_reg()),
+        tmp1,
+    ));
+    // tmp2 = src & 0b0101..
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::And,
+        RegMemImm::reg(tmp0.to_reg()),
+        tmp2,
+    ));
+    // tmp2 = (src & 0b0101..) << 1
+    ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(1), tmp2));
+    // tmp0 = (src >> 1) & 0b0101.. | (src & 0b0101..) << 1
+    ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Or,
+        RegMemImm::reg(tmp1.to_reg()),
+        tmp0,
+    ));
+
+    // Swap 2-bit units.
+    ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64));
+    ctx.emit(Inst::imm(
+        OperandSize::Size64,
+        0x3333_3333_3333_3333 & const_mask,
+        tmp2,
+    ));
+    ctx.emit(Inst::shift_r(
+        8,
+        ShiftKind::ShiftRightLogical,
+        Some(2),
+        tmp1,
+    ));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::And,
+        RegMemImm::reg(tmp2.to_reg()),
+        tmp1,
+    ));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::And,
+        RegMemImm::reg(tmp0.to_reg()),
+        tmp2,
+    ));
+    ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(2), tmp2));
+    ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Or,
+        RegMemImm::reg(tmp1.to_reg()),
+        tmp0,
+    ));
+
+    // Swap 4-bit units.
+    ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64));
+    ctx.emit(Inst::imm(
+        OperandSize::Size64,
+        0x0f0f_0f0f_0f0f_0f0f & const_mask,
+        tmp2,
+    ));
+    ctx.emit(Inst::shift_r(
+        8,
+        ShiftKind::ShiftRightLogical,
+        Some(4),
+        tmp1,
+    ));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::And,
+        RegMemImm::reg(tmp2.to_reg()),
+        tmp1,
+    ));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::And,
+        RegMemImm::reg(tmp0.to_reg()),
+        tmp2,
+    ));
+    ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(4), tmp2));
+    ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Or,
+        RegMemImm::reg(tmp1.to_reg()),
+        tmp0,
+    ));
+
+    if bits > 8 {
+        // Swap 8-bit units.
+        ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64));
+        ctx.emit(Inst::imm(
+            OperandSize::Size64,
+            0x00ff_00ff_00ff_00ff & const_mask,
+            tmp2,
+        ));
+        ctx.emit(Inst::shift_r(
+            8,
+            ShiftKind::ShiftRightLogical,
+            Some(8),
+            tmp1,
+        ));
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::And,
+            RegMemImm::reg(tmp2.to_reg()),
+            tmp1,
+        ));
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::And,
+            RegMemImm::reg(tmp0.to_reg()),
+            tmp2,
+        ));
+        ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(8), tmp2));
+        ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64));
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::Or,
+            RegMemImm::reg(tmp1.to_reg()),
+            tmp0,
+        ));
+    }
+
+    if bits > 16 {
+        // Swap 16-bit units.
+        ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64));
+        ctx.emit(Inst::imm(
+            OperandSize::Size64,
+            0x0000_ffff_0000_ffff & const_mask,
+            tmp2,
+        ));
+        ctx.emit(Inst::shift_r(
+            8,
+            ShiftKind::ShiftRightLogical,
+            Some(16),
+            tmp1,
+        ));
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::And,
+            RegMemImm::reg(tmp2.to_reg()),
+            tmp1,
+        ));
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::And,
+            RegMemImm::reg(tmp0.to_reg()),
+            tmp2,
+        ));
+        ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(16), tmp2));
+        ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64));
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::Or,
+            RegMemImm::reg(tmp1.to_reg()),
+            tmp0,
+        ));
+    }
+
+    if bits > 32 {
+        // Swap 32-bit units.
+        ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64));
+        ctx.emit(Inst::imm(
+            OperandSize::Size64,
+            0x0000_0000_ffff_ffff & const_mask,
+            tmp2,
+        ));
+        ctx.emit(Inst::shift_r(
+            8,
+            ShiftKind::ShiftRightLogical,
+            Some(32),
+            tmp1,
+        ));
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::And,
+            RegMemImm::reg(tmp2.to_reg()),
+            tmp1,
+        ));
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::And,
+            RegMemImm::reg(tmp0.to_reg()),
+            tmp2,
+        ));
+        ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(32), tmp2));
+        ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64));
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::Or,
+            RegMemImm::reg(tmp1.to_reg()),
+            tmp0,
+        ));
+    }
+
+    ctx.emit(Inst::gen_move(dst, tmp0.to_reg(), types::I64));
+}
+
+fn emit_shl_i128<C: LowerCtx<I = Inst>>(
+    ctx: &mut C,
+    src: ValueRegs<Reg>,
+    dst: ValueRegs<Writable<Reg>>,
+    amt_src: Reg,
+) {
+    let src_lo = src.regs()[0];
+    let src_hi = src.regs()[1];
+    let dst_lo = dst.regs()[0];
+    let dst_hi = dst.regs()[1];
+
+    // mov tmp1, src_lo
+    // shl tmp1, amt_src
+    // mov tmp2, src_hi
+    // shl tmp2, amt_src
+    // mov amt, 64
+    // sub amt, amt_src
+    // mov tmp3, src_lo
+    // shr tmp3, amt
+    // or tmp3, tmp2
+    // xor dst_lo, dst_lo
+    // mov amt, amt_src
+    // and amt, 64
+    // cmovz dst_hi, tmp3
+    // cmovz dst_lo, tmp1
+    // cmovnz dst_hi, tmp1
+
+    let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+    let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+    let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+    let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+
+    ctx.emit(Inst::gen_move(tmp1, src_lo, types::I64));
+    ctx.emit(Inst::gen_move(
+        Writable::from_reg(regs::rcx()),
+        amt_src,
+        types::I64,
+    ));
+    ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, None, tmp1));
+
+    ctx.emit(Inst::gen_move(tmp2, src_hi, types::I64));
+    ctx.emit(Inst::gen_move(
+        Writable::from_reg(regs::rcx()),
+        amt_src,
+        types::I64,
+    ));
+    ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, None, tmp2));
+
+    ctx.emit(Inst::imm(OperandSize::Size64, 64, amt));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Sub,
+        RegMemImm::reg(amt_src),
+        amt,
+    ));
+
+    ctx.emit(Inst::gen_move(tmp3, src_lo, types::I64));
+    ctx.emit(Inst::gen_move(
+        Writable::from_reg(regs::rcx()),
+        amt.to_reg(),
+        types::I64,
+    ));
+    ctx.emit(Inst::shift_r(8, ShiftKind::ShiftRightLogical, None, tmp3));
+
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Or,
+        RegMemImm::reg(tmp2.to_reg()),
+        tmp3,
+    ));
+
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Xor,
+        RegMemImm::reg(dst_lo.to_reg()),
+        dst_lo,
+    ));
+    // This isn't semantically necessary, but it keeps the
+    // register allocator happy, because it cannot otherwise
+    // infer that cmovz + cmovnz always defines dst_hi.
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Xor,
+        RegMemImm::reg(dst_hi.to_reg()),
+        dst_hi,
+    ));
+
+    ctx.emit(Inst::gen_move(amt, amt_src, types::I64));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::And,
+        RegMemImm::imm(64),
+        amt,
+    ));
+    ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp3.to_reg()), dst_hi));
+    ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp1.to_reg()), dst_lo));
+    ctx.emit(Inst::cmove(8, CC::NZ, RegMem::reg(tmp1.to_reg()), dst_hi));
+}
+
+fn emit_shr_i128<C: LowerCtx<I = Inst>>(
+    ctx: &mut C,
+    src: ValueRegs<Reg>,
+    dst: ValueRegs<Writable<Reg>>,
+    amt_src: Reg,
+    is_signed: bool,
+) {
+    let src_lo = src.regs()[0];
+    let src_hi = src.regs()[1];
+    let dst_lo = dst.regs()[0];
+    let dst_hi = dst.regs()[1];
+
+    // mov tmp1, src_hi
+    // {u,s}shr tmp1, amt_src
+    // mov tmp2, src_lo
+    // {u,s}shr tmp2, amt_src
+    // mov amt, 64
+    // sub amt, amt_src
+    // mov tmp3, src_hi
+    // shl tmp3, amt
+    // or tmp3, tmp2
+    // if is_signed:
+    //   mov dst_hi, src_hi
+    //   sshr dst_hi, 63  // get the sign bit
+    // else:
+    //   xor dst_hi, dst_hi
+    // mov amt, amt_src
+    // and amt, 64
+    // cmovz dst_hi, tmp1
+    // cmovz dst_lo, tmp3
+    // cmovnz dst_lo, tmp1
+
+    let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+    let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+    let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+    let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+
+    let shift_kind = if is_signed {
+        ShiftKind::ShiftRightArithmetic
+    } else {
+        ShiftKind::ShiftRightLogical
+    };
+
+    ctx.emit(Inst::gen_move(tmp1, src_hi, types::I64));
+    ctx.emit(Inst::gen_move(
+        Writable::from_reg(regs::rcx()),
+        amt_src,
+        types::I64,
+    ));
+    ctx.emit(Inst::shift_r(8, shift_kind, None, tmp1));
+
+    ctx.emit(Inst::gen_move(tmp2, src_lo, types::I64));
+    ctx.emit(Inst::gen_move(
+        Writable::from_reg(regs::rcx()),
+        amt_src,
+        types::I64,
+    ));
+    ctx.emit(Inst::shift_r(8, shift_kind, None, tmp2));
+
+    ctx.emit(Inst::imm(OperandSize::Size64, 64, amt));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Sub,
+        RegMemImm::reg(amt_src),
+        amt,
+    ));
+
+    ctx.emit(Inst::gen_move(tmp3, src_hi, types::I64));
+    ctx.emit(Inst::gen_move(
+        Writable::from_reg(regs::rcx()),
+        amt.to_reg(),
+        types::I64,
+    ));
+    ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, None, tmp3));
+
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Or,
+        RegMemImm::reg(tmp2.to_reg()),
+        tmp3,
+    ));
+
+    if is_signed {
+        ctx.emit(Inst::gen_move(dst_hi, src_hi, types::I64));
+        ctx.emit(Inst::shift_r(
+            8,
+            ShiftKind::ShiftRightArithmetic,
+            Some(63),
+            dst_hi,
+        ));
+    } else {
+        ctx.emit(Inst::alu_rmi_r(
+            true,
+            AluRmiROpcode::Xor,
+            RegMemImm::reg(dst_hi.to_reg()),
+            dst_hi,
+        ));
+    }
+    // This isn't semantically necessary, but it keeps the
+    // register allocator happy, because it cannot otherwise
+    // infer that cmovz + cmovnz always defines dst_lo.
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::Xor,
+        RegMemImm::reg(dst_lo.to_reg()),
+        dst_lo,
+    ));
+
+    ctx.emit(Inst::gen_move(amt, amt_src, types::I64));
+    ctx.emit(Inst::alu_rmi_r(
+        true,
+        AluRmiROpcode::And,
+        RegMemImm::imm(64),
+        amt,
+    ));
+    ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp1.to_reg()), dst_hi));
+    ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp3.to_reg()), dst_lo));
+    ctx.emit(Inst::cmove(8, CC::NZ, RegMem::reg(tmp1.to_reg()), dst_lo));
+}
+
 fn make_libcall_sig<C: LowerCtx<I = Inst>>(
     ctx: &mut C,
     insn: IRInst,
@@ -523,7 +1083,7 @@ fn emit_vm_call<C: LowerCtx<I = Inst>>(
     let sig = make_libcall_sig(ctx, insn, call_conv, types::I64);
     let caller_conv = ctx.abi().call_conv();
 
-    let mut abi = X64ABICaller::from_func(&sig, &extname, dist, caller_conv)?;
+    let mut abi = X64ABICaller::from_func(&sig, &extname, dist, caller_conv, flags)?;
 
     abi.emit_stack_pre_adjust(ctx);
 
@@ -667,6 +1227,101 @@ fn lower_to_amode<C: LowerCtx<I = Inst>>(ctx: &mut C, spec: InsnInput, offset: i
     Amode::imm_reg(offset as u32, input).with_flags(flags)
 }
 
+fn emit_moves<C: LowerCtx<I = Inst>>(
+    ctx: &mut C,
+    dst: ValueRegs<Writable<Reg>>,
+    src: ValueRegs<Reg>,
+    ty: Type,
+) {
+    let (_, tys) = Inst::rc_for_type(ty).unwrap();
+    for ((dst, src), ty) in dst.regs().iter().zip(src.regs().iter()).zip(tys.iter()) {
+        ctx.emit(Inst::gen_move(*dst, *src, *ty));
+    }
+}
+
+fn emit_cmoves<C: LowerCtx<I = Inst>>(
+    ctx: &mut C,
+    size: u8,
+    cc: CC,
+    src: ValueRegs<Reg>,
+    dst: ValueRegs<Writable<Reg>>,
+) {
+    let size = size / src.len() as u8;
+    let size = u8::max(size, 4); // at least 32 bits
+    for (dst, src) in dst.regs().iter().zip(src.regs().iter()) {
+        ctx.emit(Inst::cmove(size, cc, RegMem::reg(*src), *dst));
+    }
+}
+
+fn emit_clz<C: LowerCtx<I = Inst>>(
+    ctx: &mut C,
+    orig_ty: Type,
+    ty: Type,
+    src: Reg,
+    dst: Writable<Reg>,
+) {
+    let src = RegMem::reg(src);
+    let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
+    ctx.emit(Inst::imm(
+        OperandSize::from_bytes(ty.bytes()),
+        u64::max_value(),
+        dst,
+    ));
+
+    ctx.emit(Inst::unary_rm_r(
+        ty.bytes() as u8,
+        UnaryRmROpcode::Bsr,
+        src,
+        tmp,
+    ));
+
+    ctx.emit(Inst::cmove(
+        ty.bytes() as u8,
+        CC::Z,
+        RegMem::reg(dst.to_reg()),
+        tmp,
+    ));
+
+    ctx.emit(Inst::imm(
+        OperandSize::from_bytes(ty.bytes()),
+        orig_ty.bits() as u64 - 1,
+        dst,
+    ));
+
+    ctx.emit(Inst::alu_rmi_r(
+        ty == types::I64,
+        AluRmiROpcode::Sub,
+        RegMemImm::reg(tmp.to_reg()),
+        dst,
+    ));
+}
+
+fn emit_ctz<C: LowerCtx<I = Inst>>(
+    ctx: &mut C,
+    orig_ty: Type,
+    ty: Type,
+    src: Reg,
+    dst: Writable<Reg>,
+) {
+    let src = RegMem::reg(src);
+    let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
+    ctx.emit(Inst::imm(OperandSize::Size32, orig_ty.bits() as u64, tmp));
+
+    ctx.emit(Inst::unary_rm_r(
+        ty.bytes() as u8,
+        UnaryRmROpcode::Bsf,
+        src,
+        dst,
+    ));
+
+    ctx.emit(Inst::cmove(
+        ty.bytes() as u8,
+        CC::Z,
+        RegMem::reg(tmp.to_reg()),
+        dst,
+    ));
+}
+
 //=============================================================================
 // Top-level instruction lowering entry point, for one instruction.
 
@@ -675,6 +1330,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
     ctx: &mut C,
     insn: IRInst,
     flags: &Flags,
+    isa_flags: &x64_settings::Flags,
     triple: &Triple,
 ) -> CodegenResult<()> {
     let op = ctx.data(insn).opcode();
@@ -889,6 +1545,102 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 // Move the `lhs` to the same register as `dst`.
                 ctx.emit(Inst::gen_move(dst, lhs, ty));
                 ctx.emit(Inst::xmm_rm_r(sse_op, rhs, dst));
+            } else if ty == types::I128 || ty == types::B128 {
+                let alu_ops = match op {
+                    Opcode::Iadd => (AluRmiROpcode::Add, AluRmiROpcode::Adc),
+                    Opcode::Isub => (AluRmiROpcode::Sub, AluRmiROpcode::Sbb),
+                    // multiply handled specially below
+                    Opcode::Imul => (AluRmiROpcode::Mul, AluRmiROpcode::Mul),
+                    Opcode::Band => (AluRmiROpcode::And, AluRmiROpcode::And),
+                    Opcode::Bor => (AluRmiROpcode::Or, AluRmiROpcode::Or),
+                    Opcode::Bxor => (AluRmiROpcode::Xor, AluRmiROpcode::Xor),
+                    _ => panic!("Unsupported opcode with 128-bit integers: {:?}", op),
+                };
+                let lhs = put_input_in_regs(ctx, inputs[0]);
+                let rhs = put_input_in_regs(ctx, inputs[1]);
+                let dst = get_output_reg(ctx, outputs[0]);
+                assert_eq!(lhs.len(), 2);
+                assert_eq!(rhs.len(), 2);
+                assert_eq!(dst.len(), 2);
+
+                if op != Opcode::Imul {
+                    // add, sub, and, or, xor: just do ops on lower then upper half. Carry-flag
+                    // propagation is implicit (add/adc, sub/sbb).
+                    ctx.emit(Inst::gen_move(dst.regs()[0], lhs.regs()[0], types::I64));
+                    ctx.emit(Inst::gen_move(dst.regs()[1], lhs.regs()[1], types::I64));
+                    ctx.emit(Inst::alu_rmi_r(
+                        /* is_64 = */ true,
+                        alu_ops.0,
+                        RegMemImm::reg(rhs.regs()[0]),
+                        dst.regs()[0],
+                    ));
+                    ctx.emit(Inst::alu_rmi_r(
+                        /* is_64 = */ true,
+                        alu_ops.1,
+                        RegMemImm::reg(rhs.regs()[1]),
+                        dst.regs()[1],
+                    ));
+                } else {
+                    // mul:
+                    //   dst_lo = lhs_lo * rhs_lo
+                    //   dst_hi = umulhi(lhs_lo, rhs_lo) + lhs_lo * rhs_hi + lhs_hi * rhs_lo
+                    //
+                    // so we emit:
+                    //   mov dst_lo, lhs_lo
+                    //   mul dst_lo, rhs_lo
+                    //   mov dst_hi, lhs_lo
+                    //   mul dst_hi, rhs_hi
+                    //   mov tmp, lhs_hi
+                    //   mul tmp, rhs_lo
+                    //   add dst_hi, tmp
+                    //   mov rax, lhs_lo
+                    //   umulhi rhs_lo  // implicit rax arg/dst
+                    //   add dst_hi, rax
+                    let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                    ctx.emit(Inst::gen_move(dst.regs()[0], lhs.regs()[0], types::I64));
+                    ctx.emit(Inst::alu_rmi_r(
+                        /* is_64 = */ true,
+                        AluRmiROpcode::Mul,
+                        RegMemImm::reg(rhs.regs()[0]),
+                        dst.regs()[0],
+                    ));
+                    ctx.emit(Inst::gen_move(dst.regs()[1], lhs.regs()[0], types::I64));
+                    ctx.emit(Inst::alu_rmi_r(
+                        /* is_64 = */ true,
+                        AluRmiROpcode::Mul,
+                        RegMemImm::reg(rhs.regs()[1]),
+                        dst.regs()[1],
+                    ));
+                    ctx.emit(Inst::gen_move(tmp, lhs.regs()[1], types::I64));
+                    ctx.emit(Inst::alu_rmi_r(
+                        /* is_64 = */ true,
+                        AluRmiROpcode::Mul,
+                        RegMemImm::reg(rhs.regs()[0]),
+                        tmp,
+                    ));
+                    ctx.emit(Inst::alu_rmi_r(
+                        /* is_64 = */ true,
+                        AluRmiROpcode::Add,
+                        RegMemImm::reg(tmp.to_reg()),
+                        dst.regs()[1],
+                    ));
+                    ctx.emit(Inst::gen_move(
+                        Writable::from_reg(regs::rax()),
+                        lhs.regs()[0],
+                        types::I64,
+                    ));
+                    ctx.emit(Inst::mul_hi(
+                        /* size = */ 8,
+                        /* signed = */ false,
+                        RegMem::reg(rhs.regs()[0]),
+                    ));
+                    ctx.emit(Inst::alu_rmi_r(
+                        /* is_64 = */ true,
+                        AluRmiROpcode::Add,
+                        RegMemImm::reg(regs::rdx()),
+                        dst.regs()[1],
+                    ));
+                }
             } else {
                 let is_64 = ty == types::I64;
                 let alu_op = match op {
@@ -1013,17 +1765,27 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         Opcode::Bnot => {
             let ty = ty.unwrap();
             let size = ty.bytes() as u8;
-            let src = put_input_in_reg(ctx, inputs[0]);
-            let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            ctx.emit(Inst::gen_move(dst, src, ty));
 
             if ty.is_vector() {
+                let src = put_input_in_reg(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                ctx.emit(Inst::gen_move(dst, src, ty));
                 let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
                 ctx.emit(Inst::equals(ty, RegMem::from(tmp), tmp));
                 ctx.emit(Inst::xor(ty, RegMem::from(tmp), dst));
+            } else if ty == types::I128 || ty == types::B128 {
+                let src = put_input_in_regs(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]);
+                ctx.emit(Inst::gen_move(dst.regs()[0], src.regs()[0], types::I64));
+                ctx.emit(Inst::not(8, dst.regs()[0]));
+                ctx.emit(Inst::gen_move(dst.regs()[1], src.regs()[1], types::I64));
+                ctx.emit(Inst::not(8, dst.regs()[1]));
             } else if ty.is_bool() {
                 unimplemented!("bool bnot")
             } else {
+                let src = put_input_in_reg(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                ctx.emit(Inst::gen_move(dst, src, ty));
                 ctx.emit(Inst::not(size, dst));
             }
         }
@@ -1055,7 +1817,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let dst_ty = ctx.output_ty(insn, 0);
             debug_assert_eq!(ctx.input_ty(insn, 0), dst_ty);
 
-            if !dst_ty.is_vector() {
+            if !dst_ty.is_vector() && dst_ty.bits() <= 64 {
                 // Scalar shifts on x86 have various encodings:
                 // - shift by one bit, e.g. `SAL r/m8, 1` (not used here)
                 // - shift by an immediate amount, e.g. `SAL r/m8, imm8`
@@ -1109,6 +1871,89 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     ctx.emit(Inst::mov_r_r(true, rhs.unwrap(), w_rcx));
                 }
                 ctx.emit(Inst::shift_r(size, shift_kind, count, dst));
+            } else if dst_ty == types::I128 {
+                let amt_src = put_input_in_reg(ctx, inputs[1]);
+                let src = put_input_in_regs(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]);
+
+                match op {
+                    Opcode::Ishl => {
+                        emit_shl_i128(ctx, src, dst, amt_src);
+                    }
+                    Opcode::Ushr => {
+                        emit_shr_i128(ctx, src, dst, amt_src, /* is_signed = */ false);
+                    }
+                    Opcode::Sshr => {
+                        emit_shr_i128(ctx, src, dst, amt_src, /* is_signed = */ true);
+                    }
+                    Opcode::Rotl => {
+                        // (mov tmp, src)
+                        // (shl.i128 tmp, amt)
+                        // (mov dst, src)
+                        // (ushr.i128 dst, 128-amt)
+                        // (or dst, tmp)
+                        let tmp = ctx.alloc_tmp(types::I128);
+                        emit_shl_i128(ctx, src, tmp, amt_src);
+                        let inv_amt = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                        ctx.emit(Inst::imm(OperandSize::Size64, 128, inv_amt));
+                        ctx.emit(Inst::alu_rmi_r(
+                            true,
+                            AluRmiROpcode::Sub,
+                            RegMemImm::reg(amt_src),
+                            inv_amt,
+                        ));
+                        emit_shr_i128(
+                            ctx,
+                            src,
+                            dst,
+                            inv_amt.to_reg(),
+                            /* is_signed = */ false,
+                        );
+                        ctx.emit(Inst::alu_rmi_r(
+                            true,
+                            AluRmiROpcode::Or,
+                            RegMemImm::reg(tmp.regs()[0].to_reg()),
+                            dst.regs()[0],
+                        ));
+                        ctx.emit(Inst::alu_rmi_r(
+                            true,
+                            AluRmiROpcode::Or,
+                            RegMemImm::reg(tmp.regs()[1].to_reg()),
+                            dst.regs()[1],
+                        ));
+                    }
+                    Opcode::Rotr => {
+                        // (mov tmp, src)
+                        // (ushr.i128 tmp, amt)
+                        // (mov dst, src)
+                        // (shl.i128 dst, 128-amt)
+                        // (or dst, tmp)
+                        let tmp = ctx.alloc_tmp(types::I128);
+                        emit_shr_i128(ctx, src, tmp, amt_src, /* is_signed = */ false);
+                        let inv_amt = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                        ctx.emit(Inst::imm(OperandSize::Size64, 128, inv_amt));
+                        ctx.emit(Inst::alu_rmi_r(
+                            true,
+                            AluRmiROpcode::Sub,
+                            RegMemImm::reg(amt_src),
+                            inv_amt,
+                        ));
+                        emit_shl_i128(ctx, src, dst, inv_amt.to_reg());
+                        ctx.emit(Inst::alu_rmi_r(
+                            true,
+                            AluRmiROpcode::Or,
+                            RegMemImm::reg(tmp.regs()[0].to_reg()),
+                            dst.regs()[0],
+                        ));
+                        ctx.emit(Inst::alu_rmi_r(
+                            true,
+                            AluRmiROpcode::Or,
+                            RegMemImm::reg(tmp.regs()[1].to_reg()),
+                            dst.regs()[1],
+                        ));
+                    }
+                    _ => unreachable!(),
+                }
             } else if dst_ty == types::I8X16 && (op == Opcode::Ishl || op == Opcode::Ushr) {
                 // Since the x86 instruction set does not have any 8x16 shift instructions (even in higher feature sets
                 // like AVX), we lower the `ishl.i8x16` and `ushr.i8x16` to a sequence of instructions. The basic idea,
@@ -1431,7 +2276,22 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Clz => {
-            // TODO when the x86 flags have use_lzcnt, we can use LZCNT.
+            let orig_ty = ty.unwrap();
+
+            if isa_flags.use_lzcnt() && (orig_ty == types::I32 || orig_ty == types::I64) {
+                // We can use a plain lzcnt instruction here. Note no special handling is required
+                // for zero inputs, because the machine instruction does what the CLIF expects for
+                // zero, i.e. it returns zero.
+                let src = input_to_reg_mem(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                ctx.emit(Inst::unary_rm_r(
+                    orig_ty.bytes() as u8,
+                    UnaryRmROpcode::Lzcnt,
+                    src,
+                    dst,
+                ));
+                return Ok(());
+            }
 
             // General formula using bit-scan reverse (BSR):
             // mov -1, %dst
@@ -1440,358 +2300,498 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // mov $(size_bits - 1), %dst
             // sub %tmp, %dst
 
-            let (ext_spec, ty) = match ctx.input_ty(insn, 0) {
-                types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32),
-                a if a == types::I32 || a == types::I64 => (None, a),
-                _ => unreachable!(),
-            };
-
-            let src = if let Some(ext_spec) = ext_spec {
-                RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec))
+            if orig_ty == types::I128 {
+                // clz upper, tmp1
+                // clz lower, dst
+                // add dst, 64
+                // cmp tmp1, 64
+                // cmovnz tmp1, dst
+                let dsts = get_output_reg(ctx, outputs[0]);
+                let dst = dsts.regs()[0];
+                let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                let srcs = put_input_in_regs(ctx, inputs[0]);
+                let src_lo = srcs.regs()[0];
+                let src_hi = srcs.regs()[1];
+                emit_clz(ctx, types::I64, types::I64, src_hi, tmp1);
+                emit_clz(ctx, types::I64, types::I64, src_lo, dst);
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::Add,
+                    RegMemImm::imm(64),
+                    dst,
+                ));
+                ctx.emit(Inst::cmp_rmi_r(8, RegMemImm::imm(64), tmp1.to_reg()));
+                ctx.emit(Inst::cmove(8, CC::NZ, RegMem::reg(tmp1.to_reg()), dst));
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::Xor,
+                    RegMemImm::reg(dsts.regs()[1].to_reg()),
+                    dsts.regs()[1],
+                ));
             } else {
-                input_to_reg_mem(ctx, inputs[0])
-            };
-            let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                let (ext_spec, ty) = match orig_ty {
+                    types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32),
+                    a if a == types::I32 || a == types::I64 => (None, a),
+                    _ => unreachable!(),
+                };
+                let src = if let Some(ext_spec) = ext_spec {
+                    extend_input_to_reg(ctx, inputs[0], ext_spec)
+                } else {
+                    put_input_in_reg(ctx, inputs[0])
+                };
 
-            let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
-            ctx.emit(Inst::imm(
-                OperandSize::from_bytes(ty.bytes()),
-                u64::max_value(),
-                dst,
-            ));
-
-            ctx.emit(Inst::unary_rm_r(
-                ty.bytes() as u8,
-                UnaryRmROpcode::Bsr,
-                src,
-                tmp,
-            ));
-
-            ctx.emit(Inst::cmove(
-                ty.bytes() as u8,
-                CC::Z,
-                RegMem::reg(dst.to_reg()),
-                tmp,
-            ));
-
-            ctx.emit(Inst::imm(
-                OperandSize::from_bytes(ty.bytes()),
-                ty.bits() as u64 - 1,
-                dst,
-            ));
-
-            ctx.emit(Inst::alu_rmi_r(
-                ty == types::I64,
-                AluRmiROpcode::Sub,
-                RegMemImm::reg(tmp.to_reg()),
-                dst,
-            ));
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                emit_clz(ctx, orig_ty, ty, src, dst);
+            }
         }
 
         Opcode::Ctz => {
-            // TODO when the x86 flags have use_bmi1, we can use TZCNT.
+            let orig_ty = ctx.input_ty(insn, 0);
+
+            if isa_flags.use_bmi1() && (orig_ty == types::I32 || orig_ty == types::I64) {
+                // We can use a plain tzcnt instruction here. Note no special handling is required
+                // for zero inputs, because the machine instruction does what the CLIF expects for
+                // zero, i.e. it returns zero.
+                let src = input_to_reg_mem(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                ctx.emit(Inst::unary_rm_r(
+                    orig_ty.bytes() as u8,
+                    UnaryRmROpcode::Tzcnt,
+                    src,
+                    dst,
+                ));
+                return Ok(());
+            }
 
             // General formula using bit-scan forward (BSF):
             // bsf %src, %dst
             // mov $(size_bits), %tmp
             // cmovz %tmp, %dst
-            let ty = ctx.input_ty(insn, 0);
-            let ty = if ty.bits() < 32 { types::I32 } else { ty };
-            debug_assert!(ty == types::I32 || ty == types::I64);
+            if orig_ty == types::I128 {
+                // ctz src_lo, dst
+                // ctz src_hi, tmp1
+                // add tmp1, 64
+                // cmp dst, 64
+                // cmovz tmp1, dst
+                let dsts = get_output_reg(ctx, outputs[0]);
+                let dst = dsts.regs()[0];
+                let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                let srcs = put_input_in_regs(ctx, inputs[0]);
+                let src_lo = srcs.regs()[0];
+                let src_hi = srcs.regs()[1];
+                emit_ctz(ctx, types::I64, types::I64, src_lo, dst);
+                emit_ctz(ctx, types::I64, types::I64, src_hi, tmp1);
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::Add,
+                    RegMemImm::imm(64),
+                    tmp1,
+                ));
+                ctx.emit(Inst::cmp_rmi_r(8, RegMemImm::imm(64), dst.to_reg()));
+                ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp1.to_reg()), dst));
+                ctx.emit(Inst::alu_rmi_r(
+                    true,
+                    AluRmiROpcode::Xor,
+                    RegMemImm::reg(dsts.regs()[1].to_reg()),
+                    dsts.regs()[1],
+                ));
+            } else {
+                let ty = if orig_ty.bits() < 32 {
+                    types::I32
+                } else {
+                    orig_ty
+                };
+                debug_assert!(ty == types::I32 || ty == types::I64);
 
-            let src = input_to_reg_mem(ctx, inputs[0]);
-            let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-            let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
-            ctx.emit(Inst::imm(OperandSize::Size32, ty.bits() as u64, tmp));
-
-            ctx.emit(Inst::unary_rm_r(
-                ty.bytes() as u8,
-                UnaryRmROpcode::Bsf,
-                src,
-                dst,
-            ));
-
-            ctx.emit(Inst::cmove(
-                ty.bytes() as u8,
-                CC::Z,
-                RegMem::reg(tmp.to_reg()),
-                dst,
-            ));
+                let src = put_input_in_reg(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                emit_ctz(ctx, orig_ty, ty, src, dst);
+            }
         }
 
         Opcode::Popcnt => {
-            // TODO when the x86 flags have use_popcnt, we can use the popcnt instruction.
-
             let (ext_spec, ty) = match ctx.input_ty(insn, 0) {
                 types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32),
-                a if a == types::I32 || a == types::I64 => (None, a),
+                a if a == types::I32 || a == types::I64 || a == types::I128 => (None, a),
                 _ => unreachable!(),
             };
 
-            let src = if let Some(ext_spec) = ext_spec {
-                RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec))
+            if isa_flags.use_popcnt() {
+                match ty {
+                    types::I32 | types::I64 => {
+                        let src = input_to_reg_mem(ctx, inputs[0]);
+                        let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                        ctx.emit(Inst::unary_rm_r(
+                            ty.bytes() as u8,
+                            UnaryRmROpcode::Popcnt,
+                            src,
+                            dst,
+                        ));
+                        return Ok(());
+                    }
+
+                    types::I128 => {
+                        // The number of ones in a 128-bits value is the plain sum of the number of
+                        // ones in its low and high parts. No risk of overflow here.
+                        let dsts = get_output_reg(ctx, outputs[0]);
+                        let dst = dsts.regs()[0];
+                        let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                        let srcs = put_input_in_regs(ctx, inputs[0]);
+                        let src_lo = srcs.regs()[0];
+                        let src_hi = srcs.regs()[1];
+
+                        ctx.emit(Inst::unary_rm_r(
+                            8,
+                            UnaryRmROpcode::Popcnt,
+                            RegMem::reg(src_lo),
+                            dst,
+                        ));
+                        ctx.emit(Inst::unary_rm_r(
+                            8,
+                            UnaryRmROpcode::Popcnt,
+                            RegMem::reg(src_hi),
+                            tmp,
+                        ));
+                        ctx.emit(Inst::alu_rmi_r(
+                            true,
+                            AluRmiROpcode::Add,
+                            RegMemImm::reg(tmp.to_reg()),
+                            dst,
+                        ));
+
+                        // Zero the result's high component.
+                        ctx.emit(Inst::alu_rmi_r(
+                            true,
+                            AluRmiROpcode::Xor,
+                            RegMemImm::reg(dsts.regs()[1].to_reg()),
+                            dsts.regs()[1],
+                        ));
+
+                        return Ok(());
+                    }
+                    _ => {}
+                }
+            }
+
+            let (srcs, ty): (SmallVec<[RegMem; 2]>, Type) = if let Some(ext_spec) = ext_spec {
+                (
+                    smallvec![RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec))],
+                    ty,
+                )
+            } else if ty == types::I128 {
+                let regs = put_input_in_regs(ctx, inputs[0]);
+                (
+                    smallvec![RegMem::reg(regs.regs()[0]), RegMem::reg(regs.regs()[1])],
+                    types::I64,
+                )
             } else {
                 // N.B.: explicitly put input in a reg here because the width of the instruction
                 // into which this RM op goes may not match the width of the input type (in fact,
                 // it won't for i32.popcnt), and we don't want a larger than necessary load.
-                RegMem::reg(put_input_in_reg(ctx, inputs[0]))
+                (smallvec![RegMem::reg(put_input_in_reg(ctx, inputs[0]))], ty)
             };
-            let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
 
-            if ty == types::I64 {
-                let is_64 = true;
+            let mut dsts: SmallVec<[Reg; 2]> = smallvec![];
+            for src in srcs {
+                let dst = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                dsts.push(dst.to_reg());
+                if ty == types::I64 {
+                    let is_64 = true;
 
-                let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                let cst = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                    let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                    let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                    let cst = ctx.alloc_tmp(types::I64).only_reg().unwrap();
 
-                // mov src, tmp1
-                ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1));
+                    // mov src, tmp1
+                    ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1));
 
-                // shr $1, tmp1
-                ctx.emit(Inst::shift_r(
-                    8,
-                    ShiftKind::ShiftRightLogical,
-                    Some(1),
-                    tmp1,
-                ));
+                    // shr $1, tmp1
+                    ctx.emit(Inst::shift_r(
+                        8,
+                        ShiftKind::ShiftRightLogical,
+                        Some(1),
+                        tmp1,
+                    ));
 
-                // mov 0x7777_7777_7777_7777, cst
-                ctx.emit(Inst::imm(OperandSize::Size64, 0x7777777777777777, cst));
+                    // mov 0x7777_7777_7777_7777, cst
+                    ctx.emit(Inst::imm(OperandSize::Size64, 0x7777777777777777, cst));
 
-                // andq cst, tmp1
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::And,
-                    RegMemImm::reg(cst.to_reg()),
-                    tmp1,
-                ));
+                    // andq cst, tmp1
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::And,
+                        RegMemImm::reg(cst.to_reg()),
+                        tmp1,
+                    ));
 
-                // mov src, tmp2
-                ctx.emit(Inst::mov64_rm_r(src, tmp2));
+                    // mov src, tmp2
+                    ctx.emit(Inst::mov64_rm_r(src, tmp2));
 
-                // sub tmp1, tmp2
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::Sub,
-                    RegMemImm::reg(tmp1.to_reg()),
-                    tmp2,
-                ));
+                    // sub tmp1, tmp2
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Sub,
+                        RegMemImm::reg(tmp1.to_reg()),
+                        tmp2,
+                    ));
 
-                // shr $1, tmp1
-                ctx.emit(Inst::shift_r(
-                    8,
-                    ShiftKind::ShiftRightLogical,
-                    Some(1),
-                    tmp1,
-                ));
+                    // shr $1, tmp1
+                    ctx.emit(Inst::shift_r(
+                        8,
+                        ShiftKind::ShiftRightLogical,
+                        Some(1),
+                        tmp1,
+                    ));
 
-                // and cst, tmp1
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::And,
-                    RegMemImm::reg(cst.to_reg()),
-                    tmp1,
-                ));
+                    // and cst, tmp1
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::And,
+                        RegMemImm::reg(cst.to_reg()),
+                        tmp1,
+                    ));
 
-                // sub tmp1, tmp2
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::Sub,
-                    RegMemImm::reg(tmp1.to_reg()),
-                    tmp2,
-                ));
+                    // sub tmp1, tmp2
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Sub,
+                        RegMemImm::reg(tmp1.to_reg()),
+                        tmp2,
+                    ));
 
-                // shr $1, tmp1
-                ctx.emit(Inst::shift_r(
-                    8,
-                    ShiftKind::ShiftRightLogical,
-                    Some(1),
-                    tmp1,
-                ));
+                    // shr $1, tmp1
+                    ctx.emit(Inst::shift_r(
+                        8,
+                        ShiftKind::ShiftRightLogical,
+                        Some(1),
+                        tmp1,
+                    ));
 
-                // and cst, tmp1
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::And,
-                    RegMemImm::reg(cst.to_reg()),
-                    tmp1,
-                ));
+                    // and cst, tmp1
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::And,
+                        RegMemImm::reg(cst.to_reg()),
+                        tmp1,
+                    ));
 
-                // sub tmp1, tmp2
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::Sub,
-                    RegMemImm::reg(tmp1.to_reg()),
-                    tmp2,
-                ));
+                    // sub tmp1, tmp2
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Sub,
+                        RegMemImm::reg(tmp1.to_reg()),
+                        tmp2,
+                    ));
 
-                // mov tmp2, dst
-                ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst));
+                    // mov tmp2, dst
+                    ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst));
 
-                // shr $4, dst
-                ctx.emit(Inst::shift_r(8, ShiftKind::ShiftRightLogical, Some(4), dst));
+                    // shr $4, dst
+                    ctx.emit(Inst::shift_r(8, ShiftKind::ShiftRightLogical, Some(4), dst));
 
-                // add tmp2, dst
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::Add,
-                    RegMemImm::reg(tmp2.to_reg()),
-                    dst,
-                ));
+                    // add tmp2, dst
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Add,
+                        RegMemImm::reg(tmp2.to_reg()),
+                        dst,
+                    ));
 
-                // mov $0x0F0F_0F0F_0F0F_0F0F, cst
-                ctx.emit(Inst::imm(OperandSize::Size64, 0x0F0F0F0F0F0F0F0F, cst));
+                    // mov $0x0F0F_0F0F_0F0F_0F0F, cst
+                    ctx.emit(Inst::imm(OperandSize::Size64, 0x0F0F0F0F0F0F0F0F, cst));
 
-                // and cst, dst
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::And,
-                    RegMemImm::reg(cst.to_reg()),
-                    dst,
-                ));
+                    // and cst, dst
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::And,
+                        RegMemImm::reg(cst.to_reg()),
+                        dst,
+                    ));
 
-                // mov $0x0101_0101_0101_0101, cst
-                ctx.emit(Inst::imm(OperandSize::Size64, 0x0101010101010101, cst));
+                    // mov $0x0101_0101_0101_0101, cst
+                    ctx.emit(Inst::imm(OperandSize::Size64, 0x0101010101010101, cst));
 
-                // mul cst, dst
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::Mul,
-                    RegMemImm::reg(cst.to_reg()),
-                    dst,
-                ));
+                    // mul cst, dst
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Mul,
+                        RegMemImm::reg(cst.to_reg()),
+                        dst,
+                    ));
 
-                // shr $56, dst
-                ctx.emit(Inst::shift_r(
-                    8,
-                    ShiftKind::ShiftRightLogical,
-                    Some(56),
-                    dst,
-                ));
+                    // shr $56, dst
+                    ctx.emit(Inst::shift_r(
+                        8,
+                        ShiftKind::ShiftRightLogical,
+                        Some(56),
+                        dst,
+                    ));
+                } else {
+                    assert_eq!(ty, types::I32);
+                    let is_64 = false;
+
+                    let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                    let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+
+                    // mov src, tmp1
+                    ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1));
+
+                    // shr $1, tmp1
+                    ctx.emit(Inst::shift_r(
+                        4,
+                        ShiftKind::ShiftRightLogical,
+                        Some(1),
+                        tmp1,
+                    ));
+
+                    // andq $0x7777_7777, tmp1
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::And,
+                        RegMemImm::imm(0x77777777),
+                        tmp1,
+                    ));
+
+                    // mov src, tmp2
+                    ctx.emit(Inst::mov64_rm_r(src, tmp2));
+
+                    // sub tmp1, tmp2
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Sub,
+                        RegMemImm::reg(tmp1.to_reg()),
+                        tmp2,
+                    ));
+
+                    // shr $1, tmp1
+                    ctx.emit(Inst::shift_r(
+                        4,
+                        ShiftKind::ShiftRightLogical,
+                        Some(1),
+                        tmp1,
+                    ));
+
+                    // and 0x7777_7777, tmp1
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::And,
+                        RegMemImm::imm(0x77777777),
+                        tmp1,
+                    ));
+
+                    // sub tmp1, tmp2
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Sub,
+                        RegMemImm::reg(tmp1.to_reg()),
+                        tmp2,
+                    ));
+
+                    // shr $1, tmp1
+                    ctx.emit(Inst::shift_r(
+                        4,
+                        ShiftKind::ShiftRightLogical,
+                        Some(1),
+                        tmp1,
+                    ));
+
+                    // and $0x7777_7777, tmp1
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::And,
+                        RegMemImm::imm(0x77777777),
+                        tmp1,
+                    ));
+
+                    // sub tmp1, tmp2
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Sub,
+                        RegMemImm::reg(tmp1.to_reg()),
+                        tmp2,
+                    ));
+
+                    // mov tmp2, dst
+                    ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst));
+
+                    // shr $4, dst
+                    ctx.emit(Inst::shift_r(4, ShiftKind::ShiftRightLogical, Some(4), dst));
+
+                    // add tmp2, dst
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Add,
+                        RegMemImm::reg(tmp2.to_reg()),
+                        dst,
+                    ));
+
+                    // and $0x0F0F_0F0F, dst
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::And,
+                        RegMemImm::imm(0x0F0F0F0F),
+                        dst,
+                    ));
+
+                    // mul $0x0101_0101, dst
+                    ctx.emit(Inst::alu_rmi_r(
+                        is_64,
+                        AluRmiROpcode::Mul,
+                        RegMemImm::imm(0x01010101),
+                        dst,
+                    ));
+
+                    // shr $24, dst
+                    ctx.emit(Inst::shift_r(
+                        4,
+                        ShiftKind::ShiftRightLogical,
+                        Some(24),
+                        dst,
+                    ));
+                }
+            }
+
+            if dsts.len() == 1 {
+                let final_dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                ctx.emit(Inst::gen_move(final_dst, dsts[0], types::I64));
             } else {
-                assert_eq!(ty, types::I32);
-                let is_64 = false;
-
-                let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-
-                // mov src, tmp1
-                ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1));
-
-                // shr $1, tmp1
-                ctx.emit(Inst::shift_r(
-                    4,
-                    ShiftKind::ShiftRightLogical,
-                    Some(1),
-                    tmp1,
-                ));
-
-                // andq $0x7777_7777, tmp1
+                assert!(dsts.len() == 2);
+                let final_dst = get_output_reg(ctx, outputs[0]);
+                ctx.emit(Inst::gen_move(final_dst.regs()[0], dsts[0], types::I64));
                 ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::And,
-                    RegMemImm::imm(0x77777777),
-                    tmp1,
-                ));
-
-                // mov src, tmp2
-                ctx.emit(Inst::mov64_rm_r(src, tmp2));
-
-                // sub tmp1, tmp2
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::Sub,
-                    RegMemImm::reg(tmp1.to_reg()),
-                    tmp2,
-                ));
-
-                // shr $1, tmp1
-                ctx.emit(Inst::shift_r(
-                    4,
-                    ShiftKind::ShiftRightLogical,
-                    Some(1),
-                    tmp1,
-                ));
-
-                // and 0x7777_7777, tmp1
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::And,
-                    RegMemImm::imm(0x77777777),
-                    tmp1,
-                ));
-
-                // sub tmp1, tmp2
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::Sub,
-                    RegMemImm::reg(tmp1.to_reg()),
-                    tmp2,
-                ));
-
-                // shr $1, tmp1
-                ctx.emit(Inst::shift_r(
-                    4,
-                    ShiftKind::ShiftRightLogical,
-                    Some(1),
-                    tmp1,
-                ));
-
-                // and $0x7777_7777, tmp1
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::And,
-                    RegMemImm::imm(0x77777777),
-                    tmp1,
-                ));
-
-                // sub tmp1, tmp2
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::Sub,
-                    RegMemImm::reg(tmp1.to_reg()),
-                    tmp2,
-                ));
-
-                // mov tmp2, dst
-                ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst));
-
-                // shr $4, dst
-                ctx.emit(Inst::shift_r(4, ShiftKind::ShiftRightLogical, Some(4), dst));
-
-                // add tmp2, dst
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
+                    true,
                     AluRmiROpcode::Add,
-                    RegMemImm::reg(tmp2.to_reg()),
-                    dst,
+                    RegMemImm::reg(dsts[1]),
+                    final_dst.regs()[0],
                 ));
-
-                // and $0x0F0F_0F0F, dst
                 ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::And,
-                    RegMemImm::imm(0x0F0F0F0F),
-                    dst,
+                    true,
+                    AluRmiROpcode::Xor,
+                    RegMemImm::reg(final_dst.regs()[1].to_reg()),
+                    final_dst.regs()[1],
                 ));
+            }
+        }
 
-                // mul $0x0101_0101, dst
-                ctx.emit(Inst::alu_rmi_r(
-                    is_64,
-                    AluRmiROpcode::Mul,
-                    RegMemImm::imm(0x01010101),
-                    dst,
-                ));
+        Opcode::Bitrev => {
+            let ty = ctx.input_ty(insn, 0);
+            assert!(
+                ty == types::I8
+                    || ty == types::I16
+                    || ty == types::I32
+                    || ty == types::I64
+                    || ty == types::I128
+            );
 
-                // shr $24, dst
-                ctx.emit(Inst::shift_r(
-                    4,
-                    ShiftKind::ShiftRightLogical,
-                    Some(24),
-                    dst,
-                ));
+            if ty == types::I128 {
+                let src = put_input_in_regs(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]);
+                emit_bitrev(ctx, src.regs()[0], dst.regs()[1], types::I64);
+                emit_bitrev(ctx, src.regs()[1], dst.regs()[0], types::I64);
+            } else {
+                let src = put_input_in_reg(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                emit_bitrev(ctx, src, dst, ty);
             }
         }
 
@@ -1827,63 +2827,112 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let src_ty = ctx.input_ty(insn, 0);
             let dst_ty = ctx.output_ty(insn, 0);
 
-            // Sextend requires a sign-extended move, but all the other opcodes are simply a move
-            // from a zero-extended source. Here is why this works, in each case:
-            //
-            // - Bint: Bool-to-int. We always represent a bool as a 0 or 1, so we merely need to
-            // zero-extend here.
-            //
-            // - Breduce, Bextend: changing width of a boolean. We represent a bool as a 0 or 1, so
-            // again, this is a zero-extend / no-op.
-            //
-            // - Ireduce: changing width of an integer. Smaller ints are stored with undefined
-            // high-order bits, so we can simply do a copy.
+            if src_ty == types::I128 {
+                assert!(dst_ty.bits() <= 64);
+                assert!(op == Opcode::Ireduce);
+                let src = put_input_in_regs(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                ctx.emit(Inst::gen_move(dst, src.regs()[0], types::I64));
+            } else if dst_ty == types::I128 {
+                assert!(src_ty.bits() <= 64);
+                let src = put_input_in_reg(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]);
+                assert!(op == Opcode::Uextend || op == Opcode::Sextend || op == Opcode::Bint);
+                // Extend to 64 bits first.
 
-            if src_ty == types::I32 && dst_ty == types::I64 && op != Opcode::Sextend {
-                // As a particular x64 extra-pattern matching opportunity, all the ALU opcodes on
-                // 32-bits will zero-extend the upper 32-bits, so we can even not generate a
-                // zero-extended move in this case.
-                // TODO add loads and shifts here.
-                if let Some(_) = matches_input_any(
-                    ctx,
-                    inputs[0],
-                    &[
-                        Opcode::Iadd,
-                        Opcode::IaddIfcout,
-                        Opcode::Isub,
-                        Opcode::Imul,
-                        Opcode::Band,
-                        Opcode::Bor,
-                        Opcode::Bxor,
-                    ],
-                ) {
-                    let src = put_input_in_reg(ctx, inputs[0]);
-                    let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                    ctx.emit(Inst::gen_move(dst, src, types::I64));
-                    return Ok(());
-                }
-            }
-
-            let src = input_to_reg_mem(ctx, inputs[0]);
-            let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-            let ext_mode = ExtMode::new(src_ty.bits(), dst_ty.bits());
-            assert_eq!(
-                src_ty.bits() < dst_ty.bits(),
-                ext_mode.is_some(),
-                "unexpected extension: {} -> {}",
-                src_ty,
-                dst_ty
-            );
-
-            if let Some(ext_mode) = ext_mode {
-                if op == Opcode::Sextend {
-                    ctx.emit(Inst::movsx_rm_r(ext_mode, src, dst));
+                let ext_mode = ExtMode::new(src_ty.bits(), /* dst bits = */ 64);
+                if let Some(ext_mode) = ext_mode {
+                    if op == Opcode::Sextend {
+                        ctx.emit(Inst::movsx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0]));
+                    } else {
+                        ctx.emit(Inst::movzx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0]));
+                    }
                 } else {
-                    ctx.emit(Inst::movzx_rm_r(ext_mode, src, dst));
+                    ctx.emit(Inst::mov64_rm_r(RegMem::reg(src), dst.regs()[0]));
+                }
+
+                // Now generate the top 64 bits.
+                if op == Opcode::Sextend {
+                    // Sign-extend: move dst[0] into dst[1] and arithmetic-shift right by 63 bits
+                    // to spread the sign bit across all bits.
+                    ctx.emit(Inst::gen_move(
+                        dst.regs()[1],
+                        dst.regs()[0].to_reg(),
+                        types::I64,
+                    ));
+                    ctx.emit(Inst::shift_r(
+                        8,
+                        ShiftKind::ShiftRightArithmetic,
+                        Some(63),
+                        dst.regs()[1],
+                    ));
+                } else {
+                    // Zero-extend: just zero the top word.
+                    ctx.emit(Inst::alu_rmi_r(
+                        true,
+                        AluRmiROpcode::Xor,
+                        RegMemImm::reg(dst.regs()[1].to_reg()),
+                        dst.regs()[1],
+                    ));
                 }
             } else {
-                ctx.emit(Inst::mov64_rm_r(src, dst));
+                // Sextend requires a sign-extended move, but all the other opcodes are simply a move
+                // from a zero-extended source. Here is why this works, in each case:
+                //
+                // - Bint: Bool-to-int. We always represent a bool as a 0 or 1, so we merely need to
+                // zero-extend here.
+                //
+                // - Breduce, Bextend: changing width of a boolean. We represent a bool as a 0 or 1, so
+                // again, this is a zero-extend / no-op.
+                //
+                // - Ireduce: changing width of an integer. Smaller ints are stored with undefined
+                // high-order bits, so we can simply do a copy.
+                if src_ty == types::I32 && dst_ty == types::I64 && op != Opcode::Sextend {
+                    // As a particular x64 extra-pattern matching opportunity, all the ALU opcodes on
+                    // 32-bits will zero-extend the upper 32-bits, so we can even not generate a
+                    // zero-extended move in this case.
+                    // TODO add loads and shifts here.
+                    if let Some(_) = matches_input_any(
+                        ctx,
+                        inputs[0],
+                        &[
+                            Opcode::Iadd,
+                            Opcode::IaddIfcout,
+                            Opcode::Isub,
+                            Opcode::Imul,
+                            Opcode::Band,
+                            Opcode::Bor,
+                            Opcode::Bxor,
+                        ],
+                    ) {
+                        let src = put_input_in_reg(ctx, inputs[0]);
+                        let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                        ctx.emit(Inst::gen_move(dst, src, types::I64));
+                        return Ok(());
+                    }
+                }
+
+                let src = input_to_reg_mem(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+
+                let ext_mode = ExtMode::new(src_ty.bits(), dst_ty.bits());
+                assert_eq!(
+                    src_ty.bits() < dst_ty.bits(),
+                    ext_mode.is_some(),
+                    "unexpected extension: {} -> {}",
+                    src_ty,
+                    dst_ty
+                );
+
+                if let Some(ext_mode) = ext_mode {
+                    if op == Opcode::Sextend {
+                        ctx.emit(Inst::movsx_rm_r(ext_mode, src, dst));
+                    } else {
+                        ctx.emit(Inst::movzx_rm_r(ext_mode, src, dst));
+                    }
+                } else {
+                    ctx.emit(Inst::mov64_rm_r(src, dst));
+                }
             }
         }
 
@@ -1892,7 +2941,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ctx.input_ty(insn, 0);
             if !ty.is_vector() {
-                emit_cmp(ctx, insn);
+                let condcode = emit_cmp(ctx, insn, condcode);
                 let cc = CC::from_intcc(condcode);
                 ctx.emit(Inst::setcc(cc, dst));
             } else {
@@ -2099,10 +3148,19 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::FallthroughReturn | Opcode::Return => {
             for i in 0..ctx.num_inputs(insn) {
-                let src_reg = put_input_in_reg(ctx, inputs[i]);
+                let src_reg = put_input_in_regs(ctx, inputs[i]);
                 let retval_reg = ctx.retval(i);
                 let ty = ctx.input_ty(insn, i);
-                ctx.emit(Inst::gen_move(retval_reg.only_reg().unwrap(), src_reg, ty));
+                assert!(src_reg.len() == retval_reg.len());
+                let (_, tys) = Inst::rc_for_type(ty)?;
+                for ((&src, &dst), &ty) in src_reg
+                    .regs()
+                    .iter()
+                    .zip(retval_reg.regs().iter())
+                    .zip(tys.iter())
+                {
+                    ctx.emit(Inst::gen_move(dst, src, ty));
+                }
             }
             // N.B.: the Ret itself is generated by the ABI.
         }
@@ -2116,7 +3174,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert_eq!(inputs.len(), sig.params.len());
                     assert_eq!(outputs.len(), sig.returns.len());
                     (
-                        X64ABICaller::from_func(sig, &extname, dist, caller_conv)?,
+                        X64ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?,
                         &inputs[..],
                     )
                 }
@@ -2127,7 +3185,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert_eq!(inputs.len() - 1, sig.params.len());
                     assert_eq!(outputs.len(), sig.returns.len());
                     (
-                        X64ABICaller::from_ptr(sig, ptr, op, caller_conv)?,
+                        X64ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?,
                         &inputs[1..],
                     )
                 }
@@ -2137,14 +3195,15 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
             abi.emit_stack_pre_adjust(ctx);
             assert_eq!(inputs.len(), abi.num_args());
-            for (i, input) in inputs.iter().enumerate() {
-                let arg_reg = put_input_in_reg(ctx, *input);
-                abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg));
+            for i in abi.get_copy_to_arg_order() {
+                let input = inputs[i];
+                let arg_regs = put_input_in_regs(ctx, input);
+                abi.emit_copy_regs_to_arg(ctx, i, arg_regs);
             }
             abi.emit_call(ctx);
             for (i, output) in outputs.iter().enumerate() {
-                let retval_reg = get_output_reg(ctx, *output).only_reg().unwrap();
-                abi.emit_copy_retval_to_regs(ctx, i, ValueRegs::one(retval_reg));
+                let retval_regs = get_output_reg(ctx, *output);
+                abi.emit_copy_retval_to_regs(ctx, i, retval_regs);
             }
             abi.emit_stack_post_adjust(ctx);
         }
@@ -2171,11 +3230,11 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 ctx.emit_safepoint(Inst::TrapIf { trap_code, cc });
             } else if op == Opcode::Trapif {
                 let cond_code = ctx.data(insn).cond_code().unwrap();
-                let cc = CC::from_intcc(cond_code);
 
                 // Verification ensures that the input is always a single-def ifcmp.
                 let ifcmp = matches_input(ctx, inputs[0], Opcode::Ifcmp).unwrap();
-                emit_cmp(ctx, ifcmp);
+                let cond_code = emit_cmp(ctx, ifcmp, cond_code);
+                let cc = CC::from_intcc(cond_code);
 
                 ctx.emit_safepoint(Inst::TrapIf { trap_code, cc });
             } else {
@@ -2257,7 +3316,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
         Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv => {
             let lhs = put_input_in_reg(ctx, inputs[0]);
-            let rhs = input_to_reg_mem(ctx, inputs[1]);
+            // We can't guarantee the RHS (if a load) is 128-bit aligned, so we
+            // must avoid merging a load here.
+            let rhs = RegMem::reg(put_input_in_reg(ctx, inputs[1]));
             let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ty.unwrap();
 
@@ -2514,7 +3575,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::FminPseudo | Opcode::FmaxPseudo => {
-            let lhs = input_to_reg_mem(ctx, inputs[0]);
+            // We can't guarantee the RHS (if a load) is 128-bit aligned, so we
+            // must avoid merging a load here.
+            let lhs = RegMem::reg(put_input_in_reg(ctx, inputs[0]));
             let rhs = put_input_in_reg(ctx, inputs[1]);
             let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ty.unwrap();
@@ -2530,7 +3593,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Sqrt => {
-            let src = input_to_reg_mem(ctx, inputs[0]);
+            // We can't guarantee the RHS (if a load) is 128-bit aligned, so we
+            // must avoid merging a load here.
+            let src = RegMem::reg(put_input_in_reg(ctx, inputs[0]));
             let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             let ty = ty.unwrap();
 
@@ -2549,13 +3614,17 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Fpromote => {
-            let src = input_to_reg_mem(ctx, inputs[0]);
+            // We can't guarantee the RHS (if a load) is 128-bit aligned, so we
+            // must avoid merging a load here.
+            let src = RegMem::reg(put_input_in_reg(ctx, inputs[0]));
             let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtss2sd, src, dst));
         }
 
         Opcode::Fdemote => {
-            let src = input_to_reg_mem(ctx, inputs[0]);
+            // We can't guarantee the RHS (if a load) is 128-bit aligned, so we
+            // must avoid merging a load here.
+            let src = RegMem::reg(put_input_in_reg(ctx, inputs[0]));
             let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
             ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtsd2ss, src, dst));
         }
@@ -2572,7 +3641,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
                 let src = match ext_spec {
                     Some(ext_spec) => RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec)),
-                    None => input_to_reg_mem(ctx, inputs[0]),
+                    None => RegMem::reg(put_input_in_reg(ctx, inputs[0])),
                 };
 
                 let opcode = if output_ty == types::F32 {
@@ -3087,7 +4156,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Fabs | Opcode::Fneg => {
-            let src = input_to_reg_mem(ctx, inputs[0]);
+            let src = RegMem::reg(put_input_in_reg(ctx, inputs[0]));
             let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
 
             // In both cases, generate a constant and apply a single binary instruction:
@@ -3129,8 +4198,11 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     ctx.emit(Inst::gen_move(dst, src, output_ty));
 
                     // Generate an all 1s constant in an XMM register. This uses CMPPS but could
-                    // have used CMPPD with the same effect.
+                    // have used CMPPD with the same effect. Note, we zero the temp we allocate
+                    // because if not, there is a chance that the register we use could be initialized
+                    // with NaN .. in which case the CMPPS would fail since NaN != NaN.
                     let tmp = ctx.alloc_tmp(output_ty).only_reg().unwrap();
+                    ctx.emit(Inst::xmm_rm_r(SseOpcode::Xorps, RegMem::from(tmp), tmp));
                     let cond = FcmpImm::from(FloatCC::Equal);
                     let cmpps = Inst::xmm_rm_r_imm(
                         SseOpcode::Cmpps,
@@ -3222,11 +4294,29 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Ceil | Opcode::Floor | Opcode::Nearest | Opcode::Trunc => {
-            // TODO use ROUNDSS/ROUNDSD after sse4.1.
-
-            // Lower to VM calls when there's no access to SSE4.1.
             let ty = ty.unwrap();
-            if !ty.is_vector() {
+            if isa_flags.use_sse41() {
+                let mode = match op {
+                    Opcode::Ceil => RoundImm::RoundUp,
+                    Opcode::Floor => RoundImm::RoundDown,
+                    Opcode::Nearest => RoundImm::RoundNearest,
+                    Opcode::Trunc => RoundImm::RoundZero,
+                    _ => panic!("unexpected opcode {:?} in Ceil/Floor/Nearest/Trunc", op),
+                };
+                let op = match ty {
+                    types::F32 => SseOpcode::Roundss,
+                    types::F64 => SseOpcode::Roundsd,
+                    types::F32X4 => SseOpcode::Roundps,
+                    types::F64X2 => SseOpcode::Roundpd,
+                    _ => panic!("unexpected type {:?} in Ceil/Floor/Nearest/Trunc", ty),
+                };
+                let src = input_to_reg_mem(ctx, inputs[0]);
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                ctx.emit(Inst::xmm_rm_r_imm(op, src, dst, mode.encode(), false));
+            } else {
+                // Lower to VM calls when there's no access to SSE4.1.
+                // Note, for vector types on platforms that don't support sse41
+                // the execution will panic here.
                 let libcall = match (op, ty) {
                     (Opcode::Ceil, types::F32) => LibCall::CeilF32,
                     (Opcode::Ceil, types::F64) => LibCall::CeilF64,
@@ -3242,28 +4332,6 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     ),
                 };
                 emit_vm_call(ctx, flags, triple, libcall, insn, inputs, outputs)?;
-            } else {
-                let (op, mode) = match (op, ty) {
-                    (Opcode::Ceil, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundUp),
-                    (Opcode::Ceil, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundUp),
-                    (Opcode::Floor, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundDown),
-                    (Opcode::Floor, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundDown),
-                    (Opcode::Trunc, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundZero),
-                    (Opcode::Trunc, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundZero),
-                    (Opcode::Nearest, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundNearest),
-                    (Opcode::Nearest, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundNearest),
-                    _ => panic!("Unknown op/ty combination (vector){:?}", ty),
-                };
-                let src = put_input_in_reg(ctx, inputs[0]);
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                ctx.emit(Inst::gen_move(dst, src, ty));
-                ctx.emit(Inst::xmm_rm_r_imm(
-                    op,
-                    RegMem::from(dst),
-                    dst,
-                    mode.encode(),
-                    false,
-                ));
             }
         }
 
@@ -3380,62 +4448,68 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 _ => unreachable!(),
             };
 
-            let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let is_xmm = elem_ty.is_float() || elem_ty.is_vector();
-
-            match (sign_extend, is_xmm) {
-                (true, false) => {
-                    // The load is sign-extended only when the output size is lower than 64 bits,
-                    // so ext-mode is defined in this case.
-                    ctx.emit(Inst::movsx_rm_r(ext_mode.unwrap(), RegMem::mem(amode), dst));
-                }
-                (false, false) => {
-                    if elem_ty.bytes() == 8 {
-                        // Use a plain load.
-                        ctx.emit(Inst::mov64_m_r(amode, dst))
-                    } else {
-                        // Use a zero-extended load.
-                        ctx.emit(Inst::movzx_rm_r(ext_mode.unwrap(), RegMem::mem(amode), dst))
+            if elem_ty == types::I128 {
+                let dsts = get_output_reg(ctx, outputs[0]);
+                ctx.emit(Inst::mov64_m_r(amode.clone(), dsts.regs()[0]));
+                ctx.emit(Inst::mov64_m_r(amode.offset(8), dsts.regs()[1]));
+            } else {
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                let is_xmm = elem_ty.is_float() || elem_ty.is_vector();
+                match (sign_extend, is_xmm) {
+                    (true, false) => {
+                        // The load is sign-extended only when the output size is lower than 64 bits,
+                        // so ext-mode is defined in this case.
+                        ctx.emit(Inst::movsx_rm_r(ext_mode.unwrap(), RegMem::mem(amode), dst));
                     }
-                }
-                (_, true) => {
-                    ctx.emit(match elem_ty {
-                        types::F32 => Inst::xmm_mov(SseOpcode::Movss, RegMem::mem(amode), dst),
-                        types::F64 => Inst::xmm_mov(SseOpcode::Movsd, RegMem::mem(amode), dst),
-                        types::I8X8 => {
-                            if sign_extend == true {
-                                Inst::xmm_mov(SseOpcode::Pmovsxbw, RegMem::mem(amode), dst)
-                            } else {
-                                Inst::xmm_mov(SseOpcode::Pmovzxbw, RegMem::mem(amode), dst)
+                    (false, false) => {
+                        if elem_ty.bytes() == 8 {
+                            // Use a plain load.
+                            ctx.emit(Inst::mov64_m_r(amode, dst))
+                        } else {
+                            // Use a zero-extended load.
+                            ctx.emit(Inst::movzx_rm_r(ext_mode.unwrap(), RegMem::mem(amode), dst))
+                        }
+                    }
+                    (_, true) => {
+                        ctx.emit(match elem_ty {
+                            types::F32 => Inst::xmm_mov(SseOpcode::Movss, RegMem::mem(amode), dst),
+                            types::F64 => Inst::xmm_mov(SseOpcode::Movsd, RegMem::mem(amode), dst),
+                            types::I8X8 => {
+                                if sign_extend == true {
+                                    Inst::xmm_mov(SseOpcode::Pmovsxbw, RegMem::mem(amode), dst)
+                                } else {
+                                    Inst::xmm_mov(SseOpcode::Pmovzxbw, RegMem::mem(amode), dst)
+                                }
                             }
-                        }
-                        types::I16X4 => {
-                            if sign_extend == true {
-                                Inst::xmm_mov(SseOpcode::Pmovsxwd, RegMem::mem(amode), dst)
-                            } else {
-                                Inst::xmm_mov(SseOpcode::Pmovzxwd, RegMem::mem(amode), dst)
+                            types::I16X4 => {
+                                if sign_extend == true {
+                                    Inst::xmm_mov(SseOpcode::Pmovsxwd, RegMem::mem(amode), dst)
+                                } else {
+                                    Inst::xmm_mov(SseOpcode::Pmovzxwd, RegMem::mem(amode), dst)
+                                }
                             }
-                        }
-                        types::I32X2 => {
-                            if sign_extend == true {
-                                Inst::xmm_mov(SseOpcode::Pmovsxdq, RegMem::mem(amode), dst)
-                            } else {
-                                Inst::xmm_mov(SseOpcode::Pmovzxdq, RegMem::mem(amode), dst)
+                            types::I32X2 => {
+                                if sign_extend == true {
+                                    Inst::xmm_mov(SseOpcode::Pmovsxdq, RegMem::mem(amode), dst)
+                                } else {
+                                    Inst::xmm_mov(SseOpcode::Pmovzxdq, RegMem::mem(amode), dst)
+                                }
                             }
-                        }
-                        _ if elem_ty.is_vector() && elem_ty.bits() == 128 => {
-                            Inst::xmm_mov(SseOpcode::Movups, RegMem::mem(amode), dst)
-                        }
-                        // TODO Specialize for different types: MOVUPD, MOVDQU
-                        _ => unreachable!(
-                            "unexpected type for load: {:?} - {:?}",
-                            elem_ty,
-                            elem_ty.bits()
-                        ),
-                    });
+                            _ if elem_ty.is_vector() && elem_ty.bits() == 128 => {
+                                Inst::xmm_mov(SseOpcode::Movups, RegMem::mem(amode), dst)
+                            }
+                            // TODO Specialize for different types: MOVUPD, MOVDQU
+                            _ => unreachable!(
+                                "unexpected type for load: {:?} - {:?}",
+                                elem_ty,
+                                elem_ty.bits()
+                            ),
+                        });
+                    }
                 }
             }
         }
+
         Opcode::Store
         | Opcode::Istore8
         | Opcode::Istore16
@@ -3479,17 +4553,23 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 _ => unreachable!(),
             };
 
-            let src = put_input_in_reg(ctx, inputs[0]);
+            if elem_ty == types::I128 {
+                let srcs = put_input_in_regs(ctx, inputs[0]);
+                ctx.emit(Inst::mov_r_m(8, srcs.regs()[0], addr.clone()));
+                ctx.emit(Inst::mov_r_m(8, srcs.regs()[1], addr.offset(8)));
+            } else {
+                let src = put_input_in_reg(ctx, inputs[0]);
 
-            ctx.emit(match elem_ty {
-                types::F32 => Inst::xmm_mov_r_m(SseOpcode::Movss, src, addr),
-                types::F64 => Inst::xmm_mov_r_m(SseOpcode::Movsd, src, addr),
-                _ if elem_ty.is_vector() && elem_ty.bits() == 128 => {
-                    // TODO Specialize for different types: MOVUPD, MOVDQU, etc.
-                    Inst::xmm_mov_r_m(SseOpcode::Movups, src, addr)
-                }
-                _ => Inst::mov_r_m(elem_ty.bytes() as u8, src, addr),
-            });
+                ctx.emit(match elem_ty {
+                    types::F32 => Inst::xmm_mov_r_m(SseOpcode::Movss, src, addr),
+                    types::F64 => Inst::xmm_mov_r_m(SseOpcode::Movsd, src, addr),
+                    _ if elem_ty.is_vector() && elem_ty.bits() == 128 => {
+                        // TODO Specialize for different types: MOVUPD, MOVDQU, etc.
+                        Inst::xmm_mov_r_m(SseOpcode::Movups, src, addr)
+                    }
+                    _ => Inst::mov_r_m(elem_ty.bytes() as u8, src, addr),
+                });
+            }
         }
 
         Opcode::AtomicRmw => {
@@ -3656,17 +4736,9 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 };
 
                 let ty = ctx.output_ty(insn, 0);
-                let rhs = put_input_in_reg(ctx, rhs_input);
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                let lhs = if is_int_or_ref_ty(ty) && ty.bytes() < 4 {
-                    // Special case: since the higher bits are undefined per CLIF semantics, we
-                    // can just apply a 32-bit cmove here. Force inputs into registers, to
-                    // avoid partial spilling out-of-bounds with memory accesses, though.
-                    // Sign-extend operands to 32, then do a cmove of size 4.
-                    RegMem::reg(put_input_in_reg(ctx, lhs_input))
-                } else {
-                    input_to_reg_mem(ctx, lhs_input)
-                };
+                let rhs = put_input_in_regs(ctx, rhs_input);
+                let dst = get_output_reg(ctx, outputs[0]);
+                let lhs = put_input_in_regs(ctx, lhs_input);
 
                 // We request inversion of Equal to NotEqual here: taking LHS if equal would mean
                 // take it if both CC::NP and CC::Z are set, the conjunction of which can't be
@@ -3679,15 +4751,20 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert_eq!(cond_code, FloatCC::Equal);
                 }
 
-                ctx.emit(Inst::gen_move(dst, rhs, ty));
+                emit_moves(ctx, dst, rhs, ty);
 
                 match fcmp_results {
                     FcmpCondResult::Condition(cc) => {
-                        if is_int_or_ref_ty(ty) {
-                            let size = u8::max(ty.bytes() as u8, 4);
-                            ctx.emit(Inst::cmove(size, cc, lhs, dst));
+                        if is_int_or_ref_ty(ty) || ty == types::I128 || ty == types::B128 {
+                            let size = ty.bytes() as u8;
+                            emit_cmoves(ctx, size, cc, lhs, dst);
                         } else {
-                            ctx.emit(Inst::xmm_cmove(ty == types::F64, cc, lhs, dst));
+                            ctx.emit(Inst::xmm_cmove(
+                                ty == types::F64,
+                                cc,
+                                RegMem::reg(lhs.only_reg().unwrap()),
+                                dst.only_reg().unwrap(),
+                            ));
                         }
                     }
                     FcmpCondResult::AndConditions(_, _) => {
@@ -3697,40 +4774,37 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     }
                     FcmpCondResult::InvertedEqualOrConditions(cc1, cc2)
                     | FcmpCondResult::OrConditions(cc1, cc2) => {
-                        if is_int_or_ref_ty(ty) {
-                            let size = u8::max(ty.bytes() as u8, 4);
-                            ctx.emit(Inst::cmove(size, cc1, lhs.clone(), dst));
-                            ctx.emit(Inst::cmove(size, cc2, lhs, dst));
+                        if is_int_or_ref_ty(ty) || ty == types::I128 {
+                            let size = ty.bytes() as u8;
+                            emit_cmoves(ctx, size, cc1, lhs.clone(), dst);
+                            emit_cmoves(ctx, size, cc2, lhs, dst);
                         } else {
-                            ctx.emit(Inst::xmm_cmove(ty == types::F64, cc1, lhs.clone(), dst));
-                            ctx.emit(Inst::xmm_cmove(ty == types::F64, cc2, lhs, dst));
+                            ctx.emit(Inst::xmm_cmove(
+                                ty == types::F64,
+                                cc1,
+                                RegMem::reg(lhs.only_reg().unwrap()),
+                                dst.only_reg().unwrap(),
+                            ));
+                            ctx.emit(Inst::xmm_cmove(
+                                ty == types::F64,
+                                cc2,
+                                RegMem::reg(lhs.only_reg().unwrap()),
+                                dst.only_reg().unwrap(),
+                            ));
                         }
                     }
                 }
             } else {
                 let ty = ty.unwrap();
 
-                let mut size = ty.bytes() as u8;
-                let lhs = if is_int_or_ref_ty(ty) {
-                    if size < 4 {
-                        // Special case: since the higher bits are undefined per CLIF semantics, we
-                        // can just apply a 32-bit cmove here. Force inputs into registers, to
-                        // avoid partial spilling out-of-bounds with memory accesses, though.
-                        size = 4;
-                        RegMem::reg(put_input_in_reg(ctx, inputs[1]))
-                    } else {
-                        input_to_reg_mem(ctx, inputs[1])
-                    }
-                } else {
-                    input_to_reg_mem(ctx, inputs[1])
-                };
-
-                let rhs = put_input_in_reg(ctx, inputs[2]);
-                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                let size = ty.bytes() as u8;
+                let lhs = put_input_in_regs(ctx, inputs[1]);
+                let rhs = put_input_in_regs(ctx, inputs[2]);
+                let dst = get_output_reg(ctx, outputs[0]);
 
                 let cc = if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) {
-                    emit_cmp(ctx, icmp);
                     let cond_code = ctx.data(icmp).cond_code().unwrap();
+                    let cond_code = emit_cmp(ctx, icmp, cond_code);
                     CC::from_intcc(cond_code)
                 } else {
                     let sel_ty = ctx.input_ty(insn, 0);
@@ -3756,21 +4830,26 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 };
 
                 // This doesn't affect the flags.
-                ctx.emit(Inst::gen_move(dst, rhs, ty));
+                emit_moves(ctx, dst, rhs, ty);
 
-                if is_int_or_ref_ty(ty) {
-                    ctx.emit(Inst::cmove(size, cc, lhs, dst));
+                if is_int_or_ref_ty(ty) || ty == types::I128 {
+                    emit_cmoves(ctx, size, cc, lhs, dst);
                 } else {
                     debug_assert!(ty == types::F32 || ty == types::F64);
-                    ctx.emit(Inst::xmm_cmove(ty == types::F64, cc, lhs, dst));
+                    ctx.emit(Inst::xmm_cmove(
+                        ty == types::F64,
+                        cc,
+                        RegMem::reg(lhs.only_reg().unwrap()),
+                        dst.only_reg().unwrap(),
+                    ));
                 }
             }
         }
 
         Opcode::Selectif | Opcode::SelectifSpectreGuard => {
-            let lhs = input_to_reg_mem(ctx, inputs[1]);
-            let rhs = put_input_in_reg(ctx, inputs[2]);
-            let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+            let lhs = put_input_in_regs(ctx, inputs[1]);
+            let rhs = put_input_in_regs(ctx, inputs[2]);
+            let dst = get_output_reg(ctx, outputs[0]);
             let ty = ctx.output_ty(insn, 0);
 
             // Verification ensures that the input is always a single-def ifcmp.
@@ -3780,26 +4859,24 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 .unwrap()
                 .0;
             debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp);
-            emit_cmp(ctx, cmp_insn);
+            let cond_code = ctx.data(insn).cond_code().unwrap();
+            let cond_code = emit_cmp(ctx, cmp_insn, cond_code);
 
-            let cc = CC::from_intcc(ctx.data(insn).cond_code().unwrap());
+            let cc = CC::from_intcc(cond_code);
 
-            if is_int_or_ref_ty(ty) {
+            if is_int_or_ref_ty(ty) || ty == types::I128 {
                 let size = ty.bytes() as u8;
-                if size == 1 {
-                    // Sign-extend operands to 32, then do a cmove of size 4.
-                    let lhs_se = ctx.alloc_tmp(types::I32).only_reg().unwrap();
-                    ctx.emit(Inst::movsx_rm_r(ExtMode::BL, lhs, lhs_se));
-                    ctx.emit(Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(rhs), dst));
-                    ctx.emit(Inst::cmove(4, cc, RegMem::reg(lhs_se.to_reg()), dst));
-                } else {
-                    ctx.emit(Inst::gen_move(dst, rhs, ty));
-                    ctx.emit(Inst::cmove(size, cc, lhs, dst));
-                }
+                emit_moves(ctx, dst, rhs, ty);
+                emit_cmoves(ctx, size, cc, lhs, dst);
             } else {
                 debug_assert!(ty == types::F32 || ty == types::F64);
-                ctx.emit(Inst::gen_move(dst, rhs, ty));
-                ctx.emit(Inst::xmm_cmove(ty == types::F64, cc, lhs, dst));
+                emit_moves(ctx, dst, rhs, ty);
+                ctx.emit(Inst::xmm_cmove(
+                    ty == types::F64,
+                    cc,
+                    RegMem::reg(lhs.only_reg().unwrap()),
+                    dst.only_reg().unwrap(),
+                ));
             }
         }
 
@@ -3882,8 +4959,19 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                 // The quotient is in rax.
                 ctx.emit(Inst::gen_move(dst, regs::rax(), input_ty));
             } else {
-                // The remainder is in rdx.
-                ctx.emit(Inst::gen_move(dst, regs::rdx(), input_ty));
+                if size == 1 {
+                    // The remainder is in AH. Right-shift by 8 bits then move from rax.
+                    ctx.emit(Inst::shift_r(
+                        8,
+                        ShiftKind::ShiftRightLogical,
+                        Some(8),
+                        Writable::from_reg(regs::rax()),
+                    ));
+                    ctx.emit(Inst::gen_move(dst, regs::rax(), input_ty));
+                } else {
+                    // The remainder is in rdx.
+                    ctx.emit(Inst::gen_move(dst, regs::rdx(), input_ty));
+                }
             }
         }
 
@@ -4285,6 +5373,61 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             }
         }
 
+        Opcode::Iconcat => {
+            let ty = ctx.output_ty(insn, 0);
+            assert_eq!(
+                ty,
+                types::I128,
+                "Iconcat not expected to be used for non-128-bit type"
+            );
+            assert_eq!(ctx.input_ty(insn, 0), types::I64);
+            assert_eq!(ctx.input_ty(insn, 1), types::I64);
+            let lo = put_input_in_reg(ctx, inputs[0]);
+            let hi = put_input_in_reg(ctx, inputs[1]);
+            let dst = get_output_reg(ctx, outputs[0]);
+            ctx.emit(Inst::gen_move(dst.regs()[0], lo, types::I64));
+            ctx.emit(Inst::gen_move(dst.regs()[1], hi, types::I64));
+        }
+
+        Opcode::Isplit => {
+            let ty = ctx.input_ty(insn, 0);
+            assert_eq!(
+                ty,
+                types::I128,
+                "Iconcat not expected to be used for non-128-bit type"
+            );
+            assert_eq!(ctx.output_ty(insn, 0), types::I64);
+            assert_eq!(ctx.output_ty(insn, 1), types::I64);
+            let src = put_input_in_regs(ctx, inputs[0]);
+            let dst_lo = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+            let dst_hi = get_output_reg(ctx, outputs[1]).only_reg().unwrap();
+            ctx.emit(Inst::gen_move(dst_lo, src.regs()[0], types::I64));
+            ctx.emit(Inst::gen_move(dst_hi, src.regs()[1], types::I64));
+        }
+
+        Opcode::TlsValue => match flags.tls_model() {
+            TlsModel::ElfGd => {
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                let (name, _, _) = ctx.symbol_value(insn).unwrap();
+                let symbol = name.clone();
+                ctx.emit(Inst::ElfTlsGetAddr { symbol });
+                ctx.emit(Inst::gen_move(dst, regs::rax(), types::I64));
+            }
+            TlsModel::Macho => {
+                let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
+                let (name, _, _) = ctx.symbol_value(insn).unwrap();
+                let symbol = name.clone();
+                ctx.emit(Inst::MachOTlsGetAddr { symbol });
+                ctx.emit(Inst::gen_move(dst, regs::rax(), types::I64));
+            }
+            _ => {
+                todo!(
+                    "Unimplemented TLS model in x64 backend: {:?}",
+                    flags.tls_model()
+                );
+            }
+        },
+
         Opcode::IaddImm
         | Opcode::ImulImm
         | Opcode::UdivImm
@@ -4326,7 +5469,7 @@ impl LowerBackend for X64Backend {
     type MInst = Inst;
 
     fn lower<C: LowerCtx<I = Inst>>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> {
-        lower_insn_to_regs(ctx, ir_inst, &self.flags, &self.triple)
+        lower_insn_to_regs(ctx, ir_inst, &self.flags, &self.x64_flags, &self.triple)
     }
 
     fn lower_branch_group<C: LowerCtx<I = Inst>>(
@@ -4372,9 +5515,9 @@ impl LowerBackend for X64Backend {
                     let src_ty = ctx.input_ty(branches[0], 0);
 
                     if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) {
-                        emit_cmp(ctx, icmp);
-
                         let cond_code = ctx.data(icmp).cond_code().unwrap();
+                        let cond_code = emit_cmp(ctx, icmp, cond_code);
+
                         let cond_code = if op0 == Opcode::Brz {
                             cond_code.inverse()
                         } else {
@@ -4404,6 +5547,32 @@ impl LowerBackend for X64Backend {
                             }
                             FcmpCondResult::InvertedEqualOrConditions(_, _) => unreachable!(),
                         }
+                    } else if src_ty == types::I128 {
+                        let src = put_input_in_regs(
+                            ctx,
+                            InsnInput {
+                                insn: branches[0],
+                                input: 0,
+                            },
+                        );
+                        let (half_cc, comb_op) = match op0 {
+                            Opcode::Brz => (CC::Z, AluRmiROpcode::And8),
+                            Opcode::Brnz => (CC::NZ, AluRmiROpcode::Or8),
+                            _ => unreachable!(),
+                        };
+                        let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                        let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                        ctx.emit(Inst::cmp_rmi_r(8, RegMemImm::imm(0), src.regs()[0]));
+                        ctx.emit(Inst::setcc(half_cc, tmp1));
+                        ctx.emit(Inst::cmp_rmi_r(8, RegMemImm::imm(0), src.regs()[1]));
+                        ctx.emit(Inst::setcc(half_cc, tmp2));
+                        ctx.emit(Inst::alu_rmi_r(
+                            false,
+                            comb_op,
+                            RegMemImm::reg(tmp1.to_reg()),
+                            tmp2,
+                        ));
+                        ctx.emit(Inst::jmp_cond(CC::NZ, taken, not_taken));
                     } else if is_int_or_ref_ty(src_ty) || is_bool_ty(src_ty) {
                         let src = put_input_in_reg(
                             ctx,
@@ -4471,8 +5640,8 @@ impl LowerBackend for X64Backend {
                     };
 
                     if let Some(ifcmp) = matches_input(ctx, flag_input, Opcode::Ifcmp) {
-                        emit_cmp(ctx, ifcmp);
                         let cond_code = ctx.data(branches[0]).cond_code().unwrap();
+                        let cond_code = emit_cmp(ctx, ifcmp, cond_code);
                         let cc = CC::from_intcc(cond_code);
                         ctx.emit(Inst::jmp_cond(cc, taken, not_taken));
                     } else if let Some(ifcmp_sp) = matches_input(ctx, flag_input, Opcode::IfcmpSp) {
diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs
index 73183f79e8..28cd503615 100644
--- a/cranelift/codegen/src/isa/x64/mod.rs
+++ b/cranelift/codegen/src/isa/x64/mod.rs
@@ -4,13 +4,15 @@ use self::inst::EmitInfo;
 
 use super::TargetIsa;
 use crate::ir::{condcodes::IntCC, Function};
+use crate::isa::unwind::systemv::RegisterMappingError;
 use crate::isa::x64::{inst::regs::create_reg_universe_systemv, settings as x64_settings};
 use crate::isa::Builder as IsaBuilder;
 use crate::machinst::{compile, MachBackend, MachCompileResult, TargetIsaAdapter, VCode};
 use crate::result::CodegenResult;
 use crate::settings::{self as shared_settings, Flags};
 use alloc::boxed::Box;
-use regalloc::{PrettyPrint, RealRegUniverse};
+use core::hash::{Hash, Hasher};
+use regalloc::{PrettyPrint, RealRegUniverse, Reg};
 use target_lexicon::Triple;
 
 mod abi;
@@ -60,6 +62,7 @@ impl MachBackend for X64Backend {
         let buffer = buffer.finish();
         let frame_size = vcode.frame_size();
         let unwind_info = vcode.unwind_info()?;
+        let value_labels_ranges = vcode.value_labels_ranges()?;
 
         let disasm = if want_disasm {
             Some(vcode.show_rru(Some(&create_reg_universe_systemv(flags))))
@@ -72,6 +75,7 @@ impl MachBackend for X64Backend {
             frame_size,
             disasm,
             unwind_info,
+            value_labels_ranges,
         })
     }
 
@@ -79,6 +83,11 @@ impl MachBackend for X64Backend {
         &self.flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.flags.hash(&mut hasher);
+        self.x64_flags.hash(&mut hasher);
+    }
+
     fn name(&self) -> &'static str {
         "x64"
     }
@@ -127,6 +136,11 @@ impl MachBackend for X64Backend {
     fn create_systemv_cie(&self) -> Option<gimli::write::CommonInformationEntry> {
         Some(inst::unwind::systemv::create_cie())
     }
+
+    #[cfg(feature = "unwind")]
+    fn map_reg_to_dwarf(&self, reg: Reg) -> Result<u16, RegisterMappingError> {
+        inst::unwind::systemv::map_reg(reg).map(|reg| reg.0)
+    }
 }
 
 /// Create a new `isa::Builder`.
diff --git a/cranelift/codegen/src/isa/x86/mod.rs b/cranelift/codegen/src/isa/x86/mod.rs
index cbdeb3069d..272c3dfe5d 100644
--- a/cranelift/codegen/src/isa/x86/mod.rs
+++ b/cranelift/codegen/src/isa/x86/mod.rs
@@ -25,6 +25,7 @@ use alloc::borrow::Cow;
 use alloc::boxed::Box;
 use core::any::Any;
 use core::fmt;
+use core::hash::{Hash, Hasher};
 use target_lexicon::{PointerWidth, Triple};
 
 #[allow(dead_code)]
@@ -78,6 +79,11 @@ impl TargetIsa for Isa {
         &self.shared_flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.shared_flags.hash(&mut hasher);
+        self.isa_flags.hash(&mut hasher);
+    }
+
     fn uses_cpu_flags(&self) -> bool {
         true
     }
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index 59738bd3a5..ca4b2414df 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -1,7 +1,7 @@
 //! ABI definitions.
 
 use crate::binemit::StackMap;
-use crate::ir::StackSlot;
+use crate::ir::{Signature, StackSlot};
 use crate::isa::CallConv;
 use crate::machinst::*;
 use crate::settings;
@@ -27,6 +27,9 @@ pub trait ABICallee {
     /// lowering context exists.
     fn init(&mut self, maybe_tmp: Option<Writable<Reg>>);
 
+    /// Access the (possibly legalized) signature.
+    fn signature(&self) -> &Signature;
+
     /// Accumulate outgoing arguments.  This ensures that at least SIZE bytes
     /// are allocated in the prologue to be available for use in function calls
     /// to hold arguments and/or return values.  If this function is called
@@ -215,6 +218,9 @@ pub trait ABICaller {
     /// Get the number of arguments expected.
     fn num_args(&self) -> usize;
 
+    /// Access the (possibly legalized) signature.
+    fn signature(&self) -> &Signature;
+
     /// Emit a copy of an argument value from a source register, prior to the call.
     fn emit_copy_regs_to_arg<C: LowerCtx<I = Self::I>>(
         &self,
@@ -223,6 +229,11 @@ pub trait ABICaller {
         from_reg: ValueRegs<Reg>,
     );
 
+    /// Specific order for copying into arguments at callsites. We must be
+    /// careful to copy into StructArgs first, because we need to be able
+    /// to invoke memcpy() before we've loaded other arg regs (see above).
+    fn get_copy_to_arg_order(&self) -> SmallVec<[usize; 8]>;
+
     /// Emit a copy a return value into a destination register, after the call returns.
     fn emit_copy_retval_to_regs<C: LowerCtx<I = Self::I>>(
         &self,
diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs
index d315c3defb..439e93e2d9 100644
--- a/cranelift/codegen/src/machinst/abi_impl.rs
+++ b/cranelift/codegen/src/machinst/abi_impl.rs
@@ -111,7 +111,7 @@
 use super::abi::*;
 use crate::binemit::StackMap;
 use crate::ir::types::*;
-use crate::ir::{ArgumentExtension, StackSlot};
+use crate::ir::{ArgumentExtension, ArgumentPurpose, StackSlot};
 use crate::machinst::*;
 use crate::settings;
 use crate::CodegenResult;
@@ -128,22 +128,58 @@ use std::mem;
 #[derive(Clone, Copy, Debug)]
 pub enum ABIArg {
     /// In a real register (or set of registers).
-    Reg(
-        ValueRegs<RealReg>,
-        ir::Type,
-        ir::ArgumentExtension,
-        ir::ArgumentPurpose,
-    ),
+    Reg {
+        /// Register(s) that hold this arg.
+        regs: ValueRegs<RealReg>,
+        /// Value type of this arg.
+        ty: ir::Type,
+        /// Should this arg be zero- or sign-extended?
+        extension: ir::ArgumentExtension,
+        /// Purpose of this arg.
+        purpose: ir::ArgumentPurpose,
+    },
     /// Arguments only: on stack, at given offset from SP at entry.
-    Stack(i64, ir::Type, ir::ArgumentExtension, ir::ArgumentPurpose),
+    Stack {
+        /// Offset of this arg relative to the base of stack args.
+        offset: i64,
+        /// Value type of this arg.
+        ty: ir::Type,
+        /// Should this arg be zero- or sign-extended?
+        extension: ir::ArgumentExtension,
+        /// Purpose of this arg.
+        purpose: ir::ArgumentPurpose,
+    },
+    /// Structure argument. We reserve stack space for it, but the CLIF-level
+    /// semantics are a little weird: the value passed to the call instruction,
+    /// and received in the corresponding block param, is a *pointer*. On the
+    /// caller side, we memcpy the data from the passed-in pointer to the stack
+    /// area; on the callee side, we compute a pointer to this stack area and
+    /// provide that as the argument's value.
+    StructArg {
+        /// Offset of this arg relative to base of stack args.
+        offset: i64,
+        /// Size of this arg on the stack.
+        size: u64,
+        /// Purpose of this arg.
+        purpose: ir::ArgumentPurpose,
+    },
 }
 
 impl ABIArg {
     /// Get the purpose of this arg.
     fn get_purpose(self) -> ir::ArgumentPurpose {
         match self {
-            ABIArg::Reg(_, _, _, purpose) => purpose,
-            ABIArg::Stack(_, _, _, purpose) => purpose,
+            ABIArg::Reg { purpose, .. } => purpose,
+            ABIArg::Stack { purpose, .. } => purpose,
+            ABIArg::StructArg { purpose, .. } => purpose,
+        }
+    }
+
+    /// Is this a StructArg?
+    fn is_struct_arg(self) -> bool {
+        match self {
+            ABIArg::StructArg { .. } => true,
+            _ => false,
         }
     }
 }
@@ -371,6 +407,16 @@ pub trait ABIMachineSpec {
         callee_conv: isa::CallConv,
     ) -> SmallVec<[(InstIsSafepoint, Self::I); 2]>;
 
+    /// Generate a memcpy invocation. Used to set up struct args. May clobber
+    /// caller-save registers; we only memcpy before we start to set up args for
+    /// a call.
+    fn gen_memcpy(
+        call_conv: isa::CallConv,
+        dst: Reg,
+        src: Reg,
+        size: usize,
+    ) -> SmallVec<[Self::I; 8]>;
+
     /// Get the number of spillslots required for the given register-class and
     /// type.
     fn get_number_of_spillslots_for_value(rc: RegClass, ty: Type) -> u32;
@@ -455,6 +501,8 @@ impl ABISig {
 
 /// ABI object for a function body.
 pub struct ABICalleeImpl<M: ABIMachineSpec> {
+    /// CLIF-level signature, possibly normalized.
+    ir_sig: ir::Signature,
     /// Signature: arg and retval regs.
     sig: ABISig,
     /// Offsets to each stackslot.
@@ -510,8 +558,8 @@ fn get_special_purpose_param_register(
 ) -> Option<Reg> {
     let idx = f.signature.special_param_index(purpose)?;
     match abi.args[idx] {
-        ABIArg::Reg(regs, ..) => Some(regs.only_reg().unwrap().to_reg()),
-        ABIArg::Stack(..) => None,
+        ABIArg::Reg { regs, .. } => Some(regs.only_reg().unwrap().to_reg()),
+        _ => None,
     }
 }
 
@@ -520,7 +568,8 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
     pub fn new(f: &ir::Function, flags: settings::Flags) -> CodegenResult<Self> {
         debug!("ABI: func signature {:?}", f.signature);
 
-        let sig = ABISig::from_func_sig::<M>(&f.signature)?;
+        let ir_sig = ensure_struct_return_ptr_is_returned(&f.signature);
+        let sig = ABISig::from_func_sig::<M>(&ir_sig)?;
 
         let call_conv = f.signature.call_conv;
         // Only these calling conventions are supported.
@@ -567,6 +616,7 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
         };
 
         Ok(Self {
+            ir_sig,
             sig,
             stackslots,
             stackslots_size: stack_offset,
@@ -787,9 +837,30 @@ fn gen_store_base_offset_multi<M: ABIMachineSpec>(
     ret
 }
 
+fn ensure_struct_return_ptr_is_returned(sig: &ir::Signature) -> ir::Signature {
+    let params_structret = sig
+        .params
+        .iter()
+        .find(|p| p.purpose == ArgumentPurpose::StructReturn);
+    let rets_have_structret = sig.returns.len() > 0
+        && sig
+            .returns
+            .iter()
+            .any(|arg| arg.purpose == ArgumentPurpose::StructReturn);
+    let mut sig = sig.clone();
+    if params_structret.is_some() && !rets_have_structret {
+        sig.returns.insert(0, params_structret.unwrap().clone());
+    }
+    sig
+}
+
 impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
     type I = M::I;
 
+    fn signature(&self) -> &ir::Signature {
+        &self.ir_sig
+    }
+
     fn temp_needed(&self) -> Option<Type> {
         if self.sig.stack_ret_arg.is_some() {
             Some(M::word_type())
@@ -822,7 +893,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
     fn liveins(&self) -> Set<RealReg> {
         let mut set: Set<RealReg> = Set::empty();
         for &arg in &self.sig.args {
-            if let ABIArg::Reg(regs, ..) = arg {
+            if let ABIArg::Reg { regs, .. } = arg {
                 for &r in regs.regs() {
                     set.insert(r);
                 }
@@ -834,7 +905,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
     fn liveouts(&self) -> Set<RealReg> {
         let mut set: Set<RealReg> = Set::empty();
         for &ret in &self.sig.rets {
-            if let ABIArg::Reg(regs, ..) = ret {
+            if let ABIArg::Reg { regs, .. } = ret {
                 for &r in regs.regs() {
                     set.insert(r);
                 }
@@ -863,14 +934,25 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         match &self.sig.args[idx] {
             // Extension mode doesn't matter (we're copying out, not in; we
             // ignore high bits by convention).
-            &ABIArg::Reg(regs, ty, ..) => {
+            &ABIArg::Reg { regs, ty, .. } => {
                 gen_move_multi::<M>(into_regs, regs.map(|r| r.to_reg()), ty)
             }
-            &ABIArg::Stack(off, ty, ..) => gen_load_stack_multi::<M>(
-                StackAMode::FPOffset(M::fp_to_arg_offset(self.call_conv, &self.flags) + off, ty),
+            &ABIArg::Stack { offset, ty, .. } => gen_load_stack_multi::<M>(
+                StackAMode::FPOffset(
+                    M::fp_to_arg_offset(self.call_conv, &self.flags) + offset,
+                    ty,
+                ),
                 into_regs,
                 ty,
             ),
+            &ABIArg::StructArg { offset, .. } => smallvec![M::gen_get_stack_addr(
+                StackAMode::FPOffset(
+                    M::fp_to_arg_offset(self.call_conv, &self.flags) + offset,
+                    I8,
+                ),
+                into_regs.only_reg().unwrap(),
+                I8,
+            )],
         }
     }
 
@@ -892,10 +974,15 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         let mut ret = smallvec![];
         let word_bits = M::word_bits() as u8;
         match &self.sig.rets[idx] {
-            &ABIArg::Reg(regs, ty, ext, ..) => {
+            &ABIArg::Reg {
+                regs,
+                ty,
+                extension,
+                ..
+            } => {
                 let from_bits = ty_bits(ty) as u8;
                 let dest_regs = writable_value_regs(regs.map(|r| r.to_reg()));
-                let ext = M::get_ext_mode(self.sig.call_conv, ext);
+                let ext = M::get_ext_mode(self.sig.call_conv, extension);
                 match (ext, from_bits) {
                     (ArgumentExtension::Uext, n) | (ArgumentExtension::Sext, n)
                         if n < word_bits =>
@@ -921,14 +1008,20 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
                     ),
                 };
             }
-            &ABIArg::Stack(off, mut ty, ext, ..) => {
+            &ABIArg::Stack {
+                offset,
+                ty,
+                extension,
+                ..
+            } => {
+                let mut ty = ty;
                 let from_bits = ty_bits(ty) as u8;
                 // A machine ABI implementation should ensure that stack frames
                 // have "reasonable" size. All current ABIs for machinst
                 // backends (aarch64 and x64) enforce a 128MB limit.
-                let off = i32::try_from(off)
+                let off = i32::try_from(offset)
                     .expect("Argument stack offset greater than 2GB; should hit impl limit first");
-                let ext = M::get_ext_mode(self.sig.call_conv, ext);
+                let ext = M::get_ext_mode(self.sig.call_conv, extension);
                 // Trash the from_reg; it should be its last use.
                 match (ext, from_bits) {
                     (ArgumentExtension::Uext, n) | (ArgumentExtension::Sext, n)
@@ -961,6 +1054,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
                     .into_iter(),
                 );
             }
+            &ABIArg::StructArg { .. } => panic!("Unexpected StructArg location for return value"),
         }
         ret
     }
@@ -1248,7 +1342,7 @@ fn abisig_to_uses_and_defs<M: ABIMachineSpec>(sig: &ABISig) -> (Vec<Reg>, Vec<Wr
     let mut uses = Vec::new();
     for arg in &sig.args {
         match arg {
-            &ABIArg::Reg(regs, ..) => uses.extend(regs.regs().iter().map(|r| r.to_reg())),
+            &ABIArg::Reg { regs, .. } => uses.extend(regs.regs().iter().map(|r| r.to_reg())),
             _ => {}
         }
     }
@@ -1257,7 +1351,7 @@ fn abisig_to_uses_and_defs<M: ABIMachineSpec>(sig: &ABISig) -> (Vec<Reg>, Vec<Wr
     let mut defs = M::get_regs_clobbered_by_call(sig.call_conv);
     for ret in &sig.rets {
         match ret {
-            &ABIArg::Reg(regs, ..) => {
+            &ABIArg::Reg { regs, .. } => {
                 defs.extend(regs.regs().iter().map(|r| Writable::from_reg(r.to_reg())))
             }
             _ => {}
@@ -1269,6 +1363,8 @@ fn abisig_to_uses_and_defs<M: ABIMachineSpec>(sig: &ABISig) -> (Vec<Reg>, Vec<Wr
 
 /// ABI object for a callsite.
 pub struct ABICallerImpl<M: ABIMachineSpec> {
+    /// CLIF-level signature, possibly normalized.
+    ir_sig: ir::Signature,
     /// The called function's signature.
     sig: ABISig,
     /// All uses for the callsite, i.e., function args.
@@ -1281,6 +1377,8 @@ pub struct ABICallerImpl<M: ABIMachineSpec> {
     opcode: ir::Opcode,
     /// Caller's calling convention.
     caller_conv: isa::CallConv,
+    /// The settings controlling this compilation.
+    flags: settings::Flags,
 
     _mach: PhantomData<M>,
 }
@@ -1301,16 +1399,20 @@ impl<M: ABIMachineSpec> ABICallerImpl<M> {
         extname: &ir::ExternalName,
         dist: RelocDistance,
         caller_conv: isa::CallConv,
+        flags: &settings::Flags,
     ) -> CodegenResult<ABICallerImpl<M>> {
-        let sig = ABISig::from_func_sig::<M>(sig)?;
+        let ir_sig = ensure_struct_return_ptr_is_returned(sig);
+        let sig = ABISig::from_func_sig::<M>(&ir_sig)?;
         let (uses, defs) = abisig_to_uses_and_defs::<M>(&sig);
         Ok(ABICallerImpl {
+            ir_sig,
             sig,
             uses,
             defs,
             dest: CallDest::ExtName(extname.clone(), dist),
             opcode: ir::Opcode::Call,
             caller_conv,
+            flags: flags.clone(),
             _mach: PhantomData,
         })
     }
@@ -1322,16 +1424,20 @@ impl<M: ABIMachineSpec> ABICallerImpl<M> {
         ptr: Reg,
         opcode: ir::Opcode,
         caller_conv: isa::CallConv,
+        flags: &settings::Flags,
     ) -> CodegenResult<ABICallerImpl<M>> {
-        let sig = ABISig::from_func_sig::<M>(sig)?;
+        let ir_sig = ensure_struct_return_ptr_is_returned(sig);
+        let sig = ABISig::from_func_sig::<M>(&ir_sig)?;
         let (uses, defs) = abisig_to_uses_and_defs::<M>(&sig);
         Ok(ABICallerImpl {
+            ir_sig,
             sig,
             uses,
             defs,
             dest: CallDest::Reg(ptr),
             opcode,
             caller_conv,
+            flags: flags.clone(),
             _mach: PhantomData,
         })
     }
@@ -1355,6 +1461,10 @@ fn adjust_stack_and_nominal_sp<M: ABIMachineSpec, C: LowerCtx<I = M::I>>(
 impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
     type I = M::I;
 
+    fn signature(&self) -> &ir::Signature {
+        &self.ir_sig
+    }
+
     fn num_args(&self) -> usize {
         if self.sig.stack_ret_arg.is_some() {
             self.sig.args.len() - 1
@@ -1387,8 +1497,13 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
         let word_rc = M::word_reg_class();
         let word_bits = M::word_bits() as usize;
         match &self.sig.args[idx] {
-            &ABIArg::Reg(regs, ty, ext, _) => {
-                let ext = M::get_ext_mode(self.sig.call_conv, ext);
+            &ABIArg::Reg {
+                regs,
+                ty,
+                extension,
+                ..
+            } => {
+                let ext = M::get_ext_mode(self.sig.call_conv, extension);
                 if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits {
                     let reg = regs.only_reg().unwrap();
                     assert_eq!(word_rc, reg.get_class());
@@ -1414,8 +1529,14 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
                     }
                 }
             }
-            &ABIArg::Stack(off, mut ty, ext, _) => {
-                let ext = M::get_ext_mode(self.sig.call_conv, ext);
+            &ABIArg::Stack {
+                offset,
+                ty,
+                extension,
+                ..
+            } => {
+                let mut ty = ty;
+                let ext = M::get_ext_mode(self.sig.call_conv, extension);
                 if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits {
                     let from_reg = from_regs
                         .only_reg()
@@ -1439,7 +1560,28 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
                     // Store the extended version.
                     ty = M::word_type();
                 }
-                for insn in gen_store_stack_multi::<M>(StackAMode::SPOffset(off, ty), from_regs, ty)
+                for insn in
+                    gen_store_stack_multi::<M>(StackAMode::SPOffset(offset, ty), from_regs, ty)
+                {
+                    ctx.emit(insn);
+                }
+            }
+            &ABIArg::StructArg { offset, size, .. } => {
+                let src_ptr = from_regs.only_reg().unwrap();
+                let dst_ptr = ctx.alloc_tmp(M::word_type()).only_reg().unwrap();
+                ctx.emit(M::gen_get_stack_addr(
+                    StackAMode::SPOffset(offset, I8),
+                    dst_ptr,
+                    I8,
+                ));
+                // Emit a memcpy from `src_ptr` to `dst_ptr` of `size` bytes.
+                // N.B.: because we process StructArg params *first*, this is
+                // safe w.r.t. clobbers: we have not yet filled in any other
+                // arg regs.
+                let memcpy_call_conv = isa::CallConv::for_libcall(&self.flags, self.sig.call_conv);
+                for insn in
+                    M::gen_memcpy(memcpy_call_conv, dst_ptr.to_reg(), src_ptr, size as usize)
+                        .into_iter()
                 {
                     ctx.emit(insn);
                 }
@@ -1447,6 +1589,24 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
         }
     }
 
+    fn get_copy_to_arg_order(&self) -> SmallVec<[usize; 8]> {
+        let mut ret = SmallVec::new();
+        for (i, arg) in self.sig.args.iter().enumerate() {
+            // Struct args.
+            if arg.is_struct_arg() {
+                ret.push(i);
+            }
+        }
+        for (i, arg) in self.sig.args.iter().enumerate() {
+            // Non-struct args. Skip an appended return-area arg for multivalue
+            // returns, if any.
+            if !arg.is_struct_arg() && i < self.ir_sig.params.len() {
+                ret.push(i);
+            }
+        }
+        ret
+    }
+
     fn emit_copy_retval_to_regs<C: LowerCtx<I = Self::I>>(
         &self,
         ctx: &mut C,
@@ -1456,21 +1616,22 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
         match &self.sig.rets[idx] {
             // Extension mode doesn't matter because we're copying out, not in,
             // and we ignore high bits in our own registers by convention.
-            &ABIArg::Reg(regs, ty, _, _) => {
+            &ABIArg::Reg { regs, ty, .. } => {
                 for insn in gen_move_multi::<M>(into_regs, regs.map(|r| r.to_reg()), ty) {
                     ctx.emit(insn);
                 }
             }
-            &ABIArg::Stack(off, ty, _, _) => {
+            &ABIArg::Stack { offset, ty, .. } => {
                 let ret_area_base = self.sig.stack_arg_space;
                 for insn in gen_load_stack_multi::<M>(
-                    StackAMode::SPOffset(off + ret_area_base, ty),
+                    StackAMode::SPOffset(offset + ret_area_base, ty),
                     into_regs,
                     ty,
                 ) {
                     ctx.emit(insn);
                 }
             }
+            &ABIArg::StructArg { .. } => panic!("Unexpected StructArg location for return value"),
         }
     }
 
diff --git a/cranelift/codegen/src/machinst/adapter.rs b/cranelift/codegen/src/machinst/adapter.rs
index 43ff9e51a4..eb4760fae5 100644
--- a/cranelift/codegen/src/machinst/adapter.rs
+++ b/cranelift/codegen/src/machinst/adapter.rs
@@ -10,7 +10,11 @@ use crate::settings::Flags;
 #[cfg(feature = "testing_hooks")]
 use crate::regalloc::RegDiversions;
 
+#[cfg(feature = "unwind")]
+use crate::isa::unwind::systemv::RegisterMappingError;
+
 use core::any::Any;
+use core::hash::Hasher;
 use std::borrow::Cow;
 use std::fmt;
 use target_lexicon::Triple;
@@ -55,6 +59,10 @@ impl TargetIsa for TargetIsaAdapter {
         self.backend.flags()
     }
 
+    fn hash_all_flags(&self, hasher: &mut dyn Hasher) {
+        self.backend.hash_all_flags(hasher)
+    }
+
     fn register_info(&self) -> RegInfo {
         // Called from function's Display impl, so we need a stub here.
         RegInfo {
@@ -134,6 +142,11 @@ impl TargetIsa for TargetIsaAdapter {
         self.backend.create_systemv_cie()
     }
 
+    #[cfg(feature = "unwind")]
+    fn map_regalloc_reg_to_dwarf(&self, r: Reg) -> Result<u16, RegisterMappingError> {
+        self.backend.map_reg_to_dwarf(r)
+    }
+
     fn as_any(&self) -> &dyn Any {
         self as &dyn Any
     }
diff --git a/cranelift/codegen/src/machinst/debug.rs b/cranelift/codegen/src/machinst/debug.rs
new file mode 100644
index 0000000000..4768b8d06e
--- /dev/null
+++ b/cranelift/codegen/src/machinst/debug.rs
@@ -0,0 +1,500 @@
+//! Debug info analysis: computes value-label ranges from value-label markers in
+//! generated VCode.
+//!
+//! We "reverse-engineer" debug info like this because it is far more reliable
+//! than generating it while emitting code and keeping it in sync.
+//!
+//! This works by (i) observing "value-label marker" instructions, which are
+//! semantically just an assignment from a register to a "value label" (which
+//! one can think of as another register; they represent, e.g., Wasm locals) at
+//! a certain point in the code, and (ii) observing loads and stores to the
+//! stack and register moves.
+//!
+//! We track, at every program point, the correspondence between each value
+//! label and *all* locations in which it resides. E.g., if it is stored to the
+//! stack, we remember that it is in both a register and the stack slot; but if
+//! the register is later overwritten, then we have it just in the stack slot.
+//! This allows us to avoid false-positives observing loads/stores that we think
+//! are spillslots but really aren't.
+//!
+//! We do a standard forward dataflow analysis to compute this info.
+
+use crate::ir::ValueLabel;
+use crate::machinst::*;
+use crate::value_label::{LabelValueLoc, ValueLabelsRanges, ValueLocRange};
+use log::trace;
+use regalloc::{Reg, RegUsageCollector};
+use std::collections::{HashMap, HashSet};
+use std::hash::Hash;
+
+/// Location of a labeled value: in a register or in a stack slot. Note that a
+/// value may live in more than one location; `AnalysisInfo` maps each
+/// value-label to multiple `ValueLoc`s.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+enum ValueLoc {
+    Reg(Reg),
+    /// Nominal-SP offset.
+    Stack(i64),
+}
+
+impl From<ValueLoc> for LabelValueLoc {
+    fn from(v: ValueLoc) -> Self {
+        match v {
+            ValueLoc::Reg(r) => LabelValueLoc::Reg(r),
+            ValueLoc::Stack(off) => LabelValueLoc::SPOffset(off),
+        }
+    }
+}
+
+impl ValueLoc {
+    fn is_reg(self) -> bool {
+        match self {
+            ValueLoc::Reg(_) => true,
+            _ => false,
+        }
+    }
+    fn is_stack(self) -> bool {
+        match self {
+            ValueLoc::Stack(_) => true,
+            _ => false,
+        }
+    }
+}
+
+/// Mappings at one program point.
+#[derive(Clone, Debug)]
+struct AnalysisInfo {
+    /// Nominal SP relative to real SP. If `None`, then the offset is
+    /// indeterminate (i.e., we merged to the lattice 'bottom' element). This
+    /// should not happen in well-formed code.
+    nominal_sp_offset: Option<i64>,
+    /// Forward map from labeled values to sets of locations.
+    label_to_locs: HashMap<ValueLabel, HashSet<ValueLoc>>,
+    /// Reverse map for each register indicating the value it holds, if any.
+    reg_to_label: HashMap<Reg, ValueLabel>,
+    /// Reverse map for each stack offset indicating the value it holds, if any.
+    stack_to_label: HashMap<i64, ValueLabel>,
+}
+
+/// Get the registers written (mod'd or def'd) by a machine instruction.
+fn get_inst_writes<M: MachInst>(m: &M) -> Vec<Reg> {
+    // TODO: expose this part of regalloc.rs's interface publicly.
+    let mut vecs = RegUsageCollector::get_empty_reg_vecs_test_framework_only(false);
+    let mut coll = RegUsageCollector::new(&mut vecs);
+    m.get_regs(&mut coll);
+    vecs.defs.extend(vecs.mods.into_iter());
+    vecs.defs
+}
+
+impl AnalysisInfo {
+    /// Create a new analysis state. This is the "top" lattice element at which
+    /// the fixpoint dataflow analysis starts.
+    fn new() -> Self {
+        AnalysisInfo {
+            nominal_sp_offset: Some(0),
+            label_to_locs: HashMap::new(),
+            reg_to_label: HashMap::new(),
+            stack_to_label: HashMap::new(),
+        }
+    }
+
+    /// Remove all locations for a given labeled value. Used when the labeled
+    /// value is redefined (so old values become stale).
+    fn clear_label(&mut self, label: ValueLabel) {
+        if let Some(locs) = self.label_to_locs.remove(&label) {
+            for loc in locs {
+                match loc {
+                    ValueLoc::Reg(r) => {
+                        self.reg_to_label.remove(&r);
+                    }
+                    ValueLoc::Stack(off) => {
+                        self.stack_to_label.remove(&off);
+                    }
+                }
+            }
+        }
+    }
+
+    /// Remove a label from a register, if any. Used, e.g., if the register is
+    /// overwritten.
+    fn clear_reg(&mut self, reg: Reg) {
+        if let Some(label) = self.reg_to_label.remove(&reg) {
+            if let Some(locs) = self.label_to_locs.get_mut(&label) {
+                locs.remove(&ValueLoc::Reg(reg));
+            }
+        }
+    }
+
+    /// Remove a label from a stack offset, if any. Used, e.g., when the stack
+    /// slot is overwritten.
+    fn clear_stack_off(&mut self, off: i64) {
+        if let Some(label) = self.stack_to_label.remove(&off) {
+            if let Some(locs) = self.label_to_locs.get_mut(&label) {
+                locs.remove(&ValueLoc::Stack(off));
+            }
+        }
+    }
+
+    /// Indicate that a labeled value is newly defined and its new value is in
+    /// `reg`.
+    fn def_label_at_reg(&mut self, label: ValueLabel, reg: Reg) {
+        self.clear_label(label);
+        self.label_to_locs
+            .entry(label)
+            .or_insert_with(|| HashSet::new())
+            .insert(ValueLoc::Reg(reg));
+        self.reg_to_label.insert(reg, label);
+    }
+
+    /// Process a store from a register to a stack slot (offset).
+    fn store_reg(&mut self, reg: Reg, off: i64) {
+        self.clear_stack_off(off);
+        if let Some(label) = self.reg_to_label.get(&reg) {
+            if let Some(locs) = self.label_to_locs.get_mut(label) {
+                locs.insert(ValueLoc::Stack(off));
+            }
+            self.stack_to_label.insert(off, *label);
+        }
+    }
+
+    /// Process a load from a stack slot (offset) to a register.
+    fn load_reg(&mut self, reg: Reg, off: i64) {
+        self.clear_reg(reg);
+        if let Some(&label) = self.stack_to_label.get(&off) {
+            if let Some(locs) = self.label_to_locs.get_mut(&label) {
+                locs.insert(ValueLoc::Reg(reg));
+            }
+            self.reg_to_label.insert(reg, label);
+        }
+    }
+
+    /// Process a move from one register to another.
+    fn move_reg(&mut self, to: Reg, from: Reg) {
+        self.clear_reg(to);
+        if let Some(&label) = self.reg_to_label.get(&from) {
+            if let Some(locs) = self.label_to_locs.get_mut(&label) {
+                locs.insert(ValueLoc::Reg(to));
+            }
+            self.reg_to_label.insert(to, label);
+        }
+    }
+
+    /// Update the analysis state w.r.t. an instruction's effects. Given the
+    /// state just before `inst`, this method updates `self` to be the state
+    /// just after `inst`.
+    fn step<M: MachInst>(&mut self, inst: &M) {
+        for write in get_inst_writes(inst) {
+            self.clear_reg(write);
+        }
+        if let Some((label, reg)) = inst.defines_value_label() {
+            self.def_label_at_reg(label, reg);
+        }
+        match inst.stack_op_info() {
+            Some(MachInstStackOpInfo::LoadNomSPOff(reg, offset)) => {
+                self.load_reg(reg, offset + self.nominal_sp_offset.unwrap());
+            }
+            Some(MachInstStackOpInfo::StoreNomSPOff(reg, offset)) => {
+                self.store_reg(reg, offset + self.nominal_sp_offset.unwrap());
+            }
+            Some(MachInstStackOpInfo::NomSPAdj(offset)) => {
+                if self.nominal_sp_offset.is_some() {
+                    self.nominal_sp_offset = Some(self.nominal_sp_offset.unwrap() + offset);
+                }
+            }
+            _ => {}
+        }
+        if let Some((to, from)) = inst.is_move() {
+            let to = to.to_reg();
+            self.move_reg(to, from);
+        }
+    }
+}
+
+/// Trait used to implement the dataflow analysis' meet (intersect) function
+/// onthe `AnalysisInfo` components. For efficiency, this is implemented as a
+/// mutation on the LHS, rather than a pure functional operation.
+trait IntersectFrom {
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult;
+}
+
+/// Result of an intersection operation. Indicates whether the mutated LHS
+/// (which becomes the intersection result) differs from the original LHS. Also
+/// indicates if the value has become "empty" and should be removed from a
+/// parent container, if any.
+struct IntersectResult {
+    /// Did the intersection change the LHS input (the one that was mutated into
+    /// the result)? This is needed to drive the fixpoint loop; when no more
+    /// changes occur, then we have converted.
+    changed: bool,
+    /// Is the resulting value "empty"? This can be used when a container, such
+    /// as a map, holds values of this (intersection result) type; when
+    /// `is_empty` is true for the merge of the values at a particular key, we
+    /// can remove that key from the merged (intersected) result. This is not
+    /// necessary for analysis correctness but reduces the memory and runtime
+    /// cost of the fixpoint loop.
+    is_empty: bool,
+}
+
+impl IntersectFrom for AnalysisInfo {
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        let mut changed = false;
+        changed |= self
+            .nominal_sp_offset
+            .intersect_from(&other.nominal_sp_offset)
+            .changed;
+        changed |= self
+            .label_to_locs
+            .intersect_from(&other.label_to_locs)
+            .changed;
+        changed |= self
+            .reg_to_label
+            .intersect_from(&other.reg_to_label)
+            .changed;
+        changed |= self
+            .stack_to_label
+            .intersect_from(&other.stack_to_label)
+            .changed;
+        IntersectResult {
+            changed,
+            is_empty: false,
+        }
+    }
+}
+
+impl<K, V> IntersectFrom for HashMap<K, V>
+where
+    K: Copy + Eq + Hash,
+    V: IntersectFrom,
+{
+    /// Intersection for hashmap: remove keys that are not in both inputs;
+    /// recursively intersect values for keys in common.
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        let mut changed = false;
+        let mut remove_keys = vec![];
+        for k in self.keys() {
+            if !other.contains_key(k) {
+                remove_keys.push(*k);
+            }
+        }
+        for k in &remove_keys {
+            changed = true;
+            self.remove(k);
+        }
+
+        remove_keys.clear();
+        for k in other.keys() {
+            if let Some(v) = self.get_mut(k) {
+                let result = v.intersect_from(other.get(k).unwrap());
+                changed |= result.changed;
+                if result.is_empty {
+                    remove_keys.push(*k);
+                }
+            }
+        }
+        for k in &remove_keys {
+            changed = true;
+            self.remove(k);
+        }
+
+        IntersectResult {
+            changed,
+            is_empty: self.len() == 0,
+        }
+    }
+}
+impl<T> IntersectFrom for HashSet<T>
+where
+    T: Copy + Eq + Hash,
+{
+    /// Intersection for hashset: just take the set intersection.
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        let mut changed = false;
+        let mut remove = vec![];
+        for val in self.iter() {
+            if !other.contains(val) {
+                remove.push(*val);
+            }
+        }
+        for val in remove {
+            changed = true;
+            self.remove(&val);
+        }
+
+        IntersectResult {
+            changed,
+            is_empty: self.len() == 0,
+        }
+    }
+}
+impl IntersectFrom for ValueLabel {
+    // Intersection for labeled value: remove if not equal. This is equivalent
+    // to a three-level lattice with top, bottom, and unordered set of
+    // individual labels in between.
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        IntersectResult {
+            changed: false,
+            is_empty: *self != *other,
+        }
+    }
+}
+impl<T> IntersectFrom for Option<T>
+where
+    T: Copy + Eq,
+{
+    /// Intersectino for Option<T>: recursively intersect if both `Some`, else
+    /// `None`.
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        let mut changed = false;
+        if !(self.is_some() && other.is_some() && self == other) {
+            changed = true;
+            *self = None;
+        }
+        IntersectResult {
+            changed,
+            is_empty: self.is_none(),
+        }
+    }
+}
+
+/// Compute the value-label ranges (locations for program-point ranges for
+/// labeled values) from a given `VCode` compilation result.
+///
+/// In order to compute this information, we perform a dataflow analysis on the
+/// machine code. To do so, and translate the results into a form usable by the
+/// debug-info consumers, we need to know two additional things:
+///
+/// - The machine-code layout (code offsets) of the instructions. DWARF is
+///   encoded in terms of instruction *ends* (and we reason about value
+///   locations at program points *after* instructions, to match this), so we
+///   take an array `inst_ends`, giving us code offsets for each instruction's
+///   end-point. (Note that this is one *past* the last byte; so a 4-byte
+///   instruction at offset 0 has an end offset of 4.)
+///
+/// - The locations of the labels to which branches will jump. Branches can tell
+///   us about their targets in terms of `MachLabel`s, but we don't know where
+///   those `MachLabel`s will be placed in the linear array of instructions.  We
+///   take the array `label_insn_index` to provide this info: for a label with
+///   index `l`, `label_insn_index[l]` is the index of the instruction before
+///   which that label is bound.
+pub(crate) fn compute<I: VCodeInst>(
+    insts: &[I],
+    inst_ends: &[u32],
+    label_insn_index: &[u32],
+) -> ValueLabelsRanges {
+    let inst_start = |idx: usize| if idx == 0 { 0 } else { inst_ends[idx - 1] };
+
+    trace!("compute: insts =");
+    for i in 0..insts.len() {
+        trace!(" #{} end: {} -> {:?}", i, inst_ends[i], insts[i]);
+    }
+    trace!("label_insn_index: {:?}", label_insn_index);
+
+    // Info at each block head, indexed by label.
+    let mut block_starts: HashMap<u32, AnalysisInfo> = HashMap::new();
+
+    // Initialize state at entry.
+    block_starts.insert(0, AnalysisInfo::new());
+
+    // Worklist: label indices for basic blocks.
+    let mut worklist = Vec::new();
+    let mut worklist_set = HashSet::new();
+    worklist.push(0);
+    worklist_set.insert(0);
+
+    while !worklist.is_empty() {
+        let block = worklist.pop().unwrap();
+        worklist_set.remove(&block);
+
+        let mut state = block_starts.get(&block).unwrap().clone();
+        trace!("at block {} -> state: {:?}", block, state);
+        // Iterate for each instruction in the block (we break at the first
+        // terminator we see).
+        let mut index = label_insn_index[block as usize];
+        while index < insts.len() as u32 {
+            state.step(&insts[index as usize]);
+            trace!(" -> inst #{}: {:?}", index, insts[index as usize]);
+            trace!("    --> state: {:?}", state);
+
+            let term = insts[index as usize].is_term();
+            if term.is_term() {
+                for succ in term.get_succs() {
+                    trace!("    SUCCESSOR block {}", succ.get());
+                    if let Some(succ_state) = block_starts.get_mut(&succ.get()) {
+                        trace!("       orig state: {:?}", succ_state);
+                        if succ_state.intersect_from(&state).changed {
+                            if worklist_set.insert(succ.get()) {
+                                worklist.push(succ.get());
+                            }
+                            trace!("        (changed)");
+                        }
+                        trace!("       new state: {:?}", succ_state);
+                    } else {
+                        // First time seeing this block
+                        block_starts.insert(succ.get(), state.clone());
+                        worklist.push(succ.get());
+                        worklist_set.insert(succ.get());
+                    }
+                }
+                break;
+            }
+
+            index += 1;
+        }
+    }
+
+    // Now iterate over blocks one last time, collecting
+    // value-label locations.
+
+    let mut value_labels_ranges: ValueLabelsRanges = HashMap::new();
+    for block in 0..label_insn_index.len() {
+        let start_index = label_insn_index[block];
+        let end_index = if block == label_insn_index.len() - 1 {
+            insts.len() as u32
+        } else {
+            label_insn_index[block + 1]
+        };
+        let block = block as u32;
+        let mut state = block_starts.get(&block).unwrap().clone();
+        for index in start_index..end_index {
+            let offset = inst_start(index as usize);
+            let end = inst_ends[index as usize];
+            state.step(&insts[index as usize]);
+
+            for (label, locs) in &state.label_to_locs {
+                trace!("   inst {} has label {:?} -> locs {:?}", index, label, locs);
+                // Find an appropriate loc: a register if possible, otherwise pick the first stack
+                // loc.
+                let reg = locs.iter().cloned().find(|l| l.is_reg());
+                let loc = reg.or_else(|| locs.iter().cloned().find(|l| l.is_stack()));
+                if let Some(loc) = loc {
+                    let loc = LabelValueLoc::from(loc);
+                    let list = value_labels_ranges.entry(*label).or_insert_with(|| vec![]);
+                    // If the existing location list for this value-label is
+                    // either empty, or has an end location that does not extend
+                    // to the current offset, then we have to append a new
+                    // entry. Otherwise, we can extend the current entry.
+                    //
+                    // Note that `end` is one past the end of the instruction;
+                    // it appears that `end` is exclusive, so a mapping valid at
+                    // offset 5 will have start = 5, end = 6.
+                    if list
+                        .last()
+                        .map(|last| last.end <= offset || last.loc != loc)
+                        .unwrap_or(true)
+                    {
+                        list.push(ValueLocRange {
+                            loc,
+                            start: end,
+                            end: end + 1,
+                        });
+                    } else {
+                        list.last_mut().unwrap().end = end + 1;
+                    }
+                }
+            }
+        }
+    }
+
+    trace!("ret: {:?}", value_labels_ranges);
+    value_labels_ranges
+}
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index 28e4edd0c7..89e184fd4e 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -13,6 +13,7 @@ use crate::ir::instructions::BranchInfo;
 use crate::ir::{
     ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst,
     InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef,
+    ValueLabelAssignments, ValueLabelStart,
 };
 use crate::machinst::{
     writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode,
@@ -24,7 +25,7 @@ use alloc::vec::Vec;
 use core::convert::TryInto;
 use log::debug;
 use regalloc::{Reg, StackmapRequestInfo, Writable};
-use smallvec::SmallVec;
+use smallvec::{smallvec, SmallVec};
 use std::fmt::Debug;
 
 /// An "instruction color" partitions CLIF instructions by side-effecting ops.
@@ -375,8 +376,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             }
         }
 
-        let vm_context = f
-            .signature
+        let vm_context = vcode
+            .abi()
+            .signature()
             .special_param_index(ArgumentPurpose::VMContext)
             .map(|vm_context_index| {
                 let entry_block = f.layout.entry_block().unwrap();
@@ -386,7 +388,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
 
         // Assign vreg(s) to each return value.
         let mut retval_regs = vec![];
-        for ret in &f.signature.returns {
+        for ret in &vcode.abi().signature().returns.clone() {
             let regs = alloc_vregs(ret.value_type, &mut next_vreg, &mut vcode)?;
             retval_regs.push(regs);
             debug!("retval gets regs {:?}", regs);
@@ -465,6 +467,24 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
                 for insn in self.vcode.abi().gen_copy_arg_to_regs(i, regs).into_iter() {
                     self.emit(insn);
                 }
+                if self.abi().signature().params[i].purpose == ArgumentPurpose::StructReturn {
+                    assert!(regs.len() == 1);
+                    let ty = self.abi().signature().params[i].value_type;
+                    // The ABI implementation must have ensured that a StructReturn
+                    // arg is present in the return values.
+                    let struct_ret_idx = self
+                        .abi()
+                        .signature()
+                        .returns
+                        .iter()
+                        .position(|ret| ret.purpose == ArgumentPurpose::StructReturn)
+                        .expect("StructReturn return value not present!");
+                    self.emit(I::gen_move(
+                        Writable::from_reg(self.retval_regs[struct_ret_idx].regs()[0]),
+                        regs.regs()[0].to_reg(),
+                        ty,
+                    ));
+                }
             }
             if let Some(insn) = self.vcode.abi().gen_retval_area_setup() {
                 self.emit(insn);
@@ -473,6 +493,14 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
     }
 
     fn gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn) {
+        // Hack: to keep `vmctx` alive, if it exists, we emit a value label here
+        // for it if debug info is requested. This ensures that it exists either
+        // in a register or spillslot throughout the entire function body, and
+        // allows for a better debugging experience.
+        if let Some(vmctx_val) = self.f.special_param(ArgumentPurpose::VMContext) {
+            self.emit_value_label_marks_for_value(vmctx_val);
+        }
+
         let retval_regs = self.retval_regs.clone();
         for (i, regs) in retval_regs.into_iter().enumerate() {
             let regs = writable_value_regs(regs);
@@ -706,6 +734,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             if has_side_effect || value_needed {
                 debug!("lowering: inst {}: {:?}", inst, self.f.dfg[inst]);
                 backend.lower(self, inst)?;
+                // Emit value-label markers if needed, to later recover debug
+                // mappings.
+                self.emit_value_label_markers_for_inst(inst);
             }
             if data.opcode().is_return() {
                 // Return: handle specially, using ABI-appropriate sequence.
@@ -725,6 +756,80 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         Ok(())
     }
 
+    fn get_value_labels<'a>(&'a self, val: Value, depth: usize) -> Option<&'a [ValueLabelStart]> {
+        if let Some(ref values_labels) = self.f.dfg.values_labels {
+            debug!(
+                "get_value_labels: val {} -> {} -> {:?}",
+                val,
+                self.f.dfg.resolve_aliases(val),
+                values_labels.get(&self.f.dfg.resolve_aliases(val))
+            );
+            let val = self.f.dfg.resolve_aliases(val);
+            match values_labels.get(&val) {
+                Some(&ValueLabelAssignments::Starts(ref list)) => Some(&list[..]),
+                Some(&ValueLabelAssignments::Alias { value, .. }) if depth < 10 => {
+                    self.get_value_labels(value, depth + 1)
+                }
+                _ => None,
+            }
+        } else {
+            None
+        }
+    }
+
+    fn emit_value_label_marks_for_value(&mut self, val: Value) {
+        let mut markers: SmallVec<[I; 4]> = smallvec![];
+        let regs = self.value_regs[val];
+        if regs.len() > 1 {
+            return;
+        }
+        let reg = regs.only_reg().unwrap();
+
+        if let Some(label_starts) = self.get_value_labels(val, 0) {
+            let labels = label_starts
+                .iter()
+                .map(|&ValueLabelStart { label, .. }| label)
+                .collect::<FxHashSet<_>>();
+            for label in labels {
+                debug!(
+                    "value labeling: defines val {:?} -> reg {:?} -> label {:?}",
+                    val, reg, label,
+                );
+                markers.push(I::gen_value_label_marker(label, reg));
+            }
+        }
+        for marker in markers {
+            self.emit(marker);
+        }
+    }
+
+    fn emit_value_label_markers_for_inst(&mut self, inst: Inst) {
+        if self.f.dfg.values_labels.is_none() {
+            return;
+        }
+
+        debug!(
+            "value labeling: srcloc {}: inst {}",
+            self.srcloc(inst),
+            inst
+        );
+        for &val in self.f.dfg.inst_results(inst) {
+            self.emit_value_label_marks_for_value(val);
+        }
+    }
+
+    fn emit_value_label_markers_for_block_args(&mut self, block: Block) {
+        if self.f.dfg.values_labels.is_none() {
+            return;
+        }
+
+        debug!("value labeling: block {}", block);
+        for &arg in self.f.dfg.block_params(block) {
+            self.emit_value_label_marks_for_value(arg);
+        }
+        self.finish_ir_inst(SourceLoc::default());
+    }
+
     fn finish_ir_inst(&mut self, loc: SourceLoc) {
         // `bb_insts` is kept in reverse order, so emit the instructions in
         // reverse order.
@@ -866,6 +971,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             // Original block body.
             if let Some(bb) = lb.orig_block() {
                 self.lower_clif_block(backend, bb)?;
+                self.emit_value_label_markers_for_block_args(bb);
             }
             // In-edge phi moves.
             if let Some((pred, inst, succ)) = lb.in_edge() {
diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs
index 764531d54f..c7e86437fa 100644
--- a/cranelift/codegen/src/machinst/mod.rs
+++ b/cranelift/codegen/src/machinst/mod.rs
@@ -52,45 +52,9 @@
 //!         |                        - all symbolic stack references to
 //!         |                          stackslots and spillslots are resolved
 //!         |                          to concrete FP-offset mem addresses.)
-//!         | [block/insn ordering]
 //!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - vcode.final_block_order is filled in.
-//!         |                        - new insn sequence from regalloc is
-//!         |                          placed back into vcode and block
-//!         |                          boundaries are updated.)
-//!         | [redundant branch/block
-//!         |  removal]
-//!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - all blocks that were just an
-//!         |                          unconditional branch are removed.)
-//!         |
-//!         | [branch finalization
-//!         |  (fallthroughs)]
-//!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - all branches are in lowered one-
-//!         |                          target form, but targets are still
-//!         |                          block indices.)
-//!         |
-//!         | [branch finalization
-//!         |  (offsets)]
-//!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - all branch offsets from start of
-//!         |                          function are known, and all branches
-//!         |                          have resolved-offset targets.)
-//!         |
-//!         | [MemArg finalization]
-//!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - all MemArg references to the constant
-//!         |                          pool are replaced with offsets.
-//!         |                        - all constant-pool data is collected
-//!         |                          in the VCode.)
-//!         |
-//!         | [binary emission]
+//!         | [binary emission via MachBuffer
+//!         |  with streaming branch resolution/simplification]
 //!         |
 //!     Vec<u8>                     (machine code!)
 //!
@@ -98,11 +62,11 @@
 
 use crate::binemit::{CodeInfo, CodeOffset, StackMap};
 use crate::ir::condcodes::IntCC;
-use crate::ir::{Function, SourceLoc, Type};
+use crate::ir::{Function, SourceLoc, Type, ValueLabel};
 use crate::isa::unwind::input as unwind_input;
 use crate::result::CodegenResult;
 use crate::settings::Flags;
-
+use crate::value_label::ValueLabelsRanges;
 use alloc::boxed::Box;
 use alloc::vec::Vec;
 use core::fmt::Debug;
@@ -111,10 +75,14 @@ use regalloc::RegUsageCollector;
 use regalloc::{
     RealReg, RealRegUniverse, Reg, RegClass, RegUsageMapper, SpillSlot, VirtualReg, Writable,
 };
-use smallvec::SmallVec;
+use smallvec::{smallvec, SmallVec};
+use std::hash::Hasher;
 use std::string::String;
 use target_lexicon::Triple;
 
+#[cfg(feature = "unwind")]
+use crate::isa::unwind::systemv::RegisterMappingError;
+
 pub mod lower;
 pub use lower::*;
 pub mod vcode;
@@ -137,6 +105,7 @@ pub mod inst_common;
 pub use inst_common::*;
 pub mod valueregs;
 pub use valueregs::*;
+pub mod debug;
 
 /// A machine instruction.
 pub trait MachInst: Clone + Debug {
@@ -163,6 +132,11 @@ pub trait MachInst: Clone + Debug {
         true
     }
 
+    /// If this is a load or store to the stack, return that info.
+    fn stack_op_info(&self) -> Option<MachInstStackOpInfo> {
+        None
+    }
+
     /// Generate a move.
     fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self;
 
@@ -174,9 +148,6 @@ pub trait MachInst: Clone + Debug {
         alloc_tmp: F,
     ) -> SmallVec<[Self; 4]>;
 
-    /// Generate a zero-length no-op.
-    fn gen_zero_len_nop() -> Self;
-
     /// Possibly operate on a value directly in a spill-slot rather than a
     /// register. Useful if the machine has register-memory instruction forms
     /// (e.g., add directly from or directly to memory), like x86.
@@ -204,7 +175,7 @@ pub trait MachInst: Clone + Debug {
     /// request a NOP of that size, or as close to it as possible. The machine
     /// backend may return a NOP whose binary encoding is smaller than the
     /// preferred size, but must not return a NOP that is larger. However,
-    /// the instruction must have a nonzero size.
+    /// the instruction must have a nonzero size if preferred_size is nonzero.
     fn gen_nop(preferred_size: usize) -> Self;
 
     /// Get the register universe for this backend.
@@ -223,6 +194,17 @@ pub trait MachInst: Clone + Debug {
     /// be dependent on compilation flags.
     fn ref_type_regclass(_flags: &Flags) -> RegClass;
 
+    /// Does this instruction define a ValueLabel? Returns the `Reg` whose value
+    /// becomes the new value of the `ValueLabel` after this instruction.
+    fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> {
+        None
+    }
+
+    /// Create a marker instruction that defines a value label.
+    fn gen_value_label_marker(_label: ValueLabel, _reg: Reg) -> Self {
+        Self::gen_nop(0)
+    }
+
     /// A label-use kind: a type that describes the types of label references that
     /// can occur in an instruction.
     type LabelUse: MachInstLabelUse;
@@ -285,6 +267,35 @@ pub enum MachTerminator<'a> {
     Indirect(&'a [MachLabel]),
 }
 
+impl<'a> MachTerminator<'a> {
+    /// Get the successor labels named in a `MachTerminator`.
+    pub fn get_succs(&self) -> SmallVec<[MachLabel; 2]> {
+        let mut ret = smallvec![];
+        match self {
+            &MachTerminator::Uncond(l) => {
+                ret.push(l);
+            }
+            &MachTerminator::Cond(l1, l2) => {
+                ret.push(l1);
+                ret.push(l2);
+            }
+            &MachTerminator::Indirect(ls) => {
+                ret.extend(ls.iter().cloned());
+            }
+            _ => {}
+        }
+        ret
+    }
+
+    /// Is this a terminator?
+    pub fn is_term(&self) -> bool {
+        match self {
+            MachTerminator::None => false,
+            _ => true,
+        }
+    }
+}
+
 /// A trait describing the ability to encode a MachInst into binary machine code.
 pub trait MachInstEmit: MachInst {
     /// Persistent state carried across `emit` invocations.
@@ -330,6 +341,8 @@ pub struct MachCompileResult {
     pub disasm: Option<String>,
     /// Unwind info.
     pub unwind_info: Option<unwind_input::UnwindInfo<Reg>>,
+    /// Debug info: value labels to registers/stackslots at code offsets.
+    pub value_labels_ranges: Option<ValueLabelsRanges>,
 }
 
 impl MachCompileResult {
@@ -358,6 +371,10 @@ pub trait MachBackend {
     /// Return flags for this backend.
     fn flags(&self) -> &Flags;
 
+    /// Hashes all flags, both ISA-independent and ISA-specific, into the
+    /// specified hasher.
+    fn hash_all_flags(&self, hasher: &mut dyn Hasher);
+
     /// Return triple for this backend.
     fn triple(&self) -> Triple;
 
@@ -386,13 +403,17 @@ pub trait MachBackend {
         Ok(None)
     }
 
-    /// Machine-specific condcode info needed by TargetIsa.
     /// Creates a new System V Common Information Entry for the ISA.
     #[cfg(feature = "unwind")]
     fn create_systemv_cie(&self) -> Option<gimli::write::CommonInformationEntry> {
         // By default, an ISA cannot create a System V CIE
         None
     }
+    /// Maps a regalloc::Reg to a DWARF register number.
+    #[cfg(feature = "unwind")]
+    fn map_reg_to_dwarf(&self, _: Reg) -> Result<u16, RegisterMappingError> {
+        Err(RegisterMappingError::UnsupportedArchitecture)
+    }
 }
 
 /// Expected unwind info type.
@@ -431,3 +452,15 @@ pub trait UnwindInfoGenerator<I: MachInstEmit> {
         context: UnwindInfoContext<I>,
     ) -> CodegenResult<Option<unwind_input::UnwindInfo<Reg>>>;
 }
+
+/// Info about an operation that loads or stores from/to the stack.
+#[derive(Clone, Copy, Debug)]
+pub enum MachInstStackOpInfo {
+    /// Load from an offset from the nominal stack pointer into the given reg.
+    LoadNomSPOff(Reg, i64),
+    /// Store to an offset from the nominal stack pointer from the given reg.
+    StoreNomSPOff(Reg, i64),
+    /// Adjustment of nominal-SP up or down. This value is added to subsequent
+    /// offsets in loads/stores above to produce real-SP offsets.
+    NomSPAdj(i64),
+}
diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs
index c57f018e35..ab1cca0f47 100644
--- a/cranelift/codegen/src/machinst/vcode.rs
+++ b/cranelift/codegen/src/machinst/vcode.rs
@@ -21,7 +21,6 @@ use crate::ir::{self, types, Constant, ConstantData, SourceLoc};
 use crate::machinst::*;
 use crate::settings;
 use crate::timing;
-
 use regalloc::Function as RegallocFunction;
 use regalloc::Set as RegallocSet;
 use regalloc::{
@@ -110,11 +109,19 @@ pub struct VCode<I: VCodeInst> {
     /// Ranges for prologue and epilogue instructions.
     prologue_epilogue_ranges: Option<(InsnRange, Box<[InsnRange]>)>,
 
-    /// Instruction end offsets
-    insts_layout: RefCell<(Vec<u32>, u32)>,
+    /// Do we generate debug info?
+    generate_debug_info: bool,
+
+    /// Instruction end offsets, instruction indices at each label, and total
+    /// buffer size.  Only present if `generate_debug_info` is set.
+    insts_layout: RefCell<(Vec<u32>, Vec<u32>, u32)>,
 
     /// Constants.
     constants: VCodeConstants,
+
+    /// Are any debug value-labels present? If not, we can skip the
+    /// post-emission analysis.
+    has_value_labels: bool,
 }
 
 /// A builder for a VCode function body. This builder is designed for the
@@ -157,7 +164,13 @@ impl<I: VCodeInst> VCodeBuilder<I> {
         constants: VCodeConstants,
     ) -> VCodeBuilder<I> {
         let reftype_class = I::ref_type_regclass(abi.flags());
-        let vcode = VCode::new(abi, emit_info, block_order, constants);
+        let vcode = VCode::new(
+            abi,
+            emit_info,
+            block_order,
+            constants,
+            /* generate_debug_info = */ true,
+        );
         let stack_map_info = StackmapRequestInfo {
             reftype_class,
             reftyped_vregs: vec![],
@@ -242,6 +255,9 @@ impl<I: VCodeInst> VCodeBuilder<I> {
                 }
             }
         }
+        if insn.defines_value_label().is_some() {
+            self.vcode.has_value_labels = true;
+        }
         self.vcode.insts.push(insn);
         self.vcode.srclocs.push(self.cur_srcloc);
         if is_safepoint {
@@ -296,6 +312,7 @@ impl<I: VCodeInst> VCode<I> {
         emit_info: I::Info,
         block_order: BlockLoweringOrder,
         constants: VCodeConstants,
+        generate_debug_info: bool,
     ) -> VCode<I> {
         VCode {
             liveins: abi.liveins(),
@@ -314,8 +331,10 @@ impl<I: VCodeInst> VCode<I> {
             safepoint_insns: vec![],
             safepoint_slots: vec![],
             prologue_epilogue_ranges: None,
-            insts_layout: RefCell::new((vec![], 0)),
+            generate_debug_info,
+            insts_layout: RefCell::new((vec![], vec![], 0)),
             constants,
+            has_value_labels: false,
         }
     }
 
@@ -484,7 +503,8 @@ impl<I: VCodeInst> VCode<I> {
         buffer.reserve_labels_for_blocks(self.num_blocks() as BlockIndex);
         buffer.reserve_labels_for_constants(&self.constants);
 
-        let mut insts_layout = vec![0; self.insts.len()];
+        let mut inst_ends = vec![0; self.insts.len()];
+        let mut label_insn_iix = vec![0; self.num_blocks()];
 
         let mut safepoint_idx = 0;
         let mut cur_srcloc = None;
@@ -500,6 +520,7 @@ impl<I: VCodeInst> VCode<I> {
 
             let (start, end) = self.block_ranges[block as usize];
             buffer.bind_label(MachLabel::from_block(block));
+            label_insn_iix[block as usize] = start;
             for iix in start..end {
                 let srcloc = self.srclocs[iix as usize];
                 if cur_srcloc != Some(srcloc) {
@@ -526,7 +547,19 @@ impl<I: VCodeInst> VCode<I> {
 
                 self.insts[iix as usize].emit(&mut buffer, &self.emit_info, &mut state);
 
-                insts_layout[iix as usize] = buffer.cur_offset();
+                if self.generate_debug_info {
+                    // Buffer truncation may have happened since last inst append; trim inst-end
+                    // layout info as appropriate.
+                    let l = &mut inst_ends[0..iix as usize];
+                    for end in l.iter_mut().rev() {
+                        if *end > buffer.cur_offset() {
+                            *end = buffer.cur_offset();
+                        } else {
+                            break;
+                        }
+                    }
+                    inst_ends[iix as usize] = buffer.cur_offset();
+                }
             }
 
             if cur_srcloc.is_some() {
@@ -553,7 +586,16 @@ impl<I: VCodeInst> VCode<I> {
             buffer.defer_constant(label, data.alignment(), data.as_slice(), u32::max_value());
         }
 
-        *self.insts_layout.borrow_mut() = (insts_layout, buffer.cur_offset());
+        if self.generate_debug_info {
+            for end in inst_ends.iter_mut().rev() {
+                if *end > buffer.cur_offset() {
+                    *end = buffer.cur_offset();
+                } else {
+                    break;
+                }
+            }
+            *self.insts_layout.borrow_mut() = (inst_ends, label_insn_iix, buffer.cur_offset());
+        }
 
         buffer
     }
@@ -567,13 +609,27 @@ impl<I: VCodeInst> VCode<I> {
         let context = UnwindInfoContext {
             insts: &self.insts,
             insts_layout: &layout.0,
-            len: layout.1,
+            len: layout.2,
             prologue: prologue.clone(),
             epilogues,
         };
         I::UnwindInfo::create_unwind_info(context)
     }
 
+    /// Generates value-label ranges.
+    pub fn value_labels_ranges(&self) -> crate::result::CodegenResult<Option<ValueLabelsRanges>> {
+        if !self.has_value_labels {
+            return Ok(None);
+        }
+
+        let layout = &self.insts_layout.borrow();
+        Ok(Some(debug::compute(
+            &self.insts,
+            &layout.0[..],
+            &layout.1[..],
+        )))
+    }
+
     /// Get the IR block for a BlockIndex, if one exists.
     pub fn bindex_to_bb(&self, block: BlockIndex) -> Option<ir::Block> {
         self.block_order.lowered_order()[block as usize].orig_block()
@@ -670,7 +726,7 @@ impl<I: VCodeInst> RegallocFunction for VCode<I> {
     }
 
     fn gen_zero_len_nop(&self) -> I {
-        I::gen_zero_len_nop()
+        I::gen_nop(0)
     }
 
     fn maybe_direct_reload(&self, insn: &I, reg: VirtualReg, slot: SpillSlot) -> Option<I> {
diff --git a/cranelift/codegen/src/settings.rs b/cranelift/codegen/src/settings.rs
index 6f25b134af..a1bc954c54 100644
--- a/cranelift/codegen/src/settings.rs
+++ b/cranelift/codegen/src/settings.rs
@@ -188,7 +188,7 @@ pub type SetResult<T> = Result<T, SetError>;
 /// The settings objects themselves are generated and appear in the `isa/*/settings.rs` modules.
 /// Each settings object provides a `predicate_view()` method that makes it possible to query
 /// ISA predicates by number.
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Hash)]
 pub struct PredicateView<'a>(&'a [u8]);
 
 impl<'a> PredicateView<'a> {
diff --git a/cranelift/codegen/src/value_label.rs b/cranelift/codegen/src/value_label.rs
index e3daeb0f7a..3d3ca2ea99 100644
--- a/cranelift/codegen/src/value_label.rs
+++ b/cranelift/codegen/src/value_label.rs
@@ -1,13 +1,16 @@
 use crate::ir::{Function, SourceLoc, Value, ValueLabel, ValueLabelAssignments, ValueLoc};
 use crate::isa::TargetIsa;
+use crate::machinst::MachCompileResult;
 use crate::regalloc::{Context, RegDiversions};
 use crate::HashMap;
 use alloc::collections::BTreeMap;
 use alloc::vec::Vec;
 use core::cmp::Ordering;
+use core::convert::From;
 use core::iter::Iterator;
 use core::ops::Bound::*;
 use core::ops::Deref;
+use regalloc::Reg;
 
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
@@ -17,13 +20,31 @@ use serde::{Deserialize, Serialize};
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct ValueLocRange {
     /// The ValueLoc containing a ValueLabel during this range.
-    pub loc: ValueLoc,
+    pub loc: LabelValueLoc,
     /// The start of the range. It is an offset in the generated code.
     pub start: u32,
     /// The end of the range. It is an offset in the generated code.
     pub end: u32,
 }
 
+/// The particular location for a value.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub enum LabelValueLoc {
+    /// Old-backend location: RegUnit, StackSlot, or Unassigned.
+    ValueLoc(ValueLoc),
+    /// New-backend Reg.
+    Reg(Reg),
+    /// New-backend offset from stack pointer.
+    SPOffset(i64),
+}
+
+impl From<ValueLoc> for LabelValueLoc {
+    fn from(v: ValueLoc) -> Self {
+        LabelValueLoc::ValueLoc(v)
+    }
+}
+
 /// Resulting map of Value labels and their ranges/locations.
 pub type ValueLabelsRanges = HashMap<ValueLabel, Vec<ValueLocRange>>;
 
@@ -86,14 +107,18 @@ where
 pub fn build_value_labels_ranges<T>(
     func: &Function,
     regalloc: &Context,
+    mach_compile_result: Option<&MachCompileResult>,
     isa: &dyn TargetIsa,
 ) -> ValueLabelsRanges
 where
     T: From<SourceLoc> + Deref<Target = SourceLoc> + Ord + Copy,
 {
-    // FIXME(#1523): New-style backend does not yet have debug info.
-    if isa.get_mach_backend().is_some() {
-        return HashMap::new();
+    if mach_compile_result.is_some() && mach_compile_result.unwrap().value_labels_ranges.is_some() {
+        return mach_compile_result
+            .unwrap()
+            .value_labels_ranges
+            .clone()
+            .unwrap();
     }
 
     let values_labels = build_value_labels_index::<T>(func);
@@ -113,7 +138,7 @@ where
             .entry(label)
             .or_insert_with(Vec::new)
             .push(ValueLocRange {
-                loc,
+                loc: loc.into(),
                 start: range.0,
                 end: range.1,
             });
diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs
index 8d73e2d1e4..d7528beef4 100644
--- a/cranelift/codegen/src/write.rs
+++ b/cranelift/codegen/src/write.rs
@@ -11,7 +11,7 @@ use crate::ir::{
 };
 use crate::isa::{RegInfo, TargetIsa};
 use crate::packed_option::ReservedValue;
-use crate::value_label::ValueLabelsRanges;
+use crate::value_label::{LabelValueLoc, ValueLabelsRanges};
 use crate::HashSet;
 use alloc::string::String;
 use alloc::vec::Vec;
@@ -278,11 +278,13 @@ pub fn write_block_header(
     writeln!(w, "):")
 }
 
-fn write_valueloc(w: &mut dyn Write, loc: ValueLoc, regs: &RegInfo) -> fmt::Result {
+fn write_valueloc(w: &mut dyn Write, loc: LabelValueLoc, regs: &RegInfo) -> fmt::Result {
     match loc {
-        ValueLoc::Reg(r) => write!(w, "{}", regs.display_regunit(r)),
-        ValueLoc::Stack(ss) => write!(w, "{}", ss),
-        ValueLoc::Unassigned => write!(w, "?"),
+        LabelValueLoc::ValueLoc(ValueLoc::Reg(r)) => write!(w, "{}", regs.display_regunit(r)),
+        LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => write!(w, "{}", ss),
+        LabelValueLoc::ValueLoc(ValueLoc::Unassigned) => write!(w, "?"),
+        LabelValueLoc::Reg(r) => write!(w, "{:?}", r),
+        LabelValueLoc::SPOffset(off) => write!(w, "[sp+{}]", off),
     }
 }
 
diff --git a/cranelift/entity/src/list.rs b/cranelift/entity/src/list.rs
index 68cab8166b..dfb8fac646 100644
--- a/cranelift/entity/src/list.rs
+++ b/cranelift/entity/src/list.rs
@@ -91,17 +91,20 @@ type SizeClass = u8;
 
 /// Get the size of a given size class. The size includes the length field, so the maximum list
 /// length is one less than the class size.
+#[inline]
 fn sclass_size(sclass: SizeClass) -> usize {
     4 << sclass
 }
 
 /// Get the size class to use for a given list length.
 /// This always leaves room for the length element in addition to the list elements.
+#[inline]
 fn sclass_for_length(len: usize) -> SizeClass {
     30 - (len as u32 | 3).leading_zeros() as SizeClass
 }
 
 /// Is `len` the minimum length in its size class?
+#[inline]
 fn is_sclass_min_length(len: usize) -> bool {
     len > 3 && len.is_power_of_two()
 }
@@ -387,14 +390,34 @@ impl<T: EntityRef + ReservedValue> EntityList<T> {
         &mut pool.data[block + 1..block + 1 + new_len]
     }
 
+    /// Constructs a list from an iterator.
+    pub fn from_iter<I>(elements: I, pool: &mut ListPool<T>) -> Self
+    where
+        I: IntoIterator<Item = T>,
+    {
+        let mut list = Self::new();
+        list.extend(elements, pool);
+        list
+    }
+
     /// Appends multiple elements to the back of the list.
     pub fn extend<I>(&mut self, elements: I, pool: &mut ListPool<T>)
     where
         I: IntoIterator<Item = T>,
     {
-        // TODO: use `size_hint()` to reduce reallocations.
-        for x in elements {
-            self.push(x, pool);
+        let iterator = elements.into_iter();
+        let (len, upper) = iterator.size_hint();
+        // On most iterators this check is optimized down to `true`.
+        if upper == Some(len) {
+            let data = self.grow(len, pool);
+            let offset = data.len() - len;
+            for (src, dst) in iterator.zip(data[offset..].iter_mut()) {
+                *dst = src;
+            }
+        } else {
+            for x in iterator {
+                self.push(x, pool);
+            }
         }
     }
 
@@ -630,6 +653,10 @@ mod tests {
             list.as_slice(pool),
             &[i1, i2, i3, i4, i1, i1, i2, i2, i3, i3, i4, i4]
         );
+
+        let list2 = EntityList::from_iter([i1, i1, i2, i2, i3, i3, i4, i4].iter().cloned(), pool);
+        assert_eq!(list2.len(pool), 8);
+        assert_eq!(list2.as_slice(pool), &[i1, i1, i2, i2, i3, i3, i4, i4]);
     }
 
     #[test]
diff --git a/cranelift/filetests/filetests/isa/aarch64/bitops.clif b/cranelift/filetests/filetests/isa/aarch64/bitops.clif
index ab1c113104..8730128cc5 100644
--- a/cranelift/filetests/filetests/isa/aarch64/bitops.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/bitops.clif
@@ -230,19 +230,10 @@ block0(v0: i64):
 
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: lsr x1, x0, #1
-; nextln: and x1, x1, #6148914691236517205
-; nextln: sub x1, x0, x1
-; nextln: and x0, x1, #3689348814741910323
-; nextln: lsr x1, x1, #2
-; nextln: and x1, x1, #3689348814741910323
-; nextln: add x0, x1, x0
-; nextln: add x0, x0, x0, LSR 4
-; nextln: and x0, x0, #1085102592571150095
-; nextln: add x0, x0, x0, LSL 8
-; nextln: add x0, x0, x0, LSL 16
-; nextln: add x0, x0, x0, LSL 32
-; nextln: lsr x0, x0, #56
+; nextln: fmov d0, x0
+; nextln: cnt v0.8b, v0.8b
+; nextln: addv b0, v0.8b
+; nextln: umov w0, v0.b[0]
 ; nextln: mov sp, fp
 ; nextln: ldp fp, lr, [sp], #16
 ; nextln: ret
@@ -255,20 +246,10 @@ block0(v0: i32):
 
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: mov w0, w0
-; nextln: lsr w1, w0, #1
-; nextln: and x1, x1, #6148914691236517205
-; nextln: sub x1, x0, x1
-; nextln: and x0, x1, #3689348814741910323
-; nextln: lsr x1, x1, #2
-; nextln: and x1, x1, #3689348814741910323
-; nextln: add x0, x1, x0
-; nextln: add x0, x0, x0, LSR 4
-; nextln: and x0, x0, #1085102592571150095
-; nextln: add x0, x0, x0, LSL 8
-; nextln: add x0, x0, x0, LSL 16
-; nextln: add x0, x0, x0, LSL 32
-; nextln: lsr x0, x0, #56
+; nextln: fmov s0, w0
+; nextln: cnt v0.8b, v0.8b
+; nextln: addv b0, v0.8b
+; nextln: umov w0, v0.b[0]
 ; nextln: mov sp, fp
 ; nextln: ldp fp, lr, [sp], #16
 ; nextln: ret
@@ -281,20 +262,10 @@ block0(v0: i16):
 
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: uxth w0, w0
-; nextln: lsr w1, w0, #1
-; nextln: and x1, x1, #6148914691236517205
-; nextln: sub x1, x0, x1
-; nextln: and x0, x1, #3689348814741910323
-; nextln: lsr x1, x1, #2
-; nextln: and x1, x1, #3689348814741910323
-; nextln: add x0, x1, x0
-; nextln: add x0, x0, x0, LSR 4
-; nextln: and x0, x0, #1085102592571150095
-; nextln: add x0, x0, x0, LSL 8
-; nextln: add x0, x0, x0, LSL 16
-; nextln: add x0, x0, x0, LSL 32
-; nextln: lsr x0, x0, #56
+; nextln: fmov s0, w0
+; nextln: cnt v0.8b, v0.8b
+; nextln: addp v0.8b, v0.8b, v0.8b
+; nextln: umov w0, v0.b[0]
 ; nextln: mov sp, fp
 ; nextln: ldp fp, lr, [sp], #16
 ; nextln: ret
@@ -307,20 +278,9 @@ block0(v0: i8):
 
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: uxtb w0, w0
-; nextln: lsr w1, w0, #1
-; nextln: and x1, x1, #6148914691236517205
-; nextln: sub x1, x0, x1
-; nextln: and x0, x1, #3689348814741910323
-; nextln: lsr x1, x1, #2
-; nextln: and x1, x1, #3689348814741910323
-; nextln: add x0, x1, x0
-; nextln: add x0, x0, x0, LSR 4
-; nextln: and x0, x0, #1085102592571150095
-; nextln: add x0, x0, x0, LSL 8
-; nextln: add x0, x0, x0, LSL 16
-; nextln: add x0, x0, x0, LSL 32
-; nextln: lsr x0, x0, #56
+; nextln: fmov s0, w0
+; nextln: cnt v0.8b, v0.8b
+; nextln: umov w0, v0.b[0]
 ; nextln: mov sp, fp
 ; nextln: ldp fp, lr, [sp], #16
 ; nextln: ret
diff --git a/cranelift/filetests/filetests/isa/x64/bitops-i128-run.clif b/cranelift/filetests/filetests/isa/x64/bitops-i128-run.clif
new file mode 100644
index 0000000000..5795900438
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/bitops-i128-run.clif
@@ -0,0 +1,27 @@
+test run
+target x86_64
+feature "experimental_x64"
+
+function %ctz(i64, i64) -> i8 {
+block0(v0: i64, v1: i64):
+    v2 = iconcat v0, v1
+    v3 = ctz.i128 v2
+    v4 = ireduce.i8 v3
+    return v4
+}
+; run: %ctz(0x00000000_00000000, 0x00000001_00000000) == 96
+; run: %ctz(0x00000000_00010000, 0x00000001_00000000) == 16
+; run: %ctz(0x00000000_00010000, 0x00000000_00000000) == 16
+; run: %ctz(0x00000000_00000000, 0x00000000_00000000) == 128
+
+function %clz(i64, i64) -> i8 {
+block0(v0: i64, v1: i64):
+    v2 = iconcat v0, v1
+    v3 = clz.i128 v2
+    v4 = ireduce.i8 v3
+    return v4
+}
+; run: %clz(0x00000000_00000000, 0x00000001_00000000) == 31
+; run: %clz(0x00000000_00010000, 0x00000001_00000000) == 31
+; run: %clz(0x00000000_00010000, 0x00000000_00000000) == 111
+; run: %clz(0x00000000_00000000, 0x00000000_00000000) == 128
diff --git a/cranelift/filetests/filetests/isa/x64/bitrev-i128-run.clif b/cranelift/filetests/filetests/isa/x64/bitrev-i128-run.clif
new file mode 100644
index 0000000000..64ea96716c
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/bitrev-i128-run.clif
@@ -0,0 +1,47 @@
+test run
+target x86_64
+feature "experimental_x64"
+
+function %reverse_bits_zero() -> b1 {
+block0:
+    v0 = iconst.i64 0
+    v1 = iconcat v0, v0
+    v2 = bitrev.i128 v1
+    v3 = icmp eq v2, v1
+    return v3
+}
+; run
+
+function %reverse_bits_one() -> b1 {
+block0:
+    v0 = iconst.i64 0
+    v1 = iconst.i64 1
+    v2 = iconcat v0, v1
+
+    v3 = bitrev.i128 v2
+
+    v4 = iconst.i64 0x8000_0000_0000_0000
+    v5 = iconst.i64 0
+    v6 = iconcat v4, v5
+
+    v7 = icmp eq v3, v6
+    return v7
+}
+; run
+
+function %reverse_bits() -> b1 {
+block0:
+    v0 = iconst.i64 0x06AD_8667_69EC_41BA
+    v1 = iconst.i64 0x6C83_D81A_6E28_83AB
+    v2 = iconcat v0, v1
+
+    v3 = bitrev.i128 v2
+
+    v4 = iconst.i64 0xD5C11476581BC136
+    v5 = iconst.i64 0x5D823796E661B560
+    v6 = iconcat v4, v5
+
+    v7 = icmp eq v3, v6
+    return v7
+}
+; run
diff --git a/cranelift/filetests/filetests/isa/x64/clz-lzcnt.clif b/cranelift/filetests/filetests/isa/x64/clz-lzcnt.clif
new file mode 100644
index 0000000000..ac0df03384
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/clz-lzcnt.clif
@@ -0,0 +1,31 @@
+test compile
+target x86_64 has_lzcnt
+feature "experimental_x64"
+
+function %clz(i64) -> i64 {
+block0(v0: i64):
+    v1 = clz v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: lzcntq  %rdi, %rsi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret
+
+function %clz(i32) -> i32 {
+block0(v0: i32):
+    v1 = clz v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: lzcntl  %edi, %esi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret
diff --git a/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif b/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif
new file mode 100644
index 0000000000..9d05e04b04
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif
@@ -0,0 +1,49 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function %f0(i64, i64) -> i64, i64 {
+block0(v0: i64, v1: i64):
+    v2 = load.i64 v1
+; check:  movq    0(%rsi), %rax
+
+    v3 = icmp eq v0, v2
+
+    v4 = bint.i64 v3
+; nextln: cmpq    %rax, %rdi
+; nextln: setz    %cl
+; nextln: movzbq  %cl, %rcx
+
+    v5 = select.i64 v3, v0, v1
+; nextln: cmpq    %rax, %rdi
+; nextln: cmovzq  %rdi, %rsi
+
+    return v4, v5
+; nextln: movq    %rcx, %rax
+; nextln: movq    %rsi, %rdx
+}
+
+function %f1(f64, i64) -> i64, f64 {
+block0(v0: f64, v1: i64):
+    v2 = load.f64 v1
+; check:  movsd   0(%rdi), %xmm1
+
+    v3 = fcmp eq v0, v2
+
+    v4 = bint.i64 v3
+; nextln: ucomisd %xmm1, %xmm0
+; nextln: setnp   %dil
+; nextln: setz    %sil
+; nextln: andl    %edi, %esi
+; nextln: movzbq  %sil, %rsi
+
+    v5 = select.f64 v3, v0, v0
+; nextln: ucomisd %xmm1, %xmm0
+; nextln: movaps  %xmm0, %xmm1
+; nextln: jnp $$next; movsd %xmm0, %xmm1; $$next:
+; nextln: jz $$next; movsd %xmm0, %xmm1; $$next:
+
+    return v4, v5
+; nextln: movq    %rsi, %rax
+; nextln: movaps  %xmm1, %xmm0
+}
diff --git a/cranelift/filetests/filetests/isa/x64/ctz-bmi1.clif b/cranelift/filetests/filetests/isa/x64/ctz-bmi1.clif
new file mode 100644
index 0000000000..b50b10107a
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/ctz-bmi1.clif
@@ -0,0 +1,31 @@
+test compile
+target x86_64 has_bmi1
+feature "experimental_x64"
+
+function %ctz(i64) -> i64 {
+block0(v0: i64):
+    v1 = ctz v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: tzcntq  %rdi, %rsi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret
+
+function %ctz(i32) -> i32 {
+block0(v0: i32):
+    v1 = ctz v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: tzcntl  %edi, %esi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret
diff --git a/cranelift/filetests/filetests/isa/x64/floating-point.clif b/cranelift/filetests/filetests/isa/x64/floating-point.clif
new file mode 100644
index 0000000000..b3b5907210
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/floating-point.clif
@@ -0,0 +1,26 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function %f(f64) -> f64 {
+block0(v0: f64):
+    v1 = fabs.f64 v0
+    return v1
+}
+; check:  movabsq $$9223372036854775807, %rsi
+; nextln: movq    %rsi, %xmm1
+; nextln: andpd   %xmm0, %xmm1
+; nextln: movaps  %xmm1, %xmm0
+
+
+function %f(i64) -> f64 {
+block0(v0: i64):
+    v1 = load.f64 v0
+    v2 = fabs.f64 v1
+    return v2
+}
+; check:  movsd   0(%rdi), %xmm0
+; nextln: movabsq $$9223372036854775807, %rsi
+; nextln: movq    %rsi, %xmm1
+; nextln: andpd   %xmm0, %xmm1
+; nextln: movaps  %xmm1, %xmm0
diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif
new file mode 100644
index 0000000000..5e5d2ffb86
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/i128.clif
@@ -0,0 +1,1082 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function %f0(i128, i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i128):
+
+    v2 = iadd v0, v1
+; nextln:  addq    %rdx, %rdi
+; nextln:  adcq    %rcx, %rsi
+
+    return v2
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f1(i128, i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i128):
+
+    v2 = isub v0, v1
+; nextln:  subq    %rdx, %rdi
+; nextln:  sbbq    %rcx, %rsi
+
+    return v2
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f2(i128, i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i128):
+
+    v2 = band v0, v1
+; nextln:  andq    %rdx, %rdi
+; nextln:  andq    %rcx, %rsi
+
+    return v2
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f3(i128, i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i128):
+
+    v2 = bor v0, v1
+; nextln:  orq     %rdx, %rdi
+; nextln:  orq     %rcx, %rsi
+
+    return v2
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f4(i128, i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i128):
+
+    v2 = bxor v0, v1
+; nextln:  xorq    %rdx, %rdi
+; nextln:  xorq    %rcx, %rsi
+
+    return v2
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f5(i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128):
+
+    v1 = bnot v0
+; nextln:  notq    %rdi
+; nextln:  notq    %rsi
+
+    return v1
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f6(i128, i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i128):
+; v0 in rdi:rsi, v1 in rdx:rcx
+
+    v2 = imul v0, v1
+; nextln:  movq    %rsi, %rax
+; nextln:  movq    %rcx, %r8
+; nextln:  movq    %rdi, %rsi
+; nextln:  imulq   %rdx, %rsi
+; nextln:  movq    %rdi, %rcx
+; nextln:  imulq   %r8, %rcx
+; nextln:  imulq   %rdx, %rax
+; nextln:  addq    %rax, %rcx
+; nextln:  movq    %rdi, %rax
+; nextln:  mul     %rdx
+; nextln:  addq    %rdx, %rcx
+; nextln:  movq    %rsi, %rax
+; nextln:  movq    %rcx, %rdx
+
+    return v2
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f7(i64, i64) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i64, v1: i64):
+    v2 = iconcat.i64 v0, v1
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+
+    return v2
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f8(i128) -> i64, i64 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128):
+    v1, v2 = isplit.i128 v0
+; nextln:  movq    %rdi, %rax
+; nextln:  movq    %rsi, %rdx
+
+    return v1, v2
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f9(i128, i128) -> b1 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i128):
+    v2 = icmp eq v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setz    %al
+; nextln: cmpq    %rdx, %rdi
+; nextln: setz    %r8b
+; nextln: andq    %rax, %r8
+; nextln: andq    $$1, %r8
+; nextln: setnz   %al
+
+    v3 = icmp ne v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setnz   %al
+; nextln: cmpq    %rdx, %rdi
+; nextln: setnz   %r8b
+; nextln: orq     %rax, %r8
+; nextln: andq    $$1, %r8
+; nextln: setnz   %r8b
+ 
+    v4 = icmp slt v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setl    %r9b
+; nextln: setz    %al
+; nextln: cmpq    %rdx, %rdi
+; nextln: setb    %r10b
+; nextln: andq    %rax, %r10
+; nextln: orq     %r9, %r10
+; nextln: andq    $$1, %r10
+; nextln: setnz   %r9b
+ 
+    v5 = icmp sle v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setl    %r10b
+; nextln: setz    %al
+; nextln: cmpq    %rdx, %rdi
+; nextln: setbe   %r11b
+; nextln: andq    %rax, %r11
+; nextln: orq     %r10, %r11
+; nextln: andq    $$1, %r11
+; nextln: setnz   %r10b
+ 
+    v6 = icmp sgt v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setnle  %r11b
+; nextln: setz    %al
+; nextln: cmpq    %rdx, %rdi
+; nextln: setnbe  %r12b
+; nextln: andq    %rax, %r12
+; nextln: orq     %r11, %r12
+; nextln: andq    $$1, %r12
+; nextln: setnz   %r11b
+
+	v7 = icmp sge v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setnle  %r12b
+; nextln: setz    %al
+; nextln: cmpq    %rdx, %rdi
+; nextln: setnb   %r13b
+; nextln: andq    %rax, %r13
+; nextln: orq     %r12, %r13
+; nextln: andq    $$1, %r13
+; nextln: setnz   %r12b
+
+    v8 = icmp ult v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setb    %r13b
+; nextln: setz    %al
+; nextln: cmpq    %rdx, %rdi
+; nextln: setb    %r14b
+; nextln: andq    %rax, %r14
+; nextln: orq     %r13, %r14
+; nextln: andq    $$1, %r14
+; nextln: setnz   %r13b
+
+    v9 = icmp ule v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setb    %r14b
+; nextln: setz    %al
+; nextln: cmpq    %rdx, %rdi
+; nextln: setbe   %bl
+; nextln: andq    %rax, %rbx
+; nextln: orq     %r14, %rbx
+; nextln: andq    $$1, %rbx
+; nextln: setnz   %r14b
+
+    v10 = icmp ugt v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setnbe  %bl
+; nextln: setz    %r15b
+; nextln: cmpq    %rdx, %rdi
+; nextln: setnbe  %al
+; nextln: andq    %r15, %rax
+; nextln: orq     %rbx, %rax
+; nextln: andq    $$1, %rax
+; nextln: setnz   %bl
+
+    v11 = icmp uge v0, v1
+; check:  cmpq    %rcx, %rsi
+; nextln: setnbe  %sil
+; nextln: setz    %cl
+; nextln: cmpq    %rdx, %rdi
+; nextln: setnb   %dil
+; nextln: andq    %rcx, %rdi
+; nextln: orq     %rsi, %rdi
+; nextln: andq    $$1, %rdi
+; nextln: setnz   %sil
+
+    v12 = band v2, v3
+    v13 = band v4, v5
+    v14 = band v6, v7
+    v15 = band v8, v9
+    v16 = band v10, v11
+    v17 = band v12, v13
+    v18 = band v14, v15
+    v19 = band v17, v18
+    v20 = band v19, v16
+
+    return v20
+; check:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f10(i128) -> i32 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128):
+    brz v0, block1
+; check:  cmpq    $$0, %rdi
+; nextln: setz    %dil
+; nextln: cmpq    $$0, %rsi
+; nextln: setz    %sil
+; nextln: andb    %dil, %sil
+; nextln: jnz     label1; j label2
+ 
+    jump block2
+
+block1:
+    v1 = iconst.i32 1
+    return v1
+
+block2:
+    v2 = iconst.i32 2
+    return v2
+
+; check:   movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f11(i128) -> i32 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128):
+    brnz v0, block1
+; check:  cmpq    $$0, %rdi
+; nextln: setnz   %dil
+; nextln: cmpq    $$0, %rsi
+; nextln: setnz   %sil
+; nextln: orb     %dil, %sil
+; nextln: jnz     label1; j label2
+    jump block2
+
+block1:
+    v1 = iconst.i32 1
+    return v1
+
+block2:
+    v2 = iconst.i32 2
+    return v2
+
+; check:   movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f12(i64) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i64):
+    v1 = uextend.i128 v0
+    return v1
+
+; nextln:  movq    %rdi, %rsi
+; nextln:  xorq    %rdi, %rdi
+; nextln:  movq    %rsi, %rax
+; nextln:  movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f13(i64) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i64):
+    v1 = sextend.i128 v0
+    return v1
+
+; nextln:  movq    %rdi, %rsi
+; nextln:  movq    %rsi, %rdi
+; nextln:  sarq    $$63, %rdi
+; nextln:  movq    %rsi, %rax
+; nextln:  movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f14(i8) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i8):
+    v1 = sextend.i128 v0
+    return v1
+
+; nextln:  movsbq  %dil, %rsi
+; nextln:  movq    %rsi, %rdi
+; nextln:  sarq    $$63, %rdi
+; nextln:  movq    %rsi, %rax
+; nextln:  movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f15(i8) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i8):
+    v1 = uextend.i128 v0
+    return v1
+
+; nextln:  movzbq  %dil, %rsi
+; nextln:  xorq    %rdi, %rdi
+; nextln:  movq    %rsi, %rax
+; nextln:  movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+
+}
+
+function %f16(i128) -> i64 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128):
+    v1 = ireduce.i64 v0
+    return v1
+
+; nextln:  movq    %rdi, %rax
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f17(i128) -> i8 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128):
+    v1 = ireduce.i8 v0
+    return v1
+
+; nextln:  movq    %rdi, %rax
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f18(b1) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: b1):
+    v1 = bint.i128 v0
+    return v1
+
+; check:  movzbq  %dil, %rsi
+; nextln: xorq    %rdi, %rdi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f19(i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128):
+    v1 = popcnt.i128 v0
+    return v1
+
+; check:  movq    %rsi, %rdx
+; nextln: movq    %rdi, %rsi
+; nextln: shrq    $$1, %rsi
+; nextln: movabsq $$8608480567731124087, %rcx
+; nextln: andq    %rcx, %rsi
+; nextln: movq    %rdi, %rax
+; nextln: subq    %rsi, %rax
+; nextln: shrq    $$1, %rsi
+; nextln: andq    %rcx, %rsi
+; nextln: subq    %rsi, %rax
+; nextln: shrq    $$1, %rsi
+; nextln: andq    %rcx, %rsi
+; nextln: subq    %rsi, %rax
+; nextln: movq    %rax, %rsi
+; nextln: shrq    $$4, %rsi
+; nextln: addq    %rax, %rsi
+; nextln: movabsq $$1085102592571150095, %rdi
+; nextln: andq    %rdi, %rsi
+; nextln: movabsq $$72340172838076673, %rdi
+; nextln: imulq   %rdi, %rsi
+; nextln: shrq    $$56, %rsi
+; nextln: movq    %rdx, %rax
+; nextln: shrq    $$1, %rax
+; nextln: movabsq $$8608480567731124087, %rcx
+; nextln: andq    %rcx, %rax
+; nextln: movq    %rdx, %rdi
+; nextln: subq    %rax, %rdi
+; nextln: shrq    $$1, %rax
+; nextln: andq    %rcx, %rax
+; nextln: subq    %rax, %rdi
+; nextln: shrq    $$1, %rax
+; nextln: andq    %rcx, %rax
+; nextln: subq    %rax, %rdi
+; nextln: movq    %rdi, %rax
+; nextln: shrq    $$4, %rax
+; nextln: addq    %rdi, %rax
+; nextln: movabsq $$1085102592571150095, %rdi
+; nextln: andq    %rdi, %rax
+; nextln: movabsq $$72340172838076673, %rdi
+; nextln: imulq   %rdi, %rax
+; nextln: shrq    $$56, %rax
+; nextln: addq    %rax, %rsi
+; nextln: xorq    %rdi, %rdi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rdi, %rdx
+
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+
+function %f20(i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128):
+    v1 = bitrev.i128 v0
+    return v1
+
+; check:  movq    %rdi, %rcx
+; nextln: movq    %rcx, %rdi
+; nextln: movabsq $$6148914691236517205, %rax
+; nextln: shrq    $$1, %rdi
+; nextln: andq    %rax, %rdi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$1, %rax
+; nextln: movq    %rax, %rcx
+; nextln: orq     %rdi, %rcx
+; nextln: movq    %rcx, %rdi
+; nextln: movabsq $$3689348814741910323, %rax
+; nextln: shrq    $$2, %rdi
+; nextln: andq    %rax, %rdi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$2, %rax
+; nextln: movq    %rax, %rcx
+; nextln: orq     %rdi, %rcx
+; nextln: movq    %rcx, %rdi
+; nextln: movabsq $$1085102592571150095, %rax
+; nextln: shrq    $$4, %rdi
+; nextln: andq    %rax, %rdi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$4, %rax
+; nextln: movq    %rax, %rcx
+; nextln: orq     %rdi, %rcx
+; nextln: movq    %rcx, %rdi
+; nextln: movabsq $$71777214294589695, %rax
+; nextln: shrq    $$8, %rdi
+; nextln: andq    %rax, %rdi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$8, %rax
+; nextln: movq    %rax, %rcx
+; nextln: orq     %rdi, %rcx
+; nextln: movq    %rcx, %rdi
+; nextln: movabsq $$281470681808895, %rax
+; nextln: shrq    $$16, %rdi
+; nextln: andq    %rax, %rdi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$16, %rax
+; nextln: orq     %rdi, %rax
+; nextln: movq    %rax, %rcx
+; nextln: movl    $$-1, %edi
+; nextln: shrq    $$32, %rcx
+; nextln: andq    %rdi, %rcx
+; nextln: andq    %rax, %rdi
+; nextln: shlq    $$32, %rdi
+; nextln: orq     %rcx, %rdi
+; nextln: movq    %rsi, %rcx
+; nextln: movq    %rcx, %rsi
+; nextln: movabsq $$6148914691236517205, %rax
+; nextln: shrq    $$1, %rsi
+; nextln: andq    %rax, %rsi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$1, %rax
+; nextln: movq    %rax, %rcx
+; nextln: orq     %rsi, %rcx
+; nextln: movq    %rcx, %rsi
+; nextln: movabsq $$3689348814741910323, %rax
+; nextln: shrq    $$2, %rsi
+; nextln: andq    %rax, %rsi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$2, %rax
+; nextln: movq    %rax, %rcx
+; nextln: orq     %rsi, %rcx
+; nextln: movq    %rcx, %rsi
+; nextln: movabsq $$1085102592571150095, %rax
+; nextln: shrq    $$4, %rsi
+; nextln: andq    %rax, %rsi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$4, %rax
+; nextln: movq    %rax, %rcx
+; nextln: orq     %rsi, %rcx
+; nextln: movq    %rcx, %rsi
+; nextln: movabsq $$71777214294589695, %rax
+; nextln: shrq    $$8, %rsi
+; nextln: andq    %rax, %rsi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$8, %rax
+; nextln: movq    %rax, %rcx
+; nextln: orq     %rsi, %rcx
+; nextln: movq    %rcx, %rsi
+; nextln: movabsq $$281470681808895, %rax
+; nextln: shrq    $$16, %rsi
+; nextln: andq    %rax, %rsi
+; nextln: andq    %rcx, %rax
+; nextln: shlq    $$16, %rax
+; nextln: orq     %rsi, %rax
+; nextln: movq    %rax, %rsi
+; nextln: movl    $$-1, %ecx
+; nextln: shrq    $$32, %rsi
+; nextln: andq    %rcx, %rsi
+; nextln: andq    %rax, %rcx
+; nextln: shlq    $$32, %rcx
+; nextln: orq     %rsi, %rcx
+; nextln: movq    %rcx, %rax
+; nextln: movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f21(i128, i32) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i32):
+    v2 = ushr v0, v1
+    return v2
+
+; check:  movq    %rdi, %rax
+; nextln: movq    %rsi, %rdi
+; nextln: movq    %rdi, %rsi
+; nextln: movq    %rdx, %rcx
+; nextln: shrq    %cl, %rsi
+; nextln: movq    %rdx, %rcx
+; nextln: shrq    %cl, %rax
+; nextln: movl    $$64, %ecx
+; nextln: subq    %rdx, %rcx
+; nextln: shlq    %cl, %rdi
+; nextln: orq     %rax, %rdi
+; nextln: xorq    %rax, %rax
+; nextln: xorq    %rcx, %rcx
+; nextln: andq    $$64, %rdx
+; nextln: cmovzq  %rsi, %rax
+; nextln: cmovzq  %rdi, %rcx
+; nextln: cmovnzq %rsi, %rcx
+; nextln: movq    %rax, %rdx
+; nextln: movq    %rcx, %rax
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f22(i128, i32) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i32):
+    v2 = ishl v0, v1
+    return v2
+
+; check:  movq    %rsi, %rax
+; nextln: movq    %rdi, %rsi
+; nextln: movq    %rdx, %rcx
+; nextln: shlq    %cl, %rsi
+; nextln: movq    %rdx, %rcx
+; nextln: shlq    %cl, %rax
+; nextln: movl    $$64, %ecx
+; nextln: subq    %rdx, %rcx
+; nextln: shrq    %cl, %rdi
+; nextln: orq     %rax, %rdi
+; nextln: xorq    %rax, %rax
+; nextln: xorq    %rcx, %rcx
+; nextln: andq    $$64, %rdx
+; nextln: cmovzq  %rdi, %rcx
+; nextln: cmovzq  %rsi, %rax
+; nextln: cmovnzq %rsi, %rcx
+; nextln: movq    %rcx, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f23(i128, i32) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i32):
+    v2 = sshr v0, v1
+    return v2
+
+; check:  movq    %rdi, %r8
+; nextln: movq    %rsi, %rdi
+; nextln: movq    %rdi, %rsi
+; nextln: movq    %rdx, %rcx
+; nextln: sarq    %cl, %rsi
+; nextln: movq    %rdx, %rcx
+; nextln: sarq    %cl, %r8
+; nextln: movl    $$64, %ecx
+; nextln: subq    %rdx, %rcx
+; nextln: movq    %rdi, %rax
+; nextln: shlq    %cl, %rax
+; nextln: orq     %r8, %rax
+; nextln: sarq    $$63, %rdi
+; nextln: xorq    %rcx, %rcx
+; nextln: andq    $$64, %rdx
+; nextln: cmovzq  %rsi, %rdi
+; nextln: cmovzq  %rax, %rcx
+; nextln: cmovnzq %rsi, %rcx
+; nextln: movq    %rcx, %rax
+; nextln: movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f24(i128, i32) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i32):
+    v2 = rotr.i128 v0, v1
+    return v2
+
+; check:  movq    %rsi, %r9
+; nextln: movq    %rdx, %rcx
+; nextln: shrq    %cl, %r9
+; nextln: movq    %rdi, %rax
+; nextln: movq    %rdx, %rcx
+; nextln: shrq    %cl, %rax
+; nextln: movl    $$64, %ecx
+; nextln: subq    %rdx, %rcx
+; nextln: movq    %rsi, %r10
+; nextln: shlq    %cl, %r10
+; nextln: orq     %rax, %r10
+; nextln: xorq    %r8, %r8
+; nextln: xorq    %rax, %rax
+; nextln: movq    %rdx, %rcx
+; nextln: andq    $$64, %rcx
+; nextln: cmovzq  %r9, %r8
+; nextln: cmovzq  %r10, %rax
+; nextln: cmovnzq %r9, %rax
+; nextln: movl    $$128, %r9d
+; nextln: subq    %rdx, %r9
+; nextln: movq    %rdi, %rdx
+; nextln: movq    %r9, %rcx
+; nextln: shlq    %cl, %rdx
+; nextln: movq    %r9, %rcx
+; nextln: shlq    %cl, %rsi
+; nextln: movl    $$64, %ecx
+; nextln: subq    %r9, %rcx
+; nextln: movq    %rdi, %r10
+; nextln: shrq    %cl, %r10
+; nextln: orq     %rsi, %r10
+; nextln: xorq    %rsi, %rsi
+; nextln: xorq    %rdi, %rdi
+; nextln: andq    $$64, %r9
+; nextln: cmovzq  %r10, %rdi
+; nextln: cmovzq  %rdx, %rsi
+; nextln: cmovnzq %rdx, %rdi
+; nextln: orq     %rax, %rsi
+; nextln: orq     %r8, %rdi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f25(i128, i32) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i32):
+    v2 = rotl.i128 v0, v1
+    return v2
+
+; check:  movq    %rdi, %r9
+; nextln: movq    %rdx, %rcx
+; nextln: shlq    %cl, %r9
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rdx, %rcx
+; nextln: shlq    %cl, %rax
+; nextln: movl    $$64, %ecx
+; nextln: subq    %rdx, %rcx
+; nextln: movq    %rdi, %r10
+; nextln: shrq    %cl, %r10
+; nextln: orq     %rax, %r10
+; nextln: xorq    %r8, %r8
+; nextln: xorq    %rax, %rax
+; nextln: movq    %rdx, %rcx
+; nextln: andq    $$64, %rcx
+; nextln: cmovzq  %r10, %rax
+; nextln: cmovzq  %r9, %r8
+; nextln: cmovnzq %r9, %rax
+; nextln: movl    $$128, %r9d
+; nextln: subq    %rdx, %r9
+; nextln: movq    %rsi, %rdx
+; nextln: movq    %r9, %rcx
+; nextln: shrq    %cl, %rdx
+; nextln: movq    %r9, %rcx
+; nextln: shrq    %cl, %rdi
+; nextln: movl    $$64, %ecx
+; nextln: subq    %r9, %rcx
+; nextln: shlq    %cl, %rsi
+; nextln: orq     %rdi, %rsi
+; nextln: xorq    %rdi, %rdi
+; nextln: xorq    %rcx, %rcx
+; nextln: andq    $$64, %r9
+; nextln: cmovzq  %rdx, %rdi
+; nextln: cmovzq  %rsi, %rcx
+; nextln: cmovnzq %rdx, %rcx
+; nextln: orq     %r8, %rcx
+; nextln: orq     %rax, %rdi
+; nextln: movq    %rcx, %rax
+; nextln: movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f26(i128, i64) {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128, v1: i64):
+    store.i128 v0, v1
+    return
+
+; check:  movq    %rdi, 0(%rdx)
+; nextln: movq    %rsi, 8(%rdx)
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f27(i64) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i64):
+    v1 = load.i128 v0
+    return v1
+
+; check:  movq    0(%rdi), %rsi
+; nextln: movq    8(%rdi), %rdi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rdi, %rdx
+
+; nextln:  movq    %rbp, %rsp
+; nextln:  popq    %rbp
+; nextln:  ret
+}
+
+function %f28(i128, b1) -> i128 {
+block0(v0: i128, v1: b1):
+    v2 = iconst.i128 0
+    brnz v1, block1(v2)
+    jump block2(v2)
+
+block1(v3: i128):
+    v4 = iconst.i128 1
+    v5 = iadd.i128 v3, v4
+    return v5
+
+block2(v6: i128):
+    v7 = iconst.i128 2
+    v8 = iadd.i128 v6, v7
+    return v8
+
+; check: pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: testb   $$1, %dl
+; nextln: jnz     label1; j label2
+; check: Block 1:
+; check:  movl    $$0, %esi
+; nextln: movl    $$0, %edi
+; nextln: movl    $$1, %eax
+; nextln: movl    $$0, %ecx
+; nextln: addq    %rax, %rsi
+; nextln: adcq    %rcx, %rdi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rdi, %rdx
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+; check: Block 2:
+; check:  movl    $$0, %esi
+; nextln: movl    $$0, %edi
+; nextln: movl    $$2, %eax
+; nextln: movl    $$0, %ecx
+; nextln: addq    %rax, %rsi
+; nextln: adcq    %rcx, %rdi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rdi, %rdx
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+ 
+}
+
+function %f29(i128, i128, i64, i128, i128, i128) -> i128 {
+
+block0(v0: i128, v1: i128, v2: i64, v3: i128, v4: i128, v5: i128):
+    v6 = iadd.i128 v0, v1
+    v7 = uextend.i128 v2
+    v8 = iadd.i128 v3, v7
+    v9 = iadd.i128 v4, v5
+    v10 = iadd.i128 v6, v8
+    v11 = iadd.i128 v9, v10
+    return v11
+
+; check:  movq    %rsp, %rbp
+; nextln: subq    $$16, %rsp
+; nextln: movq    %r12, 0(%rsp)
+; nextln: movq    %r13, 8(%rsp)
+; nextln: virtual_sp_offset_adjust 16
+; nextln: movq    16(%rbp), %r9
+; nextln: movq    24(%rbp), %r10
+; nextln: movq    32(%rbp), %r12
+; nextln: movq    40(%rbp), %r11
+; nextln: movq    48(%rbp), %rax
+; nextln: movq    56(%rbp), %r13
+; nextln: addq    %rdx, %rdi
+; nextln: adcq    %rcx, %rsi
+; nextln: xorq    %rcx, %rcx
+; nextln: addq    %r8, %r9
+; nextln: adcq    %rcx, %r10
+; nextln: addq    %rax, %r12
+; nextln: adcq    %r13, %r11
+; nextln: addq    %r9, %rdi
+; nextln: adcq    %r10, %rsi
+; nextln: addq    %rdi, %r12
+; nextln: adcq    %rsi, %r11
+; nextln: movq    %r12, %rax
+; nextln: movq    %r11, %rdx
+; nextln: movq    0(%rsp), %r12
+; nextln: movq    8(%rsp), %r13
+; nextln: addq    $$16, %rsp
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+}
+
+function %f30(i128) -> i128, i128, i128, i64, i128, i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i128):
+    v1 = ireduce.i64 v0
+    return v0, v0, v0, v1, v0, v0
+
+; likely to change with regalloc -- just check the stores into the retval area:
+
+; check:  movq    %r8, 0(%rsi)
+; nextln: movq    %r9, 8(%rsi)
+; nextln: movq    %r10, 16(%rsi)
+; nextln: movq    %r11, 24(%rsi)
+; nextln: movq    %r12, 32(%rsi)
+; nextln: movq    %r13, 48(%rsi)
+; nextln: movq    %r14, 56(%rsi)
+; nextln: movq    %rdi, 64(%rsi)
+; nextln: movq    %rbx, 72(%rsi)
+
+}
+
+function %f31(i128, i128) -> i128, i128 {
+    fn0 = %g(i128, i128) -> i128, i128
+block0(v0: i128, v1: i128):
+    v2, v3 = call fn0(v0, v1)
+    return v2, v3
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: subq    $$16, %rsp
+; nextln: movq    %r12, 0(%rsp)
+; nextln: virtual_sp_offset_adjust 16
+; nextln: movq    %r8, %r12
+; nextln: subq    $$16, %rsp
+; nextln: virtual_sp_offset_adjust 16
+; nextln: lea     0(%rsp), %r8
+; nextln: load_ext_name %g+0, %rax
+; nextln: call    *%rax
+; nextln: movq    0(%rsp), %rsi
+; nextln: movq    8(%rsp), %rdi
+; nextln: addq    $$16, %rsp
+; nextln: virtual_sp_offset_adjust -16
+; nextln: movq    %rsi, 0(%r12)
+; nextln: movq    %rdi, 8(%r12)
+; nextln: movq    0(%rsp), %r12
+; nextln: addq    $$16, %rsp
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+}
+
+function %f32(i128) -> i128 {
+block0(v0: i128):
+    v1 = clz.i128 v0
+    return v1
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: movabsq $$-1, %rcx
+; nextln: bsrq    %rsi, %rax
+; nextln: cmovzq  %rcx, %rax
+; nextln: movl    $$63, %esi
+; nextln: subq    %rax, %rsi
+; nextln: movabsq $$-1, %rcx
+; nextln: bsrq    %rdi, %rax
+; nextln: cmovzq  %rcx, %rax
+; nextln: movl    $$63, %edi
+; nextln: subq    %rax, %rdi
+; nextln: addq    $$64, %rdi
+; nextln: cmpq    $$64, %rsi
+; nextln: cmovnzq %rsi, %rdi
+; nextln: xorq    %rsi, %rsi
+; nextln: movq    %rdi, %rax
+; nextln: movq    %rsi, %rdx
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+}
+
+function %f33(i128) -> i128 {
+block0(v0: i128):
+    v1 = ctz.i128 v0
+    return v1
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: movq    %rsi, %rax
+; nextln: movl    $$64, %ecx
+; nextln: bsfq    %rdi, %rsi
+; nextln: cmovzq  %rcx, %rsi
+; nextln: movl    $$64, %ecx
+; nextln: bsfq    %rax, %rdi
+; nextln: cmovzq  %rcx, %rdi
+; nextln: addq    $$64, %rdi
+; nextln: cmpq    $$64, %rsi
+; nextln: cmovzq  %rdi, %rsi
+; nextln: xorq    %rdi, %rdi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rdi, %rdx
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
diff --git a/cranelift/filetests/filetests/isa/x64/icmp-i128-run.clif b/cranelift/filetests/filetests/isa/x64/icmp-i128-run.clif
new file mode 100644
index 0000000000..2f9e2d5331
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/icmp-i128-run.clif
@@ -0,0 +1,95 @@
+test run
+target x86_64
+feature "experimental_x64"
+
+function %test_icmp_eq_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x0
+    v12 = iconst.i64 0x0
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x0
+    v22 = iconst.i64 0x0
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 eq v1, v2
+    return v10
+}
+
+; run
+
+function %test_icmp_imm_eq_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x0
+    v12 = iconst.i64 0x0
+    v1 = iconcat v11, v12
+    v10 = icmp_imm.i128 eq v1, 0x0
+    return v10
+}
+
+; run
+
+function %test_icmp_ne_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x0
+    v12 = iconst.i64 0x0
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x0
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 ne v1, v2
+    return v10
+}
+
+; run
+
+function %test_icmp_imm_ne_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x0
+    v12 = iconst.i64 0x0
+    v1 = iconcat v11, v12
+    v10 = icmp_imm.i128 ne v1, 0x1
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_eq_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 eq v1, v2
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_gt_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x2
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 ugt v2, v1
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_ge_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 uge v1, v2
+    return v10
+}
+
+; run
diff --git a/cranelift/filetests/filetests/isa/x64/popcnt-use-popcnt.clif b/cranelift/filetests/filetests/isa/x64/popcnt-use-popcnt.clif
new file mode 100644
index 0000000000..4e49cd6d4f
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/popcnt-use-popcnt.clif
@@ -0,0 +1,31 @@
+test compile
+target x86_64 has_popcnt has_sse42
+feature "experimental_x64"
+
+function %popcnt(i64) -> i64 {
+block0(v0: i64):
+    v1 = popcnt v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: popcntq %rdi, %rsi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret
+
+function %popcnt(i32) -> i32 {
+block0(v0: i32):
+    v1 = popcnt v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: popcntl %edi, %esi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret
diff --git a/cranelift/filetests/filetests/isa/x64/select-i128.clif b/cranelift/filetests/filetests/isa/x64/select-i128.clif
new file mode 100644
index 0000000000..3492a71997
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/select-i128.clif
@@ -0,0 +1,29 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function %f0(i32, i128, i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i32, v1: i128, v2: i128):
+
+    v3 = iconst.i32 42
+    v4 = icmp.i32 eq v0, v3
+; nextln: movl    $$42, %eax
+; nextln: cmpl    %eax, %edi
+
+    v5 = select.i128 v4, v1, v2
+; nextln: cmovzq  %rsi, %rcx
+; nextln: cmovzq  %rdx, %r8
+
+    return v5
+; nextln: movq    %rcx, %rax
+; nextln: movq    %r8, %rdx
+
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+ 
+}
+
diff --git a/cranelift/filetests/filetests/isa/x64/shift-i128-run.clif b/cranelift/filetests/filetests/isa/x64/shift-i128-run.clif
new file mode 100644
index 0000000000..37bc4667e7
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/shift-i128-run.clif
@@ -0,0 +1,106 @@
+test run
+target x86_64
+feature "experimental_x64"
+
+function %ishl1() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconcat v0, v0
+    v2 = iconst.i32 2
+    v3 = ishl.i128 v1, v2
+    v4 = iconst.i64 0x04040404_04040404
+    v5 = iconcat v4, v4
+    v6 = icmp eq v3, v5
+    return v6
+}
+; run
+
+function %ishl2() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0x01010101_01010101
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 9
+    v4 = ishl.i128 v2, v3
+    v5 = iconst.i64 0x02020202_02020200
+    v6 = iconst.i64 0x02020202_02020202
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %ishl3() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0xffffffff_ffffffff
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 66
+    v4 = ishl.i128 v2, v3
+    v5 = iconst.i64 0x00000000_00000000
+    v6 = iconst.i64 0x04040404_04040404
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %ushr1() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0x01010101_01010101
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 2
+    v4 = ushr.i128 v2, v3
+    v5 = iconst.i64 0x40404040_40404040
+    v6 = iconst.i64 0x00404040_40404040
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %ushr2() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0x01010101_01010101
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 66
+    v4 = ushr.i128 v2, v3
+    v5 = iconst.i64 0x00404040_40404040
+    v6 = iconst.i64 0x00000000_00000000
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %sshr1() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0x81010101_01010101
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 2
+    v4 = sshr.i128 v2, v3
+    v5 = iconst.i64 0x40404040_40404040
+    v6 = iconst.i64 0xe0404040_40404040
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %sshr2() -> b1 {
+block0:
+    v0 = iconst.i64 0x12345678_9abcdef0
+    v1 = iconst.i64 0x80101010_10101010
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 66
+    v4 = sshr.i128 v2, v3
+    v5 = iconst.i64 0xe0040404_04040404
+    v6 = iconst.i64 0xffffffff_ffffffff
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
diff --git a/cranelift/filetests/filetests/isa/x64/struct-arg.clif b/cranelift/filetests/filetests/isa/x64/struct-arg.clif
new file mode 100644
index 0000000000..3c867038a7
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/struct-arg.clif
@@ -0,0 +1,147 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function u0:0(i64 sarg(64)) -> i8 system_v {
+block0(v0: i64):
+    v1 = load.i8 v0
+    return v1
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: lea     16(%rbp), %rsi
+; nextln: movzbq  0(%rsi), %rsi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:1(i64 sarg(64), i64) -> i8 system_v {
+block0(v0: i64, v1: i64):
+    v2 = load.i8 v1
+	v3 = load.i8 v0
+	v4 = iadd.i8 v2, v3
+    return v4
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: lea     16(%rbp), %rsi
+; nextln: movzbq  0(%rdi), %rdi
+; nextln: movzbq  0(%rsi), %rsi
+; nextln: addl    %esi, %edi
+; nextln: movq    %rdi, %rax
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:2(i64) -> i8 system_v {
+fn1 = colocated u0:0(i64 sarg(64)) -> i8 system_v
+
+block0(v0: i64):
+    v1 = call fn1(v0)
+    return v1
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: movq    %rdi, %rsi
+; nextln: subq    $$64, %rsp
+; nextln: virtual_sp_offset_adjust 64
+; nextln: lea     0(%rsp), %rdi
+; nextln: movl    $$64, %edx
+; nextln: load_ext_name %Memcpy+0, %rcx
+; nextln: call    *%rcx
+; nextln: call    User { namespace: 0, index: 0 }
+; nextln: addq    $$64, %rsp
+; nextln: virtual_sp_offset_adjust -64
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:3(i64, i64) -> i8 system_v {
+fn1 = colocated u0:0(i64, i64 sarg(64)) -> i8 system_v
+
+block0(v0: i64, v1: i64):
+    v2 = call fn1(v0, v1)
+    return v2
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: subq    $$16, %rsp
+; nextln: movq    %r12, 0(%rsp)
+; nextln: virtual_sp_offset_adjust 16
+; nextln: movq    %rdi, %r12
+; nextln: subq    $$64, %rsp
+; nextln: virtual_sp_offset_adjust 64
+; nextln: lea     0(%rsp), %rdi
+; nextln: movl    $$64, %edx
+; nextln: load_ext_name %Memcpy+0, %rcx
+; nextln: call    *%rcx
+; nextln: movq    %r12, %rdi
+; nextln: call    User { namespace: 0, index: 0 }
+; nextln: addq    $$64, %rsp
+; nextln: virtual_sp_offset_adjust -64
+; nextln: movq    0(%rsp), %r12
+; nextln: addq    $$16, %rsp
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:4(i64 sarg(128), i64 sarg(64)) -> i8 system_v {
+block0(v0: i64, v1: i64):
+    v2 = load.i8 v0
+    v3 = load.i8 v1
+    v4 = iadd.i8 v2, v3
+    return v4
+}
+
+; check:  movq    %rsp, %rbp
+; nextln: lea     16(%rbp), %rsi
+; nextln: lea     144(%rbp), %rdi
+; nextln: movzbq  0(%rsi), %rsi
+; nextln: movzbq  0(%rdi), %rdi
+; nextln: addl    %edi, %esi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:5(i64, i64, i64) -> i8 system_v {
+fn1 = colocated u0:0(i64, i64 sarg(128), i64 sarg(64)) -> i8 system_v
+
+block0(v0: i64, v1: i64, v2: i64):
+    v3 = call fn1(v0, v1, v2)
+    return v3
+}
+
+; check:  movq    %rsp, %rbp
+; nextln: subq    $$16, %rsp
+; nextln: movq    %r12, 0(%rsp)
+; nextln: movq    %r13, 8(%rsp)
+; nextln: virtual_sp_offset_adjust 16
+; nextln: movq    %rdi, %r12
+; nextln: movq    %rdx, %r13
+; nextln: subq    $$192, %rsp
+; nextln: virtual_sp_offset_adjust 192
+; nextln: lea     0(%rsp), %rdi
+; nextln: movl    $$128, %edx
+; nextln: load_ext_name %Memcpy+0, %rcx
+; nextln: call    *%rcx
+; nextln: lea     128(%rsp), %rdi
+; nextln: movq    %r13, %rsi
+; nextln: movl    $$64, %edx
+; nextln: load_ext_name %Memcpy+0, %rcx
+; nextln: call    *%rcx
+; nextln: movq    %r12, %rdi
+; nextln: call    User { namespace: 0, index: 0 }
+; nextln: addq    $$192, %rsp
+; nextln: virtual_sp_offset_adjust -192
+; nextln: movq    0(%rsp), %r12
+; nextln: movq    8(%rsp), %r13
+; nextln: addq    $$16, %rsp
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
diff --git a/cranelift/filetests/filetests/isa/x64/struct-ret.clif b/cranelift/filetests/filetests/isa/x64/struct-ret.clif
new file mode 100644
index 0000000000..05ebbd100b
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/struct-ret.clif
@@ -0,0 +1,19 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function %f0(i64 sret) {
+block0(v0: i64):
+    v1 = iconst.i64 42
+    store v1, v0
+    return
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: movq    %rdi, %rax
+; nextln: movl    $$42, %esi
+; nextln: movq    %rsi, 0(%rdi)
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
diff --git a/cranelift/filetests/filetests/isa/x64/tls_elf.clif b/cranelift/filetests/filetests/isa/x64/tls_elf.clif
new file mode 100644
index 0000000000..6136e98e7d
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/tls_elf.clif
@@ -0,0 +1,19 @@
+test compile
+set tls_model=elf_gd
+target x86_64
+feature "experimental_x64"
+
+function u0:0(i32) -> i64 {
+gv0 = symbol colocated tls u1:0
+
+block0(v0: i32):
+    v1 = global_value.i64 gv0
+    return v1
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: elf_tls_get_addr User { namespace: 1, index: 0 }
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
diff --git a/cranelift/filetests/filetests/isa/x86/icmp-i128.clif b/cranelift/filetests/filetests/isa/x86/icmp-i128.clif
index dce0e1db87..789fcc6ea3 100644
--- a/cranelift/filetests/filetests/isa/x86/icmp-i128.clif
+++ b/cranelift/filetests/filetests/isa/x86/icmp-i128.clif
@@ -50,3 +50,45 @@ block0:
 }
 
 ; run
+
+function %test_icmp_nz_eq_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 eq v1, v2
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_gt_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x2
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 ugt v2, v1
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_ge_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 uge v1, v2
+    return v10
+}
+
+; run
diff --git a/cranelift/native/Cargo.toml b/cranelift/native/Cargo.toml
index e578bccf12..3bcd72d699 100644
--- a/cranelift/native/Cargo.toml
+++ b/cranelift/native/Cargo.toml
@@ -14,16 +14,10 @@ edition = "2018"
 cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false }
 target-lexicon = "0.11"
 
-[target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies]
-raw-cpuid = "8.1.2"
-
 [features]
 default = ["std"]
 std = ["cranelift-codegen/std"]
-# when compiling with the "core" feature, nightly must be enabled
-# enabling the "nightly" feature for raw-cpuid allows avoiding
-# linking in a c-library.
-core = ["cranelift-codegen/core", "raw-cpuid/nightly"]
+core = ["cranelift-codegen/core"]
 
 [badges]
 maintenance = { status = "experimental" }
diff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rs
index 3bd5716dac..43938bd97e 100644
--- a/cranelift/native/src/lib.rs
+++ b/cranelift/native/src/lib.rs
@@ -22,19 +22,15 @@
         clippy::use_self
     )
 )]
-#![no_std]
 
 use cranelift_codegen::isa;
 use target_lexicon::Triple;
 
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-use raw_cpuid::CpuId;
-
 /// Return an `isa` builder configured for the current host
 /// machine, or `Err(())` if the host machine is not supported
 /// in the current configuration.
 pub fn builder() -> Result<isa::Builder, &'static str> {
-    builder_with_backend_variant(isa::BackendVariant::Any)
+    builder_with_options(isa::BackendVariant::Any, true)
 }
 
 /// Return an `isa` builder configured for the current host
@@ -44,8 +40,9 @@ pub fn builder() -> Result<isa::Builder, &'static str> {
 /// Selects the given backend variant specifically; this is
 /// useful when more than oen backend exists for a given target
 /// (e.g., on x86-64).
-pub fn builder_with_backend_variant(
+pub fn builder_with_options(
     variant: isa::BackendVariant,
+    infer_native_flags: bool,
 ) -> Result<isa::Builder, &'static str> {
     let mut isa_builder =
         isa::lookup_variant(Triple::host(), variant).map_err(|err| match err {
@@ -55,72 +52,64 @@ pub fn builder_with_backend_variant(
             isa::LookupError::Unsupported => "unsupported architecture",
         })?;
 
-    if cfg!(any(target_arch = "x86", target_arch = "x86_64")) {
-        parse_x86_cpuid(&mut isa_builder)?;
-    }
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    {
+        use cranelift_codegen::settings::Configurable;
 
-    Ok(isa_builder)
-}
-
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-fn parse_x86_cpuid(isa_builder: &mut isa::Builder) -> Result<(), &'static str> {
-    use cranelift_codegen::settings::Configurable;
-    let cpuid = CpuId::new();
-
-    if let Some(info) = cpuid.get_feature_info() {
-        if !info.has_sse2() {
+        if !std::is_x86_feature_detected!("sse2") {
             return Err("x86 support requires SSE2");
         }
-        if info.has_sse3() {
+
+        if !infer_native_flags {
+            return Ok(isa_builder);
+        }
+
+        if std::is_x86_feature_detected!("sse3") {
             isa_builder.enable("has_sse3").unwrap();
         }
-        if info.has_ssse3() {
+        if std::is_x86_feature_detected!("ssse3") {
             isa_builder.enable("has_ssse3").unwrap();
         }
-        if info.has_sse41() {
+        if std::is_x86_feature_detected!("sse4.1") {
             isa_builder.enable("has_sse41").unwrap();
         }
-        if info.has_sse42() {
+        if std::is_x86_feature_detected!("sse4.2") {
             isa_builder.enable("has_sse42").unwrap();
         }
-        if info.has_popcnt() {
+        if std::is_x86_feature_detected!("popcnt") {
             isa_builder.enable("has_popcnt").unwrap();
         }
-        if info.has_avx() {
+        if std::is_x86_feature_detected!("avx") {
             isa_builder.enable("has_avx").unwrap();
         }
-    }
-    if let Some(info) = cpuid.get_extended_feature_info() {
-        if info.has_bmi1() {
-            isa_builder.enable("has_bmi1").unwrap();
-        }
-        if info.has_bmi2() {
-            isa_builder.enable("has_bmi2").unwrap();
-        }
-        if info.has_avx2() {
+        if std::is_x86_feature_detected!("avx2") {
             isa_builder.enable("has_avx2").unwrap();
         }
-        if info.has_avx512dq() {
+        if std::is_x86_feature_detected!("bmi1") {
+            isa_builder.enable("has_bmi1").unwrap();
+        }
+        if std::is_x86_feature_detected!("bmi2") {
+            isa_builder.enable("has_bmi2").unwrap();
+        }
+        if std::is_x86_feature_detected!("avx512dq") {
             isa_builder.enable("has_avx512dq").unwrap();
         }
-        if info.has_avx512vl() {
+        if std::is_x86_feature_detected!("avx512vl") {
             isa_builder.enable("has_avx512vl").unwrap();
         }
-        if info.has_avx512f() {
+        if std::is_x86_feature_detected!("avx512f") {
             isa_builder.enable("has_avx512f").unwrap();
         }
-    }
-    if let Some(info) = cpuid.get_extended_function_info() {
-        if info.has_lzcnt() {
+        if std::is_x86_feature_detected!("lzcnt") {
             isa_builder.enable("has_lzcnt").unwrap();
         }
     }
-    Ok(())
-}
 
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
-fn parse_x86_cpuid(_isa_builder: &mut isa::Builder) -> Result<(), &'static str> {
-    unreachable!();
+    // squelch warnings about unused mut/variables on some platforms.
+    drop(&mut isa_builder);
+    drop(infer_native_flags);
+
+    Ok(isa_builder)
 }
 
 #[cfg(test)]
diff --git a/cranelift/object/Cargo.toml b/cranelift/object/Cargo.toml
index 95c5b07447..bb27e3ee6c 100644
--- a/cranelift/object/Cargo.toml
+++ b/cranelift/object/Cargo.toml
@@ -12,7 +12,7 @@ edition = "2018"
 [dependencies]
 cranelift-module = { path = "../module", version = "0.69.0" }
 cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false, features = ["std"] }
-object = { version = "0.22.0", default-features = false, features = ["write"] }
+object = { version = "0.23.0", default-features = false, features = ["write"] }
 target-lexicon = "0.11"
 anyhow = "1.0"
 log = { version = "0.4.6", default-features = false }
diff --git a/cranelift/peepmatic/Cargo.toml b/cranelift/peepmatic/Cargo.toml
index 9028e7105a..9bd6540673 100644
--- a/cranelift/peepmatic/Cargo.toml
+++ b/cranelift/peepmatic/Cargo.toml
@@ -15,7 +15,7 @@ peepmatic-macro = { version = "0.69.0", path = "crates/macro" }
 peepmatic-runtime = { version = "0.69.0", path = "crates/runtime", features = ["construct"] }
 peepmatic-traits = { version = "0.69.0", path = "crates/traits" }
 serde = { version = "1.0.105", features = ["derive"] }
-wast = "29.0.0"
+wast = "32.0.0"
 z3 = { version = "0.7.1", features = ["static-link-z3"] }
 
 [dev-dependencies]
diff --git a/cranelift/peepmatic/crates/fuzzing/Cargo.toml b/cranelift/peepmatic/crates/fuzzing/Cargo.toml
index 1611b75c6d..b84b8aef3c 100644
--- a/cranelift/peepmatic/crates/fuzzing/Cargo.toml
+++ b/cranelift/peepmatic/crates/fuzzing/Cargo.toml
@@ -19,6 +19,6 @@ peepmatic-runtime = { path = "../runtime", features = ["construct"] }
 peepmatic-test = { path = "../test" }
 peepmatic-test-operator = { path = "../test-operator" }
 peepmatic-traits = { path = "../traits" }
-rand = { version = "0.7.3", features = ["small_rng"] }
+rand = { version = "0.8.2", features = ["small_rng"] }
 serde = "1.0.106"
-wast = "29.0.0"
+wast = "32.0.0"
diff --git a/cranelift/peepmatic/crates/runtime/Cargo.toml b/cranelift/peepmatic/crates/runtime/Cargo.toml
index 8e15ad7527..729c3678c3 100644
--- a/cranelift/peepmatic/crates/runtime/Cargo.toml
+++ b/cranelift/peepmatic/crates/runtime/Cargo.toml
@@ -16,7 +16,7 @@ peepmatic-automata = { version = "0.69.0", path = "../automata", features = ["se
 peepmatic-traits = { version = "0.69.0", path = "../traits" }
 serde = { version = "1.0.105", features = ["derive"] }
 thiserror = "1.0.15"
-wast = { version = "29.0.0", optional = true }
+wast = { version = "32.0.0", optional = true }
 
 [dev-dependencies]
 peepmatic-test-operator = { version = "0.69.0", path = "../test-operator" }
diff --git a/cranelift/peepmatic/crates/souper/Cargo.toml b/cranelift/peepmatic/crates/souper/Cargo.toml
index 811363651c..1bec224d06 100644
--- a/cranelift/peepmatic/crates/souper/Cargo.toml
+++ b/cranelift/peepmatic/crates/souper/Cargo.toml
@@ -16,4 +16,4 @@ log = "0.4.8"
 [dev-dependencies]
 peepmatic = { path = "../..", version = "0.69.0" }
 peepmatic-test-operator = { version = "0.69.0", path = "../test-operator" }
-wast = "29.0.0"
+wast = "32.0.0"
diff --git a/cranelift/peepmatic/crates/test-operator/Cargo.toml b/cranelift/peepmatic/crates/test-operator/Cargo.toml
index 295610c1bc..6574827a44 100644
--- a/cranelift/peepmatic/crates/test-operator/Cargo.toml
+++ b/cranelift/peepmatic/crates/test-operator/Cargo.toml
@@ -11,4 +11,4 @@ edition = "2018"
 [dependencies]
 peepmatic-traits = { version = "0.69.0", path = "../traits" }
 serde = { version = "1.0.105", features = ["derive"] }
-wast = "29.0.0"
+wast = "32.0.0"
diff --git a/cranelift/src/souper_harvest.rs b/cranelift/src/souper_harvest.rs
index 5bfa1fe629..e92a502302 100644
--- a/cranelift/src/souper_harvest.rs
+++ b/cranelift/src/souper_harvest.rs
@@ -12,7 +12,7 @@ static WASM_MAGIC: &[u8] = &[0x00, 0x61, 0x73, 0x6D];
 /// Harvest candidates for superoptimization from a Wasm or Clif file.
 ///
 /// Candidates are emitted in Souper's text format:
-/// https://github.com/google/souper
+/// <https://github.com/google/souper>
 #[derive(StructOpt)]
 pub struct Options {
     /// Specify an input file to be used. Use '-' for stdin.
diff --git a/cranelift/wasm/Cargo.toml b/cranelift/wasm/Cargo.toml
index 413d2ee004..b5549453f8 100644
--- a/cranelift/wasm/Cargo.toml
+++ b/cranelift/wasm/Cargo.toml
@@ -12,12 +12,12 @@ keywords = ["webassembly", "wasm"]
 edition = "2018"
 
 [dependencies]
-wasmparser = { version = "0.71", default-features = false }
+wasmparser = { version = "0.73", default-features = false }
 cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false }
 cranelift-entity = { path = "../entity", version = "0.69.0" }
 cranelift-frontend = { path = "../frontend", version = "0.69.0", default-features = false }
 hashbrown = { version = "0.9.1", optional = true }
-itertools = "0.9.0"
+itertools = "0.10.0"
 log = { version = "0.4.6", default-features = false }
 serde = { version = "1.0.94", features = ["derive"], optional = true }
 smallvec = "1.6.1"
diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs
index bb22a333ca..a20975b25b 100644
--- a/cranelift/wasm/src/code_translator.rs
+++ b/cranelift/wasm/src/code_translator.rs
@@ -261,7 +261,7 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
                 .extend_from_slice(builder.block_params(loop_body));
 
             builder.switch_to_block(loop_body);
-            environ.translate_loop_header(builder.cursor())?;
+            environ.translate_loop_header(builder)?;
         }
         Operator::If { ty } => {
             let val = state.pop1();
@@ -1626,7 +1626,7 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
             // operands must match (hence the bitcast).
             state.push1(builder.ins().bitselect(bitcast_c, bitcast_a, bitcast_b))
         }
-        Operator::I8x16AnyTrue | Operator::I16x8AnyTrue | Operator::I32x4AnyTrue => {
+        Operator::V128AnyTrue => {
             let a = pop1_with_bitcast(state, type_of(op), builder);
             let bool_result = builder.ins().vany_true(a);
             state.push1(builder.ins().bint(I32, bool_result))
@@ -1824,7 +1824,21 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
             let (a, b) = pop2_with_bitcast(state, I16X8, builder);
             state.push1(builder.ins().widening_pairwise_dot_product_s(a, b));
         }
-        Operator::I16x8ExtMulLowI8x16S
+        Operator::I64x2Bitmask
+        | Operator::I64x2WidenLowI32x4S
+        | Operator::I64x2WidenHighI32x4S
+        | Operator::I64x2WidenLowI32x4U
+        | Operator::I64x2WidenHighI32x4U
+        | Operator::V128Load8Lane { .. }
+        | Operator::V128Load16Lane { .. }
+        | Operator::V128Load32Lane { .. }
+        | Operator::V128Load64Lane { .. }
+        | Operator::V128Store8Lane { .. }
+        | Operator::V128Store16Lane { .. }
+        | Operator::V128Store32Lane { .. }
+        | Operator::V128Store64Lane { .. }
+        | Operator::I16x8Q15MulrSatS
+        | Operator::I16x8ExtMulLowI8x16S
         | Operator::I16x8ExtMulHighI8x16S
         | Operator::I16x8ExtMulLowI8x16U
         | Operator::I16x8ExtMulHighI8x16U
@@ -2522,7 +2536,6 @@ fn type_of(operator: &Operator) -> Type {
         | Operator::I8x16GeU
         | Operator::I8x16Neg
         | Operator::I8x16Abs
-        | Operator::I8x16AnyTrue
         | Operator::I8x16AllTrue
         | Operator::I8x16Shl
         | Operator::I8x16ShrS
@@ -2557,7 +2570,6 @@ fn type_of(operator: &Operator) -> Type {
         | Operator::I16x8GeU
         | Operator::I16x8Neg
         | Operator::I16x8Abs
-        | Operator::I16x8AnyTrue
         | Operator::I16x8AllTrue
         | Operator::I16x8Shl
         | Operator::I16x8ShrS
@@ -2592,7 +2604,6 @@ fn type_of(operator: &Operator) -> Type {
         | Operator::I32x4GeU
         | Operator::I32x4Neg
         | Operator::I32x4Abs
-        | Operator::I32x4AnyTrue
         | Operator::I32x4AllTrue
         | Operator::I32x4Shl
         | Operator::I32x4ShrS
diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs
index f7577de5d3..0bd3c48745 100644
--- a/cranelift/wasm/src/environ/spec.rs
+++ b/cranelift/wasm/src/environ/spec.rs
@@ -205,25 +205,34 @@ pub enum ReturnMode {
 
 /// An entry in the alias section of a wasm module (from the module linking
 /// proposal)
-pub enum Alias {
-    /// A parent's module is being aliased into our own index space.
-    ///
-    /// Note that the index here is in the parent's index space, not our own.
-    ParentModule(ModuleIndex),
+pub enum Alias<'a> {
+    /// An outer module's module is being aliased into our own index space.
+    OuterModule {
+        /// The number of modules above us that we're referencing.
+        relative_depth: u32,
+        /// The module index in the outer module's index space we're referencing.
+        index: ModuleIndex,
+    },
 
-    /// A parent's type is being aliased into our own index space
+    /// An outer module's type is being aliased into our own index space
     ///
-    /// Note that the index here is in the parent's index space, not our own.
-    ParentType(TypeIndex),
+    /// Note that the index here is in the outer module's index space, not our
+    /// own.
+    OuterType {
+        /// The number of modules above us that we're referencing.
+        relative_depth: u32,
+        /// The type index in the outer module's index space we're referencing.
+        index: TypeIndex,
+    },
 
     /// A previously created instance is having one of its exports aliased into
     /// our index space.
-    Child {
+    InstanceExport {
         /// The index we're aliasing.
         instance: InstanceIndex,
         /// The nth export that we're inserting into our own index space
         /// locally.
-        export: usize,
+        export: &'a str,
     },
 }
 
@@ -286,6 +295,12 @@ pub trait FuncEnvironment: TargetEnvironment {
         ReturnMode::NormalReturns
     }
 
+    /// Called after the locals for a function have been parsed, and the number
+    /// of variables defined by this function is provided.
+    fn after_locals(&mut self, num_locals_defined: usize) {
+        drop(num_locals_defined);
+    }
+
     /// Set up the necessary preamble definitions in `func` to access the global variable
     /// identified by `index`.
     ///
@@ -628,7 +643,7 @@ pub trait FuncEnvironment: TargetEnvironment {
     ///
     /// This can be used to insert explicit interrupt or safepoint checking at
     /// the beginnings of loops.
-    fn translate_loop_header(&mut self, _pos: FuncCursor) -> WasmResult<()> {
+    fn translate_loop_header(&mut self, _builder: &mut FunctionBuilder) -> WasmResult<()> {
         // By default, don't emit anything.
         Ok(())
     }
@@ -1014,30 +1029,15 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
         drop(amount);
     }
 
-    /// Declares that a module will come later with the type signature provided.
-    fn declare_module(&mut self, ty: TypeIndex) -> WasmResult<()> {
-        drop(ty);
-        Err(WasmError::Unsupported("module linking".to_string()))
-    }
-
     /// Called at the beginning of translating a module.
     ///
-    /// The `index` argument is a monotonically increasing index which
-    /// corresponds to the nth module that's being translated. This is not the
-    /// 32-bit index in the current module's index space. For example the first
-    /// call to `module_start` will have index 0.
-    ///
     /// Note that for nested modules this may be called multiple times.
-    fn module_start(&mut self, index: usize) {
-        drop(index);
-    }
+    fn module_start(&mut self) {}
 
     /// Called at the end of translating a module.
     ///
     /// Note that for nested modules this may be called multiple times.
-    fn module_end(&mut self, index: usize) {
-        drop(index);
-    }
+    fn module_end(&mut self) {}
 
     /// Indicates that this module will have `amount` instances.
     fn reserve_instances(&mut self, amount: u32) {
@@ -1046,7 +1046,11 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
 
     /// Declares a new instance which this module will instantiate before it's
     /// instantiated.
-    fn declare_instance(&mut self, module: ModuleIndex, args: Vec<EntityIndex>) -> WasmResult<()> {
+    fn declare_instance(
+        &mut self,
+        module: ModuleIndex,
+        args: Vec<(&'data str, EntityIndex)>,
+    ) -> WasmResult<()> {
         drop((module, args));
         Err(WasmError::Unsupported("wasm instance".to_string()))
     }
@@ -1056,7 +1060,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
     /// The alias comes from the `instance` specified (or the parent if `None`
     /// is supplied) and the index is either in the module's own index spaces
     /// for the parent or an index into the exports for nested instances.
-    fn declare_alias(&mut self, alias: Alias) -> WasmResult<()> {
+    fn declare_alias(&mut self, alias: Alias<'data>) -> WasmResult<()> {
         drop(alias);
         Err(WasmError::Unsupported("wasm alias".to_string()))
     }
diff --git a/cranelift/wasm/src/func_translator.rs b/cranelift/wasm/src/func_translator.rs
index 7a8f9c19d2..97e5354c6e 100644
--- a/cranelift/wasm/src/func_translator.rs
+++ b/cranelift/wasm/src/func_translator.rs
@@ -170,6 +170,8 @@ fn parse_local_decls<FE: FuncEnvironment + ?Sized>(
         declare_locals(builder, count, ty, &mut next_local, environ)?;
     }
 
+    environ.after_locals(next_local);
+
     Ok(())
 }
 
diff --git a/cranelift/wasm/src/module_translator.rs b/cranelift/wasm/src/module_translator.rs
index c3e27dd820..ed71d7ce1c 100644
--- a/cranelift/wasm/src/module_translator.rs
+++ b/cranelift/wasm/src/module_translator.rs
@@ -4,8 +4,8 @@ use crate::environ::{ModuleEnvironment, WasmResult};
 use crate::sections_translator::{
     parse_alias_section, parse_data_section, parse_element_section, parse_event_section,
     parse_export_section, parse_function_section, parse_global_section, parse_import_section,
-    parse_instance_section, parse_memory_section, parse_module_section, parse_name_section,
-    parse_start_section, parse_table_section, parse_type_section,
+    parse_instance_section, parse_memory_section, parse_name_section, parse_start_section,
+    parse_table_section, parse_type_section,
 };
 use crate::state::ModuleTranslationState;
 use cranelift_codegen::timing;
@@ -22,23 +22,16 @@ pub fn translate_module<'data>(
     let mut module_translation_state = ModuleTranslationState::new();
     let mut validator = Validator::new();
     validator.wasm_features(environ.wasm_features());
-    let mut stack = Vec::new();
-    let mut modules = 1;
-    let mut cur_module = 0;
 
     for payload in Parser::new(0).parse_all(data) {
         match payload? {
             Payload::Version { num, range } => {
                 validator.version(num, &range)?;
-                environ.module_start(cur_module);
+                environ.module_start();
             }
             Payload::End => {
                 validator.end()?;
-                environ.module_end(cur_module);
-                if let Some((other, other_index)) = stack.pop() {
-                    validator = other;
-                    cur_module = other_index;
-                }
+                environ.module_end();
             }
 
             Payload::TypeSection(types) => {
@@ -111,10 +104,6 @@ pub fn translate_module<'data>(
                 environ.reserve_passive_data(count)?;
             }
 
-            Payload::ModuleSection(s) => {
-                validator.module_section(&s)?;
-                parse_module_section(s, environ)?;
-            }
             Payload::InstanceSection(s) => {
                 validator.instance_section(&s)?;
                 parse_instance_section(s, environ)?;
@@ -123,20 +112,17 @@ pub fn translate_module<'data>(
                 validator.alias_section(&s)?;
                 parse_alias_section(s, environ)?;
             }
-            Payload::ModuleCodeSectionStart {
+            Payload::ModuleSectionStart {
                 count,
                 range,
                 size: _,
             } => {
-                validator.module_code_section_start(count, &range)?;
+                validator.module_section_start(count, &range)?;
+                environ.reserve_modules(count);
             }
 
-            Payload::ModuleCodeSectionEntry { .. } => {
-                let subvalidator = validator.module_code_section_entry();
-                stack.push((validator, cur_module));
-                validator = subvalidator;
-                cur_module = modules;
-                modules += 1;
+            Payload::ModuleSectionEntry { .. } => {
+                validator.module_section_entry();
             }
 
             Payload::CustomSection {
diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs
index 0e791c1a43..3906c02393 100644
--- a/cranelift/wasm/src/sections_translator.rs
+++ b/cranelift/wasm/src/sections_translator.rs
@@ -504,19 +504,6 @@ pub fn parse_name_section<'data>(
     Ok(())
 }
 
-/// Parses the Module section of the wasm module.
-pub fn parse_module_section<'data>(
-    section: wasmparser::ModuleSectionReader<'data>,
-    environ: &mut dyn ModuleEnvironment<'data>,
-) -> WasmResult<()> {
-    environ.reserve_modules(section.get_count());
-
-    for module_ty in section {
-        environ.declare_module(TypeIndex::from_u32(module_ty?))?;
-    }
-    Ok(())
-}
-
 /// Parses the Instance section of the wasm module.
 pub fn parse_instance_section<'data>(
     section: wasmparser::InstanceSectionReader<'data>,
@@ -530,20 +517,23 @@ pub fn parse_instance_section<'data>(
         let args = instance
             .args()?
             .into_iter()
-            .map(|result| {
-                let (kind, idx) = result?;
-                Ok(match kind {
-                    ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(idx)),
-                    ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(idx)),
-                    ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(idx)),
-                    ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(idx)),
-                    ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(idx)),
-                    ExternalKind::Instance => EntityIndex::Instance(InstanceIndex::from_u32(idx)),
+            .map(|arg| {
+                let arg = arg?;
+                let index = match arg.kind {
+                    ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(arg.index)),
+                    ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(arg.index)),
+                    ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(arg.index)),
+                    ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(arg.index)),
+                    ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(arg.index)),
+                    ExternalKind::Instance => {
+                        EntityIndex::Instance(InstanceIndex::from_u32(arg.index))
+                    }
                     ExternalKind::Event => unimplemented!(),
 
                     // this won't pass validation
                     ExternalKind::Type => unreachable!(),
-                })
+                };
+                Ok((arg.name, index))
             })
             .collect::<WasmResult<Vec<_>>>()?;
         environ.declare_instance(module, args)?;
@@ -557,19 +547,28 @@ pub fn parse_alias_section<'data>(
     environ: &mut dyn ModuleEnvironment<'data>,
 ) -> WasmResult<()> {
     for alias in section {
-        let alias = alias?;
-        let alias = match alias.instance {
-            wasmparser::AliasedInstance::Parent => {
-                match alias.kind {
-                    ExternalKind::Module => Alias::ParentModule(ModuleIndex::from_u32(alias.index)),
-                    ExternalKind::Type => Alias::ParentType(TypeIndex::from_u32(alias.index)),
-                    // shouldn't get past validation
-                    _ => unreachable!(),
-                }
-            }
-            wasmparser::AliasedInstance::Child(i) => Alias::Child {
-                instance: InstanceIndex::from_u32(i),
-                export: alias.index as usize,
+        let alias = match alias? {
+            wasmparser::Alias::OuterType {
+                relative_depth,
+                index,
+            } => Alias::OuterType {
+                relative_depth,
+                index: TypeIndex::from_u32(index),
+            },
+            wasmparser::Alias::OuterModule {
+                relative_depth,
+                index,
+            } => Alias::OuterModule {
+                relative_depth,
+                index: ModuleIndex::from_u32(index),
+            },
+            wasmparser::Alias::InstanceExport {
+                instance,
+                export,
+                kind: _,
+            } => Alias::InstanceExport {
+                instance: InstanceIndex::from_u32(instance),
+                export,
             },
         };
         environ.declare_alias(alias)?;
diff --git a/crates/bench-api/Cargo.toml b/crates/bench-api/Cargo.toml
index 02dfd986be..c5c70bf6df 100644
--- a/crates/bench-api/Cargo.toml
+++ b/crates/bench-api/Cargo.toml
@@ -16,10 +16,13 @@ crate-type = ["rlib", "cdylib"]
 
 [dependencies]
 anyhow = "1.0"
+shuffling-allocator = { version = "1.1.1", optional = true }
 wasmtime = { path = "../wasmtime", default-features = false }
 wasmtime-wasi = { path = "../wasi" }
 wasi-common = { path = "../wasi-common" }
 
-
 [dev-dependencies]
 wat = "1.0"
+
+[features]
+default = ["shuffling-allocator"]
diff --git a/crates/bench-api/src/lib.rs b/crates/bench-api/src/lib.rs
index a2958f09a8..8923b4c788 100644
--- a/crates/bench-api/src/lib.rs
+++ b/crates/bench-api/src/lib.rs
@@ -21,10 +21,16 @@
 //! # Example
 //!
 //! ```
+//! use std::ptr;
 //! use wasmtime_bench_api::*;
 //!
-//! let engine = unsafe { wasm_bench_create() };
-//! assert!(!engine.is_null());
+//! let working_dir = std::env::current_dir().unwrap().display().to_string();
+//! let mut bench_api = ptr::null_mut();
+//! unsafe {
+//!     let code = wasm_bench_create(working_dir.as_ptr(), working_dir.len(), &mut bench_api);
+//!     assert_eq!(code, OK);
+//!     assert!(!bench_api.is_null());
+//! };
 //!
 //! let wasm = wat::parse_bytes(br#"
 //!     (module
@@ -42,7 +48,7 @@
 //! "#).unwrap();
 //!
 //! // Start your compilation timer here.
-//! let code = unsafe { wasm_bench_compile(engine, wasm.as_ptr(), wasm.len()) };
+//! let code = unsafe { wasm_bench_compile(bench_api, wasm.as_ptr(), wasm.len()) };
 //! // End your compilation timer here.
 //! assert_eq!(code, OK);
 //!
@@ -57,23 +63,25 @@
 //! }
 //!
 //! // Start your instantiation timer here.
-//! let code = unsafe { wasm_bench_instantiate(engine, bench_start, bench_stop) };
+//! let code = unsafe { wasm_bench_instantiate(bench_api, bench_start, bench_stop) };
 //! // End your instantiation timer here.
 //! assert_eq!(code, OK);
 //!
 //! // No need to start timers for the execution since, by convention, the timer
 //! // functions we passed during instantiation will be called by the benchmark
 //! // at the appropriate time (before and after the benchmarked section).
-//! let code = unsafe { wasm_bench_execute(engine) };
+//! let code = unsafe { wasm_bench_execute(bench_api) };
 //! assert_eq!(code, OK);
 //!
 //! unsafe {
-//!     wasm_bench_free(engine);
+//!     wasm_bench_free(bench_api);
 //! }
 //! ```
 
 use anyhow::{anyhow, Context, Result};
+use std::env;
 use std::os::raw::{c_int, c_void};
+use std::path::Path;
 use std::slice;
 use wasi_common::WasiCtxBuilder;
 use wasmtime::{Config, Engine, Instance, Linker, Module, Store};
@@ -83,15 +91,43 @@ pub type ExitCode = c_int;
 pub const OK: ExitCode = 0;
 pub const ERR: ExitCode = -1;
 
+// Randomize the location of heap objects to avoid accidental locality being an
+// uncontrolled variable that obscures performance evaluation in our
+// experiments.
+#[cfg(feature = "shuffling-allocator")]
+#[global_allocator]
+static ALLOC: shuffling_allocator::ShufflingAllocator<std::alloc::System> =
+    shuffling_allocator::wrap!(&std::alloc::System);
+
 /// Exposes a C-compatible way of creating the engine from the bytes of a single
 /// Wasm module.
 ///
-/// This function returns a pointer to a structure that contains the engine's
-/// initialized state.
+/// On success, the `out_bench_ptr` is initialized to a pointer to a structure
+/// that contains the engine's initialized state, and `0` is returned. On
+/// failure, a non-zero status code is returned and `out_bench_ptr` is left
+/// untouched.
 #[no_mangle]
-pub extern "C" fn wasm_bench_create() -> *mut c_void {
-    let state = Box::new(BenchState::new());
-    Box::into_raw(state) as _
+pub extern "C" fn wasm_bench_create(
+    working_dir_ptr: *const u8,
+    working_dir_len: usize,
+    out_bench_ptr: *mut *mut c_void,
+) -> ExitCode {
+    let result = (|| -> Result<_> {
+        let working_dir = unsafe { std::slice::from_raw_parts(working_dir_ptr, working_dir_len) };
+        let working_dir = std::str::from_utf8(working_dir)
+            .context("given working directory is not valid UTF-8")?;
+        let state = Box::new(BenchState::new(working_dir)?);
+        Ok(Box::into_raw(state) as _)
+    })();
+
+    if let Ok(bench_ptr) = result {
+        unsafe {
+            assert!(!out_bench_ptr.is_null());
+            *out_bench_ptr = bench_ptr;
+        }
+    }
+
+    to_exit_code(result.map(|_| ()))
 }
 
 /// Free the engine state allocated by this library.
@@ -155,27 +191,49 @@ fn to_exit_code<T>(result: impl Into<Result<T>>) -> ExitCode {
 /// to manage the Wasmtime engine between calls.
 struct BenchState {
     engine: Engine,
-    store: Store,
+    linker: Linker,
     module: Option<Module>,
     instance: Option<Instance>,
     did_execute: bool,
 }
 
 impl BenchState {
-    fn new() -> Self {
+    fn new(working_dir: impl AsRef<Path>) -> Result<Self> {
         let mut config = Config::new();
         config.wasm_simd(true);
         // NB: do not configure a code cache.
 
         let engine = Engine::new(&config);
         let store = Store::new(&engine);
-        Self {
+
+        let mut linker = Linker::new(&store);
+
+        // Create a WASI environment.
+
+        let mut cx = WasiCtxBuilder::new();
+        cx.inherit_stdio();
+        // Allow access to the working directory so that the benchmark can read
+        // its input workload(s).
+        let working_dir = wasi_common::preopen_dir(working_dir)
+            .context("failed to preopen the working directory")?;
+        cx.preopened_dir(working_dir, ".");
+        // Pass this env var along so that the benchmark program can use smaller
+        // input workload(s) if it has them and that has been requested.
+        if let Ok(val) = env::var("WASM_BENCH_USE_SMALL_WORKLOAD") {
+            cx.env("WASM_BENCH_USE_SMALL_WORKLOAD", &val);
+        }
+
+        let cx = cx.build()?;
+        let wasi = Wasi::new(&store, cx);
+        wasi.add_to_linker(&mut linker)?;
+
+        Ok(Self {
             engine,
-            store,
+            linker,
             module: None,
             instance: None,
             did_execute: false,
-        }
+        })
     }
 
     fn compile(&mut self, bytes: &[u8]) -> Result<()> {
@@ -201,20 +259,11 @@ impl BenchState {
             .as_mut()
             .expect("compile the module before instantiating it");
 
-        let mut linker = Linker::new(&self.store);
-
-        // Import a very restricted WASI environment.
-        let mut cx = WasiCtxBuilder::new();
-        cx.inherit_stdio();
-        let cx = cx.build()?;
-        let wasi = Wasi::new(linker.store(), cx);
-        wasi.add_to_linker(&mut linker)?;
-
         // Import the specialized benchmarking functions.
-        linker.func("bench", "start", move || bench_start())?;
-        linker.func("bench", "end", move || bench_end())?;
+        self.linker.func("bench", "start", move || bench_start())?;
+        self.linker.func("bench", "end", move || bench_end())?;
 
-        self.instance = Some(linker.instantiate(&module)?);
+        self.instance = Some(self.linker.instantiate(&module)?);
         Ok(())
     }
 
diff --git a/crates/c-api/include/doc-wasm.h b/crates/c-api/include/doc-wasm.h
index 73d57a647b..356db1e1bc 100644
--- a/crates/c-api/include/doc-wasm.h
+++ b/crates/c-api/include/doc-wasm.h
@@ -316,6 +316,12 @@
  *
  * \fn wasm_name_new_new_uninitialized
  * \brief Convenience alias
+ * 
+ * \fn wasm_name_new_from_string
+ * \brief Create a new name from a C string.
+ * 
+ * \fn wasm_name_new_from_string_nt
+ * \brief Create a new name from a C string with null terminator.
  *
  * \fn wasm_name_copy
  * \brief Convenience alias
@@ -407,7 +413,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_valtype_t* wasm_valtype_copy(wasm_valtype_t *)
+ * \fn own wasm_valtype_t* wasm_valtype_copy(const wasm_valtype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -477,7 +483,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_functype_t* wasm_functype_copy(wasm_functype_t *)
+ * \fn own wasm_functype_t* wasm_functype_copy(const wasm_functype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -548,7 +554,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_globaltype_t* wasm_globaltype_copy(wasm_globaltype_t *)
+ * \fn own wasm_globaltype_t* wasm_globaltype_copy(const wasm_globaltype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -625,7 +631,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_tabletype_t* wasm_tabletype_copy(wasm_tabletype_t *)
+ * \fn own wasm_tabletype_t* wasm_tabletype_copy(const wasm_tabletype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -711,7 +717,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_memorytype_t* wasm_memorytype_copy(wasm_memorytype_t *)
+ * \fn own wasm_memorytype_t* wasm_memorytype_copy(const wasm_memorytype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -780,7 +786,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_externtype_t* wasm_externtype_copy(wasm_externtype_t *)
+ * \fn own wasm_externtype_t* wasm_externtype_copy(const wasm_externtype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -957,7 +963,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_importtype_t* wasm_importtype_copy(wasm_importtype_t *)
+ * \fn own wasm_importtype_t* wasm_importtype_copy(const wasm_importtype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -1038,7 +1044,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_exporttype_t* wasm_exporttype_copy(wasm_exporttype_t *)
+ * \fn own wasm_exporttype_t* wasm_exporttype_copy(const wasm_exporttype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -1520,8 +1526,6 @@
  * and types of results as the original type signature. It is undefined behavior
  * to return other types or different numbers of values.
  *
- * This function takes ownership of all of the parameters given. It's expected
- * that the caller will invoke `wasm_val_delete` for each one provided.
  * Ownership of the results and the trap returned, if any, is passed to the
  * caller of this function.
  *
@@ -1608,7 +1612,7 @@
  * \fn size_t wasm_func_result_arity(const wasm_func_t *);
  * \brief Returns the number of results returned by this function.
  *
- * \fn own wasm_trap_t *wasm_func_call(const wasm_func_t *, const wasm_val_t args[], const wasm_val_t results[]);
+* \fn own wasm_trap_t *wasm_func_call(const wasm_func_t *, const wasm_val_vec_t *args, wasm_val_vec_t *results);
  * \brief Calls the provided function with the arguments given.
  *
  * This function is used to call WebAssembly from the host. The parameter array
@@ -2164,7 +2168,7 @@
  * \fn wasm_ref_as_instance_const(const wasm_ref_t *);
  * \brief Unimplemented in Wasmtime, aborts the process if called.
  *
- * \fn own wasm_instance_t *wasm_instance_new(wasm_store_t *, const wasm_module_t *, const wasm_extern_t *const[], wasm_trap_t **);
+ * \fn own wasm_instance_t *wasm_instance_new(wasm_store_t *, const wasm_module_t *, const wasm_extern_vec_t *, wasm_trap_t **);
  * \brief Instantiates a module with the provided imports.
  *
  * This function will instantiate the provided #wasm_module_t into the provided
@@ -2194,3 +2198,29 @@
  * the same length as #wasm_module_exports called on the original module. Each
  * element is 1:1 matched with the elements in the list of #wasm_module_exports.
  */
+
+/**
+ * \def WASM_EMPTY_VEC
+ * \brief Used to initialize an empty vector type.
+ *
+ * \def WASM_ARRAY_VEC
+ * \brief Used to initialize a vector type from a C array.
+ *
+ * \def WASM_I32_VAL
+ * \brief Used to initialize a 32-bit integer wasm_val_t value.
+ *
+ * \def WASM_I64_VAL
+ * \brief Used to initialize a 64-bit integer wasm_val_t value.
+ *
+ * \def WASM_F32_VAL
+ * \brief Used to initialize a 32-bit floating point wasm_val_t value.
+ *
+ * \def WASM_F64_VAL
+ * \brief Used to initialize a 64-bit floating point wasm_val_t value.
+ *
+ * \def WASM_REF_VAL
+ * \brief Used to initialize an externref wasm_val_t value.
+ *
+ * \def WASM_INIT_VAL
+ * \brief Used to initialize a null externref wasm_val_t value.
+ */
diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h
index 769cfc1268..948552b633 100644
--- a/crates/c-api/include/wasmtime.h
+++ b/crates/c-api/include/wasmtime.h
@@ -274,6 +274,14 @@ WASMTIME_CONFIG_PROP(void, static_memory_guard_size, uint64_t)
  */
 WASMTIME_CONFIG_PROP(void, dynamic_memory_guard_size, uint64_t)
 
+/**
+ * \brief Configures the maximum number of instances that can be created.
+ *
+ * For more information see the Rust documentation at
+ * https://bytecodealliance.github.io/wasmtime/api/wasmtime/struct.Config.html#method.max_instances.
+ */
+WASMTIME_CONFIG_PROP(void, max_instances, size_t)
+
 /**
  * \brief Enables Wasmtime's cache and loads configuration from the specified
  * path.
@@ -527,7 +535,7 @@ typedef struct wasmtime_caller_t wasmtime_caller_t;
  * argument is a #wasmtime_caller_t which allows learning information about the
  * caller.
  */
-typedef own wasm_trap_t* (*wasmtime_func_callback_t)(const wasmtime_caller_t* caller, const wasm_val_t args[], wasm_val_t results[]);
+typedef own wasm_trap_t* (*wasmtime_func_callback_t)(const wasmtime_caller_t* caller, const wasm_val_vec_t *args, wasm_val_vec_t *results);
 
 /**
  * \brief Callback signature for #wasmtime_func_new_with_env.
@@ -536,7 +544,7 @@ typedef own wasm_trap_t* (*wasmtime_func_callback_t)(const wasmtime_caller_t* ca
  * first argument is a #wasmtime_caller_t which allows learning information
  * about the caller.
  */
-typedef own wasm_trap_t* (*wasmtime_func_callback_with_env_t)(const wasmtime_caller_t* caller, void* env, const wasm_val_t args[], wasm_val_t results[]);
+typedef own wasm_trap_t* (*wasmtime_func_callback_with_env_t)(const wasmtime_caller_t* caller, void* env, const wasm_val_vec_t *args, wasm_val_vec_t *results);
 
 /**
  * \brief Creates a new host-defined function.
@@ -671,7 +679,6 @@ WASM_API_EXTERN const wasm_name_t *wasmtime_frame_module_name(const wasm_frame_t
  *
  * This function is similar to #wasm_func_call, but with a few tweaks:
  *
- * * `args` and `results` have a size parameter saying how big the arrays are
  * * An error *and* a trap can be returned
  * * Errors are returned if `args` have the wrong types, if the args/results
  *   arrays have the wrong lengths, or if values come from the wrong store.
@@ -697,10 +704,8 @@ WASM_API_EXTERN const wasm_name_t *wasmtime_frame_module_name(const wasm_frame_t
  */
 WASM_API_EXTERN own wasmtime_error_t *wasmtime_func_call(
     wasm_func_t *func,
-    const wasm_val_t *args,
-    size_t num_args,
-    wasm_val_t *results,
-    size_t num_results,
+    const wasm_val_vec_t *args,
+    wasm_val_vec_t *results,
     own wasm_trap_t **trap
 );
 
@@ -741,7 +746,6 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_global_set(
  * This function is similar to #wasm_instance_new, but with a few tweaks:
  *
  * * An error message can be returned from this function.
- * * The number of imports specified is passed as an argument
  * * The `trap` pointer is required to not be NULL.
  *
  * The states of return values from this function are similar to
@@ -759,8 +763,7 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_global_set(
 WASM_API_EXTERN own wasmtime_error_t *wasmtime_instance_new(
     wasm_store_t *store,
     const wasm_module_t *module,
-    const wasm_extern_t* const imports[],
-    size_t num_imports,
+    const wasm_extern_vec_t* imports,
     own wasm_instance_t **instance,
     own wasm_trap_t **trap
 );
@@ -1016,7 +1019,7 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_module_deserialize(
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_instancetype_t* wasm_instancetype_copy(wasm_instancetype_t *)
+ * \fn own wasm_instancetype_t* wasm_instancetype_copy(const wasm_instancetype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -1113,7 +1116,7 @@ WASM_API_EXTERN const wasm_instancetype_t* wasm_externtype_as_instancetype_const
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_moduletype_t* wasm_moduletype_copy(wasm_moduletype_t *)
+ * \fn own wasm_moduletype_t* wasm_moduletype_copy(const wasm_moduletype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs
index 08da598ec1..5ca7e7a3ba 100644
--- a/crates/c-api/src/config.rs
+++ b/crates/c-api/src/config.rs
@@ -171,3 +171,8 @@ pub extern "C" fn wasmtime_config_static_memory_guard_size_set(c: &mut wasm_conf
 pub extern "C" fn wasmtime_config_dynamic_memory_guard_size_set(c: &mut wasm_config_t, size: u64) {
     c.config.dynamic_memory_guard_size(size);
 }
+
+#[no_mangle]
+pub extern "C" fn wasmtime_config_max_instances_set(c: &mut wasm_config_t, limit: usize) {
+    c.config.max_instances(limit);
+}
diff --git a/crates/c-api/src/func.rs b/crates/c-api/src/func.rs
index 5c63e9782f..87351e99ca 100644
--- a/crates/c-api/src/func.rs
+++ b/crates/c-api/src/func.rs
@@ -1,4 +1,4 @@
-use crate::{wasm_extern_t, wasm_functype_t, wasm_store_t, wasm_val_t};
+use crate::{wasm_extern_t, wasm_functype_t, wasm_store_t, wasm_val_t, wasm_val_vec_t};
 use crate::{wasm_name_t, wasm_trap_t, wasmtime_error_t};
 use anyhow::anyhow;
 use std::ffi::c_void;
@@ -21,26 +21,28 @@ pub struct wasmtime_caller_t<'a> {
     caller: Caller<'a>,
 }
 
-pub type wasm_func_callback_t =
-    extern "C" fn(args: *const wasm_val_t, results: *mut wasm_val_t) -> Option<Box<wasm_trap_t>>;
+pub type wasm_func_callback_t = extern "C" fn(
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
+) -> Option<Box<wasm_trap_t>>;
 
 pub type wasm_func_callback_with_env_t = extern "C" fn(
     env: *mut std::ffi::c_void,
-    args: *const wasm_val_t,
-    results: *mut wasm_val_t,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
 ) -> Option<Box<wasm_trap_t>>;
 
 pub type wasmtime_func_callback_t = extern "C" fn(
     caller: *const wasmtime_caller_t,
-    args: *const wasm_val_t,
-    results: *mut wasm_val_t,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
 ) -> Option<Box<wasm_trap_t>>;
 
 pub type wasmtime_func_callback_with_env_t = extern "C" fn(
     caller: *const wasmtime_caller_t,
     env: *mut std::ffi::c_void,
-    args: *const wasm_val_t,
-    results: *mut wasm_val_t,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
 ) -> Option<Box<wasm_trap_t>>;
 
 struct Finalizer {
@@ -83,21 +85,25 @@ impl From<Func> for wasm_func_t {
 fn create_function(
     store: &wasm_store_t,
     ty: &wasm_functype_t,
-    func: impl Fn(Caller<'_>, *const wasm_val_t, *mut wasm_val_t) -> Option<Box<wasm_trap_t>> + 'static,
+    func: impl Fn(Caller<'_>, *const wasm_val_vec_t, *mut wasm_val_vec_t) -> Option<Box<wasm_trap_t>>
+        + 'static,
 ) -> Box<wasm_func_t> {
     let store = &store.store;
     let ty = ty.ty().ty.clone();
     let func = Func::new(store, ty, move |caller, params, results| {
-        let params = params
+        let params: wasm_val_vec_t = params
             .iter()
             .cloned()
             .map(|p| wasm_val_t::from_val(p))
-            .collect::<Vec<_>>();
-        let mut out_results = vec![wasm_val_t::default(); results.len()];
-        let out = func(caller, params.as_ptr(), out_results.as_mut_ptr());
+            .collect::<Vec<_>>()
+            .into();
+        let mut out_results: wasm_val_vec_t = vec![wasm_val_t::default(); results.len()].into();
+        let out = func(caller, &params, &mut out_results);
         if let Some(trap) = out {
             return Err(trap.trap.clone());
         }
+
+        let out_results = out_results.as_slice();
         for i in 0..results.len() {
             results[i] = out_results[i].val();
         }
@@ -164,17 +170,14 @@ pub extern "C" fn wasmtime_func_new_with_env(
 #[no_mangle]
 pub unsafe extern "C" fn wasm_func_call(
     wasm_func: &wasm_func_t,
-    args: *const wasm_val_t,
-    results: *mut MaybeUninit<wasm_val_t>,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
 ) -> *mut wasm_trap_t {
-    let func = wasm_func.func();
     let mut trap = ptr::null_mut();
-    let error = wasmtime_func_call(
+    let error = _wasmtime_func_call(
         wasm_func,
-        args,
-        func.param_arity(),
-        results,
-        func.result_arity(),
+        (*args).as_slice(),
+        (*results).as_uninit_slice(),
         &mut trap,
     );
     match error {
@@ -186,16 +189,14 @@ pub unsafe extern "C" fn wasm_func_call(
 #[no_mangle]
 pub unsafe extern "C" fn wasmtime_func_call(
     func: &wasm_func_t,
-    args: *const wasm_val_t,
-    num_args: usize,
-    results: *mut MaybeUninit<wasm_val_t>,
-    num_results: usize,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
     trap_ptr: &mut *mut wasm_trap_t,
 ) -> Option<Box<wasmtime_error_t>> {
     _wasmtime_func_call(
         func,
-        std::slice::from_raw_parts(args, num_args),
-        std::slice::from_raw_parts_mut(results, num_results),
+        (*args).as_slice(),
+        (*results).as_uninit_slice(),
         trap_ptr,
     )
 }
diff --git a/crates/c-api/src/instance.rs b/crates/c-api/src/instance.rs
index 5a41d68a1c..9a2dd3b2ca 100644
--- a/crates/c-api/src/instance.rs
+++ b/crates/c-api/src/instance.rs
@@ -40,16 +40,15 @@ impl wasm_instance_t {
 pub unsafe extern "C" fn wasm_instance_new(
     store: &wasm_store_t,
     wasm_module: &wasm_module_t,
-    imports: *const Box<wasm_extern_t>,
+    imports: *const wasm_extern_vec_t,
     result: Option<&mut *mut wasm_trap_t>,
 ) -> Option<Box<wasm_instance_t>> {
     let mut instance = ptr::null_mut();
     let mut trap = ptr::null_mut();
-    let err = wasmtime_instance_new(
+    let err = _wasmtime_instance_new(
         store,
         wasm_module,
-        imports,
-        wasm_module.module().imports().len(),
+        (*imports).as_slice(),
         &mut instance,
         &mut trap,
     );
@@ -83,31 +82,27 @@ pub unsafe extern "C" fn wasm_instance_new(
 pub unsafe extern "C" fn wasmtime_instance_new(
     store: &wasm_store_t,
     module: &wasm_module_t,
-    imports: *const Box<wasm_extern_t>,
-    num_imports: usize,
+    imports: *const wasm_extern_vec_t,
     instance_ptr: &mut *mut wasm_instance_t,
     trap_ptr: &mut *mut wasm_trap_t,
 ) -> Option<Box<wasmtime_error_t>> {
-    _wasmtime_instance_new(
-        store,
-        module,
-        std::slice::from_raw_parts(imports, num_imports),
-        instance_ptr,
-        trap_ptr,
-    )
+    _wasmtime_instance_new(store, module, (*imports).as_slice(), instance_ptr, trap_ptr)
 }
 
 fn _wasmtime_instance_new(
     store: &wasm_store_t,
     module: &wasm_module_t,
-    imports: &[Box<wasm_extern_t>],
+    imports: &[Option<Box<wasm_extern_t>>],
     instance_ptr: &mut *mut wasm_instance_t,
     trap_ptr: &mut *mut wasm_trap_t,
 ) -> Option<Box<wasmtime_error_t>> {
     let store = &store.store;
     let imports = imports
         .iter()
-        .map(|import| import.which.clone())
+        .filter_map(|import| match import {
+            Some(i) => Some(i.which.clone()),
+            None => None,
+        })
         .collect::<Vec<_>>();
     handle_instantiate(
         Instance::new(store, module.module(), &imports),
diff --git a/crates/c-api/src/vec.rs b/crates/c-api/src/vec.rs
index 21a47f0717..fec8c7dd0d 100644
--- a/crates/c-api/src/vec.rs
+++ b/crates/c-api/src/vec.rs
@@ -4,6 +4,7 @@ use crate::{
     wasm_moduletype_t, wasm_tabletype_t, wasm_val_t, wasm_valtype_t,
 };
 use std::mem;
+use std::mem::MaybeUninit;
 use std::ptr;
 use std::slice;
 
@@ -54,6 +55,18 @@ macro_rules! declare_vecs {
                 }
             }
 
+            pub fn as_uninit_slice(&mut self) -> &mut [MaybeUninit<$elem_ty>] {
+                // Note that we're careful to not create a slice with a null
+                // pointer as the data pointer, since that isn't defined
+                // behavior in Rust.
+                if self.size == 0 {
+                    &mut []
+                } else {
+                    assert!(!self.data.is_null());
+                    unsafe { slice::from_raw_parts_mut(self.data as _, self.size) }
+                }
+            }
+
             pub fn take(&mut self) -> Vec<$elem_ty> {
                 if self.data.is_null() {
                     return Vec::new();
diff --git a/crates/c-api/wasm-c-api b/crates/c-api/wasm-c-api
index d9a80099d4..c9d3128465 160000
--- a/crates/c-api/wasm-c-api
+++ b/crates/c-api/wasm-c-api
@@ -1 +1 @@
-Subproject commit d9a80099d496b5cdba6f3fe8fc77586e0e505ddc
+Subproject commit c9d31284651b975f05ac27cee0bab1377560b87e
diff --git a/crates/cache/Cargo.toml b/crates/cache/Cargo.toml
index 970ea91b93..9995dbfb60 100644
--- a/crates/cache/Cargo.toml
+++ b/crates/cache/Cargo.toml
@@ -18,7 +18,7 @@ log = { version = "0.4.8", default-features = false }
 serde = { version = "1.0.94", features = ["derive"] }
 sha2 = "0.9.0"
 toml = "0.5.5"
-zstd = "0.5"
+zstd = { version = "0.6", default-features = false }
 
 [target.'cfg(target_os = "windows")'.dependencies]
 winapi = "0.3.7"
diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml
index 99ccc5a497..9b4e0972bb 100644
--- a/crates/cranelift/Cargo.toml
+++ b/crates/cranelift/Cargo.toml
@@ -17,3 +17,4 @@ cranelift-wasm = { path = "../../cranelift/wasm", version = "0.69.0" }
 cranelift-codegen = { path = "../../cranelift/codegen", version = "0.69.0" }
 cranelift-frontend = { path = "../../cranelift/frontend", version = "0.69.0" }
 cranelift-entity = { path = "../../cranelift/entity", version = "0.69.0" }
+wasmparser = "0.73.0"
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 6b94824f93..a9481aca93 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -7,11 +7,14 @@ use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Si
 use cranelift_codegen::isa::{self, TargetFrontendConfig};
 use cranelift_entity::{EntityRef, PrimaryMap};
 use cranelift_frontend::FunctionBuilder;
+use cranelift_frontend::Variable;
 use cranelift_wasm::{
-    self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableIndex,
-    TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType,
+    self, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, MemoryIndex,
+    SignatureIndex, TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType,
 };
 use std::convert::TryFrom;
+use std::mem;
+use wasmparser::Operator;
 use wasmtime_environ::{
     BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, TableStyle, Tunables, VMOffsets,
     INTERRUPTED, WASM_PAGE_SIZE,
@@ -125,6 +128,20 @@ pub struct FuncEnvironment<'module_environment> {
     pub(crate) offsets: VMOffsets,
 
     tunables: &'module_environment Tunables,
+
+    /// A function-local variable which stores the cached value of the amount of
+    /// fuel remaining to execute. If used this is modified frequently so it's
+    /// stored locally as a variable instead of always referenced from the field
+    /// in `*const VMInterrupts`
+    fuel_var: cranelift_frontend::Variable,
+
+    /// A function-local variable which caches the value of `*const
+    /// VMInterrupts` for this function's vmctx argument. This pointer is stored
+    /// in the vmctx itself, but never changes for the lifetime of the function,
+    /// so if we load it up front we can continue to use it throughout.
+    vminterrupts_ptr: cranelift_frontend::Variable,
+
+    fuel_consumed: i64,
 }
 
 impl<'module_environment> FuncEnvironment<'module_environment> {
@@ -151,6 +168,12 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             builtin_function_signatures,
             offsets: VMOffsets::new(target_config.pointer_bytes(), module),
             tunables,
+            fuel_var: Variable::new(0),
+            vminterrupts_ptr: Variable::new(0),
+
+            // Start with at least one fuel being consumed because even empty
+            // functions should consume at least some fuel.
+            fuel_consumed: 1,
         }
     }
 
@@ -418,6 +441,241 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             (global, 0)
         }
     }
+
+    fn declare_vminterrupts_ptr(&mut self, builder: &mut FunctionBuilder<'_>) {
+        // We load the `*const VMInterrupts` value stored within vmctx at the
+        // head of the function and reuse the same value across the entire
+        // function. This is possible since we know that the pointer never
+        // changes for the lifetime of the function.
+        let pointer_type = self.pointer_type();
+        builder.declare_var(self.vminterrupts_ptr, pointer_type);
+        let vmctx = self.vmctx(builder.func);
+        let base = builder.ins().global_value(pointer_type, vmctx);
+        let offset = i32::try_from(self.offsets.vmctx_interrupts()).unwrap();
+        let interrupt_ptr = builder
+            .ins()
+            .load(pointer_type, ir::MemFlags::trusted(), base, offset);
+        builder.def_var(self.vminterrupts_ptr, interrupt_ptr);
+    }
+
+    fn fuel_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) {
+        // On function entry we load the amount of fuel into a function-local
+        // `self.fuel_var` to make fuel modifications fast locally. This cache
+        // is then periodically flushed to the Store-defined location in
+        // `VMInterrupts` later.
+        builder.declare_var(self.fuel_var, ir::types::I64);
+        self.fuel_load_into_var(builder);
+        self.fuel_check(builder);
+    }
+
+    fn fuel_function_exit(&mut self, builder: &mut FunctionBuilder<'_>) {
+        // On exiting the function we need to be sure to save the fuel we have
+        // cached locally in `self.fuel_var` back into the Store-defined
+        // location.
+        self.fuel_save_from_var(builder);
+    }
+
+    fn fuel_before_op(
+        &mut self,
+        op: &Operator<'_>,
+        builder: &mut FunctionBuilder<'_>,
+        reachable: bool,
+    ) {
+        if !reachable {
+            // In unreachable code we shouldn't have any leftover fuel we
+            // haven't accounted for since the reason for us to become
+            // unreachable should have already added it to `self.fuel_var`.
+            debug_assert_eq!(self.fuel_consumed, 0);
+            return;
+        }
+
+        self.fuel_consumed += match op {
+            // Nop and drop generate no code, so don't consume fuel for them.
+            Operator::Nop | Operator::Drop => 0,
+
+            // Control flow may create branches, but is generally cheap and
+            // free, so don't consume fuel. Note the lack of `if` since some
+            // cost is incurred with the conditional check.
+            Operator::Block { .. }
+            | Operator::Loop { .. }
+            | Operator::Unreachable
+            | Operator::Return
+            | Operator::Else
+            | Operator::End => 0,
+
+            // everything else, just call it one operation.
+            _ => 1,
+        };
+
+        match op {
+            // Exiting a function (via a return or unreachable) or otherwise
+            // entering a different function (via a call) means that we need to
+            // update the fuel consumption in `VMInterrupts` because we're
+            // about to move control out of this function itself and the fuel
+            // may need to be read.
+            //
+            // Before this we need to update the fuel counter from our own cost
+            // leading up to this function call, and then we can store
+            // `self.fuel_var` into `VMInterrupts`.
+            Operator::Unreachable
+            | Operator::Return
+            | Operator::CallIndirect { .. }
+            | Operator::Call { .. }
+            | Operator::ReturnCall { .. }
+            | Operator::ReturnCallIndirect { .. } => {
+                self.fuel_increment_var(builder);
+                self.fuel_save_from_var(builder);
+            }
+
+            // To ensure all code preceding a loop is only counted once we
+            // update the fuel variable on entry.
+            Operator::Loop { .. }
+
+            // Entering into an `if` block means that the edge we take isn't
+            // known until runtime, so we need to update our fuel consumption
+            // before we take the branch.
+            | Operator::If { .. }
+
+            // Control-flow instructions mean that we're moving to the end/exit
+            // of a block somewhere else. That means we need to update the fuel
+            // counter since we're effectively terminating our basic block.
+            | Operator::Br { .. }
+            | Operator::BrIf { .. }
+            | Operator::BrTable { .. }
+
+            // Exiting a scope means that we need to update the fuel
+            // consumption because there are multiple ways to exit a scope and
+            // this is the only time we have to account for instructions
+            // executed so far.
+            | Operator::End
+
+            // This is similar to `end`, except that it's only the terminator
+            // for an `if` block. The same reasoning applies though in that we
+            // are terminating a basic block and need to update the fuel
+            // variable.
+            | Operator::Else => self.fuel_increment_var(builder),
+
+            // This is a normal instruction where the fuel is buffered to later
+            // get added to `self.fuel_var`.
+            //
+            // Note that we generally ignore instructions which may trap and
+            // therefore result in exiting a block early. Current usage of fuel
+            // means that it's not too important to account for a precise amount
+            // of fuel consumed but rather "close to the actual amount" is good
+            // enough. For 100% precise counting, however, we'd probably need to
+            // not only increment but also save the fuel amount more often
+            // around trapping instructions. (see the `unreachable` instruction
+            // case above)
+            //
+            // Note that `Block` is specifically omitted from incrementing the
+            // fuel variable. Control flow entering a `block` is unconditional
+            // which means it's effectively executing straight-line code. We'll
+            // update the counter when exiting a block, but we shouldn't need to
+            // do so upon entering a block.
+            _ => {}
+        }
+    }
+
+    fn fuel_after_op(&mut self, op: &Operator<'_>, builder: &mut FunctionBuilder<'_>) {
+        // After a function call we need to reload our fuel value since the
+        // function may have changed it.
+        match op {
+            Operator::Call { .. } | Operator::CallIndirect { .. } => {
+                self.fuel_load_into_var(builder);
+            }
+            _ => {}
+        }
+    }
+
+    /// Adds `self.fuel_consumed` to the `fuel_var`, zero-ing out the amount of
+    /// fuel consumed at that point.
+    fn fuel_increment_var(&mut self, builder: &mut FunctionBuilder<'_>) {
+        let consumption = mem::replace(&mut self.fuel_consumed, 0);
+        if consumption == 0 {
+            return;
+        }
+
+        let fuel = builder.use_var(self.fuel_var);
+        let fuel = builder.ins().iadd_imm(fuel, consumption);
+        builder.def_var(self.fuel_var, fuel);
+    }
+
+    /// Loads the fuel consumption value from `VMInterrupts` into `self.fuel_var`
+    fn fuel_load_into_var(&mut self, builder: &mut FunctionBuilder<'_>) {
+        let (addr, offset) = self.fuel_addr_offset(builder);
+        let fuel = builder
+            .ins()
+            .load(ir::types::I64, ir::MemFlags::trusted(), addr, offset);
+        builder.def_var(self.fuel_var, fuel);
+    }
+
+    /// Stores the fuel consumption value from `self.fuel_var` into
+    /// `VMInterrupts`.
+    fn fuel_save_from_var(&mut self, builder: &mut FunctionBuilder<'_>) {
+        let (addr, offset) = self.fuel_addr_offset(builder);
+        let fuel_consumed = builder.use_var(self.fuel_var);
+        builder
+            .ins()
+            .store(ir::MemFlags::trusted(), fuel_consumed, addr, offset);
+    }
+
+    /// Returns the `(address, offset)` of the fuel consumption within
+    /// `VMInterrupts`, used to perform loads/stores later.
+    fn fuel_addr_offset(
+        &mut self,
+        builder: &mut FunctionBuilder<'_>,
+    ) -> (ir::Value, ir::immediates::Offset32) {
+        (
+            builder.use_var(self.vminterrupts_ptr),
+            i32::from(self.offsets.vminterrupts_fuel_consumed()).into(),
+        )
+    }
+
+    /// Checks the amount of remaining, and if we've run out of fuel we call
+    /// the out-of-fuel function.
+    fn fuel_check(&mut self, builder: &mut FunctionBuilder) {
+        self.fuel_increment_var(builder);
+        let out_of_gas_block = builder.create_block();
+        let continuation_block = builder.create_block();
+
+        // Note that our fuel is encoded as adding positive values to a
+        // negative number. Whenever the negative number goes positive that
+        // means we ran out of fuel.
+        //
+        // Compare to see if our fuel is positive, and if so we ran out of gas.
+        // Otherwise we can continue on like usual.
+        let zero = builder.ins().iconst(ir::types::I64, 0);
+        let fuel = builder.use_var(self.fuel_var);
+        let cmp = builder.ins().ifcmp(fuel, zero);
+        builder
+            .ins()
+            .brif(IntCC::SignedGreaterThanOrEqual, cmp, out_of_gas_block, &[]);
+        builder.ins().jump(continuation_block, &[]);
+        builder.seal_block(out_of_gas_block);
+
+        // If we ran out of gas then we call our out-of-gas intrinsic and it
+        // figures out what to do. Note that this may raise a trap, or do
+        // something like yield to an async runtime. In either case we don't
+        // assume what happens and handle the case the intrinsic returns.
+        //
+        // Note that we save/reload fuel around this since the out-of-gas
+        // intrinsic may alter how much fuel is in the system.
+        builder.switch_to_block(out_of_gas_block);
+        self.fuel_save_from_var(builder);
+        let out_of_gas_sig = self.builtin_function_signatures.out_of_gas(builder.func);
+        let (vmctx, out_of_gas) = self.translate_load_builtin_function_address(
+            &mut builder.cursor(),
+            BuiltinFunctionIndex::out_of_gas(),
+        );
+        builder
+            .ins()
+            .call_indirect(out_of_gas_sig, out_of_gas, &[vmctx]);
+        self.fuel_load_into_var(builder);
+        builder.ins().jump(continuation_block, &[]);
+        builder.seal_block(continuation_block);
+
+        builder.switch_to_block(continuation_block);
+    }
 }
 
 impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environment> {
@@ -437,6 +695,11 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         index >= 2
     }
 
+    fn after_locals(&mut self, num_locals: usize) {
+        self.vminterrupts_ptr = Variable::new(num_locals);
+        self.fuel_var = Variable::new(num_locals + 1);
+    }
+
     fn make_table(&mut self, func: &mut ir::Function, index: TableIndex) -> WasmResult<ir::Table> {
         let pointer_type = self.pointer_type();
 
@@ -1482,36 +1745,90 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         Ok(*pos.func.dfg.inst_results(call_inst).first().unwrap())
     }
 
-    fn translate_loop_header(&mut self, mut pos: FuncCursor) -> WasmResult<()> {
-        if !self.tunables.interruptable {
-            return Ok(());
-        }
-
-        // Start out each loop with a check to the interupt flag to allow
-        // interruption of long or infinite loops.
+    fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {
+        // If enabled check the interrupt flag to prevent long or infinite
+        // loops.
         //
         // For more information about this see comments in
         // `crates/environ/src/cranelift.rs`
-        let vmctx = self.vmctx(&mut pos.func);
-        let pointer_type = self.pointer_type();
-        let base = pos.ins().global_value(pointer_type, vmctx);
-        let offset = i32::try_from(self.offsets.vmctx_interrupts()).unwrap();
-        let interrupt_ptr = pos
-            .ins()
-            .load(pointer_type, ir::MemFlags::trusted(), base, offset);
-        let interrupt = pos.ins().load(
-            pointer_type,
-            ir::MemFlags::trusted(),
-            interrupt_ptr,
-            i32::from(self.offsets.vminterrupts_stack_limit()),
-        );
-        // Note that the cast to `isize` happens first to allow sign-extension,
-        // if necessary, to `i64`.
-        let interrupted_sentinel = pos.ins().iconst(pointer_type, INTERRUPTED as isize as i64);
-        let cmp = pos
-            .ins()
-            .icmp(IntCC::Equal, interrupt, interrupted_sentinel);
-        pos.ins().trapnz(cmp, ir::TrapCode::Interrupt);
+        if self.tunables.interruptable {
+            let pointer_type = self.pointer_type();
+            let interrupt_ptr = builder.use_var(self.vminterrupts_ptr);
+            let interrupt = builder.ins().load(
+                pointer_type,
+                ir::MemFlags::trusted(),
+                interrupt_ptr,
+                i32::from(self.offsets.vminterrupts_stack_limit()),
+            );
+            // Note that the cast to `isize` happens first to allow sign-extension,
+            // if necessary, to `i64`.
+            let interrupted_sentinel = builder
+                .ins()
+                .iconst(pointer_type, INTERRUPTED as isize as i64);
+            let cmp = builder
+                .ins()
+                .icmp(IntCC::Equal, interrupt, interrupted_sentinel);
+            builder.ins().trapnz(cmp, ir::TrapCode::Interrupt);
+        }
+
+        // Additionally if enabled check how much fuel we have remaining to see
+        // if we've run out by this point.
+        if self.tunables.consume_fuel {
+            self.fuel_check(builder);
+        }
+
+        Ok(())
+    }
+
+    fn before_translate_operator(
+        &mut self,
+        op: &Operator,
+        builder: &mut FunctionBuilder,
+        state: &FuncTranslationState,
+    ) -> WasmResult<()> {
+        if self.tunables.consume_fuel {
+            self.fuel_before_op(op, builder, state.reachable());
+        }
+        Ok(())
+    }
+
+    fn after_translate_operator(
+        &mut self,
+        op: &Operator,
+        builder: &mut FunctionBuilder,
+        state: &FuncTranslationState,
+    ) -> WasmResult<()> {
+        if self.tunables.consume_fuel && state.reachable() {
+            self.fuel_after_op(op, builder);
+        }
+        Ok(())
+    }
+
+    fn before_translate_function(
+        &mut self,
+        builder: &mut FunctionBuilder,
+        _state: &FuncTranslationState,
+    ) -> WasmResult<()> {
+        // If the `vminterrupts_ptr` variable will get used then we initialize
+        // it here.
+        if self.tunables.consume_fuel || self.tunables.interruptable {
+            self.declare_vminterrupts_ptr(builder);
+        }
+        // Additionally we initialize `fuel_var` if it will get used.
+        if self.tunables.consume_fuel {
+            self.fuel_function_entry(builder);
+        }
+        Ok(())
+    }
+
+    fn after_translate_function(
+        &mut self,
+        builder: &mut FunctionBuilder,
+        state: &FuncTranslationState,
+    ) -> WasmResult<()> {
+        if self.tunables.consume_fuel && state.reachable() {
+            self.fuel_function_exit(builder);
+        }
         Ok(())
     }
 }
diff --git a/crates/debug/Cargo.toml b/crates/debug/Cargo.toml
index 523c136cba..28e1297ea8 100644
--- a/crates/debug/Cargo.toml
+++ b/crates/debug/Cargo.toml
@@ -13,8 +13,8 @@ edition = "2018"
 
 [dependencies]
 gimli = "0.23.0"
-wasmparser = "0.71"
-object = { version = "0.22.0", default-features = false, features = ["read_core", "elf", "write"] }
+wasmparser = "0.73"
+object = { version = "0.23.0", default-features = false, features = ["read_core", "elf", "write"] }
 wasmtime-environ = { path = "../environ", version = "0.22.0" }
 target-lexicon = { version = "0.11.0", default-features = false }
 anyhow = "1.0"
diff --git a/crates/debug/src/transform/expression.rs b/crates/debug/src/transform/expression.rs
index 0a286aa616..57f3cc7a21 100644
--- a/crates/debug/src/transform/expression.rs
+++ b/crates/debug/src/transform/expression.rs
@@ -7,7 +7,7 @@ use std::collections::{HashMap, HashSet};
 use std::hash::{Hash, Hasher};
 use std::rc::Rc;
 use wasmtime_environ::entity::EntityRef;
-use wasmtime_environ::ir::{StackSlots, ValueLabel, ValueLabelsRanges, ValueLoc};
+use wasmtime_environ::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLabelsRanges, ValueLoc};
 use wasmtime_environ::isa::TargetIsa;
 use wasmtime_environ::wasm::{get_vmctx_value_label, DefinedFuncIndex};
 use wasmtime_environ::ModuleMemoryOffset;
@@ -131,27 +131,24 @@ impl CompiledExpression {
 const X86_64_STACK_OFFSET: i64 = 16;
 
 fn translate_loc(
-    loc: ValueLoc,
+    loc: LabelValueLoc,
     frame_info: Option<&FunctionFrameInfo>,
     isa: &dyn TargetIsa,
     add_stack_value: bool,
 ) -> Result<Option<Vec<u8>>> {
     Ok(match loc {
-        ValueLoc::Reg(reg) if add_stack_value => {
+        LabelValueLoc::ValueLoc(ValueLoc::Reg(reg)) => {
             let machine_reg = isa.map_dwarf_register(reg)?;
             let mut writer = ExpressionWriter::new();
-            writer.write_op_reg(machine_reg)?;
+            if add_stack_value {
+                writer.write_op_reg(machine_reg)?;
+            } else {
+                writer.write_op_breg(machine_reg)?;
+                writer.write_sleb128(0)?;
+            }
             Some(writer.into_vec())
         }
-        ValueLoc::Reg(reg) => {
-            assert!(!add_stack_value);
-            let machine_reg = isa.map_dwarf_register(reg)?;
-            let mut writer = ExpressionWriter::new();
-            writer.write_op_breg(machine_reg)?;
-            writer.write_sleb128(0)?;
-            Some(writer.into_vec())
-        }
-        ValueLoc::Stack(ss) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => {
             if let Some(frame_info) = frame_info {
                 if let Some(ss_offset) = frame_info.stack_slots[ss].offset {
                     let mut writer = ExpressionWriter::new();
@@ -165,6 +162,27 @@ fn translate_loc(
             }
             None
         }
+        LabelValueLoc::Reg(r) => {
+            let machine_reg = isa.map_regalloc_reg_to_dwarf(r)?;
+            let mut writer = ExpressionWriter::new();
+            if add_stack_value {
+                writer.write_op_reg(machine_reg)?;
+            } else {
+                writer.write_op_breg(machine_reg)?;
+                writer.write_sleb128(0)?;
+            }
+            Some(writer.into_vec())
+        }
+        LabelValueLoc::SPOffset(off) => {
+            let mut writer = ExpressionWriter::new();
+            writer.write_op_breg(X86_64::RSP.0)?;
+            writer.write_sleb128(off)?;
+            if !add_stack_value {
+                writer.write_op(gimli::constants::DW_OP_deref)?;
+            }
+            return Ok(Some(writer.into_vec()));
+        }
+
         _ => None,
     })
 }
@@ -172,13 +190,13 @@ fn translate_loc(
 fn append_memory_deref(
     buf: &mut Vec<u8>,
     frame_info: &FunctionFrameInfo,
-    vmctx_loc: ValueLoc,
+    vmctx_loc: LabelValueLoc,
     isa: &dyn TargetIsa,
 ) -> Result<bool> {
     let mut writer = ExpressionWriter::new();
     // FIXME for imported memory
     match vmctx_loc {
-        ValueLoc::Reg(vmctx_reg) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Reg(vmctx_reg)) => {
             let reg = isa.map_dwarf_register(vmctx_reg)? as u8;
             writer.write_u8(gimli::constants::DW_OP_breg0.0 + reg)?;
             let memory_offset = match frame_info.vmctx_memory_offset() {
@@ -189,7 +207,7 @@ fn append_memory_deref(
             };
             writer.write_sleb128(memory_offset)?;
         }
-        ValueLoc::Stack(ss) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => {
             if let Some(ss_offset) = frame_info.stack_slots[ss].offset {
                 writer.write_op_breg(X86_64::RBP.0)?;
                 writer.write_sleb128(ss_offset as i64 + X86_64_STACK_OFFSET)?;
@@ -207,6 +225,31 @@ fn append_memory_deref(
                 return Ok(false);
             }
         }
+        LabelValueLoc::Reg(r) => {
+            let reg = isa.map_regalloc_reg_to_dwarf(r)?;
+            writer.write_op_breg(reg)?;
+            let memory_offset = match frame_info.vmctx_memory_offset() {
+                Some(offset) => offset,
+                None => {
+                    return Ok(false);
+                }
+            };
+            writer.write_sleb128(memory_offset)?;
+        }
+        LabelValueLoc::SPOffset(off) => {
+            writer.write_op_breg(X86_64::RSP.0)?;
+            writer.write_sleb128(off)?;
+            writer.write_op(gimli::constants::DW_OP_deref)?;
+            writer.write_op(gimli::constants::DW_OP_consts)?;
+            let memory_offset = match frame_info.vmctx_memory_offset() {
+                Some(offset) => offset,
+                None => {
+                    return Ok(false);
+                }
+            };
+            writer.write_sleb128(memory_offset)?;
+            writer.write_op(gimli::constants::DW_OP_plus)?;
+        }
         _ => {
             return Ok(false);
         }
@@ -468,7 +511,7 @@ where
                 let _ = code_chunk; // suppresses warning for final flush
             }
         };
-    };
+    }
     // Find all landing pads by scanning bytes, do not care about
     // false location at this moment.
     // Looks hacky but it is fast; does not need to be really exact.
@@ -653,7 +696,7 @@ struct CachedValueLabelRange {
     func_index: DefinedFuncIndex,
     start: usize,
     end: usize,
-    label_location: HashMap<ValueLabel, ValueLoc>,
+    label_location: HashMap<ValueLabel, LabelValueLoc>,
 }
 
 struct ValueLabelRangesBuilder<'a, 'b> {
@@ -1179,7 +1222,7 @@ mod tests {
     fn create_mock_value_ranges() -> (ValueLabelsRanges, (ValueLabel, ValueLabel, ValueLabel)) {
         use std::collections::HashMap;
         use wasmtime_environ::entity::EntityRef;
-        use wasmtime_environ::ir::{ValueLoc, ValueLocRange};
+        use wasmtime_environ::ir::{LabelValueLoc, ValueLoc, ValueLocRange};
         let mut value_ranges = HashMap::new();
         let value_0 = ValueLabel::new(0);
         let value_1 = ValueLabel::new(1);
@@ -1187,7 +1230,7 @@ mod tests {
         value_ranges.insert(
             value_0,
             vec![ValueLocRange {
-                loc: ValueLoc::Unassigned,
+                loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned),
                 start: 0,
                 end: 25,
             }],
@@ -1195,7 +1238,7 @@ mod tests {
         value_ranges.insert(
             value_1,
             vec![ValueLocRange {
-                loc: ValueLoc::Unassigned,
+                loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned),
                 start: 5,
                 end: 30,
             }],
@@ -1204,12 +1247,12 @@ mod tests {
             value_2,
             vec![
                 ValueLocRange {
-                    loc: ValueLoc::Unassigned,
+                    loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned),
                     start: 0,
                     end: 10,
                 },
                 ValueLocRange {
-                    loc: ValueLoc::Unassigned,
+                    loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned),
                     start: 20,
                     end: 30,
                 },
diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml
index caa6ba3b58..81b775402c 100644
--- a/crates/environ/Cargo.toml
+++ b/crates/environ/Cargo.toml
@@ -16,7 +16,7 @@ anyhow = "1.0"
 cranelift-codegen = { path = "../../cranelift/codegen", version = "0.69.0", features = ["enable-serde"] }
 cranelift-entity = { path = "../../cranelift/entity", version = "0.69.0", features = ["enable-serde"] }
 cranelift-wasm = { path = "../../cranelift/wasm", version = "0.69.0", features = ["enable-serde"] }
-wasmparser = "0.71"
+wasmparser = "0.73"
 indexmap = { version = "1.0.2", features = ["serde-1"] }
 thiserror = "1.0.4"
 serde = { version = "1.0.94", features = ["derive"] }
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index f2bb330217..1bbc064241 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -57,6 +57,8 @@ macro_rules! foreach_builtin_function {
             memory_atomic_wait64(vmctx, i32, i32, i64, i64) -> (i32);
             /// Returns an index for wasm's `memory.atomic.wait64` for imported memories.
             imported_memory_atomic_wait64(vmctx, i32, i32, i64, i64) -> (i32);
+            /// Invoked when fuel has run out while executing a function.
+            out_of_gas(vmctx) -> ();
         }
     };
 }
diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs
index 9e1a164675..2fc9e9ecd0 100644
--- a/crates/environ/src/data_structures.rs
+++ b/crates/environ/src/data_structures.rs
@@ -3,8 +3,8 @@
 pub mod ir {
     pub use cranelift_codegen::binemit::{Reloc, StackMap};
     pub use cranelift_codegen::ir::{
-        types, AbiParam, ArgumentPurpose, JumpTableOffsets, LibCall, Signature, SourceLoc,
-        StackSlots, TrapCode, Type, ValueLabel, ValueLoc,
+        types, AbiParam, ArgumentPurpose, JumpTableOffsets, LabelValueLoc, LibCall, Signature,
+        SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc,
     };
     pub use cranelift_codegen::{ValueLabelsRanges, ValueLocRange};
 }
diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs
index 226d96134d..b25682acf2 100644
--- a/crates/environ/src/module.rs
+++ b/crates/environ/src/module.rs
@@ -142,12 +142,6 @@ impl ModuleType {
 /// memory initializers.
 #[derive(Default, Debug, Clone, Serialize, Deserialize)]
 pub struct Module {
-    /// The parent index of this module, used for the module linking proposal.
-    ///
-    /// This index is into the list of modules returned from compilation of a
-    /// single wasm file with nested modules.
-    pub parent: Option<usize>,
-
     /// The name of this wasm module, often found in the wasm file.
     pub name: Option<String>,
 
@@ -213,25 +207,26 @@ pub struct Module {
 pub enum Initializer {
     /// An imported item is required to be provided.
     Import {
-        /// Module name of this import
-        module: String,
-        /// Optional field name of this import
+        /// Name of this import
+        name: String,
+        /// The field name projection of this import. When module-linking is
+        /// enabled this is always `None`. Otherwise this is always `Some`.
         field: Option<String>,
         /// Where this import will be placed, which also has type information
         /// about the import.
         index: EntityIndex,
     },
 
-    /// A module from the parent's declared modules is inserted into our own
+    /// An export from a previously defined instance is being inserted into our
     /// index space.
-    AliasParentModule(ModuleIndex),
-
-    /// A module from the parent's declared modules is inserted into our own
-    /// index space.
-    #[allow(missing_docs)]
+    ///
+    /// Note that when the module linking proposal is enabled two-level imports
+    /// will implicitly desugar to this initializer.
     AliasInstanceExport {
+        /// The instance that we're referencing.
         instance: InstanceIndex,
-        export: usize,
+        /// Which export is being inserted into our index space.
+        export: String,
     },
 
     /// A module is being instantiated with previously configured intializers
@@ -239,15 +234,36 @@ pub enum Initializer {
     Instantiate {
         /// The module that this instance is instantiating.
         module: ModuleIndex,
-        /// The arguments provided to instantiation.
-        args: Vec<EntityIndex>,
+        /// The arguments provided to instantiation, along with their name in
+        /// the instance being instantiated.
+        args: IndexMap<String, EntityIndex>,
     },
 
-    /// A module is defined into the module index space, and which module is
-    /// being defined is specified by the index payload.
+    /// A module is being created from a set of compiled artifacts.
+    CreateModule {
+        /// The index of the artifact that's being convereted into a module.
+        artifact_index: usize,
+        /// The list of artifacts that this module value will be inheriting.
+        artifacts: Vec<usize>,
+        /// The list of modules that this module value will inherit.
+        modules: Vec<ModuleUpvar>,
+    },
+
+    /// A module is created from a closed-over-module value, defined when this
+    /// module was created.
     DefineModule(usize),
 }
 
+/// Where module values can come from when creating a new module from a compiled
+/// artifact.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum ModuleUpvar {
+    /// A module value is inherited from the module creating the new module.
+    Inherit(usize),
+    /// A module value comes from the instance-to-be-created module index space.
+    Local(ModuleIndex),
+}
+
 impl Module {
     /// Allocates the module data structures.
     pub fn new() -> Self {
@@ -351,11 +367,9 @@ impl Module {
     /// module name, field name, and type that's being imported.
     pub fn imports(&self) -> impl Iterator<Item = (&str, Option<&str>, EntityType)> {
         self.initializers.iter().filter_map(move |i| match i {
-            Initializer::Import {
-                module,
-                field,
-                index,
-            } => Some((module.as_str(), field.as_deref(), self.type_of(*index))),
+            Initializer::Import { name, field, index } => {
+                Some((name.as_str(), field.as_deref(), self.type_of(*index)))
+            }
             _ => None,
         })
     }
@@ -389,16 +403,16 @@ pub struct TypeTables {
 /// The type signature of known modules.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ModuleSignature {
-    /// All imports in this module, listed in order with their module/name and
+    /// All imports in this module, listed in order with their name and
     /// what type they're importing.
-    pub imports: Vec<(String, Option<String>, EntityType)>,
+    pub imports: IndexMap<String, EntityType>,
     /// Exports are what an instance type conveys, so we go through an
     /// indirection over there.
     pub exports: InstanceTypeIndex,
 }
 
 /// The type signature of known instances.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct InstanceSignature {
     /// The name of what's being exported as well as its type signature.
     pub exports: IndexMap<String, EntityType>,
diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs
index 7b2cca6110..9d1f89cb1f 100644
--- a/crates/environ/src/module_environ.rs
+++ b/crates/environ/src/module_environ.rs
@@ -1,6 +1,6 @@
 use crate::module::{
-    Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, TableElements,
-    TablePlan, TypeTables,
+    Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, ModuleUpvar,
+    TableElements, TablePlan, TypeTables,
 };
 use crate::tunables::Tunables;
 use cranelift_codegen::ir;
@@ -14,7 +14,7 @@ use cranelift_wasm::{
     WasmError, WasmFuncType, WasmResult,
 };
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{hash_map::Entry, HashMap};
 use std::convert::TryFrom;
 use std::mem;
 use std::path::PathBuf;
@@ -31,6 +31,10 @@ pub struct ModuleEnvironment<'data> {
     /// the module linking proposal.
     results: Vec<ModuleTranslation<'data>>,
 
+    /// Modules which are in-progress being translated, or otherwise also known
+    /// as the outer modules of the current module being processed.
+    in_progress: Vec<ModuleTranslation<'data>>,
+
     /// How many modules that have not yet made their way into `results` which
     /// are coming at some point.
     modules_to_be: usize,
@@ -38,13 +42,11 @@ pub struct ModuleEnvironment<'data> {
     /// Intern'd types for this entire translation, shared by all modules.
     types: TypeTables,
 
-    /// Where our module will get pushed into `results` after it's finished.
-    cur: usize,
-
     // Various bits and pieces of configuration
     features: WasmFeatures,
     target_config: TargetFrontendConfig,
     tunables: Tunables,
+    first_module: bool,
 }
 
 /// The result of translating via `ModuleEnvironment`. Function bodies are not
@@ -72,15 +74,15 @@ pub struct ModuleTranslation<'data> {
     /// which function is currently being defined.
     code_index: u32,
 
-    /// When local modules are declared an entry is pushed onto this list which
-    /// indicates that the initializer at the specified position needs to be
-    /// rewritten with the module's final index in the global list of compiled
-    /// modules.
-    module_initializer_indexes: Vec<usize>,
+    implicit_instances: HashMap<&'data str, InstanceIndex>,
 
-    /// Used as a pointer into the above list as the module code section is
-    /// parsed.
-    num_modules_defined: usize,
+    /// The artifacts which are needed from the parent module when this module
+    /// is created. This is used to insert into `Initializer::CreateModule` when
+    /// this module is defined in the parent.
+    creation_artifacts: Vec<usize>,
+
+    /// Same as `creation_artifacts`, but for modules instead of artifacts.
+    creation_modules: Vec<ModuleUpvar>,
 }
 
 /// Contains function data: byte code and its offset in the module.
@@ -142,12 +144,13 @@ impl<'data> ModuleEnvironment<'data> {
         Self {
             result: ModuleTranslation::default(),
             results: Vec::with_capacity(1),
+            in_progress: Vec::new(),
             modules_to_be: 1,
-            cur: 0,
             types: Default::default(),
             target_config,
             tunables: tunables.clone(),
             features: *features,
+            first_module: true,
         }
     }
 
@@ -163,7 +166,8 @@ impl<'data> ModuleEnvironment<'data> {
     /// Note that for MVP modules this will always be a list with one element,
     /// but with the module linking proposal this may have many elements.
     ///
-    /// For the module linking proposal the top-level module is at index 0.
+    /// For the module linking proposal the top-level module is returned as the
+    /// first return value.
     ///
     /// The `TypeTables` structure returned contains intern'd versions of types
     /// referenced from each module translation. This primarily serves as the
@@ -173,10 +177,10 @@ impl<'data> ModuleEnvironment<'data> {
     pub fn translate(
         mut self,
         data: &'data [u8],
-    ) -> WasmResult<(Vec<ModuleTranslation<'data>>, TypeTables)> {
+    ) -> WasmResult<(usize, Vec<ModuleTranslation<'data>>, TypeTables)> {
         translate_module(data, &mut self)?;
         assert!(self.results.len() > 0);
-        Ok((self.results, self.types))
+        Ok((self.results.len() - 1, self.results, self.types))
     }
 
     fn declare_export(&mut self, export: EntityIndex, name: &str) -> WasmResult<()> {
@@ -224,6 +228,136 @@ impl<'data> ModuleEnvironment<'data> {
         dwarf.ranges = gimli::RangeLists::new(info.debug_ranges, info.debug_rnglists);
         dwarf.locations = gimli::LocationLists::new(info.debug_loc, info.debug_loclists);
     }
+
+    /// Declares a new import with the `module` and `field` names, importing the
+    /// `ty` specified.
+    ///
+    /// Note that this method is somewhat tricky due to the implementation of
+    /// the module linking proposal. In the module linking proposal two-level
+    /// imports are recast as single-level imports of instances. That recasting
+    /// happens here by recording an import of an instance for the first time
+    /// we see a two-level import.
+    ///
+    /// When the module linking proposal is disabled, however, disregard this
+    /// logic and instead work directly with two-level imports since no
+    /// instances are defined.
+    fn declare_import(&mut self, module: &'data str, field: Option<&'data str>, ty: EntityType) {
+        if !self.features.module_linking {
+            assert!(field.is_some());
+            let index = self.push_type(ty);
+            self.result.module.initializers.push(Initializer::Import {
+                name: module.to_owned(),
+                field: field.map(|s| s.to_string()),
+                index,
+            });
+            return;
+        }
+
+        match field {
+            Some(field) => {
+                // If this is a two-level import then this is actually an
+                // implicit import of an instance, where each two-level import
+                // is an alias directive from the original instance. The first
+                // thing we do here is lookup our implicit instance, creating a
+                // blank one if it wasn't already created.
+                let instance = match self.result.implicit_instances.entry(module) {
+                    Entry::Occupied(e) => *e.get(),
+                    Entry::Vacant(v) => {
+                        let ty = self
+                            .types
+                            .instance_signatures
+                            .push(InstanceSignature::default());
+                        let idx = self.result.module.instances.push(ty);
+                        self.result.module.initializers.push(Initializer::Import {
+                            name: module.to_owned(),
+                            field: None,
+                            index: EntityIndex::Instance(idx),
+                        });
+                        *v.insert(idx)
+                    }
+                };
+
+                // Update the implicit instance's type signature with this new
+                // field and its type.
+                self.types.instance_signatures[self.result.module.instances[instance]]
+                    .exports
+                    .insert(field.to_string(), ty.clone());
+
+                // Record our implicit alias annotation which corresponds to
+                // this import that we're processing.
+                self.result
+                    .module
+                    .initializers
+                    .push(Initializer::AliasInstanceExport {
+                        instance,
+                        export: field.to_string(),
+                    });
+
+                // And then record the type information for the item that we're
+                // processing.
+                self.push_type(ty);
+            }
+            None => {
+                // Without a field then this is a single-level import (a feature
+                // of module linking) which means we're simply importing that
+                // name with the specified type. Record the type information and
+                // then the name that we're importing.
+                let index = self.push_type(ty);
+                self.result.module.initializers.push(Initializer::Import {
+                    name: module.to_owned(),
+                    field: None,
+                    index,
+                });
+            }
+        }
+    }
+
+    fn push_type(&mut self, ty: EntityType) -> EntityIndex {
+        match ty {
+            EntityType::Function(ty) => {
+                EntityIndex::Function(self.result.module.functions.push(ty))
+            }
+            EntityType::Table(ty) => {
+                let plan = TablePlan::for_table(ty, &self.tunables);
+                EntityIndex::Table(self.result.module.table_plans.push(plan))
+            }
+            EntityType::Memory(ty) => {
+                let plan = MemoryPlan::for_memory(ty, &self.tunables);
+                EntityIndex::Memory(self.result.module.memory_plans.push(plan))
+            }
+            EntityType::Global(ty) => EntityIndex::Global(self.result.module.globals.push(ty)),
+            EntityType::Instance(ty) => {
+                EntityIndex::Instance(self.result.module.instances.push(ty))
+            }
+            EntityType::Module(ty) => EntityIndex::Module(self.result.module.modules.push(ty)),
+            EntityType::Event(_) => unimplemented!(),
+        }
+    }
+
+    fn gen_type_of_module(&mut self, module: &Module) -> ModuleTypeIndex {
+        let imports = module
+            .imports()
+            .map(|(s, field, ty)| {
+                assert!(field.is_none());
+                (s.to_string(), ty)
+            })
+            .collect();
+        let exports = module
+            .exports
+            .iter()
+            .map(|(name, idx)| (name.clone(), module.type_of(*idx)))
+            .collect();
+
+        // FIXME(#2469): this instance/module signature insertion should likely
+        // be deduplicated.
+        let exports = self
+            .types
+            .instance_signatures
+            .push(InstanceSignature { exports });
+        self.types
+            .module_signatures
+            .push(ModuleSignature { imports, exports })
+    }
 }
 
 impl<'data> TargetEnvironment for ModuleEnvironment<'data> {
@@ -267,13 +401,29 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
 
     fn declare_type_module(
         &mut self,
-        imports: &[(&'data str, Option<&'data str>, EntityType)],
+        declared_imports: &[(&'data str, Option<&'data str>, EntityType)],
         exports: &[(&'data str, EntityType)],
     ) -> WasmResult<()> {
-        let imports = imports
-            .iter()
-            .map(|i| (i.0.to_string(), i.1.map(|s| s.to_string()), i.2.clone()))
-            .collect();
+        let mut imports = indexmap::IndexMap::new();
+        let mut instance_types = HashMap::new();
+        for (module, field, ty) in declared_imports {
+            match field {
+                Some(field) => {
+                    let idx = *instance_types
+                        .entry(module)
+                        .or_insert_with(|| self.types.instance_signatures.push(Default::default()));
+                    self.types.instance_signatures[idx]
+                        .exports
+                        .insert(field.to_string(), ty.clone());
+                    if !imports.contains_key(*module) {
+                        imports.insert(module.to_string(), EntityType::Instance(idx));
+                    }
+                }
+                None => {
+                    imports.insert(module.to_string(), ty.clone());
+                }
+            }
+        }
         let exports = exports
             .iter()
             .map(|e| (e.0.to_string(), e.1.clone()))
@@ -344,8 +494,8 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
     fn declare_func_import(
         &mut self,
         index: TypeIndex,
-        module: &str,
-        field: Option<&str>,
+        module: &'data str,
+        field: Option<&'data str>,
     ) -> WasmResult<()> {
         debug_assert_eq!(
             self.result.module.functions.len(),
@@ -353,12 +503,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
             "Imported functions must be declared first"
         );
         let sig_index = self.result.module.types[index].unwrap_function();
-        let func_index = self.result.module.functions.push(sig_index);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Function(func_index),
-        });
+        self.declare_import(module, field, EntityType::Function(sig_index));
         self.result.module.num_imported_funcs += 1;
         self.result.debuginfo.wasm_file.imported_func_count += 1;
         Ok(())
@@ -367,21 +512,15 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
     fn declare_table_import(
         &mut self,
         table: Table,
-        module: &str,
-        field: Option<&str>,
+        module: &'data str,
+        field: Option<&'data str>,
     ) -> WasmResult<()> {
         debug_assert_eq!(
             self.result.module.table_plans.len(),
             self.result.module.num_imported_tables,
             "Imported tables must be declared first"
         );
-        let plan = TablePlan::for_table(table, &self.tunables);
-        let table_index = self.result.module.table_plans.push(plan);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Table(table_index),
-        });
+        self.declare_import(module, field, EntityType::Table(table));
         self.result.module.num_imported_tables += 1;
         Ok(())
     }
@@ -389,8 +528,8 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
     fn declare_memory_import(
         &mut self,
         memory: Memory,
-        module: &str,
-        field: Option<&str>,
+        module: &'data str,
+        field: Option<&'data str>,
     ) -> WasmResult<()> {
         debug_assert_eq!(
             self.result.module.memory_plans.len(),
@@ -400,13 +539,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
         if memory.shared {
             return Err(WasmError::Unsupported("shared memories".to_owned()));
         }
-        let plan = MemoryPlan::for_memory(memory, &self.tunables);
-        let memory_index = self.result.module.memory_plans.push(plan);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Memory(memory_index),
-        });
+        self.declare_import(module, field, EntityType::Memory(memory));
         self.result.module.num_imported_memories += 1;
         Ok(())
     }
@@ -414,20 +547,15 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
     fn declare_global_import(
         &mut self,
         global: Global,
-        module: &str,
-        field: Option<&str>,
+        module: &'data str,
+        field: Option<&'data str>,
     ) -> WasmResult<()> {
         debug_assert_eq!(
             self.result.module.globals.len(),
             self.result.module.num_imported_globals,
             "Imported globals must be declared first"
         );
-        let global_index = self.result.module.globals.push(global);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Global(global_index),
-        });
+        self.declare_import(module, field, EntityType::Global(global));
         self.result.module.num_imported_globals += 1;
         Ok(())
     }
@@ -439,12 +567,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
         field: Option<&'data str>,
     ) -> WasmResult<()> {
         let signature = self.type_to_module_type(ty_index)?;
-        let module_index = self.result.module.modules.push(signature);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Module(module_index),
-        });
+        self.declare_import(module, field, EntityType::Module(signature));
         Ok(())
     }
 
@@ -455,12 +578,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
         field: Option<&'data str>,
     ) -> WasmResult<()> {
         let signature = self.type_to_instance_type(ty_index)?;
-        let instance_index = self.result.module.instances.push(signature);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Instance(instance_index),
-        });
+        self.declare_import(module, field, EntityType::Instance(signature));
         Ok(())
     }
 
@@ -755,62 +873,57 @@ and for re-adding support for interface types you can see this issue:
         self.result.module.initializers.reserve(amount as usize);
     }
 
-    fn declare_module(&mut self, ty: TypeIndex) -> WasmResult<()> {
-        // Record the type signature of this module ...
-        let signature = self.type_to_module_type(ty)?;
-        self.result.module.modules.push(signature);
-
-        // ... and then record that in the initialization steps of this module
-        // we're inserting this module into the module index space. At this
-        // point we don't know the final index of the module we're defining, so
-        // we leave a placeholder to get rewritten later.
-        let loc = self.result.module.initializers.len();
-        self.result
-            .module
-            .initializers
-            .push(Initializer::DefineModule(usize::max_value()));
-        self.result.module_initializer_indexes.push(loc);
-        Ok(())
-    }
-
-    fn module_start(&mut self, index: usize) {
-        // Reset the contents of `self.result` for a new module that's getting
-        // translataed.
-        let mut prev = mem::replace(&mut self.result, ModuleTranslation::default());
-
-        // If this is a nested submodule then we record the final destination of
-        // the child in parent (we store `index` into `prev`) in the appropriate
-        // initialization slot as dicated by `num_modules_defined` (our index of
-        // iteration through the code section).
-        // Record that the `num_modules_defined`-th module is defined at index
-        // by updating the initializer entry.
-        if index > 0 {
-            let initializer_idx = prev.module_initializer_indexes[prev.num_modules_defined];
-            prev.num_modules_defined += 1;
-            debug_assert!(match &prev.module.initializers[initializer_idx] {
-                Initializer::DefineModule(usize::MAX) => true,
-                _ => false,
-            });
-            prev.module.initializers[initializer_idx] = Initializer::DefineModule(index);
-            self.result.module.parent = Some(self.cur);
+    fn module_start(&mut self) {
+        // If this is the first time this method is called, nothing to do.
+        if self.first_module {
+            self.first_module = false;
+            return;
         }
-
-        // Update our current index counter and save our parent's translation
-        // where this current translation will end up, which we'll swap back as
-        // part of `module_end`.
-        self.cur = index;
-        assert_eq!(index, self.results.len());
-        self.results.push(prev);
+        // Reset our internal state for a new module by saving the current
+        // module in `results`.
+        let in_progress = mem::replace(&mut self.result, ModuleTranslation::default());
+        self.in_progress.push(in_progress);
         self.modules_to_be -= 1;
     }
 
-    fn module_end(&mut self, index: usize) {
-        assert!(self.result.num_modules_defined == self.result.module_initializer_indexes.len());
+    fn module_end(&mut self) {
+        self.result.creation_artifacts.shrink_to_fit();
+        self.result.creation_modules.shrink_to_fit();
 
-        // Move our finished module into its final location, swapping it with
-        // what was this module's parent.
-        self.cur = self.result.module.parent.unwrap_or(0);
-        mem::swap(&mut self.result, &mut self.results[index]);
+        let (record_initializer, mut done) = match self.in_progress.pop() {
+            Some(m) => (true, mem::replace(&mut self.result, m)),
+            None => (false, mem::take(&mut self.result)),
+        };
+
+        if record_initializer {
+            // Record the type of the module we just finished in our own
+            // module's list of modules.
+            let sig = self.gen_type_of_module(&done.module);
+            self.result.module.modules.push(sig);
+
+            // The root module will store the artifacts for this finished
+            // module at `artifact_index`. This then needs to be inherited by
+            // all later modules coming down to our now-current `self.result`...
+            let mut artifact_index = self.results.len();
+            for result in self.in_progress.iter_mut().chain(Some(&mut self.result)) {
+                result.creation_artifacts.push(artifact_index);
+                artifact_index = result.creation_artifacts.len() - 1;
+            }
+            // ... and then `self.result` needs to create a new module with
+            // whatever was record to save off as its own artifacts/modules.
+            self.result
+                .module
+                .initializers
+                .push(Initializer::CreateModule {
+                    artifact_index,
+                    artifacts: mem::take(&mut done.creation_artifacts),
+                    modules: mem::take(&mut done.creation_modules),
+                });
+        }
+
+        // And the final step is to insert the module into the list of finished
+        // modules to get returned at the end.
+        self.results.push(done);
     }
 
     fn reserve_instances(&mut self, amt: u32) {
@@ -818,7 +931,12 @@ and for re-adding support for interface types you can see this issue:
         self.result.module.initializers.reserve(amt as usize);
     }
 
-    fn declare_instance(&mut self, module: ModuleIndex, args: Vec<EntityIndex>) -> WasmResult<()> {
+    fn declare_instance(
+        &mut self,
+        module: ModuleIndex,
+        args: Vec<(&'data str, EntityIndex)>,
+    ) -> WasmResult<()> {
+        let args = args.into_iter().map(|(s, i)| (s.to_string(), i)).collect();
         // Record the type of this instance with the type signature of the
         // module we're instantiating and then also add an initializer which
         // records that we'll be adding to the instance index space here.
@@ -839,29 +957,60 @@ and for re-adding support for interface types you can see this issue:
             //
             // Note that we don't add an initializer for this alias because
             // we statically know where all types point to.
-            Alias::ParentType(parent_idx) => {
-                let ty = self.results[self.cur].module.types[parent_idx];
+            Alias::OuterType {
+                relative_depth,
+                index,
+            } => {
+                let module_idx = self.in_progress.len() - 1 - (relative_depth as usize);
+                let ty = self.in_progress[module_idx].module.types[index];
                 self.result.module.types.push(ty);
             }
 
-            // This is similar to types in that it's easy for us to record the
-            // type of the module that's being aliased, but we also need to add
-            // an initializer so during instantiation we can prepare the index
-            // space appropriately.
-            Alias::ParentModule(parent_idx) => {
-                let module_idx = self.results[self.cur].module.modules[parent_idx];
-                self.result.module.modules.push(module_idx);
+            // Modules are a bit trickier since we need to record how to track
+            // the state from the original module down to our own.
+            Alias::OuterModule {
+                relative_depth,
+                index,
+            } => {
+                // First we can copy the type from the parent module into our
+                // own module to record what type our module definition will
+                // have.
+                let module_idx = self.in_progress.len() - 1 - (relative_depth as usize);
+                let module_ty = self.in_progress[module_idx].module.modules[index];
+                self.result.module.modules.push(module_ty);
+
+                // Next we'll be injecting a module value that is closed over,
+                // and that will be used to define the module into the index
+                // space. Record an initializer about where our module is
+                // sourced from (which will be stored within each module value
+                // itself).
+                let module_index = self.result.creation_modules.len();
                 self.result
                     .module
                     .initializers
-                    .push(Initializer::AliasParentModule(parent_idx));
+                    .push(Initializer::DefineModule(module_index));
+
+                // And finally we need to record a breadcrumb trail of how to
+                // get the module value into `module_index`. The module just
+                // after our destination module will use a `ModuleIndex` to
+                // fetch the module value, and everything else inbetween will
+                // inherit that module's closed-over value.
+                let mut upvar = ModuleUpvar::Local(index);
+                for outer in self.in_progress[module_idx + 1..].iter_mut() {
+                    let upvar = mem::replace(
+                        &mut upvar,
+                        ModuleUpvar::Inherit(outer.creation_modules.len()),
+                    );
+                    outer.creation_modules.push(upvar);
+                }
+                self.result.creation_modules.push(upvar);
             }
 
             // This case is slightly more involved, we'll be recording all the
             // type information for each kind of entity, and then we also need
             // to record an initialization step to get the export from the
             // instance.
-            Alias::Child { instance, export } => {
+            Alias::InstanceExport { instance, export } => {
                 let ty = self.result.module.instances[instance];
                 match &self.types.instance_signatures[ty].exports[export] {
                     EntityType::Global(g) => {
@@ -894,7 +1043,10 @@ and for re-adding support for interface types you can see this issue:
                 self.result
                     .module
                     .initializers
-                    .push(Initializer::AliasInstanceExport { instance, export })
+                    .push(Initializer::AliasInstanceExport {
+                        instance,
+                        export: export.to_string(),
+                    })
             }
         }
 
diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs
index 939861504d..bd86fef237 100644
--- a/crates/environ/src/tunables.rs
+++ b/crates/environ/src/tunables.rs
@@ -23,6 +23,10 @@ pub struct Tunables {
     /// calls and interrupts are implemented through the `VMInterrupts`
     /// structure, or `InterruptHandle` in the `wasmtime` crate.
     pub interruptable: bool,
+
+    /// Whether or not fuel is enabled for generated code, meaning that fuel
+    /// will be consumed every time a wasm instruction is executed.
+    pub consume_fuel: bool,
 }
 
 impl Default for Tunables {
@@ -57,6 +61,7 @@ impl Default for Tunables {
             generate_native_debuginfo: false,
             parse_wasm_debuginfo: true,
             interruptable: false,
+            consume_fuel: false,
         }
     }
 }
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 8673a38d3c..7f74f46754 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -258,6 +258,11 @@ impl VMOffsets {
     pub fn vminterrupts_stack_limit(&self) -> u8 {
         0
     }
+
+    /// Return the offset of the `fuel_consumed` field of `VMInterrupts`
+    pub fn vminterrupts_fuel_consumed(&self) -> u8 {
+        self.pointer_size
+    }
 }
 
 /// Offsets for `VMCallerCheckedAnyfunc`.
diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml
index 2cc157704a..140922809e 100644
--- a/crates/fuzzing/Cargo.toml
+++ b/crates/fuzzing/Cargo.toml
@@ -13,12 +13,12 @@ arbitrary = { version = "0.4.1", features = ["derive"] }
 env_logger = "0.8.1"
 log = "0.4.8"
 rayon = "1.2.1"
-wasmparser = "0.71"
-wasmprinter = "0.2.17"
+wasmparser = "0.73"
+wasmprinter = "0.2.20"
 wasmtime = { path = "../wasmtime" }
 wasmtime-wast = { path = "../wast" }
-wasm-encoder = "0.2"
-wasm-smith = "0.3.0"
+wasm-encoder = "0.4"
+wasm-smith = "0.3.1"
 wasmi = "0.7.0"
 
 [dev-dependencies]
diff --git a/crates/fuzzing/src/generators.rs b/crates/fuzzing/src/generators.rs
index d31eba04cb..b44af8daf5 100644
--- a/crates/fuzzing/src/generators.rs
+++ b/crates/fuzzing/src/generators.rs
@@ -64,6 +64,8 @@ pub struct Config {
     debug_info: bool,
     canonicalize_nans: bool,
     interruptable: bool,
+    #[allow(missing_docs)]
+    pub consume_fuel: bool,
 
     // Note that we use 32-bit values here to avoid blowing the 64-bit address
     // space by requesting ungodly-large sizes/guards.
@@ -75,14 +77,15 @@ pub struct Config {
 impl Config {
     /// Converts this to a `wasmtime::Config` object
     pub fn to_wasmtime(&self) -> wasmtime::Config {
-        let mut cfg = wasmtime::Config::new();
+        let mut cfg = crate::fuzz_default_config(wasmtime::Strategy::Auto).unwrap();
         cfg.debug_info(self.debug_info)
             .static_memory_maximum_size(self.static_memory_maximum_size.unwrap_or(0).into())
             .static_memory_guard_size(self.static_memory_guard_size.unwrap_or(0).into())
             .dynamic_memory_guard_size(self.dynamic_memory_guard_size.unwrap_or(0).into())
             .cranelift_nan_canonicalization(self.canonicalize_nans)
             .cranelift_opt_level(self.opt_level.to_wasmtime())
-            .interruptable(self.interruptable);
+            .interruptable(self.interruptable)
+            .consume_fuel(self.consume_fuel);
         return cfg;
     }
 }
diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs
index 7ef3382411..d131149ddb 100644
--- a/crates/fuzzing/src/lib.rs
+++ b/crates/fuzzing/src/lib.rs
@@ -39,6 +39,9 @@ pub fn fuzz_default_config(strategy: wasmtime::Strategy) -> anyhow::Result<wasmt
         .wasm_bulk_memory(true)
         .wasm_reference_types(true)
         .wasm_module_linking(true)
+        .max_instances(100)
+        .max_tables(100)
+        .max_memories(100)
         .strategy(strategy)?;
     Ok(config)
 }
diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs
index 6581f9e0dc..2fc4532de4 100644
--- a/crates/fuzzing/src/oracles.rs
+++ b/crates/fuzzing/src/oracles.rs
@@ -40,6 +40,20 @@ fn log_wasm(wasm: &[u8]) {
     }
 }
 
+/// Methods of timing out execution of a WebAssembly module
+#[derive(Debug)]
+pub enum Timeout {
+    /// No timeout is used, it should be guaranteed via some other means that
+    /// the input does not infinite loop.
+    None,
+    /// A time-based timeout is used with a sleeping thread sending a signal
+    /// after the specified duration.
+    Time(Duration),
+    /// Fuel-based timeouts are used where the specified fuel is all that the
+    /// provided wasm module is allowed to consume.
+    Fuel(u64),
+}
+
 /// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected
 /// panic or segfault or anything else that can be detected "passively".
 ///
@@ -47,12 +61,11 @@ fn log_wasm(wasm: &[u8]) {
 ///
 /// You can control which compiler is used via passing a `Strategy`.
 pub fn instantiate(wasm: &[u8], known_valid: bool, strategy: Strategy) {
-    instantiate_with_config(
-        wasm,
-        known_valid,
-        crate::fuzz_default_config(strategy).unwrap(),
-        None,
-    );
+    // Explicitly disable module linking for now since it's a breaking change to
+    // pre-module-linking modules due to imports
+    let mut cfg = crate::fuzz_default_config(strategy).unwrap();
+    cfg.wasm_module_linking(false);
+    instantiate_with_config(wasm, known_valid, cfg, Timeout::None);
 }
 
 /// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected
@@ -65,28 +78,38 @@ pub fn instantiate_with_config(
     wasm: &[u8],
     known_valid: bool,
     mut config: Config,
-    timeout: Option<Duration>,
+    timeout: Timeout,
 ) {
     crate::init_fuzzing();
 
-    config.interruptable(timeout.is_some());
+    config.interruptable(match &timeout {
+        Timeout::Time(_) => true,
+        _ => false,
+    });
+    config.consume_fuel(match &timeout {
+        Timeout::Fuel(_) => true,
+        _ => false,
+    });
     let engine = Engine::new(&config);
     let store = Store::new(&engine);
 
-    // If a timeout is requested then we spawn a helper thread to wait for the
-    // requested time and then send us a signal to get interrupted. We also
-    // arrange for the thread's sleep to get interrupted if we return early (or
-    // the wasm returns within the time limit), which allows the thread to get
-    // torn down.
-    //
-    // This prevents us from creating a huge number of sleeping threads if this
-    // function is executed in a loop, like it does on nightly fuzzing
-    // infrastructure.
-
     let mut timeout_state = SignalOnDrop::default();
-    if let Some(timeout) = timeout {
-        let handle = store.interrupt_handle().unwrap();
-        timeout_state.spawn_timeout(timeout, move || handle.interrupt());
+    match timeout {
+        Timeout::Fuel(fuel) => store.add_fuel(fuel),
+        // If a timeout is requested then we spawn a helper thread to wait for
+        // the requested time and then send us a signal to get interrupted. We
+        // also arrange for the thread's sleep to get interrupted if we return
+        // early (or the wasm returns within the time limit), which allows the
+        // thread to get torn down.
+        //
+        // This prevents us from creating a huge number of sleeping threads if
+        // this function is executed in a loop, like it does on nightly fuzzing
+        // infrastructure.
+        Timeout::Time(timeout) => {
+            let handle = store.interrupt_handle().unwrap();
+            timeout_state.spawn_timeout(timeout, move || handle.interrupt());
+        }
+        Timeout::None => {}
     }
 
     log_wasm(wasm);
@@ -99,8 +122,14 @@ pub fn instantiate_with_config(
 
     match Instance::new(&store, &module, &imports) {
         Ok(_) => {}
-        // Allow traps which can happen normally with `unreachable`
+        // Allow traps which can happen normally with `unreachable` or a timeout
         Err(e) if e.downcast_ref::<Trap>().is_some() => {}
+        // Allow resource exhaustion since this is something that our wasm-smith
+        // generator doesn't guarantee is forbidden.
+        Err(e) if e.to_string().contains("resource limit exceeded") => {}
+        // Also allow errors related to fuel consumption
+        Err(e) if e.to_string().contains("all fuel consumed") => {}
+        // Everything else should be a bug in the fuzzer
         Err(e) => panic!("failed to instantiate {}", e),
     }
 }
@@ -381,13 +410,16 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) {
 /// Executes the wast `test` spectest with the `config` specified.
 ///
 /// Ensures that spec tests pass regardless of the `Config`.
-pub fn spectest(config: crate::generators::Config, test: crate::generators::SpecTest) {
+pub fn spectest(fuzz_config: crate::generators::Config, test: crate::generators::SpecTest) {
     crate::init_fuzzing();
-    log::debug!("running {:?} with {:?}", test.file, config);
-    let mut config = config.to_wasmtime();
+    log::debug!("running {:?} with {:?}", test.file, fuzz_config);
+    let mut config = fuzz_config.to_wasmtime();
     config.wasm_reference_types(false);
     config.wasm_bulk_memory(false);
     let store = Store::new(&Engine::new(&config));
+    if fuzz_config.consume_fuel {
+        store.add_fuel(u64::max_value());
+    }
     let mut wast_context = WastContext::new(store);
     wast_context.register_spectest().unwrap();
     wast_context
@@ -396,16 +428,22 @@ pub fn spectest(config: crate::generators::Config, test: crate::generators::Spec
 }
 
 /// Execute a series of `table.get` and `table.set` operations.
-pub fn table_ops(config: crate::generators::Config, ops: crate::generators::table_ops::TableOps) {
+pub fn table_ops(
+    fuzz_config: crate::generators::Config,
+    ops: crate::generators::table_ops::TableOps,
+) {
     let _ = env_logger::try_init();
 
     let num_dropped = Rc::new(Cell::new(0));
 
     {
-        let mut config = config.to_wasmtime();
+        let mut config = fuzz_config.to_wasmtime();
         config.wasm_reference_types(true);
         let engine = Engine::new(&config);
         let store = Store::new(&engine);
+        if fuzz_config.consume_fuel {
+            store.add_fuel(u64::max_value());
+        }
 
         let wasm = ops.to_wasm_binary();
         log_wasm(&wasm);
@@ -518,6 +556,9 @@ pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Con
     wasmtime_config.cranelift_nan_canonicalization(true);
     let wasmtime_engine = Engine::new(&wasmtime_config);
     let wasmtime_store = Store::new(&wasmtime_engine);
+    if config.consume_fuel {
+        wasmtime_store.add_fuel(u64::max_value());
+    }
     let wasmtime_module =
         Module::new(&wasmtime_engine, &wasm).expect("Wasmtime can compile module");
     let wasmtime_instance = Instance::new(&wasmtime_store, &wasmtime_module, &[])
diff --git a/crates/fuzzing/src/oracles/dummy.rs b/crates/fuzzing/src/oracles/dummy.rs
index 561a8fb329..f49cee8c5b 100644
--- a/crates/fuzzing/src/oracles/dummy.rs
+++ b/crates/fuzzing/src/oracles/dummy.rs
@@ -349,3 +349,232 @@ fn wat_ty(ty: &ValType) -> &'static str {
         ValType::FuncRef => "funcref",
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashSet;
+
+    fn store() -> Store {
+        let mut config = Config::default();
+        config.wasm_module_linking(true);
+        config.wasm_multi_memory(true);
+        let engine = wasmtime::Engine::new(&config);
+        Store::new(&engine)
+    }
+
+    #[test]
+    fn dummy_table_import() {
+        let store = store();
+        let table = dummy_table(
+            &store,
+            TableType::new(ValType::ExternRef, Limits::at_least(10)),
+        );
+        assert_eq!(table.size(), 10);
+        for i in 0..10 {
+            assert!(table.get(i).unwrap().unwrap_externref().is_none());
+        }
+    }
+
+    #[test]
+    fn dummy_global_import() {
+        let store = store();
+        let global = dummy_global(&store, GlobalType::new(ValType::I32, Mutability::Const));
+        assert_eq!(global.val_type(), ValType::I32);
+        assert_eq!(global.mutability(), Mutability::Const);
+    }
+
+    #[test]
+    fn dummy_memory_import() {
+        let store = store();
+        let memory = dummy_memory(&store, MemoryType::new(Limits::at_least(1)));
+        assert_eq!(memory.size(), 1);
+    }
+
+    #[test]
+    fn dummy_function_import() {
+        let store = store();
+        let func_ty = FuncType::new(vec![ValType::I32], vec![ValType::I64]);
+        let func = dummy_func(&store, func_ty.clone());
+        assert_eq!(func.ty(), func_ty);
+    }
+
+    #[test]
+    fn dummy_instance_import() {
+        let store = store();
+
+        let mut instance_ty = InstanceType::new();
+
+        // Functions.
+        instance_ty.add_named_export("func0", FuncType::new(vec![ValType::I32], vec![]).into());
+        instance_ty.add_named_export("func1", FuncType::new(vec![], vec![ValType::I64]).into());
+
+        // Globals.
+        instance_ty.add_named_export(
+            "global0",
+            GlobalType::new(ValType::I32, Mutability::Const).into(),
+        );
+        instance_ty.add_named_export(
+            "global1",
+            GlobalType::new(ValType::I64, Mutability::Var).into(),
+        );
+
+        // Tables.
+        instance_ty.add_named_export(
+            "table0",
+            TableType::new(ValType::ExternRef, Limits::at_least(1)).into(),
+        );
+        instance_ty.add_named_export(
+            "table1",
+            TableType::new(ValType::ExternRef, Limits::at_least(1)).into(),
+        );
+
+        // Memories.
+        instance_ty.add_named_export("memory0", MemoryType::new(Limits::at_least(1)).into());
+        instance_ty.add_named_export("memory1", MemoryType::new(Limits::at_least(1)).into());
+
+        // Modules.
+        instance_ty.add_named_export("module0", ModuleType::new().into());
+        instance_ty.add_named_export("module1", ModuleType::new().into());
+
+        // Instances.
+        instance_ty.add_named_export("instance0", InstanceType::new().into());
+        instance_ty.add_named_export("instance1", InstanceType::new().into());
+
+        let instance = dummy_instance(&store, instance_ty.clone());
+
+        let mut expected_exports = vec![
+            "func0",
+            "func1",
+            "global0",
+            "global1",
+            "table0",
+            "table1",
+            "memory0",
+            "memory1",
+            "module0",
+            "module1",
+            "instance0",
+            "instance1",
+        ]
+        .into_iter()
+        .collect::<HashSet<_>>();
+        for exp in instance.ty().exports() {
+            let was_expected = expected_exports.remove(exp.name());
+            assert!(was_expected);
+        }
+        assert!(expected_exports.is_empty());
+    }
+
+    #[test]
+    fn dummy_module_import() {
+        let store = store();
+
+        let mut module_ty = ModuleType::new();
+
+        // Multiple exported and imported functions.
+        module_ty.add_named_export("func0", FuncType::new(vec![ValType::I32], vec![]).into());
+        module_ty.add_named_export("func1", FuncType::new(vec![], vec![ValType::I64]).into());
+        module_ty.add_named_import(
+            "func2",
+            None,
+            FuncType::new(vec![ValType::I64], vec![]).into(),
+        );
+        module_ty.add_named_import(
+            "func3",
+            None,
+            FuncType::new(vec![], vec![ValType::I32]).into(),
+        );
+
+        // Multiple exported and imported globals.
+        module_ty.add_named_export(
+            "global0",
+            GlobalType::new(ValType::I32, Mutability::Const).into(),
+        );
+        module_ty.add_named_export(
+            "global1",
+            GlobalType::new(ValType::I64, Mutability::Var).into(),
+        );
+        module_ty.add_named_import(
+            "global2",
+            None,
+            GlobalType::new(ValType::I32, Mutability::Var).into(),
+        );
+        module_ty.add_named_import(
+            "global3",
+            None,
+            GlobalType::new(ValType::I64, Mutability::Const).into(),
+        );
+
+        // Multiple exported and imported tables.
+        module_ty.add_named_export(
+            "table0",
+            TableType::new(ValType::ExternRef, Limits::at_least(1)).into(),
+        );
+        module_ty.add_named_export(
+            "table1",
+            TableType::new(ValType::ExternRef, Limits::at_least(1)).into(),
+        );
+        module_ty.add_named_import(
+            "table2",
+            None,
+            TableType::new(ValType::ExternRef, Limits::at_least(1)).into(),
+        );
+        module_ty.add_named_import(
+            "table3",
+            None,
+            TableType::new(ValType::ExternRef, Limits::at_least(1)).into(),
+        );
+
+        // Multiple exported and imported memories.
+        module_ty.add_named_export("memory0", MemoryType::new(Limits::at_least(1)).into());
+        module_ty.add_named_export("memory1", MemoryType::new(Limits::at_least(1)).into());
+        module_ty.add_named_import("memory2", None, MemoryType::new(Limits::at_least(1)).into());
+        module_ty.add_named_import("memory3", None, MemoryType::new(Limits::at_least(1)).into());
+
+        // An exported and an imported module.
+        module_ty.add_named_export("module0", ModuleType::new().into());
+        module_ty.add_named_import("module1", None, ModuleType::new().into());
+
+        // An exported and an imported instance.
+        module_ty.add_named_export("instance0", InstanceType::new().into());
+        module_ty.add_named_import("instance1", None, InstanceType::new().into());
+
+        // Create the module.
+        let module = dummy_module(&store, module_ty);
+
+        // Check that we have the expected exports.
+        assert!(module.get_export("func0").is_some());
+        assert!(module.get_export("func1").is_some());
+        assert!(module.get_export("global0").is_some());
+        assert!(module.get_export("global1").is_some());
+        assert!(module.get_export("table0").is_some());
+        assert!(module.get_export("table1").is_some());
+        assert!(module.get_export("memory0").is_some());
+        assert!(module.get_export("memory1").is_some());
+        assert!(module.get_export("instance0").is_some());
+        assert!(module.get_export("module0").is_some());
+
+        // Check that we have the exported imports.
+        let mut expected_imports = vec![
+            "func2",
+            "func3",
+            "global2",
+            "global3",
+            "table2",
+            "table3",
+            "memory2",
+            "memory3",
+            "instance1",
+            "module1",
+        ]
+        .into_iter()
+        .collect::<HashSet<_>>();
+        for imp in module.imports() {
+            assert!(imp.name().is_none());
+            let was_expected = expected_imports.remove(imp.module());
+            assert!(was_expected);
+        }
+        assert!(expected_imports.is_empty());
+    }
+}
diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml
index bebb4d2841..2353d98ed5 100644
--- a/crates/jit/Cargo.toml
+++ b/crates/jit/Cargo.toml
@@ -28,13 +28,13 @@ rayon = { version = "1.0", optional = true }
 region = "2.1.0"
 thiserror = "1.0.4"
 target-lexicon = { version = "0.11.0", default-features = false }
-wasmparser = "0.71"
+wasmparser = "0.73"
 more-asserts = "0.2.1"
 anyhow = "1.0"
 cfg-if = "1.0"
 log = "0.4"
 gimli = { version = "0.23.0", default-features = false, features = ["write"] }
-object = { version = "0.22.0", default-features = false, features = ["write"] }
+object = { version = "0.23.0", default-features = false, features = ["write"] }
 serde = { version = "1.0.94", features = ["derive"] }
 addr2line = { version = "0.14", default-features = false }
 
diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs
index 5adb0e8312..fe94c27c02 100644
--- a/crates/jit/src/compiler.rs
+++ b/crates/jit/src/compiler.rs
@@ -182,14 +182,10 @@ impl Hash for Compiler {
         // misc tunables.
         strategy.hash(hasher);
         isa.triple().hash(hasher);
-        features.hash(hasher);
-        // TODO: if this `to_string()` is too expensive then we should upstream
-        // a native hashing ability of flags into cranelift itself, but
-        // compilation and/or cache loading is relatively expensive so seems
-        // unlikely.
-        isa.flags().to_string().hash(hasher);
+        isa.hash_all_flags(hasher);
         isa.frontend_config().hash(hasher);
         tunables.hash(hasher);
+        features.hash(hasher);
 
         // Catch accidental bugs of reusing across crate versions.
         env!("CARGO_PKG_VERSION").hash(hasher);
diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs
index f739b79d1a..655432e00f 100644
--- a/crates/jit/src/instantiate.rs
+++ b/crates/jit/src/instantiate.rs
@@ -105,8 +105,8 @@ impl CompilationArtifacts {
     pub fn build(
         compiler: &Compiler,
         data: &[u8],
-    ) -> Result<(Vec<CompilationArtifacts>, TypeTables), SetupError> {
-        let (translations, types) = ModuleEnvironment::new(
+    ) -> Result<(usize, Vec<CompilationArtifacts>, TypeTables), SetupError> {
+        let (main_module, translations, types) = ModuleEnvironment::new(
             compiler.frontend_config(),
             compiler.tunables(),
             compiler.features(),
@@ -166,6 +166,7 @@ impl CompilationArtifacts {
             })
             .collect::<Result<Vec<_>, SetupError>>()?;
         Ok((
+            main_module,
             list,
             TypeTables {
                 wasm_signatures: types.wasm_signatures,
@@ -220,7 +221,7 @@ impl CompiledModule {
         artifacts: Vec<CompilationArtifacts>,
         isa: &dyn TargetIsa,
         profiler: &dyn ProfilingAgent,
-    ) -> Result<Vec<Self>, SetupError> {
+    ) -> Result<Vec<Arc<Self>>, SetupError> {
         maybe_parallel!(artifacts.(into_iter | into_par_iter))
             .map(|a| CompiledModule::from_artifacts(a, isa, profiler))
             .collect()
@@ -231,7 +232,7 @@ impl CompiledModule {
         artifacts: CompilationArtifacts,
         isa: &dyn TargetIsa,
         profiler: &dyn ProfilingAgent,
-    ) -> Result<Self, SetupError> {
+    ) -> Result<Arc<Self>, SetupError> {
         // Allocate all of the compiled functions into executable memory,
         // copying over their contents.
         let (code_memory, code_range, finished_functions, trampolines) = build_code_memory(
@@ -265,7 +266,7 @@ impl CompiledModule {
 
         let finished_functions = FinishedFunctions(finished_functions);
 
-        Ok(Self {
+        Ok(Arc::new(Self {
             module: Arc::new(artifacts.module.clone()),
             artifacts,
             code: Arc::new(ModuleCode {
@@ -274,7 +275,7 @@ impl CompiledModule {
             }),
             finished_functions,
             trampolines,
-        })
+        }))
     }
 
     /// Crate an `Instance` from this `CompiledModule`.
diff --git a/crates/jit/src/native.rs b/crates/jit/src/native.rs
index 9d1fdd7b66..afcf83d3cc 100644
--- a/crates/jit/src/native.rs
+++ b/crates/jit/src/native.rs
@@ -6,6 +6,11 @@ pub fn builder() -> cranelift_codegen::isa::Builder {
     cranelift_native::builder().expect("host machine is not a supported target")
 }
 
+pub fn builder_without_flags() -> cranelift_codegen::isa::Builder {
+    cranelift_native::builder_with_options(cranelift_codegen::isa::BackendVariant::Any, false)
+        .expect("host machine is not a supported target")
+}
+
 pub fn call_conv() -> cranelift_codegen::isa::CallConv {
     use target_lexicon::HOST;
     cranelift_codegen::isa::CallConv::triple_default(&HOST)
diff --git a/crates/lightbeam/Cargo.toml b/crates/lightbeam/Cargo.toml
index 89a18280fa..2cde3bd067 100644
--- a/crates/lightbeam/Cargo.toml
+++ b/crates/lightbeam/Cargo.toml
@@ -18,18 +18,18 @@ derive_more = "0.99"
 dynasm = "1.0.0"
 dynasmrt = "1.0.0"
 iter-enum = "0.2"
-itertools = "0.9.0"
+itertools = "0.10.0"
 memoffset = "0.6.0"
 more-asserts = "0.2.1"
 smallvec = "1.6.1"
 thiserror = "1.0.9"
 typemap = "0.3"
-wasmparser = "0.71"
+wasmparser = "0.73"
 
 [dev-dependencies]
 lazy_static = "1.2"
 wat = "1.0.23"
-quickcheck = "0.9.0"
+quickcheck = "1.0.0"
 anyhow = "1.0"
 
 [badges]
diff --git a/crates/lightbeam/wasmtime/Cargo.toml b/crates/lightbeam/wasmtime/Cargo.toml
index 71f43c8a3c..a2b8b06e9d 100644
--- a/crates/lightbeam/wasmtime/Cargo.toml
+++ b/crates/lightbeam/wasmtime/Cargo.toml
@@ -13,6 +13,6 @@ edition = "2018"
 
 [dependencies]
 lightbeam = { path = "..", version = "0.22.0" }
-wasmparser = "0.71"
+wasmparser = "0.73"
 cranelift-codegen = { path = "../../../cranelift/codegen", version = "0.69.0" }
 wasmtime-environ = { path = "../../environ", version = "0.22.0" }
diff --git a/crates/misc/run-examples/src/main.rs b/crates/misc/run-examples/src/main.rs
index 94e5b56dc9..12746b2a38 100644
--- a/crates/misc/run-examples/src/main.rs
+++ b/crates/misc/run-examples/src/main.rs
@@ -83,7 +83,8 @@ fn main() -> anyhow::Result<()> {
                     .arg("userenv.lib")
                     .arg("ntdll.lib")
                     .arg("shell32.lib")
-                    .arg("ole32.lib");
+                    .arg("ole32.lib")
+                    .arg("bcrypt.lib");
                 if is_dir {
                     "main.exe".to_string()
                 } else {
diff --git a/crates/obj/Cargo.toml b/crates/obj/Cargo.toml
index 917ea45236..d88d388fc6 100644
--- a/crates/obj/Cargo.toml
+++ b/crates/obj/Cargo.toml
@@ -13,7 +13,7 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0"
 wasmtime-environ = { path = "../environ", version = "0.22.0" }
-object = { version = "0.22.0", default-features = false, features = ["write"] }
+object = { version = "0.23.0", default-features = false, features = ["write"] }
 more-asserts = "0.2.1"
 target-lexicon = { version = "0.11.0", default-features = false }
 wasmtime-debug = { path = "../debug", version = "0.22.0" }
diff --git a/crates/profiling/Cargo.toml b/crates/profiling/Cargo.toml
index dfc853b3d7..072cb0fbb5 100644
--- a/crates/profiling/Cargo.toml
+++ b/crates/profiling/Cargo.toml
@@ -24,7 +24,7 @@ wasmtime-runtime = { path = "../runtime", version = "0.22.0" }
 ittapi-rs = { version = "0.1.5", optional = true }
 
 [dependencies.object]
-version = "0.22.0"
+version = "0.23.0"
 optional = true
 default-features = false
 features = ['read_core', 'elf', 'std']
diff --git a/crates/profiling/src/jitdump_linux.rs b/crates/profiling/src/jitdump_linux.rs
index ef6684c95f..0f3a4bb00d 100644
--- a/crates/profiling/src/jitdump_linux.rs
+++ b/crates/profiling/src/jitdump_linux.rs
@@ -1,6 +1,6 @@
 //! Support for jitdump files which can be used by perf for profiling jitted code.
 //! Spec definitions for the output format is as described here:
-//! https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt
+//! <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt>
 //!
 //! Usage Example:
 //!     Record
diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml
index 78564d6faf..40266d3793 100644
--- a/crates/runtime/Cargo.toml
+++ b/crates/runtime/Cargo.toml
@@ -14,7 +14,7 @@ edition = "2018"
 [dependencies]
 wasmtime-environ = { path = "../environ", version = "0.22.0" }
 region = "2.1.0"
-libc = { version = "0.2.70", default-features = false }
+libc = { version = "0.2.82", default-features = false }
 log = "0.4.8"
 memoffset = "0.6.0"
 indexmap = "1.0.2"
diff --git a/crates/runtime/src/export.rs b/crates/runtime/src/export.rs
index 0161e56a68..b9564e571a 100644
--- a/crates/runtime/src/export.rs
+++ b/crates/runtime/src/export.rs
@@ -1,14 +1,14 @@
 use crate::vmcontext::{
     VMCallerCheckedAnyfunc, VMContext, VMGlobalDefinition, VMMemoryDefinition, VMTableDefinition,
 };
-use crate::InstanceHandle;
+use crate::RuntimeInstance;
 use std::any::Any;
 use std::ptr::NonNull;
 use wasmtime_environ::wasm::Global;
 use wasmtime_environ::{MemoryPlan, TablePlan};
 
 /// The value of an export passed from one instance to another.
-pub enum Export<'a> {
+pub enum Export {
     /// A function export value.
     Function(ExportFunction),
 
@@ -22,10 +22,10 @@ pub enum Export<'a> {
     Global(ExportGlobal),
 
     /// An instance
-    Instance(&'a InstanceHandle),
+    Instance(RuntimeInstance),
 
     /// A module
-    Module(&'a dyn Any),
+    Module(Box<dyn Any>),
 }
 
 /// A function export value.
@@ -38,8 +38,8 @@ pub struct ExportFunction {
     pub anyfunc: NonNull<VMCallerCheckedAnyfunc>,
 }
 
-impl<'a> From<ExportFunction> for Export<'a> {
-    fn from(func: ExportFunction) -> Export<'a> {
+impl From<ExportFunction> for Export {
+    fn from(func: ExportFunction) -> Export {
         Export::Function(func)
     }
 }
@@ -55,8 +55,8 @@ pub struct ExportTable {
     pub table: TablePlan,
 }
 
-impl<'a> From<ExportTable> for Export<'a> {
-    fn from(func: ExportTable) -> Export<'a> {
+impl From<ExportTable> for Export {
+    fn from(func: ExportTable) -> Export {
         Export::Table(func)
     }
 }
@@ -72,8 +72,8 @@ pub struct ExportMemory {
     pub memory: MemoryPlan,
 }
 
-impl<'a> From<ExportMemory> for Export<'a> {
-    fn from(func: ExportMemory) -> Export<'a> {
+impl From<ExportMemory> for Export {
+    fn from(func: ExportMemory) -> Export {
         Export::Memory(func)
     }
 }
@@ -89,8 +89,8 @@ pub struct ExportGlobal {
     pub global: Global,
 }
 
-impl<'a> From<ExportGlobal> for Export<'a> {
-    fn from(func: ExportGlobal) -> Export<'a> {
+impl From<ExportGlobal> for Export {
+    fn from(func: ExportGlobal) -> Export {
         Export::Global(func)
     }
 }
diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs
index dd0a4f1475..b5a8ef30bb 100644
--- a/crates/runtime/src/externref.rs
+++ b/crates/runtime/src/externref.rs
@@ -97,7 +97,7 @@
 //!
 //! For more general information on deferred reference counting, see *An
 //! Examination of Deferred Reference Counting and Cycle Detection* by Quinane:
-//! https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf
+//! <https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf>
 
 use std::alloc::Layout;
 use std::any::Any;
diff --git a/crates/runtime/src/imports.rs b/crates/runtime/src/imports.rs
index 1969630750..fe6031293e 100644
--- a/crates/runtime/src/imports.rs
+++ b/crates/runtime/src/imports.rs
@@ -1,8 +1,4 @@
 use crate::vmcontext::{VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport};
-use crate::InstanceHandle;
-use std::any::Any;
-use wasmtime_environ::entity::PrimaryMap;
-use wasmtime_environ::wasm::{InstanceIndex, ModuleIndex};
 
 /// Resolved import pointers.
 ///
@@ -28,15 +24,4 @@ pub struct Imports<'a> {
 
     /// Resolved addresses for imported globals.
     pub globals: &'a [VMGlobalImport],
-
-    /// Resolved imported instances.
-    pub instances: PrimaryMap<InstanceIndex, InstanceHandle>,
-
-    /// Resolved imported modules.
-    ///
-    /// Note that `Box<Any>` here is chosen to allow the embedder of this crate
-    /// to pick an appropriate representation of what module type should be. For
-    /// example for the `wasmtime` crate it's `wasmtime::Module` but that's not
-    /// defined way down here in this low crate.
-    pub modules: PrimaryMap<ModuleIndex, Box<dyn Any>>,
 }
diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs
index 9ff7c4ee6d..1df1cfb640 100644
--- a/crates/runtime/src/instance.rs
+++ b/crates/runtime/src/instance.rs
@@ -14,6 +14,7 @@ use crate::vmcontext::{
     VMSharedSignatureIndex, VMTableDefinition, VMTableImport,
 };
 use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable};
+use indexmap::IndexMap;
 use memoffset::offset_of;
 use more_asserts::assert_lt;
 use std::alloc::{self, Layout};
@@ -22,17 +23,22 @@ use std::cell::RefCell;
 use std::collections::HashMap;
 use std::convert::TryFrom;
 use std::ptr::NonNull;
+use std::rc::Rc;
 use std::sync::Arc;
 use std::{mem, ptr, slice};
 use thiserror::Error;
 use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap};
 use wasmtime_environ::wasm::{
     DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex,
-    ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, InstanceIndex, MemoryIndex,
-    ModuleIndex, SignatureIndex, TableElementType, TableIndex, WasmType,
+    ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex,
+    TableElementType, TableIndex, WasmType,
 };
 use wasmtime_environ::{ir, DataInitializer, Module, ModuleType, TableElements, VMOffsets};
 
+/// Runtime representation of an instance value, which erases all `Instance`
+/// information since instances are just a collection of values.
+pub type RuntimeInstance = Rc<IndexMap<String, Export>>;
+
 /// A WebAssembly instance.
 ///
 /// This is repr(C) to ensure that the vmctx field is last.
@@ -50,15 +56,6 @@ pub(crate) struct Instance {
     /// WebAssembly table data.
     tables: BoxedSlice<DefinedTableIndex, Table>,
 
-    /// Instances our module defined and their handles.
-    instances: PrimaryMap<InstanceIndex, InstanceHandle>,
-
-    /// Modules that are located in our index space.
-    ///
-    /// For now these are `Box<Any>` so the caller can define the type of what a
-    /// module looks like.
-    modules: PrimaryMap<ModuleIndex, Box<dyn Any>>,
-
     /// Passive elements in this instantiation. As `elem.drop`s happen, these
     /// entries get removed. A missing entry is considered equivalent to an
     /// empty slice.
@@ -266,18 +263,8 @@ impl Instance {
         self.vmctx() as *const VMContext as *mut VMContext
     }
 
-    /// Lookup an export with the given name.
-    pub fn lookup(&self, field: &str) -> Option<Export> {
-        let export = if let Some(export) = self.module.exports.get(field) {
-            export.clone()
-        } else {
-            return None;
-        };
-        Some(self.lookup_by_declaration(&export))
-    }
-
     /// Lookup an export with the given export declaration.
-    pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export<'_> {
+    pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export {
         match export {
             EntityIndex::Function(index) => {
                 let anyfunc = self.get_caller_checked_anyfunc(*index).unwrap();
@@ -326,8 +313,9 @@ impl Instance {
             }
             .into(),
 
-            EntityIndex::Instance(index) => Export::Instance(&self.instances[*index]),
-            EntityIndex::Module(index) => Export::Module(&*self.modules[*index]),
+            EntityIndex::Instance(_) | EntityIndex::Module(_) => {
+                panic!("can't use this api for modules/instances")
+            }
         }
     }
 
@@ -855,8 +843,6 @@ impl InstanceHandle {
                 passive_elements: Default::default(),
                 passive_data,
                 host_state,
-                instances: imports.instances,
-                modules: imports.modules,
                 vmctx: VMContext {},
             };
             let layout = instance.alloc_layout();
@@ -1015,11 +1001,6 @@ impl InstanceHandle {
         self.instance().module()
     }
 
-    /// Lookup an export with the given name.
-    pub fn lookup(&self, field: &str) -> Option<Export> {
-        self.instance().lookup(field)
-    }
-
     /// Lookup an export with the given export declaration.
     pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export {
         self.instance().lookup_by_declaration(export)
diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs
index 97e3934ef0..00539c36b4 100644
--- a/crates/runtime/src/lib.rs
+++ b/crates/runtime/src/lib.rs
@@ -37,7 +37,7 @@ pub mod libcalls;
 pub use crate::export::*;
 pub use crate::externref::*;
 pub use crate::imports::Imports;
-pub use crate::instance::{InstanceHandle, InstantiationError, LinkError};
+pub use crate::instance::{InstanceHandle, InstantiationError, LinkError, RuntimeInstance};
 pub use crate::jit_int::GdbJitImageRegistration;
 pub use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator};
 pub use crate::mmap::Mmap;
diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs
index 61e8ca4a21..026738a8a1 100644
--- a/crates/runtime/src/libcalls.rs
+++ b/crates/runtime/src/libcalls.rs
@@ -581,3 +581,8 @@ pub unsafe extern "C" fn wasmtime_imported_memory_atomic_wait64(
         "wasm atomics (fn wasmtime_imported_memory_atomic_wait64) unsupported",
     ))));
 }
+
+/// Hook for when an instance runs out of fuel.
+pub unsafe extern "C" fn wasmtime_out_of_gas(_vmctx: *mut VMContext) {
+    crate::traphandlers::out_of_gas()
+}
diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs
index 948a24d530..feaf2c4a9a 100644
--- a/crates/runtime/src/traphandlers.rs
+++ b/crates/runtime/src/traphandlers.rs
@@ -164,9 +164,15 @@ cfg_if::cfg_if! {
                 } else if #[cfg(all(any(target_os = "linux", target_os = "android"), target_arch = "aarch64"))] {
                     let cx = &*(cx as *const libc::ucontext_t);
                     cx.uc_mcontext.pc as *const u8
-                } else if #[cfg(target_os = "macos")] {
+                } else if #[cfg(all(target_os = "macos", target_arch = "x86_64"))] {
                     let cx = &*(cx as *const libc::ucontext_t);
                     (*cx.uc_mcontext).__ss.__rip as *const u8
+                } else if #[cfg(all(target_os = "macos", target_arch = "x86"))] {
+                    let cx = &*(cx as *const libc::ucontext_t);
+                    (*cx.uc_mcontext).__ss.__eip as *const u8
+                } else if #[cfg(all(target_os = "macos", target_arch = "aarch64"))] {
+                    let cx = &*(cx as *const libc::ucontext_t);
+                    (*cx.uc_mcontext).__ss.__pc as *const u8
                 } else if #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] {
                     let cx = &*(cx as *const libc::ucontext_t);
                     cx.uc_mcontext.mc_rip as *const u8
@@ -404,6 +410,13 @@ pub fn with_last_info<R>(func: impl FnOnce(Option<&dyn Any>) -> R) -> R {
     tls::with(|state| func(state.map(|s| s.trap_info.as_any())))
 }
 
+/// Invokes the contextually-defined context's out-of-gas function.
+///
+/// (basically delegates to `wasmtime::Store::out_of_gas`)
+pub fn out_of_gas() {
+    tls::with(|state| state.unwrap().trap_info.out_of_gas())
+}
+
 /// Temporary state stored on the stack which is registered in the `tls` module
 /// below for calls into wasm.
 pub struct CallThreadState<'a> {
@@ -426,7 +439,7 @@ pub unsafe trait TrapInfo {
 
     /// Returns whether the given program counter lies within wasm code,
     /// indicating whether we should handle a trap or not.
-    fn is_wasm_code(&self, pc: usize) -> bool;
+    fn is_wasm_trap(&self, pc: usize) -> bool;
 
     /// Uses `call` to call a custom signal handler, if one is specified.
     ///
@@ -436,6 +449,12 @@ pub unsafe trait TrapInfo {
     /// Returns the maximum size, in bytes, the wasm native stack is allowed to
     /// grow to.
     fn max_wasm_stack(&self) -> usize;
+
+    /// Callback invoked whenever WebAssembly has entirely consumed the fuel
+    /// that it was allotted.
+    ///
+    /// This function may return, and it may also `raise_lib_trap`.
+    fn out_of_gas(&self);
 }
 
 enum UnwindReason {
@@ -629,7 +648,7 @@ impl<'a> CallThreadState<'a> {
         }
 
         // If this fault wasn't in wasm code, then it's not our problem
-        if !self.trap_info.is_wasm_code(pc as usize) {
+        if !self.trap_info.is_wasm_trap(pc as usize) {
             return ptr::null();
         }
 
diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs
index 3eebdabd4e..c20a42b5ed 100644
--- a/crates/runtime/src/vmcontext.rs
+++ b/crates/runtime/src/vmcontext.rs
@@ -4,6 +4,7 @@
 use crate::externref::VMExternRef;
 use crate::instance::Instance;
 use std::any::Any;
+use std::cell::UnsafeCell;
 use std::ptr::NonNull;
 use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
 use std::u32;
@@ -612,6 +613,7 @@ impl VMBuiltinFunctionsArray {
             wasmtime_memory_atomic_wait64 as usize;
         ptrs[BuiltinFunctionIndex::imported_memory_atomic_wait64().index() as usize] =
             wasmtime_imported_memory_atomic_wait64 as usize;
+        ptrs[BuiltinFunctionIndex::out_of_gas().index() as usize] = wasmtime_out_of_gas as usize;
 
         if cfg!(debug_assertions) {
             for i in 0..ptrs.len() {
@@ -658,8 +660,7 @@ impl VMInvokeArgument {
     }
 }
 
-/// Structure used to control interrupting wasm code, currently with only one
-/// atomic flag internally used.
+/// Structure used to control interrupting wasm code.
 #[derive(Debug)]
 #[repr(C)]
 pub struct VMInterrupts {
@@ -668,6 +669,14 @@ pub struct VMInterrupts {
     /// This is used to control both stack overflow as well as interrupting wasm
     /// modules. For more information see `crates/environ/src/cranelift.rs`.
     pub stack_limit: AtomicUsize,
+
+    /// Indicator of how much fuel has been consumed and is remaining to
+    /// WebAssembly.
+    ///
+    /// This field is typically negative and increments towards positive. Upon
+    /// turning positive a wasm trap will be generated. This field is only
+    /// modified if wasm is configured to consume fuel.
+    pub fuel_consumed: UnsafeCell<i64>,
 }
 
 impl VMInterrupts {
@@ -682,6 +691,7 @@ impl Default for VMInterrupts {
     fn default() -> VMInterrupts {
         VMInterrupts {
             stack_limit: AtomicUsize::new(usize::max_value()),
+            fuel_consumed: UnsafeCell::new(0),
         }
     }
 }
diff --git a/crates/test-programs/Cargo.toml b/crates/test-programs/Cargo.toml
index 20a6efc653..ca7359ebeb 100644
--- a/crates/test-programs/Cargo.toml
+++ b/crates/test-programs/Cargo.toml
@@ -21,7 +21,7 @@ tempfile = "3.1.0"
 os_pipe = "0.9"
 anyhow = "1.0.19"
 wat = "1.0.23"
-cap-std = "0.11"
+cap-std = "0.12"
 
 [features]
 test_programs = []
diff --git a/crates/wasi-common/Cargo.toml b/crates/wasi-common/Cargo.toml
index f815cb7568..8980f20bec 100644
--- a/crates/wasi-common/Cargo.toml
+++ b/crates/wasi-common/Cargo.toml
@@ -24,8 +24,8 @@ thiserror = "1.0"
 wiggle = { path = "../wiggle", default-features = false, version = "0.22.0" }
 tracing = "0.1.19"
 system-interface = { version = "0.5.4", features = ["cap_std_impls"] }
-cap-std = "0.11"
-cap-rand = "0.11"
+cap-std = "0.12"
+cap-rand = "0.12"
 bitflags = "1.2"
 
 [target.'cfg(unix)'.dependencies]
diff --git a/crates/wasi-common/cap-std-sync/Cargo.toml b/crates/wasi-common/cap-std-sync/Cargo.toml
index 65c060f818..17320baf17 100644
--- a/crates/wasi-common/cap-std-sync/Cargo.toml
+++ b/crates/wasi-common/cap-std-sync/Cargo.toml
@@ -15,10 +15,10 @@ publish = false
 [dependencies]
 wasi-common = { path = "../", version = "0.22.0" }
 anyhow = "1.0"
-cap-std = "0.11"
-cap-fs-ext = "0.11"
-cap-time-ext = "0.11"
-cap-rand = "0.11"
+cap-std = "0.12"
+cap-fs-ext = "0.12"
+cap-time-ext = "0.12"
+cap-rand = "0.12"
 fs-set-times = "0.2.2"
 unsafe-io = "0.3"
 system-interface = { version = "0.5.4", features = ["cap_std_impls"] }
diff --git a/crates/wasi-crypto/Cargo.toml b/crates/wasi-crypto/Cargo.toml
new file mode 100644
index 0000000000..a0df36082d
--- /dev/null
+++ b/crates/wasi-crypto/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "wasmtime-wasi-crypto"
+version = "0.22.0"
+authors = ["The Wasmtime Project Developers"]
+description = "Wasmtime implementation of the wasi-crypto API"
+documentation = "https://docs.rs/wasmtime-wasi-crypto"
+license = "Apache-2.0 WITH LLVM-exception"
+categories = ["wasm", "cryptography"]
+keywords = ["webassembly", "wasm", "crypto"]
+repository = "https://github.com/bytecodealliance/wasmtime"
+readme = "README.md"
+edition = "2018"
+
+[dependencies]
+anyhow = "1.0"
+wasi-crypto = { path = "spec/implementations/hostcalls/rust", version = "0.1.4" }
+wasmtime = { path = "../wasmtime", version = "0.22.0", default-features = false }
+wasmtime-wiggle = { path = "../wiggle/wasmtime", version = "0.22.0" }
+wiggle = { path = "../wiggle", version = "0.22.0" }
+
+[badges]
+maintenance = { status = "experimental" }
diff --git a/crates/wasi-crypto/LICENSE b/crates/wasi-crypto/LICENSE
new file mode 100644
index 0000000000..f9d81955f4
--- /dev/null
+++ b/crates/wasi-crypto/LICENSE
@@ -0,0 +1,220 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+--- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+
diff --git a/crates/wasi-crypto/README.md b/crates/wasi-crypto/README.md
new file mode 100644
index 0000000000..ca8766e1a0
--- /dev/null
+++ b/crates/wasi-crypto/README.md
@@ -0,0 +1,56 @@
+# wasmtime-wasi-crypto
+
+This crate enables support for the [wasi-crypto] APIs in Wasmtime.
+
+The sole purpose of the implementation is to allow bindings and
+application developers to test the proposed APIs. This implementation
+is not meant to be used in production. Like the specification, it is
+currently experimental and its functionality can quickly change.
+
+Since the [wasi-crypto] API is expected to be an optional feature of
+WASI, this crate is currently separate from the [wasi-common] crate.
+
+* [documentation]
+* [interfaces reference]
+* [interfaces reference (compact)]
+
+[wasi-crypto]: https://github.com/WebAssembly/wasi-crypto
+[wasi-common]: ../../wasi-common
+[documentation]: ../spec/docs/wasi-crypto.md
+[interfaces reference]: ../spec/witx/wasi_ephemeral_crypto.md
+[interfaces reference (compact)]: ../spec/witx/wasi_ephemeral_crypto.txt
+
+## Wasmtime integration
+
+Use the Wasmtime APIs to instantiate a Wasm module and link the
+`wasi-crypto` modules as follows:
+
+```rust
+use wasmtime_wasi_crypto::{
+    WasiCryptoAsymmetricCommon, WasiCryptoCommon, WasiCryptoCtx, WasiCryptoSignatures,
+    WasiCryptoSymmetric,
+};
+
+let cx_crypto = WasiCryptoCtx::new();
+WasiCryptoCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?;
+WasiCryptoAsymmetricCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?;
+WasiCryptoSignatures::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?;
+WasiCryptoSymmetric::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?;
+
+let wasi = wasmtime_wasi::old::snapshot_0::Wasi::new(linker.store(), mk_cx()?);
+wasi.add_to_linker(linker)?;
+```
+
+## Building Wasmtime
+
+Wasmtime must be compiled with the `wasi-crypto` feature flag
+(disabled by default) in order to include the crypto APIs.
+
+## Examples
+
+Example [rust bindings] and [assemblyscript bindings] are provided to
+demonstrate how these APIs can be used and exposed to applications in
+an idiomatic way.
+
+[rust bindings]: ../spec/implementations/bindings/rust
+[assemblyscript bindings]: ../spec/implementations/bindings/assemblyscript
diff --git a/crates/wasi-crypto/spec b/crates/wasi-crypto/spec
new file mode 160000
index 0000000000..6d7821dec3
--- /dev/null
+++ b/crates/wasi-crypto/spec
@@ -0,0 +1 @@
+Subproject commit 6d7821dec301a11dcf3c0d50e5a51af5169eaee3
diff --git a/crates/wasi-crypto/src/lib.rs b/crates/wasi-crypto/src/lib.rs
new file mode 100644
index 0000000000..42a21f36d4
--- /dev/null
+++ b/crates/wasi-crypto/src/lib.rs
@@ -0,0 +1,31 @@
+mod wiggle_interfaces;
+
+pub use wiggle_interfaces::WasiCryptoCtx;
+
+wasmtime_wiggle::wasmtime_integration!({
+    target: wiggle_interfaces::wasi_modules,
+    witx: ["$CARGO_MANIFEST_DIR/spec/witx/wasi_ephemeral_crypto.witx"],
+    ctx: WasiCryptoCtx,
+    modules: {
+        wasi_ephemeral_crypto_common =>
+            {
+                name: WasiCryptoCommon,
+                docs: "wasi-crypto - Common module."
+            },
+        wasi_ephemeral_crypto_asymmetric_common =>
+            {
+                name: WasiCryptoAsymmetricCommon,
+                docs: "wasi-crypto - Common module for asymmetric operations."
+            },
+        wasi_ephemeral_crypto_signatures =>
+            {
+                name: WasiCryptoSignatures,
+                docs: "wasi-crypto - Signature module."
+            },
+        wasi_ephemeral_crypto_symmetric =>
+            {
+                name: WasiCryptoSymmetric,
+                docs: "wasi-crypto - Symmetric cryptography module."
+            }
+    }
+});
diff --git a/crates/wasi-crypto/src/wiggle_interfaces/asymmetric_common.rs b/crates/wasi-crypto/src/wiggle_interfaces/asymmetric_common.rs
new file mode 100644
index 0000000000..8de31ee429
--- /dev/null
+++ b/crates/wasi-crypto/src/wiggle_interfaces/asymmetric_common.rs
@@ -0,0 +1,292 @@
+use super::{guest_types, WasiCryptoCtx};
+
+use std::convert::TryInto;
+use wasi_crypto::{ensure, CryptoError, KeyPairEncoding, PublicKeyEncoding, SecretKeyEncoding};
+
+impl super::wasi_ephemeral_crypto_asymmetric_common::WasiEphemeralCryptoAsymmetricCommon
+    for WasiCryptoCtx
+{
+    // --- keypair_manager
+
+    fn keypair_generate_managed(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+        alg_type: guest_types::AlgorithmType,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        options_handle: &guest_types::OptOptions,
+    ) -> Result<guest_types::Keypair, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let options_handle = match *options_handle {
+            guest_types::OptOptions::Some(options_handle) => Some(options_handle),
+            guest_types::OptOptions::None => None,
+        };
+        Ok(self
+            .ctx
+            .keypair_generate_managed(
+                secrets_manager_handle.into(),
+                alg_type.into(),
+                alg_str,
+                options_handle.map(Into::into),
+            )?
+            .into())
+    }
+
+    fn keypair_store_managed(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+        kp_handle: guest_types::Keypair,
+        kp_id_ptr: &wiggle::GuestPtr<'_, u8>,
+        kp_id_max_len: guest_types::Size,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let key_id_buf = &mut *kp_id_ptr.as_array(kp_id_max_len).as_slice_mut()?;
+        Ok(self.ctx.keypair_store_managed(
+            secrets_manager_handle.into(),
+            kp_handle.into(),
+            key_id_buf,
+        )?)
+    }
+
+    fn keypair_replace_managed(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+        kp_old_handle: guest_types::Keypair,
+        kp_new_handle: guest_types::Keypair,
+    ) -> Result<guest_types::Version, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .keypair_replace_managed(
+                secrets_manager_handle.into(),
+                kp_old_handle.into(),
+                kp_new_handle.into(),
+            )?
+            .into())
+    }
+
+    fn keypair_from_id(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+        kp_id_ptr: &wiggle::GuestPtr<'_, u8>,
+        kp_id_len: guest_types::Size,
+        kp_version: guest_types::Version,
+    ) -> Result<guest_types::Keypair, guest_types::CryptoErrno> {
+        let kp_id = &*kp_id_ptr.as_array(kp_id_len).as_slice()?;
+        Ok(self
+            .ctx
+            .keypair_from_id(secrets_manager_handle.into(), kp_id, kp_version.into())?
+            .into())
+    }
+
+    // --- keypair
+
+    fn keypair_generate(
+        &self,
+        alg_type: guest_types::AlgorithmType,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        options_handle: &guest_types::OptOptions,
+    ) -> Result<guest_types::Keypair, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let options_handle = match *options_handle {
+            guest_types::OptOptions::Some(options_handle) => Some(options_handle),
+            guest_types::OptOptions::None => None,
+        };
+        Ok(self
+            .ctx
+            .keypair_generate(alg_type.into(), alg_str, options_handle.map(Into::into))?
+            .into())
+    }
+
+    fn keypair_import(
+        &self,
+        alg_type: guest_types::AlgorithmType,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        encoded_ptr: &wiggle::GuestPtr<'_, u8>,
+        encoded_len: guest_types::Size,
+        encoding: guest_types::KeypairEncoding,
+    ) -> Result<guest_types::Keypair, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let encoded = &*encoded_ptr.as_array(encoded_len).as_slice()?;
+        Ok(self
+            .ctx
+            .keypair_import(alg_type.into(), alg_str, encoded, encoding.into())?
+            .into())
+    }
+
+    fn keypair_id(
+        &self,
+        kp_handle: guest_types::Keypair,
+        kp_id_ptr: &wiggle::GuestPtr<'_, u8>,
+        kp_id_max_len: guest_types::Size,
+    ) -> Result<(guest_types::Size, guest_types::Version), guest_types::CryptoErrno> {
+        let kp_id_buf = &mut *kp_id_ptr.as_array(kp_id_max_len as _).as_slice_mut()?;
+        let (kp_id, version) = self.ctx.keypair_id(kp_handle.into())?;
+        ensure!(kp_id.len() <= kp_id_buf.len(), CryptoError::Overflow.into());
+        kp_id_buf.copy_from_slice(&kp_id);
+        Ok((kp_id.len().try_into()?, version.into()))
+    }
+
+    fn keypair_export(
+        &self,
+        kp_handle: guest_types::Keypair,
+        encoding: guest_types::KeypairEncoding,
+    ) -> Result<guest_types::ArrayOutput, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .keypair_export(kp_handle.into(), encoding.into())?
+            .into())
+    }
+
+    fn keypair_publickey(
+        &self,
+        kp_handle: guest_types::Keypair,
+    ) -> Result<guest_types::Publickey, guest_types::CryptoErrno> {
+        Ok(self.ctx.keypair_publickey(kp_handle.into())?.into())
+    }
+
+    fn keypair_close(
+        &self,
+        kp_handle: guest_types::Keypair,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self.ctx.keypair_close(kp_handle.into())?)
+    }
+
+    // --- publickey
+
+    fn publickey_import(
+        &self,
+        alg_type: guest_types::AlgorithmType,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        encoded_ptr: &wiggle::GuestPtr<'_, u8>,
+        encoded_len: guest_types::Size,
+        encoding: guest_types::PublickeyEncoding,
+    ) -> Result<guest_types::Publickey, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let encoded = &*encoded_ptr.as_array(encoded_len).as_slice()?;
+        Ok(self
+            .ctx
+            .publickey_import(alg_type.into(), alg_str, encoded, encoding.into())?
+            .into())
+    }
+
+    fn publickey_export(
+        &self,
+        pk_handle: guest_types::Publickey,
+        encoding: guest_types::PublickeyEncoding,
+    ) -> Result<guest_types::ArrayOutput, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .publickey_export(pk_handle.into(), encoding.into())?
+            .into())
+    }
+
+    fn publickey_from_secretkey(
+        &self,
+        sk_handle: guest_types::Secretkey,
+    ) -> Result<guest_types::Publickey, guest_types::CryptoErrno> {
+        Ok(self.ctx.keypair_publickey(sk_handle.into())?.into())
+    }
+
+    fn publickey_verify(
+        &self,
+        pk_handle: guest_types::Publickey,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self.ctx.publickey_verify(pk_handle.into())?)
+    }
+
+    fn publickey_close(
+        &self,
+        pk_handle: guest_types::Publickey,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self.ctx.publickey_close(pk_handle.into())?)
+    }
+
+    // --- secretkey
+
+    fn secretkey_import(
+        &self,
+        alg_type: guest_types::AlgorithmType,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        encoded_ptr: &wiggle::GuestPtr<'_, u8>,
+        encoded_len: guest_types::Size,
+        encoding: guest_types::SecretkeyEncoding,
+    ) -> Result<guest_types::Secretkey, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let encoded = &*encoded_ptr.as_array(encoded_len).as_slice()?;
+        Ok(self
+            .ctx
+            .secretkey_import(alg_type.into(), alg_str, encoded, encoding.into())?
+            .into())
+    }
+
+    fn secretkey_export(
+        &self,
+        sk_handle: guest_types::Secretkey,
+        encoding: guest_types::SecretkeyEncoding,
+    ) -> Result<guest_types::ArrayOutput, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .secretkey_export(sk_handle.into(), encoding.into())?
+            .into())
+    }
+
+    fn secretkey_close(
+        &self,
+        sk_handle: guest_types::Secretkey,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self.ctx.secretkey_close(sk_handle.into())?)
+    }
+
+    fn keypair_from_pk_and_sk(
+        &self,
+        pk_handle: guest_types::Publickey,
+        sk_handle: guest_types::Secretkey,
+    ) -> Result<guest_types::Keypair, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .keypair_from_pk_and_sk(pk_handle.into(), sk_handle.into())?
+            .into())
+    }
+
+    fn keypair_secretkey(
+        &self,
+        kp_handle: guest_types::Keypair,
+    ) -> Result<guest_types::Secretkey, guest_types::CryptoErrno> {
+        Ok(self.ctx.keypair_secretkey(kp_handle.into())?.into())
+    }
+}
+
+impl From<guest_types::KeypairEncoding> for KeyPairEncoding {
+    fn from(encoding: guest_types::KeypairEncoding) -> Self {
+        match encoding {
+            guest_types::KeypairEncoding::Raw => KeyPairEncoding::Raw,
+            guest_types::KeypairEncoding::Pkcs8 => KeyPairEncoding::Pkcs8,
+            guest_types::KeypairEncoding::Pem => KeyPairEncoding::Pem,
+            guest_types::KeypairEncoding::Local => KeyPairEncoding::Local,
+        }
+    }
+}
+
+impl From<guest_types::PublickeyEncoding> for PublicKeyEncoding {
+    fn from(encoding: guest_types::PublickeyEncoding) -> Self {
+        match encoding {
+            guest_types::PublickeyEncoding::Raw => PublicKeyEncoding::Raw,
+            guest_types::PublickeyEncoding::Pkcs8 => PublicKeyEncoding::Pkcs8,
+            guest_types::PublickeyEncoding::Pem => PublicKeyEncoding::Pem,
+            guest_types::PublickeyEncoding::Sec => PublicKeyEncoding::Sec,
+            guest_types::PublickeyEncoding::CompressedSec => PublicKeyEncoding::CompressedSec,
+            guest_types::PublickeyEncoding::Local => PublicKeyEncoding::Local,
+        }
+    }
+}
+
+impl From<guest_types::SecretkeyEncoding> for SecretKeyEncoding {
+    fn from(encoding: guest_types::SecretkeyEncoding) -> Self {
+        match encoding {
+            guest_types::SecretkeyEncoding::Raw => SecretKeyEncoding::Raw,
+            guest_types::SecretkeyEncoding::Pkcs8 => SecretKeyEncoding::Pkcs8,
+            guest_types::SecretkeyEncoding::Pem => SecretKeyEncoding::Pem,
+            guest_types::SecretkeyEncoding::Sec => SecretKeyEncoding::Sec,
+            guest_types::SecretkeyEncoding::CompressedSec => SecretKeyEncoding::CompressedSec,
+            guest_types::SecretkeyEncoding::Local => SecretKeyEncoding::Local,
+        }
+    }
+}
diff --git a/crates/wasi-crypto/src/wiggle_interfaces/common.rs b/crates/wasi-crypto/src/wiggle_interfaces/common.rs
new file mode 100644
index 0000000000..f0d5a32ff8
--- /dev/null
+++ b/crates/wasi-crypto/src/wiggle_interfaces/common.rs
@@ -0,0 +1,150 @@
+use super::{guest_types, WasiCryptoCtx};
+
+use std::convert::TryInto;
+use wasi_crypto::{AlgorithmType, Version};
+
+impl super::wasi_ephemeral_crypto_common::WasiEphemeralCryptoCommon for WasiCryptoCtx {
+    // --- options
+
+    fn options_open(
+        &self,
+        options_type: guest_types::AlgorithmType,
+    ) -> Result<guest_types::Options, guest_types::CryptoErrno> {
+        Ok(self.ctx.options_open(options_type.into())?.into())
+    }
+
+    fn options_close(
+        &self,
+        options_handle: guest_types::Options,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self.ctx.options_close(options_handle.into())?)
+    }
+
+    fn options_set(
+        &self,
+        options_handle: guest_types::Options,
+        name_str: &wiggle::GuestPtr<'_, str>,
+        value_ptr: &wiggle::GuestPtr<'_, u8>,
+        value_len: guest_types::Size,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let name_str: &str = &*name_str.as_str()?;
+        let value: &[u8] = { &*value_ptr.as_array(value_len).as_slice()? };
+        Ok(self
+            .ctx
+            .options_set(options_handle.into(), name_str, value)?)
+    }
+
+    fn options_set_guest_buffer(
+        &self,
+        options_handle: guest_types::Options,
+        name_str: &wiggle::GuestPtr<'_, str>,
+        buffer_ptr: &wiggle::GuestPtr<'_, u8>,
+        buffer_len: guest_types::Size,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let name_str: &str = &*name_str.as_str()?;
+        let buffer: &'static mut [u8] =
+            unsafe { std::mem::transmute(&mut *buffer_ptr.as_array(buffer_len).as_slice_mut()?) };
+        Ok(self
+            .ctx
+            .options_set_guest_buffer(options_handle.into(), name_str, buffer)?)
+    }
+
+    fn options_set_u64(
+        &self,
+        options_handle: guest_types::Options,
+        name_str: &wiggle::GuestPtr<'_, str>,
+        value: u64,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let name_str: &str = &*name_str.as_str()?;
+        Ok(self
+            .ctx
+            .options_set_u64(options_handle.into(), name_str, value)?)
+    }
+
+    // --- array
+
+    fn array_output_len(
+        &self,
+        array_output_handle: guest_types::ArrayOutput,
+    ) -> Result<guest_types::Size, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .array_output_len(array_output_handle.into())?
+            .try_into()?)
+    }
+
+    fn array_output_pull(
+        &self,
+        array_output_handle: guest_types::ArrayOutput,
+        buf_ptr: &wiggle::GuestPtr<'_, u8>,
+        buf_len: guest_types::Size,
+    ) -> Result<guest_types::Size, guest_types::CryptoErrno> {
+        let buf: &mut [u8] = { &mut *buf_ptr.as_array(buf_len).as_slice_mut()? };
+        Ok(self
+            .ctx
+            .array_output_pull(array_output_handle.into(), buf)?
+            .try_into()?)
+    }
+
+    // --- secrets_manager
+
+    fn secrets_manager_open(
+        &self,
+        options_handle: &guest_types::OptOptions,
+    ) -> Result<guest_types::SecretsManager, guest_types::CryptoErrno> {
+        let options_handle = match *options_handle {
+            guest_types::OptOptions::Some(options_handle) => Some(options_handle),
+            guest_types::OptOptions::None => None,
+        };
+        Ok(self
+            .ctx
+            .secrets_manager_open(options_handle.map(Into::into))?
+            .into())
+    }
+
+    fn secrets_manager_close(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .secrets_manager_close(secrets_manager_handle.into())?)
+    }
+
+    fn secrets_manager_invalidate(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+        key_id_ptr: &wiggle::GuestPtr<'_, u8>,
+        key_id_len: guest_types::Size,
+        key_version: guest_types::Version,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let key_id: &[u8] = { &*key_id_ptr.as_array(key_id_len).as_slice()? };
+        Ok(self.ctx.secrets_manager_invalidate(
+            secrets_manager_handle.into(),
+            key_id,
+            key_version.into(),
+        )?)
+    }
+}
+
+impl From<guest_types::AlgorithmType> for AlgorithmType {
+    fn from(options_type: guest_types::AlgorithmType) -> Self {
+        match options_type {
+            guest_types::AlgorithmType::Signatures => AlgorithmType::Signatures,
+            guest_types::AlgorithmType::Symmetric => AlgorithmType::Symmetric,
+            guest_types::AlgorithmType::KeyExchange => AlgorithmType::KeyExchange,
+        }
+    }
+}
+
+impl From<guest_types::Version> for Version {
+    fn from(version: guest_types::Version) -> Self {
+        Version(version.into())
+    }
+}
+
+impl From<Version> for guest_types::Version {
+    fn from(version: Version) -> Self {
+        version.into()
+    }
+}
diff --git a/crates/wasi-crypto/src/wiggle_interfaces/error.rs b/crates/wasi-crypto/src/wiggle_interfaces/error.rs
new file mode 100644
index 0000000000..e3651f6db9
--- /dev/null
+++ b/crates/wasi-crypto/src/wiggle_interfaces/error.rs
@@ -0,0 +1,67 @@
+use super::{guest_types, WasiCryptoCtx};
+
+use std::num::TryFromIntError;
+use wasi_crypto::CryptoError;
+
+impl From<CryptoError> for guest_types::CryptoErrno {
+    fn from(e: CryptoError) -> Self {
+        match e {
+            CryptoError::Success => guest_types::CryptoErrno::Success,
+            CryptoError::GuestError(_wiggle_error) => guest_types::CryptoErrno::GuestError,
+            CryptoError::NotImplemented => guest_types::CryptoErrno::NotImplemented,
+            CryptoError::UnsupportedFeature => guest_types::CryptoErrno::UnsupportedFeature,
+            CryptoError::ProhibitedOperation => guest_types::CryptoErrno::ProhibitedOperation,
+            CryptoError::UnsupportedEncoding => guest_types::CryptoErrno::UnsupportedEncoding,
+            CryptoError::UnsupportedAlgorithm => guest_types::CryptoErrno::UnsupportedAlgorithm,
+            CryptoError::UnsupportedOption => guest_types::CryptoErrno::UnsupportedOption,
+            CryptoError::InvalidKey => guest_types::CryptoErrno::InvalidKey,
+            CryptoError::InvalidLength => guest_types::CryptoErrno::InvalidLength,
+            CryptoError::VerificationFailed => guest_types::CryptoErrno::VerificationFailed,
+            CryptoError::RNGError => guest_types::CryptoErrno::RngError,
+            CryptoError::AlgorithmFailure => guest_types::CryptoErrno::AlgorithmFailure,
+            CryptoError::InvalidSignature => guest_types::CryptoErrno::InvalidSignature,
+            CryptoError::Closed => guest_types::CryptoErrno::Closed,
+            CryptoError::InvalidHandle => guest_types::CryptoErrno::InvalidHandle,
+            CryptoError::Overflow => guest_types::CryptoErrno::Overflow,
+            CryptoError::InternalError => guest_types::CryptoErrno::InternalError,
+            CryptoError::TooManyHandles => guest_types::CryptoErrno::TooManyHandles,
+            CryptoError::KeyNotSupported => guest_types::CryptoErrno::KeyNotSupported,
+            CryptoError::KeyRequired => guest_types::CryptoErrno::KeyRequired,
+            CryptoError::InvalidTag => guest_types::CryptoErrno::InvalidTag,
+            CryptoError::InvalidOperation => guest_types::CryptoErrno::InvalidOperation,
+            CryptoError::NonceRequired => guest_types::CryptoErrno::NonceRequired,
+            CryptoError::InvalidNonce => guest_types::CryptoErrno::InvalidNonce,
+            CryptoError::OptionNotSet => guest_types::CryptoErrno::OptionNotSet,
+            CryptoError::NotFound => guest_types::CryptoErrno::NotFound,
+            CryptoError::ParametersMissing => guest_types::CryptoErrno::ParametersMissing,
+            CryptoError::IncompatibleKeys => guest_types::CryptoErrno::IncompatibleKeys,
+            CryptoError::Expired => guest_types::CryptoErrno::Expired,
+        }
+    }
+}
+
+impl From<TryFromIntError> for guest_types::CryptoErrno {
+    fn from(_: TryFromIntError) -> Self {
+        CryptoError::Overflow.into()
+    }
+}
+
+impl<'a> wiggle::GuestErrorType for guest_types::CryptoErrno {
+    fn success() -> Self {
+        guest_types::CryptoErrno::Success
+    }
+}
+
+impl guest_types::GuestErrorConversion for WasiCryptoCtx {
+    fn into_crypto_errno(&self, e: wiggle::GuestError) -> guest_types::CryptoErrno {
+        eprintln!("GuestError (witx) {:?}", e);
+        guest_types::CryptoErrno::GuestError
+    }
+}
+
+impl From<wiggle::GuestError> for guest_types::CryptoErrno {
+    fn from(e: wiggle::GuestError) -> Self {
+        eprintln!("GuestError (impl) {:?}", e);
+        guest_types::CryptoErrno::GuestError
+    }
+}
diff --git a/crates/wasi-crypto/src/wiggle_interfaces/key_exchange.rs b/crates/wasi-crypto/src/wiggle_interfaces/key_exchange.rs
new file mode 100644
index 0000000000..a1685de116
--- /dev/null
+++ b/crates/wasi-crypto/src/wiggle_interfaces/key_exchange.rs
@@ -0,0 +1,40 @@
+use super::{guest_types, WasiCryptoCtx};
+
+impl super::wasi_ephemeral_crypto_kx::WasiEphemeralCryptoKx for WasiCryptoCtx {
+    // --- key exchange
+
+    fn kx_dh(
+        &self,
+        pk_handle: guest_types::Publickey,
+        sk_handle: guest_types::Secretkey,
+    ) -> Result<guest_types::ArrayOutput, guest_types::CryptoErrno> {
+        Ok(self.ctx.kx_dh(pk_handle.into(), sk_handle.into())?.into())
+    }
+
+    // --- Key encapsulation
+
+    fn kx_encapsulate(
+        &self,
+        pk_handle: guest_types::Publickey,
+    ) -> Result<(guest_types::ArrayOutput, guest_types::ArrayOutput), guest_types::CryptoErrno>
+    {
+        let (secret_handle, encapsulated_secret_handle) =
+            self.ctx.kx_encapsulate(pk_handle.into())?;
+        Ok((secret_handle.into(), encapsulated_secret_handle.into()))
+    }
+
+    fn kx_decapsulate(
+        &self,
+        sk_handle: guest_types::Secretkey,
+        encapsulated_secret_ptr: &wiggle::GuestPtr<'_, u8>,
+        encapsulated_secret_len: guest_types::Size,
+    ) -> Result<guest_types::ArrayOutput, guest_types::CryptoErrno> {
+        let encapsulated_secret = &*encapsulated_secret_ptr
+            .as_array(encapsulated_secret_len)
+            .as_slice()?;
+        Ok(self
+            .ctx
+            .kx_decapsulate(sk_handle.into(), encapsulated_secret)?
+            .into())
+    }
+}
diff --git a/crates/wasi-crypto/src/wiggle_interfaces/mod.rs b/crates/wasi-crypto/src/wiggle_interfaces/mod.rs
new file mode 100644
index 0000000000..8784a10bd1
--- /dev/null
+++ b/crates/wasi-crypto/src/wiggle_interfaces/mod.rs
@@ -0,0 +1,38 @@
+use std::rc::Rc;
+
+use wasi_crypto::CryptoCtx;
+
+wiggle::from_witx!({
+    witx: ["$CARGO_MANIFEST_DIR/spec/witx/wasi_ephemeral_crypto.witx"],
+    ctx: WasiCryptoCtx
+});
+
+pub mod wasi_modules {
+    pub use super::{
+        wasi_ephemeral_crypto_asymmetric_common, wasi_ephemeral_crypto_common,
+        wasi_ephemeral_crypto_kx, wasi_ephemeral_crypto_signatures,
+        wasi_ephemeral_crypto_symmetric,
+    };
+}
+
+pub use types as guest_types;
+
+#[derive(Clone)]
+pub struct WasiCryptoCtx {
+    ctx: Rc<CryptoCtx>,
+}
+
+impl WasiCryptoCtx {
+    pub fn new() -> Self {
+        WasiCryptoCtx {
+            ctx: Rc::new(CryptoCtx::new()),
+        }
+    }
+}
+
+mod asymmetric_common;
+mod common;
+mod error;
+mod key_exchange;
+mod signatures;
+mod symmetric;
diff --git a/crates/wasi-crypto/src/wiggle_interfaces/signatures.rs b/crates/wasi-crypto/src/wiggle_interfaces/signatures.rs
new file mode 100644
index 0000000000..64fc56ed2e
--- /dev/null
+++ b/crates/wasi-crypto/src/wiggle_interfaces/signatures.rs
@@ -0,0 +1,129 @@
+use super::{guest_types, WasiCryptoCtx};
+
+use wasi_crypto::SignatureEncoding;
+
+impl super::wasi_ephemeral_crypto_signatures::WasiEphemeralCryptoSignatures for WasiCryptoCtx {
+    // --- signature
+
+    fn signature_export(
+        &self,
+        signature_handle: guest_types::Signature,
+        encoding: guest_types::SignatureEncoding,
+    ) -> Result<guest_types::ArrayOutput, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .signature_export(signature_handle.into(), encoding.into())?
+            .into())
+    }
+
+    fn signature_import(
+        &self,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        encoded_ptr: &wiggle::GuestPtr<'_, u8>,
+        encoded_len: guest_types::Size,
+        encoding: guest_types::SignatureEncoding,
+    ) -> Result<guest_types::Signature, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let encoded = &*encoded_ptr.as_array(encoded_len).as_slice()?;
+        Ok(self
+            .ctx
+            .signature_import(alg_str, encoded, encoding.into())?
+            .into())
+    }
+
+    fn signature_state_open(
+        &self,
+        kp_handle: guest_types::Keypair,
+    ) -> Result<guest_types::SignatureState, guest_types::CryptoErrno> {
+        Ok(self.ctx.signature_state_open(kp_handle.into())?.into())
+    }
+
+    fn signature_state_update(
+        &self,
+        state_handle: guest_types::SignatureState,
+        input_ptr: &wiggle::GuestPtr<'_, u8>,
+        input_len: guest_types::Size,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let input = &*input_ptr.as_array(input_len).as_slice()?;
+        Ok(self
+            .ctx
+            .signature_state_update(state_handle.into(), input)?)
+    }
+
+    fn signature_state_sign(
+        &self,
+        signature_state_handle: guest_types::SignatureState,
+    ) -> Result<guest_types::ArrayOutput, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .signature_state_sign(signature_state_handle.into())?
+            .into())
+    }
+
+    fn signature_state_close(
+        &self,
+        signature_state_handle: guest_types::SignatureState,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .signature_state_close(signature_state_handle.into())?)
+    }
+
+    fn signature_verification_state_open(
+        &self,
+        pk_handle: guest_types::Publickey,
+    ) -> Result<guest_types::SignatureVerificationState, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .signature_verification_state_open(pk_handle.into())?
+            .into())
+    }
+
+    fn signature_verification_state_update(
+        &self,
+        verification_state_handle: guest_types::SignatureVerificationState,
+        input_ptr: &wiggle::GuestPtr<'_, u8>,
+        input_len: guest_types::Size,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let input: &[u8] = &*input_ptr.as_array(input_len).as_slice()?;
+        Ok(self
+            .ctx
+            .signature_verification_state_update(verification_state_handle.into(), input)?)
+    }
+
+    fn signature_verification_state_verify(
+        &self,
+        verification_state_handle: guest_types::SignatureVerificationState,
+        signature_handle: guest_types::Signature,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self.ctx.signature_verification_state_verify(
+            verification_state_handle.into(),
+            signature_handle.into(),
+        )?)
+    }
+
+    fn signature_verification_state_close(
+        &self,
+        verification_state_handle: guest_types::SignatureVerificationState,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .signature_verification_state_close(verification_state_handle.into())?)
+    }
+
+    fn signature_close(
+        &self,
+        signature_handle: guest_types::Signature,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self.ctx.signature_close(signature_handle.into())?)
+    }
+}
+
+impl From<guest_types::SignatureEncoding> for SignatureEncoding {
+    fn from(encoding: guest_types::SignatureEncoding) -> Self {
+        match encoding {
+            guest_types::SignatureEncoding::Raw => SignatureEncoding::Raw,
+            guest_types::SignatureEncoding::Der => SignatureEncoding::Der,
+        }
+    }
+}
diff --git a/crates/wasi-crypto/src/wiggle_interfaces/symmetric.rs b/crates/wasi-crypto/src/wiggle_interfaces/symmetric.rs
new file mode 100644
index 0000000000..c4f1d8f32b
--- /dev/null
+++ b/crates/wasi-crypto/src/wiggle_interfaces/symmetric.rs
@@ -0,0 +1,384 @@
+use super::{guest_types, WasiCryptoCtx};
+
+use std::convert::TryInto;
+use wasi_crypto::{ensure, CryptoError};
+
+impl super::wasi_ephemeral_crypto_symmetric::WasiEphemeralCryptoSymmetric for WasiCryptoCtx {
+    // --- secrets_manager
+
+    fn symmetric_key_generate_managed(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        options_handle: &guest_types::OptOptions,
+    ) -> Result<guest_types::SymmetricKey, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let options_handle = match *options_handle {
+            guest_types::OptOptions::Some(options_handle) => Some(options_handle),
+            guest_types::OptOptions::None => None,
+        };
+        Ok(self
+            .ctx
+            .symmetric_key_generate_managed(
+                secrets_manager_handle.into(),
+                alg_str,
+                options_handle.map(Into::into),
+            )?
+            .into())
+    }
+
+    fn symmetric_key_store_managed(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+        symmetric_key_handle: guest_types::SymmetricKey,
+        symmetric_key_id_ptr: &wiggle::GuestPtr<'_, u8>,
+        symmetric_key_id_max_len: guest_types::Size,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let key_id_buf = &mut *symmetric_key_id_ptr
+            .as_array(symmetric_key_id_max_len)
+            .as_slice_mut()?;
+        Ok(self.ctx.symmetric_key_store_managed(
+            secrets_manager_handle.into(),
+            symmetric_key_handle.into(),
+            key_id_buf,
+        )?)
+    }
+
+    fn symmetric_key_replace_managed(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+        symmetric_key_old_handle: guest_types::SymmetricKey,
+        symmetric_key_new_handle: guest_types::SymmetricKey,
+    ) -> Result<guest_types::Version, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .symmetric_key_replace_managed(
+                secrets_manager_handle.into(),
+                symmetric_key_old_handle.into(),
+                symmetric_key_new_handle.into(),
+            )?
+            .into())
+    }
+
+    fn symmetric_key_from_id(
+        &self,
+        secrets_manager_handle: guest_types::SecretsManager,
+        symmetric_key_id_ptr: &wiggle::GuestPtr<'_, u8>,
+        symmetric_key_id_len: guest_types::Size,
+        symmetric_key_version: guest_types::Version,
+    ) -> Result<guest_types::SymmetricKey, guest_types::CryptoErrno> {
+        let symmetric_key_id = &*symmetric_key_id_ptr
+            .as_array(symmetric_key_id_len)
+            .as_slice()?;
+        Ok(self
+            .ctx
+            .symmetric_key_from_id(
+                secrets_manager_handle.into(),
+                symmetric_key_id,
+                symmetric_key_version.into(),
+            )?
+            .into())
+    }
+
+    // --- key
+
+    fn symmetric_key_generate(
+        &self,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        options_handle: &guest_types::OptOptions,
+    ) -> Result<guest_types::SymmetricKey, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let options_handle = match *options_handle {
+            guest_types::OptOptions::Some(options_handle) => Some(options_handle),
+            guest_types::OptOptions::None => None,
+        };
+        Ok(self
+            .ctx
+            .symmetric_key_generate(alg_str, options_handle.map(Into::into))?
+            .into())
+    }
+
+    fn symmetric_key_import(
+        &self,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        raw_ptr: &wiggle::GuestPtr<'_, u8>,
+        raw_len: guest_types::Size,
+    ) -> Result<guest_types::SymmetricKey, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let raw = &*raw_ptr.as_array(raw_len).as_slice()?;
+        Ok(self.ctx.symmetric_key_import(alg_str, raw)?.into())
+    }
+
+    fn symmetric_key_export(
+        &self,
+        symmetric_key_handle: guest_types::SymmetricKey,
+    ) -> Result<guest_types::ArrayOutput, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .symmetric_key_export(symmetric_key_handle.into())?
+            .into())
+    }
+
+    fn symmetric_key_id(
+        &self,
+        symmetric_key_handle: guest_types::SymmetricKey,
+        symmetric_key_id_ptr: &wiggle::GuestPtr<'_, u8>,
+        symmetric_key_id_max_len: guest_types::Size,
+    ) -> Result<(guest_types::Size, guest_types::Version), guest_types::CryptoErrno> {
+        let key_id_buf = &mut *symmetric_key_id_ptr
+            .as_array(symmetric_key_id_max_len)
+            .as_slice_mut()?;
+        let (key_id, version) = self.ctx.symmetric_key_id(symmetric_key_handle.into())?;
+        ensure!(
+            key_id.len() <= key_id_buf.len(),
+            CryptoError::Overflow.into()
+        );
+        key_id_buf.copy_from_slice(&key_id);
+        Ok((key_id.len().try_into()?, version.into()))
+    }
+
+    fn symmetric_key_close(
+        &self,
+        key_handle: guest_types::SymmetricKey,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self.ctx.symmetric_key_close(key_handle.into())?)
+    }
+
+    // --- state
+
+    fn symmetric_state_open(
+        &self,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+        key_handle: &guest_types::OptSymmetricKey,
+        options_handle: &guest_types::OptOptions,
+    ) -> Result<guest_types::SymmetricState, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        let key_handle = match *key_handle {
+            guest_types::OptSymmetricKey::Some(key_handle) => Some(key_handle),
+            guest_types::OptSymmetricKey::None => None,
+        };
+        let options_handle = match *options_handle {
+            guest_types::OptOptions::Some(options_handle) => Some(options_handle),
+            guest_types::OptOptions::None => None,
+        };
+        Ok(self
+            .ctx
+            .symmetric_state_open(
+                alg_str,
+                key_handle.map(Into::into),
+                options_handle.map(Into::into),
+            )?
+            .into())
+    }
+
+    fn symmetric_state_options_get(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+        name_str: &wiggle::GuestPtr<'_, str>,
+        value_ptr: &wiggle::GuestPtr<'_, u8>,
+        value_max_len: guest_types::Size,
+    ) -> Result<guest_types::Size, guest_types::CryptoErrno> {
+        let name_str: &str = &*name_str.as_str()?;
+        let value = &mut *value_ptr.as_array(value_max_len).as_slice_mut()?;
+        Ok(self
+            .ctx
+            .options_get(symmetric_state_handle.into(), name_str, value)?
+            .try_into()?)
+    }
+
+    fn symmetric_state_options_get_u64(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+        name_str: &wiggle::GuestPtr<'_, str>,
+    ) -> Result<u64, guest_types::CryptoErrno> {
+        let name_str: &str = &*name_str.as_str()?;
+        Ok(self
+            .ctx
+            .options_get_u64(symmetric_state_handle.into(), name_str)?)
+    }
+
+    fn symmetric_state_close(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .symmetric_state_close(symmetric_state_handle.into())?)
+    }
+
+    fn symmetric_state_absorb(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+        data_ptr: &wiggle::GuestPtr<'_, u8>,
+        data_len: guest_types::Size,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let data = &*data_ptr.as_array(data_len).as_slice()?;
+        Ok(self
+            .ctx
+            .symmetric_state_absorb(symmetric_state_handle.into(), data)?)
+    }
+
+    fn symmetric_state_squeeze(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+        out_ptr: &wiggle::GuestPtr<'_, u8>,
+        out_len: guest_types::Size,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?;
+        Ok(self
+            .ctx
+            .symmetric_state_squeeze(symmetric_state_handle.into(), out)?)
+    }
+
+    fn symmetric_state_squeeze_tag(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+    ) -> Result<guest_types::SymmetricTag, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .symmetric_state_squeeze_tag(symmetric_state_handle.into())?
+            .into())
+    }
+
+    fn symmetric_state_squeeze_key(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+        alg_str: &wiggle::GuestPtr<'_, str>,
+    ) -> Result<guest_types::SymmetricKey, guest_types::CryptoErrno> {
+        let alg_str = &*alg_str.as_str()?;
+        Ok(self
+            .ctx
+            .symmetric_state_squeeze_key(symmetric_state_handle.into(), alg_str)?
+            .into())
+    }
+
+    fn symmetric_state_max_tag_len(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+    ) -> Result<guest_types::Size, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .symmetric_state_max_tag_len(symmetric_state_handle.into())?
+            .try_into()?)
+    }
+
+    fn symmetric_state_encrypt(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+        out_ptr: &wiggle::GuestPtr<'_, u8>,
+        out_len: guest_types::Size,
+        data_ptr: &wiggle::GuestPtr<'_, u8>,
+        data_len: guest_types::Size,
+    ) -> Result<guest_types::Size, guest_types::CryptoErrno> {
+        let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?;
+        let data = &*data_ptr.as_array(data_len).as_slice()?;
+        Ok(self
+            .ctx
+            .symmetric_state_encrypt(symmetric_state_handle.into(), out, data)?
+            .try_into()?)
+    }
+
+    fn symmetric_state_encrypt_detached(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+        out_ptr: &wiggle::GuestPtr<'_, u8>,
+        out_len: guest_types::Size,
+        data_ptr: &wiggle::GuestPtr<'_, u8>,
+        data_len: guest_types::Size,
+    ) -> Result<guest_types::SymmetricTag, guest_types::CryptoErrno> {
+        let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?;
+        let data = &*data_ptr.as_array(data_len).as_slice()?;
+        Ok(self
+            .ctx
+            .symmetric_state_encrypt_detached(symmetric_state_handle.into(), out, data)?
+            .into())
+    }
+
+    fn symmetric_state_decrypt(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+        out_ptr: &wiggle::GuestPtr<'_, u8>,
+        out_len: guest_types::Size,
+        data_ptr: &wiggle::GuestPtr<'_, u8>,
+        data_len: guest_types::Size,
+    ) -> Result<guest_types::Size, guest_types::CryptoErrno> {
+        let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?;
+        let data = &*data_ptr.as_array(data_len).as_slice()?;
+        Ok(self
+            .ctx
+            .symmetric_state_decrypt(symmetric_state_handle.into(), out, data)?
+            .try_into()?)
+    }
+
+    fn symmetric_state_decrypt_detached(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+        out_ptr: &wiggle::GuestPtr<'_, u8>,
+        out_len: guest_types::Size,
+        data_ptr: &wiggle::GuestPtr<'_, u8>,
+        data_len: guest_types::Size,
+        raw_tag_ptr: &wiggle::GuestPtr<'_, u8>,
+        raw_tag_len: guest_types::Size,
+    ) -> Result<guest_types::Size, guest_types::CryptoErrno> {
+        let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?;
+        let data = &*data_ptr.as_array(data_len).as_slice()?;
+        let raw_tag: &[u8] = &*raw_tag_ptr.as_array(raw_tag_len).as_slice()?;
+        Ok(self
+            .ctx
+            .symmetric_state_decrypt_detached(symmetric_state_handle.into(), out, data, raw_tag)?
+            .try_into()?)
+    }
+
+    fn symmetric_state_ratchet(
+        &self,
+        symmetric_state_handle: guest_types::SymmetricState,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .symmetric_state_ratchet(symmetric_state_handle.into())?)
+    }
+
+    // --- tag
+
+    fn symmetric_tag_len(
+        &self,
+        symmetric_tag_handle: guest_types::SymmetricTag,
+    ) -> Result<guest_types::Size, guest_types::CryptoErrno> {
+        Ok(self
+            .ctx
+            .symmetric_tag_len(symmetric_tag_handle.into())?
+            .try_into()?)
+    }
+
+    fn symmetric_tag_pull(
+        &self,
+        symmetric_tag_handle: guest_types::SymmetricTag,
+        buf_ptr: &wiggle::GuestPtr<'_, u8>,
+        buf_len: guest_types::Size,
+    ) -> Result<guest_types::Size, guest_types::CryptoErrno> {
+        let buf = &mut *buf_ptr.as_array(buf_len).as_slice_mut()?;
+        Ok(self
+            .ctx
+            .symmetric_tag_pull(symmetric_tag_handle.into(), buf)?
+            .try_into()?)
+    }
+
+    fn symmetric_tag_verify(
+        &self,
+        symmetric_tag_handle: guest_types::SymmetricTag,
+        expected_raw_ptr: &wiggle::GuestPtr<'_, u8>,
+        expected_raw_len: guest_types::Size,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        let expected_raw = &*expected_raw_ptr.as_array(expected_raw_len).as_slice()?;
+        Ok(self
+            .ctx
+            .symmetric_tag_verify(symmetric_tag_handle.into(), expected_raw)?)
+    }
+
+    fn symmetric_tag_close(
+        &self,
+        symmetric_tag_handle: guest_types::SymmetricTag,
+    ) -> Result<(), guest_types::CryptoErrno> {
+        Ok(self.ctx.symmetric_tag_close(symmetric_tag_handle.into())?)
+    }
+}
diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml
index 54da1d9873..52b36464cf 100644
--- a/crates/wasmtime/Cargo.toml
+++ b/crates/wasmtime/Cargo.toml
@@ -16,7 +16,7 @@ wasmtime-jit = { path = "../jit", version = "0.22.0" }
 wasmtime-cache = { path = "../cache", version = "0.22.0", optional = true }
 wasmtime-profiling = { path = "../profiling", version = "0.22.0" }
 target-lexicon = { version = "0.11.0", default-features = false }
-wasmparser = "0.71"
+wasmparser = "0.73"
 anyhow = "1.0.19"
 region = "2.2.0"
 libc = "0.2"
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index 5a0686de91..a662599dc8 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -1,4 +1,4 @@
-use crate::externals::MemoryCreator;
+use crate::memory::MemoryCreator;
 use crate::trampoline::MemoryCreatorProxy;
 use anyhow::{bail, Result};
 use std::cmp;
@@ -33,6 +33,9 @@ pub struct Config {
     pub(crate) max_wasm_stack: usize,
     pub(crate) features: WasmFeatures,
     pub(crate) wasm_backtrace_details_env_used: bool,
+    pub(crate) max_instances: usize,
+    pub(crate) max_tables: usize,
+    pub(crate) max_memories: usize,
 }
 
 impl Config {
@@ -79,6 +82,9 @@ impl Config {
                 multi_value: true,
                 ..WasmFeatures::default()
             },
+            max_instances: 10_000,
+            max_tables: 10_000,
+            max_memories: 10_000,
         };
         ret.wasm_backtrace_details(WasmBacktraceDetails::Environment);
         return ret;
@@ -131,6 +137,28 @@ impl Config {
         self
     }
 
+    /// Configures whether execution of WebAssembly will "consume fuel" to
+    /// either halt or yield execution as desired.
+    ///
+    /// This option is similar in purpose to [`Config::interruptable`] where
+    /// you can prevent infinitely-executing WebAssembly code. The difference
+    /// is that this option allows deterministic execution of WebAssembly code
+    /// by instrumenting generated code consume fuel as it executes. When fuel
+    /// runs out the behavior is defined by configuration within a [`Store`],
+    /// and by default a trap is raised.
+    ///
+    /// Note that a [`Store`] starts with no fuel, so if you enable this option
+    /// you'll have to be sure to pour some fuel into [`Store`] before
+    /// executing some code.
+    ///
+    /// By default this option is `false`.
+    ///
+    /// [`Store`]: crate::Store
+    pub fn consume_fuel(&mut self, enable: bool) -> &mut Self {
+        self.tunables.consume_fuel = enable;
+        self
+    }
+
     /// Configures the maximum amount of native stack space available to
     /// executing WebAssembly code.
     ///
@@ -383,6 +411,20 @@ impl Config {
         self
     }
 
+    /// Clears native CPU flags inferred from the host.
+    ///
+    /// By default Wasmtime will tune generated code for the host that Wasmtime
+    /// itself is running on. If you're compiling on one host, however, and
+    /// shipping artifacts to another host then this behavior may not be
+    /// desired. This function will clear all inferred native CPU features.
+    ///
+    /// To enable CPU features afterwards it's recommended to use the
+    /// [`Config::cranelift_other_flag`] method.
+    pub fn cranelift_clear_cpu_flags(&mut self) -> &mut Self {
+        self.isa_flags = native::builder_without_flags();
+        self
+    }
+
     /// Allows settings another Cranelift flag defined by a flag name and value. This allows
     /// fine-tuning of Cranelift settings.
     ///
@@ -635,6 +677,39 @@ impl Config {
         self
     }
 
+    /// Configures the maximum number of instances which can be created within
+    /// this `Store`.
+    ///
+    /// Instantiation will fail with an error if this limit is exceeded.
+    ///
+    /// This value defaults to 10,000.
+    pub fn max_instances(&mut self, instances: usize) -> &mut Self {
+        self.max_instances = instances;
+        self
+    }
+
+    /// Configures the maximum number of tables which can be created within
+    /// this `Store`.
+    ///
+    /// Instantiation will fail with an error if this limit is exceeded.
+    ///
+    /// This value defaults to 10,000.
+    pub fn max_tables(&mut self, tables: usize) -> &mut Self {
+        self.max_tables = tables;
+        self
+    }
+
+    /// Configures the maximum number of memories which can be created within
+    /// this `Store`.
+    ///
+    /// Instantiation will fail with an error if this limit is exceeded.
+    ///
+    /// This value defaults to 10,000.
+    pub fn max_memories(&mut self, memories: usize) -> &mut Self {
+        self.max_memories = memories;
+        self
+    }
+
     pub(crate) fn target_isa(&self) -> Box<dyn TargetIsa> {
         self.isa_flags
             .clone()
diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs
index 9a075e308e..00bcaa499b 100644
--- a/crates/wasmtime/src/externals.rs
+++ b/crates/wasmtime/src/externals.rs
@@ -1,15 +1,13 @@
-use crate::trampoline::{
-    generate_global_export, generate_memory_export, generate_table_export, StoreInstanceHandle,
-};
+use crate::memory::Memory;
+use crate::trampoline::{generate_global_export, generate_table_export, StoreInstanceHandle};
 use crate::values::{from_checked_anyfunc, into_checked_anyfunc, Val};
 use crate::{
-    ExternRef, ExternType, Func, GlobalType, Instance, MemoryType, Module, Mutability, Store,
-    TableType, Trap, ValType,
+    ExternRef, ExternType, Func, GlobalType, Instance, Module, Mutability, Store, TableType, Trap,
+    ValType,
 };
 use anyhow::{anyhow, bail, Result};
 use std::mem;
 use std::ptr;
-use std::slice;
 use wasmtime_environ::wasm;
 use wasmtime_runtime::{self as runtime, InstanceHandle};
 
@@ -112,26 +110,25 @@ impl Extern {
         }
     }
 
-    pub(crate) fn from_wasmtime_export(
-        wasmtime_export: wasmtime_runtime::Export,
-        instance: StoreInstanceHandle,
+    pub(crate) unsafe fn from_wasmtime_export(
+        wasmtime_export: &wasmtime_runtime::Export,
+        store: &Store,
     ) -> Extern {
         match wasmtime_export {
             wasmtime_runtime::Export::Function(f) => {
-                Extern::Func(Func::from_wasmtime_function(f, instance))
+                Extern::Func(Func::from_wasmtime_function(f, store))
             }
             wasmtime_runtime::Export::Memory(m) => {
-                Extern::Memory(Memory::from_wasmtime_memory(m, instance))
+                Extern::Memory(Memory::from_wasmtime_memory(m, store))
             }
             wasmtime_runtime::Export::Global(g) => {
-                Extern::Global(Global::from_wasmtime_global(g, instance))
+                Extern::Global(Global::from_wasmtime_global(g, store))
             }
             wasmtime_runtime::Export::Table(t) => {
-                Extern::Table(Table::from_wasmtime_table(t, instance))
+                Extern::Table(Table::from_wasmtime_table(t, store))
             }
             wasmtime_runtime::Export::Instance(i) => {
-                let handle = unsafe { instance.store.existing_instance_handle(i.clone()) };
-                Extern::Instance(Instance::from_wasmtime(handle))
+                Extern::Instance(Instance::from_wasmtime(i, store))
             }
             wasmtime_runtime::Export::Module(m) => {
                 Extern::Module(m.downcast_ref::<Module>().unwrap().clone())
@@ -335,13 +332,13 @@ impl Global {
         Ok(())
     }
 
-    pub(crate) fn from_wasmtime_global(
-        wasmtime_export: wasmtime_runtime::ExportGlobal,
-        instance: StoreInstanceHandle,
+    pub(crate) unsafe fn from_wasmtime_global(
+        wasmtime_export: &wasmtime_runtime::ExportGlobal,
+        store: &Store,
     ) -> Global {
         Global {
-            instance,
-            wasmtime_export,
+            instance: store.existing_vmctx(wasmtime_export.vmctx),
+            wasmtime_export: wasmtime_export.clone(),
         }
     }
 
@@ -354,6 +351,10 @@ impl Global {
             from: self.wasmtime_export.definition,
         }
     }
+
+    pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportGlobal {
+        &self.wasmtime_export
+    }
 }
 
 /// A WebAssembly `table`, or an array of values.
@@ -576,13 +577,13 @@ impl Table {
         Ok(())
     }
 
-    pub(crate) fn from_wasmtime_table(
-        wasmtime_export: wasmtime_runtime::ExportTable,
-        instance: StoreInstanceHandle,
+    pub(crate) unsafe fn from_wasmtime_table(
+        wasmtime_export: &wasmtime_runtime::ExportTable,
+        store: &Store,
     ) -> Table {
         Table {
-            instance,
-            wasmtime_export,
+            instance: store.existing_vmctx(wasmtime_export.vmctx),
+            wasmtime_export: wasmtime_export.clone(),
         }
     }
 
@@ -596,510 +597,9 @@ impl Table {
             vmctx: self.wasmtime_export.vmctx,
         }
     }
-}
 
-/// A WebAssembly linear memory.
-///
-/// WebAssembly memories represent a contiguous array of bytes that have a size
-/// that is always a multiple of the WebAssembly page size, currently 64
-/// kilobytes.
-///
-/// WebAssembly memory is used for global data, statics in C/C++/Rust, shadow
-/// stack memory, etc. Accessing wasm memory is generally quite fast!
-///
-/// # `Memory` and `Clone`
-///
-/// Memories are internally reference counted so you can `clone` a `Memory`. The
-/// cloning process only performs a shallow clone, so two cloned `Memory`
-/// instances are equivalent in their functionality.
-///
-/// # `Memory` and threads
-///
-/// It is intended that `Memory` is safe to share between threads. At this time
-/// this is not implemented in `wasmtime`, however. This is planned to be
-/// implemented though!
-///
-/// # `Memory` and Safety
-///
-/// Linear memory is a lynchpin of safety for WebAssembly, but it turns out
-/// there are very few ways to safely inspect the contents of a memory from the
-/// host (Rust). This is because memory safety is quite tricky when working with
-/// a `Memory` and we're still working out the best idioms to encapsulate
-/// everything safely where it's efficient and ergonomic. This section of
-/// documentation, however, is intended to help educate a bit what is and isn't
-/// safe when working with `Memory`.
-///
-/// For safety purposes you can think of a `Memory` as a glorified
-/// `Rc<UnsafeCell<Vec<u8>>>`. There are a few consequences of this
-/// interpretation:
-///
-/// * At any time someone else may have access to the memory (hence the `Rc`).
-///   This could be a wasm instance, other host code, or a set of wasm instances
-///   which all reference a `Memory`. When in doubt assume someone else has a
-///   handle to your `Memory`.
-///
-/// * At any time, memory can be read from or written to (hence the
-///   `UnsafeCell`). Anyone with a handle to a wasm memory can read/write to it.
-///   Primarily other instances can execute the `load` and `store` family of
-///   instructions, as well as any other which modifies or reads memory.
-///
-/// * At any time memory may grow (hence the `Vec<..>`). Growth may relocate the
-///   base memory pointer (similar to how `vec.push(...)` can change the result
-///   of `.as_ptr()`)
-///
-/// So given that we're working roughly with `Rc<UnsafeCell<Vec<u8>>>` that's a
-/// lot to keep in mind! It's hopefully though sort of setting the stage as to
-/// what you can safely do with memories.
-///
-/// Let's run through a few safe examples first of how you can use a `Memory`.
-///
-/// ```rust
-/// use wasmtime::Memory;
-///
-/// fn safe_examples(mem: &Memory) {
-///     // Just like wasm, it's safe to read memory almost at any time. The
-///     // gotcha here is that we need to be sure to load from the correct base
-///     // pointer and perform the bounds check correctly. So long as this is
-///     // all self contained here (e.g. not arbitrary code in the middle) we're
-///     // good to go.
-///     let byte = unsafe { mem.data_unchecked()[0x123] };
-///
-///     // Short-lived borrows of memory are safe, but they must be scoped and
-///     // not have code which modifies/etc `Memory` while the borrow is active.
-///     // For example if you want to read a string from memory it is safe to do
-///     // so:
-///     let string_base = 0xdead;
-///     let string_len = 0xbeef;
-///     let string = unsafe {
-///         let bytes = &mem.data_unchecked()[string_base..][..string_len];
-///         match std::str::from_utf8(bytes) {
-///             Ok(s) => s.to_string(), // copy out of wasm memory
-///             Err(_) => panic!("not valid utf-8"),
-///         }
-///     };
-///
-///     // Additionally like wasm you can write to memory at any point in time,
-///     // again making sure that after you get the unchecked slice you don't
-///     // execute code which could read/write/modify `Memory`:
-///     unsafe {
-///         mem.data_unchecked_mut()[0x123] = 3;
-///     }
-///
-///     // When working with *borrows* that point directly into wasm memory you
-///     // need to be extremely careful. Any functionality that operates on a
-///     // borrow into wasm memory needs to be thoroughly audited to effectively
-///     // not touch the `Memory` at all
-///     let data_base = 0xfeed;
-///     let data_len = 0xface;
-///     unsafe {
-///         let data = &mem.data_unchecked()[data_base..][..data_len];
-///         host_function_that_doesnt_touch_memory(data);
-///
-///         // effectively the same rules apply to mutable borrows
-///         let data_mut = &mut mem.data_unchecked_mut()[data_base..][..data_len];
-///         host_function_that_doesnt_touch_memory(data);
-///     }
-/// }
-/// # fn host_function_that_doesnt_touch_memory(_: &[u8]){}
-/// ```
-///
-/// It's worth also, however, covering some examples of **incorrect**,
-/// **unsafe** usages of `Memory`. Do not do these things!
-///
-/// ```rust
-/// # use anyhow::Result;
-/// use wasmtime::Memory;
-///
-/// // NOTE: All code in this function is not safe to execute and may cause
-/// // segfaults/undefined behavior at runtime. Do not copy/paste these examples
-/// // into production code!
-/// unsafe fn unsafe_examples(mem: &Memory) -> Result<()> {
-///     // First and foremost, any borrow can be invalidated at any time via the
-///     // `Memory::grow` function. This can relocate memory which causes any
-///     // previous pointer to be possibly invalid now.
-///     let pointer: &u8 = &mem.data_unchecked()[0x100];
-///     mem.grow(1)?; // invalidates `pointer`!
-///     // println!("{}", *pointer); // FATAL: use-after-free
-///
-///     // Note that the use-after-free also applies to slices, whether they're
-///     // slices of bytes or strings.
-///     let slice: &[u8] = &mem.data_unchecked()[0x100..0x102];
-///     mem.grow(1)?; // invalidates `slice`!
-///     // println!("{:?}", slice); // FATAL: use-after-free
-///
-///     // Due to the reference-counted nature of `Memory` note that literal
-///     // calls to `Memory::grow` are not sufficient to audit for. You'll need
-///     // to be careful that any mutation of `Memory` doesn't happen while
-///     // you're holding an active borrow.
-///     let slice: &[u8] = &mem.data_unchecked()[0x100..0x102];
-///     some_other_function(); // may invalidate `slice` through another `mem` reference
-///     // println!("{:?}", slice); // FATAL: maybe a use-after-free
-///
-///     // An especially subtle aspect of accessing a wasm instance's memory is
-///     // that you need to be extremely careful about aliasing. Anyone at any
-///     // time can call `data_unchecked()` or `data_unchecked_mut()`, which
-///     // means you can easily have aliasing mutable references:
-///     let ref1: &u8 = &mem.data_unchecked()[0x100];
-///     let ref2: &mut u8 = &mut mem.data_unchecked_mut()[0x100];
-///     // *ref2 = *ref1; // FATAL: violates Rust's aliasing rules
-///
-///     // Note that aliasing applies to strings as well, for example this is
-///     // not valid because the slices overlap.
-///     let slice1: &mut [u8] = &mut mem.data_unchecked_mut()[0x100..][..3];
-///     let slice2: &mut [u8] = &mut mem.data_unchecked_mut()[0x102..][..4];
-///     // println!("{:?} {:?}", slice1, slice2); // FATAL: aliasing mutable pointers
-///
-///     Ok(())
-/// }
-/// # fn some_other_function() {}
-/// ```
-///
-/// Overall there's some general rules of thumb when working with `Memory` and
-/// getting raw pointers inside of it:
-///
-/// * If you never have a "long lived" pointer into memory, you're likely in the
-///   clear. Care still needs to be taken in threaded scenarios or when/where
-///   data is read, but you'll be shielded from many classes of issues.
-/// * Long-lived pointers must always respect Rust'a aliasing rules. It's ok for
-///   shared borrows to overlap with each other, but mutable borrows must
-///   overlap with nothing.
-/// * Long-lived pointers are only valid if `Memory` isn't used in an unsafe way
-///   while the pointer is valid. This includes both aliasing and growth.
-///
-/// At this point it's worth reiterating again that working with `Memory` is
-/// pretty tricky and that's not great! Proposals such as [interface types] are
-/// intended to prevent wasm modules from even needing to import/export memory
-/// in the first place, which obviates the need for all of these safety caveats!
-/// Additionally over time we're still working out the best idioms to expose in
-/// `wasmtime`, so if you've got ideas or questions please feel free to [open an
-/// issue]!
-///
-/// ## `Memory` Safety and Threads
-///
-/// Currently the `wasmtime` crate does not implement the wasm threads proposal,
-/// but it is planned to do so. It's additionally worthwhile discussing how this
-/// affects memory safety and what was previously just discussed as well.
-///
-/// Once threads are added into the mix, all of the above rules still apply.
-/// There's an additional, rule, however, that all reads and writes can
-/// happen *concurrently*. This effectively means that long-lived borrows into
-/// wasm memory are virtually never safe to have.
-///
-/// Mutable pointers are fundamentally unsafe to have in a concurrent scenario
-/// in the face of arbitrary wasm code. Only if you dynamically know for sure
-/// that wasm won't access a region would it be safe to construct a mutable
-/// pointer. Additionally even shared pointers are largely unsafe because their
-/// underlying contents may change, so unless `UnsafeCell` in one form or
-/// another is used everywhere there's no safety.
-///
-/// One important point about concurrency is that `Memory::grow` can indeed
-/// happen concurrently. This, however, will never relocate the base pointer.
-/// Shared memories must always have a maximum size and they will be
-/// preallocated such that growth will never relocate the base pointer. The
-/// maximum length of the memory, however, will change over time.
-///
-/// Overall the general rule of thumb for shared memories is that you must
-/// atomically read and write everything. Nothing can be borrowed and everything
-/// must be eagerly copied out.
-///
-/// [interface types]: https://github.com/webassembly/interface-types
-/// [open an issue]: https://github.com/bytecodealliance/wasmtime/issues/new
-#[derive(Clone)]
-pub struct Memory {
-    instance: StoreInstanceHandle,
-    wasmtime_export: wasmtime_runtime::ExportMemory,
-}
-
-impl Memory {
-    /// Creates a new WebAssembly memory given the configuration of `ty`.
-    ///
-    /// The `store` argument is a general location for cache information, and
-    /// otherwise the memory will immediately be allocated according to the
-    /// type's configuration. All WebAssembly memory is initialized to zero.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # use wasmtime::*;
-    /// # fn main() -> anyhow::Result<()> {
-    /// let engine = Engine::default();
-    /// let store = Store::new(&engine);
-    ///
-    /// let memory_ty = MemoryType::new(Limits::new(1, None));
-    /// let memory = Memory::new(&store, memory_ty);
-    ///
-    /// let module = Module::new(&engine, "(module (memory (import \"\" \"\") 1))")?;
-    /// let instance = Instance::new(&store, &module, &[memory.into()])?;
-    /// // ...
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub fn new(store: &Store, ty: MemoryType) -> Memory {
-        let (instance, wasmtime_export) =
-            generate_memory_export(store, &ty).expect("generated memory");
-        Memory {
-            instance,
-            wasmtime_export,
-        }
-    }
-
-    /// Returns the underlying type of this memory.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # use wasmtime::*;
-    /// # fn main() -> anyhow::Result<()> {
-    /// let engine = Engine::default();
-    /// let store = Store::new(&engine);
-    /// let module = Module::new(&engine, "(module (memory (export \"mem\") 1))")?;
-    /// let instance = Instance::new(&store, &module, &[])?;
-    /// let memory = instance.get_memory("mem").unwrap();
-    /// let ty = memory.ty();
-    /// assert_eq!(ty.limits().min(), 1);
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub fn ty(&self) -> MemoryType {
-        MemoryType::from_wasmtime_memory(&self.wasmtime_export.memory.memory)
-    }
-
-    /// Returns this memory as a slice view that can be read natively in Rust.
-    ///
-    /// # Safety
-    ///
-    /// This is an unsafe operation because there is no guarantee that the
-    /// following operations do not happen concurrently while the slice is in
-    /// use:
-    ///
-    /// * Data could be modified by calling into a wasm module.
-    /// * Memory could be relocated through growth by calling into a wasm
-    ///   module.
-    /// * When threads are supported, non-atomic reads will race with other
-    ///   writes.
-    ///
-    /// Extreme care need be taken when the data of a `Memory` is read. The
-    /// above invariants all need to be upheld at a bare minimum, and in
-    /// general you'll need to ensure that while you're looking at slice you're
-    /// the only one who can possibly look at the slice and read/write it.
-    ///
-    /// Be sure to keep in mind that `Memory` is reference counted, meaning
-    /// that there may be other users of this `Memory` instance elsewhere in
-    /// your program. Additionally `Memory` can be shared and used in any number
-    /// of wasm instances, so calling any wasm code should be considered
-    /// dangerous while you're holding a slice of memory.
-    ///
-    /// For more information and examples see the documentation on the
-    /// [`Memory`] type.
-    pub unsafe fn data_unchecked(&self) -> &[u8] {
-        self.data_unchecked_mut()
-    }
-
-    /// Returns this memory as a slice view that can be read and written
-    /// natively in Rust.
-    ///
-    /// # Safety
-    ///
-    /// All of the same safety caveats of [`Memory::data_unchecked`] apply
-    /// here, doubly so because this is returning a mutable slice! As a
-    /// double-extra reminder, remember that `Memory` is reference counted, so
-    /// you can very easily acquire two mutable slices by simply calling this
-    /// function twice. Extreme caution should be used when using this method,
-    /// and in general you probably want to result to unsafe accessors and the
-    /// `data` methods below.
-    ///
-    /// For more information and examples see the documentation on the
-    /// [`Memory`] type.
-    pub unsafe fn data_unchecked_mut(&self) -> &mut [u8] {
-        let definition = &*self.wasmtime_export.definition;
-        slice::from_raw_parts_mut(definition.base, definition.current_length)
-    }
-
-    /// Returns the base pointer, in the host's address space, that the memory
-    /// is located at.
-    ///
-    /// When reading and manipulating memory be sure to read up on the caveats
-    /// of [`Memory::data_unchecked`] to make sure that you can safely
-    /// read/write the memory.
-    ///
-    /// For more information and examples see the documentation on the
-    /// [`Memory`] type.
-    pub fn data_ptr(&self) -> *mut u8 {
-        unsafe { (*self.wasmtime_export.definition).base }
-    }
-
-    /// Returns the byte length of this memory.
-    ///
-    /// The returned value will be a multiple of the wasm page size, 64k.
-    ///
-    /// For more information and examples see the documentation on the
-    /// [`Memory`] type.
-    pub fn data_size(&self) -> usize {
-        unsafe { (*self.wasmtime_export.definition).current_length }
-    }
-
-    /// Returns the size, in pages, of this wasm memory.
-    pub fn size(&self) -> u32 {
-        (self.data_size() / wasmtime_environ::WASM_PAGE_SIZE as usize) as u32
-    }
-
-    /// Grows this WebAssembly memory by `delta` pages.
-    ///
-    /// This will attempt to add `delta` more pages of memory on to the end of
-    /// this `Memory` instance. If successful this may relocate the memory and
-    /// cause [`Memory::data_ptr`] to return a new value. Additionally previous
-    /// slices into this memory may no longer be valid.
-    ///
-    /// On success returns the number of pages this memory previously had
-    /// before the growth succeeded.
-    ///
-    /// # Errors
-    ///
-    /// Returns an error if memory could not be grown, for example if it exceeds
-    /// the maximum limits of this memory.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # use wasmtime::*;
-    /// # fn main() -> anyhow::Result<()> {
-    /// let engine = Engine::default();
-    /// let store = Store::new(&engine);
-    /// let module = Module::new(&engine, "(module (memory (export \"mem\") 1 2))")?;
-    /// let instance = Instance::new(&store, &module, &[])?;
-    /// let memory = instance.get_memory("mem").unwrap();
-    ///
-    /// assert_eq!(memory.size(), 1);
-    /// assert_eq!(memory.grow(1)?, 1);
-    /// assert_eq!(memory.size(), 2);
-    /// assert!(memory.grow(1).is_err());
-    /// assert_eq!(memory.size(), 2);
-    /// assert_eq!(memory.grow(0)?, 2);
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub fn grow(&self, delta: u32) -> Result<u32> {
-        let index = self
-            .instance
-            .memory_index(unsafe { &*self.wasmtime_export.definition });
-        self.instance
-            .memory_grow(index, delta)
-            .ok_or_else(|| anyhow!("failed to grow memory"))
-    }
-
-    pub(crate) fn from_wasmtime_memory(
-        wasmtime_export: wasmtime_runtime::ExportMemory,
-        instance: StoreInstanceHandle,
-    ) -> Memory {
-        Memory {
-            instance,
-            wasmtime_export,
-        }
-    }
-
-    pub(crate) fn wasmtime_ty(&self) -> &wasmtime_environ::wasm::Memory {
-        &self.wasmtime_export.memory.memory
-    }
-
-    pub(crate) fn vmimport(&self) -> wasmtime_runtime::VMMemoryImport {
-        wasmtime_runtime::VMMemoryImport {
-            from: self.wasmtime_export.definition,
-            vmctx: self.wasmtime_export.vmctx,
-        }
-    }
-}
-
-/// A linear memory. This trait provides an interface for raw memory buffers which are used
-/// by wasmtime, e.g. inside ['Memory']. Such buffers are in principle not thread safe.
-/// By implementing this trait together with MemoryCreator,
-/// one can supply wasmtime with custom allocated host managed memory.
-///
-/// # Safety
-/// The memory should be page aligned and a multiple of page size.
-/// To prevent possible silent overflows, the memory should be protected by a guard page.
-/// Additionally the safety concerns explained in ['Memory'], for accessing the memory
-/// apply here as well.
-///
-/// Note that this is a relatively new and experimental feature and it is recommended
-/// to be familiar with wasmtime runtime code to use it.
-pub unsafe trait LinearMemory {
-    /// Returns the number of allocated wasm pages.
-    fn size(&self) -> u32;
-
-    /// Grow memory by the specified amount of wasm pages.
-    ///
-    /// Returns `None` if memory can't be grown by the specified amount
-    /// of wasm pages.
-    fn grow(&self, delta: u32) -> Option<u32>;
-
-    /// Return the allocated memory as a mutable pointer to u8.
-    fn as_ptr(&self) -> *mut u8;
-}
-
-/// A memory creator. Can be used to provide a memory creator
-/// to wasmtime which supplies host managed memory.
-///
-/// # Safety
-/// This trait is unsafe, as the memory safety depends on proper implementation of
-/// memory management. Memories created by the MemoryCreator should always be treated
-/// as owned by wasmtime instance, and any modification of them outside of wasmtime
-/// invoked routines is unsafe and may lead to corruption.
-///
-/// Note that this is a relatively new and experimental feature and it is recommended
-/// to be familiar with wasmtime runtime code to use it.
-pub unsafe trait MemoryCreator: Send + Sync {
-    /// Create a new `LinearMemory` object from the specified parameters.
-    ///
-    /// The type of memory being created is specified by `ty` which indicates
-    /// both the minimum and maximum size, in wasm pages.
-    ///
-    /// The `reserved_size_in_bytes` value indicates the expected size of the
-    /// reservation that is to be made for this memory. If this value is `None`
-    /// than the implementation is free to allocate memory as it sees fit. If
-    /// the value is `Some`, however, then the implementation is expected to
-    /// reserve that many bytes for the memory's allocation, plus the guard
-    /// size at the end. Note that this reservation need only be a virtual
-    /// memory reservation, physical memory does not need to be allocated
-    /// immediately. In this case `grow` should never move the base pointer and
-    /// the maximum size of `ty` is guaranteed to fit within `reserved_size_in_bytes`.
-    ///
-    /// The `guard_size_in_bytes` parameter indicates how many bytes of space, after the
-    /// memory allocation, is expected to be unmapped. JIT code will elide
-    /// bounds checks based on the `guard_size_in_bytes` provided, so for JIT code to
-    /// work correctly the memory returned will need to be properly guarded with
-    /// `guard_size_in_bytes` bytes left unmapped after the base allocation.
-    ///
-    /// Note that the `reserved_size_in_bytes` and `guard_size_in_bytes` options are tuned from
-    /// the various [`Config`](crate::Config) methods about memory
-    /// sizes/guards. Additionally these two values are guaranteed to be
-    /// multiples of the system page size.
-    fn new_memory(
-        &self,
-        ty: MemoryType,
-        reserved_size_in_bytes: Option<u64>,
-        guard_size_in_bytes: u64,
-    ) -> Result<Box<dyn LinearMemory>, String>;
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::*;
-
-    // Assert that creating a memory via `Memory::new` respects the limits/tunables
-    // in `Config`.
-    #[test]
-    fn respect_tunables() {
-        let mut cfg = Config::new();
-        cfg.static_memory_maximum_size(0)
-            .dynamic_memory_guard_size(0);
-        let store = Store::new(&Engine::new(&cfg));
-        let ty = MemoryType::new(Limits::new(1, None));
-        let mem = Memory::new(&store, ty);
-        assert_eq!(mem.wasmtime_export.memory.offset_guard_size, 0);
-        match mem.wasmtime_export.memory.style {
-            wasmtime_environ::MemoryStyle::Dynamic => {}
-            other => panic!("unexpected style {:?}", other),
-        }
+    pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportTable {
+        &self.wasmtime_export
     }
 }
 
diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs
index 630c5f736d..7f68b97964 100644
--- a/crates/wasmtime/src/func.rs
+++ b/crates/wasmtime/src/func.rs
@@ -1,6 +1,6 @@
 use crate::store::StoreInner;
 use crate::trampoline::StoreInstanceHandle;
-use crate::{Extern, ExternRef, FuncType, Memory, Store, Trap, Val, ValType};
+use crate::{Extern, ExternRef, FuncType, Store, Trap, Val, ValType};
 use anyhow::{bail, ensure, Context as _, Result};
 use smallvec::{smallvec, SmallVec};
 use std::cmp::max;
@@ -9,8 +9,9 @@ use std::mem;
 use std::panic::{self, AssertUnwindSafe};
 use std::ptr::{self, NonNull};
 use std::rc::Weak;
+use wasmtime_environ::wasm::EntityIndex;
 use wasmtime_runtime::{
-    raise_user_trap, Export, InstanceHandle, VMContext, VMFunctionBody, VMSharedSignatureIndex,
+    raise_user_trap, InstanceHandle, VMContext, VMFunctionBody, VMSharedSignatureIndex,
     VMTrampoline,
 };
 
@@ -331,11 +332,8 @@ impl Func {
         debug_assert!(
             anyfunc.as_ref().type_index != wasmtime_runtime::VMSharedSignatureIndex::default()
         );
-
-        let instance_handle = wasmtime_runtime::InstanceHandle::from_vmctx(anyfunc.as_ref().vmctx);
         let export = wasmtime_runtime::ExportFunction { anyfunc };
-        let instance = store.existing_instance_handle(instance_handle);
-        let f = Func::from_wasmtime_function(export, instance);
+        let f = Func::from_wasmtime_function(&export, store);
         Some(f)
     }
 
@@ -373,8 +371,8 @@ impl Func {
     ///
     /// Finally you can also optionally take [`Caller`] as the first argument of
     /// your closure. If inserted then you're able to inspect the caller's
-    /// state, for example the [`Memory`] it has exported so you can read what
-    /// pointers point to.
+    /// state, for example the [`Memory`](crate::Memory) it has exported so you
+    /// can read what pointers point to.
     ///
     /// Note that when using this API, the intention is to create as thin of a
     /// layer as possible for when WebAssembly calls the function provided. With
@@ -649,24 +647,24 @@ impl Func {
         self.export.anyfunc
     }
 
-    pub(crate) fn from_wasmtime_function(
-        export: wasmtime_runtime::ExportFunction,
-        instance: StoreInstanceHandle,
+    pub(crate) unsafe fn from_wasmtime_function(
+        export: &wasmtime_runtime::ExportFunction,
+        store: &Store,
     ) -> Self {
         // Each function signature in a module should have a trampoline stored
         // on that module as well, so unwrap the result here since otherwise
         // it's a bug in wasmtime.
-        let trampoline = instance
-            .store
+        let anyfunc = export.anyfunc.as_ref();
+        let trampoline = store
             .signatures()
             .borrow()
-            .lookup_shared(unsafe { export.anyfunc.as_ref().type_index })
+            .lookup_shared(anyfunc.type_index)
             .expect("failed to retrieve trampoline from module")
             .1;
 
         Func {
-            instance,
-            export,
+            instance: store.existing_vmctx(anyfunc.vmctx),
+            export: export.clone(),
             trampoline,
         }
     }
@@ -807,6 +805,10 @@ impl Func {
             }
         }
     }
+
+    pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportFunction {
+        &self.export
+    }
 }
 
 impl fmt::Debug for Func {
@@ -1506,10 +1508,16 @@ impl Caller<'_> {
             debug_assert!(self.store.upgrade().is_some());
             let handle =
                 Store::from_inner(self.store.upgrade()?).existing_instance_handle(instance);
-            let export = handle.lookup(name)?;
-            match export {
-                Export::Memory(m) => Some(Extern::Memory(Memory::from_wasmtime_memory(m, handle))),
-                Export::Function(f) => Some(Extern::Func(Func::from_wasmtime_function(f, handle))),
+            let index = handle.module().exports.get(name)?;
+            match index {
+                // Only allow memory/functions for now to emulate what interface
+                // types will once provide
+                EntityIndex::Memory(_) | EntityIndex::Function(_) => {
+                    Some(Extern::from_wasmtime_export(
+                        &handle.lookup_by_declaration(&index),
+                        &handle.store,
+                    ))
+                }
                 _ => None,
             }
         }
diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs
index 0dff161052..47115d90ae 100644
--- a/crates/wasmtime/src/instance.rs
+++ b/crates/wasmtime/src/instance.rs
@@ -1,241 +1,22 @@
 use crate::trampoline::StoreInstanceHandle;
 use crate::types::matching;
 use crate::{
-    Engine, Export, Extern, ExternType, Func, Global, InstanceType, Memory, Module, Store, Table,
-    Trap,
+    Engine, Export, Extern, Func, Global, InstanceType, Memory, Module, Store, Table, Trap,
 };
 use anyhow::{bail, Context, Error, Result};
 use std::mem;
-use std::sync::Arc;
+use std::rc::Rc;
 use wasmtime_environ::entity::PrimaryMap;
 use wasmtime_environ::wasm::{
-    EntityIndex, EntityType, FuncIndex, GlobalIndex, InstanceIndex, MemoryIndex, ModuleIndex,
-    TableIndex,
+    EntityIndex, FuncIndex, GlobalIndex, InstanceIndex, MemoryIndex, ModuleIndex, TableIndex,
 };
 use wasmtime_environ::Initializer;
-use wasmtime_jit::TypeTables;
 use wasmtime_runtime::{
-    Imports, InstanceHandle, InstantiationError, StackMapRegistry, VMContext,
+    Imports, InstantiationError, RuntimeInstance, StackMapRegistry, VMContext,
     VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport,
     VMTableImport,
 };
 
-/// Performs all low-level steps necessary for instantiation.
-///
-/// This function will take all the arguments and attempt to do everything
-/// necessary to instantiate the referenced instance. The trickiness of this
-/// function stems from the implementation of the module-linking proposal where
-/// we're handling nested instances, interleaved imports/aliases, etc. That's
-/// all an internal implementation here ideally though!
-///
-/// * `store` - the store we're instantiating into
-/// * `compiled_module` - the module that we're instantiating
-/// * `all_modules` - the list of all modules that were part of the compilation
-///   of `compiled_module`. This is only applicable in the module linking
-///   proposal, otherwise this will just be a list containing `compiled_module`
-///   itself.
-/// * `type` - the type tables produced during compilation which
-///   `compiled_module`'s metadata references.
-/// * `parent_modules` - this is the list of compiled modules the parent has.
-///   This is only applicable on recursive instantiations.
-/// * `define_import` - this function, like the name implies, defines an import
-///   into the provided builder. The expected entity that it's defining is also
-///   passed in for the top-level case where type-checking is performed. This is
-///   fallible because type checks may fail.
-fn instantiate(
-    store: &Store,
-    module: &Module,
-    parent_modules: &PrimaryMap<ModuleIndex, Module>,
-    define_import: &mut dyn FnMut(&EntityIndex, &mut ImportsBuilder<'_>) -> Result<()>,
-) -> Result<StoreInstanceHandle, Error> {
-    let compiled_module = module.compiled_module();
-    let env_module = compiled_module.module();
-
-    let mut imports = ImportsBuilder::new(store, module);
-    for initializer in env_module.initializers.iter() {
-        match initializer {
-            // Definition of an import depends on how our parent is providing
-            // imports, so we delegate to our custom closure. This will resolve
-            // to fetching from the import list for the top-level module and
-            // otherwise fetching from each nested instance's argument list for
-            // submodules.
-            Initializer::Import {
-                index,
-                module,
-                field,
-            } => {
-                define_import(index, &mut imports).with_context(|| match field {
-                    Some(name) => format!("incompatible import type for `{}::{}`", module, name),
-                    None => format!("incompatible import type for `{}`", module),
-                })?;
-            }
-
-            // This one's pretty easy, we're just picking up our parent's module
-            // and putting it into our own index space.
-            Initializer::AliasParentModule(idx) => {
-                imports.modules.push(parent_modules[*idx].clone());
-            }
-
-            // Turns out defining any kind of module is pretty easy, we're just
-            // slinging around pointers.
-            Initializer::DefineModule(idx) => {
-                imports.modules.push(module.submodule(*idx));
-            }
-
-            // Here we lookup our instance handle, find the right export,
-            // and then push that item into our own index space. We eschew
-            // type-checking since only valid modules reach this point.
-            //
-            // Note that export lookup here needs to happen by name. The
-            // `export` index is an index into our local type definition of the
-            // type of the instance to figure out what name it was assigned.
-            // This is where the subtyping happens!
-            //
-            // Note that the unsafety here is because we're asserting that the
-            // handle comes from our same store, but this should be true because
-            // we acquired the handle from an instance in the store.
-            Initializer::AliasInstanceExport { instance, export } => {
-                let instance_ty = env_module.instances[*instance];
-                let export_name = module.types().instance_signatures[instance_ty]
-                    .exports
-                    .get_index(*export)
-                    .expect("validation bug - should be valid")
-                    .0;
-                let handle = &imports.instances[*instance];
-                let entity_index = &handle.module().exports[export_name];
-                let item = Extern::from_wasmtime_export(
-                    handle.lookup_by_declaration(entity_index),
-                    unsafe { store.existing_instance_handle(handle.clone()) },
-                );
-                imports.push_extern(&item);
-            }
-
-            // Oh boy a recursive instantiation! The recursive arguments here
-            // are pretty simple, and the only slightly-meaty one is how
-            // arguments are pulled from `args` and pushed directly into the
-            // builder specified, which should be an easy enough
-            // copy-the-pointer operation in all cases.
-            //
-            // Note that this recursive call shouldn't result in an infinite
-            // loop because of wasm module validation which requires everything
-            // to be a DAG. Additionally the recursion should also be bounded
-            // due to validation. We may one day need to make this an iterative
-            // loop, however.
-            //
-            // Also note that there's some unsafety here around cloning
-            // `InstanceHandle` because the handle may not live long enough, but
-            // we're doing all of this in the context of our `Store` argument
-            // above so we should be safe here.
-            Initializer::Instantiate { module, args } => {
-                let mut args = args.iter();
-                let handle = instantiate(
-                    store,
-                    &imports.modules[*module],
-                    &imports.modules,
-                    &mut |_, builder| {
-                        match *args.next().unwrap() {
-                            EntityIndex::Global(i) => {
-                                builder.globals.push(imports.globals[i]);
-                            }
-                            EntityIndex::Function(i) => {
-                                builder.functions.push(imports.functions[i]);
-                            }
-                            EntityIndex::Table(i) => {
-                                builder.tables.push(imports.tables[i]);
-                            }
-                            EntityIndex::Memory(i) => {
-                                builder.memories.push(imports.memories[i]);
-                            }
-                            EntityIndex::Module(i) => {
-                                builder.modules.push(imports.modules[i].clone());
-                            }
-                            EntityIndex::Instance(i) => {
-                                builder
-                                    .instances
-                                    .push(unsafe { imports.instances[i].clone() });
-                            }
-                        }
-                        Ok(())
-                    },
-                )?;
-                imports.instances.push(unsafe { (*handle).clone() });
-            }
-        }
-    }
-
-    // With the above initialization done we've now acquired the final set of
-    // imports in all the right index spaces and everything. Time to carry on
-    // with the creation of our own instance.
-    let imports = imports.build();
-
-    // Register the module just before instantiation to ensure we have a
-    // trampoline registered for every signature and to preserve the module's
-    // compiled JIT code within the `Store`.
-    store.register_module(module);
-
-    let config = store.engine().config();
-    let instance = unsafe {
-        let instance = compiled_module.instantiate(
-            imports,
-            &store.lookup_shared_signature(module.types()),
-            config.memory_creator.as_ref().map(|a| a as _),
-            store.interrupts(),
-            Box::new(module.types().clone()),
-            store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _,
-            store.stack_map_registry() as *const StackMapRegistry as *mut _,
-        )?;
-
-        // After we've created the `InstanceHandle` we still need to run
-        // initialization to set up data/elements/etc. We do this after adding
-        // the `InstanceHandle` to the store though. This is required for safety
-        // because the start function (for example) may trap, but element
-        // initializers may have run which placed elements into other instance's
-        // tables. This means that from this point on, regardless of whether
-        // initialization is successful, we need to keep the instance alive.
-        let instance = store.add_instance(instance);
-        instance
-            .initialize(
-                config.features.bulk_memory,
-                &compiled_module.data_initializers(),
-            )
-            .map_err(|e| -> Error {
-                match e {
-                    InstantiationError::Trap(trap) => Trap::from_runtime(store, trap).into(),
-                    other => other.into(),
-                }
-            })?;
-
-        instance
-    };
-
-    let start_func = instance.handle.module().start_func;
-
-    // If a start function is present, invoke it. Make sure we use all the
-    // trap-handling configuration in `store` as well.
-    if let Some(start) = start_func {
-        let f = match instance
-            .handle
-            .lookup_by_declaration(&EntityIndex::Function(start))
-        {
-            wasmtime_runtime::Export::Function(f) => f,
-            _ => unreachable!(), // valid modules shouldn't hit this
-        };
-        let vmctx_ptr = instance.handle.vmctx_ptr();
-        unsafe {
-            super::func::invoke_wasm_and_catch_traps(vmctx_ptr, store, || {
-                mem::transmute::<
-                    *const VMFunctionBody,
-                    unsafe extern "C" fn(*mut VMContext, *mut VMContext),
-                >(f.anyfunc.as_ref().func_ptr.as_ptr())(
-                    f.anyfunc.as_ref().vmctx, vmctx_ptr
-                )
-            })?;
-        }
-    }
-
-    Ok(instance)
-}
-
 /// An instantiated WebAssembly module.
 ///
 /// This type represents the instantiation of a [`Module`]. Once instantiated
@@ -254,7 +35,8 @@ fn instantiate(
 /// call any code or execute anything!
 #[derive(Clone)]
 pub struct Instance {
-    pub(crate) handle: StoreInstanceHandle,
+    pub(crate) store: Store,
+    pub(crate) items: RuntimeInstance,
 }
 
 impl Instance {
@@ -312,49 +94,29 @@ impl Instance {
     /// [issue]: https://github.com/bytecodealliance/wasmtime/issues/727
     /// [`ExternType`]: crate::ExternType
     pub fn new(store: &Store, module: &Module, imports: &[Extern]) -> Result<Instance, Error> {
-        if !Engine::same(store.engine(), module.engine()) {
-            bail!("cross-`Engine` instantiation is not currently supported");
-        }
-
-        // Perform some pre-flight checks before we get into the meat of
-        // instantiation.
-        let expected = module.compiled_module().module().imports().count();
-        if expected != imports.len() {
-            bail!("expected {} imports, found {}", expected, imports.len());
-        }
-        for import in imports {
-            if !import.comes_from_same_store(store) {
-                bail!("cross-`Store` instantiation is not currently supported");
+        let mut i = Instantiator::new(store, module, imports)?;
+        loop {
+            if let Some((instance, items)) = i.step()? {
+                Instantiator::start_raw(&instance)?;
+                if let Some(items) = items {
+                    break Ok(Instance::from_wasmtime(&items, store));
+                }
             }
         }
-
-        let mut imports = imports.iter();
-        let handle = instantiate(store, module, &PrimaryMap::new(), &mut |idx, builder| {
-            let import = imports.next().expect("already checked the length");
-            builder.define_extern(idx, import)
-        })?;
-
-        Ok(Instance { handle })
     }
 
-    pub(crate) fn from_wasmtime(handle: StoreInstanceHandle) -> Instance {
-        Instance { handle }
+    pub(crate) fn from_wasmtime(handle: &RuntimeInstance, store: &Store) -> Instance {
+        Instance {
+            items: handle.clone(),
+            store: store.clone(),
+        }
     }
 
     /// Returns the type signature of this instance.
     pub fn ty(&self) -> InstanceType {
         let mut ty = InstanceType::new();
-        let module = self.handle.module();
-        let types = self
-            .handle
-            .host_state()
-            .downcast_ref::<Arc<TypeTables>>()
-            .unwrap();
-        for (name, index) in module.exports.iter() {
-            ty.add_named_export(
-                name,
-                ExternType::from_wasmtime(types, &module.type_of(*index)),
-            );
+        for export in self.exports() {
+            ty.add_named_export(export.name(), export.ty());
         }
         ty
     }
@@ -364,16 +126,15 @@ impl Instance {
     /// This is the [`Store`] that generally serves as a sort of global cache
     /// for various instance-related things.
     pub fn store(&self) -> &Store {
-        &self.handle.store
+        &self.store
     }
 
     /// Returns the list of exported items from this [`Instance`].
     pub fn exports<'instance>(
         &'instance self,
     ) -> impl ExactSizeIterator<Item = Export<'instance>> + 'instance {
-        self.handle.exports().map(move |(name, entity_index)| {
-            let export = self.handle.lookup_by_declaration(entity_index);
-            let extern_ = Extern::from_wasmtime_export(export, self.handle.clone());
+        self.items.iter().map(move |(name, item)| {
+            let extern_ = unsafe { Extern::from_wasmtime_export(item, &self.store) };
             Export::new(name, extern_)
         })
     }
@@ -385,8 +146,8 @@ impl Instance {
     ///
     /// Returns `None` if there was no export named `name`.
     pub fn get_export(&self, name: &str) -> Option<Extern> {
-        let export = self.handle.lookup(&name)?;
-        Some(Extern::from_wasmtime_export(export, self.handle.clone()))
+        let export = self.items.get(name)?;
+        Some(unsafe { Extern::from_wasmtime_export(export, &self.store) })
     }
 
     /// Looks up an exported [`Func`] value by name.
@@ -422,71 +183,366 @@ impl Instance {
     }
 }
 
+struct Instantiator<'a> {
+    in_progress: Vec<ImportsBuilder<'a>>,
+    cur: ImportsBuilder<'a>,
+    store: &'a Store,
+}
+
 struct ImportsBuilder<'a> {
+    src: ImportSource<'a>,
     functions: PrimaryMap<FuncIndex, VMFunctionImport>,
     tables: PrimaryMap<TableIndex, VMTableImport>,
     memories: PrimaryMap<MemoryIndex, VMMemoryImport>,
     globals: PrimaryMap<GlobalIndex, VMGlobalImport>,
-    instances: PrimaryMap<InstanceIndex, InstanceHandle>,
+    instances: PrimaryMap<InstanceIndex, RuntimeInstance>,
     modules: PrimaryMap<ModuleIndex, Module>,
-
-    module: &'a wasmtime_environ::Module,
-    matcher: matching::MatchCx<'a>,
+    initializer: usize,
+    module: Module,
 }
 
-impl<'a> ImportsBuilder<'a> {
-    fn new(store: &'a Store, module: &'a Module) -> ImportsBuilder<'a> {
-        let types = module.types();
-        let module = module.compiled_module().module();
-        ImportsBuilder {
-            module,
-            matcher: matching::MatchCx { store, types },
-            functions: PrimaryMap::with_capacity(module.num_imported_funcs),
-            tables: PrimaryMap::with_capacity(module.num_imported_tables),
-            memories: PrimaryMap::with_capacity(module.num_imported_memories),
-            globals: PrimaryMap::with_capacity(module.num_imported_globals),
-            instances: PrimaryMap::with_capacity(module.instances.len()),
-            modules: PrimaryMap::with_capacity(module.modules.len()),
+enum ImportSource<'a> {
+    Runtime(&'a [Extern]),
+    Outer { initializer: usize },
+}
+
+impl<'a> Instantiator<'a> {
+    /// Creates a new instantiation context used to process all the initializer
+    /// directives of a module.
+    ///
+    /// This doesn't do much work itself beyond setting things up.
+    fn new(store: &'a Store, module: &Module, imports: &'a [Extern]) -> Result<Instantiator<'a>> {
+        if !Engine::same(store.engine(), module.engine()) {
+            bail!("cross-`Engine` instantiation is not currently supported");
+        }
+
+        // Perform some pre-flight checks before we get into the meat of
+        // instantiation.
+        let expected = module.compiled_module().module().imports().count();
+        if expected != imports.len() {
+            bail!("expected {} imports, found {}", expected, imports.len());
+        }
+        for import in imports {
+            if !import.comes_from_same_store(store) {
+                bail!("cross-`Store` instantiation is not currently supported");
+            }
+        }
+
+        Ok(Instantiator {
+            in_progress: Vec::new(),
+            cur: ImportsBuilder::new(module, ImportSource::Runtime(imports)),
+            store,
+        })
+    }
+
+    /// Processes the next initializer for the next instance being created
+    /// without running any wasm code.
+    ///
+    /// This function will process module initializers, handling recursive
+    /// instantiations of modules for module linking if necessary as well. This
+    /// does not actually execute any WebAssembly code, which means that it
+    /// will return whenever an instance is created (because its `start`
+    /// function may need to be executed).
+    ///
+    /// If this function returns `None`, then it simply needs to be called
+    /// again to execute the next initializer. Otherwise this function has two
+    /// return values:
+    ///
+    /// * The first is the raw handle to the instance that was just created.
+    ///   This instance must have its start function executed by the caller.
+    /// * The second is an optional list of items to get wrapped up in an
+    ///   `Instance`. This is only `Some` for the outermost instance that was
+    ///   created. If this is `None` callers need to keep calling this function
+    ///   since the instance created was simply for a recursive instance
+    ///   defined here.
+    fn step(&mut self) -> Result<Option<(StoreInstanceHandle, Option<RuntimeInstance>)>> {
+        if self.cur.initializer == 0 {
+            self.store.bump_resource_counts(&self.cur.module)?;
+        }
+
+        // Read the current module's initializer and move forward the
+        // initializer pointer as well.
+        self.cur.initializer += 1;
+        match self
+            .cur
+            .module
+            .env_module()
+            .initializers
+            .get(self.cur.initializer - 1)
+        {
+            Some(Initializer::Import { index, name, field }) => {
+                match &mut self.cur.src {
+                    // If imports are coming from the runtime-provided list
+                    // (e.g. the root module being instantiated) then we
+                    // need to typecheck each item here before recording it.
+                    //
+                    // Note the `unwrap` here should be ok given the validation
+                    // above in `Instantiation::new`.
+                    ImportSource::Runtime(list) => {
+                        let (head, remaining) = list.split_first().unwrap();
+                        *list = remaining;
+                        let expected_ty =
+                            self.cur.module.compiled_module().module().type_of(*index);
+                        matching::MatchCx {
+                            types: self.cur.module.types(),
+                            store: self.store,
+                        }
+                        .extern_(&expected_ty, head)
+                        .with_context(|| {
+                            let extra = match field {
+                                Some(name) => format!("::{}", name),
+                                None => String::new(),
+                            };
+                            format!("incompatible import type for `{}{}`", name, extra)
+                        })?;
+                        self.cur.push(head);
+                    }
+
+                    // Otherwise if arguments are coming from our outer
+                    // instance due to a recursive instantiation then we
+                    // look in the previous initializer's mapping of
+                    // arguments to figure out where to load the item from.
+                    // Note that no typechecking is necessary here due to
+                    // validation.
+                    ImportSource::Outer { initializer } => {
+                        debug_assert!(field.is_none());
+                        let outer = self.in_progress.last().unwrap();
+                        let args = match &outer.module.env_module().initializers[*initializer] {
+                            Initializer::Instantiate { args, .. } => args,
+                            _ => unreachable!(),
+                        };
+                        let index = args.get(name).expect("should be present after validation");
+                        match *index {
+                            EntityIndex::Global(i) => {
+                                self.cur.globals.push(outer.globals[i]);
+                            }
+                            EntityIndex::Function(i) => {
+                                self.cur.functions.push(outer.functions[i]);
+                            }
+                            EntityIndex::Table(i) => {
+                                self.cur.tables.push(outer.tables[i]);
+                            }
+                            EntityIndex::Memory(i) => {
+                                self.cur.memories.push(outer.memories[i]);
+                            }
+                            EntityIndex::Module(i) => {
+                                self.cur.modules.push(outer.modules[i].clone());
+                            }
+                            EntityIndex::Instance(i) => {
+                                self.cur.instances.push(outer.instances[i].clone());
+                            }
+                        }
+                    }
+                }
+            }
+
+            // Here we lookup our instance handle, find the right export,
+            // and then push that item into our own index space. We eschew
+            // type-checking since only valid modules should reach this point.
+            Some(Initializer::AliasInstanceExport { instance, export }) => {
+                let export = &self.cur.instances[*instance][export];
+                let item = unsafe { Extern::from_wasmtime_export(export, self.store) };
+                self.cur.push(&item);
+            }
+
+            // A recursive instantiation of an instance.
+            //
+            // The `module` argument is used to create an import builder
+            // object, and we specify that the source of imports for the builder is
+            // this initializer's position so we can look at the `args` payload
+            // later.
+            //
+            // Once that's set up we save off `self.cur` into
+            // `self.in_progress` and start the instantiation of the child
+            // instance on the next execution of this function.
+            Some(Initializer::Instantiate { module, args: _ }) => {
+                let module = &self.cur.modules[*module];
+                let imports = ImportsBuilder::new(
+                    module,
+                    ImportSource::Outer {
+                        initializer: self.cur.initializer - 1,
+                    },
+                );
+                let prev = mem::replace(&mut self.cur, imports);
+                self.in_progress.push(prev);
+            }
+
+            // A new module is being defined, and the source of this module is
+            // our module's list of closed-over-modules.
+            //
+            // This is used for outer aliases.
+            Some(Initializer::DefineModule(upvar_index)) => {
+                self.cur
+                    .modules
+                    .push(self.cur.module.module_upvar(*upvar_index).clone());
+            }
+
+            // A new module is defined, created from a set of compiled
+            // artifacts. The new module value will be created with the
+            // specified artifacts being closed over as well as the specified
+            // set of module values in our index/upvar index spaces being closed
+            // over.
+            //
+            // This is used for defining submodules.
+            Some(Initializer::CreateModule {
+                artifact_index,
+                artifacts,
+                modules,
+            }) => {
+                let submodule = self.cur.module.create_submodule(
+                    *artifact_index,
+                    artifacts,
+                    modules,
+                    &self.cur.modules,
+                );
+                self.cur.modules.push(submodule);
+            }
+
+            // All initializers have been processed, which means we're ready to
+            // perform the actual raw instantiation with the raw import values.
+            // Once that's done if there's an in-progress module we record the
+            // instance in the index space. Otherwise this is the final module
+            // and we return the items out.
+            //
+            // Note that in all cases we return the raw instance handle to get
+            // the start function executed by the outer context.
+            None => {
+                let instance = self.instantiate_raw()?;
+                let items = self.runtime_instance(&instance);
+                let items = match self.in_progress.pop() {
+                    Some(imports) => {
+                        self.cur = imports;
+                        self.cur.instances.push(items);
+                        None
+                    }
+                    None => Some(items),
+                };
+                return Ok(Some((instance, items)));
+            }
+        }
+
+        Ok(None)
+    }
+
+    fn instantiate_raw(&self) -> Result<StoreInstanceHandle> {
+        let compiled_module = self.cur.module.compiled_module();
+
+        // Register the module just before instantiation to ensure we have a
+        // trampoline registered for every signature and to preserve the module's
+        // compiled JIT code within the `Store`.
+        self.store.register_module(&self.cur.module);
+
+        let config = self.store.engine().config();
+        unsafe {
+            let instance = compiled_module.instantiate(
+                self.cur.build(),
+                &self.store.lookup_shared_signature(self.cur.module.types()),
+                config.memory_creator.as_ref().map(|a| a as _),
+                self.store.interrupts(),
+                Box::new(()),
+                self.store.externref_activations_table() as *const VMExternRefActivationsTable
+                    as *mut _,
+                self.store.stack_map_registry() as *const StackMapRegistry as *mut _,
+            )?;
+
+            // After we've created the `InstanceHandle` we still need to run
+            // initialization to set up data/elements/etc. We do this after adding
+            // the `InstanceHandle` to the store though. This is required for safety
+            // because the start function (for example) may trap, but element
+            // initializers may have run which placed elements into other instance's
+            // tables. This means that from this point on, regardless of whether
+            // initialization is successful, we need to keep the instance alive.
+            let instance = self.store.add_instance(instance);
+            instance
+                .initialize(
+                    config.features.bulk_memory,
+                    &compiled_module.data_initializers(),
+                )
+                .map_err(|e| -> Error {
+                    match e {
+                        InstantiationError::Trap(trap) => {
+                            Trap::from_runtime(self.store, trap).into()
+                        }
+                        other => other.into(),
+                    }
+                })?;
+
+            Ok(instance)
         }
     }
 
-    fn define_extern(&mut self, expected: &EntityIndex, actual: &Extern) -> Result<()> {
-        let expected_ty = self.module.type_of(*expected);
-        let compatible = match &expected_ty {
-            EntityType::Table(i) => match actual {
-                Extern::Table(e) => self.matcher.table(i, e),
-                _ => bail!("expected table, but found {}", actual.desc()),
-            },
-            EntityType::Memory(i) => match actual {
-                Extern::Memory(e) => self.matcher.memory(i, e),
-                _ => bail!("expected memory, but found {}", actual.desc()),
-            },
-            EntityType::Global(i) => match actual {
-                Extern::Global(e) => self.matcher.global(i, e),
-                _ => bail!("expected global, but found {}", actual.desc()),
-            },
-            EntityType::Function(i) => match actual {
-                Extern::Func(e) => self.matcher.func(*i, e),
-                _ => bail!("expected func, but found {}", actual.desc()),
-            },
-            EntityType::Instance(i) => match actual {
-                Extern::Instance(e) => self.matcher.instance(*i, e),
-                _ => bail!("expected instance, but found {}", actual.desc()),
-            },
-            EntityType::Module(i) => match actual {
-                Extern::Module(e) => self.matcher.module(*i, e),
-                _ => bail!("expected module, but found {}", actual.desc()),
-            },
-            EntityType::Event(_) => unimplemented!(),
-        };
-        if !compatible {
-            bail!("{} types incompatible", actual.desc());
+    fn start_raw(instance: &StoreInstanceHandle) -> Result<()> {
+        let start_func = instance.handle.module().start_func;
+
+        // If a start function is present, invoke it. Make sure we use all the
+        // trap-handling configuration in `store` as well.
+        if let Some(start) = start_func {
+            let f = match instance
+                .handle
+                .lookup_by_declaration(&EntityIndex::Function(start))
+            {
+                wasmtime_runtime::Export::Function(f) => f,
+                _ => unreachable!(), // valid modules shouldn't hit this
+            };
+            let vmctx_ptr = instance.handle.vmctx_ptr();
+            unsafe {
+                super::func::invoke_wasm_and_catch_traps(vmctx_ptr, &instance.store, || {
+                    mem::transmute::<
+                        *const VMFunctionBody,
+                        unsafe extern "C" fn(*mut VMContext, *mut VMContext),
+                    >(f.anyfunc.as_ref().func_ptr.as_ptr())(
+                        f.anyfunc.as_ref().vmctx, vmctx_ptr
+                    )
+                })?;
+            }
         }
-        self.push_extern(actual);
         Ok(())
     }
 
-    fn push_extern(&mut self, item: &Extern) {
+    fn runtime_instance(&self, instance: &StoreInstanceHandle) -> RuntimeInstance {
+        let exports = instance
+            .handle
+            .module()
+            .exports
+            .iter()
+            .map(|(name, index)| {
+                // Note that instances and modules are not handled by
+                // `wasmtime_runtime`, they're handled by us in this crate. That
+                // means we need to handle that here, otherwise we defer to the
+                // instance to load the values.
+                let item = match index {
+                    EntityIndex::Instance(i) => {
+                        wasmtime_runtime::Export::Instance(self.cur.instances[*i].clone())
+                    }
+                    EntityIndex::Module(i) => {
+                        wasmtime_runtime::Export::Module(Box::new(self.cur.modules[*i].clone()))
+                    }
+                    index => instance.handle.lookup_by_declaration(index),
+                };
+                (name.clone(), item)
+            })
+            .collect();
+        Rc::new(exports)
+    }
+}
+
+impl<'a> ImportsBuilder<'a> {
+    fn new(module: &Module, src: ImportSource<'a>) -> ImportsBuilder<'a> {
+        let raw = module.compiled_module().module();
+        ImportsBuilder {
+            src,
+            functions: PrimaryMap::with_capacity(raw.num_imported_funcs),
+            tables: PrimaryMap::with_capacity(raw.num_imported_tables),
+            memories: PrimaryMap::with_capacity(raw.num_imported_memories),
+            globals: PrimaryMap::with_capacity(raw.num_imported_globals),
+            instances: PrimaryMap::with_capacity(raw.instances.len()),
+            modules: PrimaryMap::with_capacity(raw.modules.len()),
+            module: module.clone(),
+            initializer: 0,
+        }
+    }
+
+    fn push(&mut self, item: &Extern) {
         match item {
             Extern::Func(i) => {
                 self.functions.push(i.vmimport());
@@ -501,8 +557,7 @@ impl<'a> ImportsBuilder<'a> {
                 self.memories.push(i.vmimport());
             }
             Extern::Instance(i) => {
-                debug_assert!(Store::same(i.store(), self.matcher.store));
-                self.instances.push(unsafe { (*i.handle).clone() });
+                self.instances.push(i.items.clone());
             }
             Extern::Module(m) => {
                 self.modules.push(m.clone());
@@ -510,17 +565,46 @@ impl<'a> ImportsBuilder<'a> {
         }
     }
 
-    fn build(&mut self) -> Imports<'_> {
+    fn build(&self) -> Imports<'_> {
         Imports {
             tables: self.tables.values().as_slice(),
             globals: self.globals.values().as_slice(),
             memories: self.memories.values().as_slice(),
             functions: self.functions.values().as_slice(),
-            instances: mem::take(&mut self.instances),
-            modules: mem::take(&mut self.modules)
-                .into_iter()
-                .map(|(_, m)| Box::new(m) as Box<_>)
-                .collect(),
+        }
+    }
+}
+
+/// An internal structure to this crate to build an `Instance` from a list of
+/// items with names. This is intended to stay private for now, it'll need an
+/// audit of APIs if publicly exported.
+#[derive(Default)]
+pub(crate) struct InstanceBuilder {
+    items: RuntimeInstance,
+}
+
+impl InstanceBuilder {
+    pub(crate) fn new() -> InstanceBuilder {
+        InstanceBuilder::default()
+    }
+
+    pub(crate) fn insert(&mut self, name: &str, item: impl Into<Extern>) {
+        let items = Rc::get_mut(&mut self.items).unwrap();
+        let export = match item.into() {
+            Extern::Func(i) => wasmtime_runtime::Export::Function(i.wasmtime_export().clone()),
+            Extern::Memory(i) => wasmtime_runtime::Export::Memory(i.wasmtime_export().clone()),
+            Extern::Table(i) => wasmtime_runtime::Export::Table(i.wasmtime_export().clone()),
+            Extern::Global(i) => wasmtime_runtime::Export::Global(i.wasmtime_export().clone()),
+            Extern::Instance(i) => wasmtime_runtime::Export::Instance(i.items.clone()),
+            Extern::Module(i) => wasmtime_runtime::Export::Module(Box::new(i.clone())),
+        };
+        items.insert(name.to_string(), export);
+    }
+
+    pub(crate) fn finish(self, store: &Store) -> Instance {
+        Instance {
+            store: store.clone(),
+            items: self.items,
         }
     }
 }
diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs
index 2248eec5a7..9deb3638d0 100644
--- a/crates/wasmtime/src/lib.rs
+++ b/crates/wasmtime/src/lib.rs
@@ -241,6 +241,7 @@ mod frame_info;
 mod func;
 mod instance;
 mod linker;
+mod memory;
 mod module;
 mod r#ref;
 mod sig_registry;
@@ -257,6 +258,7 @@ pub use crate::frame_info::{FrameInfo, FrameSymbol};
 pub use crate::func::*;
 pub use crate::instance::Instance;
 pub use crate::linker::*;
+pub use crate::memory::*;
 pub use crate::module::Module;
 pub use crate::r#ref::ExternRef;
 pub use crate::store::*;
diff --git a/crates/wasmtime/src/linker.rs b/crates/wasmtime/src/linker.rs
index 763d0fa3cb..396327b5d5 100644
--- a/crates/wasmtime/src/linker.rs
+++ b/crates/wasmtime/src/linker.rs
@@ -1,3 +1,4 @@
+use crate::instance::InstanceBuilder;
 use crate::{
     Extern, ExternType, Func, FuncType, GlobalType, ImportType, Instance, IntoFunc, Module, Store,
     Trap,
@@ -654,7 +655,37 @@ impl Linker {
             },
             kind: self.import_kind(import.ty()),
         };
-        self.map.get(&key).cloned()
+        if let Some(result) = self.map.get(&key).cloned() {
+            return Some(result);
+        }
+
+        // This is a key location where the module linking proposal is
+        // implemented. This logic allows single-level imports of an instance to
+        // get satisfied by multiple definitions of items within this `Linker`.
+        //
+        // The instance being import is iterated over to load the names from
+        // this `Linker` (recursively calling `get`). If anything isn't defined
+        // we return `None` since the entire value isn't defined. Otherwise when
+        // all values are loaded it's assembled into an `Instance` and
+        // returned`.
+        //
+        // Note that this isn't exactly the speediest implementation in the
+        // world. Ideally we would pre-create the `Instance` instead of creating
+        // it each time a module is instantiated. For now though while the
+        // module linking proposal is under development this should hopefully
+        // suffice.
+        if let ExternType::Instance(t) = import.ty() {
+            if import.name().is_none() {
+                let mut builder = InstanceBuilder::new();
+                for export in t.exports() {
+                    let item = self.get(&export.as_import(import.module()))?;
+                    builder.insert(export.name(), item);
+                }
+                return Some(builder.finish(&self.store).into());
+            }
+        }
+
+        None
     }
 
     /// Returns all items defined for the `module` and `name` pair.
diff --git a/crates/wasmtime/src/memory.rs b/crates/wasmtime/src/memory.rs
new file mode 100644
index 0000000000..0b2d21e2b6
--- /dev/null
+++ b/crates/wasmtime/src/memory.rs
@@ -0,0 +1,578 @@
+use crate::trampoline::{generate_memory_export, StoreInstanceHandle};
+use crate::{MemoryType, Store};
+use anyhow::{anyhow, Result};
+use std::slice;
+
+/// Error for out of bounds [`Memory`] access.
+#[derive(Debug)]
+#[non_exhaustive]
+pub struct MemoryAccessError {
+    // Keep struct internals private for future extensibility.
+    _private: (),
+}
+
+impl std::fmt::Display for MemoryAccessError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "out of bounds memory access")
+    }
+}
+
+impl std::error::Error for MemoryAccessError {}
+
+/// A WebAssembly linear memory.
+///
+/// WebAssembly memories represent a contiguous array of bytes that have a size
+/// that is always a multiple of the WebAssembly page size, currently 64
+/// kilobytes.
+///
+/// WebAssembly memory is used for global data, statics in C/C++/Rust, shadow
+/// stack memory, etc. Accessing wasm memory is generally quite fast!
+///
+/// # `Memory` and `Clone`
+///
+/// Memories are internally reference counted so you can `clone` a `Memory`. The
+/// cloning process only performs a shallow clone, so two cloned `Memory`
+/// instances are equivalent in their functionality.
+///
+/// # `Memory` and threads
+///
+/// It is intended that `Memory` is safe to share between threads. At this time
+/// this is not implemented in `wasmtime`, however. This is planned to be
+/// implemented though!
+///
+/// # `Memory` and Safety
+///
+/// Linear memory is a lynchpin of safety for WebAssembly, but it turns out
+/// there are very few ways to safely inspect the contents of a memory from the
+/// host (Rust). This is because memory safety is quite tricky when working with
+/// a `Memory` and we're still working out the best idioms to encapsulate
+/// everything safely where it's efficient and ergonomic. This section of
+/// documentation, however, is intended to help educate a bit what is and isn't
+/// safe when working with `Memory`.
+///
+/// For safety purposes you can think of a `Memory` as a glorified
+/// `Rc<UnsafeCell<Vec<u8>>>`. There are a few consequences of this
+/// interpretation:
+///
+/// * At any time someone else may have access to the memory (hence the `Rc`).
+///   This could be a wasm instance, other host code, or a set of wasm instances
+///   which all reference a `Memory`. When in doubt assume someone else has a
+///   handle to your `Memory`.
+///
+/// * At any time, memory can be read from or written to (hence the
+///   `UnsafeCell`). Anyone with a handle to a wasm memory can read/write to it.
+///   Primarily other instances can execute the `load` and `store` family of
+///   instructions, as well as any other which modifies or reads memory.
+///
+/// * At any time memory may grow (hence the `Vec<..>`). Growth may relocate the
+///   base memory pointer (similar to how `vec.push(...)` can change the result
+///   of `.as_ptr()`)
+///
+/// So given that we're working roughly with `Rc<UnsafeCell<Vec<u8>>>` that's a
+/// lot to keep in mind! It's hopefully though sort of setting the stage as to
+/// what you can safely do with memories.
+///
+/// Let's run through a few safe examples first of how you can use a `Memory`.
+///
+/// ```rust
+/// use wasmtime::{Memory, MemoryAccessError};
+///
+/// // Memory can be read and written safely with the `Memory::read` and
+/// // `Memory::write` methods.
+/// // An error is returned if the copy did not succeed.
+/// fn safe_examples(mem: &Memory) -> Result<(), MemoryAccessError> {
+///     let offset = 5;
+///     mem.write(offset, b"hello")?;
+///     let mut buffer = [0u8; 5];
+///     mem.read(offset, &mut buffer)?;
+///     assert_eq!(b"hello", &buffer);
+///     Ok(())
+/// }
+///
+/// // You can also get direct, unsafe access to the memory, but must manually
+/// // ensure that safety invariants are upheld.
+///
+/// fn correct_unsafe_examples(mem: &Memory) {
+///     // Just like wasm, it's safe to read memory almost at any time. The
+///     // gotcha here is that we need to be sure to load from the correct base
+///     // pointer and perform the bounds check correctly. So long as this is
+///     // all self contained here (e.g. not arbitrary code in the middle) we're
+///     // good to go.
+///     let byte = unsafe { mem.data_unchecked()[0x123] };
+///
+///     // Short-lived borrows of memory are safe, but they must be scoped and
+///     // not have code which modifies/etc `Memory` while the borrow is active.
+///     // For example if you want to read a string from memory it is safe to do
+///     // so:
+///     let string_base = 0xdead;
+///     let string_len = 0xbeef;
+///     let string = unsafe {
+///         let bytes = &mem.data_unchecked()[string_base..][..string_len];
+///         match std::str::from_utf8(bytes) {
+///             Ok(s) => s.to_string(), // copy out of wasm memory
+///             Err(_) => panic!("not valid utf-8"),
+///         }
+///     };
+///
+///     // Additionally like wasm you can write to memory at any point in time,
+///     // again making sure that after you get the unchecked slice you don't
+///     // execute code which could read/write/modify `Memory`:
+///     unsafe {
+///         mem.data_unchecked_mut()[0x123] = 3;
+///     }
+///
+///     // When working with *borrows* that point directly into wasm memory you
+///     // need to be extremely careful. Any functionality that operates on a
+///     // borrow into wasm memory needs to be thoroughly audited to effectively
+///     // not touch the `Memory` at all
+///     let data_base = 0xfeed;
+///     let data_len = 0xface;
+///     unsafe {
+///         let data = &mem.data_unchecked()[data_base..][..data_len];
+///         host_function_that_doesnt_touch_memory(data);
+///
+///         // effectively the same rules apply to mutable borrows
+///         let data_mut = &mut mem.data_unchecked_mut()[data_base..][..data_len];
+///         host_function_that_doesnt_touch_memory(data);
+///     }
+/// }
+/// # fn host_function_that_doesnt_touch_memory(_: &[u8]){}
+/// ```
+///
+/// It's worth also, however, covering some examples of **incorrect**,
+/// **unsafe** usages of `Memory`. Do not do these things!
+///
+/// ```rust
+/// # use anyhow::Result;
+/// use wasmtime::Memory;
+///
+/// // NOTE: All code in this function is not safe to execute and may cause
+/// // segfaults/undefined behavior at runtime. Do not copy/paste these examples
+/// // into production code!
+/// unsafe fn unsafe_examples(mem: &Memory) -> Result<()> {
+///     // First and foremost, any borrow can be invalidated at any time via the
+///     // `Memory::grow` function. This can relocate memory which causes any
+///     // previous pointer to be possibly invalid now.
+///     let pointer: &u8 = &mem.data_unchecked()[0x100];
+///     mem.grow(1)?; // invalidates `pointer`!
+///     // println!("{}", *pointer); // FATAL: use-after-free
+///
+///     // Note that the use-after-free also applies to slices, whether they're
+///     // slices of bytes or strings.
+///     let slice: &[u8] = &mem.data_unchecked()[0x100..0x102];
+///     mem.grow(1)?; // invalidates `slice`!
+///     // println!("{:?}", slice); // FATAL: use-after-free
+///
+///     // Due to the reference-counted nature of `Memory` note that literal
+///     // calls to `Memory::grow` are not sufficient to audit for. You'll need
+///     // to be careful that any mutation of `Memory` doesn't happen while
+///     // you're holding an active borrow.
+///     let slice: &[u8] = &mem.data_unchecked()[0x100..0x102];
+///     some_other_function(); // may invalidate `slice` through another `mem` reference
+///     // println!("{:?}", slice); // FATAL: maybe a use-after-free
+///
+///     // An especially subtle aspect of accessing a wasm instance's memory is
+///     // that you need to be extremely careful about aliasing. Anyone at any
+///     // time can call `data_unchecked()` or `data_unchecked_mut()`, which
+///     // means you can easily have aliasing mutable references:
+///     let ref1: &u8 = &mem.data_unchecked()[0x100];
+///     let ref2: &mut u8 = &mut mem.data_unchecked_mut()[0x100];
+///     // *ref2 = *ref1; // FATAL: violates Rust's aliasing rules
+///
+///     // Note that aliasing applies to strings as well, for example this is
+///     // not valid because the slices overlap.
+///     let slice1: &mut [u8] = &mut mem.data_unchecked_mut()[0x100..][..3];
+///     let slice2: &mut [u8] = &mut mem.data_unchecked_mut()[0x102..][..4];
+///     // println!("{:?} {:?}", slice1, slice2); // FATAL: aliasing mutable pointers
+///
+///     Ok(())
+/// }
+/// # fn some_other_function() {}
+/// ```
+///
+/// Overall there's some general rules of thumb when working with `Memory` and
+/// getting raw pointers inside of it:
+///
+/// * If you never have a "long lived" pointer into memory, you're likely in the
+///   clear. Care still needs to be taken in threaded scenarios or when/where
+///   data is read, but you'll be shielded from many classes of issues.
+/// * Long-lived pointers must always respect Rust'a aliasing rules. It's ok for
+///   shared borrows to overlap with each other, but mutable borrows must
+///   overlap with nothing.
+/// * Long-lived pointers are only valid if `Memory` isn't used in an unsafe way
+///   while the pointer is valid. This includes both aliasing and growth.
+///
+/// At this point it's worth reiterating again that working with `Memory` is
+/// pretty tricky and that's not great! Proposals such as [interface types] are
+/// intended to prevent wasm modules from even needing to import/export memory
+/// in the first place, which obviates the need for all of these safety caveats!
+/// Additionally over time we're still working out the best idioms to expose in
+/// `wasmtime`, so if you've got ideas or questions please feel free to [open an
+/// issue]!
+///
+/// ## `Memory` Safety and Threads
+///
+/// Currently the `wasmtime` crate does not implement the wasm threads proposal,
+/// but it is planned to do so. It's additionally worthwhile discussing how this
+/// affects memory safety and what was previously just discussed as well.
+///
+/// Once threads are added into the mix, all of the above rules still apply.
+/// There's an additional, rule, however, that all reads and writes can
+/// happen *concurrently*. This effectively means that long-lived borrows into
+/// wasm memory are virtually never safe to have.
+///
+/// Mutable pointers are fundamentally unsafe to have in a concurrent scenario
+/// in the face of arbitrary wasm code. Only if you dynamically know for sure
+/// that wasm won't access a region would it be safe to construct a mutable
+/// pointer. Additionally even shared pointers are largely unsafe because their
+/// underlying contents may change, so unless `UnsafeCell` in one form or
+/// another is used everywhere there's no safety.
+///
+/// One important point about concurrency is that `Memory::grow` can indeed
+/// happen concurrently. This, however, will never relocate the base pointer.
+/// Shared memories must always have a maximum size and they will be
+/// preallocated such that growth will never relocate the base pointer. The
+/// maximum length of the memory, however, will change over time.
+///
+/// Overall the general rule of thumb for shared memories is that you must
+/// atomically read and write everything. Nothing can be borrowed and everything
+/// must be eagerly copied out.
+///
+/// [interface types]: https://github.com/webassembly/interface-types
+/// [open an issue]: https://github.com/bytecodealliance/wasmtime/issues/new
+#[derive(Clone)]
+pub struct Memory {
+    pub(crate) instance: StoreInstanceHandle,
+    wasmtime_export: wasmtime_runtime::ExportMemory,
+}
+
+impl Memory {
+    /// Creates a new WebAssembly memory given the configuration of `ty`.
+    ///
+    /// The `store` argument is a general location for cache information, and
+    /// otherwise the memory will immediately be allocated according to the
+    /// type's configuration. All WebAssembly memory is initialized to zero.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use wasmtime::*;
+    /// # fn main() -> anyhow::Result<()> {
+    /// let engine = Engine::default();
+    /// let store = Store::new(&engine);
+    ///
+    /// let memory_ty = MemoryType::new(Limits::new(1, None));
+    /// let memory = Memory::new(&store, memory_ty);
+    ///
+    /// let module = Module::new(&engine, "(module (memory (import \"\" \"\") 1))")?;
+    /// let instance = Instance::new(&store, &module, &[memory.into()])?;
+    /// // ...
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn new(store: &Store, ty: MemoryType) -> Memory {
+        let (instance, wasmtime_export) =
+            generate_memory_export(store, &ty).expect("generated memory");
+        Memory {
+            instance,
+            wasmtime_export,
+        }
+    }
+
+    /// Returns the underlying type of this memory.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use wasmtime::*;
+    /// # fn main() -> anyhow::Result<()> {
+    /// let engine = Engine::default();
+    /// let store = Store::new(&engine);
+    /// let module = Module::new(&engine, "(module (memory (export \"mem\") 1))")?;
+    /// let instance = Instance::new(&store, &module, &[])?;
+    /// let memory = instance.get_memory("mem").unwrap();
+    /// let ty = memory.ty();
+    /// assert_eq!(ty.limits().min(), 1);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn ty(&self) -> MemoryType {
+        MemoryType::from_wasmtime_memory(&self.wasmtime_export.memory.memory)
+    }
+
+    /// Safely reads memory contents at the given offset into a buffer.
+    ///
+    /// The entire buffer will be filled.
+    ///
+    /// If offset + buffer length exceed the current memory capacity, then the
+    /// buffer is left untouched and a [`MemoryAccessError`] is returned.
+    pub fn read(&self, offset: usize, buffer: &mut [u8]) -> Result<(), MemoryAccessError> {
+        unsafe {
+            let slice = self
+                .data_unchecked()
+                .get(offset..)
+                .and_then(|s| s.get(..buffer.len()))
+                .ok_or(MemoryAccessError { _private: () })?;
+            buffer.copy_from_slice(slice);
+            Ok(())
+        }
+    }
+
+    /// Safely writes contents of a buffer to this memory at the given offset.
+    ///
+    /// If the offset + buffer length exceed current memory capacity, then none
+    /// of the buffer is written to memory and a [`MemoryAccessError`] is
+    /// returned.
+    pub fn write(&self, offset: usize, buffer: &[u8]) -> Result<(), MemoryAccessError> {
+        unsafe {
+            self.data_unchecked_mut()
+                .get_mut(offset..)
+                .and_then(|s| s.get_mut(..buffer.len()))
+                .ok_or(MemoryAccessError { _private: () })?
+                .copy_from_slice(buffer);
+            Ok(())
+        }
+    }
+
+    /// Returns this memory as a slice view that can be read natively in Rust.
+    ///
+    /// # Safety
+    ///
+    /// This is an unsafe operation because there is no guarantee that the
+    /// following operations do not happen concurrently while the slice is in
+    /// use:
+    ///
+    /// * Data could be modified by calling into a wasm module.
+    /// * Memory could be relocated through growth by calling into a wasm
+    ///   module.
+    /// * When threads are supported, non-atomic reads will race with other
+    ///   writes.
+    ///
+    /// Extreme care need be taken when the data of a `Memory` is read. The
+    /// above invariants all need to be upheld at a bare minimum, and in
+    /// general you'll need to ensure that while you're looking at slice you're
+    /// the only one who can possibly look at the slice and read/write it.
+    ///
+    /// Be sure to keep in mind that `Memory` is reference counted, meaning
+    /// that there may be other users of this `Memory` instance elsewhere in
+    /// your program. Additionally `Memory` can be shared and used in any number
+    /// of wasm instances, so calling any wasm code should be considered
+    /// dangerous while you're holding a slice of memory.
+    ///
+    /// For more information and examples see the documentation on the
+    /// [`Memory`] type.
+    pub unsafe fn data_unchecked(&self) -> &[u8] {
+        self.data_unchecked_mut()
+    }
+
+    /// Returns this memory as a slice view that can be read and written
+    /// natively in Rust.
+    ///
+    /// # Safety
+    ///
+    /// All of the same safety caveats of [`Memory::data_unchecked`] apply
+    /// here, doubly so because this is returning a mutable slice! As a
+    /// double-extra reminder, remember that `Memory` is reference counted, so
+    /// you can very easily acquire two mutable slices by simply calling this
+    /// function twice. Extreme caution should be used when using this method,
+    /// and in general you probably want to result to unsafe accessors and the
+    /// `data` methods below.
+    ///
+    /// For more information and examples see the documentation on the
+    /// [`Memory`] type.
+    pub unsafe fn data_unchecked_mut(&self) -> &mut [u8] {
+        let definition = &*self.wasmtime_export.definition;
+        slice::from_raw_parts_mut(definition.base, definition.current_length)
+    }
+
+    /// Returns the base pointer, in the host's address space, that the memory
+    /// is located at.
+    ///
+    /// When reading and manipulating memory be sure to read up on the caveats
+    /// of [`Memory::data_unchecked`] to make sure that you can safely
+    /// read/write the memory.
+    ///
+    /// For more information and examples see the documentation on the
+    /// [`Memory`] type.
+    pub fn data_ptr(&self) -> *mut u8 {
+        unsafe { (*self.wasmtime_export.definition).base }
+    }
+
+    /// Returns the byte length of this memory.
+    ///
+    /// The returned value will be a multiple of the wasm page size, 64k.
+    ///
+    /// For more information and examples see the documentation on the
+    /// [`Memory`] type.
+    pub fn data_size(&self) -> usize {
+        unsafe { (*self.wasmtime_export.definition).current_length }
+    }
+
+    /// Returns the size, in pages, of this wasm memory.
+    pub fn size(&self) -> u32 {
+        (self.data_size() / wasmtime_environ::WASM_PAGE_SIZE as usize) as u32
+    }
+
+    /// Grows this WebAssembly memory by `delta` pages.
+    ///
+    /// This will attempt to add `delta` more pages of memory on to the end of
+    /// this `Memory` instance. If successful this may relocate the memory and
+    /// cause [`Memory::data_ptr`] to return a new value. Additionally previous
+    /// slices into this memory may no longer be valid.
+    ///
+    /// On success returns the number of pages this memory previously had
+    /// before the growth succeeded.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if memory could not be grown, for example if it exceeds
+    /// the maximum limits of this memory.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use wasmtime::*;
+    /// # fn main() -> anyhow::Result<()> {
+    /// let engine = Engine::default();
+    /// let store = Store::new(&engine);
+    /// let module = Module::new(&engine, "(module (memory (export \"mem\") 1 2))")?;
+    /// let instance = Instance::new(&store, &module, &[])?;
+    /// let memory = instance.get_memory("mem").unwrap();
+    ///
+    /// assert_eq!(memory.size(), 1);
+    /// assert_eq!(memory.grow(1)?, 1);
+    /// assert_eq!(memory.size(), 2);
+    /// assert!(memory.grow(1).is_err());
+    /// assert_eq!(memory.size(), 2);
+    /// assert_eq!(memory.grow(0)?, 2);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn grow(&self, delta: u32) -> Result<u32> {
+        let index = self
+            .instance
+            .memory_index(unsafe { &*self.wasmtime_export.definition });
+        self.instance
+            .memory_grow(index, delta)
+            .ok_or_else(|| anyhow!("failed to grow memory"))
+    }
+
+    pub(crate) unsafe fn from_wasmtime_memory(
+        wasmtime_export: &wasmtime_runtime::ExportMemory,
+        store: &Store,
+    ) -> Memory {
+        Memory {
+            instance: store.existing_vmctx(wasmtime_export.vmctx),
+            wasmtime_export: wasmtime_export.clone(),
+        }
+    }
+
+    pub(crate) fn wasmtime_ty(&self) -> &wasmtime_environ::wasm::Memory {
+        &self.wasmtime_export.memory.memory
+    }
+
+    pub(crate) fn vmimport(&self) -> wasmtime_runtime::VMMemoryImport {
+        wasmtime_runtime::VMMemoryImport {
+            from: self.wasmtime_export.definition,
+            vmctx: self.wasmtime_export.vmctx,
+        }
+    }
+
+    pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportMemory {
+        &self.wasmtime_export
+    }
+}
+
+/// A linear memory. This trait provides an interface for raw memory buffers which are used
+/// by wasmtime, e.g. inside ['Memory']. Such buffers are in principle not thread safe.
+/// By implementing this trait together with MemoryCreator,
+/// one can supply wasmtime with custom allocated host managed memory.
+///
+/// # Safety
+/// The memory should be page aligned and a multiple of page size.
+/// To prevent possible silent overflows, the memory should be protected by a guard page.
+/// Additionally the safety concerns explained in ['Memory'], for accessing the memory
+/// apply here as well.
+///
+/// Note that this is a relatively new and experimental feature and it is recommended
+/// to be familiar with wasmtime runtime code to use it.
+pub unsafe trait LinearMemory {
+    /// Returns the number of allocated wasm pages.
+    fn size(&self) -> u32;
+
+    /// Grow memory by the specified amount of wasm pages.
+    ///
+    /// Returns `None` if memory can't be grown by the specified amount
+    /// of wasm pages.
+    fn grow(&self, delta: u32) -> Option<u32>;
+
+    /// Return the allocated memory as a mutable pointer to u8.
+    fn as_ptr(&self) -> *mut u8;
+}
+
+/// A memory creator. Can be used to provide a memory creator
+/// to wasmtime which supplies host managed memory.
+///
+/// # Safety
+/// This trait is unsafe, as the memory safety depends on proper implementation of
+/// memory management. Memories created by the MemoryCreator should always be treated
+/// as owned by wasmtime instance, and any modification of them outside of wasmtime
+/// invoked routines is unsafe and may lead to corruption.
+///
+/// Note that this is a relatively new and experimental feature and it is recommended
+/// to be familiar with wasmtime runtime code to use it.
+pub unsafe trait MemoryCreator: Send + Sync {
+    /// Create a new `LinearMemory` object from the specified parameters.
+    ///
+    /// The type of memory being created is specified by `ty` which indicates
+    /// both the minimum and maximum size, in wasm pages.
+    ///
+    /// The `reserved_size_in_bytes` value indicates the expected size of the
+    /// reservation that is to be made for this memory. If this value is `None`
+    /// than the implementation is free to allocate memory as it sees fit. If
+    /// the value is `Some`, however, then the implementation is expected to
+    /// reserve that many bytes for the memory's allocation, plus the guard
+    /// size at the end. Note that this reservation need only be a virtual
+    /// memory reservation, physical memory does not need to be allocated
+    /// immediately. In this case `grow` should never move the base pointer and
+    /// the maximum size of `ty` is guaranteed to fit within `reserved_size_in_bytes`.
+    ///
+    /// The `guard_size_in_bytes` parameter indicates how many bytes of space, after the
+    /// memory allocation, is expected to be unmapped. JIT code will elide
+    /// bounds checks based on the `guard_size_in_bytes` provided, so for JIT code to
+    /// work correctly the memory returned will need to be properly guarded with
+    /// `guard_size_in_bytes` bytes left unmapped after the base allocation.
+    ///
+    /// Note that the `reserved_size_in_bytes` and `guard_size_in_bytes` options are tuned from
+    /// the various [`Config`](crate::Config) methods about memory
+    /// sizes/guards. Additionally these two values are guaranteed to be
+    /// multiples of the system page size.
+    fn new_memory(
+        &self,
+        ty: MemoryType,
+        reserved_size_in_bytes: Option<u64>,
+        guard_size_in_bytes: u64,
+    ) -> Result<Box<dyn LinearMemory>, String>;
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::*;
+
+    // Assert that creating a memory via `Memory::new` respects the limits/tunables
+    // in `Config`.
+    #[test]
+    fn respect_tunables() {
+        let mut cfg = Config::new();
+        cfg.static_memory_maximum_size(0)
+            .dynamic_memory_guard_size(0);
+        let store = Store::new(&Engine::new(&cfg));
+        let ty = MemoryType::new(Limits::new(1, None));
+        let mem = Memory::new(&store, ty);
+        assert_eq!(mem.wasmtime_export.memory.offset_guard_size, 0);
+        match mem.wasmtime_export.memory.style {
+            wasmtime_environ::MemoryStyle::Dynamic => {}
+            other => panic!("unexpected style {:?}", other),
+        }
+    }
+}
diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs
index d30309c5fc..0bcf8f5686 100644
--- a/crates/wasmtime/src/module.rs
+++ b/crates/wasmtime/src/module.rs
@@ -2,12 +2,16 @@ use crate::types::{ExportType, ExternType, ImportType};
 use crate::{Engine, ModuleType};
 use anyhow::{bail, Context, Result};
 use bincode::Options;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use std::hash::Hash;
 use std::path::Path;
 use std::sync::Arc;
 use wasmparser::Validator;
 #[cfg(feature = "cache")]
 use wasmtime_cache::ModuleCacheEntry;
+use wasmtime_environ::entity::PrimaryMap;
+use wasmtime_environ::wasm::ModuleIndex;
 use wasmtime_jit::{CompilationArtifacts, CompiledModule, TypeTables};
 
 /// A compiled WebAssembly module, ready to be instantiated.
@@ -80,14 +84,72 @@ use wasmtime_jit::{CompilationArtifacts, CompiledModule, TypeTables};
 /// [`Config`]: crate::Config
 #[derive(Clone)]
 pub struct Module {
-    engine: Engine,
-    data: Arc<ModuleData>,
-    index: usize,
+    inner: Arc<ModuleInner>,
 }
 
-pub(crate) struct ModuleData {
-    pub(crate) types: Arc<TypeTables>,
-    pub(crate) modules: Vec<CompiledModule>,
+struct ModuleInner {
+    engine: Engine,
+    /// The compiled artifacts for this module that will be instantiated and
+    /// executed.
+    module: Arc<CompiledModule>,
+    /// Closed-over compilation artifacts used to create submodules when this
+    /// module is instantiated.
+    artifact_upvars: Vec<Arc<CompiledModule>>,
+    /// Closed-over module values which are used when this module is
+    /// instantiated.
+    module_upvars: Vec<Module>,
+    /// Type information of this module and all `artifact_upvars` compiled
+    /// modules.
+    types: Arc<TypeTables>,
+}
+
+/// A small helper struct which defines modules are serialized.
+#[derive(serde::Serialize, serde::Deserialize)]
+struct ModuleSerialized<'a> {
+    /// All compiled artifacts neeeded by this module, where the last entry in
+    /// this list is the artifacts for the module itself.
+    artifacts: Vec<MyCow<'a, CompilationArtifacts>>,
+    /// Closed-over module values that are also needed for this module.
+    modules: Vec<ModuleSerialized<'a>>,
+    /// The index into the list of type tables that are used for this module's
+    /// type tables.
+    type_tables: usize,
+}
+
+// This is like `std::borrow::Cow` but it doesn't have a `Clone` bound on `T`
+enum MyCow<'a, T> {
+    Borrowed(&'a T),
+    Owned(T),
+}
+
+impl<'a, T> MyCow<'a, T> {
+    fn unwrap_owned(self) -> T {
+        match self {
+            MyCow::Owned(val) => val,
+            MyCow::Borrowed(_) => unreachable!(),
+        }
+    }
+}
+
+impl<'a, T: Serialize> Serialize for MyCow<'a, T> {
+    fn serialize<S>(&self, dst: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::ser::Serializer,
+    {
+        match self {
+            MyCow::Borrowed(val) => val.serialize(dst),
+            MyCow::Owned(val) => val.serialize(dst),
+        }
+    }
+}
+
+impl<'a, 'b, T: Deserialize<'a>> Deserialize<'a> for MyCow<'b, T> {
+    fn deserialize<D>(src: D) -> Result<Self, D::Error>
+    where
+        D: serde::de::Deserializer<'a>,
+    {
+        Ok(MyCow::Owned(T::deserialize(src)?))
+    }
 }
 
 impl Module {
@@ -169,7 +231,8 @@ impl Module {
     /// See [`Module::new`] for other details.
     pub fn new_with_name(engine: &Engine, bytes: impl AsRef<[u8]>, name: &str) -> Result<Module> {
         let mut module = Module::new(engine, bytes.as_ref())?;
-        Arc::get_mut(&mut module.data).unwrap().modules[module.index]
+        Arc::get_mut(&mut Arc::get_mut(&mut module.inner).unwrap().module)
+            .unwrap()
             .module_mut()
             .expect("mutable module")
             .name = Some(name.to_string());
@@ -245,24 +308,30 @@ impl Module {
     /// ```
     pub fn from_binary(engine: &Engine, binary: &[u8]) -> Result<Module> {
         #[cfg(feature = "cache")]
-        let (artifacts, types) = ModuleCacheEntry::new("wasmtime", engine.cache_config())
-            .get_data((engine.compiler(), binary), |(compiler, binary)| {
-                CompilationArtifacts::build(compiler, binary)
-            })?;
+        let (main_module, artifacts, types) =
+            ModuleCacheEntry::new("wasmtime", engine.cache_config())
+                .get_data((engine.compiler(), binary), |(compiler, binary)| {
+                    CompilationArtifacts::build(compiler, binary)
+                })?;
         #[cfg(not(feature = "cache"))]
-        let (artifacts, types) = CompilationArtifacts::build(engine.compiler(), binary)?;
+        let (main_module, artifacts, types) =
+            CompilationArtifacts::build(engine.compiler(), binary)?;
 
-        let modules = CompiledModule::from_artifacts_list(
+        let mut modules = CompiledModule::from_artifacts_list(
             artifacts,
             engine.compiler().isa(),
             &*engine.config().profiler,
         )?;
+        let module = modules.remove(main_module);
 
-        let types = Arc::new(types);
         Ok(Module {
-            engine: engine.clone(),
-            index: 0,
-            data: Arc::new(ModuleData { types, modules }),
+            inner: Arc::new(ModuleInner {
+                engine: engine.clone(),
+                module,
+                types: Arc::new(types),
+                artifact_upvars: modules,
+                module_upvars: Vec::new(),
+            }),
         })
     }
 
@@ -311,21 +380,46 @@ impl Module {
 
     /// Serialize compilation artifacts to the buffer. See also `deseriaize`.
     pub fn serialize(&self) -> Result<Vec<u8>> {
-        let artifacts = (
-            compiler_fingerprint(&self.engine),
-            self.data
-                .modules
-                .iter()
-                .map(|i| i.compilation_artifacts())
-                .collect::<Vec<_>>(),
-            &*self.data.types,
-            self.index,
-        );
-
+        let mut pushed = HashMap::new();
+        let mut tables = Vec::new();
+        let module = self.serialized_module(&mut pushed, &mut tables);
+        let artifacts = (compiler_fingerprint(self.engine()), tables, module);
         let buffer = bincode_options().serialize(&artifacts)?;
         Ok(buffer)
     }
 
+    fn serialized_module<'a>(
+        &'a self,
+        type_tables_pushed: &mut HashMap<usize, usize>,
+        type_tables: &mut Vec<&'a TypeTables>,
+    ) -> ModuleSerialized<'a> {
+        // Deduplicate `Arc<TypeTables>` using our two parameters to ensure we
+        // serialize type tables as little as possible.
+        let ptr = Arc::as_ptr(self.types());
+        let type_tables_idx = *type_tables_pushed.entry(ptr as usize).or_insert_with(|| {
+            type_tables.push(self.types());
+            type_tables.len() - 1
+        });
+        ModuleSerialized {
+            artifacts: self
+                .inner
+                .artifact_upvars
+                .iter()
+                .map(|i| MyCow::Borrowed(i.compilation_artifacts()))
+                .chain(Some(MyCow::Borrowed(
+                    self.compiled_module().compilation_artifacts(),
+                )))
+                .collect(),
+            modules: self
+                .inner
+                .module_upvars
+                .iter()
+                .map(|i| i.serialized_module(type_tables_pushed, type_tables))
+                .collect(),
+            type_tables: type_tables_idx,
+        }
+    }
+
     /// Deserializes and creates a module from the compilation artifacts.
     /// The `serialize` saves the compilation artifacts along with the host
     /// fingerprint, which consists of target, compiler flags, and wasmtime
@@ -336,44 +430,117 @@ impl Module {
     /// for modifications or curruptions. All responsibily of signing and its
     /// verification falls on the embedder.
     pub fn deserialize(engine: &Engine, serialized: &[u8]) -> Result<Module> {
-        let expected_fingerprint = compiler_fingerprint(engine);
-
-        let (fingerprint, artifacts, types, index) = bincode_options()
-            .deserialize::<(u64, _, _, _)>(serialized)
+        let (fingerprint, types, serialized) = bincode_options()
+            .deserialize::<(u64, Vec<TypeTables>, _)>(serialized)
             .context("Deserialize compilation artifacts")?;
-        if fingerprint != expected_fingerprint {
+
+        if fingerprint != compiler_fingerprint(engine) {
             bail!("Incompatible compilation artifact");
         }
 
-        let modules = CompiledModule::from_artifacts_list(
-            artifacts,
-            engine.compiler().isa(),
-            &*engine.config().profiler,
-        )?;
+        let types = types.into_iter().map(Arc::new).collect::<Vec<_>>();
+        return mk(engine, &types, serialized);
 
-        let types = Arc::new(types);
-        Ok(Module {
-            engine: engine.clone(),
-            index,
-            data: Arc::new(ModuleData { modules, types }),
-        })
-    }
-
-    pub(crate) fn compiled_module(&self) -> &CompiledModule {
-        &self.data.modules[self.index]
-    }
-
-    pub(crate) fn submodule(&self, index: usize) -> Module {
-        assert!(index < self.data.modules.len());
-        Module {
-            engine: self.engine.clone(),
-            data: self.data.clone(),
-            index,
+        fn mk(
+            engine: &Engine,
+            types: &Vec<Arc<TypeTables>>,
+            module: ModuleSerialized<'_>,
+        ) -> Result<Module> {
+            let mut artifacts = CompiledModule::from_artifacts_list(
+                module
+                    .artifacts
+                    .into_iter()
+                    .map(|i| i.unwrap_owned())
+                    .collect(),
+                engine.compiler().isa(),
+                &*engine.config().profiler,
+            )?;
+            let inner = ModuleInner {
+                engine: engine.clone(),
+                types: types[module.type_tables].clone(),
+                module: artifacts.pop().unwrap(),
+                artifact_upvars: artifacts,
+                module_upvars: module
+                    .modules
+                    .into_iter()
+                    .map(|m| mk(engine, types, m))
+                    .collect::<Result<Vec<_>>>()?,
+            };
+            Ok(Module {
+                inner: Arc::new(inner),
+            })
         }
     }
 
+    /// Creates a submodule `Module` value from the specified parameters.
+    ///
+    /// This is used for creating submodules as part of module instantiation.
+    ///
+    /// * `artifact_index` - the index in `artifact_upvars` that we're creating
+    ///   a module for
+    /// * `artifact_upvars` - the mapping of indices of what artifact upvars are
+    ///   needed for the submodule. The length of this array is the length of
+    ///   the upvars array in the submodule to be created, and each element of
+    ///   this array is an index into this module's upvar array.
+    /// * `module_upvars` - similar to `artifact_upvars` this is a mapping of
+    ///   how to create the e`module_upvars` of the submodule being created.
+    ///   Each entry in this array is either an index into this module's own
+    ///   module upvars array or it's an index into `modules`, the list of
+    ///   modules so far for the instance where this submodule is being
+    ///   created.
+    /// * `modules` - array indexed by `module_upvars`.
+    ///
+    /// Note that the real meat of this happens in `ModuleEnvironment`
+    /// translation inside of `wasmtime_environ`. This just does the easy thing
+    /// of handling all the indices, over there is where the indices are
+    /// actually calculated and such.
+    pub(crate) fn create_submodule(
+        &self,
+        artifact_index: usize,
+        artifact_upvars: &[usize],
+        module_upvars: &[wasmtime_environ::ModuleUpvar],
+        modules: &PrimaryMap<ModuleIndex, Module>,
+    ) -> Module {
+        Module {
+            inner: Arc::new(ModuleInner {
+                types: self.types().clone(),
+                engine: self.engine().clone(),
+                module: self.inner.artifact_upvars[artifact_index].clone(),
+                artifact_upvars: artifact_upvars
+                    .iter()
+                    .map(|i| self.inner.artifact_upvars[*i].clone())
+                    .collect(),
+                module_upvars: module_upvars
+                    .iter()
+                    .map(|i| match *i {
+                        wasmtime_environ::ModuleUpvar::Inherit(i) => {
+                            self.inner.module_upvars[i].clone()
+                        }
+                        wasmtime_environ::ModuleUpvar::Local(i) => modules[i].clone(),
+                    })
+                    .collect(),
+            }),
+        }
+    }
+
+    pub(crate) fn compiled_module(&self) -> &CompiledModule {
+        &self.inner.module
+    }
+
+    pub(crate) fn env_module(&self) -> &wasmtime_environ::Module {
+        self.compiled_module().module()
+    }
+
     pub(crate) fn types(&self) -> &Arc<TypeTables> {
-        &self.data.types
+        &self.inner.types
+    }
+
+    /// Looks up the module upvar value at the `index` specified.
+    ///
+    /// Note that this panics if `index` is out of bounds since this should
+    /// only be called for valid indices as part of instantiation.
+    pub(crate) fn module_upvar(&self, index: usize) -> &Module {
+        &self.inner.module_upvars[index]
     }
 
     /// Returns identifier/name that this [`Module`] has. This name
@@ -583,7 +750,7 @@ impl Module {
 
     /// Returns the [`Engine`] that this [`Module`] was compiled by.
     pub fn engine(&self) -> &Engine {
-        &self.engine
+        &self.inner.engine
     }
 }
 
diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs
index 6b314f6ac1..10564d79dd 100644
--- a/crates/wasmtime/src/store.rs
+++ b/crates/wasmtime/src/store.rs
@@ -4,8 +4,9 @@ use crate::trampoline::StoreInstanceHandle;
 use crate::{Engine, Module};
 use anyhow::{bail, Result};
 use std::any::Any;
-use std::cell::RefCell;
+use std::cell::{Cell, RefCell};
 use std::collections::HashSet;
+use std::convert::TryFrom;
 use std::fmt;
 use std::hash::{Hash, Hasher};
 use std::rc::{Rc, Weak};
@@ -13,8 +14,8 @@ use std::sync::Arc;
 use wasmtime_environ::wasm;
 use wasmtime_jit::{CompiledModule, ModuleCode, TypeTables};
 use wasmtime_runtime::{
-    InstanceHandle, RuntimeMemoryCreator, SignalHandler, StackMapRegistry, TrapInfo, VMExternRef,
-    VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex,
+    InstanceHandle, RuntimeMemoryCreator, SignalHandler, StackMapRegistry, TrapInfo, VMContext,
+    VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex,
 };
 
 /// A `Store` is a collection of WebAssembly instances and host-defined items.
@@ -67,6 +68,15 @@ pub(crate) struct StoreInner {
     /// Set of all compiled modules that we're holding a strong reference to
     /// the module's code for. This includes JIT functions, trampolines, etc.
     modules: RefCell<HashSet<ArcModuleCode>>,
+
+    // Numbers of resources instantiated in this store.
+    instance_count: Cell<usize>,
+    memory_count: Cell<usize>,
+    table_count: Cell<usize>,
+
+    /// An adjustment to add to the fuel consumed value in `interrupts` above
+    /// to get the true amount of fuel consumed.
+    fuel_adj: Cell<i64>,
 }
 
 struct HostInfoKey(VMExternRef);
@@ -109,6 +119,10 @@ impl Store {
                 stack_map_registry: StackMapRegistry::default(),
                 frame_info: Default::default(),
                 modules: Default::default(),
+                instance_count: Default::default(),
+                memory_count: Default::default(),
+                table_count: Default::default(),
+                fuel_adj: Cell::new(0),
             }),
         }
     }
@@ -188,8 +202,9 @@ impl Store {
             None => return,
         };
         // Only register this module if it hasn't already been registered.
-        if !self.is_wasm_code(first_pc) {
-            self.inner.frame_info.borrow_mut().register(module);
+        let mut info = self.inner.frame_info.borrow_mut();
+        if !info.contains_pc(first_pc) {
+            info.register(module);
         }
     }
 
@@ -213,6 +228,43 @@ impl Store {
         }
     }
 
+    pub(crate) fn bump_resource_counts(&self, module: &Module) -> Result<()> {
+        let config = self.engine().config();
+
+        fn bump(slot: &Cell<usize>, max: usize, amt: usize, desc: &str) -> Result<()> {
+            let new = slot.get().saturating_add(amt);
+            if new > max {
+                bail!(
+                    "resource limit exceeded: {} count too high at {}",
+                    desc,
+                    new
+                );
+            }
+            slot.set(new);
+            Ok(())
+        }
+
+        let module = module.env_module();
+        let memories = module.memory_plans.len() - module.num_imported_memories;
+        let tables = module.table_plans.len() - module.num_imported_tables;
+
+        bump(
+            &self.inner.instance_count,
+            config.max_instances,
+            1,
+            "instance",
+        )?;
+        bump(
+            &self.inner.memory_count,
+            config.max_memories,
+            memories,
+            "memory",
+        )?;
+        bump(&self.inner.table_count, config.max_tables, tables, "table")?;
+
+        Ok(())
+    }
+
     pub(crate) unsafe fn add_instance(&self, handle: InstanceHandle) -> StoreInstanceHandle {
         self.inner.instances.borrow_mut().push(handle.clone());
         StoreInstanceHandle {
@@ -234,6 +286,10 @@ impl Store {
         }
     }
 
+    pub(crate) unsafe fn existing_vmctx(&self, cx: *mut VMContext) -> StoreInstanceHandle {
+        self.existing_instance_handle(InstanceHandle::from_vmctx(cx))
+    }
+
     pub(crate) fn weak(&self) -> Weak<StoreInner> {
         Rc::downgrade(&self.inner)
     }
@@ -377,6 +433,64 @@ impl Store {
             );
         }
     }
+
+    /// Returns the amount of fuel consumed by this store's execution so far.
+    ///
+    /// If fuel consumption is not enabled via
+    /// [`Config::consume_fuel`](crate::Config::consume_fuel) then this
+    /// function will return `None`. Also note that fuel, if enabled, must be
+    /// originally configured via [`Store::add_fuel`].
+    pub fn fuel_consumed(&self) -> Option<u64> {
+        if !self.engine().config().tunables.consume_fuel {
+            return None;
+        }
+        let consumed = unsafe { *self.inner.interrupts.fuel_consumed.get() };
+        Some(u64::try_from(self.inner.fuel_adj.get() + consumed).unwrap())
+    }
+
+    /// Adds fuel to this [`Store`] for wasm to consume while executing.
+    ///
+    /// For this method to work fuel consumption must be enabled via
+    /// [`Config::consume_fuel`](crate::Config::consume_fuel). By default a
+    /// [`Store`] starts with 0 fuel for wasm to execute with (meaning it will
+    /// immediately trap). This function must be called for the store to have
+    /// some fuel to allow WebAssembly to execute.
+    ///
+    /// Note that at this time when fuel is entirely consumed it will cause
+    /// wasm to trap. More usages of fuel are planned for the future.
+    ///
+    /// # Panics
+    ///
+    /// This function will panic if the store's [`Config`](crate::Config) did
+    /// not have fuel consumption enabled.
+    pub fn add_fuel(&self, fuel: u64) {
+        assert!(self.engine().config().tunables.consume_fuel);
+
+        // Fuel is stored as an i64, so we need to cast it. If the provided fuel
+        // value overflows that just assume that i64::max will suffice. Wasm
+        // execution isn't fast enough to burn through i64::max fuel in any
+        // reasonable amount of time anyway.
+        let fuel = i64::try_from(fuel).unwrap_or(i64::max_value());
+        let adj = self.inner.fuel_adj.get();
+        let consumed_ptr = unsafe { &mut *self.inner.interrupts.fuel_consumed.get() };
+
+        match (consumed_ptr.checked_sub(fuel), adj.checked_add(fuel)) {
+            // If we succesfully did arithmetic without overflowing then we can
+            // just update our fields.
+            (Some(consumed), Some(adj)) => {
+                self.inner.fuel_adj.set(adj);
+                *consumed_ptr = consumed;
+            }
+
+            // Otherwise something overflowed. Make sure that we preserve the
+            // amount of fuel that's already consumed, but otherwise assume that
+            // we were given infinite fuel.
+            _ => {
+                self.inner.fuel_adj.set(i64::max_value());
+                *consumed_ptr = (*consumed_ptr + adj) - i64::max_value();
+            }
+        }
+    }
 }
 
 unsafe impl TrapInfo for Store {
@@ -384,8 +498,8 @@ unsafe impl TrapInfo for Store {
         self
     }
 
-    fn is_wasm_code(&self, addr: usize) -> bool {
-        self.frame_info().borrow().contains_pc(addr)
+    fn is_wasm_trap(&self, addr: usize) -> bool {
+        self.frame_info().borrow().lookup_trap_info(addr).is_some()
     }
 
     fn custom_signal_handler(&self, call: &dyn Fn(&SignalHandler) -> bool) -> bool {
@@ -398,6 +512,23 @@ unsafe impl TrapInfo for Store {
     fn max_wasm_stack(&self) -> usize {
         self.engine().config().max_wasm_stack
     }
+
+    fn out_of_gas(&self) {
+        #[derive(Debug)]
+        struct OutOfGas;
+
+        impl fmt::Display for OutOfGas {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                f.write_str("all fuel consumed by WebAssembly")
+            }
+        }
+
+        impl std::error::Error for OutOfGas {}
+
+        unsafe {
+            wasmtime_runtime::raise_lib_trap(wasmtime_runtime::Trap::User(Box::new(OutOfGas)))
+        }
+    }
 }
 
 impl Default for Store {
@@ -431,6 +562,13 @@ pub struct InterruptHandle {
     interrupts: Arc<VMInterrupts>,
 }
 
+// The `VMInterrupts` type is a pod-type with no destructor, and we only access
+// `interrupts` from other threads, so add in these trait impls which are
+// otherwise not available due to the `fuel_consumed` variable in
+// `VMInterrupts`.
+unsafe impl Send for InterruptHandle {}
+unsafe impl Sync for InterruptHandle {}
+
 impl InterruptHandle {
     /// Flags that execution within this handle's original [`Store`] should be
     /// interrupted.
diff --git a/crates/wasmtime/src/trampoline/global.rs b/crates/wasmtime/src/trampoline/global.rs
index 00a91bdb91..bb20bb1569 100644
--- a/crates/wasmtime/src/trampoline/global.rs
+++ b/crates/wasmtime/src/trampoline/global.rs
@@ -48,7 +48,7 @@ pub fn create_global(store: &Store, gt: &GlobalType, val: Val) -> Result<StoreIn
                 module
                     .initializers
                     .push(wasmtime_environ::Initializer::Import {
-                        module: "".into(),
+                        name: "".into(),
                         field: None,
                         index: wasm::EntityIndex::Function(func_index),
                     });
@@ -80,7 +80,7 @@ pub fn create_global(store: &Store, gt: &GlobalType, val: Val) -> Result<StoreIn
     )?;
 
     if let Some(x) = externref_init {
-        match handle.lookup("").unwrap() {
+        match handle.lookup_by_declaration(&wasm::EntityIndex::Global(global_id)) {
             wasmtime_runtime::Export::Global(g) => unsafe {
                 *(*g.definition).as_externref_mut() = Some(x.inner);
             },
diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs
index 7993c8e87c..0bfb2bfff9 100644
--- a/crates/wasmtime/src/trampoline/memory.rs
+++ b/crates/wasmtime/src/trampoline/memory.rs
@@ -1,5 +1,5 @@
 use super::create_handle::create_handle;
-use crate::externals::{LinearMemory, MemoryCreator};
+use crate::memory::{LinearMemory, MemoryCreator};
 use crate::trampoline::StoreInstanceHandle;
 use crate::Store;
 use crate::{Limits, MemoryType};
diff --git a/crates/wasmtime/src/trampoline/mod.rs b/crates/wasmtime/src/trampoline/mod.rs
index e8c836b2ec..4772d7481a 100644
--- a/crates/wasmtime/src/trampoline/mod.rs
+++ b/crates/wasmtime/src/trampoline/mod.rs
@@ -16,6 +16,7 @@ use crate::{FuncType, GlobalType, MemoryType, Store, TableType, Trap, Val};
 use anyhow::Result;
 use std::any::Any;
 use std::ops::Deref;
+use wasmtime_environ::wasm;
 use wasmtime_runtime::{InstanceHandle, VMContext, VMFunctionBody, VMTrampoline};
 
 /// A wrapper around `wasmtime_runtime::InstanceHandle` which pairs it with the
@@ -55,7 +56,8 @@ pub fn generate_func_export(
     VMTrampoline,
 )> {
     let (instance, trampoline) = create_handle_with_function(ft, func, store)?;
-    match instance.lookup("").expect("trampoline export") {
+    let idx = wasm::EntityIndex::Function(wasm::FuncIndex::from_u32(0));
+    match instance.lookup_by_declaration(&idx) {
         wasmtime_runtime::Export::Function(f) => Ok((instance, f, trampoline)),
         _ => unreachable!(),
     }
@@ -72,7 +74,8 @@ pub unsafe fn generate_raw_func_export(
     state: Box<dyn Any>,
 ) -> Result<(StoreInstanceHandle, wasmtime_runtime::ExportFunction)> {
     let instance = func::create_handle_with_raw_function(ft, func, trampoline, store, state)?;
-    match instance.lookup("").expect("trampoline export") {
+    let idx = wasm::EntityIndex::Function(wasm::FuncIndex::from_u32(0));
+    match instance.lookup_by_declaration(&idx) {
         wasmtime_runtime::Export::Function(f) => Ok((instance, f)),
         _ => unreachable!(),
     }
@@ -84,7 +87,8 @@ pub fn generate_global_export(
     val: Val,
 ) -> Result<(StoreInstanceHandle, wasmtime_runtime::ExportGlobal)> {
     let instance = create_global(store, gt, val)?;
-    match instance.lookup("").expect("global export") {
+    let idx = wasm::EntityIndex::Global(wasm::GlobalIndex::from_u32(0));
+    match instance.lookup_by_declaration(&idx) {
         wasmtime_runtime::Export::Global(g) => Ok((instance, g)),
         _ => unreachable!(),
     }
@@ -95,7 +99,8 @@ pub fn generate_memory_export(
     m: &MemoryType,
 ) -> Result<(StoreInstanceHandle, wasmtime_runtime::ExportMemory)> {
     let instance = create_handle_with_memory(store, m)?;
-    match instance.lookup("").expect("memory export") {
+    let idx = wasm::EntityIndex::Memory(wasm::MemoryIndex::from_u32(0));
+    match instance.lookup_by_declaration(&idx) {
         wasmtime_runtime::Export::Memory(m) => Ok((instance, m)),
         _ => unreachable!(),
     }
@@ -106,7 +111,8 @@ pub fn generate_table_export(
     t: &TableType,
 ) -> Result<(StoreInstanceHandle, wasmtime_runtime::ExportTable)> {
     let instance = create_handle_with_table(store, t)?;
-    match instance.lookup("").expect("table export") {
+    let idx = wasm::EntityIndex::Table(wasm::TableIndex::from_u32(0));
+    match instance.lookup_by_declaration(&idx) {
         wasmtime_runtime::Export::Table(t) => Ok((instance, t)),
         _ => unreachable!(),
     }
diff --git a/crates/wasmtime/src/trampoline/table.rs b/crates/wasmtime/src/trampoline/table.rs
index 7c451f5cb0..c151fca51d 100644
--- a/crates/wasmtime/src/trampoline/table.rs
+++ b/crates/wasmtime/src/trampoline/table.rs
@@ -23,6 +23,7 @@ pub fn create_handle_with_table(store: &Store, table: &TableType) -> Result<Stor
 
     let table_plan = wasmtime_environ::TablePlan::for_table(table, &tunable);
     let table_id = module.table_plans.push(table_plan);
+    // TODO: can this `exports.insert` get removed?
     module
         .exports
         .insert(String::new(), wasm::EntityIndex::Table(table_id));
diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs
index 3a7076a764..5a090d794c 100644
--- a/crates/wasmtime/src/types.rs
+++ b/crates/wasmtime/src/types.rs
@@ -478,9 +478,9 @@ impl ModuleType {
 
     /// Returns the list of imports associated with this module type.
     pub fn imports(&self) -> impl ExactSizeIterator<Item = ImportType<'_>> {
-        self.imports.iter().map(|(module, name, ty)| ImportType {
-            module,
-            name: name.as_deref(),
+        self.imports.iter().map(|(name, field, ty)| ImportType {
+            module: name,
+            name: field.as_deref(),
             ty: EntityOrExtern::Extern(ty),
         })
     }
@@ -506,13 +506,7 @@ impl ModuleType {
             imports: ty
                 .imports
                 .iter()
-                .map(|(m, name, ty)| {
-                    (
-                        m.to_string(),
-                        name.as_ref().map(|n| n.to_string()),
-                        ExternType::from_wasmtime(types, ty),
-                    )
-                })
+                .map(|(m, ty)| (m.to_string(), None, ExternType::from_wasmtime(types, ty)))
                 .collect(),
         }
     }
@@ -615,8 +609,9 @@ impl<'module> ImportType<'module> {
     /// Returns the field name of the module that this import is expected to
     /// come from.
     ///
-    /// Note that the name can be `None` for the module linking proposal. If the
-    /// module linking proposal is not enabled it's safe to unwrap this.
+    /// Note that this is optional due to the module linking proposal. If the
+    /// module linking proposal is enabled this is always `None`, otherwise this
+    /// is always `Some`.
     pub fn name(&self) -> Option<&'module str> {
         self.name
     }
@@ -683,6 +678,17 @@ impl<'module> ExportType<'module> {
             EntityOrExtern::Extern(e) => (*e).clone(),
         }
     }
+
+    pub(crate) fn as_import<'a>(&self, module: &'a str) -> ImportType<'a>
+    where
+        'module: 'a,
+    {
+        ImportType {
+            module,
+            name: Some(self.name),
+            ty: self.ty.clone(),
+        }
+    }
 }
 
 impl<'module> fmt::Debug for ExportType<'module> {
diff --git a/crates/wasmtime/src/types/matching.rs b/crates/wasmtime/src/types/matching.rs
index 018ab35fd9..a767bbb7e1 100644
--- a/crates/wasmtime/src/types/matching.rs
+++ b/crates/wasmtime/src/types/matching.rs
@@ -1,5 +1,5 @@
-use crate::Store;
-use std::sync::Arc;
+use crate::{Extern, Store};
+use anyhow::{bail, Context, Result};
 use wasmtime_environ::wasm::{
     EntityType, Global, InstanceTypeIndex, Memory, ModuleTypeIndex, SignatureIndex, Table,
 };
@@ -11,22 +11,27 @@ pub struct MatchCx<'a> {
 }
 
 impl MatchCx<'_> {
-    pub fn global(&self, expected: &Global, actual: &crate::Global) -> bool {
+    pub fn global(&self, expected: &Global, actual: &crate::Global) -> Result<()> {
         self.global_ty(expected, actual.wasmtime_ty())
     }
 
-    fn global_ty(&self, expected: &Global, actual: &Global) -> bool {
-        expected.ty == actual.ty
+    fn global_ty(&self, expected: &Global, actual: &Global) -> Result<()> {
+        if expected.ty == actual.ty
             && expected.wasm_ty == actual.wasm_ty
             && expected.mutability == actual.mutability
+        {
+            Ok(())
+        } else {
+            bail!("global types incompatible")
+        }
     }
 
-    pub fn table(&self, expected: &Table, actual: &crate::Table) -> bool {
+    pub fn table(&self, expected: &Table, actual: &crate::Table) -> Result<()> {
         self.table_ty(expected, actual.wasmtime_ty())
     }
 
-    fn table_ty(&self, expected: &Table, actual: &Table) -> bool {
-        expected.wasm_ty == actual.wasm_ty
+    fn table_ty(&self, expected: &Table, actual: &Table) -> Result<()> {
+        if expected.wasm_ty == actual.wasm_ty
             && expected.ty == actual.ty
             && expected.minimum <= actual.minimum
             && match expected.maximum {
@@ -36,14 +41,19 @@ impl MatchCx<'_> {
                 },
                 None => true,
             }
+        {
+            Ok(())
+        } else {
+            bail!("table types incompatible")
+        }
     }
 
-    pub fn memory(&self, expected: &Memory, actual: &crate::Memory) -> bool {
+    pub fn memory(&self, expected: &Memory, actual: &crate::Memory) -> Result<()> {
         self.memory_ty(expected, actual.wasmtime_ty())
     }
 
-    fn memory_ty(&self, expected: &Memory, actual: &Memory) -> bool {
-        expected.shared == actual.shared
+    fn memory_ty(&self, expected: &Memory, actual: &Memory) -> Result<()> {
+        if expected.shared == actual.shared
             && expected.minimum <= actual.minimum
             && match expected.maximum {
                 Some(expected) => match actual.maximum {
@@ -52,10 +62,15 @@ impl MatchCx<'_> {
                 },
                 None => true,
             }
+        {
+            Ok(())
+        } else {
+            bail!("memory types incompatible")
+        }
     }
 
-    pub fn func(&self, expected: SignatureIndex, actual: &crate::Func) -> bool {
-        match self
+    pub fn func(&self, expected: SignatureIndex, actual: &crate::Func) -> Result<()> {
+        let matches = match self
             .store
             .signatures()
             .borrow()
@@ -65,31 +80,50 @@ impl MatchCx<'_> {
             // If our expected signature isn't registered, then there's no way
             // that `actual` can match it.
             None => false,
+        };
+        if matches {
+            Ok(())
+        } else {
+            bail!("function types incompatible")
         }
     }
 
-    pub fn instance(&self, expected: InstanceTypeIndex, actual: &crate::Instance) -> bool {
-        let module = actual.handle.module();
-        self.exports_match(
-            expected,
-            actual
-                .handle
-                .host_state()
-                .downcast_ref::<Arc<TypeTables>>()
-                .unwrap(),
-            |name| module.exports.get(name).map(|idx| module.type_of(*idx)),
-        )
+    pub fn instance(&self, expected: InstanceTypeIndex, actual: &crate::Instance) -> Result<()> {
+        for (name, expected) in self.types.instance_signatures[expected].exports.iter() {
+            match actual.items.get(name) {
+                Some(item) => {
+                    let item = unsafe { Extern::from_wasmtime_export(item, self.store) };
+                    self.extern_(expected, &item)
+                        .with_context(|| format!("instance export {:?} incompatible", name))?;
+                }
+                None => bail!("instance type missing export {:?}", name),
+            }
+        }
+        Ok(())
     }
 
     /// Validates that the type signature of `actual` matches the `expected`
     /// module type signature.
-    pub fn module(&self, expected: ModuleTypeIndex, actual: &crate::Module) -> bool {
+    pub fn module(&self, expected: ModuleTypeIndex, actual: &crate::Module) -> Result<()> {
+        // This should only ever be invoked with module linking, and this is an
+        // early check that our `field` assertion below should always work as
+        // well.
+        assert!(self.store.engine().config().features.module_linking);
+
         let expected_sig = &self.types.module_signatures[expected];
         let module = actual.compiled_module().module();
-        self.imports_match(expected, actual.types(), module.imports())
-            && self.exports_match(expected_sig.exports, actual.types(), |name| {
-                module.exports.get(name).map(|idx| module.type_of(*idx))
-            })
+        self.imports_match(
+            expected,
+            actual.types(),
+            module.imports().map(|(name, field, ty)| {
+                assert!(field.is_none()); // should be true if module linking is enabled
+                (name, ty)
+            }),
+        )?;
+        self.exports_match(expected_sig.exports, actual.types(), |name| {
+            module.exports.get(name).map(|idx| module.type_of(*idx))
+        })?;
+        Ok(())
     }
 
     /// Validates that the `actual_imports` list of module imports matches the
@@ -100,19 +134,25 @@ impl MatchCx<'_> {
         &self,
         expected: ModuleTypeIndex,
         actual_types: &TypeTables,
-        mut actual_imports: impl Iterator<Item = (&'a str, Option<&'a str>, EntityType)>,
-    ) -> bool {
+        actual_imports: impl Iterator<Item = (&'a str, EntityType)>,
+    ) -> Result<()> {
+        // Imports match if all of the actual imports are satisfied by the
+        // expected set of imports. Note that we're reversing the order of the
+        // subtytpe matching here too.
         let expected_sig = &self.types.module_signatures[expected];
-        for (_, _, expected) in expected_sig.imports.iter() {
-            let (_, _, ty) = match actual_imports.next() {
-                Some(e) => e,
-                None => return false,
+        for (name, actual_ty) in actual_imports {
+            let expected_ty = match expected_sig.imports.get(name) {
+                Some(ty) => ty,
+                None => bail!("expected type doesn't import {:?}", name),
             };
-            if !self.extern_ty_matches(expected, &ty, actual_types) {
-                return false;
+            MatchCx {
+                types: actual_types,
+                store: self.store,
             }
+            .extern_ty_matches(&actual_ty, expected_ty, self.types)
+            .with_context(|| format!("module import {:?} incompatible", name))?;
         }
-        actual_imports.next().is_none()
+        Ok(())
     }
 
     /// Validates that all exports in `expected` are defined by `lookup` within
@@ -122,16 +162,19 @@ impl MatchCx<'_> {
         expected: InstanceTypeIndex,
         actual_types: &TypeTables,
         lookup: impl Fn(&str) -> Option<EntityType>,
-    ) -> bool {
+    ) -> Result<()> {
         // The `expected` type must be a subset of `actual`, meaning that all
         // names in `expected` must be present in `actual`. Note that we do
         // name-based lookup here instead of index-based lookup.
-        self.types.instance_signatures[expected].exports.iter().all(
-            |(name, expected)| match lookup(name) {
-                Some(ty) => self.extern_ty_matches(expected, &ty, actual_types),
-                None => false,
-            },
-        )
+        for (name, expected) in self.types.instance_signatures[expected].exports.iter() {
+            match lookup(name) {
+                Some(ty) => self
+                    .extern_ty_matches(expected, &ty, actual_types)
+                    .with_context(|| format!("export {:?} incompatible", name))?,
+                None => bail!("failed to find export {:?}", name),
+            }
+        }
+        Ok(())
     }
 
     /// Validates that the `expected` entity matches the `actual_ty` defined
@@ -141,34 +184,49 @@ impl MatchCx<'_> {
         expected: &EntityType,
         actual_ty: &EntityType,
         actual_types: &TypeTables,
-    ) -> bool {
+    ) -> Result<()> {
+        let actual_desc = match actual_ty {
+            EntityType::Global(_) => "global",
+            EntityType::Module(_) => "module",
+            EntityType::Memory(_) => "memory",
+            EntityType::Event(_) => "event",
+            EntityType::Instance(_) => "instance",
+            EntityType::Table(_) => "table",
+            EntityType::Function(_) => "function",
+        };
         match expected {
             EntityType::Global(expected) => match actual_ty {
                 EntityType::Global(actual) => self.global_ty(expected, actual),
-                _ => false,
+                _ => bail!("expected global, but found {}", actual_desc),
             },
             EntityType::Table(expected) => match actual_ty {
                 EntityType::Table(actual) => self.table_ty(expected, actual),
-                _ => false,
+                _ => bail!("expected table, but found {}", actual_desc),
             },
             EntityType::Memory(expected) => match actual_ty {
                 EntityType::Memory(actual) => self.memory_ty(expected, actual),
-                _ => false,
+                _ => bail!("expected memory, but found {}", actual_desc),
             },
             EntityType::Function(expected) => match *actual_ty {
                 EntityType::Function(actual) => {
-                    self.types.wasm_signatures[*expected] == actual_types.wasm_signatures[actual]
+                    if self.types.wasm_signatures[*expected] == actual_types.wasm_signatures[actual]
+                    {
+                        Ok(())
+                    } else {
+                        bail!("function types incompatible")
+                    }
                 }
-                _ => false,
+                _ => bail!("expected function, but found {}", actual_desc),
             },
             EntityType::Instance(expected) => match actual_ty {
                 EntityType::Instance(actual) => {
                     let sig = &actual_types.instance_signatures[*actual];
                     self.exports_match(*expected, actual_types, |name| {
                         sig.exports.get(name).cloned()
-                    })
+                    })?;
+                    Ok(())
                 }
-                _ => false,
+                _ => bail!("expected instance, but found {}", actual_desc),
             },
             EntityType::Module(expected) => match actual_ty {
                 EntityType::Module(actual) => {
@@ -180,14 +238,48 @@ impl MatchCx<'_> {
                     self.imports_match(
                         *expected,
                         actual_types,
-                        actual_module_sig.imports.iter().map(|(module, field, ty)| {
-                            (module.as_str(), field.as_deref(), ty.clone())
-                        }),
-                    ) && self.exports_match(expected_module_sig.exports, actual_types, |name| {
+                        actual_module_sig
+                            .imports
+                            .iter()
+                            .map(|(module, ty)| (module.as_str(), ty.clone())),
+                    )?;
+                    self.exports_match(expected_module_sig.exports, actual_types, |name| {
                         actual_instance_sig.exports.get(name).cloned()
-                    })
+                    })?;
+                    Ok(())
                 }
-                _ => false,
+                _ => bail!("expected module, but found {}", actual_desc),
+            },
+            EntityType::Event(_) => unimplemented!(),
+        }
+    }
+
+    /// Validates that the `expected` type matches the type of `actual`
+    pub fn extern_(&self, expected: &EntityType, actual: &Extern) -> Result<()> {
+        match expected {
+            EntityType::Global(expected) => match actual {
+                Extern::Global(actual) => self.global(expected, actual),
+                _ => bail!("expected global, but found {}", actual.desc()),
+            },
+            EntityType::Table(expected) => match actual {
+                Extern::Table(actual) => self.table(expected, actual),
+                _ => bail!("expected table, but found {}", actual.desc()),
+            },
+            EntityType::Memory(expected) => match actual {
+                Extern::Memory(actual) => self.memory(expected, actual),
+                _ => bail!("expected memory, but found {}", actual.desc()),
+            },
+            EntityType::Function(expected) => match actual {
+                Extern::Func(actual) => self.func(*expected, actual),
+                _ => bail!("expected func, but found {}", actual.desc()),
+            },
+            EntityType::Instance(expected) => match actual {
+                Extern::Instance(actual) => self.instance(*expected, actual),
+                _ => bail!("expected instance, but found {}", actual.desc()),
+            },
+            EntityType::Module(expected) => match actual {
+                Extern::Module(actual) => self.module(*expected, actual),
+                _ => bail!("expected module, but found {}", actual.desc()),
             },
             EntityType::Event(_) => unimplemented!(),
         }
diff --git a/crates/wast/Cargo.toml b/crates/wast/Cargo.toml
index bbe1277f4c..8c1dea6331 100644
--- a/crates/wast/Cargo.toml
+++ b/crates/wast/Cargo.toml
@@ -13,7 +13,7 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0.19"
 wasmtime = { path = "../wasmtime", version = "0.22.0", default-features = false }
-wast = "29.0.0"
+wast = "32.0.0"
 
 [badges]
 maintenance = { status = "actively-developed" }
diff --git a/deny.toml b/deny.toml
index 37e5a3a0f8..5fbdd14a0e 100644
--- a/deny.toml
+++ b/deny.toml
@@ -42,4 +42,8 @@ skip = [
     { name = "humantime" }, # caused by env_logger
     { name = "getrandom" }, # rand not updates to 0.2 yet
     { name = "wast" }, # old one pulled in by witx
+    { name = "rand" }, # 0.7 pulled in by transitive deps
+    { name = "rand_core" }, # 0.7 pulled in by transitive deps
+    { name = "rand_chacha" }, # 0.7 pulled in by transitive deps
+    { name = "itertools" }, # 0.9 pulled in by zstd-sys
 ]
diff --git a/examples/externref.c b/examples/externref.c
index 5828d17665..725e2e0e2f 100644
--- a/examples/externref.c
+++ b/examples/externref.c
@@ -75,7 +75,8 @@ int main() {
   printf("Instantiating module...\n");
   wasm_trap_t *trap = NULL;
   wasm_instance_t *instance = NULL;
-  error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap);
+  wasm_extern_vec_t imports = WASM_EMPTY_VEC;
+  error = wasmtime_instance_new(store, module, &imports, &instance, &trap);
   if (instance == NULL)
     exit_with_error("failed to instantiate", error, trap);
 
@@ -139,10 +140,11 @@ int main() {
   assert(func != NULL);
 
   // And call it!
-  wasm_val_t args[1];
-  wasm_val_copy(&args[0], &externref);
+  wasm_val_t args[1] = { externref };
   wasm_val_t results[1];
-  error = wasmtime_func_call(func, args, 1, results, 1, &trap);
+  wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args);
+  wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results);
+  error = wasmtime_func_call(func, &args_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call function", error, trap);
 
@@ -161,7 +163,6 @@ int main() {
   ret = 0;
 
   wasm_val_delete(&results[0]);
-  wasm_val_delete(&args[0]);
   wasm_val_delete(&global_val);
   wasm_val_delete(&elem);
   wasm_extern_vec_delete(&externs);
diff --git a/examples/fib-debug/main.c b/examples/fib-debug/main.c
index a4e22dee3c..722ddb01d9 100644
--- a/examples/fib-debug/main.c
+++ b/examples/fib-debug/main.c
@@ -71,7 +71,8 @@ int main(int argc, const char* argv[]) {
   printf("Instantiating module...\n");
   wasm_instance_t* instance = NULL;
   wasm_trap_t *trap = NULL;
-  error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap);
+  wasm_extern_vec_t imports = WASM_EMPTY_VEC;
+  error = wasmtime_instance_new(store, module, &imports, &instance, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to instantiate", error, trap);
   wasm_module_delete(module);
@@ -95,9 +96,11 @@ int main(int argc, const char* argv[]) {
 
   // Call.
   printf("Calling fib...\n");
-  wasm_val_t params[1] = { {.kind = WASM_I32, .of = {.i32 = 6}} };
+  wasm_val_t params[1] = { WASM_I32_VAL(6) };
   wasm_val_t results[1];
-  error = wasmtime_func_call(run_func, params, 1, results, 1, &trap);
+  wasm_val_vec_t params_vec = WASM_ARRAY_VEC(params);
+  wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results);
+  error = wasmtime_func_call(run_func, &params_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call function", error, trap);
 
diff --git a/examples/gcd.c b/examples/gcd.c
index 76ca77faf7..d811458bb4 100644
--- a/examples/gcd.c
+++ b/examples/gcd.c
@@ -65,7 +65,8 @@ int main() {
   wasm_byte_vec_delete(&wasm);
   wasm_trap_t *trap = NULL;
   wasm_instance_t *instance = NULL;
-  error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap);
+  wasm_extern_vec_t imports = WASM_EMPTY_VEC;
+  error = wasmtime_instance_new(store, module, &imports, &instance, &trap);
   if (instance == NULL)
     exit_with_error("failed to instantiate", error, trap);
 
@@ -79,13 +80,11 @@ int main() {
   // And call it!
   int a = 6;
   int b = 27;
-  wasm_val_t params[2];
+  wasm_val_t params[2] = { WASM_I32_VAL(a), WASM_I32_VAL(b) };
   wasm_val_t results[1];
-  params[0].kind = WASM_I32;
-  params[0].of.i32 = a;
-  params[1].kind = WASM_I32;
-  params[1].of.i32 = b;
-  error = wasmtime_func_call(gcd, params, 2, results, 1, &trap);
+  wasm_val_vec_t params_vec = WASM_ARRAY_VEC(params);
+  wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results);
+  error = wasmtime_func_call(gcd, &params_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call gcd", error, trap);
   assert(results[0].kind == WASM_I32);
diff --git a/examples/hello.c b/examples/hello.c
index a14ef18aca..7c14612945 100644
--- a/examples/hello.c
+++ b/examples/hello.c
@@ -26,7 +26,7 @@ to tweak the `-lpthread` and such annotations as well as the name of the
 
 static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap);
 
-static wasm_trap_t* hello_callback(const wasm_val_t args[], wasm_val_t results[]) {
+static wasm_trap_t* hello_callback(const wasm_val_vec_t* args, wasm_val_vec_t* results) {
   printf("Calling back...\n");
   printf("> Hello World!\n");
   return NULL;
@@ -86,8 +86,9 @@ int main() {
   printf("Instantiating module...\n");
   wasm_trap_t *trap = NULL;
   wasm_instance_t *instance = NULL;
-  const wasm_extern_t *imports[] = { wasm_func_as_extern(hello) };
-  error = wasmtime_instance_new(store, module, imports, 1, &instance, &trap);
+  wasm_extern_t* imports[] = { wasm_func_as_extern(hello) };
+  wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports);
+  error = wasmtime_instance_new(store, module, &imports_vec, &instance, &trap);
   if (instance == NULL)
     exit_with_error("failed to instantiate", error, trap);
 
@@ -101,7 +102,9 @@ int main() {
 
   // And call it!
   printf("Calling export...\n");
-  error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap);
+  wasm_val_vec_t args_vec = WASM_EMPTY_VEC;
+  wasm_val_vec_t results_vec = WASM_EMPTY_VEC;
+  error = wasmtime_func_call(run, &args_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call function", error, trap);
 
diff --git a/examples/hello.cc b/examples/hello.cc
index c83be42576..f44b49c1fc 100644
--- a/examples/hello.cc
+++ b/examples/hello.cc
@@ -26,7 +26,7 @@ to tweak the `-lpthread` and such annotations as well as the name of the
 
 static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap);
 
-static wasm_trap_t* hello_callback(const wasm_val_t args[], wasm_val_t results[]) {
+static wasm_trap_t* hello_callback(const wasm_val_vec_t* args, wasm_val_vec_t* results) {
   printf("Calling back...\n");
   printf("> Hello World!\n");
   return NULL;
@@ -86,8 +86,9 @@ int main() {
   printf("Instantiating module...\n");
   wasm_trap_t *trap = NULL;
   wasm_instance_t *instance = NULL;
-  const wasm_extern_t *imports[] = { wasm_func_as_extern(hello) };
-  error = wasmtime_instance_new(store, module, imports, 1, &instance, &trap);
+  wasm_extern_t* imports[] = { wasm_func_as_extern(hello) };
+  wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports);
+  error = wasmtime_instance_new(store, module, &imports_vec, &instance, &trap);
   if (instance == NULL)
     exit_with_error("failed to instantiate", error, trap);
 
@@ -101,7 +102,9 @@ int main() {
 
   // And call it!
   printf("Calling export...\n");
-  error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap);
+  wasm_val_vec_t args_vec = WASM_EMPTY_VEC;
+  wasm_val_vec_t results_vec = WASM_EMPTY_VEC;
+  error = wasmtime_func_call(run, &args_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call function", error, trap);
 
diff --git a/examples/interrupt.c b/examples/interrupt.c
index d8ccd9a8f4..e6540a2d8d 100644
--- a/examples/interrupt.c
+++ b/examples/interrupt.c
@@ -42,6 +42,7 @@ static void* helper(void *_handle) {
   printf("Sending an interrupt\n");
   wasmtime_interrupt_handle_interrupt(handle);
   wasmtime_interrupt_handle_delete(handle);
+  return 0;
 }
 
 static void spawn_interrupt(wasmtime_interrupt_handle_t *handle) {
@@ -89,11 +90,12 @@ int main() {
   wasm_module_t *module = NULL;
   wasm_trap_t *trap = NULL;
   wasm_instance_t *instance = NULL;
+  wasm_extern_vec_t imports = WASM_EMPTY_VEC;
   error = wasmtime_module_new(engine, &wasm, &module);
   wasm_byte_vec_delete(&wasm);
   if (error != NULL)
     exit_with_error("failed to compile module", error, NULL);
-  error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap);
+  error = wasmtime_instance_new(store, module, &imports, &instance, &trap);
   if (instance == NULL)
     exit_with_error("failed to instantiate", error, trap);
 
@@ -109,7 +111,9 @@ int main() {
 
   // And call it!
   printf("Entering infinite loop...\n");
-  error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap);
+  wasm_val_vec_t args_vec = WASM_EMPTY_VEC;
+  wasm_val_vec_t results_vec = WASM_EMPTY_VEC;
+  error = wasmtime_func_call(run, &args_vec, &results_vec, &trap);
   assert(error == NULL);
   assert(trap != NULL);
   printf("Got a trap!...\n");
diff --git a/examples/linking.c b/examples/linking.c
index f4c53b1f5a..d9bc0f93c1 100644
--- a/examples/linking.c
+++ b/examples/linking.c
@@ -100,7 +100,9 @@ int main() {
   assert(linking1_externs.size == 1);
   wasm_func_t *run = wasm_extern_as_func(linking1_externs.data[0]);
   assert(run != NULL);
-  error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap);
+  wasm_val_vec_t args_vec = WASM_EMPTY_VEC;
+  wasm_val_vec_t results_vec = WASM_EMPTY_VEC;
+  error = wasmtime_func_call(run, &args_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call run", error, trap);
 
diff --git a/examples/memory.c b/examples/memory.c
index a8391e8c20..24e26dbd4c 100644
--- a/examples/memory.c
+++ b/examples/memory.c
@@ -54,10 +54,11 @@ void check(bool success) {
   }
 }
 
-void check_call(wasm_func_t* func, wasm_val_t args[], size_t num_args, int32_t expected) {
+void check_call(wasm_func_t* func, const wasm_val_vec_t* args_vec, int32_t expected) {
   wasm_val_t results[1];
+  wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results);
   wasm_trap_t *trap = NULL;
-  wasmtime_error_t *error = wasmtime_func_call(func, args, num_args, results, 1, &trap);
+  wasmtime_error_t *error = wasmtime_func_call(func, args_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call function", error, trap);
   if (results[0].of.i32 != expected) {
@@ -67,42 +68,44 @@ void check_call(wasm_func_t* func, wasm_val_t args[], size_t num_args, int32_t e
 }
 
 void check_call0(wasm_func_t* func, int32_t expected) {
-  check_call(func, NULL, 0, expected);
+  wasm_val_vec_t args_vec = WASM_EMPTY_VEC;
+  check_call(func, &args_vec, expected);
 }
 
 void check_call1(wasm_func_t* func, int32_t arg, int32_t expected) {
-  wasm_val_t args[] = { {.kind = WASM_I32, .of = {.i32 = arg}} };
-  check_call(func, args, 1, expected);
+  wasm_val_t args[] = { WASM_I32_VAL(arg) };
+  wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args);
+  check_call(func, &args_vec, expected);
 }
 
 void check_call2(wasm_func_t* func, int32_t arg1, int32_t arg2, int32_t expected) {
-  wasm_val_t args[2] = {
-    {.kind = WASM_I32, .of = {.i32 = arg1}},
-    {.kind = WASM_I32, .of = {.i32 = arg2}}
-  };
-  check_call(func, args, 2, expected);
+  wasm_val_t args[] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) };
+  wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args);
+  check_call(func, &args_vec, expected);
 }
 
-void check_ok(wasm_func_t* func, wasm_val_t args[], size_t num_args) {
+void check_ok(wasm_func_t* func, const wasm_val_vec_t* args_vec) {
   wasm_trap_t *trap = NULL;
-  wasmtime_error_t *error = wasmtime_func_call(func, args, num_args, NULL, 0, &trap);
+  wasm_val_vec_t results_vec = WASM_EMPTY_VEC;
+  wasmtime_error_t *error = wasmtime_func_call(func, args_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call function", error, trap);
 }
 
 void check_ok2(wasm_func_t* func, int32_t arg1, int32_t arg2) {
-  wasm_val_t args[2] = {
-    {.kind = WASM_I32, .of = {.i32 = arg1}},
-    {.kind = WASM_I32, .of = {.i32 = arg2}}
-  };
-  check_ok(func, args, 2);
+  wasm_val_t args[] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) };
+  wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args);
+  check_ok(func, &args_vec);
 }
 
-void check_trap(wasm_func_t* func, wasm_val_t args[], size_t num_args, size_t num_results) {
-  wasm_val_t results[1];
+void check_trap(wasm_func_t* func, const wasm_val_vec_t* args_vec, size_t num_results) {
   assert(num_results <= 1);
+  wasm_val_t results[1];
+  wasm_val_vec_t results_vec;
+  results_vec.data = results;
+  results_vec.size = num_results;
   wasm_trap_t *trap = NULL;
-  wasmtime_error_t *error = wasmtime_func_call(func, args, num_args, results, num_results, &trap);
+  wasmtime_error_t *error = wasmtime_func_call(func, args_vec, &results_vec, &trap);
   if (error != NULL)
     exit_with_error("failed to call function", error, NULL);
   if (trap == NULL) {
@@ -113,16 +116,15 @@ void check_trap(wasm_func_t* func, wasm_val_t args[], size_t num_args, size_t nu
 }
 
 void check_trap1(wasm_func_t* func, int32_t arg) {
-  wasm_val_t args[1] = { {.kind = WASM_I32, .of = {.i32 = arg}} };
-  check_trap(func, args, 1, 1);
+  wasm_val_t args[] = { WASM_I32_VAL(arg) };
+  wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args);
+  check_trap(func, &args_vec, 1);
 }
 
 void check_trap2(wasm_func_t* func, int32_t arg1, int32_t arg2) {
-  wasm_val_t args[2] = {
-    {.kind = WASM_I32, .of = {.i32 = arg1}},
-    {.kind = WASM_I32, .of = {.i32 = arg2}}
-  };
-  check_trap(func, args, 2, 0);
+  wasm_val_t args[] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) };
+  wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args);
+  check_trap(func, &args_vec, 0);
 }
 
 int main(int argc, const char* argv[]) {
@@ -167,7 +169,8 @@ int main(int argc, const char* argv[]) {
   printf("Instantiating module...\n");
   wasm_instance_t* instance = NULL;
   wasm_trap_t *trap = NULL;
-  error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap);
+  wasm_extern_vec_t imports = WASM_EMPTY_VEC;
+  error = wasmtime_instance_new(store, module, &imports, &instance, &trap);
   if (!instance)
     exit_with_error("failed to instantiate", error, trap);
 
diff --git a/examples/multi.c b/examples/multi.c
index a1514b1833..3553706447 100644
--- a/examples/multi.c
+++ b/examples/multi.c
@@ -32,14 +32,14 @@ static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_t
 
 // A function to be called from Wasm code.
 wasm_trap_t* callback(
-  const wasm_val_t args[], wasm_val_t results[]
+  const wasm_val_vec_t* args, wasm_val_vec_t* results
 ) {
   printf("Calling back...\n");
-  printf("> %"PRIu32" %"PRIu64"\n", args[0].of.i32, args[1].of.i64);
+  printf("> %"PRIu32" %"PRIu64"\n", args->data[0].of.i32, args->data[1].of.i64);
   printf("\n");
 
-  wasm_val_copy(&results[0], &args[1]);
-  wasm_val_copy(&results[1], &args[0]);
+  wasm_val_copy(&results->data[0], &args->data[1]);
+  wasm_val_copy(&results->data[1], &args->data[0]);
   return NULL;
 }
 
@@ -112,10 +112,11 @@ int main(int argc, const char* argv[]) {
 
   // Instantiate.
   printf("Instantiating module...\n");
-  const wasm_extern_t* imports[] = {wasm_func_as_extern(callback_func)};
+  wasm_extern_t* imports[] = {wasm_func_as_extern(callback_func)};
+  wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports);
   wasm_instance_t* instance = NULL;
   wasm_trap_t* trap = NULL;
-  error = wasmtime_instance_new(store, module, imports, 1, &instance, &trap);
+  error = wasmtime_instance_new(store, module, &imports_vec, &instance, &trap);
   if (!instance)
     exit_with_error("failed to instantiate", error, trap);
 
@@ -140,13 +141,11 @@ int main(int argc, const char* argv[]) {
 
   // Call.
   printf("Calling export...\n");
-  wasm_val_t args[2];
-  args[0].kind = WASM_I32;
-  args[0].of.i32 = 1;
-  args[1].kind = WASM_I64;
-  args[1].of.i64 = 2;
+  wasm_val_t args[2] = { WASM_I32_VAL(1), WASM_I64_VAL(2) };
   wasm_val_t results[2];
-  error = wasmtime_func_call(run_func, args, 2, results, 2, &trap);
+  wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args);
+  wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results);
+  error = wasmtime_func_call(run_func, &args_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call run", error, trap);
 
diff --git a/examples/serialize.c b/examples/serialize.c
index c25a1bf401..023e1db30b 100644
--- a/examples/serialize.c
+++ b/examples/serialize.c
@@ -26,7 +26,7 @@ to tweak the `-lpthread` and such annotations as well as the name of the
 
 static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap);
 
-static wasm_trap_t* hello_callback(const wasm_val_t args[], wasm_val_t results[]) {
+static wasm_trap_t* hello_callback(const wasm_val_vec_t* args, wasm_val_vec_t* results) {
   printf("Calling back...\n");
   printf("> Hello World!\n");
   return NULL;
@@ -111,8 +111,9 @@ int deserialize(wasm_byte_vec_t* buffer) {
   printf("Instantiating module...\n");
   wasm_trap_t *trap = NULL;
   wasm_instance_t *instance = NULL;
-  const wasm_extern_t *imports[] = { wasm_func_as_extern(hello) };
-  error = wasmtime_instance_new(store, module, imports, 1, &instance, &trap);
+  wasm_extern_t *imports[] = { wasm_func_as_extern(hello) };
+  wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports);
+  error = wasmtime_instance_new(store, module, &imports_vec, &instance, &trap);
   if (instance == NULL)
     exit_with_error("failed to instantiate", error, trap);
 
@@ -126,7 +127,9 @@ int deserialize(wasm_byte_vec_t* buffer) {
 
   // And call it!
   printf("Calling export...\n");
-  error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap);
+  wasm_val_vec_t args_vec = WASM_EMPTY_VEC;
+  wasm_val_vec_t results_vec = WASM_EMPTY_VEC;
+  error = wasmtime_func_call(run, &args_vec, &results_vec, &trap);
   if (error != NULL || trap != NULL)
     exit_with_error("failed to call function", error, trap);
 
diff --git a/examples/threads.c b/examples/threads.c
index bc04ba69c6..06aa1f0aa7 100644
--- a/examples/threads.c
+++ b/examples/threads.c
@@ -17,9 +17,9 @@ const int N_THREADS = 10;
 const int N_REPS = 3;
 
 // A function to be called from Wasm code.
-own wasm_trap_t* callback(const wasm_val_t args[], wasm_val_t results[]) {
-  assert(args[0].kind == WASM_I32);
-  printf("> Thread %d running\n", args[0].of.i32);
+own wasm_trap_t* callback(const wasm_val_vec_t* args, wasm_val_vec_t* results) {
+  assert(args->data[0].kind == WASM_I32);
+  printf("> Thread %d running\n", args->data[0].of.i32);
   return NULL;
 }
 
@@ -53,11 +53,12 @@ void* run(void* args_abs) {
     wasm_globaltype_delete(global_type);
 
     // Instantiate.
-    const wasm_extern_t* imports[] = {
+    wasm_extern_t* imports[] = {
       wasm_func_as_extern(func), wasm_global_as_extern(global),
     };
+    wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports);
     own wasm_instance_t* instance =
-      wasm_instance_new(store, module, imports, NULL);
+      wasm_instance_new(store, module, &imports_vec, NULL);
     if (!instance) {
       printf("> Error instantiating module!\n");
       return NULL;
@@ -82,7 +83,9 @@ void* run(void* args_abs) {
     wasm_instance_delete(instance);
 
     // Call.
-    if (wasm_func_call(run_func, NULL, NULL)) {
+    wasm_val_vec_t args_vec = WASM_EMPTY_VEC;
+    wasm_val_vec_t results_vec = WASM_EMPTY_VEC;
+    if (wasm_func_call(run_func, &args_vec, &results_vec)) {
       printf("> Error calling function!\n");
       return NULL;
     }
diff --git a/examples/wasi-fs/main.c b/examples/wasi-fs/main.c
index 6452cb7d41..8633bfbd6c 100644
--- a/examples/wasi-fs/main.c
+++ b/examples/wasi-fs/main.c
@@ -91,7 +91,10 @@ int main() {
   wasmtime_linker_get_default(linker, &empty, &func);
   if (error != NULL)
     exit_with_error("failed to locate default export for module", error, NULL);
-  error = wasmtime_func_call(func, NULL, 0, NULL, 0, &trap);
+
+  wasm_val_vec_t args_vec = WASM_EMPTY_VEC;
+  wasm_val_vec_t results_vec = WASM_EMPTY_VEC;
+  error = wasmtime_func_call(func, &args_vec, &results_vec, &trap);
   if (error != NULL)
     exit_with_error("error calling default export", error, trap);
 
diff --git a/examples/wasi/main.c b/examples/wasi/main.c
index a8514c69ca..3bb71ec4de 100644
--- a/examples/wasi/main.c
+++ b/examples/wasi/main.c
@@ -90,7 +90,10 @@ int main() {
   wasmtime_linker_get_default(linker, &empty, &func);
   if (error != NULL)
     exit_with_error("failed to locate default export for module", error, NULL);
-  error = wasmtime_func_call(func, NULL, 0, NULL, 0, &trap);
+
+  wasm_val_vec_t args_vec = WASM_EMPTY_VEC;
+  wasm_val_vec_t results_vec = WASM_EMPTY_VEC;
+  error = wasmtime_func_call(func, &args_vec, &results_vec, &trap);
   if (error != NULL)
     exit_with_error("error calling default export", error, trap);
 
diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml
index 73bd003e1e..df6905a23f 100644
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -17,7 +17,7 @@ target-lexicon = "0.11"
 peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing", optional = true }
 wasmtime = { path = "../crates/wasmtime" }
 wasmtime-fuzzing = { path = "../crates/fuzzing" }
-wasm-smith = "0.3.0"
+wasm-smith = "0.3.1"
 
 [features]
 experimental_x64 = ["wasmtime-fuzzing/experimental_x64"]
diff --git a/fuzz/fuzz_targets/instantiate-maybe-invalid.rs b/fuzz/fuzz_targets/instantiate-maybe-invalid.rs
index 219986a25a..46a749f5e8 100644
--- a/fuzz/fuzz_targets/instantiate-maybe-invalid.rs
+++ b/fuzz/fuzz_targets/instantiate-maybe-invalid.rs
@@ -4,13 +4,18 @@ use libfuzzer_sys::fuzz_target;
 use std::time::Duration;
 use wasm_smith::MaybeInvalidModule;
 use wasmtime::Strategy;
-use wasmtime_fuzzing::oracles;
+use wasmtime_fuzzing::oracles::{self, Timeout};
 
-fuzz_target!(|module: MaybeInvalidModule| {
+fuzz_target!(|pair: (bool, MaybeInvalidModule)| {
+    let (timeout_with_time, module) = pair;
     oracles::instantiate_with_config(
         &module.to_bytes(),
         false,
         wasmtime_fuzzing::fuzz_default_config(Strategy::Auto).unwrap(),
-        Some(Duration::from_secs(20)),
+        if timeout_with_time {
+            Timeout::Time(Duration::from_secs(20))
+        } else {
+            Timeout::Fuel(100_000)
+        },
     );
 });
diff --git a/fuzz/fuzz_targets/instantiate-swarm.rs b/fuzz/fuzz_targets/instantiate-swarm.rs
index 092a32e925..b534cc4970 100644
--- a/fuzz/fuzz_targets/instantiate-swarm.rs
+++ b/fuzz/fuzz_targets/instantiate-swarm.rs
@@ -2,12 +2,23 @@
 
 use libfuzzer_sys::fuzz_target;
 use std::time::Duration;
-use wasm_smith::{ConfiguredModule, SwarmConfig};
+use wasm_smith::{Config, ConfiguredModule, SwarmConfig};
 use wasmtime::Strategy;
-use wasmtime_fuzzing::oracles;
+use wasmtime_fuzzing::oracles::{self, Timeout};
 
-fuzz_target!(|module: ConfiguredModule<SwarmConfig>| {
+fuzz_target!(|pair: (bool, ConfiguredModule<SwarmConfig>)| {
+    let (timeout_with_time, module) = pair;
     let mut cfg = wasmtime_fuzzing::fuzz_default_config(Strategy::Auto).unwrap();
     cfg.wasm_multi_memory(true);
-    oracles::instantiate_with_config(&module.to_bytes(), true, cfg, Some(Duration::from_secs(20)));
+    cfg.wasm_module_linking(module.config().module_linking_enabled());
+    oracles::instantiate_with_config(
+        &module.to_bytes(),
+        true,
+        cfg,
+        if timeout_with_time {
+            Timeout::Time(Duration::from_secs(20))
+        } else {
+            Timeout::Fuel(100_000)
+        },
+    );
 });
diff --git a/scripts/publish.rs b/scripts/publish.rs
index 330a7e0dfb..75a8d652cd 100644
--- a/scripts/publish.rs
+++ b/scripts/publish.rs
@@ -65,6 +65,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[
     "wasmtime-wiggle",
     "wasmtime-wasi",
     "wasmtime-wasi-nn",
+    "wasmtime-wasi-crypto",
     "wasmtime-rust-macro",
     "wasmtime-rust",
     "wasmtime-wast",
@@ -176,7 +177,7 @@ fn read_crate(manifest: &Path) -> Crate {
     } else {
         version.clone()
     };
-    if name == "witx" {
+    if ["witx", "wasi-crypto"].contains(&&name[..]) {
         publish = false;
     }
     Crate {
@@ -299,6 +300,13 @@ fn verify(crates: &[Crate]) {
         .unwrap();
     verify_and_vendor(&witx);
 
+    // Vendor wasi-crypto which is also a path dependency
+    let wasi_crypto = crates
+        .iter()
+        .find(|c| c.name == "wasi-crypto")
+        .unwrap();
+    verify_and_vendor(&wasi_crypto);
+
     for krate in crates {
         if !krate.publish {
             continue;
diff --git a/src/commands/run.rs b/src/commands/run.rs
index 1caec3532d..7f655b1982 100644
--- a/src/commands/run.rs
+++ b/src/commands/run.rs
@@ -19,6 +19,12 @@ use wasmtime_wasi::snapshots::preview_1::Wasi as WasiSnapshot1;
 #[cfg(feature = "wasi-nn")]
 use wasmtime_wasi_nn::{WasiNn, WasiNnCtx};
 
+#[cfg(feature = "wasi-crypto")]
+use wasmtime_wasi_crypto::{
+    WasiCryptoAsymmetricCommon, WasiCryptoCommon, WasiCryptoCtx, WasiCryptoSignatures,
+    WasiCryptoSymmetric,
+};
+
 fn parse_module(s: &OsStr) -> Result<PathBuf, OsString> {
     // Do not accept wasmtime subcommand names as the module name
     match s.to_str() {
@@ -368,5 +374,14 @@ fn populate_with_wasi(
         wasi_nn.add_to_linker(linker)?;
     }
 
+    #[cfg(feature = "wasi-crypto")]
+    {
+        let cx_crypto = WasiCryptoCtx::new();
+        WasiCryptoCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?;
+        WasiCryptoAsymmetricCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?;
+        WasiCryptoSignatures::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?;
+        WasiCryptoSymmetric::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?;
+    }
+
     Ok(())
 }
diff --git a/src/obj.rs b/src/obj.rs
index e957fb6afd..529d1c3c6d 100644
--- a/src/obj.rs
+++ b/src/obj.rs
@@ -65,7 +65,7 @@ pub fn compile_to_obj(
     );
 
     let environ = ModuleEnvironment::new(compiler.isa().frontend_config(), &tunables, &features);
-    let (mut translation, types) = environ
+    let (_main_module, mut translation, types) = environ
         .translate(wasm)
         .context("failed to translate module")?;
     assert_eq!(translation.len(), 1);
diff --git a/tests/all/debug/lldb.rs b/tests/all/debug/lldb.rs
index c29e757b31..7807732700 100644
--- a/tests/all/debug/lldb.rs
+++ b/tests/all/debug/lldb.rs
@@ -137,7 +137,11 @@ check: exited with status
 #[ignore]
 #[cfg(all(
     any(target_os = "linux", target_os = "macos"),
-    target_pointer_width = "64"
+    target_pointer_width = "64",
+    // Ignore test on new backend. The value this is looking for is
+    // not available at the point that the breakpoint is set when
+    // compiled by the new backend.
+    not(feature = "experimental_x64"),
 ))]
 pub fn test_debug_dwarf_ptr() -> Result<()> {
     let output = lldb_with_script(
diff --git a/tests/all/debug/translate.rs b/tests/all/debug/translate.rs
index e8ceb7bd9e..7253989d57 100644
--- a/tests/all/debug/translate.rs
+++ b/tests/all/debug/translate.rs
@@ -114,7 +114,11 @@ check:        DW_AT_decl_line	(10)
 #[cfg(all(
     any(target_os = "linux", target_os = "macos"),
     target_arch = "x86_64",
-    target_pointer_width = "64"
+    target_pointer_width = "64",
+    // Ignore test on new backend. This is a specific test with hardcoded
+    // offsets and the new backend compiles the return basic-block at a different
+    // offset, causing mismatches.
+    not(feature = "experimental_x64"),
 ))]
 fn test_debug_dwarf5_translate_lines() -> Result<()> {
     check_line_program(
diff --git a/tests/all/externals.rs b/tests/all/externals.rs
index 365d5b2611..db70f7e3cb 100644
--- a/tests/all/externals.rs
+++ b/tests/all/externals.rs
@@ -340,3 +340,47 @@ fn grow_externref_tables_via_api() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+#[test]
+fn read_write_memory_via_api() {
+    let cfg = Config::new();
+    let store = Store::new(&Engine::new(&cfg));
+    let ty = MemoryType::new(Limits::new(1, None));
+    let mem = Memory::new(&store, ty);
+    mem.grow(1).unwrap();
+
+    let value = b"hello wasm";
+    mem.write(mem.data_size() - value.len(), value).unwrap();
+
+    let mut buffer = [0u8; 10];
+    mem.read(mem.data_size() - buffer.len(), &mut buffer)
+        .unwrap();
+    assert_eq!(value, &buffer);
+
+    // Error conditions.
+
+    // Out of bounds write.
+
+    let res = mem.write(mem.data_size() - value.len() + 1, value);
+    assert!(res.is_err());
+    assert_ne!(
+        unsafe { mem.data_unchecked()[mem.data_size() - value.len() + 1] },
+        value[0],
+        "no data is written",
+    );
+
+    // Out of bounds read.
+
+    buffer[0] = 0x42;
+    let res = mem.read(mem.data_size() - buffer.len() + 1, &mut buffer);
+    assert!(res.is_err());
+    assert_eq!(buffer[0], 0x42, "no data is read");
+
+    // Read offset overflow.
+    let res = mem.read(usize::MAX, &mut buffer);
+    assert!(res.is_err());
+
+    // Write offset overflow.
+    let res = mem.write(usize::MAX, &mut buffer);
+    assert!(res.is_err());
+}
diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs
new file mode 100644
index 0000000000..b1b73bc6a8
--- /dev/null
+++ b/tests/all/fuel.rs
@@ -0,0 +1,124 @@
+use anyhow::Result;
+use wasmtime::*;
+use wast::parser::{self, Parse, ParseBuffer, Parser};
+
+mod kw {
+    wast::custom_keyword!(assert_fuel);
+}
+
+struct FuelWast<'a> {
+    assertions: Vec<(wast::Span, u64, wast::Module<'a>)>,
+}
+
+impl<'a> Parse<'a> for FuelWast<'a> {
+    fn parse(parser: Parser<'a>) -> parser::Result<Self> {
+        let mut assertions = Vec::new();
+        while !parser.is_empty() {
+            assertions.push(parser.parens(|p| {
+                let span = p.parse::<kw::assert_fuel>()?.0;
+                Ok((span, p.parse()?, p.parens(|p| p.parse())?))
+            })?);
+        }
+        Ok(FuelWast { assertions })
+    }
+}
+
+#[test]
+fn run() -> Result<()> {
+    let test = std::fs::read_to_string("tests/all/fuel.wast")?;
+    let buf = ParseBuffer::new(&test)?;
+    let mut wast = parser::parse::<FuelWast<'_>>(&buf)?;
+    for (span, fuel, module) in wast.assertions.iter_mut() {
+        let consumed = fuel_consumed(&module.encode()?);
+        if consumed == *fuel {
+            continue;
+        }
+        let (line, col) = span.linecol_in(&test);
+        panic!(
+            "tests/all/fuel.wast:{}:{} - expected {} fuel, found {}",
+            line + 1,
+            col + 1,
+            fuel,
+            consumed
+        );
+    }
+    Ok(())
+}
+
+fn fuel_consumed(wasm: &[u8]) -> u64 {
+    let mut config = Config::new();
+    config.consume_fuel(true);
+    let engine = Engine::new(&config);
+    let module = Module::new(&engine, wasm).unwrap();
+    let store = Store::new(&engine);
+    store.add_fuel(u64::max_value());
+    drop(Instance::new(&store, &module, &[]));
+    store.fuel_consumed().unwrap()
+}
+
+#[test]
+fn iloop() {
+    iloop_aborts(
+        r#"
+            (module
+                (start 0)
+                (func loop br 0 end)
+            )
+        "#,
+    );
+    iloop_aborts(
+        r#"
+            (module
+                (start 0)
+                (func loop i32.const 1 br_if 0 end)
+            )
+        "#,
+    );
+    iloop_aborts(
+        r#"
+            (module
+                (start 0)
+                (func loop i32.const 0 br_table 0 end)
+            )
+        "#,
+    );
+    iloop_aborts(
+        r#"
+            (module
+                (start 0)
+                (func $f0 call $f1 call $f1)
+                (func $f1 call $f2 call $f2)
+                (func $f2 call $f3 call $f3)
+                (func $f3 call $f4 call $f4)
+                (func $f4 call $f5 call $f5)
+                (func $f5 call $f6 call $f6)
+                (func $f6 call $f7 call $f7)
+                (func $f7 call $f8 call $f8)
+                (func $f8 call $f9 call $f9)
+                (func $f9 call $f10 call $f10)
+                (func $f10 call $f11 call $f11)
+                (func $f11 call $f12 call $f12)
+                (func $f12 call $f13 call $f13)
+                (func $f13 call $f14 call $f14)
+                (func $f14 call $f15 call $f15)
+                (func $f15 call $f16 call $f16)
+                (func $f16)
+            )
+        "#,
+    );
+
+    fn iloop_aborts(wat: &str) {
+        let mut config = Config::new();
+        config.consume_fuel(true);
+        let engine = Engine::new(&config);
+        let module = Module::new(&engine, wat).unwrap();
+        let store = Store::new(&engine);
+        store.add_fuel(10_000);
+        let error = Instance::new(&store, &module, &[]).err().unwrap();
+        assert!(
+            error.to_string().contains("all fuel consumed"),
+            "bad error: {}",
+            error
+        );
+    }
+}
diff --git a/tests/all/fuel.wast b/tests/all/fuel.wast
new file mode 100644
index 0000000000..bb3450b6cc
--- /dev/null
+++ b/tests/all/fuel.wast
@@ -0,0 +1,208 @@
+(assert_fuel 0 (module))
+
+(assert_fuel 1
+  (module
+    (func $f)
+    (start $f)))
+
+(assert_fuel 2
+  (module
+    (func $f
+      i32.const 0
+      drop
+    )
+    (start $f)))
+
+(assert_fuel 1
+  (module
+    (func $f
+      block
+      end
+    )
+    (start $f)))
+
+(assert_fuel 1
+  (module
+    (func $f
+      unreachable
+    )
+    (start $f)))
+
+(assert_fuel 7
+  (module
+    (func $f
+      i32.const 0
+      i32.const 0
+      i32.const 0
+      i32.const 0
+      i32.const 0
+      i32.const 0
+      unreachable
+    )
+    (start $f)))
+
+(assert_fuel 1
+  (module
+    (func $f
+      return
+      i32.const 0
+      i32.const 0
+      i32.const 0
+      i32.const 0
+      i32.const 0
+      i32.const 0
+      unreachable
+    )
+    (start $f)))
+
+(assert_fuel 3
+  (module
+    (func $f
+      i32.const 0
+      if
+        call $f
+      end
+    )
+    (start $f)))
+
+(assert_fuel 4
+  (module
+    (func $f
+      i32.const 1
+      if
+        i32.const 0
+        drop
+      end
+    )
+    (start $f)))
+
+(assert_fuel 4
+  (module
+    (func $f
+      i32.const 1
+      if
+        i32.const 0
+        drop
+      else
+        call $f
+      end
+    )
+    (start $f)))
+
+(assert_fuel 4
+  (module
+    (func $f
+      i32.const 0
+      if
+        call $f
+      else
+        i32.const 0
+        drop
+      end
+    )
+    (start $f)))
+
+(assert_fuel 3
+  (module
+    (func $f
+      block
+        i32.const 1
+        br_if 0
+        i32.const 0
+        drop
+      end
+    )
+    (start $f)))
+
+(assert_fuel 4
+  (module
+    (func $f
+      block
+        i32.const 0
+        br_if 0
+        i32.const 0
+        drop
+      end
+    )
+    (start $f)))
+
+;; count code before unreachable
+(assert_fuel 2
+  (module
+    (func $f
+      i32.const 0
+      unreachable
+    )
+    (start $f)))
+
+;; count code before return
+(assert_fuel 2
+  (module
+    (func $f
+      i32.const 0
+      return
+    )
+    (start $f)))
+
+;; cross-function fuel works
+(assert_fuel 3
+  (module
+    (func $f
+      call $other
+    )
+    (func $other)
+    (start $f)))
+(assert_fuel 5
+  (module
+    (func $f
+      i32.const 0
+      call $other
+      i32.const 0
+      drop
+    )
+    (func $other (param i32))
+    (start $f)))
+(assert_fuel 4
+  (module
+    (func $f
+      call $other
+      drop
+    )
+    (func $other (result i32)
+      i32.const 0
+    )
+    (start $f)))
+(assert_fuel 4
+  (module
+    (func $f
+      i32.const 0
+      call_indirect
+    )
+    (func $other)
+    (table funcref (elem $other))
+    (start $f)))
+
+;; loops!
+(assert_fuel 1
+  (module
+    (func $f
+      loop
+      end
+    )
+    (start $f)))
+(assert_fuel 53 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func
+  (module
+    (func $f
+      (local i32)
+      i32.const 10
+      local.set 0
+
+      loop
+        local.get 0
+        i32.const 1
+        i32.sub
+        local.tee 0
+        br_if 0
+      end
+    )
+    (start $f)))
diff --git a/tests/all/fuzzing.rs b/tests/all/fuzzing.rs
index 3e46af2d72..e5c1906886 100644
--- a/tests/all/fuzzing.rs
+++ b/tests/all/fuzzing.rs
@@ -6,7 +6,7 @@
 //! `include_bytes!("./fuzzing/some-descriptive-name.wasm")`.
 
 use wasmtime::{Config, Strategy};
-use wasmtime_fuzzing::oracles;
+use wasmtime_fuzzing::oracles::{self, Timeout};
 
 #[test]
 fn instantiate_empty_module() {
@@ -26,5 +26,5 @@ fn instantiate_module_that_compiled_to_x64_has_register_32() {
     let mut config = Config::new();
     config.debug_info(true);
     let data = wat::parse_str(include_str!("./fuzzing/issue694.wat")).unwrap();
-    oracles::instantiate_with_config(&data, true, config, None);
+    oracles::instantiate_with_config(&data, true, config, Timeout::None);
 }
diff --git a/tests/all/main.rs b/tests/all/main.rs
index 414708144f..c20b2dddce 100644
--- a/tests/all/main.rs
+++ b/tests/all/main.rs
@@ -2,6 +2,7 @@ mod cli_tests;
 mod custom_signal_handler;
 mod debug;
 mod externals;
+mod fuel;
 mod func;
 mod fuzzing;
 mod globals;
@@ -12,6 +13,7 @@ mod instance;
 mod invoke_func_via_table;
 mod linker;
 mod memory_creator;
+mod module;
 mod module_linking;
 mod module_serialize;
 mod name;
diff --git a/tests/all/module.rs b/tests/all/module.rs
new file mode 100644
index 0000000000..64384e0de6
--- /dev/null
+++ b/tests/all/module.rs
@@ -0,0 +1,54 @@
+use wasmtime::*;
+
+#[test]
+fn caches_across_engines() {
+    let mut c = Config::new();
+    c.cranelift_clear_cpu_flags();
+
+    let bytes = Module::new(&Engine::new(&c), "(module)")
+        .unwrap()
+        .serialize()
+        .unwrap();
+
+    let res = Module::deserialize(
+        &Engine::new(&Config::new().cranelift_clear_cpu_flags()),
+        &bytes,
+    );
+    assert!(res.is_ok());
+
+    // differ in shared cranelift flags
+    let res = Module::deserialize(
+        &Engine::new(
+            &Config::new()
+                .cranelift_clear_cpu_flags()
+                .cranelift_nan_canonicalization(true),
+        ),
+        &bytes,
+    );
+    assert!(res.is_err());
+
+    // differ in cranelift settings
+    let res = Module::deserialize(
+        &Engine::new(
+            &Config::new()
+                .cranelift_clear_cpu_flags()
+                .cranelift_opt_level(OptLevel::None),
+        ),
+        &bytes,
+    );
+    assert!(res.is_err());
+
+    // differ in cpu-specific flags
+    if cfg!(target_arch = "x86_64") {
+        let res = Module::deserialize(
+            &Engine::new(unsafe {
+                &Config::new()
+                    .cranelift_clear_cpu_flags()
+                    .cranelift_other_flag("has_sse3", "true")
+                    .unwrap()
+            }),
+            &bytes,
+        );
+        assert!(res.is_err());
+    }
+}
diff --git a/tests/all/module_linking.rs b/tests/all/module_linking.rs
index f352eb7f54..829ddd9b8e 100644
--- a/tests/all/module_linking.rs
+++ b/tests/all/module_linking.rs
@@ -105,8 +105,13 @@ fn imports_exports() -> Result<()> {
     assert_eq!(i.len(), 1);
     let import = i.next().unwrap();
     assert_eq!(import.module(), "");
-    assert_eq!(import.name(), Some("a"));
-    let module_ty = match import.ty() {
+    assert_eq!(import.name(), None);
+    let instance_ty = match import.ty() {
+        ExternType::Instance(t) => t,
+        _ => panic!("unexpected type"),
+    };
+    assert_eq!(instance_ty.exports().len(), 1);
+    let module_ty = match instance_ty.exports().next().unwrap().ty() {
         ExternType::Module(m) => m,
         _ => panic!("unexpected type"),
     };
@@ -148,8 +153,13 @@ fn imports_exports() -> Result<()> {
     assert_eq!(i.len(), 1);
     let import = i.next().unwrap();
     assert_eq!(import.module(), "");
-    assert_eq!(import.name(), Some("b"));
+    assert_eq!(import.name(), None);
     let instance_ty = match import.ty() {
+        ExternType::Instance(t) => t,
+        _ => panic!("unexpected type"),
+    };
+    assert_eq!(instance_ty.exports().len(), 1);
+    let instance_ty = match instance_ty.exports().next().unwrap().ty() {
         ExternType::Instance(m) => m,
         _ => panic!("unexpected type"),
     };
@@ -175,3 +185,114 @@ fn imports_exports() -> Result<()> {
     }
     Ok(())
 }
+
+#[test]
+fn limit_instances() -> Result<()> {
+    let mut config = Config::new();
+    config.wasm_module_linking(true);
+    config.max_instances(10);
+    let engine = Engine::new(&config);
+    let module = Module::new(
+        &engine,
+        r#"
+            (module $PARENT
+              (module $m0)
+              (module $m1
+                (instance (instantiate (module outer $PARENT $m0)))
+                (instance (instantiate (module outer $PARENT $m0))))
+              (module $m2
+                (instance (instantiate (module outer $PARENT $m1)))
+                (instance (instantiate (module outer $PARENT $m1))))
+              (module $m3
+                (instance (instantiate (module outer $PARENT $m2)))
+                (instance (instantiate (module outer $PARENT $m2))))
+              (module $m4
+                (instance (instantiate (module outer $PARENT $m3)))
+                (instance (instantiate (module outer $PARENT $m3))))
+              (module $m5
+                (instance (instantiate (module outer $PARENT $m4)))
+                (instance (instantiate (module outer $PARENT $m4))))
+              (instance (instantiate $m5))
+            )
+        "#,
+    )?;
+    let store = Store::new(&engine);
+    let err = Instance::new(&store, &module, &[]).err().unwrap();
+    assert!(
+        err.to_string().contains("resource limit exceeded"),
+        "bad error: {}",
+        err
+    );
+    Ok(())
+}
+
+#[test]
+fn limit_memories() -> Result<()> {
+    let mut config = Config::new();
+    config.wasm_module_linking(true);
+    config.wasm_multi_memory(true);
+    config.max_memories(10);
+    let engine = Engine::new(&config);
+    let module = Module::new(
+        &engine,
+        r#"
+            (module
+              (module $m0
+                (memory 1 1)
+                (memory 1 1)
+                (memory 1 1)
+                (memory 1 1)
+                (memory 1 1)
+              )
+
+              (instance (instantiate $m0))
+              (instance (instantiate $m0))
+              (instance (instantiate $m0))
+              (instance (instantiate $m0))
+            )
+        "#,
+    )?;
+    let store = Store::new(&engine);
+    let err = Instance::new(&store, &module, &[]).err().unwrap();
+    assert!(
+        err.to_string().contains("resource limit exceeded"),
+        "bad error: {}",
+        err
+    );
+    Ok(())
+}
+
+#[test]
+fn limit_tables() -> Result<()> {
+    let mut config = Config::new();
+    config.wasm_module_linking(true);
+    config.max_tables(10);
+    let engine = Engine::new(&config);
+    let module = Module::new(
+        &engine,
+        r#"
+            (module
+              (module $m0
+                (table 1 1 funcref)
+                (table 1 1 funcref)
+                (table 1 1 funcref)
+                (table 1 1 funcref)
+                (table 1 1 funcref)
+              )
+
+              (instance (instantiate $m0))
+              (instance (instantiate $m0))
+              (instance (instantiate $m0))
+              (instance (instantiate $m0))
+            )
+        "#,
+    )?;
+    let store = Store::new(&engine);
+    let err = Instance::new(&store, &module, &[]).err().unwrap();
+    assert!(
+        err.to_string().contains("resource limit exceeded"),
+        "bad error: {}",
+        err
+    );
+    Ok(())
+}
diff --git a/tests/misc_testsuite/module-linking/alias-outer.wast b/tests/misc_testsuite/module-linking/alias-outer.wast
new file mode 100644
index 0000000000..e6dc6255a3
--- /dev/null
+++ b/tests/misc_testsuite/module-linking/alias-outer.wast
@@ -0,0 +1,67 @@
+(module $a
+  (module $m1)
+  (module $b
+    (module $m2)
+    (module $c
+      (instance (instantiate (module outer $a $m1)))
+      (instance (instantiate (module outer $b $m2)))
+    )
+    (instance (instantiate $c))
+  )
+  (instance (instantiate $b))
+)
+
+(module $a
+  (module (export "m"))
+)
+
+(module $PARENT
+  (import "a" "m" (module $b))
+  (module $c
+    (module $d
+      (instance (instantiate (module outer $PARENT $b)))
+    )
+    (instance (instantiate $d))
+  )
+  (instance (instantiate $c))
+)
+
+;; Instantiate `$b` here below twice with two different imports. Ensure the
+;; exported modules close over the captured state correctly to ensure that we
+;; get the right functions.
+(module $a
+  (module $b (export "close_over_imports")
+    (import "m" (module $m (export "f" (func (result i32)))))
+    (module (export "m")
+      (instance $a (instantiate (module outer $b $m)))
+      (func (export "f") (result i32)
+        call (func $a "f"))
+    )
+  )
+)
+
+(module
+  (import "a" "close_over_imports" (module $m0
+    (import "m" (module (export "f" (func (result i32)))))
+    (export "m" (module (export "f" (func (result i32)))))
+  ))
+
+  (module $m1
+    (func (export "f") (result i32)
+      i32.const 0))
+  (instance $m_g1 (instantiate $m0 "m" (module $m1)))
+  (instance $g1 (instantiate (module $m_g1 "m")))
+  (module $m2
+    (func (export "f") (result i32)
+      i32.const 1))
+  (instance $m_g2 (instantiate $m0 "m" (module $m2)))
+  (instance $g2 (instantiate (module $m_g2 "m")))
+
+  (func (export "get1") (result i32)
+    call (func $g1 "f"))
+  (func (export "get2") (result i32)
+    call (func $g2 "f"))
+)
+
+(assert_return (invoke "get1") (i32.const 0))
+(assert_return (invoke "get2") (i32.const 1))
diff --git a/tests/misc_testsuite/module-linking/alias.wast b/tests/misc_testsuite/module-linking/alias.wast
index 0c69b4084c..2fcad01777 100644
--- a/tests/misc_testsuite/module-linking/alias.wast
+++ b/tests/misc_testsuite/module-linking/alias.wast
@@ -7,7 +7,7 @@
   (instance $a (instantiate $m))
 
   (func (export "get") (result i32)
-    call $a.$foo)
+    call (func $a "foo"))
 )
 (assert_return (invoke "get") (i32.const 1))
 
@@ -19,7 +19,7 @@
   (instance $a (instantiate $m))
 
   (func (export "get") (result i32)
-    global.get $a.$g)
+    global.get (global $a "g"))
 )
 (assert_return (invoke "get") (i32.const 2))
 
@@ -30,7 +30,7 @@
     (data (i32.const 0) "\03\00\00\00")
   )
   (instance $a (instantiate $m))
-  (alias (instance $a) (memory $m))
+  (alias $m (memory $a "m"))
 
   (func (export "get") (result i32)
     i32.const 0
@@ -50,7 +50,7 @@
 
   (func (export "get") (result i32)
     i32.const 0
-    call_indirect $a.$t (result i32))
+    call_indirect (table $a "t") (result i32))
 )
 (assert_return (invoke "get") (i32.const 4))
 
@@ -62,11 +62,10 @@
         i32.const 5))
   )
   (instance $a (instantiate $m))
-  (instance $b (instantiate $a.$sub))
-  (alias $b.$f (instance $b) (func 0))
+  (instance $b (instantiate (module $a "module")))
 
   (func (export "get") (result i32)
-    call $b.$f)
+    call (func $b ""))
 )
 (assert_return (invoke "get") (i32.const 5))
 
@@ -79,11 +78,9 @@
     (instance $i (export "") (instantiate $sub))
   )
   (instance $a (instantiate $m))
-  (alias $a.$i (instance $a) (instance 0))
-  (alias $a.$i.$f (instance $a.$i) (func 0))
 
   (func (export "get") (result i32)
-    call $a.$i.$f)
+    call (func $a "" ""))
 )
 (assert_return (invoke "get") (i32.const 6))
 
@@ -91,48 +88,49 @@
 (module
   (type $t (func))
   (module $m
-    (func $f (type $t))
+    (func $f (type outer 0 $t))
   )
   (instance $a (instantiate $m))
 )
 
-;; alias parent -- module
+;; alias outer -- module
 (module
   (module $a)
   (module $m
-    (instance (instantiate $a))
+    (instance (instantiate (module outer 0 $a)))
   )
   (instance (instantiate $m))
 )
 
 ;; The alias, import, type, module, and instance sections can all be interleaved
-(module
+(module $ROOT
   (module $a)
   (type $t (func))
   (module $m
     ;; alias
-    (alias $thunk parent (type $t))
+    (alias $thunk (type outer 0 $t))
     ;; import
     (import "" "" (func (type $thunk)))
     ;; module (referencing parent type)
     (module
-      (func (type $thunk))
+      (func (type outer $m $thunk))
+      (func (type outer $ROOT $t))
     )
     ;; type
     (type $thunk2 (func))
     ;; module (referencing previous alias)
     (module $m2
-      (func (export "") (type $thunk2))
+      (func (export "") (type outer $m $thunk2))
     )
     ;; instance
     (instance $i (instantiate $m2))
     ;; alias that instance
-    (alias $my_f (instance $i) (func 0))
+    (alias $my_f (func $i ""))
     ;; module
     (module $m3
       (import "" (func)))
     ;; use our aliased function to create the module
-    (instance $i2 (instantiate $m3 (func $my_f)))
+    (instance $i2 (instantiate $m3 "" (func $my_f)))
     ;; module
     (module $m4
       (import "" (func)))
@@ -141,5 +139,5 @@
   ;; instantiate the above module
   (module $smol (func $f (export "")))
   (instance $smol (instantiate $smol))
-  (instance (instantiate $m (func $smol.$f)))
+  (instance (instantiate $m "" (instance $smol)))
 )
diff --git a/tests/misc_testsuite/module-linking/import-subtyping.wast b/tests/misc_testsuite/module-linking/import-subtyping.wast
index 4ac1658070..2055f848f3 100644
--- a/tests/misc_testsuite/module-linking/import-subtyping.wast
+++ b/tests/misc_testsuite/module-linking/import-subtyping.wast
@@ -9,22 +9,36 @@
 
 (module
   (import "a" "m" (module))
+)
+(module
   (import "a" "m" (module (export "" (func))))
+)
+(module
   (import "a" "m" (module (export "a" (func))))
+)
+(module
   (import "a" "m" (module (export "b" (global i32))))
+)
+(module
   (import "a" "m" (module
     (export "" (func))
     (export "a" (func))
   ))
+)
+(module
   (import "a" "m" (module
     (export "a" (func))
     (export "" (func))
   ))
+)
+(module
   (import "a" "m" (module
     (export "a" (func))
     (export "" (func))
     (export "b" (global i32))
   ))
+)
+(module
   (import "a" "m" (module
     (export "b" (global i32))
     (export "a" (func))
@@ -37,64 +51,60 @@
   (module (export "m")
     (func (export ""))))
 
-(module
-  (import "a" "m" (module))
-  (import "a" "m" (module (export "" (func))))
-)
+(module (import "a" "m" (module)))
+(module (import "a" "m" (module (export "" (func)))))
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (func (param i32))))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (func (result i32))))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (global i32)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (table 1 funcref)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (memory 1)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (module)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (instance)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 
 (module $a
   (module (export "m")
     (global (export "") i32 (i32.const 0))))
 
 ;; globals
-(module
-  (import "a" "m" (module))
-  (import "a" "m" (module (export "" (global i32))))
-)
+(module (import "a" "m" (module)))
+(module (import "a" "m" (module (export "" (global i32)))))
 (assert_unlinkable
   (module
     (import "a" "m" (module (export "" (global (mut i32)))))
   )
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (global f32)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (func)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (table 1 funcref)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (memory 1)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (module)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (instance)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 
 ;; tables
 (module $a
@@ -105,40 +115,52 @@
 )
 (module
   (import "a" "m" (module))
+)
+(module
   (import "a" "m" (module (export "" (table 1 funcref))))
+)
+(module
   (import "a" "m" (module (export "" (table 0 funcref))))
+)
+(module
   (import "a" "m" (module (export "max" (table 1 10 funcref))))
+)
+(module
   (import "a" "m" (module (export "max" (table 0 10 funcref))))
+)
+(module
   (import "a" "m" (module (export "max" (table 0 11 funcref))))
+)
+(module
   (import "a" "m" (module (export "max" (table 0 funcref))))
 )
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (global f32)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (func)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (table 2 funcref)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (table 1 10 funcref)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "max" (table 2 10 funcref)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "max" (table 1 9 funcref)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (memory 1)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (module)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (instance)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 
 ;; memories
 (module $a
@@ -149,40 +171,52 @@
 )
 (module
   (import "a" "m" (module))
+)
+(module
   (import "a" "m" (module (export "" (memory 1))))
+)
+(module
   (import "a" "m" (module (export "" (memory 0))))
+)
+(module
   (import "a" "m" (module (export "max" (memory 1 10))))
+)
+(module
   (import "a" "m" (module (export "max" (memory 0 10))))
+)
+(module
   (import "a" "m" (module (export "max" (memory 0 11))))
+)
+(module
   (import "a" "m" (module (export "max" (memory 0))))
 )
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (global f32)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (func)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (table 1 funcref)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (memory 2)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (memory 1 10)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "max" (memory 2 10)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "max" (memory 2)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (module)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (instance)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 
 ;; modules
 (module $a
@@ -206,72 +240,102 @@
     )
     ;; import a mixture
     (module (export "e")
-      (import "" (func))
-      (import "" (func))
-      (import "" (global i32))
+      (import "a" (func))
+      (import "b" (func))
+      (import "c" (global i32))
     )
   )
 )
 (module
   (import "a" "m" (module))
+)
+(module
   (import "a" "m" (module (export "a" (module))))
+)
+(module
   (import "a" "m" (module (export "b" (module))))
+)
+(module
   (import "a" "m" (module (export "b" (module (export "" (func))))))
+)
+(module
   (import "a" "m" (module (export "c" (module))))
+)
+(module
   (import "a" "m" (module (export "c" (module
     (export "a" (func))
   ))))
+)
+(module
   (import "a" "m" (module (export "c" (module
     (export "a" (func))
     (export "b" (func (result i32)))
   ))))
+)
+(module
   (import "a" "m" (module (export "c" (module
     (export "c" (global i32))
   ))))
+)
+(module
   (import "a" "m" (module (export "c" (module
     (export "c" (global i32))
     (export "a" (func))
   ))))
-
-  ;; for now import strings aren't matched at all, imports must simply pairwise
-  ;; line up
-  (import "a" "m" (module (export "d" (module (import "" (func))))))
-  (import "a" "m" (module (export "d" (module (import "x" (func))))))
-  (import "a" "m" (module (export "d" (module (import "x" "y" (func))))))
-
-  (import "a" "m" (module (export "e" (module
-    (import "x" "y" (func))
+)
+(module
+  (import "a" "m" (module (export "d" (module
+    (import "" (func))
     (import "a" (func))
-    (import "z" (global i32))
+  ))))
+)
+(module
+  (import "a" "m" (module (export "d" (module (import "" (func))))))
+)
+(assert_unlinkable
+  (module
+    (import "a" "m" (module (export "d" (module (import "x" (func))))))
+  )
+  "incompatible import type for `a`")
+(assert_unlinkable
+  (module
+    (import "a" "m" (module (export "d" (module (import "x" "y" (func))))))
+  )
+  "incompatible import type for `a`")
+(module
+  (import "a" "m" (module (export "e" (module
+    (import "a" (func))
+    (import "b" (func))
+    (import "c" (global i32))
   ))))
 )
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (module (export "a" (func)))))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "d" (module)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "d" (module (import "" (module)))))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (global f32)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (func)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (table 1 funcref)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (memory 2)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (module (export "foo" (func)))))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (instance)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 
 ;; instances
 (module $a
@@ -303,46 +367,65 @@
 )
 (module
   (import "a" "a" (instance))
+)
+(module
   (import "a" "b" (instance))
+)
+(module
   (import "a" "b" (instance (export "" (func))))
+)
+(module
   (import "a" "c" (instance))
+)
+(module
   (import "a" "c" (instance (export "a" (func))))
+)
+(module
   (import "a" "c" (instance (export "b" (func (result i32)))))
+)
+(module
   (import "a" "c" (instance (export "c" (global i32))))
+)
+(module
   (import "a" "c" (instance
     (export "a" (func))
     (export "b" (func (result i32)))
     (export "c" (global i32))
   ))
+)
+(module
   (import "a" "c" (instance
     (export "c" (global i32))
     (export "a" (func))
   ))
-
+)
+(module
   (import "a" "m" (module (export "i" (instance))))
+)
+(module
   (import "a" "m" (module (export "i" (instance (export "" (func))))))
 )
 (assert_unlinkable
   (module (import "a" "a" (instance (export "" (global f32)))))
-  "instance types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "i" (instance (export "x" (func)))))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (func)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (table 1 funcref)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (memory 2)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (memory 1 10)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "max" (memory 2 10)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
 (assert_unlinkable
   (module (import "a" "m" (module (export "" (module)))))
-  "module types incompatible")
+  "incompatible import type for `a`")
diff --git a/tests/misc_testsuite/module-linking/instantiate.wast b/tests/misc_testsuite/module-linking/instantiate.wast
index 8a61684438..257aeb90dc 100644
--- a/tests/misc_testsuite/module-linking/instantiate.wast
+++ b/tests/misc_testsuite/module-linking/instantiate.wast
@@ -34,7 +34,7 @@
   (module
     (import "" (func))
     (start 0))
-  (instance $a (instantiate 0 (func $set)))
+  (instance $a (instantiate 0 "" (func $set)))
 )
 
 (assert_return (invoke $a "get") (i32.const 1))
@@ -49,7 +49,7 @@
       global.set 0)
     (start 0))
 
-  (instance $a (instantiate 0 (global $g)))
+  (instance $a (instantiate 0 "" (global $g)))
 )
 (assert_return (invoke $a "get") (i32.const 2))
 
@@ -63,7 +63,7 @@
       call_indirect)
     (start 0))
 
-  (instance $a (instantiate 0 (table $t)))
+  (instance $a (instantiate 0 "" (table $t)))
 )
 (assert_return (invoke $a "get") (i32.const 3))
 
@@ -78,7 +78,7 @@
       i32.store)
     (start 0))
 
-  (instance $a (instantiate 0 (memory $m)))
+  (instance $a (instantiate 0 "" (memory $m)))
 )
 (assert_return (invoke $a "load") (i32.const 100))
 
@@ -88,13 +88,13 @@
 
   (module $m1
     (import "" (instance (export "" (func))))
-    (alias (instance 0) (func 0))
+    (alias (func 0 ""))
     (start 0))
 
   (module $m2
     (func (export "") (import "")))
-  (instance $i (instantiate $m2 (func $set)))
-  (instance (instantiate $m1 (instance $i)))
+  (instance $i (instantiate $m2 "" (func $set)))
+  (instance (instantiate $m1 "" (instance $i)))
 )
 (assert_return (invoke $a "get") (i32.const 4))
 
@@ -106,14 +106,14 @@
     (import "" (module $m (export "" (func $f (result i32)))))
     (instance $i (instantiate $m))
     (func $get (export "") (result i32)
-      call $i.$f))
+      call (func $i "")))
 
   (module $m2
     (func (export "") (result i32)
       i32.const 5))
-  (instance $i (instantiate $m1 (module $m2)))
+  (instance $i (instantiate $m1 "" (module $m2)))
   (func (export "get") (result i32)
-    call $i.$get)
+    call (func $i ""))
 )
 (assert_return (invoke "get") (i32.const 5))
 
@@ -122,16 +122,16 @@
   (module $m
     (import "" (module $m (export "get" (func (result i32)))))
     (instance $i (instantiate $m))
-    (alias $f (instance $i) (func 0))
+    (alias $f (func $i "get"))
     (export "" (func $f))
   )
   (module $m2
     (func (export "get") (result i32)
       i32.const 6))
-  (instance $a (instantiate $m (module $m2)))
+  (instance $a (instantiate $m "" (module $m2)))
 
   (func (export "get") (result i32)
-    call $a.$f)
+    call (func $a ""))
 )
 (assert_return (invoke "get") (i32.const 6))
 
@@ -143,10 +143,10 @@
   (import "a" "memory" (memory $m 1))
 
   (module
-    (import "" (memory 1))
-    (import "" (global (mut i32)))
-    (import "" (table 1 funcref))
-    (import "" (func))
+    (import "m" (memory 1))
+    (import "g" (global (mut i32)))
+    (import "t" (table 1 funcref))
+    (import "f" (func))
     (func $start
       call 0
 
@@ -163,10 +163,10 @@
 
   (instance $a
     (instantiate 0
-      (memory $m)
-      (global $g)
-      (table $t)
-      (func $f)
+      "m" (memory $m)
+      "g" (global $g)
+      "t" (table $t)
+      "f" (func $f)
     )
   )
 )
@@ -183,10 +183,10 @@
   (module $mt (import "" (table 1 funcref)))
   (module $mg (import "" (global (mut i32))))
 
-  (instance (instantiate $mm (memory $m)))
-  (instance (instantiate $mf (func $f)))
-  (instance (instantiate $mt (table $t)))
-  (instance (instantiate $mg (global $g)))
+  (instance (instantiate $mm "" (memory $m)))
+  (instance (instantiate $mf "" (func $f)))
+  (instance (instantiate $mt "" (table $t)))
+  (instance (instantiate $mg "" (global $g)))
 )
 
 ;; instantiate nested
@@ -204,13 +204,13 @@
           (import "" (func))
           (start 0)
         )
-        (instance (instantiate 0 (func 0)))
+        (instance (instantiate 0 "" (func 0)))
       )
-      (instance (instantiate 0 (func 0)))
+      (instance (instantiate 0 "" (func 0)))
     )
-    (instance (instantiate 0 (func 0)))
+    (instance (instantiate 0 "" (func 0)))
   )
-  (instance (instantiate 0 (func 0)))
+  (instance (instantiate 0 "" (func 0)))
 )
 (assert_return (invoke $a "get") (i32.const 1))
 
@@ -219,20 +219,14 @@
   (module (export "m"))
   (instance (export "i") (instantiate 0))
 )
-(module
-  (import "b" "m" (module))
-  (import "b" "i" (instance))
-)
-(assert_unlinkable
-  (module
-    (import "b" "m" (module (import "" (func))))
-  )
-  "module types incompatible")
+(module (import "b" "m" (module)))
+(module (import "b" "m" (module (import "" (func)))))
+(module (import "b" "i" (instance)))
 (assert_unlinkable
   (module
     (import "b" "i" (instance (export "" (func))))
   )
-  "instance types incompatible")
+  "incompatible import type")
 
 ;; ensure we ignore other exported items
 (module $b
@@ -250,7 +244,7 @@
   ))
 
   (func (export "get") (result i32)
-    global.get $i.$g)
+    global.get (global $i "g"))
 )
 (assert_return (invoke "get") (i32.const 0xfeed))
 
@@ -270,15 +264,45 @@
 (module
   (import "b" "i" (instance $i
     ;; notice that this order is swapped
-    (export "g" (func $g (param i32) (result i32)))
-    (export "f" (func $f (result i32)))
+    (export "g" (func (param i32) (result i32)))
+    (export "f" (func (result i32)))
   ))
 
   (func (export "f") (result i32)
-    call $i.$f)
+    call (func $i "f"))
   (func (export "g") (param i32) (result i32)
     local.get 0
-    call $i.$g)
+    call (func $i "g"))
 )
 (assert_return (invoke "f") (i32.const 300))
 (assert_return (invoke "g" (i32.const 3000)) (i32.const 3100))
+
+(module $a
+  (func (export "f")))
+
+(module
+  (import "a" "f" (func))
+
+  (module $m1
+    (import "a" "f" (func)))
+  (instance (instantiate $m1 "a" (instance 0)))
+)
+
+(module
+  (import "a" "f" (func))
+
+  ;; this module provides nothing
+  (module $m1)
+
+  ;; this module imports a module which it says imports something
+  (module $m2
+    (module $a
+      (func (export "")))
+    (instance $i (instantiate $a))
+    (import "m" (module $b (import "" (func))))
+    (instance $b (instantiate $b "" (func $i ""))))
+
+  ;; we should be able to instantiate m2 with m1 because m1 doesn't actually
+  ;; import anything (always safe to remove imports!)
+  (instance (instantiate $m2 "m" (module $m1)))
+)
diff --git a/tests/misc_testsuite/multi-memory/simple.wast b/tests/misc_testsuite/multi-memory/simple.wast
index c06c0ff420..14b203b1be 100644
--- a/tests/misc_testsuite/multi-memory/simple.wast
+++ b/tests/misc_testsuite/multi-memory/simple.wast
@@ -5,20 +5,20 @@
   (func (export "store1") (param i32 i64)
       local.get 0
       local.get 1
-      i64.store $m1)
+      i64.store (memory $m1))
 
   (func (export "store2") (param i32 i64)
       local.get 0
       local.get 1
-      i64.store $m2)
+      i64.store (memory $m2))
 
   (func (export "load1") (param i32) (result i64)
       local.get 0
-      i64.load $m1)
+      i64.load (memory $m1))
 
   (func (export "load2") (param i32) (result i64)
       local.get 0
-      i64.load $m2)
+      i64.load (memory $m2))
 )
 
 (invoke "store1" (i32.const 0) (i64.const 1))
@@ -64,20 +64,20 @@
   (func (export "store1") (param i32 i64)
       local.get 0
       local.get 1
-      i64.store $m1)
+      i64.store (memory $m1))
 
   (func (export "store2") (param i32 i64)
       local.get 0
       local.get 1
-      i64.store $m2)
+      i64.store (memory $m2))
 
   (func (export "load1") (param i32) (result i64)
       local.get 0
-      i64.load $m1)
+      i64.load (memory $m1))
 
   (func (export "load2") (param i32) (result i64)
       local.get 0
-      i64.load $m2)
+      i64.load (memory $m2))
 )
 
 (invoke "store1" (i32.const 0) (i64.const 1))
@@ -92,11 +92,11 @@
 
   (func (export "grow1") (param i32) (result i32)
       local.get 0
-      memory.grow $m1)
+      memory.grow (memory $m1))
 
   (func (export "grow2") (param i32) (result i32)
       local.get 0
-      memory.grow $m2)
+      memory.grow (memory $m2))
 
   (func (export "size1") (result i32) memory.size $m1)
   (func (export "size2") (result i32) memory.size $m2)
@@ -128,7 +128,7 @@
       i32.const 4
       memory.init $d $m2
       i32.const 1
-      i32.load $m2)
+      i32.load (memory $m2))
 
   (data $d "\01\00\00\00" "\02\00\00\00")
 )
@@ -154,7 +154,7 @@
       i32.const 2
       memory.fill $m2
       i32.const 1
-      i32.load $m2)
+      i32.load (memory $m2))
 )
 
 (assert_return (invoke "fill1") (i32.const 0x01010101))