Merge remote-tracking branch 'origin/main' into pch/wasi_common_cap_std

.github/workflows/main.yml

@@ -274,6 +274,7 @@ jobs:
             --exclude lightbeam \
             --exclude wasmtime-lightbeam \
             --exclude wasmtime-wasi-nn \
+            --exclude wasmtime-wasi-crypto \
             --exclude peepmatic \
             --exclude peepmatic-automata \
             --exclude peepmatic-fuzzing \
@@ -292,6 +293,14 @@ jobs:
       env:
         RUST_BACKTRACE: 1
 
+    # Test debug (DWARF) related functionality on new backend.
+    - run: |
+        sudo apt-get update && sudo apt-get install -y gdb
+        cargo test --features experimental_x64 test_debug_dwarf -- --ignored --test-threads 1 --test debug::
+      if: matrix.os == 'ubuntu-latest'
+      env:
+        RUST_BACKTRACE: 1
+
     # Build and test lightbeam. Note that
     # Lightbeam tests fail right now, but we don't want to block on that.
     - run: cargo build --package lightbeam
@@ -343,6 +352,21 @@ jobs:
         env:
           RUST_BACKTRACE: 1
 
+  # Build and test the wasi-crypto module.
+  test_wasi_crypto:
+    name: Test wasi-crypto module
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - run: rustup target add wasm32-wasi
+    - name: Install Rust
+      run: rustup update stable && rustup default stable
+    - run: ./ci/run-wasi-crypto-example.sh
+      env:
+        RUST_BACKTRACE: 1
+
   # Verify that cranelift's code generation is deterministic
   meta_determinist_check:
     name: Meta deterministic check
@@ -451,6 +475,7 @@ jobs:
             --exclude lightbeam \
             --exclude wasmtime-lightbeam \
             --exclude wasmtime-wasi-nn \
+            --exclude wasmtime-wasi-crypto \
             --exclude peepmatic \
             --exclude peepmatic-automata \
             --exclude peepmatic-fuzzing \

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "crates/wasi-nn/spec"]
 	path = crates/wasi-nn/spec
 	url = https://github.com/WebAssembly/wasi-nn
+[submodule "crates/wasi-crypto/spec"]
+	path = crates/wasi-crypto/spec
+	url = https://github.com/WebAssembly/wasi-crypto.git

Cargo.toml

@@ -30,22 +30,23 @@ wasmtime-jit = { path = "crates/jit", version = "0.22.0" }
 wasmtime-obj = { path = "crates/obj", version = "0.22.0" }
 wasmtime-wast = { path = "crates/wast", version = "0.22.0" }
 wasmtime-wasi = { path = "crates/wasi", version = "0.22.0" }
+wasmtime-wasi-crypto = { path = "crates/wasi-crypto", version = "0.22.0", optional = true }
 wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "0.22.0", optional = true }
 wasi-common = { path = "crates/wasi-common", version = "0.22.0" }
 wasi-cap-std-sync = { path = "crates/wasi-common/cap-std-sync", version = "0.22.0" }
 structopt = { version = "0.3.5", features = ["color", "suggestions"] }
-object = { version = "0.22.0", default-features = false, features = ["write"] }
+object = { version = "0.23.0", default-features = false, features = ["write"] }
 anyhow = "1.0.19"
 target-lexicon = { version = "0.11.0", default-features = false }
 pretty_env_logger = "0.4.0"
 file-per-thread-logger = "0.1.1"
-wat = "1.0.30"
+wat = "1.0.33"
 libc = "0.2.60"
 log = "0.4.8"
 rayon = "1.2.1"
 humantime = "2.0.0"
-wasmparser = "0.71.0"
-cap-std = "0.11.0"
+wasmparser = "0.73.0"
+cap-std = "0.12"
 
 [dev-dependencies]
 env_logger = "0.8.1"
@@ -56,6 +57,7 @@ test-programs = { path = "crates/test-programs" }
 wasmtime-fuzzing = { path = "crates/fuzzing" }
 wasmtime-runtime = { path = "crates/runtime" }
 tracing-subscriber = "0.2.0"
+wast = "32.0.0"
 
 [build-dependencies]
 anyhow = "1.0.19"
@@ -88,6 +90,7 @@ default = ["jitdump", "wasmtime/wat", "wasmtime/parallel-compilation"]
 lightbeam = ["wasmtime/lightbeam"]
 jitdump = ["wasmtime/jitdump"]
 vtune = ["wasmtime/vtune"]
+wasi-crypto = ["wasmtime-wasi-crypto"]
 wasi-nn = ["wasmtime-wasi-nn"]
 
 # Try the experimental, work-in-progress new x86_64 backend. This is not stable

build.rs

@@ -176,10 +176,52 @@ fn write_testsuite_tests(
 
 /// For experimental_x64 backend features that are not supported yet, mark tests as panicking, so
 /// they stop "passing" once the features are properly implemented.
-///
-/// TODO(#2470): removed all tests from this set as we are disabling x64 SIMD tests unconditionally
-/// instead until we resolve a nondeterminism bug. Restore when fixed.
-fn experimental_x64_should_panic(_testsuite: &str, _testname: &str, _strategy: &str) -> bool {
+fn experimental_x64_should_panic(testsuite: &str, testname: &str, strategy: &str) -> bool {
+    if !cfg!(feature = "experimental_x64") || strategy != "Cranelift" {
+        return false;
+    }
+
+    match (testsuite, testname) {
+        ("simd", "simd_address") => return false,
+        ("simd", "simd_align") => return false,
+        ("simd", "simd_bitwise") => return false,
+        ("simd", "simd_bit_shift") => return false,
+        ("simd", "simd_boolean") => return false,
+        ("simd", "simd_const") => return false,
+        ("simd", "simd_i8x16_arith") => return false,
+        ("simd", "simd_i8x16_arith2") => return false,
+        ("simd", "simd_i8x16_cmp") => return false,
+        ("simd", "simd_i8x16_sat_arith") => return false,
+        ("simd", "simd_i16x8_arith") => return false,
+        ("simd", "simd_i16x8_arith2") => return false,
+        ("simd", "simd_i16x8_cmp") => return false,
+        ("simd", "simd_i16x8_sat_arith") => return false,
+        ("simd", "simd_i32x4_arith") => return false,
+        ("simd", "simd_i32x4_arith2") => return false,
+        ("simd", "simd_i32x4_cmp") => return false,
+        ("simd", "simd_i32x4_dot_i16x8") => return false,
+        ("simd", "simd_i64x2_arith") => return false,
+        ("simd", "simd_f32x4") => return false,
+        ("simd", "simd_f32x4_arith") => return false,
+        ("simd", "simd_f32x4_cmp") => return false,
+        ("simd", "simd_f32x4_pmin_pmax") => return false,
+        ("simd", "simd_f64x2") => return false,
+        ("simd", "simd_f64x2_arith") => return false,
+        ("simd", "simd_f64x2_cmp") => return false,
+        ("simd", "simd_f64x2_pmin_pmax") => return false,
+        ("simd", "simd_lane") => return false,
+        ("simd", "simd_load") => return false,
+        ("simd", "simd_load_extend") => return false,
+        ("simd", "simd_load_splat") => return false,
+        ("simd", "simd_load_zero") => return false,
+        ("simd", "simd_splat") => return false,
+        ("simd", "simd_store") => return false,
+        ("simd", "simd_conversions") => return false,
+        ("simd", "simd_f32x4_rounding") => return false,
+        ("simd", "simd_f64x2_rounding") => return false,
+        ("simd", _) => return true,
+        _ => {}
+    }
     false
 }
 
@@ -201,14 +243,12 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
                 return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "x86_64";
             }
 
-            // Ignore all x64 SIMD tests for now (#2470).
-            ("simd", _) if cfg!(feature = "experimental_x64") => {
-                return env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "x86_64";
-            }
+            // Waiting for an update to the spec testsuite to not use old
+            // instruction names.
+            ("simd", "simd_boolean") | ("simd", "simd_lane") => return true,
 
             // These are only implemented on aarch64 and x64.
-            ("simd", "simd_boolean")
-            | ("simd", "simd_f32x4_pmin_pmax")
+            ("simd", "simd_f32x4_pmin_pmax")
             | ("simd", "simd_f64x2_pmin_pmax")
             | ("simd", "simd_f32x4_rounding")
             | ("simd", "simd_f64x2_rounding")

ci/run-experimental-x64-ci.sh

@@ -14,6 +14,7 @@ cargo $CARGO_VERSION \
             --all \
             --exclude wasmtime-lightbeam \
             --exclude wasmtime-wasi-nn \
+            --exclude wasmtime-wasi-crypto \
             --exclude peepmatic \
             --exclude peepmatic-automata \
             --exclude peepmatic-fuzzing \

ci/run-wasi-crypto-example.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+RUST_BINDINGS="crates/wasi-crypto/spec/implementations/bindings/rust"
+pushd "$RUST_BINDINGS"
+cargo build --release --target=wasm32-wasi
+popd
+
+cargo run --features wasi-crypto -- run "$RUST_BINDINGS/target/wasm32-wasi/release/wasi-crypto-guest.wasm"

cranelift/README.md

@@ -19,7 +19,7 @@ For more information, see [the documentation](docs/index.md).
 For an example of how to use the JIT, see the [JIT Demo], which
 implements a toy language.
 
-[JIT Demo]: https://github.com/bytecodealliance/simplejit-demo
+[JIT Demo]: https://github.com/bytecodealliance/cranelift-jit-demo
 
 For an example of how to use Cranelift to run WebAssembly code, see
 [Wasmtime], which implements a standalone, embeddable, VM using Cranelift.

cranelift/codegen/Cargo.toml

@@ -30,7 +30,7 @@ peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, versi
 peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.69.0" }
 regalloc = { version = "0.0.31" }
 souper-ir = { version = "2.1.0", optional = true }
-wast = { version = "29.0.0", optional = true }
+wast = { version = "32.0.0", optional = true }
 # It is a goal of the cranelift-codegen crate to have minimal external dependencies.
 # Please don't add any unless they are essential to the task of creating binary
 # machine code. Integration tests that need external dependencies can be
@@ -74,7 +74,7 @@ all-arch = [
 ]
 
 # For dependent crates that want to serialize some parts of cranelift
-enable-serde = ["serde"]
+enable-serde = ["serde", "regalloc/enable-serde"]
 
 # Allow snapshotting regalloc test cases. Useful only to report bad register
 # allocation failures, or for regalloc.rs developers.

cranelift/codegen/meta/src/gen_settings.rs

@@ -418,7 +418,7 @@ fn gen_display(group: &SettingGroup, fmt: &mut Formatter) {
 
 fn gen_group(group: &SettingGroup, parent: ParentGroup, fmt: &mut Formatter) {
     // Generate struct.
-    fmtln!(fmt, "#[derive(Clone)]");
+    fmtln!(fmt, "#[derive(Clone, Hash)]");
     fmt.doc_comment(format!("Flags group `{}`.", group.name));
     fmtln!(fmt, "pub struct Flags {");
     fmt.indent(|fmt| {

cranelift/codegen/src/context.rs

@@ -473,6 +473,7 @@ impl Context {
         Ok(build_value_labels_ranges::<ComparableSourceLoc>(
             &self.func,
             &self.regalloc,
+            self.mach_compile_result.as_ref(),
             isa,
         ))
     }

cranelift/codegen/src/ir/mod.rs

@@ -58,6 +58,7 @@ pub use crate::ir::table::TableData;
 pub use crate::ir::trapcode::TrapCode;
 pub use crate::ir::types::Type;
 pub use crate::ir::valueloc::{ArgumentLoc, ValueLoc};
+pub use crate::value_label::LabelValueLoc;
 pub use cranelift_codegen_shared::condcodes;
 
 use crate::binemit;

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -4,6 +4,8 @@ use crate::ir;
 use crate::ir::types;
 use crate::ir::types::*;
 use crate::ir::MemFlags;
+use crate::ir::Opcode;
+use crate::ir::{ExternalName, LibCall};
 use crate::isa;
 use crate::isa::aarch64::{inst::EmitState, inst::*};
 use crate::machinst::*;
@@ -76,41 +78,41 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
         match &param.purpose {
             &ir::ArgumentPurpose::VMContext => {
                 // This is SpiderMonkey's `WasmTlsReg`.
-                Some(ABIArg::Reg(
-                    ValueRegs::one(xreg(BALDRDASH_TLS_REG).to_real_reg()),
-                    ir::types::I64,
-                    param.extension,
-                    param.purpose,
-                ))
+                Some(ABIArg::Reg {
+                    regs: ValueRegs::one(xreg(BALDRDASH_TLS_REG).to_real_reg()),
+                    ty: ir::types::I64,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::SignatureId => {
                 // This is SpiderMonkey's `WasmTableCallSigReg`.
-                Some(ABIArg::Reg(
-                    ValueRegs::one(xreg(BALDRDASH_SIG_REG).to_real_reg()),
-                    ir::types::I64,
-                    param.extension,
-                    param.purpose,
-                ))
+                Some(ABIArg::Reg {
+                    regs: ValueRegs::one(xreg(BALDRDASH_SIG_REG).to_real_reg()),
+                    ty: ir::types::I64,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::CalleeTLS => {
                 // This is SpiderMonkey's callee TLS slot in the extended frame of Wasm's ABI-2020.
                 assert!(call_conv == isa::CallConv::Baldrdash2020);
-                Some(ABIArg::Stack(
-                    BALDRDASH_CALLEE_TLS_OFFSET,
-                    ir::types::I64,
-                    ir::ArgumentExtension::None,
-                    param.purpose,
-                ))
+                Some(ABIArg::Stack {
+                    offset: BALDRDASH_CALLEE_TLS_OFFSET,
+                    ty: ir::types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::CallerTLS => {
                 // This is SpiderMonkey's caller TLS slot in the extended frame of Wasm's ABI-2020.
                 assert!(call_conv == isa::CallConv::Baldrdash2020);
-                Some(ABIArg::Stack(
-                    BALDRDASH_CALLER_TLS_OFFSET,
-                    ir::types::I64,
-                    ir::ArgumentExtension::None,
-                    param.purpose,
-                ))
+                Some(ABIArg::Stack {
+                    offset: BALDRDASH_CALLER_TLS_OFFSET,
+                    ty: ir::types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: param.purpose,
+                })
             }
             _ => None,
         }
@@ -208,7 +210,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
                 | &ir::ArgumentPurpose::StackLimit
                 | &ir::ArgumentPurpose::SignatureId
                 | &ir::ArgumentPurpose::CallerTLS
-                | &ir::ArgumentPurpose::CalleeTLS => {}
+                | &ir::ArgumentPurpose::CalleeTLS
+                | &ir::ArgumentPurpose::StructReturn
+                | &ir::ArgumentPurpose::StructArgument(_) => {}
                 _ => panic!(
                     "Unsupported argument purpose {:?} in signature: {:?}",
                     param.purpose, params
@@ -233,18 +237,28 @@ impl ABIMachineSpec for AArch64MachineDeps {
             if let Some(param) = try_fill_baldrdash_reg(call_conv, param) {
                 assert!(rc == RegClass::I64);
                 ret.push(param);
+            } else if let ir::ArgumentPurpose::StructArgument(size) = param.purpose {
+                let offset = next_stack as i64;
+                let size = size as u64;
+                assert!(size % 8 == 0, "StructArgument size is not properly aligned");
+                next_stack += size;
+                ret.push(ABIArg::StructArg {
+                    offset,
+                    size,
+                    purpose: param.purpose,
+                });
             } else if *next_reg < max_per_class_reg_vals && remaining_reg_vals > 0 {
                 let reg = match rc {
                     RegClass::I64 => xreg(*next_reg),
                     RegClass::V128 => vreg(*next_reg),
                     _ => unreachable!(),
                 };
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(reg.to_real_reg()),
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(reg.to_real_reg()),
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
                 *next_reg += 1;
                 remaining_reg_vals -= 1;
             } else {
@@ -255,12 +269,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
                 // Align.
                 debug_assert!(size.is_power_of_two());
                 next_stack = (next_stack + size - 1) & !(size - 1);
-                ret.push(ABIArg::Stack(
-                    next_stack as i64,
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
+                ret.push(ABIArg::Stack {
+                    offset: next_stack as i64,
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
                 next_stack += size;
             }
         }
@@ -272,19 +286,19 @@ impl ABIMachineSpec for AArch64MachineDeps {
         let extra_arg = if add_ret_area_ptr {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
             if next_xreg < max_per_class_reg_vals && remaining_reg_vals > 0 {
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(xreg(next_xreg).to_real_reg()),
-                    I64,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(xreg(next_xreg).to_real_reg()),
+                    ty: I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
             } else {
-                ret.push(ABIArg::Stack(
-                    next_stack as i64,
-                    I64,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Stack {
+                    offset: next_stack as i64,
+                    ty: I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
                 next_stack += 8;
             }
             Some(ret.len() - 1)
@@ -708,6 +722,34 @@ impl ABIMachineSpec for AArch64MachineDeps {
         insts
     }
 
+    fn gen_memcpy(
+        call_conv: isa::CallConv,
+        dst: Reg,
+        src: Reg,
+        size: usize,
+    ) -> SmallVec<[Self::I; 8]> {
+        // Baldrdash should not use struct args.
+        assert!(!call_conv.extends_baldrdash());
+        let mut insts = SmallVec::new();
+        let arg0 = writable_xreg(0);
+        let arg1 = writable_xreg(1);
+        let arg2 = writable_xreg(2);
+        insts.push(Inst::gen_move(arg0, dst, I64));
+        insts.push(Inst::gen_move(arg1, src, I64));
+        insts.extend(Inst::load_constant(arg2, size as u64).into_iter());
+        insts.push(Inst::Call {
+            info: Box::new(CallInfo {
+                dest: ExternalName::LibCall(LibCall::Memcpy),
+                uses: vec![arg0.to_reg(), arg1.to_reg(), arg2.to_reg()],
+                defs: Self::get_regs_clobbered_by_call(call_conv),
+                opcode: Opcode::Call,
+                caller_callconv: call_conv,
+                callee_callconv: call_conv,
+            }),
+        });
+        insts
+    }
+
     fn get_number_of_spillslots_for_value(rc: RegClass, ty: Type) -> u32 {
         // We allocate in terms of 8-byte slots.
         match (rc, ty) {

cranelift/codegen/src/isa/aarch64/inst/args.rs

@@ -601,6 +601,14 @@ impl ScalarSize {
         }
     }
 
+    /// Convert from an integer operand size.
+    pub fn from_operand_size(size: OperandSize) -> ScalarSize {
+        match size {
+            OperandSize::Size32 => ScalarSize::Size32,
+            OperandSize::Size64 => ScalarSize::Size64,
+        }
+    }
+
     /// Convert from a type into the smallest size that fits.
     pub fn from_ty(ty: Type) -> ScalarSize {
         Self::from_bits(ty_bits(ty))

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -1463,12 +1463,18 @@ impl MachInstEmit for Inst {
                         debug_assert!(size == VectorSize::Size32x4 || size == VectorSize::Size64x2);
                         (0b0, 0b11000, enc_size | 0b10)
                     }
+                    VecMisc2::Cnt => {
+                        debug_assert!(size == VectorSize::Size8x8 || size == VectorSize::Size8x16);
+                        (0b0, 0b00101, enc_size)
+                    }
                 };
                 sink.put4(enc_vec_rr_misc((q << 1) | u, size, bits_12_16, rd, rn));
             }
             &Inst::VecLanes { op, rd, rn, size } => {
                 let (q, size) = match size {
+                    VectorSize::Size8x8 => (0b0, 0b00),
                     VectorSize::Size8x16 => (0b1, 0b00),
+                    VectorSize::Size16x4 => (0b0, 0b01),
                     VectorSize::Size16x8 => (0b1, 0b01),
                     VectorSize::Size32x4 => (0b1, 0b10),
                     _ => unreachable!(),
@@ -2365,6 +2371,9 @@ impl MachInstEmit for Inst {
                     sink.bind_label(jump_around_label);
                 }
             }
+            &Inst::ValueLabelMarker { .. } => {
+                // Nothing; this is only used to compute debug info.
+            }
         }
 
         let end_off = sink.cur_offset();

cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs

@@ -3792,6 +3792,28 @@ fn test_aarch64_binemit() {
         "frintp v12.2d, v17.2d",
     ));
 
+    insns.push((
+        Inst::VecMisc {
+            op: VecMisc2::Cnt,
+            rd: writable_vreg(23),
+            rn: vreg(5),
+            size: VectorSize::Size8x8,
+        },
+        "B758200E",
+        "cnt v23.8b, v5.8b",
+    ));
+
+    insns.push((
+        Inst::VecLanes {
+            op: VecLanesOp::Uminv,
+            rd: writable_vreg(0),
+            rn: vreg(31),
+            size: VectorSize::Size8x8,
+        },
+        "E0AB312E",
+        "uminv b0, v31.8b",
+    ));
+
     insns.push((
         Inst::VecLanes {
             op: VecLanesOp::Uminv,
@@ -3836,6 +3858,17 @@ fn test_aarch64_binemit() {
         "addv b2, v29.16b",
     ));
 
+    insns.push((
+        Inst::VecLanes {
+            op: VecLanesOp::Addv,
+            rd: writable_vreg(15),
+            rn: vreg(7),
+            size: VectorSize::Size16x4,
+        },
+        "EFB8710E",
+        "addv h15, v7.4h",
+    ));
+
     insns.push((
         Inst::VecLanes {
             op: VecLanesOp::Addv,

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -7,7 +7,7 @@ use crate::binemit::CodeOffset;
 use crate::ir::types::{
     B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64,
 };
-use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type};
+use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type, ValueLabel};
 use crate::isa::CallConv;
 use crate::machinst::*;
 use crate::{settings, CodegenError, CodegenResult};
@@ -331,6 +331,8 @@ pub enum VecMisc2 {
     Frintm,
     /// Floating point round to integral, rounding towards plus infinity
     Frintp,
+    /// Population count per byte
+    Cnt,
 }
 
 /// A Vector narrowing operation with two registers.
@@ -1208,6 +1210,12 @@ pub enum Inst {
         /// The needed space before the next deadline.
         needed_space: CodeOffset,
     },
+
+    /// A definition of a value label.
+    ValueLabelMarker {
+        reg: Reg,
+        label: ValueLabel,
+    },
 }
 
 fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize {
@@ -2015,6 +2023,9 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
             memarg_regs(mem, collector);
         }
         &Inst::VirtualSPOffsetAdj { .. } => {}
+        &Inst::ValueLabelMarker { reg, .. } => {
+            collector.add_use(reg);
+        }
         &Inst::EmitIsland { .. } => {}
     }
 }
@@ -2765,6 +2776,9 @@ fn aarch64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
         }
         &mut Inst::VirtualSPOffsetAdj { .. } => {}
         &mut Inst::EmitIsland { .. } => {}
+        &mut Inst::ValueLabelMarker { ref mut reg, .. } => {
+            map_use(mapper, reg);
+        }
     }
 }
 
@@ -2893,11 +2907,10 @@ impl MachInst for Inst {
         }
     }
 
-    fn gen_zero_len_nop() -> Inst {
-        Inst::Nop0
-    }
-
     fn gen_nop(preferred_size: usize) -> Inst {
+        if preferred_size == 0 {
+            return Inst::Nop0;
+        }
         // We can't give a NOP (or any insn) < 4 bytes.
         assert!(preferred_size >= 4);
         Inst::Nop4
@@ -2960,6 +2973,17 @@ impl MachInst for Inst {
     fn ref_type_regclass(_: &settings::Flags) -> RegClass {
         RegClass::I64
     }
+
+    fn gen_value_label_marker(label: ValueLabel, reg: Reg) -> Self {
+        Inst::ValueLabelMarker { label, reg }
+    }
+
+    fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> {
+        match self {
+            Inst::ValueLabelMarker { label, reg } => Some((*label, *reg)),
+            _ => None,
+        }
+    }
 }
 
 //=============================================================================
@@ -3752,6 +3776,7 @@ impl Inst {
                     VecMisc2::Frintz => ("frintz", size),
                     VecMisc2::Frintm => ("frintm", size),
                     VecMisc2::Frintp => ("frintp", size),
+                    VecMisc2::Cnt => ("cnt", size),
                 };
 
                 let rd_size = if is_shll { size.widen() } else { size };
@@ -4068,6 +4093,10 @@ impl Inst {
                 format!("virtual_sp_offset_adjust {}", offset)
             }
             &Inst::EmitIsland { needed_space } => format!("emit_island {}", needed_space),
+
+            &Inst::ValueLabelMarker { label, reg } => {
+                format!("value_label {:?}, {}", label, reg.show_rru(mb_rru))
+            }
         }
     }
 }

cranelift/codegen/src/isa/aarch64/lower.rs

@@ -1231,7 +1231,7 @@ impl LowerBackend for AArch64Backend {
     type MInst = Inst;
 
     fn lower<C: LowerCtx<I = Inst>>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> {
-        lower_inst::lower_insn_to_regs(ctx, ir_inst)
+        lower_inst::lower_insn_to_regs(ctx, ir_inst, &self.flags)
     }
 
     fn lower_branch_group<C: LowerCtx<I = Inst>>(

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -7,6 +7,7 @@ use crate::ir::Inst as IRInst;
 use crate::ir::{InstructionData, Opcode, TrapCode};
 use crate::machinst::lower::*;
 use crate::machinst::*;
+use crate::settings::Flags;
 use crate::{CodegenError, CodegenResult};
 
 use crate::isa::aarch64::abi::*;
@@ -24,6 +25,7 @@ use super::lower::*;
 pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
     ctx: &mut C,
     insn: IRInst,
+    flags: &Flags,
 ) -> CodegenResult<()> {
     let op = ctx.data(insn).opcode();
     let inputs = insn_inputs(ctx, insn);
@@ -960,143 +962,57 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         }
 
         Opcode::Popcnt => {
-            // Lower popcount using the following algorithm:
-            //
-            //   x -= (x >> 1) & 0x5555555555555555
-            //   x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333)
-            //   x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f
-            //   x += x << 8
-            //   x += x << 16
-            //   x += x << 32
-            //   x >> 56
-            let ty = ty.unwrap();
             let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            // FIXME(#1537): zero-extend 8/16/32-bit operands only to 32 bits,
-            // and fix the sequence below to work properly for this.
-            let narrow_mode = NarrowValueMode::ZeroExtend64;
-            let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-            let tmp = ctx.alloc_tmp(I64).only_reg().unwrap();
+            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
+            let ty = ty.unwrap();
+            let size = ScalarSize::from_operand_size(OperandSize::from_ty(ty));
+            let tmp = ctx.alloc_tmp(I8X16).only_reg().unwrap();
 
-            // If this is a 32-bit Popcnt, use Lsr32 to clear the top 32 bits of the register, then
-            // the rest of the code is identical to the 64-bit version.
-            // lsr [wx]d, [wx]n, #1
-            ctx.emit(Inst::AluRRImmShift {
-                alu_op: choose_32_64(ty, ALUOp::Lsr32, ALUOp::Lsr64),
-                rd: rd,
+            // fmov tmp, rn
+            // cnt tmp.8b, tmp.8b
+            // addp tmp.8b, tmp.8b, tmp.8b / addv tmp, tmp.8b / (no instruction for 8-bit inputs)
+            // umov rd, tmp.b[0]
+
+            ctx.emit(Inst::MovToFpu {
+                rd: tmp,
                 rn: rn,
-                immshift: ImmShift::maybe_from_u64(1).unwrap(),
+                size,
             });
-
-            // and xd, xd, #0x5555555555555555
-            ctx.emit(Inst::AluRRImmLogic {
-                alu_op: ALUOp::And64,
-                rd: rd,
-                rn: rd.to_reg(),
-                imml: ImmLogic::maybe_from_u64(0x5555555555555555, I64).unwrap(),
-            });
-
-            // sub xd, xn, xd
-            ctx.emit(Inst::AluRRR {
-                alu_op: ALUOp::Sub64,
-                rd: rd,
-                rn: rn,
-                rm: rd.to_reg(),
-            });
-
-            // and xt, xd, #0x3333333333333333
-            ctx.emit(Inst::AluRRImmLogic {
-                alu_op: ALUOp::And64,
+            ctx.emit(Inst::VecMisc {
+                op: VecMisc2::Cnt,
                 rd: tmp,
-                rn: rd.to_reg(),
-                imml: ImmLogic::maybe_from_u64(0x3333333333333333, I64).unwrap(),
+                rn: tmp.to_reg(),
+                size: VectorSize::Size8x8,
             });
 
-            // lsr xd, xd, #2
-            ctx.emit(Inst::AluRRImmShift {
-                alu_op: ALUOp::Lsr64,
-                rd: rd,
-                rn: rd.to_reg(),
-                immshift: ImmShift::maybe_from_u64(2).unwrap(),
-            });
-
-            // and xd, xd, #0x3333333333333333
-            ctx.emit(Inst::AluRRImmLogic {
-                alu_op: ALUOp::And64,
-                rd: rd,
-                rn: rd.to_reg(),
-                imml: ImmLogic::maybe_from_u64(0x3333333333333333, I64).unwrap(),
-            });
-
-            // add xt, xd, xt
-            ctx.emit(Inst::AluRRR {
-                alu_op: ALUOp::Add64,
-                rd: tmp,
-                rn: rd.to_reg(),
-                rm: tmp.to_reg(),
-            });
-
-            // add xt, xt, xt LSR #4
-            ctx.emit(Inst::AluRRRShift {
-                alu_op: ALUOp::Add64,
+            match ScalarSize::from_ty(ty) {
+                ScalarSize::Size8 => {}
+                ScalarSize::Size16 => {
+                    // ADDP is usually cheaper than ADDV.
+                    ctx.emit(Inst::VecRRR {
+                        alu_op: VecALUOp::Addp,
                         rd: tmp,
                         rn: tmp.to_reg(),
                         rm: tmp.to_reg(),
-                shiftop: ShiftOpAndAmt::new(
-                    ShiftOp::LSR,
-                    ShiftOpShiftImm::maybe_from_shift(4).unwrap(),
-                ),
+                        size: VectorSize::Size8x8,
                     });
-
-            // and xt, xt, #0x0f0f0f0f0f0f0f0f
-            ctx.emit(Inst::AluRRImmLogic {
-                alu_op: ALUOp::And64,
+                }
+                ScalarSize::Size32 | ScalarSize::Size64 => {
+                    ctx.emit(Inst::VecLanes {
+                        op: VecLanesOp::Addv,
                         rd: tmp,
                         rn: tmp.to_reg(),
-                imml: ImmLogic::maybe_from_u64(0x0f0f0f0f0f0f0f0f, I64).unwrap(),
+                        size: VectorSize::Size8x8,
                     });
+                }
+                sz => panic!("Unexpected scalar FP operand size: {:?}", sz),
+            }
 
-            // add xt, xt, xt, LSL #8
-            ctx.emit(Inst::AluRRRShift {
-                alu_op: ALUOp::Add64,
-                rd: tmp,
+            ctx.emit(Inst::MovFromVec {
+                rd,
                 rn: tmp.to_reg(),
-                rm: tmp.to_reg(),
-                shiftop: ShiftOpAndAmt::new(
-                    ShiftOp::LSL,
-                    ShiftOpShiftImm::maybe_from_shift(8).unwrap(),
-                ),
-            });
-
-            // add xt, xt, xt, LSL #16
-            ctx.emit(Inst::AluRRRShift {
-                alu_op: ALUOp::Add64,
-                rd: tmp,
-                rn: tmp.to_reg(),
-                rm: tmp.to_reg(),
-                shiftop: ShiftOpAndAmt::new(
-                    ShiftOp::LSL,
-                    ShiftOpShiftImm::maybe_from_shift(16).unwrap(),
-                ),
-            });
-
-            // add xt, xt, xt, LSL #32
-            ctx.emit(Inst::AluRRRShift {
-                alu_op: ALUOp::Add64,
-                rd: tmp,
-                rn: tmp.to_reg(),
-                rm: tmp.to_reg(),
-                shiftop: ShiftOpAndAmt::new(
-                    ShiftOp::LSL,
-                    ShiftOpShiftImm::maybe_from_shift(32).unwrap(),
-                ),
-            });
-
-            // lsr xd, xt, #56
-            ctx.emit(Inst::AluRRImmShift {
-                alu_op: ALUOp::Lsr64,
-                rd: rd,
-                rn: tmp.to_reg(),
-                immshift: ImmShift::maybe_from_u64(56).unwrap(),
+                idx: 0,
+                size: VectorSize::Size8x16,
             });
         }
 
@@ -1803,7 +1719,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert!(inputs.len() == sig.params.len());
                     assert!(outputs.len() == sig.returns.len());
                     (
-                        AArch64ABICaller::from_func(sig, &extname, dist, caller_conv)?,
+                        AArch64ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?,
                         &inputs[..],
                     )
                 }
@@ -1813,7 +1729,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert!(inputs.len() - 1 == sig.params.len());
                     assert!(outputs.len() == sig.returns.len());
                     (
-                        AArch64ABICaller::from_ptr(sig, ptr, op, caller_conv)?,
+                        AArch64ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?,
                         &inputs[1..],
                     )
                 }
@@ -1822,8 +1738,9 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
             abi.emit_stack_pre_adjust(ctx);
             assert!(inputs.len() == abi.num_args());
-            for (i, input) in inputs.iter().enumerate() {
-                let arg_reg = put_input_in_reg(ctx, *input, NarrowValueMode::None);
+            for i in abi.get_copy_to_arg_order() {
+                let input = inputs[i];
+                let arg_reg = put_input_in_reg(ctx, input, NarrowValueMode::None);
                 abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg));
             }
             abi.emit_call(ctx);

cranelift/codegen/src/isa/aarch64/mod.rs

@@ -8,6 +8,7 @@ use crate::result::CodegenResult;
 use crate::settings;
 
 use alloc::boxed::Box;
+use core::hash::{Hash, Hasher};
 
 use regalloc::{PrettyPrint, RealRegUniverse};
 use target_lexicon::{Aarch64Architecture, Architecture, Triple};
@@ -79,6 +80,7 @@ impl MachBackend for AArch64Backend {
             frame_size,
             disasm,
             unwind_info,
+            value_labels_ranges: None,
         })
     }
 
@@ -94,6 +96,10 @@ impl MachBackend for AArch64Backend {
         &self.flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.flags.hash(&mut hasher);
+    }
+
     fn reg_universe(&self) -> &RealRegUniverse {
         &self.reg_universe
     }

cranelift/codegen/src/isa/arm32/abi.rs

@@ -81,12 +81,12 @@ impl ABIMachineSpec for Arm32MachineDeps {
             if next_rreg < max_reg_val {
                 let reg = rreg(next_rreg);
 
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(reg.to_real_reg()),
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(reg.to_real_reg()),
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
                 next_rreg += 1;
             } else {
                 // Arguments are stored on stack in reversed order.
@@ -101,12 +101,12 @@ impl ABIMachineSpec for Arm32MachineDeps {
         let extra_arg = if add_ret_area_ptr {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
             if next_rreg < max_reg_val {
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(rreg(next_rreg).to_real_reg()),
-                    I32,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(rreg(next_rreg).to_real_reg()),
+                    ty: I32,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
             } else {
                 stack_args.push((
                     I32,
@@ -124,12 +124,12 @@ impl ABIMachineSpec for Arm32MachineDeps {
         let max_stack = next_stack;
         for (ty, ext, purpose) in stack_args.into_iter().rev() {
             next_stack -= 4;
-            ret.push(ABIArg::Stack(
-                (max_stack - next_stack) as i64,
+            ret.push(ABIArg::Stack {
+                offset: (max_stack - next_stack) as i64,
                 ty,
-                ext,
+                extension: ext,
                 purpose,
-            ));
+            });
         }
         assert_eq!(next_stack, 0);
 
@@ -426,6 +426,15 @@ impl ABIMachineSpec for Arm32MachineDeps {
         insts
     }
 
+    fn gen_memcpy(
+        _call_conv: isa::CallConv,
+        _dst: Reg,
+        _src: Reg,
+        _size: usize,
+    ) -> SmallVec<[Self::I; 8]> {
+        unimplemented!("StructArgs not implemented for ARM32 yet");
+    }
+
     fn get_number_of_spillslots_for_value(rc: RegClass, _ty: Type) -> u32 {
         match rc {
             RegClass::I32 => 1,

cranelift/codegen/src/isa/arm32/inst/mod.rs

@@ -831,11 +831,10 @@ impl MachInst for Inst {
         }
     }
 
-    fn gen_zero_len_nop() -> Inst {
-        Inst::Nop0
-    }
-
     fn gen_nop(preferred_size: usize) -> Inst {
+        if preferred_size == 0 {
+            return Inst::Nop0;
+        }
         assert!(preferred_size >= 2);
         Inst::Nop2
     }

cranelift/codegen/src/isa/arm32/lower.rs

@@ -224,7 +224,7 @@ impl LowerBackend for Arm32Backend {
     type MInst = Inst;
 
     fn lower<C: LowerCtx<I = Inst>>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> {
-        lower_inst::lower_insn_to_regs(ctx, ir_inst)
+        lower_inst::lower_insn_to_regs(ctx, ir_inst, &self.flags)
     }
 
     fn lower_branch_group<C: LowerCtx<I = Inst>>(

cranelift/codegen/src/isa/arm32/lower_inst.rs

@@ -5,6 +5,7 @@ use crate::ir::Inst as IRInst;
 use crate::ir::Opcode;
 use crate::machinst::lower::*;
 use crate::machinst::*;
+use crate::settings::Flags;
 use crate::CodegenResult;
 
 use crate::isa::arm32::abi::*;
@@ -18,6 +19,7 @@ use super::lower::*;
 pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
     ctx: &mut C,
     insn: IRInst,
+    flags: &Flags,
 ) -> CodegenResult<()> {
     let op = ctx.data(insn).opcode();
     let inputs: SmallVec<[InsnInput; 4]> = (0..ctx.num_inputs(insn))
@@ -502,7 +504,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert_eq!(inputs.len(), sig.params.len());
                     assert_eq!(outputs.len(), sig.returns.len());
                     (
-                        Arm32ABICaller::from_func(sig, &extname, dist, caller_conv)?,
+                        Arm32ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?,
                         &inputs[..],
                     )
                 }
@@ -512,7 +514,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                     assert_eq!(inputs.len() - 1, sig.params.len());
                     assert_eq!(outputs.len(), sig.returns.len());
                     (
-                        Arm32ABICaller::from_ptr(sig, ptr, op, caller_conv)?,
+                        Arm32ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?,
                         &inputs[1..],
                     )
                 }

cranelift/codegen/src/isa/arm32/mod.rs

@@ -8,6 +8,7 @@ use crate::result::CodegenResult;
 use crate::settings;
 
 use alloc::boxed::Box;
+use core::hash::{Hash, Hasher};
 use regalloc::{PrettyPrint, RealRegUniverse};
 use target_lexicon::{Architecture, ArmArchitecture, Triple};
 
@@ -74,6 +75,7 @@ impl MachBackend for Arm32Backend {
             frame_size,
             disasm,
             unwind_info: None,
+            value_labels_ranges: None,
         })
     }
 
@@ -89,6 +91,10 @@ impl MachBackend for Arm32Backend {
         &self.flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.flags.hash(&mut hasher);
+    }
+
     fn reg_universe(&self) -> &RealRegUniverse {
         &self.reg_universe
     }

cranelift/codegen/src/isa/mod.rs

@@ -69,6 +69,7 @@ use alloc::boxed::Box;
 use core::any::Any;
 use core::fmt;
 use core::fmt::{Debug, Formatter};
+use core::hash::Hasher;
 use target_lexicon::{triple, Architecture, PointerWidth, Triple};
 use thiserror::Error;
 
@@ -265,6 +266,10 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
     /// Get the ISA-independent flags that were used to make this trait object.
     fn flags(&self) -> &settings::Flags;
 
+    /// Hashes all flags, both ISA-independent and ISA-specific, into the
+    /// specified hasher.
+    fn hash_all_flags(&self, hasher: &mut dyn Hasher);
+
     /// Get the default calling convention of this target.
     fn default_call_conv(&self) -> CallConv {
         CallConv::triple_default(self.triple())
@@ -325,6 +330,12 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
         Err(RegisterMappingError::UnsupportedArchitecture)
     }
 
+    #[cfg(feature = "unwind")]
+    /// Map a regalloc::Reg to its corresponding DWARF register.
+    fn map_regalloc_reg_to_dwarf(&self, _: ::regalloc::Reg) -> Result<u16, RegisterMappingError> {
+        Err(RegisterMappingError::UnsupportedArchitecture)
+    }
+
     /// Returns an iterator over legal encodings for the instruction.
     fn legal_encodings<'a>(
         &'a self,

cranelift/codegen/src/isa/riscv/mod.rs

@@ -19,6 +19,7 @@ use alloc::borrow::Cow;
 use alloc::boxed::Box;
 use core::any::Any;
 use core::fmt;
+use core::hash::{Hash, Hasher};
 use target_lexicon::{PointerWidth, Triple};
 
 #[allow(dead_code)]
@@ -69,6 +70,11 @@ impl TargetIsa for Isa {
         &self.shared_flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.shared_flags.hash(&mut hasher);
+        self.isa_flags.hash(&mut hasher);
+    }
+
     fn register_info(&self) -> RegInfo {
         registers::INFO.clone()
     }

cranelift/codegen/src/isa/x64/abi.rs

@@ -31,41 +31,41 @@ fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option<A
         match &param.purpose {
             &ir::ArgumentPurpose::VMContext => {
                 // This is SpiderMonkey's `WasmTlsReg`.
-                Some(ABIArg::Reg(
-                    ValueRegs::one(regs::r14().to_real_reg()),
-                    types::I64,
-                    param.extension,
-                    param.purpose,
-                ))
+                Some(ABIArg::Reg {
+                    regs: ValueRegs::one(regs::r14().to_real_reg()),
+                    ty: types::I64,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::SignatureId => {
                 // This is SpiderMonkey's `WasmTableCallSigReg`.
-                Some(ABIArg::Reg(
-                    ValueRegs::one(regs::r10().to_real_reg()),
-                    types::I64,
-                    param.extension,
-                    param.purpose,
-                ))
+                Some(ABIArg::Reg {
+                    regs: ValueRegs::one(regs::r10().to_real_reg()),
+                    ty: types::I64,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::CalleeTLS => {
                 // This is SpiderMonkey's callee TLS slot in the extended frame of Wasm's ABI-2020.
                 assert!(call_conv == isa::CallConv::Baldrdash2020);
-                Some(ABIArg::Stack(
-                    BALDRDASH_CALLEE_TLS_OFFSET,
-                    ir::types::I64,
-                    ir::ArgumentExtension::None,
-                    param.purpose,
-                ))
+                Some(ABIArg::Stack {
+                    offset: BALDRDASH_CALLEE_TLS_OFFSET,
+                    ty: ir::types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: param.purpose,
+                })
             }
             &ir::ArgumentPurpose::CallerTLS => {
                 // This is SpiderMonkey's caller TLS slot in the extended frame of Wasm's ABI-2020.
                 assert!(call_conv == isa::CallConv::Baldrdash2020);
-                Some(ABIArg::Stack(
-                    BALDRDASH_CALLER_TLS_OFFSET,
-                    ir::types::I64,
-                    ir::ArgumentExtension::None,
-                    param.purpose,
-                ))
+                Some(ABIArg::Stack {
+                    offset: BALDRDASH_CALLER_TLS_OFFSET,
+                    ty: ir::types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: param.purpose,
+                })
             }
             _ => None,
         }
@@ -131,49 +131,84 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                 | &ir::ArgumentPurpose::StackLimit
                 | &ir::ArgumentPurpose::SignatureId
                 | &ir::ArgumentPurpose::CalleeTLS
-                | &ir::ArgumentPurpose::CallerTLS => {}
+                | &ir::ArgumentPurpose::CallerTLS
+                | &ir::ArgumentPurpose::StructReturn
+                | &ir::ArgumentPurpose::StructArgument(_) => {}
                 _ => panic!(
                     "Unsupported argument purpose {:?} in signature: {:?}",
                     param.purpose, params
                 ),
             }
 
-            let intreg = in_int_reg(param.value_type);
-            let vecreg = in_vec_reg(param.value_type);
-            debug_assert!(intreg || vecreg);
-            debug_assert!(!(intreg && vecreg));
-
-            let (next_reg, candidate) = if intreg {
-                let candidate = match args_or_rets {
-                    ArgsOrRets::Args => get_intreg_for_arg_systemv(&call_conv, next_gpr),
-                    ArgsOrRets::Rets => get_intreg_for_retval_systemv(&call_conv, next_gpr, i),
-                };
-                debug_assert!(candidate
-                    .map(|r| r.get_class() == RegClass::I64)
-                    .unwrap_or(true));
-                (&mut next_gpr, candidate)
-            } else {
-                let candidate = match args_or_rets {
-                    ArgsOrRets::Args => get_fltreg_for_arg_systemv(&call_conv, next_vreg),
-                    ArgsOrRets::Rets => get_fltreg_for_retval_systemv(&call_conv, next_vreg, i),
-                };
-                debug_assert!(candidate
-                    .map(|r| r.get_class() == RegClass::V128)
-                    .unwrap_or(true));
-                (&mut next_vreg, candidate)
-            };
-
             if let Some(param) = try_fill_baldrdash_reg(call_conv, param) {
-                assert!(intreg);
                 ret.push(param);
-            } else if let Some(reg) = candidate {
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(reg.to_real_reg()),
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
-                *next_reg += 1;
+                continue;
+            }
+
+            if let ir::ArgumentPurpose::StructArgument(size) = param.purpose {
+                let offset = next_stack as i64;
+                let size = size as u64;
+                assert!(size % 8 == 0, "StructArgument size is not properly aligned");
+                next_stack += size;
+                ret.push(ABIArg::StructArg {
+                    offset,
+                    size,
+                    purpose: param.purpose,
+                });
+                continue;
+            }
+
+            // Find regclass(es) of the register(s) used to store a value of this type.
+            let (rcs, _) = Inst::rc_for_type(param.value_type)?;
+            let intreg = rcs[0] == RegClass::I64;
+            let num_regs = rcs.len();
+            assert!(num_regs <= 2);
+            if num_regs == 2 {
+                assert_eq!(rcs[0], rcs[1]);
+            }
+
+            let mut regs: SmallVec<[RealReg; 2]> = smallvec![];
+            for j in 0..num_regs {
+                let nextreg = if intreg {
+                    match args_or_rets {
+                        ArgsOrRets::Args => get_intreg_for_arg_systemv(&call_conv, next_gpr + j),
+                        ArgsOrRets::Rets => {
+                            get_intreg_for_retval_systemv(&call_conv, next_gpr + j, i + j)
+                        }
+                    }
+                } else {
+                    match args_or_rets {
+                        ArgsOrRets::Args => get_fltreg_for_arg_systemv(&call_conv, next_vreg + j),
+                        ArgsOrRets::Rets => {
+                            get_fltreg_for_retval_systemv(&call_conv, next_vreg + j, i + j)
+                        }
+                    }
+                };
+                if let Some(reg) = nextreg {
+                    regs.push(reg.to_real_reg());
+                } else {
+                    regs.clear();
+                    break;
+                }
+            }
+
+            if regs.len() > 0 {
+                let regs = match num_regs {
+                    1 => ValueRegs::one(regs[0]),
+                    2 => ValueRegs::two(regs[0], regs[1]),
+                    _ => panic!("More than two registers unexpected"),
+                };
+                ret.push(ABIArg::Reg {
+                    regs,
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
+                if intreg {
+                    next_gpr += num_regs;
+                } else {
+                    next_vreg += num_regs;
+                }
             } else {
                 // Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte
                 // stack alignment happens separately after all args.)
@@ -182,12 +217,12 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                 // Align.
                 debug_assert!(size.is_power_of_two());
                 next_stack = (next_stack + size - 1) & !(size - 1);
-                ret.push(ABIArg::Stack(
-                    next_stack as i64,
-                    param.value_type,
-                    param.extension,
-                    param.purpose,
-                ));
+                ret.push(ABIArg::Stack {
+                    offset: next_stack as i64,
+                    ty: param.value_type,
+                    extension: param.extension,
+                    purpose: param.purpose,
+                });
                 next_stack += size;
             }
         }
@@ -199,19 +234,19 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         let extra_arg = if add_ret_area_ptr {
             debug_assert!(args_or_rets == ArgsOrRets::Args);
             if let Some(reg) = get_intreg_for_arg_systemv(&call_conv, next_gpr) {
-                ret.push(ABIArg::Reg(
-                    ValueRegs::one(reg.to_real_reg()),
-                    types::I64,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Reg {
+                    regs: ValueRegs::one(reg.to_real_reg()),
+                    ty: types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
             } else {
-                ret.push(ABIArg::Stack(
-                    next_stack as i64,
-                    types::I64,
-                    ir::ArgumentExtension::None,
-                    ir::ArgumentPurpose::Normal,
-                ));
+                ret.push(ABIArg::Stack {
+                    offset: next_stack as i64,
+                    ty: types::I64,
+                    extension: ir::ArgumentExtension::None,
+                    purpose: ir::ArgumentPurpose::Normal,
+                });
                 next_stack += 8;
             }
             Some(ret.len() - 1)
@@ -421,6 +456,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         let stack_size = clobbered_size + fixed_frame_storage_size;
         // Align to 16 bytes.
         let stack_size = (stack_size + 15) & !15;
+        let clobbered_size = stack_size - fixed_frame_storage_size;
         // Adjust the stack pointer downward with one `sub rsp, IMM`
         // instruction.
         if stack_size > 0 {
@@ -547,6 +583,51 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         insts
     }
 
+    fn gen_memcpy(
+        call_conv: isa::CallConv,
+        dst: Reg,
+        src: Reg,
+        size: usize,
+    ) -> SmallVec<[Self::I; 8]> {
+        // Baldrdash should not use struct args.
+        assert!(!call_conv.extends_baldrdash());
+        let mut insts = SmallVec::new();
+        let arg0 = get_intreg_for_arg_systemv(&call_conv, 0).unwrap();
+        let arg1 = get_intreg_for_arg_systemv(&call_conv, 1).unwrap();
+        let arg2 = get_intreg_for_arg_systemv(&call_conv, 2).unwrap();
+        // We need a register to load the address of `memcpy()` below and we
+        // don't have a lowering context to allocate a temp here; so just use a
+        // register we know we are free to mutate as part of this sequence
+        // (because it is clobbered by the call as per the ABI anyway).
+        let memcpy_addr = get_intreg_for_arg_systemv(&call_conv, 3).unwrap();
+        insts.push(Inst::gen_move(Writable::from_reg(arg0), dst, I64));
+        insts.push(Inst::gen_move(Writable::from_reg(arg1), src, I64));
+        insts.extend(
+            Inst::gen_constant(
+                ValueRegs::one(Writable::from_reg(arg2)),
+                size as u128,
+                I64,
+                |_| panic!("tmp should not be needed"),
+            )
+            .into_iter(),
+        );
+        // We use an indirect call and a full LoadExtName because we do not have
+        // information about the libcall `RelocDistance` here, so we
+        // conservatively use the more flexible calling sequence.
+        insts.push(Inst::LoadExtName {
+            dst: Writable::from_reg(memcpy_addr),
+            name: Box::new(ExternalName::LibCall(LibCall::Memcpy)),
+            offset: 0,
+        });
+        insts.push(Inst::call_unknown(
+            RegMem::reg(memcpy_addr),
+            /* uses = */ vec![arg0, arg1, arg2],
+            /* defs = */ Self::get_regs_clobbered_by_call(call_conv),
+            Opcode::Call,
+        ));
+        insts
+    }
+
     fn get_number_of_spillslots_for_value(rc: RegClass, ty: Type) -> u32 {
         // We allocate in terms of 8-byte slots.
         match (rc, ty) {
@@ -658,31 +739,6 @@ impl From<StackAMode> for SyntheticAmode {
     }
 }
 
-fn in_int_reg(ty: types::Type) -> bool {
-    match ty {
-        types::I8
-        | types::I16
-        | types::I32
-        | types::I64
-        | types::B1
-        | types::B8
-        | types::B16
-        | types::B32
-        | types::B64
-        | types::R64 => true,
-        types::R32 => panic!("unexpected 32-bits refs on x64!"),
-        _ => false,
-    }
-}
-
-fn in_vec_reg(ty: types::Type) -> bool {
-    match ty {
-        types::F32 | types::F64 => true,
-        _ if ty.is_vector() => true,
-        _ => false,
-    }
-}
-
 fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> {
     match call_conv {
         CallConv::Fast

cranelift/codegen/src/isa/x64/inst/args.rs

@@ -346,23 +346,35 @@ impl PrettyPrintSized for RegMem {
 #[derive(Copy, Clone, PartialEq)]
 pub enum AluRmiROpcode {
     Add,
+    Adc,
     Sub,
+    Sbb,
     And,
     Or,
     Xor,
     /// The signless, non-extending (N x N -> N, for N in {32,64}) variant.
     Mul,
+    /// 8-bit form of And. Handled separately as we don't have full 8-bit op
+    /// support (we just use wider instructions). Used only with some sequences
+    /// with SETcc.
+    And8,
+    /// 8-bit form of Or.
+    Or8,
 }
 
 impl fmt::Debug for AluRmiROpcode {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
         let name = match self {
             AluRmiROpcode::Add => "add",
+            AluRmiROpcode::Adc => "adc",
             AluRmiROpcode::Sub => "sub",
+            AluRmiROpcode::Sbb => "sbb",
             AluRmiROpcode::And => "and",
             AluRmiROpcode::Or => "or",
             AluRmiROpcode::Xor => "xor",
             AluRmiROpcode::Mul => "imul",
+            AluRmiROpcode::And8 => "and",
+            AluRmiROpcode::Or8 => "or",
         };
         write!(fmt, "{}", name)
     }
@@ -374,12 +386,28 @@ impl fmt::Display for AluRmiROpcode {
     }
 }
 
+impl AluRmiROpcode {
+    /// Is this a special-cased 8-bit ALU op?
+    pub fn is_8bit(self) -> bool {
+        match self {
+            AluRmiROpcode::And8 | AluRmiROpcode::Or8 => true,
+            _ => false,
+        }
+    }
+}
+
 #[derive(Clone, PartialEq)]
 pub enum UnaryRmROpcode {
     /// Bit-scan reverse.
     Bsr,
     /// Bit-scan forward.
     Bsf,
+    /// Counts leading zeroes (Leading Zero CouNT).
+    Lzcnt,
+    /// Counts trailing zeroes (Trailing Zero CouNT).
+    Tzcnt,
+    /// Counts the number of ones (POPulation CouNT).
+    Popcnt,
 }
 
 impl fmt::Debug for UnaryRmROpcode {
@@ -387,6 +415,9 @@ impl fmt::Debug for UnaryRmROpcode {
         match self {
             UnaryRmROpcode::Bsr => write!(fmt, "bsr"),
             UnaryRmROpcode::Bsf => write!(fmt, "bsf"),
+            UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"),
+            UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"),
+            UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"),
         }
     }
 }
@@ -1010,7 +1041,7 @@ impl fmt::Display for ExtMode {
 }
 
 /// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right.
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub enum ShiftKind {
     ShiftLeft,
     /// Inserts zeros in the most significant bits.

cranelift/codegen/src/isa/x64/inst/emit.rs

@@ -1,5 +1,6 @@
 use crate::binemit::{Addend, Reloc};
 use crate::ir::immediates::{Ieee32, Ieee64};
+use crate::ir::LibCall;
 use crate::ir::TrapCode;
 use crate::isa::x64::inst::args::*;
 use crate::isa::x64::inst::*;
@@ -83,6 +84,14 @@ impl RexFlags {
         self
     }
 
+    #[inline(always)]
+    fn always_emit_if_8bit_needed(&mut self, reg: u8) -> &mut Self {
+        if reg >= 4 && reg <= 7 {
+            self.always_emit();
+        }
+        self
+    }
+
     #[inline(always)]
     fn must_clear_w(&self) -> bool {
         (self.0 & 1) != 0
@@ -120,18 +129,20 @@ impl RexFlags {
 /// We may need to include one or more legacy prefix bytes before the REX prefix.  This enum
 /// covers only the small set of possibilities that we actually need.
 enum LegacyPrefixes {
-    /// No prefix bytes
+    /// No prefix bytes.
     None,
-    /// Operand Size Override -- here, denoting "16-bit operation"
+    /// Operand Size Override -- here, denoting "16-bit operation".
     _66,
-    /// The Lock prefix
+    /// The Lock prefix.
     _F0,
-    /// Operand size override and Lock
+    /// Operand size override and Lock.
     _66F0,
-    /// REPNE, but no specific meaning here -- is just an opcode extension
+    /// REPNE, but no specific meaning here -- is just an opcode extension.
     _F2,
-    /// REP/REPE, but no specific meaning here -- is just an opcode extension
+    /// REP/REPE, but no specific meaning here -- is just an opcode extension.
     _F3,
+    /// Operand size override and same effect as F3.
+    _66F3,
 }
 
 impl LegacyPrefixes {
@@ -148,6 +159,10 @@ impl LegacyPrefixes {
             }
             LegacyPrefixes::_F2 => sink.put1(0xF2),
             LegacyPrefixes::_F3 => sink.put1(0xF3),
+            LegacyPrefixes::_66F3 => {
+                sink.put1(0x66);
+                sink.put1(0xF3);
+            }
             LegacyPrefixes::None => (),
         }
     }
@@ -527,7 +542,7 @@ pub(crate) fn emit(
             src,
             dst: reg_g,
         } => {
-            let rex = if *is_64 {
+            let mut rex = if *is_64 {
                 RexFlags::set_w()
             } else {
                 RexFlags::clear_w()
@@ -581,17 +596,26 @@ pub(crate) fn emit(
                     }
                 }
             } else {
-                let (opcode_r, opcode_m, subopcode_i) = match op {
-                    AluRmiROpcode::Add => (0x01, 0x03, 0),
-                    AluRmiROpcode::Sub => (0x29, 0x2B, 5),
-                    AluRmiROpcode::And => (0x21, 0x23, 4),
-                    AluRmiROpcode::Or => (0x09, 0x0B, 1),
-                    AluRmiROpcode::Xor => (0x31, 0x33, 6),
+                let (opcode_r, opcode_m, subopcode_i, is_8bit) = match op {
+                    AluRmiROpcode::Add => (0x01, 0x03, 0, false),
+                    AluRmiROpcode::Adc => (0x11, 0x03, 0, false),
+                    AluRmiROpcode::Sub => (0x29, 0x2B, 5, false),
+                    AluRmiROpcode::Sbb => (0x19, 0x2B, 5, false),
+                    AluRmiROpcode::And => (0x21, 0x23, 4, false),
+                    AluRmiROpcode::Or => (0x09, 0x0B, 1, false),
+                    AluRmiROpcode::Xor => (0x31, 0x33, 6, false),
+                    AluRmiROpcode::And8 => (0x20, 0x22, 4, true),
+                    AluRmiROpcode::Or8 => (0x08, 0x0A, 1, true),
                     AluRmiROpcode::Mul => panic!("unreachable"),
                 };
+                assert!(!(is_8bit && *is_64));
 
                 match src {
                     RegMemImm::Reg { reg: reg_e } => {
+                        if is_8bit {
+                            rex.always_emit_if_8bit_needed(int_reg_enc(*reg_e));
+                            rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg()));
+                        }
                         // GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R
                         // duality). Do this too, so as to be able to compare generated machine
                         // code easily.
@@ -604,11 +628,12 @@ pub(crate) fn emit(
                             reg_g.to_reg(),
                             rex,
                         );
-                        // NB: if this is ever extended to handle byte size ops, be sure to retain
-                        // redundant REX prefixes.
                     }
 
                     RegMemImm::Mem { addr } => {
+                        if is_8bit {
+                            rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg()));
+                        }
                         // Here we revert to the "normal" G-E ordering.
                         let amode = addr.finalize(state, sink);
                         emit_std_reg_mem(
@@ -625,6 +650,7 @@ pub(crate) fn emit(
                     }
 
                     RegMemImm::Imm { simm32 } => {
+                        assert!(!is_8bit);
                         let use_imm8 = low8_will_sign_extend_to_32(*simm32);
                         let opcode = if use_imm8 { 0x83 } else { 0x81 };
                         // And also here we use the "normal" G-E ordering.
@@ -645,16 +671,31 @@ pub(crate) fn emit(
         }
 
         Inst::UnaryRmR { size, op, src, dst } => {
-            let (prefix, rex_flags) = match size {
-                2 => (LegacyPrefixes::_66, RexFlags::clear_w()),
-                4 => (LegacyPrefixes::None, RexFlags::clear_w()),
-                8 => (LegacyPrefixes::None, RexFlags::set_w()),
+            let rex_flags = match size {
+                2 | 4 => RexFlags::clear_w(),
+                8 => RexFlags::set_w(),
+                _ => unreachable!(),
+            };
+
+            use UnaryRmROpcode::*;
+            let prefix = match size {
+                2 => match op {
+                    Bsr | Bsf => LegacyPrefixes::_66,
+                    Lzcnt | Tzcnt | Popcnt => LegacyPrefixes::_66F3,
+                },
+                4 | 8 => match op {
+                    Bsr | Bsf => LegacyPrefixes::None,
+                    Lzcnt | Tzcnt | Popcnt => LegacyPrefixes::_F3,
+                },
                 _ => unreachable!(),
             };
 
             let (opcode, num_opcodes) = match op {
-                UnaryRmROpcode::Bsr => (0x0fbd, 2),
-                UnaryRmROpcode::Bsf => (0x0fbc, 2),
+                Bsr => (0x0fbd, 2),
+                Bsf => (0x0fbc, 2),
+                Lzcnt => (0x0fbd, 2),
+                Tzcnt => (0x0fbc, 2),
+                Popcnt => (0x0fb8, 2),
             };
 
             match src {
@@ -685,8 +726,13 @@ pub(crate) fn emit(
         }
 
         Inst::Not { size, src } => {
+            let src = int_reg_enc(src.to_reg());
             let (opcode, prefix, rex_flags) = match size {
-                1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()),
+                1 => (
+                    0xF6,
+                    LegacyPrefixes::None,
+                    *RexFlags::clear_w().always_emit_if_8bit_needed(src),
+                ),
                 2 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
                 4 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
                 8 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()),
@@ -694,13 +740,17 @@ pub(crate) fn emit(
             };
 
             let subopcode = 2;
-            let src = int_reg_enc(src.to_reg());
             emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags)
         }
 
         Inst::Neg { size, src } => {
+            let src = int_reg_enc(src.to_reg());
             let (opcode, prefix, rex_flags) = match size {
-                1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()),
+                1 => (
+                    0xF6,
+                    LegacyPrefixes::None,
+                    *RexFlags::clear_w().always_emit_if_8bit_needed(src),
+                ),
                 2 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
                 4 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
                 8 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()),
@@ -708,7 +758,6 @@ pub(crate) fn emit(
             };
 
             let subopcode = 3;
-            let src = int_reg_enc(src.to_reg());
             emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags)
         }
 
@@ -717,7 +766,7 @@ pub(crate) fn emit(
             signed,
             divisor,
         } => {
-            let (opcode, prefix, rex_flags) = match size {
+            let (opcode, prefix, mut rex_flags) = match size {
                 1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()),
                 2 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
                 4 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
@@ -732,6 +781,9 @@ pub(crate) fn emit(
             match divisor {
                 RegMem::Reg { reg } => {
                     let src = int_reg_enc(*reg);
+                    if *size == 1 {
+                        rex_flags.always_emit_if_8bit_needed(src);
+                    }
                     emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags)
                 }
                 RegMem::Mem { addr: src } => {
@@ -987,9 +1039,7 @@ pub(crate) fn emit(
                         ExtMode::BL | ExtMode::BQ => {
                             // A redundant REX prefix must be emitted for certain register inputs.
                             let enc_src = int_reg_enc(*src);
-                            if enc_src >= 4 && enc_src <= 7 {
-                                rex_flags.always_emit();
-                            };
+                            rex_flags.always_emit_if_8bit_needed(enc_src);
                         }
                         _ => {}
                     }
@@ -1084,9 +1134,7 @@ pub(crate) fn emit(
                         ExtMode::BL | ExtMode::BQ => {
                             // A redundant REX prefix must be emitted for certain register inputs.
                             let enc_src = int_reg_enc(*src);
-                            if enc_src >= 4 && enc_src <= 7 {
-                                rex_flags.always_emit();
-                            };
+                            rex_flags.always_emit_if_8bit_needed(enc_src);
                         }
                         _ => {}
                     }
@@ -1130,9 +1178,7 @@ pub(crate) fn emit(
                     let mut rex = RexFlags::clear_w();
 
                     let enc_src = int_reg_enc(*src);
-                    if enc_src >= 4 && enc_src <= 7 {
-                        rex.always_emit();
-                    };
+                    rex.always_emit_if_8bit_needed(enc_src);
 
                     // MOV r8, r/m8 is (REX.W==0) 88 /r
                     emit_std_reg_mem(
@@ -1215,7 +1261,11 @@ pub(crate) fn emit(
             match num_bits {
                 None => {
                     let (opcode, prefix, rex_flags) = match size {
-                        1 => (0xD2, LegacyPrefixes::None, RexFlags::clear_w()),
+                        1 => (
+                            0xD2,
+                            LegacyPrefixes::None,
+                            *RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst),
+                        ),
                         2 => (0xD3, LegacyPrefixes::_66, RexFlags::clear_w()),
                         4 => (0xD3, LegacyPrefixes::None, RexFlags::clear_w()),
                         8 => (0xD3, LegacyPrefixes::None, RexFlags::set_w()),
@@ -1231,7 +1281,11 @@ pub(crate) fn emit(
 
                 Some(num_bits) => {
                     let (opcode, prefix, rex_flags) = match size {
-                        1 => (0xC0, LegacyPrefixes::None, RexFlags::clear_w()),
+                        1 => (
+                            0xC0,
+                            LegacyPrefixes::None,
+                            *RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst),
+                        ),
                         2 => (0xC1, LegacyPrefixes::_66, RexFlags::clear_w()),
                         4 => (0xC1, LegacyPrefixes::None, RexFlags::clear_w()),
                         8 => (0xC1, LegacyPrefixes::None, RexFlags::set_w()),
@@ -1330,9 +1384,7 @@ pub(crate) fn emit(
                     let mut rex = RexFlags::clear_w();
                     // Here, a redundant REX prefix changes the meaning of the instruction.
                     let enc_g = int_reg_enc(*reg_g);
-                    if enc_g >= 4 && enc_g <= 7 {
-                        rex.always_emit();
-                    }
+                    rex.always_emit_if_8bit_needed(enc_g);
                     rex
                 }
                 _ => panic!("x64::Inst::Cmp_RMI_R::emit: unreachable"),
@@ -1343,9 +1395,7 @@ pub(crate) fn emit(
                     if *size == 1 {
                         // Check whether the E register forces the use of a redundant REX.
                         let enc_e = int_reg_enc(*reg_e);
-                        if enc_e >= 4 && enc_e <= 7 {
-                            rex.always_emit();
-                        }
+                        rex.always_emit_if_8bit_needed(enc_e);
                     }
 
                     // Use the swapped operands encoding for CMP, to stay consistent with the output of
@@ -2065,7 +2115,9 @@ pub(crate) fn emit(
                 SseOpcode::Pextrd => (LegacyPrefixes::_66, 0x0F3A16, 3),
                 SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2),
                 SseOpcode::Roundps => (LegacyPrefixes::_66, 0x0F3A08, 3),
+                SseOpcode::Roundss => (LegacyPrefixes::_66, 0x0F3A0A, 3),
                 SseOpcode::Roundpd => (LegacyPrefixes::_66, 0x0F3A09, 3),
+                SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3),
                 _ => unimplemented!("Opcode {:?} not implemented", op),
             };
             let rex = if *is64 {
@@ -2761,9 +2813,7 @@ pub(crate) fn emit(
                 types::I8 => {
                     let mut rex_flags = RexFlags::clear_w();
                     let enc_src = int_reg_enc(*src);
-                    if enc_src >= 4 && enc_src <= 7 {
-                        rex_flags.always_emit();
-                    };
+                    rex_flags.always_emit_if_8bit_needed(enc_src);
                     (LegacyPrefixes::_F0, rex_flags, 0x0FB0)
                 }
                 types::I16 => (LegacyPrefixes::_66F0, RexFlags::clear_w(), 0x0FB1),
@@ -2962,6 +3012,50 @@ pub(crate) fn emit(
         Inst::EpiloguePlaceholder => {
             // Generate no code.
         }
+
+        Inst::ElfTlsGetAddr { ref symbol } => {
+            // N.B.: Must be exactly this byte sequence; the linker requires it,
+            // because it must know how to rewrite the bytes.
+
+            // data16 lea gv@tlsgd(%rip),%rdi
+            sink.put1(0x66); // data16
+            sink.put1(0b01001000); // REX.W
+            sink.put1(0x8d); // LEA
+            sink.put1(0x3d); // ModRM byte
+            emit_reloc(sink, state, Reloc::ElfX86_64TlsGd, symbol, -4);
+            sink.put4(0); // offset
+
+            // data16 data16 callq __tls_get_addr-4
+            sink.put1(0x66); // data16
+            sink.put1(0x66); // data16
+            sink.put1(0b01001000); // REX.W
+            sink.put1(0xe8); // CALL
+            emit_reloc(
+                sink,
+                state,
+                Reloc::X86CallPLTRel4,
+                &ExternalName::LibCall(LibCall::ElfTlsGetAddr),
+                -4,
+            );
+            sink.put4(0); // offset
+        }
+
+        Inst::MachOTlsGetAddr { ref symbol } => {
+            // movq gv@tlv(%rip), %rdi
+            sink.put1(0x48); // REX.w
+            sink.put1(0x8b); // MOV
+            sink.put1(0x3d); // ModRM byte
+            emit_reloc(sink, state, Reloc::MachOX86_64Tlv, symbol, -4);
+            sink.put4(0); // offset
+
+            // callq *(%rdi)
+            sink.put1(0xff);
+            sink.put1(0x17);
+        }
+
+        Inst::ValueLabelMarker { .. } => {
+            // Nothing; this is only used to compute debug info.
+        }
     }
 
     state.clear_post_insn();

cranelift/codegen/src/isa/x64/inst/emit_tests.rs

@@ -1025,6 +1025,56 @@ fn test_x64_emit() {
         "4C09FA",
         "orq     %r15, %rdx",
     ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::And8, RegMemImm::reg(r15), w_rdx),
+        "4420FA",
+        "andb    %r15b, %dl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::And8, RegMemImm::reg(rax), w_rsi),
+        "4020C6",
+        "andb    %al, %sil",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::And8, RegMemImm::reg(rax), w_rbx),
+        "20C3",
+        "andb    %al, %bl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(
+            false,
+            AluRmiROpcode::And8,
+            RegMemImm::mem(Amode::imm_reg(0, rax)),
+            w_rbx,
+        ),
+        "2218",
+        "andb    0(%rax), %bl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::Or8, RegMemImm::reg(r15), w_rdx),
+        "4408FA",
+        "orb     %r15b, %dl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::Or8, RegMemImm::reg(rax), w_rsi),
+        "4008C6",
+        "orb     %al, %sil",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(false, AluRmiROpcode::Or8, RegMemImm::reg(rax), w_rbx),
+        "08C3",
+        "orb     %al, %bl",
+    ));
+    insns.push((
+        Inst::alu_rmi_r(
+            false,
+            AluRmiROpcode::Or8,
+            RegMemImm::mem(Amode::imm_reg(0, rax)),
+            w_rbx,
+        ),
+        "0A18",
+        "orb     0(%rax), %bl",
+    ));
     insns.push((
         Inst::alu_rmi_r(true, AluRmiROpcode::Xor, RegMemImm::reg(r15), w_rdx),
         "4C31FA",
@@ -1193,6 +1243,16 @@ fn test_x64_emit() {
         "66F7D7",
         "notw    %di",
     ));
+    insns.push((
+        Inst::not(1, Writable::from_reg(regs::rdi())),
+        "40F6D7",
+        "notb    %dil",
+    ));
+    insns.push((
+        Inst::not(1, Writable::from_reg(regs::rax())),
+        "F6D0",
+        "notb    %al",
+    ));
 
     // ========================================================
     // Neg
@@ -1216,6 +1276,16 @@ fn test_x64_emit() {
         "66F7DF",
         "negw    %di",
     ));
+    insns.push((
+        Inst::neg(1, Writable::from_reg(regs::rdi())),
+        "40F6DF",
+        "negb    %dil",
+    ));
+    insns.push((
+        Inst::neg(1, Writable::from_reg(regs::rax())),
+        "F6D8",
+        "negb    %al",
+    ));
 
     // ========================================================
     // Div
@@ -1239,6 +1309,16 @@ fn test_x64_emit() {
         "48F7F7",
         "div     %rdi",
     ));
+    insns.push((
+        Inst::div(1, false, RegMem::reg(regs::rax())),
+        "F6F0",
+        "div     %al",
+    ));
+    insns.push((
+        Inst::div(1, false, RegMem::reg(regs::rsi())),
+        "40F6F6",
+        "div     %sil",
+    ));
 
     // ========================================================
     // MulHi
@@ -2352,9 +2432,14 @@ fn test_x64_emit() {
     ));
     insns.push((
         Inst::shift_r(1, ShiftKind::RotateRight, None, w_rsi),
-        "D2CE",
+        "40D2CE",
         "rorb    %cl, %sil",
     ));
+    insns.push((
+        Inst::shift_r(1, ShiftKind::RotateRight, None, w_rax),
+        "D2C8",
+        "rorb    %cl, %al",
+    ));
     insns.push((
         Inst::shift_r(1, ShiftKind::RotateRight, Some(5), w_r15),
         "41C0CF05",
@@ -3819,6 +3904,28 @@ fn test_x64_emit() {
     let trap_code = TrapCode::UnreachableCodeReached;
     insns.push((Inst::Ud2 { trap_code }, "0F0B", "ud2 unreachable"));
 
+    insns.push((
+        Inst::ElfTlsGetAddr {
+            symbol: ExternalName::User {
+                namespace: 0,
+                index: 0,
+            },
+        },
+        "66488D3D00000000666648E800000000",
+        "elf_tls_get_addr User { namespace: 0, index: 0 }",
+    ));
+
+    insns.push((
+        Inst::MachOTlsGetAddr {
+            symbol: ExternalName::User {
+                namespace: 0,
+                index: 0,
+            },
+        },
+        "488B3D00000000FF17",
+        "macho_tls_get_addr User { namespace: 0, index: 0 }",
+    ));
+
     // ========================================================
     // Actually run the tests!
     let mut flag_builder = settings::builder();

cranelift/codegen/src/isa/x64/inst/mod.rs

@@ -1,8 +1,10 @@
 //! This module defines x86_64-specific machine instruction types.
 
 use crate::binemit::{CodeOffset, StackMap};
-use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type};
+use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type, ValueLabel};
+use crate::isa::x64::abi::X64ABIMachineSpec;
 use crate::isa::x64::settings as x64_settings;
+use crate::isa::CallConv;
 use crate::machinst::*;
 use crate::{settings, settings::Flags, CodegenError, CodegenResult};
 use alloc::boxed::Box;
@@ -474,6 +476,17 @@ pub enum Inst {
     /// reports its own `def`s/`use`s/`mod`s; this adds complexity (the instruction list is no
     /// longer flat) and requires knowledge about semantics and initial-value independence anyway.
     XmmUninitializedValue { dst: Writable<Reg> },
+
+    /// A call to the `ElfTlsGetAddr` libcall. Returns address
+    /// of TLS symbol in rax.
+    ElfTlsGetAddr { symbol: ExternalName },
+
+    /// A Mach-O TLS symbol access. Returns address of the TLS
+    /// symbol in rax.
+    MachOTlsGetAddr { symbol: ExternalName },
+
+    /// A definition of a value label.
+    ValueLabelMarker { reg: Reg, label: ValueLabel },
 }
 
 pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool {
@@ -532,7 +545,10 @@ impl Inst {
             | Inst::XmmCmpRmR { .. }
             | Inst::XmmLoadConst { .. }
             | Inst::XmmMinMaxSeq { .. }
-            | Inst::XmmUninitializedValue { .. } => None,
+            | Inst::XmmUninitializedValue { .. }
+            | Inst::ElfTlsGetAddr { .. }
+            | Inst::MachOTlsGetAddr { .. }
+            | Inst::ValueLabelMarker { .. } => None,
 
             // These use dynamic SSE opcodes.
             Inst::GprToXmm { op, .. }
@@ -550,7 +566,7 @@ impl Inst {
 
 impl Inst {
     pub(crate) fn nop(len: u8) -> Self {
-        debug_assert!(len <= 16);
+        debug_assert!(len <= 15);
         Self::Nop { len }
     }
 
@@ -1243,6 +1259,14 @@ impl PrettyPrint for Inst {
             (if is_64 { "q" } else { "l" }).to_string()
         }
 
+        fn suffix_lqb(is_64: bool, is_8: bool) -> String {
+            match (is_64, is_8) {
+                (_, true) => "b".to_string(),
+                (true, false) => "q".to_string(),
+                (false, false) => "l".to_string(),
+            }
+        }
+
         fn size_lq(is_64: bool) -> u8 {
             if is_64 {
                 8
@@ -1251,6 +1275,16 @@ impl PrettyPrint for Inst {
             }
         }
 
+        fn size_lqb(is_64: bool, is_8: bool) -> u8 {
+            if is_8 {
+                1
+            } else if is_64 {
+                8
+            } else {
+                4
+            }
+        }
+
         fn suffix_bwlq(size: u8) -> String {
             match size {
                 1 => "b".to_string(),
@@ -1271,9 +1305,9 @@ impl PrettyPrint for Inst {
                 dst,
             } => format!(
                 "{} {}, {}",
-                ljustify2(op.to_string(), suffix_lq(*is_64)),
-                src.show_rru_sized(mb_rru, size_lq(*is_64)),
-                show_ireg_sized(dst.to_reg(), mb_rru, size_lq(*is_64)),
+                ljustify2(op.to_string(), suffix_lqb(*is_64, op.is_8bit())),
+                src.show_rru_sized(mb_rru, size_lqb(*is_64, op.is_8bit())),
+                show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*is_64, op.is_8bit())),
             ),
 
             Inst::UnaryRmR { src, dst, op, size } => format!(
@@ -1762,6 +1796,18 @@ impl PrettyPrint for Inst {
             Inst::Hlt => "hlt".into(),
 
             Inst::Ud2 { trap_code } => format!("ud2 {}", trap_code),
+
+            Inst::ElfTlsGetAddr { ref symbol } => {
+                format!("elf_tls_get_addr {:?}", symbol)
+            }
+
+            Inst::MachOTlsGetAddr { ref symbol } => {
+                format!("macho_tls_get_addr {:?}", symbol)
+            }
+
+            Inst::ValueLabelMarker { label, reg } => {
+                format!("value_label {:?}, {}", label, reg.show_rru(mb_rru))
+            }
         }
     }
 }
@@ -1777,7 +1823,7 @@ impl fmt::Debug for Inst {
 fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
     // This is a bit subtle. If some register is in the modified set, then it may not be in either
     // the use or def sets. However, enforcing that directly is somewhat difficult. Instead,
-    // regalloc.rs will "fix" this for us by removing the the modified set from the use and def
+    // regalloc.rs will "fix" this for us by removing the modified set from the use and def
     // sets.
     match inst {
         Inst::AluRmiR { src, dst, .. } => {
@@ -1849,6 +1895,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
                 || *op == SseOpcode::Pextrw
                 || *op == SseOpcode::Pextrd
                 || *op == SseOpcode::Pshufd
+                || *op == SseOpcode::Roundss
+                || *op == SseOpcode::Roundsd
+                || *op == SseOpcode::Roundps
+                || *op == SseOpcode::Roundpd
             {
                 src.get_regs_as_uses(collector);
                 collector.add_def(*dst);
@@ -2021,6 +2071,22 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
         | Inst::Fence { .. } => {
             // No registers are used.
         }
+
+        Inst::ElfTlsGetAddr { .. } | Inst::MachOTlsGetAddr { .. } => {
+            // All caller-saves are clobbered.
+            //
+            // We use the SysV calling convention here because the
+            // pseudoinstruction (and relocation that it emits) is specific to
+            // ELF systems; other x86-64 targets with other conventions (i.e.,
+            // Windows) use different TLS strategies.
+            for reg in X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV) {
+                collector.add_def(reg);
+            }
+        }
+
+        Inst::ValueLabelMarker { reg, .. } => {
+            collector.add_use(*reg);
+        }
     }
 }
 
@@ -2065,6 +2131,17 @@ impl Amode {
             }
         }
     }
+
+    /// Offset the amode by a fixed offset.
+    pub(crate) fn offset(&self, offset: u32) -> Self {
+        let mut ret = self.clone();
+        match &mut ret {
+            &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
+            &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
+            _ => panic!("Cannot offset amode: {:?}", self),
+        }
+        ret
+    }
 }
 
 impl RegMemImm {
@@ -2163,6 +2240,10 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
                 || *op == SseOpcode::Pextrw
                 || *op == SseOpcode::Pextrd
                 || *op == SseOpcode::Pshufd
+                || *op == SseOpcode::Roundss
+                || *op == SseOpcode::Roundsd
+                || *op == SseOpcode::Roundps
+                || *op == SseOpcode::Roundpd
             {
                 src.map_uses(mapper);
                 map_def(mapper, dst);
@@ -2385,6 +2466,8 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             dst.map_uses(mapper);
         }
 
+        Inst::ValueLabelMarker { ref mut reg, .. } => map_use(mapper, reg),
+
         Inst::Ret
         | Inst::EpiloguePlaceholder
         | Inst::JmpKnown { .. }
@@ -2396,6 +2479,8 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
         | Inst::Ud2 { .. }
         | Inst::Hlt
         | Inst::AtomicRmwSeq { .. }
+        | Inst::ElfTlsGetAddr { .. }
+        | Inst::MachOTlsGetAddr { .. }
         | Inst::Fence { .. } => {
             // Instruction doesn't explicitly mention any regs, so it can't have any virtual
             // regs that we'd need to remap.  Hence no action required.
@@ -2473,6 +2558,25 @@ impl MachInst for Inst {
         }
     }
 
+    fn stack_op_info(&self) -> Option<MachInstStackOpInfo> {
+        match self {
+            Self::VirtualSPOffsetAdj { offset } => Some(MachInstStackOpInfo::NomSPAdj(*offset)),
+            Self::MovRM {
+                size: 8,
+                src,
+                dst: SyntheticAmode::NominalSPOffset { simm32 },
+            } => Some(MachInstStackOpInfo::StoreNomSPOff(*src, *simm32 as i64)),
+            Self::Mov64MR {
+                src: SyntheticAmode::NominalSPOffset { simm32 },
+                dst,
+            } => Some(MachInstStackOpInfo::LoadNomSPOff(
+                dst.to_reg(),
+                *simm32 as i64,
+            )),
+            _ => None,
+        }
+    }
+
     fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst {
         let rc_dst = dst_reg.to_reg().get_class();
         let rc_src = src_reg.get_class();
@@ -2497,12 +2601,8 @@ impl MachInst for Inst {
         }
     }
 
-    fn gen_zero_len_nop() -> Inst {
-        Inst::Nop { len: 0 }
-    }
-
     fn gen_nop(preferred_size: usize) -> Inst {
-        Inst::nop((preferred_size % 16) as u8)
+        Inst::nop(std::cmp::min(preferred_size, 15) as u8)
     }
 
     fn maybe_direct_reload(&self, _reg: VirtualReg, _slot: SpillSlot) -> Option<Inst> {
@@ -2548,9 +2648,19 @@ impl MachInst for Inst {
         ty: Type,
         mut alloc_tmp: F,
     ) -> SmallVec<[Self; 4]> {
-        // We don't support 128-bit constants.
-        assert!(value <= u64::MAX as u128);
         let mut ret = SmallVec::new();
+        if ty == types::I128 {
+            ret.push(Inst::imm(
+                OperandSize::Size64,
+                value as u64,
+                to_regs.regs()[0],
+            ));
+            ret.push(Inst::imm(
+                OperandSize::Size64,
+                (value >> 64) as u64,
+                to_regs.regs()[1],
+            ));
+        } else {
             let to_reg = to_regs
                 .only_reg()
                 .expect("multi-reg values not supported on x64");
@@ -2621,6 +2731,7 @@ impl MachInst for Inst {
                     ));
                 }
             }
+        }
         ret
     }
 
@@ -2636,6 +2747,17 @@ impl MachInst for Inst {
         RegClass::I64
     }
 
+    fn gen_value_label_marker(label: ValueLabel, reg: Reg) -> Self {
+        Inst::ValueLabelMarker { label, reg }
+    }
+
+    fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> {
+        match self {
+            Inst::ValueLabelMarker { label, reg } => Some((*label, *reg)),
+            _ => None,
+        }
+    }
+
     type LabelUse = LabelUse;
 }
 

cranelift/codegen/src/isa/x64/mod.rs

@@ -4,13 +4,15 @@ use self::inst::EmitInfo;
 
 use super::TargetIsa;
 use crate::ir::{condcodes::IntCC, Function};
+use crate::isa::unwind::systemv::RegisterMappingError;
 use crate::isa::x64::{inst::regs::create_reg_universe_systemv, settings as x64_settings};
 use crate::isa::Builder as IsaBuilder;
 use crate::machinst::{compile, MachBackend, MachCompileResult, TargetIsaAdapter, VCode};
 use crate::result::CodegenResult;
 use crate::settings::{self as shared_settings, Flags};
 use alloc::boxed::Box;
-use regalloc::{PrettyPrint, RealRegUniverse};
+use core::hash::{Hash, Hasher};
+use regalloc::{PrettyPrint, RealRegUniverse, Reg};
 use target_lexicon::Triple;
 
 mod abi;
@@ -60,6 +62,7 @@ impl MachBackend for X64Backend {
         let buffer = buffer.finish();
         let frame_size = vcode.frame_size();
         let unwind_info = vcode.unwind_info()?;
+        let value_labels_ranges = vcode.value_labels_ranges()?;
 
         let disasm = if want_disasm {
             Some(vcode.show_rru(Some(&create_reg_universe_systemv(flags))))
@@ -72,6 +75,7 @@ impl MachBackend for X64Backend {
             frame_size,
             disasm,
             unwind_info,
+            value_labels_ranges,
         })
     }
 
@@ -79,6 +83,11 @@ impl MachBackend for X64Backend {
         &self.flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.flags.hash(&mut hasher);
+        self.x64_flags.hash(&mut hasher);
+    }
+
     fn name(&self) -> &'static str {
         "x64"
     }
@@ -127,6 +136,11 @@ impl MachBackend for X64Backend {
     fn create_systemv_cie(&self) -> Option<gimli::write::CommonInformationEntry> {
         Some(inst::unwind::systemv::create_cie())
     }
+
+    #[cfg(feature = "unwind")]
+    fn map_reg_to_dwarf(&self, reg: Reg) -> Result<u16, RegisterMappingError> {
+        inst::unwind::systemv::map_reg(reg).map(|reg| reg.0)
+    }
 }
 
 /// Create a new `isa::Builder`.

cranelift/codegen/src/isa/x86/mod.rs

@@ -25,6 +25,7 @@ use alloc::borrow::Cow;
 use alloc::boxed::Box;
 use core::any::Any;
 use core::fmt;
+use core::hash::{Hash, Hasher};
 use target_lexicon::{PointerWidth, Triple};
 
 #[allow(dead_code)]
@@ -78,6 +79,11 @@ impl TargetIsa for Isa {
         &self.shared_flags
     }
 
+    fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) {
+        self.shared_flags.hash(&mut hasher);
+        self.isa_flags.hash(&mut hasher);
+    }
+
     fn uses_cpu_flags(&self) -> bool {
         true
     }

cranelift/codegen/src/machinst/abi.rs

@@ -1,7 +1,7 @@
 //! ABI definitions.
 
 use crate::binemit::StackMap;
-use crate::ir::StackSlot;
+use crate::ir::{Signature, StackSlot};
 use crate::isa::CallConv;
 use crate::machinst::*;
 use crate::settings;
@@ -27,6 +27,9 @@ pub trait ABICallee {
     /// lowering context exists.
     fn init(&mut self, maybe_tmp: Option<Writable<Reg>>);
 
+    /// Access the (possibly legalized) signature.
+    fn signature(&self) -> &Signature;
+
     /// Accumulate outgoing arguments.  This ensures that at least SIZE bytes
     /// are allocated in the prologue to be available for use in function calls
     /// to hold arguments and/or return values.  If this function is called
@@ -215,6 +218,9 @@ pub trait ABICaller {
     /// Get the number of arguments expected.
     fn num_args(&self) -> usize;
 
+    /// Access the (possibly legalized) signature.
+    fn signature(&self) -> &Signature;
+
     /// Emit a copy of an argument value from a source register, prior to the call.
     fn emit_copy_regs_to_arg<C: LowerCtx<I = Self::I>>(
         &self,
@@ -223,6 +229,11 @@ pub trait ABICaller {
         from_reg: ValueRegs<Reg>,
     );
 
+    /// Specific order for copying into arguments at callsites. We must be
+    /// careful to copy into StructArgs first, because we need to be able
+    /// to invoke memcpy() before we've loaded other arg regs (see above).
+    fn get_copy_to_arg_order(&self) -> SmallVec<[usize; 8]>;
+
     /// Emit a copy a return value into a destination register, after the call returns.
     fn emit_copy_retval_to_regs<C: LowerCtx<I = Self::I>>(
         &self,

cranelift/codegen/src/machinst/abi_impl.rs

@@ -111,7 +111,7 @@
 use super::abi::*;
 use crate::binemit::StackMap;
 use crate::ir::types::*;
-use crate::ir::{ArgumentExtension, StackSlot};
+use crate::ir::{ArgumentExtension, ArgumentPurpose, StackSlot};
 use crate::machinst::*;
 use crate::settings;
 use crate::CodegenResult;
@@ -128,22 +128,58 @@ use std::mem;
 #[derive(Clone, Copy, Debug)]
 pub enum ABIArg {
     /// In a real register (or set of registers).
-    Reg(
-        ValueRegs<RealReg>,
-        ir::Type,
-        ir::ArgumentExtension,
-        ir::ArgumentPurpose,
-    ),
+    Reg {
+        /// Register(s) that hold this arg.
+        regs: ValueRegs<RealReg>,
+        /// Value type of this arg.
+        ty: ir::Type,
+        /// Should this arg be zero- or sign-extended?
+        extension: ir::ArgumentExtension,
+        /// Purpose of this arg.
+        purpose: ir::ArgumentPurpose,
+    },
     /// Arguments only: on stack, at given offset from SP at entry.
-    Stack(i64, ir::Type, ir::ArgumentExtension, ir::ArgumentPurpose),
+    Stack {
+        /// Offset of this arg relative to the base of stack args.
+        offset: i64,
+        /// Value type of this arg.
+        ty: ir::Type,
+        /// Should this arg be zero- or sign-extended?
+        extension: ir::ArgumentExtension,
+        /// Purpose of this arg.
+        purpose: ir::ArgumentPurpose,
+    },
+    /// Structure argument. We reserve stack space for it, but the CLIF-level
+    /// semantics are a little weird: the value passed to the call instruction,
+    /// and received in the corresponding block param, is a *pointer*. On the
+    /// caller side, we memcpy the data from the passed-in pointer to the stack
+    /// area; on the callee side, we compute a pointer to this stack area and
+    /// provide that as the argument's value.
+    StructArg {
+        /// Offset of this arg relative to base of stack args.
+        offset: i64,
+        /// Size of this arg on the stack.
+        size: u64,
+        /// Purpose of this arg.
+        purpose: ir::ArgumentPurpose,
+    },
 }
 
 impl ABIArg {
     /// Get the purpose of this arg.
     fn get_purpose(self) -> ir::ArgumentPurpose {
         match self {
-            ABIArg::Reg(_, _, _, purpose) => purpose,
-            ABIArg::Stack(_, _, _, purpose) => purpose,
+            ABIArg::Reg { purpose, .. } => purpose,
+            ABIArg::Stack { purpose, .. } => purpose,
+            ABIArg::StructArg { purpose, .. } => purpose,
+        }
+    }
+
+    /// Is this a StructArg?
+    fn is_struct_arg(self) -> bool {
+        match self {
+            ABIArg::StructArg { .. } => true,
+            _ => false,
         }
     }
 }
@@ -371,6 +407,16 @@ pub trait ABIMachineSpec {
         callee_conv: isa::CallConv,
     ) -> SmallVec<[(InstIsSafepoint, Self::I); 2]>;
 
+    /// Generate a memcpy invocation. Used to set up struct args. May clobber
+    /// caller-save registers; we only memcpy before we start to set up args for
+    /// a call.
+    fn gen_memcpy(
+        call_conv: isa::CallConv,
+        dst: Reg,
+        src: Reg,
+        size: usize,
+    ) -> SmallVec<[Self::I; 8]>;
+
     /// Get the number of spillslots required for the given register-class and
     /// type.
     fn get_number_of_spillslots_for_value(rc: RegClass, ty: Type) -> u32;
@@ -455,6 +501,8 @@ impl ABISig {
 
 /// ABI object for a function body.
 pub struct ABICalleeImpl<M: ABIMachineSpec> {
+    /// CLIF-level signature, possibly normalized.
+    ir_sig: ir::Signature,
     /// Signature: arg and retval regs.
     sig: ABISig,
     /// Offsets to each stackslot.
@@ -510,8 +558,8 @@ fn get_special_purpose_param_register(
 ) -> Option<Reg> {
     let idx = f.signature.special_param_index(purpose)?;
     match abi.args[idx] {
-        ABIArg::Reg(regs, ..) => Some(regs.only_reg().unwrap().to_reg()),
-        ABIArg::Stack(..) => None,
+        ABIArg::Reg { regs, .. } => Some(regs.only_reg().unwrap().to_reg()),
+        _ => None,
     }
 }
 
@@ -520,7 +568,8 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
     pub fn new(f: &ir::Function, flags: settings::Flags) -> CodegenResult<Self> {
         debug!("ABI: func signature {:?}", f.signature);
 
-        let sig = ABISig::from_func_sig::<M>(&f.signature)?;
+        let ir_sig = ensure_struct_return_ptr_is_returned(&f.signature);
+        let sig = ABISig::from_func_sig::<M>(&ir_sig)?;
 
         let call_conv = f.signature.call_conv;
         // Only these calling conventions are supported.
@@ -567,6 +616,7 @@ impl<M: ABIMachineSpec> ABICalleeImpl<M> {
         };
 
         Ok(Self {
+            ir_sig,
             sig,
             stackslots,
             stackslots_size: stack_offset,
@@ -787,9 +837,30 @@ fn gen_store_base_offset_multi<M: ABIMachineSpec>(
     ret
 }
 
+fn ensure_struct_return_ptr_is_returned(sig: &ir::Signature) -> ir::Signature {
+    let params_structret = sig
+        .params
+        .iter()
+        .find(|p| p.purpose == ArgumentPurpose::StructReturn);
+    let rets_have_structret = sig.returns.len() > 0
+        && sig
+            .returns
+            .iter()
+            .any(|arg| arg.purpose == ArgumentPurpose::StructReturn);
+    let mut sig = sig.clone();
+    if params_structret.is_some() && !rets_have_structret {
+        sig.returns.insert(0, params_structret.unwrap().clone());
+    }
+    sig
+}
+
 impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
     type I = M::I;
 
+    fn signature(&self) -> &ir::Signature {
+        &self.ir_sig
+    }
+
     fn temp_needed(&self) -> Option<Type> {
         if self.sig.stack_ret_arg.is_some() {
             Some(M::word_type())
@@ -822,7 +893,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
     fn liveins(&self) -> Set<RealReg> {
         let mut set: Set<RealReg> = Set::empty();
         for &arg in &self.sig.args {
-            if let ABIArg::Reg(regs, ..) = arg {
+            if let ABIArg::Reg { regs, .. } = arg {
                 for &r in regs.regs() {
                     set.insert(r);
                 }
@@ -834,7 +905,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
     fn liveouts(&self) -> Set<RealReg> {
         let mut set: Set<RealReg> = Set::empty();
         for &ret in &self.sig.rets {
-            if let ABIArg::Reg(regs, ..) = ret {
+            if let ABIArg::Reg { regs, .. } = ret {
                 for &r in regs.regs() {
                     set.insert(r);
                 }
@@ -863,14 +934,25 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         match &self.sig.args[idx] {
             // Extension mode doesn't matter (we're copying out, not in; we
             // ignore high bits by convention).
-            &ABIArg::Reg(regs, ty, ..) => {
+            &ABIArg::Reg { regs, ty, .. } => {
                 gen_move_multi::<M>(into_regs, regs.map(|r| r.to_reg()), ty)
             }
-            &ABIArg::Stack(off, ty, ..) => gen_load_stack_multi::<M>(
-                StackAMode::FPOffset(M::fp_to_arg_offset(self.call_conv, &self.flags) + off, ty),
+            &ABIArg::Stack { offset, ty, .. } => gen_load_stack_multi::<M>(
+                StackAMode::FPOffset(
+                    M::fp_to_arg_offset(self.call_conv, &self.flags) + offset,
+                    ty,
+                ),
                 into_regs,
                 ty,
             ),
+            &ABIArg::StructArg { offset, .. } => smallvec![M::gen_get_stack_addr(
+                StackAMode::FPOffset(
+                    M::fp_to_arg_offset(self.call_conv, &self.flags) + offset,
+                    I8,
+                ),
+                into_regs.only_reg().unwrap(),
+                I8,
+            )],
         }
     }
 
@@ -892,10 +974,15 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         let mut ret = smallvec![];
         let word_bits = M::word_bits() as u8;
         match &self.sig.rets[idx] {
-            &ABIArg::Reg(regs, ty, ext, ..) => {
+            &ABIArg::Reg {
+                regs,
+                ty,
+                extension,
+                ..
+            } => {
                 let from_bits = ty_bits(ty) as u8;
                 let dest_regs = writable_value_regs(regs.map(|r| r.to_reg()));
-                let ext = M::get_ext_mode(self.sig.call_conv, ext);
+                let ext = M::get_ext_mode(self.sig.call_conv, extension);
                 match (ext, from_bits) {
                     (ArgumentExtension::Uext, n) | (ArgumentExtension::Sext, n)
                         if n < word_bits =>
@@ -921,14 +1008,20 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
                     ),
                 };
             }
-            &ABIArg::Stack(off, mut ty, ext, ..) => {
+            &ABIArg::Stack {
+                offset,
+                ty,
+                extension,
+                ..
+            } => {
+                let mut ty = ty;
                 let from_bits = ty_bits(ty) as u8;
                 // A machine ABI implementation should ensure that stack frames
                 // have "reasonable" size. All current ABIs for machinst
                 // backends (aarch64 and x64) enforce a 128MB limit.
-                let off = i32::try_from(off)
+                let off = i32::try_from(offset)
                     .expect("Argument stack offset greater than 2GB; should hit impl limit first");
-                let ext = M::get_ext_mode(self.sig.call_conv, ext);
+                let ext = M::get_ext_mode(self.sig.call_conv, extension);
                 // Trash the from_reg; it should be its last use.
                 match (ext, from_bits) {
                     (ArgumentExtension::Uext, n) | (ArgumentExtension::Sext, n)
@@ -961,6 +1054,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
                     .into_iter(),
                 );
             }
+            &ABIArg::StructArg { .. } => panic!("Unexpected StructArg location for return value"),
         }
         ret
     }
@@ -1248,7 +1342,7 @@ fn abisig_to_uses_and_defs<M: ABIMachineSpec>(sig: &ABISig) -> (Vec<Reg>, Vec<Wr
     let mut uses = Vec::new();
     for arg in &sig.args {
         match arg {
-            &ABIArg::Reg(regs, ..) => uses.extend(regs.regs().iter().map(|r| r.to_reg())),
+            &ABIArg::Reg { regs, .. } => uses.extend(regs.regs().iter().map(|r| r.to_reg())),
             _ => {}
         }
     }
@@ -1257,7 +1351,7 @@ fn abisig_to_uses_and_defs<M: ABIMachineSpec>(sig: &ABISig) -> (Vec<Reg>, Vec<Wr
     let mut defs = M::get_regs_clobbered_by_call(sig.call_conv);
     for ret in &sig.rets {
         match ret {
-            &ABIArg::Reg(regs, ..) => {
+            &ABIArg::Reg { regs, .. } => {
                 defs.extend(regs.regs().iter().map(|r| Writable::from_reg(r.to_reg())))
             }
             _ => {}
@@ -1269,6 +1363,8 @@ fn abisig_to_uses_and_defs<M: ABIMachineSpec>(sig: &ABISig) -> (Vec<Reg>, Vec<Wr
 
 /// ABI object for a callsite.
 pub struct ABICallerImpl<M: ABIMachineSpec> {
+    /// CLIF-level signature, possibly normalized.
+    ir_sig: ir::Signature,
     /// The called function's signature.
     sig: ABISig,
     /// All uses for the callsite, i.e., function args.
@@ -1281,6 +1377,8 @@ pub struct ABICallerImpl<M: ABIMachineSpec> {
     opcode: ir::Opcode,
     /// Caller's calling convention.
     caller_conv: isa::CallConv,
+    /// The settings controlling this compilation.
+    flags: settings::Flags,
 
     _mach: PhantomData<M>,
 }
@@ -1301,16 +1399,20 @@ impl<M: ABIMachineSpec> ABICallerImpl<M> {
         extname: &ir::ExternalName,
         dist: RelocDistance,
         caller_conv: isa::CallConv,
+        flags: &settings::Flags,
     ) -> CodegenResult<ABICallerImpl<M>> {
-        let sig = ABISig::from_func_sig::<M>(sig)?;
+        let ir_sig = ensure_struct_return_ptr_is_returned(sig);
+        let sig = ABISig::from_func_sig::<M>(&ir_sig)?;
         let (uses, defs) = abisig_to_uses_and_defs::<M>(&sig);
         Ok(ABICallerImpl {
+            ir_sig,
             sig,
             uses,
             defs,
             dest: CallDest::ExtName(extname.clone(), dist),
             opcode: ir::Opcode::Call,
             caller_conv,
+            flags: flags.clone(),
             _mach: PhantomData,
         })
     }
@@ -1322,16 +1424,20 @@ impl<M: ABIMachineSpec> ABICallerImpl<M> {
         ptr: Reg,
         opcode: ir::Opcode,
         caller_conv: isa::CallConv,
+        flags: &settings::Flags,
     ) -> CodegenResult<ABICallerImpl<M>> {
-        let sig = ABISig::from_func_sig::<M>(sig)?;
+        let ir_sig = ensure_struct_return_ptr_is_returned(sig);
+        let sig = ABISig::from_func_sig::<M>(&ir_sig)?;
         let (uses, defs) = abisig_to_uses_and_defs::<M>(&sig);
         Ok(ABICallerImpl {
+            ir_sig,
             sig,
             uses,
             defs,
             dest: CallDest::Reg(ptr),
             opcode,
             caller_conv,
+            flags: flags.clone(),
             _mach: PhantomData,
         })
     }
@@ -1355,6 +1461,10 @@ fn adjust_stack_and_nominal_sp<M: ABIMachineSpec, C: LowerCtx<I = M::I>>(
 impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
     type I = M::I;
 
+    fn signature(&self) -> &ir::Signature {
+        &self.ir_sig
+    }
+
     fn num_args(&self) -> usize {
         if self.sig.stack_ret_arg.is_some() {
             self.sig.args.len() - 1
@@ -1387,8 +1497,13 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
         let word_rc = M::word_reg_class();
         let word_bits = M::word_bits() as usize;
         match &self.sig.args[idx] {
-            &ABIArg::Reg(regs, ty, ext, _) => {
-                let ext = M::get_ext_mode(self.sig.call_conv, ext);
+            &ABIArg::Reg {
+                regs,
+                ty,
+                extension,
+                ..
+            } => {
+                let ext = M::get_ext_mode(self.sig.call_conv, extension);
                 if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits {
                     let reg = regs.only_reg().unwrap();
                     assert_eq!(word_rc, reg.get_class());
@@ -1414,8 +1529,14 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
                     }
                 }
             }
-            &ABIArg::Stack(off, mut ty, ext, _) => {
-                let ext = M::get_ext_mode(self.sig.call_conv, ext);
+            &ABIArg::Stack {
+                offset,
+                ty,
+                extension,
+                ..
+            } => {
+                let mut ty = ty;
+                let ext = M::get_ext_mode(self.sig.call_conv, extension);
                 if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits {
                     let from_reg = from_regs
                         .only_reg()
@@ -1439,7 +1560,28 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
                     // Store the extended version.
                     ty = M::word_type();
                 }
-                for insn in gen_store_stack_multi::<M>(StackAMode::SPOffset(off, ty), from_regs, ty)
+                for insn in
+                    gen_store_stack_multi::<M>(StackAMode::SPOffset(offset, ty), from_regs, ty)
+                {
+                    ctx.emit(insn);
+                }
+            }
+            &ABIArg::StructArg { offset, size, .. } => {
+                let src_ptr = from_regs.only_reg().unwrap();
+                let dst_ptr = ctx.alloc_tmp(M::word_type()).only_reg().unwrap();
+                ctx.emit(M::gen_get_stack_addr(
+                    StackAMode::SPOffset(offset, I8),
+                    dst_ptr,
+                    I8,
+                ));
+                // Emit a memcpy from `src_ptr` to `dst_ptr` of `size` bytes.
+                // N.B.: because we process StructArg params *first*, this is
+                // safe w.r.t. clobbers: we have not yet filled in any other
+                // arg regs.
+                let memcpy_call_conv = isa::CallConv::for_libcall(&self.flags, self.sig.call_conv);
+                for insn in
+                    M::gen_memcpy(memcpy_call_conv, dst_ptr.to_reg(), src_ptr, size as usize)
+                        .into_iter()
                 {
                     ctx.emit(insn);
                 }
@@ -1447,6 +1589,24 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
         }
     }
 
+    fn get_copy_to_arg_order(&self) -> SmallVec<[usize; 8]> {
+        let mut ret = SmallVec::new();
+        for (i, arg) in self.sig.args.iter().enumerate() {
+            // Struct args.
+            if arg.is_struct_arg() {
+                ret.push(i);
+            }
+        }
+        for (i, arg) in self.sig.args.iter().enumerate() {
+            // Non-struct args. Skip an appended return-area arg for multivalue
+            // returns, if any.
+            if !arg.is_struct_arg() && i < self.ir_sig.params.len() {
+                ret.push(i);
+            }
+        }
+        ret
+    }
+
     fn emit_copy_retval_to_regs<C: LowerCtx<I = Self::I>>(
         &self,
         ctx: &mut C,
@@ -1456,21 +1616,22 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
         match &self.sig.rets[idx] {
             // Extension mode doesn't matter because we're copying out, not in,
             // and we ignore high bits in our own registers by convention.
-            &ABIArg::Reg(regs, ty, _, _) => {
+            &ABIArg::Reg { regs, ty, .. } => {
                 for insn in gen_move_multi::<M>(into_regs, regs.map(|r| r.to_reg()), ty) {
                     ctx.emit(insn);
                 }
             }
-            &ABIArg::Stack(off, ty, _, _) => {
+            &ABIArg::Stack { offset, ty, .. } => {
                 let ret_area_base = self.sig.stack_arg_space;
                 for insn in gen_load_stack_multi::<M>(
-                    StackAMode::SPOffset(off + ret_area_base, ty),
+                    StackAMode::SPOffset(offset + ret_area_base, ty),
                     into_regs,
                     ty,
                 ) {
                     ctx.emit(insn);
                 }
             }
+            &ABIArg::StructArg { .. } => panic!("Unexpected StructArg location for return value"),
         }
     }
 

cranelift/codegen/src/machinst/adapter.rs

@@ -10,7 +10,11 @@ use crate::settings::Flags;
 #[cfg(feature = "testing_hooks")]
 use crate::regalloc::RegDiversions;
 
+#[cfg(feature = "unwind")]
+use crate::isa::unwind::systemv::RegisterMappingError;
+
 use core::any::Any;
+use core::hash::Hasher;
 use std::borrow::Cow;
 use std::fmt;
 use target_lexicon::Triple;
@@ -55,6 +59,10 @@ impl TargetIsa for TargetIsaAdapter {
         self.backend.flags()
     }
 
+    fn hash_all_flags(&self, hasher: &mut dyn Hasher) {
+        self.backend.hash_all_flags(hasher)
+    }
+
     fn register_info(&self) -> RegInfo {
         // Called from function's Display impl, so we need a stub here.
         RegInfo {
@@ -134,6 +142,11 @@ impl TargetIsa for TargetIsaAdapter {
         self.backend.create_systemv_cie()
     }
 
+    #[cfg(feature = "unwind")]
+    fn map_regalloc_reg_to_dwarf(&self, r: Reg) -> Result<u16, RegisterMappingError> {
+        self.backend.map_reg_to_dwarf(r)
+    }
+
     fn as_any(&self) -> &dyn Any {
         self as &dyn Any
     }

cranelift/codegen/src/machinst/debug.rs

@@ -0,0 +1,500 @@
+//! Debug info analysis: computes value-label ranges from value-label markers in
+//! generated VCode.
+//!
+//! We "reverse-engineer" debug info like this because it is far more reliable
+//! than generating it while emitting code and keeping it in sync.
+//!
+//! This works by (i) observing "value-label marker" instructions, which are
+//! semantically just an assignment from a register to a "value label" (which
+//! one can think of as another register; they represent, e.g., Wasm locals) at
+//! a certain point in the code, and (ii) observing loads and stores to the
+//! stack and register moves.
+//!
+//! We track, at every program point, the correspondence between each value
+//! label and *all* locations in which it resides. E.g., if it is stored to the
+//! stack, we remember that it is in both a register and the stack slot; but if
+//! the register is later overwritten, then we have it just in the stack slot.
+//! This allows us to avoid false-positives observing loads/stores that we think
+//! are spillslots but really aren't.
+//!
+//! We do a standard forward dataflow analysis to compute this info.
+
+use crate::ir::ValueLabel;
+use crate::machinst::*;
+use crate::value_label::{LabelValueLoc, ValueLabelsRanges, ValueLocRange};
+use log::trace;
+use regalloc::{Reg, RegUsageCollector};
+use std::collections::{HashMap, HashSet};
+use std::hash::Hash;
+
+/// Location of a labeled value: in a register or in a stack slot. Note that a
+/// value may live in more than one location; `AnalysisInfo` maps each
+/// value-label to multiple `ValueLoc`s.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+enum ValueLoc {
+    Reg(Reg),
+    /// Nominal-SP offset.
+    Stack(i64),
+}
+
+impl From<ValueLoc> for LabelValueLoc {
+    fn from(v: ValueLoc) -> Self {
+        match v {
+            ValueLoc::Reg(r) => LabelValueLoc::Reg(r),
+            ValueLoc::Stack(off) => LabelValueLoc::SPOffset(off),
+        }
+    }
+}
+
+impl ValueLoc {
+    fn is_reg(self) -> bool {
+        match self {
+            ValueLoc::Reg(_) => true,
+            _ => false,
+        }
+    }
+    fn is_stack(self) -> bool {
+        match self {
+            ValueLoc::Stack(_) => true,
+            _ => false,
+        }
+    }
+}
+
+/// Mappings at one program point.
+#[derive(Clone, Debug)]
+struct AnalysisInfo {
+    /// Nominal SP relative to real SP. If `None`, then the offset is
+    /// indeterminate (i.e., we merged to the lattice 'bottom' element). This
+    /// should not happen in well-formed code.
+    nominal_sp_offset: Option<i64>,
+    /// Forward map from labeled values to sets of locations.
+    label_to_locs: HashMap<ValueLabel, HashSet<ValueLoc>>,
+    /// Reverse map for each register indicating the value it holds, if any.
+    reg_to_label: HashMap<Reg, ValueLabel>,
+    /// Reverse map for each stack offset indicating the value it holds, if any.
+    stack_to_label: HashMap<i64, ValueLabel>,
+}
+
+/// Get the registers written (mod'd or def'd) by a machine instruction.
+fn get_inst_writes<M: MachInst>(m: &M) -> Vec<Reg> {
+    // TODO: expose this part of regalloc.rs's interface publicly.
+    let mut vecs = RegUsageCollector::get_empty_reg_vecs_test_framework_only(false);
+    let mut coll = RegUsageCollector::new(&mut vecs);
+    m.get_regs(&mut coll);
+    vecs.defs.extend(vecs.mods.into_iter());
+    vecs.defs
+}
+
+impl AnalysisInfo {
+    /// Create a new analysis state. This is the "top" lattice element at which
+    /// the fixpoint dataflow analysis starts.
+    fn new() -> Self {
+        AnalysisInfo {
+            nominal_sp_offset: Some(0),
+            label_to_locs: HashMap::new(),
+            reg_to_label: HashMap::new(),
+            stack_to_label: HashMap::new(),
+        }
+    }
+
+    /// Remove all locations for a given labeled value. Used when the labeled
+    /// value is redefined (so old values become stale).
+    fn clear_label(&mut self, label: ValueLabel) {
+        if let Some(locs) = self.label_to_locs.remove(&label) {
+            for loc in locs {
+                match loc {
+                    ValueLoc::Reg(r) => {
+                        self.reg_to_label.remove(&r);
+                    }
+                    ValueLoc::Stack(off) => {
+                        self.stack_to_label.remove(&off);
+                    }
+                }
+            }
+        }
+    }
+
+    /// Remove a label from a register, if any. Used, e.g., if the register is
+    /// overwritten.
+    fn clear_reg(&mut self, reg: Reg) {
+        if let Some(label) = self.reg_to_label.remove(&reg) {
+            if let Some(locs) = self.label_to_locs.get_mut(&label) {
+                locs.remove(&ValueLoc::Reg(reg));
+            }
+        }
+    }
+
+    /// Remove a label from a stack offset, if any. Used, e.g., when the stack
+    /// slot is overwritten.
+    fn clear_stack_off(&mut self, off: i64) {
+        if let Some(label) = self.stack_to_label.remove(&off) {
+            if let Some(locs) = self.label_to_locs.get_mut(&label) {
+                locs.remove(&ValueLoc::Stack(off));
+            }
+        }
+    }
+
+    /// Indicate that a labeled value is newly defined and its new value is in
+    /// `reg`.
+    fn def_label_at_reg(&mut self, label: ValueLabel, reg: Reg) {
+        self.clear_label(label);
+        self.label_to_locs
+            .entry(label)
+            .or_insert_with(|| HashSet::new())
+            .insert(ValueLoc::Reg(reg));
+        self.reg_to_label.insert(reg, label);
+    }
+
+    /// Process a store from a register to a stack slot (offset).
+    fn store_reg(&mut self, reg: Reg, off: i64) {
+        self.clear_stack_off(off);
+        if let Some(label) = self.reg_to_label.get(&reg) {
+            if let Some(locs) = self.label_to_locs.get_mut(label) {
+                locs.insert(ValueLoc::Stack(off));
+            }
+            self.stack_to_label.insert(off, *label);
+        }
+    }
+
+    /// Process a load from a stack slot (offset) to a register.
+    fn load_reg(&mut self, reg: Reg, off: i64) {
+        self.clear_reg(reg);
+        if let Some(&label) = self.stack_to_label.get(&off) {
+            if let Some(locs) = self.label_to_locs.get_mut(&label) {
+                locs.insert(ValueLoc::Reg(reg));
+            }
+            self.reg_to_label.insert(reg, label);
+        }
+    }
+
+    /// Process a move from one register to another.
+    fn move_reg(&mut self, to: Reg, from: Reg) {
+        self.clear_reg(to);
+        if let Some(&label) = self.reg_to_label.get(&from) {
+            if let Some(locs) = self.label_to_locs.get_mut(&label) {
+                locs.insert(ValueLoc::Reg(to));
+            }
+            self.reg_to_label.insert(to, label);
+        }
+    }
+
+    /// Update the analysis state w.r.t. an instruction's effects. Given the
+    /// state just before `inst`, this method updates `self` to be the state
+    /// just after `inst`.
+    fn step<M: MachInst>(&mut self, inst: &M) {
+        for write in get_inst_writes(inst) {
+            self.clear_reg(write);
+        }
+        if let Some((label, reg)) = inst.defines_value_label() {
+            self.def_label_at_reg(label, reg);
+        }
+        match inst.stack_op_info() {
+            Some(MachInstStackOpInfo::LoadNomSPOff(reg, offset)) => {
+                self.load_reg(reg, offset + self.nominal_sp_offset.unwrap());
+            }
+            Some(MachInstStackOpInfo::StoreNomSPOff(reg, offset)) => {
+                self.store_reg(reg, offset + self.nominal_sp_offset.unwrap());
+            }
+            Some(MachInstStackOpInfo::NomSPAdj(offset)) => {
+                if self.nominal_sp_offset.is_some() {
+                    self.nominal_sp_offset = Some(self.nominal_sp_offset.unwrap() + offset);
+                }
+            }
+            _ => {}
+        }
+        if let Some((to, from)) = inst.is_move() {
+            let to = to.to_reg();
+            self.move_reg(to, from);
+        }
+    }
+}
+
+/// Trait used to implement the dataflow analysis' meet (intersect) function
+/// onthe `AnalysisInfo` components. For efficiency, this is implemented as a
+/// mutation on the LHS, rather than a pure functional operation.
+trait IntersectFrom {
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult;
+}
+
+/// Result of an intersection operation. Indicates whether the mutated LHS
+/// (which becomes the intersection result) differs from the original LHS. Also
+/// indicates if the value has become "empty" and should be removed from a
+/// parent container, if any.
+struct IntersectResult {
+    /// Did the intersection change the LHS input (the one that was mutated into
+    /// the result)? This is needed to drive the fixpoint loop; when no more
+    /// changes occur, then we have converted.
+    changed: bool,
+    /// Is the resulting value "empty"? This can be used when a container, such
+    /// as a map, holds values of this (intersection result) type; when
+    /// `is_empty` is true for the merge of the values at a particular key, we
+    /// can remove that key from the merged (intersected) result. This is not
+    /// necessary for analysis correctness but reduces the memory and runtime
+    /// cost of the fixpoint loop.
+    is_empty: bool,
+}
+
+impl IntersectFrom for AnalysisInfo {
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        let mut changed = false;
+        changed |= self
+            .nominal_sp_offset
+            .intersect_from(&other.nominal_sp_offset)
+            .changed;
+        changed |= self
+            .label_to_locs
+            .intersect_from(&other.label_to_locs)
+            .changed;
+        changed |= self
+            .reg_to_label
+            .intersect_from(&other.reg_to_label)
+            .changed;
+        changed |= self
+            .stack_to_label
+            .intersect_from(&other.stack_to_label)
+            .changed;
+        IntersectResult {
+            changed,
+            is_empty: false,
+        }
+    }
+}
+
+impl<K, V> IntersectFrom for HashMap<K, V>
+where
+    K: Copy + Eq + Hash,
+    V: IntersectFrom,
+{
+    /// Intersection for hashmap: remove keys that are not in both inputs;
+    /// recursively intersect values for keys in common.
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        let mut changed = false;
+        let mut remove_keys = vec![];
+        for k in self.keys() {
+            if !other.contains_key(k) {
+                remove_keys.push(*k);
+            }
+        }
+        for k in &remove_keys {
+            changed = true;
+            self.remove(k);
+        }
+
+        remove_keys.clear();
+        for k in other.keys() {
+            if let Some(v) = self.get_mut(k) {
+                let result = v.intersect_from(other.get(k).unwrap());
+                changed |= result.changed;
+                if result.is_empty {
+                    remove_keys.push(*k);
+                }
+            }
+        }
+        for k in &remove_keys {
+            changed = true;
+            self.remove(k);
+        }
+
+        IntersectResult {
+            changed,
+            is_empty: self.len() == 0,
+        }
+    }
+}
+impl<T> IntersectFrom for HashSet<T>
+where
+    T: Copy + Eq + Hash,
+{
+    /// Intersection for hashset: just take the set intersection.
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        let mut changed = false;
+        let mut remove = vec![];
+        for val in self.iter() {
+            if !other.contains(val) {
+                remove.push(*val);
+            }
+        }
+        for val in remove {
+            changed = true;
+            self.remove(&val);
+        }
+
+        IntersectResult {
+            changed,
+            is_empty: self.len() == 0,
+        }
+    }
+}
+impl IntersectFrom for ValueLabel {
+    // Intersection for labeled value: remove if not equal. This is equivalent
+    // to a three-level lattice with top, bottom, and unordered set of
+    // individual labels in between.
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        IntersectResult {
+            changed: false,
+            is_empty: *self != *other,
+        }
+    }
+}
+impl<T> IntersectFrom for Option<T>
+where
+    T: Copy + Eq,
+{
+    /// Intersectino for Option<T>: recursively intersect if both `Some`, else
+    /// `None`.
+    fn intersect_from(&mut self, other: &Self) -> IntersectResult {
+        let mut changed = false;
+        if !(self.is_some() && other.is_some() && self == other) {
+            changed = true;
+            *self = None;
+        }
+        IntersectResult {
+            changed,
+            is_empty: self.is_none(),
+        }
+    }
+}
+
+/// Compute the value-label ranges (locations for program-point ranges for
+/// labeled values) from a given `VCode` compilation result.
+///
+/// In order to compute this information, we perform a dataflow analysis on the
+/// machine code. To do so, and translate the results into a form usable by the
+/// debug-info consumers, we need to know two additional things:
+///
+/// - The machine-code layout (code offsets) of the instructions. DWARF is
+///   encoded in terms of instruction *ends* (and we reason about value
+///   locations at program points *after* instructions, to match this), so we
+///   take an array `inst_ends`, giving us code offsets for each instruction's
+///   end-point. (Note that this is one *past* the last byte; so a 4-byte
+///   instruction at offset 0 has an end offset of 4.)
+///
+/// - The locations of the labels to which branches will jump. Branches can tell
+///   us about their targets in terms of `MachLabel`s, but we don't know where
+///   those `MachLabel`s will be placed in the linear array of instructions.  We
+///   take the array `label_insn_index` to provide this info: for a label with
+///   index `l`, `label_insn_index[l]` is the index of the instruction before
+///   which that label is bound.
+pub(crate) fn compute<I: VCodeInst>(
+    insts: &[I],
+    inst_ends: &[u32],
+    label_insn_index: &[u32],
+) -> ValueLabelsRanges {
+    let inst_start = |idx: usize| if idx == 0 { 0 } else { inst_ends[idx - 1] };
+
+    trace!("compute: insts =");
+    for i in 0..insts.len() {
+        trace!(" #{} end: {} -> {:?}", i, inst_ends[i], insts[i]);
+    }
+    trace!("label_insn_index: {:?}", label_insn_index);
+
+    // Info at each block head, indexed by label.
+    let mut block_starts: HashMap<u32, AnalysisInfo> = HashMap::new();
+
+    // Initialize state at entry.
+    block_starts.insert(0, AnalysisInfo::new());
+
+    // Worklist: label indices for basic blocks.
+    let mut worklist = Vec::new();
+    let mut worklist_set = HashSet::new();
+    worklist.push(0);
+    worklist_set.insert(0);
+
+    while !worklist.is_empty() {
+        let block = worklist.pop().unwrap();
+        worklist_set.remove(&block);
+
+        let mut state = block_starts.get(&block).unwrap().clone();
+        trace!("at block {} -> state: {:?}", block, state);
+        // Iterate for each instruction in the block (we break at the first
+        // terminator we see).
+        let mut index = label_insn_index[block as usize];
+        while index < insts.len() as u32 {
+            state.step(&insts[index as usize]);
+            trace!(" -> inst #{}: {:?}", index, insts[index as usize]);
+            trace!("    --> state: {:?}", state);
+
+            let term = insts[index as usize].is_term();
+            if term.is_term() {
+                for succ in term.get_succs() {
+                    trace!("    SUCCESSOR block {}", succ.get());
+                    if let Some(succ_state) = block_starts.get_mut(&succ.get()) {
+                        trace!("       orig state: {:?}", succ_state);
+                        if succ_state.intersect_from(&state).changed {
+                            if worklist_set.insert(succ.get()) {
+                                worklist.push(succ.get());
+                            }
+                            trace!("        (changed)");
+                        }
+                        trace!("       new state: {:?}", succ_state);
+                    } else {
+                        // First time seeing this block
+                        block_starts.insert(succ.get(), state.clone());
+                        worklist.push(succ.get());
+                        worklist_set.insert(succ.get());
+                    }
+                }
+                break;
+            }
+
+            index += 1;
+        }
+    }
+
+    // Now iterate over blocks one last time, collecting
+    // value-label locations.
+
+    let mut value_labels_ranges: ValueLabelsRanges = HashMap::new();
+    for block in 0..label_insn_index.len() {
+        let start_index = label_insn_index[block];
+        let end_index = if block == label_insn_index.len() - 1 {
+            insts.len() as u32
+        } else {
+            label_insn_index[block + 1]
+        };
+        let block = block as u32;
+        let mut state = block_starts.get(&block).unwrap().clone();
+        for index in start_index..end_index {
+            let offset = inst_start(index as usize);
+            let end = inst_ends[index as usize];
+            state.step(&insts[index as usize]);
+
+            for (label, locs) in &state.label_to_locs {
+                trace!("   inst {} has label {:?} -> locs {:?}", index, label, locs);
+                // Find an appropriate loc: a register if possible, otherwise pick the first stack
+                // loc.
+                let reg = locs.iter().cloned().find(|l| l.is_reg());
+                let loc = reg.or_else(|| locs.iter().cloned().find(|l| l.is_stack()));
+                if let Some(loc) = loc {
+                    let loc = LabelValueLoc::from(loc);
+                    let list = value_labels_ranges.entry(*label).or_insert_with(|| vec![]);
+                    // If the existing location list for this value-label is
+                    // either empty, or has an end location that does not extend
+                    // to the current offset, then we have to append a new
+                    // entry. Otherwise, we can extend the current entry.
+                    //
+                    // Note that `end` is one past the end of the instruction;
+                    // it appears that `end` is exclusive, so a mapping valid at
+                    // offset 5 will have start = 5, end = 6.
+                    if list
+                        .last()
+                        .map(|last| last.end <= offset || last.loc != loc)
+                        .unwrap_or(true)
+                    {
+                        list.push(ValueLocRange {
+                            loc,
+                            start: end,
+                            end: end + 1,
+                        });
+                    } else {
+                        list.last_mut().unwrap().end = end + 1;
+                    }
+                }
+            }
+        }
+    }
+
+    trace!("ret: {:?}", value_labels_ranges);
+    value_labels_ranges
+}

cranelift/codegen/src/machinst/lower.rs

@@ -13,6 +13,7 @@ use crate::ir::instructions::BranchInfo;
 use crate::ir::{
     ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst,
     InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef,
+    ValueLabelAssignments, ValueLabelStart,
 };
 use crate::machinst::{
     writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode,
@@ -24,7 +25,7 @@ use alloc::vec::Vec;
 use core::convert::TryInto;
 use log::debug;
 use regalloc::{Reg, StackmapRequestInfo, Writable};
-use smallvec::SmallVec;
+use smallvec::{smallvec, SmallVec};
 use std::fmt::Debug;
 
 /// An "instruction color" partitions CLIF instructions by side-effecting ops.
@@ -375,8 +376,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             }
         }
 
-        let vm_context = f
-            .signature
+        let vm_context = vcode
+            .abi()
+            .signature()
             .special_param_index(ArgumentPurpose::VMContext)
             .map(|vm_context_index| {
                 let entry_block = f.layout.entry_block().unwrap();
@@ -386,7 +388,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
 
         // Assign vreg(s) to each return value.
         let mut retval_regs = vec![];
-        for ret in &f.signature.returns {
+        for ret in &vcode.abi().signature().returns.clone() {
             let regs = alloc_vregs(ret.value_type, &mut next_vreg, &mut vcode)?;
             retval_regs.push(regs);
             debug!("retval gets regs {:?}", regs);
@@ -465,6 +467,24 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
                 for insn in self.vcode.abi().gen_copy_arg_to_regs(i, regs).into_iter() {
                     self.emit(insn);
                 }
+                if self.abi().signature().params[i].purpose == ArgumentPurpose::StructReturn {
+                    assert!(regs.len() == 1);
+                    let ty = self.abi().signature().params[i].value_type;
+                    // The ABI implementation must have ensured that a StructReturn
+                    // arg is present in the return values.
+                    let struct_ret_idx = self
+                        .abi()
+                        .signature()
+                        .returns
+                        .iter()
+                        .position(|ret| ret.purpose == ArgumentPurpose::StructReturn)
+                        .expect("StructReturn return value not present!");
+                    self.emit(I::gen_move(
+                        Writable::from_reg(self.retval_regs[struct_ret_idx].regs()[0]),
+                        regs.regs()[0].to_reg(),
+                        ty,
+                    ));
+                }
             }
             if let Some(insn) = self.vcode.abi().gen_retval_area_setup() {
                 self.emit(insn);
@@ -473,6 +493,14 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
     }
 
     fn gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn) {
+        // Hack: to keep `vmctx` alive, if it exists, we emit a value label here
+        // for it if debug info is requested. This ensures that it exists either
+        // in a register or spillslot throughout the entire function body, and
+        // allows for a better debugging experience.
+        if let Some(vmctx_val) = self.f.special_param(ArgumentPurpose::VMContext) {
+            self.emit_value_label_marks_for_value(vmctx_val);
+        }
+
         let retval_regs = self.retval_regs.clone();
         for (i, regs) in retval_regs.into_iter().enumerate() {
             let regs = writable_value_regs(regs);
@@ -706,6 +734,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             if has_side_effect || value_needed {
                 debug!("lowering: inst {}: {:?}", inst, self.f.dfg[inst]);
                 backend.lower(self, inst)?;
+                // Emit value-label markers if needed, to later recover debug
+                // mappings.
+                self.emit_value_label_markers_for_inst(inst);
             }
             if data.opcode().is_return() {
                 // Return: handle specially, using ABI-appropriate sequence.
@@ -725,6 +756,80 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
         Ok(())
     }
 
+    fn get_value_labels<'a>(&'a self, val: Value, depth: usize) -> Option<&'a [ValueLabelStart]> {
+        if let Some(ref values_labels) = self.f.dfg.values_labels {
+            debug!(
+                "get_value_labels: val {} -> {} -> {:?}",
+                val,
+                self.f.dfg.resolve_aliases(val),
+                values_labels.get(&self.f.dfg.resolve_aliases(val))
+            );
+            let val = self.f.dfg.resolve_aliases(val);
+            match values_labels.get(&val) {
+                Some(&ValueLabelAssignments::Starts(ref list)) => Some(&list[..]),
+                Some(&ValueLabelAssignments::Alias { value, .. }) if depth < 10 => {
+                    self.get_value_labels(value, depth + 1)
+                }
+                _ => None,
+            }
+        } else {
+            None
+        }
+    }
+
+    fn emit_value_label_marks_for_value(&mut self, val: Value) {
+        let mut markers: SmallVec<[I; 4]> = smallvec![];
+        let regs = self.value_regs[val];
+        if regs.len() > 1 {
+            return;
+        }
+        let reg = regs.only_reg().unwrap();
+
+        if let Some(label_starts) = self.get_value_labels(val, 0) {
+            let labels = label_starts
+                .iter()
+                .map(|&ValueLabelStart { label, .. }| label)
+                .collect::<FxHashSet<_>>();
+            for label in labels {
+                debug!(
+                    "value labeling: defines val {:?} -> reg {:?} -> label {:?}",
+                    val, reg, label,
+                );
+                markers.push(I::gen_value_label_marker(label, reg));
+            }
+        }
+        for marker in markers {
+            self.emit(marker);
+        }
+    }
+
+    fn emit_value_label_markers_for_inst(&mut self, inst: Inst) {
+        if self.f.dfg.values_labels.is_none() {
+            return;
+        }
+
+        debug!(
+            "value labeling: srcloc {}: inst {}",
+            self.srcloc(inst),
+            inst
+        );
+        for &val in self.f.dfg.inst_results(inst) {
+            self.emit_value_label_marks_for_value(val);
+        }
+    }
+
+    fn emit_value_label_markers_for_block_args(&mut self, block: Block) {
+        if self.f.dfg.values_labels.is_none() {
+            return;
+        }
+
+        debug!("value labeling: block {}", block);
+        for &arg in self.f.dfg.block_params(block) {
+            self.emit_value_label_marks_for_value(arg);
+        }
+        self.finish_ir_inst(SourceLoc::default());
+    }
+
     fn finish_ir_inst(&mut self, loc: SourceLoc) {
         // `bb_insts` is kept in reverse order, so emit the instructions in
         // reverse order.
@@ -866,6 +971,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
             // Original block body.
             if let Some(bb) = lb.orig_block() {
                 self.lower_clif_block(backend, bb)?;
+                self.emit_value_label_markers_for_block_args(bb);
             }
             // In-edge phi moves.
             if let Some((pred, inst, succ)) = lb.in_edge() {

cranelift/codegen/src/machinst/mod.rs

@@ -52,45 +52,9 @@
 //!         |                        - all symbolic stack references to
 //!         |                          stackslots and spillslots are resolved
 //!         |                          to concrete FP-offset mem addresses.)
-//!         | [block/insn ordering]
 //!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - vcode.final_block_order is filled in.
-//!         |                        - new insn sequence from regalloc is
-//!         |                          placed back into vcode and block
-//!         |                          boundaries are updated.)
-//!         | [redundant branch/block
-//!         |  removal]
-//!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - all blocks that were just an
-//!         |                          unconditional branch are removed.)
-//!         |
-//!         | [branch finalization
-//!         |  (fallthroughs)]
-//!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - all branches are in lowered one-
-//!         |                          target form, but targets are still
-//!         |                          block indices.)
-//!         |
-//!         | [branch finalization
-//!         |  (offsets)]
-//!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - all branch offsets from start of
-//!         |                          function are known, and all branches
-//!         |                          have resolved-offset targets.)
-//!         |
-//!         | [MemArg finalization]
-//!         |
-//!     VCode<arch_backend::Inst>   (machine instructions:
-//!         |                        - all MemArg references to the constant
-//!         |                          pool are replaced with offsets.
-//!         |                        - all constant-pool data is collected
-//!         |                          in the VCode.)
-//!         |
-//!         | [binary emission]
+//!         | [binary emission via MachBuffer
+//!         |  with streaming branch resolution/simplification]
 //!         |
 //!     Vec<u8>                     (machine code!)
 //!
@@ -98,11 +62,11 @@
 
 use crate::binemit::{CodeInfo, CodeOffset, StackMap};
 use crate::ir::condcodes::IntCC;
-use crate::ir::{Function, SourceLoc, Type};
+use crate::ir::{Function, SourceLoc, Type, ValueLabel};
 use crate::isa::unwind::input as unwind_input;
 use crate::result::CodegenResult;
 use crate::settings::Flags;
-
+use crate::value_label::ValueLabelsRanges;
 use alloc::boxed::Box;
 use alloc::vec::Vec;
 use core::fmt::Debug;
@@ -111,10 +75,14 @@ use regalloc::RegUsageCollector;
 use regalloc::{
     RealReg, RealRegUniverse, Reg, RegClass, RegUsageMapper, SpillSlot, VirtualReg, Writable,
 };
-use smallvec::SmallVec;
+use smallvec::{smallvec, SmallVec};
+use std::hash::Hasher;
 use std::string::String;
 use target_lexicon::Triple;
 
+#[cfg(feature = "unwind")]
+use crate::isa::unwind::systemv::RegisterMappingError;
+
 pub mod lower;
 pub use lower::*;
 pub mod vcode;
@@ -137,6 +105,7 @@ pub mod inst_common;
 pub use inst_common::*;
 pub mod valueregs;
 pub use valueregs::*;
+pub mod debug;
 
 /// A machine instruction.
 pub trait MachInst: Clone + Debug {
@@ -163,6 +132,11 @@ pub trait MachInst: Clone + Debug {
         true
     }
 
+    /// If this is a load or store to the stack, return that info.
+    fn stack_op_info(&self) -> Option<MachInstStackOpInfo> {
+        None
+    }
+
     /// Generate a move.
     fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Self;
 
@@ -174,9 +148,6 @@ pub trait MachInst: Clone + Debug {
         alloc_tmp: F,
     ) -> SmallVec<[Self; 4]>;
 
-    /// Generate a zero-length no-op.
-    fn gen_zero_len_nop() -> Self;
-
     /// Possibly operate on a value directly in a spill-slot rather than a
     /// register. Useful if the machine has register-memory instruction forms
     /// (e.g., add directly from or directly to memory), like x86.
@@ -204,7 +175,7 @@ pub trait MachInst: Clone + Debug {
     /// request a NOP of that size, or as close to it as possible. The machine
     /// backend may return a NOP whose binary encoding is smaller than the
     /// preferred size, but must not return a NOP that is larger. However,
-    /// the instruction must have a nonzero size.
+    /// the instruction must have a nonzero size if preferred_size is nonzero.
     fn gen_nop(preferred_size: usize) -> Self;
 
     /// Get the register universe for this backend.
@@ -223,6 +194,17 @@ pub trait MachInst: Clone + Debug {
     /// be dependent on compilation flags.
     fn ref_type_regclass(_flags: &Flags) -> RegClass;
 
+    /// Does this instruction define a ValueLabel? Returns the `Reg` whose value
+    /// becomes the new value of the `ValueLabel` after this instruction.
+    fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> {
+        None
+    }
+
+    /// Create a marker instruction that defines a value label.
+    fn gen_value_label_marker(_label: ValueLabel, _reg: Reg) -> Self {
+        Self::gen_nop(0)
+    }
+
     /// A label-use kind: a type that describes the types of label references that
     /// can occur in an instruction.
     type LabelUse: MachInstLabelUse;
@@ -285,6 +267,35 @@ pub enum MachTerminator<'a> {
     Indirect(&'a [MachLabel]),
 }
 
+impl<'a> MachTerminator<'a> {
+    /// Get the successor labels named in a `MachTerminator`.
+    pub fn get_succs(&self) -> SmallVec<[MachLabel; 2]> {
+        let mut ret = smallvec![];
+        match self {
+            &MachTerminator::Uncond(l) => {
+                ret.push(l);
+            }
+            &MachTerminator::Cond(l1, l2) => {
+                ret.push(l1);
+                ret.push(l2);
+            }
+            &MachTerminator::Indirect(ls) => {
+                ret.extend(ls.iter().cloned());
+            }
+            _ => {}
+        }
+        ret
+    }
+
+    /// Is this a terminator?
+    pub fn is_term(&self) -> bool {
+        match self {
+            MachTerminator::None => false,
+            _ => true,
+        }
+    }
+}
+
 /// A trait describing the ability to encode a MachInst into binary machine code.
 pub trait MachInstEmit: MachInst {
     /// Persistent state carried across `emit` invocations.
@@ -330,6 +341,8 @@ pub struct MachCompileResult {
     pub disasm: Option<String>,
     /// Unwind info.
     pub unwind_info: Option<unwind_input::UnwindInfo<Reg>>,
+    /// Debug info: value labels to registers/stackslots at code offsets.
+    pub value_labels_ranges: Option<ValueLabelsRanges>,
 }
 
 impl MachCompileResult {
@@ -358,6 +371,10 @@ pub trait MachBackend {
     /// Return flags for this backend.
     fn flags(&self) -> &Flags;
 
+    /// Hashes all flags, both ISA-independent and ISA-specific, into the
+    /// specified hasher.
+    fn hash_all_flags(&self, hasher: &mut dyn Hasher);
+
     /// Return triple for this backend.
     fn triple(&self) -> Triple;
 
@@ -386,13 +403,17 @@ pub trait MachBackend {
         Ok(None)
     }
 
-    /// Machine-specific condcode info needed by TargetIsa.
     /// Creates a new System V Common Information Entry for the ISA.
     #[cfg(feature = "unwind")]
     fn create_systemv_cie(&self) -> Option<gimli::write::CommonInformationEntry> {
         // By default, an ISA cannot create a System V CIE
         None
     }
+    /// Maps a regalloc::Reg to a DWARF register number.
+    #[cfg(feature = "unwind")]
+    fn map_reg_to_dwarf(&self, _: Reg) -> Result<u16, RegisterMappingError> {
+        Err(RegisterMappingError::UnsupportedArchitecture)
+    }
 }
 
 /// Expected unwind info type.
@@ -431,3 +452,15 @@ pub trait UnwindInfoGenerator<I: MachInstEmit> {
         context: UnwindInfoContext<I>,
     ) -> CodegenResult<Option<unwind_input::UnwindInfo<Reg>>>;
 }
+
+/// Info about an operation that loads or stores from/to the stack.
+#[derive(Clone, Copy, Debug)]
+pub enum MachInstStackOpInfo {
+    /// Load from an offset from the nominal stack pointer into the given reg.
+    LoadNomSPOff(Reg, i64),
+    /// Store to an offset from the nominal stack pointer from the given reg.
+    StoreNomSPOff(Reg, i64),
+    /// Adjustment of nominal-SP up or down. This value is added to subsequent
+    /// offsets in loads/stores above to produce real-SP offsets.
+    NomSPAdj(i64),
+}

cranelift/codegen/src/machinst/vcode.rs

@@ -21,7 +21,6 @@ use crate::ir::{self, types, Constant, ConstantData, SourceLoc};
 use crate::machinst::*;
 use crate::settings;
 use crate::timing;
-
 use regalloc::Function as RegallocFunction;
 use regalloc::Set as RegallocSet;
 use regalloc::{
@@ -110,11 +109,19 @@ pub struct VCode<I: VCodeInst> {
     /// Ranges for prologue and epilogue instructions.
     prologue_epilogue_ranges: Option<(InsnRange, Box<[InsnRange]>)>,
 
-    /// Instruction end offsets
-    insts_layout: RefCell<(Vec<u32>, u32)>,
+    /// Do we generate debug info?
+    generate_debug_info: bool,
+
+    /// Instruction end offsets, instruction indices at each label, and total
+    /// buffer size.  Only present if `generate_debug_info` is set.
+    insts_layout: RefCell<(Vec<u32>, Vec<u32>, u32)>,
 
     /// Constants.
     constants: VCodeConstants,
+
+    /// Are any debug value-labels present? If not, we can skip the
+    /// post-emission analysis.
+    has_value_labels: bool,
 }
 
 /// A builder for a VCode function body. This builder is designed for the
@@ -157,7 +164,13 @@ impl<I: VCodeInst> VCodeBuilder<I> {
         constants: VCodeConstants,
     ) -> VCodeBuilder<I> {
         let reftype_class = I::ref_type_regclass(abi.flags());
-        let vcode = VCode::new(abi, emit_info, block_order, constants);
+        let vcode = VCode::new(
+            abi,
+            emit_info,
+            block_order,
+            constants,
+            /* generate_debug_info = */ true,
+        );
         let stack_map_info = StackmapRequestInfo {
             reftype_class,
             reftyped_vregs: vec![],
@@ -242,6 +255,9 @@ impl<I: VCodeInst> VCodeBuilder<I> {
                 }
             }
         }
+        if insn.defines_value_label().is_some() {
+            self.vcode.has_value_labels = true;
+        }
         self.vcode.insts.push(insn);
         self.vcode.srclocs.push(self.cur_srcloc);
         if is_safepoint {
@@ -296,6 +312,7 @@ impl<I: VCodeInst> VCode<I> {
         emit_info: I::Info,
         block_order: BlockLoweringOrder,
         constants: VCodeConstants,
+        generate_debug_info: bool,
     ) -> VCode<I> {
         VCode {
             liveins: abi.liveins(),
@@ -314,8 +331,10 @@ impl<I: VCodeInst> VCode<I> {
             safepoint_insns: vec![],
             safepoint_slots: vec![],
             prologue_epilogue_ranges: None,
-            insts_layout: RefCell::new((vec![], 0)),
+            generate_debug_info,
+            insts_layout: RefCell::new((vec![], vec![], 0)),
             constants,
+            has_value_labels: false,
         }
     }
 
@@ -484,7 +503,8 @@ impl<I: VCodeInst> VCode<I> {
         buffer.reserve_labels_for_blocks(self.num_blocks() as BlockIndex);
         buffer.reserve_labels_for_constants(&self.constants);
 
-        let mut insts_layout = vec![0; self.insts.len()];
+        let mut inst_ends = vec![0; self.insts.len()];
+        let mut label_insn_iix = vec![0; self.num_blocks()];
 
         let mut safepoint_idx = 0;
         let mut cur_srcloc = None;
@@ -500,6 +520,7 @@ impl<I: VCodeInst> VCode<I> {
 
             let (start, end) = self.block_ranges[block as usize];
             buffer.bind_label(MachLabel::from_block(block));
+            label_insn_iix[block as usize] = start;
             for iix in start..end {
                 let srcloc = self.srclocs[iix as usize];
                 if cur_srcloc != Some(srcloc) {
@@ -526,7 +547,19 @@ impl<I: VCodeInst> VCode<I> {
 
                 self.insts[iix as usize].emit(&mut buffer, &self.emit_info, &mut state);
 
-                insts_layout[iix as usize] = buffer.cur_offset();
+                if self.generate_debug_info {
+                    // Buffer truncation may have happened since last inst append; trim inst-end
+                    // layout info as appropriate.
+                    let l = &mut inst_ends[0..iix as usize];
+                    for end in l.iter_mut().rev() {
+                        if *end > buffer.cur_offset() {
+                            *end = buffer.cur_offset();
+                        } else {
+                            break;
+                        }
+                    }
+                    inst_ends[iix as usize] = buffer.cur_offset();
+                }
             }
 
             if cur_srcloc.is_some() {
@@ -553,7 +586,16 @@ impl<I: VCodeInst> VCode<I> {
             buffer.defer_constant(label, data.alignment(), data.as_slice(), u32::max_value());
         }
 
-        *self.insts_layout.borrow_mut() = (insts_layout, buffer.cur_offset());
+        if self.generate_debug_info {
+            for end in inst_ends.iter_mut().rev() {
+                if *end > buffer.cur_offset() {
+                    *end = buffer.cur_offset();
+                } else {
+                    break;
+                }
+            }
+            *self.insts_layout.borrow_mut() = (inst_ends, label_insn_iix, buffer.cur_offset());
+        }
 
         buffer
     }
@@ -567,13 +609,27 @@ impl<I: VCodeInst> VCode<I> {
         let context = UnwindInfoContext {
             insts: &self.insts,
             insts_layout: &layout.0,
-            len: layout.1,
+            len: layout.2,
             prologue: prologue.clone(),
             epilogues,
         };
         I::UnwindInfo::create_unwind_info(context)
     }
 
+    /// Generates value-label ranges.
+    pub fn value_labels_ranges(&self) -> crate::result::CodegenResult<Option<ValueLabelsRanges>> {
+        if !self.has_value_labels {
+            return Ok(None);
+        }
+
+        let layout = &self.insts_layout.borrow();
+        Ok(Some(debug::compute(
+            &self.insts,
+            &layout.0[..],
+            &layout.1[..],
+        )))
+    }
+
     /// Get the IR block for a BlockIndex, if one exists.
     pub fn bindex_to_bb(&self, block: BlockIndex) -> Option<ir::Block> {
         self.block_order.lowered_order()[block as usize].orig_block()
@@ -670,7 +726,7 @@ impl<I: VCodeInst> RegallocFunction for VCode<I> {
     }
 
     fn gen_zero_len_nop(&self) -> I {
-        I::gen_zero_len_nop()
+        I::gen_nop(0)
     }
 
     fn maybe_direct_reload(&self, insn: &I, reg: VirtualReg, slot: SpillSlot) -> Option<I> {

cranelift/codegen/src/settings.rs

@@ -188,7 +188,7 @@ pub type SetResult<T> = Result<T, SetError>;
 /// The settings objects themselves are generated and appear in the `isa/*/settings.rs` modules.
 /// Each settings object provides a `predicate_view()` method that makes it possible to query
 /// ISA predicates by number.
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Hash)]
 pub struct PredicateView<'a>(&'a [u8]);
 
 impl<'a> PredicateView<'a> {

cranelift/codegen/src/value_label.rs

@@ -1,13 +1,16 @@
 use crate::ir::{Function, SourceLoc, Value, ValueLabel, ValueLabelAssignments, ValueLoc};
 use crate::isa::TargetIsa;
+use crate::machinst::MachCompileResult;
 use crate::regalloc::{Context, RegDiversions};
 use crate::HashMap;
 use alloc::collections::BTreeMap;
 use alloc::vec::Vec;
 use core::cmp::Ordering;
+use core::convert::From;
 use core::iter::Iterator;
 use core::ops::Bound::*;
 use core::ops::Deref;
+use regalloc::Reg;
 
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
@@ -17,13 +20,31 @@ use serde::{Deserialize, Serialize};
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct ValueLocRange {
     /// The ValueLoc containing a ValueLabel during this range.
-    pub loc: ValueLoc,
+    pub loc: LabelValueLoc,
     /// The start of the range. It is an offset in the generated code.
     pub start: u32,
     /// The end of the range. It is an offset in the generated code.
     pub end: u32,
 }
 
+/// The particular location for a value.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub enum LabelValueLoc {
+    /// Old-backend location: RegUnit, StackSlot, or Unassigned.
+    ValueLoc(ValueLoc),
+    /// New-backend Reg.
+    Reg(Reg),
+    /// New-backend offset from stack pointer.
+    SPOffset(i64),
+}
+
+impl From<ValueLoc> for LabelValueLoc {
+    fn from(v: ValueLoc) -> Self {
+        LabelValueLoc::ValueLoc(v)
+    }
+}
+
 /// Resulting map of Value labels and their ranges/locations.
 pub type ValueLabelsRanges = HashMap<ValueLabel, Vec<ValueLocRange>>;
 
@@ -86,14 +107,18 @@ where
 pub fn build_value_labels_ranges<T>(
     func: &Function,
     regalloc: &Context,
+    mach_compile_result: Option<&MachCompileResult>,
     isa: &dyn TargetIsa,
 ) -> ValueLabelsRanges
 where
     T: From<SourceLoc> + Deref<Target = SourceLoc> + Ord + Copy,
 {
-    // FIXME(#1523): New-style backend does not yet have debug info.
-    if isa.get_mach_backend().is_some() {
-        return HashMap::new();
+    if mach_compile_result.is_some() && mach_compile_result.unwrap().value_labels_ranges.is_some() {
+        return mach_compile_result
+            .unwrap()
+            .value_labels_ranges
+            .clone()
+            .unwrap();
     }
 
     let values_labels = build_value_labels_index::<T>(func);
@@ -113,7 +138,7 @@ where
             .entry(label)
             .or_insert_with(Vec::new)
             .push(ValueLocRange {
-                loc,
+                loc: loc.into(),
                 start: range.0,
                 end: range.1,
             });

cranelift/codegen/src/write.rs

@@ -11,7 +11,7 @@ use crate::ir::{
 };
 use crate::isa::{RegInfo, TargetIsa};
 use crate::packed_option::ReservedValue;
-use crate::value_label::ValueLabelsRanges;
+use crate::value_label::{LabelValueLoc, ValueLabelsRanges};
 use crate::HashSet;
 use alloc::string::String;
 use alloc::vec::Vec;
@@ -278,11 +278,13 @@ pub fn write_block_header(
     writeln!(w, "):")
 }
 
-fn write_valueloc(w: &mut dyn Write, loc: ValueLoc, regs: &RegInfo) -> fmt::Result {
+fn write_valueloc(w: &mut dyn Write, loc: LabelValueLoc, regs: &RegInfo) -> fmt::Result {
     match loc {
-        ValueLoc::Reg(r) => write!(w, "{}", regs.display_regunit(r)),
-        ValueLoc::Stack(ss) => write!(w, "{}", ss),
-        ValueLoc::Unassigned => write!(w, "?"),
+        LabelValueLoc::ValueLoc(ValueLoc::Reg(r)) => write!(w, "{}", regs.display_regunit(r)),
+        LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => write!(w, "{}", ss),
+        LabelValueLoc::ValueLoc(ValueLoc::Unassigned) => write!(w, "?"),
+        LabelValueLoc::Reg(r) => write!(w, "{:?}", r),
+        LabelValueLoc::SPOffset(off) => write!(w, "[sp+{}]", off),
     }
 }
 

cranelift/entity/src/list.rs

@@ -91,17 +91,20 @@ type SizeClass = u8;
 
 /// Get the size of a given size class. The size includes the length field, so the maximum list
 /// length is one less than the class size.
+#[inline]
 fn sclass_size(sclass: SizeClass) -> usize {
     4 << sclass
 }
 
 /// Get the size class to use for a given list length.
 /// This always leaves room for the length element in addition to the list elements.
+#[inline]
 fn sclass_for_length(len: usize) -> SizeClass {
     30 - (len as u32 | 3).leading_zeros() as SizeClass
 }
 
 /// Is `len` the minimum length in its size class?
+#[inline]
 fn is_sclass_min_length(len: usize) -> bool {
     len > 3 && len.is_power_of_two()
 }
@@ -387,16 +390,36 @@ impl<T: EntityRef + ReservedValue> EntityList<T> {
         &mut pool.data[block + 1..block + 1 + new_len]
     }
 
+    /// Constructs a list from an iterator.
+    pub fn from_iter<I>(elements: I, pool: &mut ListPool<T>) -> Self
+    where
+        I: IntoIterator<Item = T>,
+    {
+        let mut list = Self::new();
+        list.extend(elements, pool);
+        list
+    }
+
     /// Appends multiple elements to the back of the list.
     pub fn extend<I>(&mut self, elements: I, pool: &mut ListPool<T>)
     where
         I: IntoIterator<Item = T>,
     {
-        // TODO: use `size_hint()` to reduce reallocations.
-        for x in elements {
+        let iterator = elements.into_iter();
+        let (len, upper) = iterator.size_hint();
+        // On most iterators this check is optimized down to `true`.
+        if upper == Some(len) {
+            let data = self.grow(len, pool);
+            let offset = data.len() - len;
+            for (src, dst) in iterator.zip(data[offset..].iter_mut()) {
+                *dst = src;
+            }
+        } else {
+            for x in iterator {
                 self.push(x, pool);
             }
         }
+    }
 
     /// Inserts an element as position `index` in the list, shifting all elements after it to the
     /// right.
@@ -630,6 +653,10 @@ mod tests {
             list.as_slice(pool),
             &[i1, i2, i3, i4, i1, i1, i2, i2, i3, i3, i4, i4]
         );
+
+        let list2 = EntityList::from_iter([i1, i1, i2, i2, i3, i3, i4, i4].iter().cloned(), pool);
+        assert_eq!(list2.len(pool), 8);
+        assert_eq!(list2.as_slice(pool), &[i1, i1, i2, i2, i3, i3, i4, i4]);
     }
 
     #[test]

cranelift/filetests/filetests/isa/aarch64/bitops.clif

@@ -230,19 +230,10 @@ block0(v0: i64):
 
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: lsr x1, x0, #1
-; nextln: and x1, x1, #6148914691236517205
-; nextln: sub x1, x0, x1
-; nextln: and x0, x1, #3689348814741910323
-; nextln: lsr x1, x1, #2
-; nextln: and x1, x1, #3689348814741910323
-; nextln: add x0, x1, x0
-; nextln: add x0, x0, x0, LSR 4
-; nextln: and x0, x0, #1085102592571150095
-; nextln: add x0, x0, x0, LSL 8
-; nextln: add x0, x0, x0, LSL 16
-; nextln: add x0, x0, x0, LSL 32
-; nextln: lsr x0, x0, #56
+; nextln: fmov d0, x0
+; nextln: cnt v0.8b, v0.8b
+; nextln: addv b0, v0.8b
+; nextln: umov w0, v0.b[0]
 ; nextln: mov sp, fp
 ; nextln: ldp fp, lr, [sp], #16
 ; nextln: ret
@@ -255,20 +246,10 @@ block0(v0: i32):
 
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: mov w0, w0
-; nextln: lsr w1, w0, #1
-; nextln: and x1, x1, #6148914691236517205
-; nextln: sub x1, x0, x1
-; nextln: and x0, x1, #3689348814741910323
-; nextln: lsr x1, x1, #2
-; nextln: and x1, x1, #3689348814741910323
-; nextln: add x0, x1, x0
-; nextln: add x0, x0, x0, LSR 4
-; nextln: and x0, x0, #1085102592571150095
-; nextln: add x0, x0, x0, LSL 8
-; nextln: add x0, x0, x0, LSL 16
-; nextln: add x0, x0, x0, LSL 32
-; nextln: lsr x0, x0, #56
+; nextln: fmov s0, w0
+; nextln: cnt v0.8b, v0.8b
+; nextln: addv b0, v0.8b
+; nextln: umov w0, v0.b[0]
 ; nextln: mov sp, fp
 ; nextln: ldp fp, lr, [sp], #16
 ; nextln: ret
@@ -281,20 +262,10 @@ block0(v0: i16):
 
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: uxth w0, w0
-; nextln: lsr w1, w0, #1
-; nextln: and x1, x1, #6148914691236517205
-; nextln: sub x1, x0, x1
-; nextln: and x0, x1, #3689348814741910323
-; nextln: lsr x1, x1, #2
-; nextln: and x1, x1, #3689348814741910323
-; nextln: add x0, x1, x0
-; nextln: add x0, x0, x0, LSR 4
-; nextln: and x0, x0, #1085102592571150095
-; nextln: add x0, x0, x0, LSL 8
-; nextln: add x0, x0, x0, LSL 16
-; nextln: add x0, x0, x0, LSL 32
-; nextln: lsr x0, x0, #56
+; nextln: fmov s0, w0
+; nextln: cnt v0.8b, v0.8b
+; nextln: addp v0.8b, v0.8b, v0.8b
+; nextln: umov w0, v0.b[0]
 ; nextln: mov sp, fp
 ; nextln: ldp fp, lr, [sp], #16
 ; nextln: ret
@@ -307,20 +278,9 @@ block0(v0: i8):
 
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: uxtb w0, w0
-; nextln: lsr w1, w0, #1
-; nextln: and x1, x1, #6148914691236517205
-; nextln: sub x1, x0, x1
-; nextln: and x0, x1, #3689348814741910323
-; nextln: lsr x1, x1, #2
-; nextln: and x1, x1, #3689348814741910323
-; nextln: add x0, x1, x0
-; nextln: add x0, x0, x0, LSR 4
-; nextln: and x0, x0, #1085102592571150095
-; nextln: add x0, x0, x0, LSL 8
-; nextln: add x0, x0, x0, LSL 16
-; nextln: add x0, x0, x0, LSL 32
-; nextln: lsr x0, x0, #56
+; nextln: fmov s0, w0
+; nextln: cnt v0.8b, v0.8b
+; nextln: umov w0, v0.b[0]
 ; nextln: mov sp, fp
 ; nextln: ldp fp, lr, [sp], #16
 ; nextln: ret

cranelift/filetests/filetests/isa/x64/bitops-i128-run.clif

@@ -0,0 +1,27 @@
+test run
+target x86_64
+feature "experimental_x64"
+
+function %ctz(i64, i64) -> i8 {
+block0(v0: i64, v1: i64):
+    v2 = iconcat v0, v1
+    v3 = ctz.i128 v2
+    v4 = ireduce.i8 v3
+    return v4
+}
+; run: %ctz(0x00000000_00000000, 0x00000001_00000000) == 96
+; run: %ctz(0x00000000_00010000, 0x00000001_00000000) == 16
+; run: %ctz(0x00000000_00010000, 0x00000000_00000000) == 16
+; run: %ctz(0x00000000_00000000, 0x00000000_00000000) == 128
+
+function %clz(i64, i64) -> i8 {
+block0(v0: i64, v1: i64):
+    v2 = iconcat v0, v1
+    v3 = clz.i128 v2
+    v4 = ireduce.i8 v3
+    return v4
+}
+; run: %clz(0x00000000_00000000, 0x00000001_00000000) == 31
+; run: %clz(0x00000000_00010000, 0x00000001_00000000) == 31
+; run: %clz(0x00000000_00010000, 0x00000000_00000000) == 111
+; run: %clz(0x00000000_00000000, 0x00000000_00000000) == 128

cranelift/filetests/filetests/isa/x64/bitrev-i128-run.clif

@@ -0,0 +1,47 @@
+test run
+target x86_64
+feature "experimental_x64"
+
+function %reverse_bits_zero() -> b1 {
+block0:
+    v0 = iconst.i64 0
+    v1 = iconcat v0, v0
+    v2 = bitrev.i128 v1
+    v3 = icmp eq v2, v1
+    return v3
+}
+; run
+
+function %reverse_bits_one() -> b1 {
+block0:
+    v0 = iconst.i64 0
+    v1 = iconst.i64 1
+    v2 = iconcat v0, v1
+
+    v3 = bitrev.i128 v2
+
+    v4 = iconst.i64 0x8000_0000_0000_0000
+    v5 = iconst.i64 0
+    v6 = iconcat v4, v5
+
+    v7 = icmp eq v3, v6
+    return v7
+}
+; run
+
+function %reverse_bits() -> b1 {
+block0:
+    v0 = iconst.i64 0x06AD_8667_69EC_41BA
+    v1 = iconst.i64 0x6C83_D81A_6E28_83AB
+    v2 = iconcat v0, v1
+
+    v3 = bitrev.i128 v2
+
+    v4 = iconst.i64 0xD5C11476581BC136
+    v5 = iconst.i64 0x5D823796E661B560
+    v6 = iconcat v4, v5
+
+    v7 = icmp eq v3, v6
+    return v7
+}
+; run

cranelift/filetests/filetests/isa/x64/clz-lzcnt.clif

@@ -0,0 +1,31 @@
+test compile
+target x86_64 has_lzcnt
+feature "experimental_x64"
+
+function %clz(i64) -> i64 {
+block0(v0: i64):
+    v1 = clz v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: lzcntq  %rdi, %rsi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret
+
+function %clz(i32) -> i32 {
+block0(v0: i32):
+    v1 = clz v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: lzcntl  %edi, %esi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret

cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif

@@ -0,0 +1,49 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function %f0(i64, i64) -> i64, i64 {
+block0(v0: i64, v1: i64):
+    v2 = load.i64 v1
+; check:  movq    0(%rsi), %rax
+
+    v3 = icmp eq v0, v2
+
+    v4 = bint.i64 v3
+; nextln: cmpq    %rax, %rdi
+; nextln: setz    %cl
+; nextln: movzbq  %cl, %rcx
+
+    v5 = select.i64 v3, v0, v1
+; nextln: cmpq    %rax, %rdi
+; nextln: cmovzq  %rdi, %rsi
+
+    return v4, v5
+; nextln: movq    %rcx, %rax
+; nextln: movq    %rsi, %rdx
+}
+
+function %f1(f64, i64) -> i64, f64 {
+block0(v0: f64, v1: i64):
+    v2 = load.f64 v1
+; check:  movsd   0(%rdi), %xmm1
+
+    v3 = fcmp eq v0, v2
+
+    v4 = bint.i64 v3
+; nextln: ucomisd %xmm1, %xmm0
+; nextln: setnp   %dil
+; nextln: setz    %sil
+; nextln: andl    %edi, %esi
+; nextln: movzbq  %sil, %rsi
+
+    v5 = select.f64 v3, v0, v0
+; nextln: ucomisd %xmm1, %xmm0
+; nextln: movaps  %xmm0, %xmm1
+; nextln: jnp $$next; movsd %xmm0, %xmm1; $$next:
+; nextln: jz $$next; movsd %xmm0, %xmm1; $$next:
+
+    return v4, v5
+; nextln: movq    %rsi, %rax
+; nextln: movaps  %xmm1, %xmm0
+}

cranelift/filetests/filetests/isa/x64/ctz-bmi1.clif

@@ -0,0 +1,31 @@
+test compile
+target x86_64 has_bmi1
+feature "experimental_x64"
+
+function %ctz(i64) -> i64 {
+block0(v0: i64):
+    v1 = ctz v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: tzcntq  %rdi, %rsi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret
+
+function %ctz(i32) -> i32 {
+block0(v0: i32):
+    v1 = ctz v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: tzcntl  %edi, %esi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret

cranelift/filetests/filetests/isa/x64/floating-point.clif

@@ -0,0 +1,26 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function %f(f64) -> f64 {
+block0(v0: f64):
+    v1 = fabs.f64 v0
+    return v1
+}
+; check:  movabsq $$9223372036854775807, %rsi
+; nextln: movq    %rsi, %xmm1
+; nextln: andpd   %xmm0, %xmm1
+; nextln: movaps  %xmm1, %xmm0
+
+
+function %f(i64) -> f64 {
+block0(v0: i64):
+    v1 = load.f64 v0
+    v2 = fabs.f64 v1
+    return v2
+}
+; check:  movsd   0(%rdi), %xmm0
+; nextln: movabsq $$9223372036854775807, %rsi
+; nextln: movq    %rsi, %xmm1
+; nextln: andpd   %xmm0, %xmm1
+; nextln: movaps  %xmm1, %xmm0

cranelift/filetests/filetests/isa/x64/icmp-i128-run.clif

@@ -0,0 +1,95 @@
+test run
+target x86_64
+feature "experimental_x64"
+
+function %test_icmp_eq_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x0
+    v12 = iconst.i64 0x0
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x0
+    v22 = iconst.i64 0x0
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 eq v1, v2
+    return v10
+}
+
+; run
+
+function %test_icmp_imm_eq_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x0
+    v12 = iconst.i64 0x0
+    v1 = iconcat v11, v12
+    v10 = icmp_imm.i128 eq v1, 0x0
+    return v10
+}
+
+; run
+
+function %test_icmp_ne_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x0
+    v12 = iconst.i64 0x0
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x0
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 ne v1, v2
+    return v10
+}
+
+; run
+
+function %test_icmp_imm_ne_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x0
+    v12 = iconst.i64 0x0
+    v1 = iconcat v11, v12
+    v10 = icmp_imm.i128 ne v1, 0x1
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_eq_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 eq v1, v2
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_gt_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x2
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 ugt v2, v1
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_ge_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 uge v1, v2
+    return v10
+}
+
+; run

cranelift/filetests/filetests/isa/x64/popcnt-use-popcnt.clif

@@ -0,0 +1,31 @@
+test compile
+target x86_64 has_popcnt has_sse42
+feature "experimental_x64"
+
+function %popcnt(i64) -> i64 {
+block0(v0: i64):
+    v1 = popcnt v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: popcntq %rdi, %rsi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret
+
+function %popcnt(i32) -> i32 {
+block0(v0: i32):
+    v1 = popcnt v0
+    return v1
+}
+
+; check: pushq   %rbp
+; check: movq    %rsp, %rbp
+; check: popcntl %edi, %esi
+; check: movq    %rsi, %rax
+; check: movq    %rbp, %rsp
+; check: popq    %rbp
+; check: ret

cranelift/filetests/filetests/isa/x64/select-i128.clif

@@ -0,0 +1,29 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function %f0(i32, i128, i128) -> i128 {
+; check:   pushq   %rbp
+; nextln:  movq    %rsp, %rbp
+
+block0(v0: i32, v1: i128, v2: i128):
+
+    v3 = iconst.i32 42
+    v4 = icmp.i32 eq v0, v3
+; nextln: movl    $$42, %eax
+; nextln: cmpl    %eax, %edi
+
+    v5 = select.i128 v4, v1, v2
+; nextln: cmovzq  %rsi, %rcx
+; nextln: cmovzq  %rdx, %r8
+
+    return v5
+; nextln: movq    %rcx, %rax
+; nextln: movq    %r8, %rdx
+
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+ 
+}
+

cranelift/filetests/filetests/isa/x64/shift-i128-run.clif

@@ -0,0 +1,106 @@
+test run
+target x86_64
+feature "experimental_x64"
+
+function %ishl1() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconcat v0, v0
+    v2 = iconst.i32 2
+    v3 = ishl.i128 v1, v2
+    v4 = iconst.i64 0x04040404_04040404
+    v5 = iconcat v4, v4
+    v6 = icmp eq v3, v5
+    return v6
+}
+; run
+
+function %ishl2() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0x01010101_01010101
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 9
+    v4 = ishl.i128 v2, v3
+    v5 = iconst.i64 0x02020202_02020200
+    v6 = iconst.i64 0x02020202_02020202
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %ishl3() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0xffffffff_ffffffff
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 66
+    v4 = ishl.i128 v2, v3
+    v5 = iconst.i64 0x00000000_00000000
+    v6 = iconst.i64 0x04040404_04040404
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %ushr1() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0x01010101_01010101
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 2
+    v4 = ushr.i128 v2, v3
+    v5 = iconst.i64 0x40404040_40404040
+    v6 = iconst.i64 0x00404040_40404040
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %ushr2() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0x01010101_01010101
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 66
+    v4 = ushr.i128 v2, v3
+    v5 = iconst.i64 0x00404040_40404040
+    v6 = iconst.i64 0x00000000_00000000
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %sshr1() -> b1 {
+block0:
+    v0 = iconst.i64 0x01010101_01010101
+    v1 = iconst.i64 0x81010101_01010101
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 2
+    v4 = sshr.i128 v2, v3
+    v5 = iconst.i64 0x40404040_40404040
+    v6 = iconst.i64 0xe0404040_40404040
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run
+
+function %sshr2() -> b1 {
+block0:
+    v0 = iconst.i64 0x12345678_9abcdef0
+    v1 = iconst.i64 0x80101010_10101010
+    v2 = iconcat v0, v1
+    v3 = iconst.i32 66
+    v4 = sshr.i128 v2, v3
+    v5 = iconst.i64 0xe0040404_04040404
+    v6 = iconst.i64 0xffffffff_ffffffff
+    v7 = iconcat v5, v6
+    v8 = icmp eq v4, v7
+    return v8
+}
+; run

cranelift/filetests/filetests/isa/x64/struct-arg.clif

@@ -0,0 +1,147 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function u0:0(i64 sarg(64)) -> i8 system_v {
+block0(v0: i64):
+    v1 = load.i8 v0
+    return v1
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: lea     16(%rbp), %rsi
+; nextln: movzbq  0(%rsi), %rsi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:1(i64 sarg(64), i64) -> i8 system_v {
+block0(v0: i64, v1: i64):
+    v2 = load.i8 v1
+	v3 = load.i8 v0
+	v4 = iadd.i8 v2, v3
+    return v4
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: lea     16(%rbp), %rsi
+; nextln: movzbq  0(%rdi), %rdi
+; nextln: movzbq  0(%rsi), %rsi
+; nextln: addl    %esi, %edi
+; nextln: movq    %rdi, %rax
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:2(i64) -> i8 system_v {
+fn1 = colocated u0:0(i64 sarg(64)) -> i8 system_v
+
+block0(v0: i64):
+    v1 = call fn1(v0)
+    return v1
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: movq    %rdi, %rsi
+; nextln: subq    $$64, %rsp
+; nextln: virtual_sp_offset_adjust 64
+; nextln: lea     0(%rsp), %rdi
+; nextln: movl    $$64, %edx
+; nextln: load_ext_name %Memcpy+0, %rcx
+; nextln: call    *%rcx
+; nextln: call    User { namespace: 0, index: 0 }
+; nextln: addq    $$64, %rsp
+; nextln: virtual_sp_offset_adjust -64
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:3(i64, i64) -> i8 system_v {
+fn1 = colocated u0:0(i64, i64 sarg(64)) -> i8 system_v
+
+block0(v0: i64, v1: i64):
+    v2 = call fn1(v0, v1)
+    return v2
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: subq    $$16, %rsp
+; nextln: movq    %r12, 0(%rsp)
+; nextln: virtual_sp_offset_adjust 16
+; nextln: movq    %rdi, %r12
+; nextln: subq    $$64, %rsp
+; nextln: virtual_sp_offset_adjust 64
+; nextln: lea     0(%rsp), %rdi
+; nextln: movl    $$64, %edx
+; nextln: load_ext_name %Memcpy+0, %rcx
+; nextln: call    *%rcx
+; nextln: movq    %r12, %rdi
+; nextln: call    User { namespace: 0, index: 0 }
+; nextln: addq    $$64, %rsp
+; nextln: virtual_sp_offset_adjust -64
+; nextln: movq    0(%rsp), %r12
+; nextln: addq    $$16, %rsp
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:4(i64 sarg(128), i64 sarg(64)) -> i8 system_v {
+block0(v0: i64, v1: i64):
+    v2 = load.i8 v0
+    v3 = load.i8 v1
+    v4 = iadd.i8 v2, v3
+    return v4
+}
+
+; check:  movq    %rsp, %rbp
+; nextln: lea     16(%rbp), %rsi
+; nextln: lea     144(%rbp), %rdi
+; nextln: movzbq  0(%rsi), %rsi
+; nextln: movzbq  0(%rdi), %rdi
+; nextln: addl    %edi, %esi
+; nextln: movq    %rsi, %rax
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret
+
+function u0:5(i64, i64, i64) -> i8 system_v {
+fn1 = colocated u0:0(i64, i64 sarg(128), i64 sarg(64)) -> i8 system_v
+
+block0(v0: i64, v1: i64, v2: i64):
+    v3 = call fn1(v0, v1, v2)
+    return v3
+}
+
+; check:  movq    %rsp, %rbp
+; nextln: subq    $$16, %rsp
+; nextln: movq    %r12, 0(%rsp)
+; nextln: movq    %r13, 8(%rsp)
+; nextln: virtual_sp_offset_adjust 16
+; nextln: movq    %rdi, %r12
+; nextln: movq    %rdx, %r13
+; nextln: subq    $$192, %rsp
+; nextln: virtual_sp_offset_adjust 192
+; nextln: lea     0(%rsp), %rdi
+; nextln: movl    $$128, %edx
+; nextln: load_ext_name %Memcpy+0, %rcx
+; nextln: call    *%rcx
+; nextln: lea     128(%rsp), %rdi
+; nextln: movq    %r13, %rsi
+; nextln: movl    $$64, %edx
+; nextln: load_ext_name %Memcpy+0, %rcx
+; nextln: call    *%rcx
+; nextln: movq    %r12, %rdi
+; nextln: call    User { namespace: 0, index: 0 }
+; nextln: addq    $$192, %rsp
+; nextln: virtual_sp_offset_adjust -192
+; nextln: movq    0(%rsp), %r12
+; nextln: movq    8(%rsp), %r13
+; nextln: addq    $$16, %rsp
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret

cranelift/filetests/filetests/isa/x64/struct-ret.clif

@@ -0,0 +1,19 @@
+test compile
+target x86_64
+feature "experimental_x64"
+
+function %f0(i64 sret) {
+block0(v0: i64):
+    v1 = iconst.i64 42
+    store v1, v0
+    return
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: movq    %rdi, %rax
+; nextln: movl    $$42, %esi
+; nextln: movq    %rsi, 0(%rdi)
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret

cranelift/filetests/filetests/isa/x64/tls_elf.clif

@@ -0,0 +1,19 @@
+test compile
+set tls_model=elf_gd
+target x86_64
+feature "experimental_x64"
+
+function u0:0(i32) -> i64 {
+gv0 = symbol colocated tls u1:0
+
+block0(v0: i32):
+    v1 = global_value.i64 gv0
+    return v1
+}
+
+; check:  pushq   %rbp
+; nextln: movq    %rsp, %rbp
+; nextln: elf_tls_get_addr User { namespace: 1, index: 0 }
+; nextln: movq    %rbp, %rsp
+; nextln: popq    %rbp
+; nextln: ret

cranelift/filetests/filetests/isa/x86/icmp-i128.clif

@@ -50,3 +50,45 @@ block0:
 }
 
 ; run
+
+function %test_icmp_nz_eq_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 eq v1, v2
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_gt_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x2
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 ugt v2, v1
+    return v10
+}
+
+; run
+
+function %test_icmp_nz_ge_i128() -> b1 {
+block0:
+    v11 = iconst.i64 0x1
+    v12 = iconst.i64 0x1
+    v1 = iconcat v11, v12
+    v21 = iconst.i64 0x1
+    v22 = iconst.i64 0x1
+    v2 = iconcat v21, v22
+    v10 = icmp.i128 uge v1, v2
+    return v10
+}
+
+; run

cranelift/native/Cargo.toml

@@ -14,16 +14,10 @@ edition = "2018"
 cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false }
 target-lexicon = "0.11"
 
-[target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies]
-raw-cpuid = "8.1.2"
-
 [features]
 default = ["std"]
 std = ["cranelift-codegen/std"]
-# when compiling with the "core" feature, nightly must be enabled
-# enabling the "nightly" feature for raw-cpuid allows avoiding
-# linking in a c-library.
-core = ["cranelift-codegen/core", "raw-cpuid/nightly"]
+core = ["cranelift-codegen/core"]
 
 [badges]
 maintenance = { status = "experimental" }

cranelift/native/src/lib.rs

@@ -22,19 +22,15 @@
         clippy::use_self
     )
 )]
-#![no_std]
 
 use cranelift_codegen::isa;
 use target_lexicon::Triple;
 
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-use raw_cpuid::CpuId;
-
 /// Return an `isa` builder configured for the current host
 /// machine, or `Err(())` if the host machine is not supported
 /// in the current configuration.
 pub fn builder() -> Result<isa::Builder, &'static str> {
-    builder_with_backend_variant(isa::BackendVariant::Any)
+    builder_with_options(isa::BackendVariant::Any, true)
 }
 
 /// Return an `isa` builder configured for the current host
@@ -44,8 +40,9 @@ pub fn builder() -> Result<isa::Builder, &'static str> {
 /// Selects the given backend variant specifically; this is
 /// useful when more than oen backend exists for a given target
 /// (e.g., on x86-64).
-pub fn builder_with_backend_variant(
+pub fn builder_with_options(
     variant: isa::BackendVariant,
+    infer_native_flags: bool,
 ) -> Result<isa::Builder, &'static str> {
     let mut isa_builder =
         isa::lookup_variant(Triple::host(), variant).map_err(|err| match err {
@@ -55,72 +52,64 @@ pub fn builder_with_backend_variant(
             isa::LookupError::Unsupported => "unsupported architecture",
         })?;
 
-    if cfg!(any(target_arch = "x86", target_arch = "x86_64")) {
-        parse_x86_cpuid(&mut isa_builder)?;
-    }
-
-    Ok(isa_builder)
-}
-
     #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-fn parse_x86_cpuid(isa_builder: &mut isa::Builder) -> Result<(), &'static str> {
+    {
         use cranelift_codegen::settings::Configurable;
-    let cpuid = CpuId::new();
 
-    if let Some(info) = cpuid.get_feature_info() {
-        if !info.has_sse2() {
+        if !std::is_x86_feature_detected!("sse2") {
             return Err("x86 support requires SSE2");
         }
-        if info.has_sse3() {
+
+        if !infer_native_flags {
+            return Ok(isa_builder);
+        }
+
+        if std::is_x86_feature_detected!("sse3") {
             isa_builder.enable("has_sse3").unwrap();
         }
-        if info.has_ssse3() {
+        if std::is_x86_feature_detected!("ssse3") {
             isa_builder.enable("has_ssse3").unwrap();
         }
-        if info.has_sse41() {
+        if std::is_x86_feature_detected!("sse4.1") {
             isa_builder.enable("has_sse41").unwrap();
         }
-        if info.has_sse42() {
+        if std::is_x86_feature_detected!("sse4.2") {
             isa_builder.enable("has_sse42").unwrap();
         }
-        if info.has_popcnt() {
+        if std::is_x86_feature_detected!("popcnt") {
             isa_builder.enable("has_popcnt").unwrap();
         }
-        if info.has_avx() {
+        if std::is_x86_feature_detected!("avx") {
             isa_builder.enable("has_avx").unwrap();
         }
-    }
-    if let Some(info) = cpuid.get_extended_feature_info() {
-        if info.has_bmi1() {
-            isa_builder.enable("has_bmi1").unwrap();
-        }
-        if info.has_bmi2() {
-            isa_builder.enable("has_bmi2").unwrap();
-        }
-        if info.has_avx2() {
+        if std::is_x86_feature_detected!("avx2") {
             isa_builder.enable("has_avx2").unwrap();
         }
-        if info.has_avx512dq() {
+        if std::is_x86_feature_detected!("bmi1") {
+            isa_builder.enable("has_bmi1").unwrap();
+        }
+        if std::is_x86_feature_detected!("bmi2") {
+            isa_builder.enable("has_bmi2").unwrap();
+        }
+        if std::is_x86_feature_detected!("avx512dq") {
             isa_builder.enable("has_avx512dq").unwrap();
         }
-        if info.has_avx512vl() {
+        if std::is_x86_feature_detected!("avx512vl") {
             isa_builder.enable("has_avx512vl").unwrap();
         }
-        if info.has_avx512f() {
+        if std::is_x86_feature_detected!("avx512f") {
             isa_builder.enable("has_avx512f").unwrap();
         }
-    }
-    if let Some(info) = cpuid.get_extended_function_info() {
-        if info.has_lzcnt() {
+        if std::is_x86_feature_detected!("lzcnt") {
             isa_builder.enable("has_lzcnt").unwrap();
         }
     }
-    Ok(())
-}
 
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
-fn parse_x86_cpuid(_isa_builder: &mut isa::Builder) -> Result<(), &'static str> {
-    unreachable!();
+    // squelch warnings about unused mut/variables on some platforms.
+    drop(&mut isa_builder);
+    drop(infer_native_flags);
+
+    Ok(isa_builder)
 }
 
 #[cfg(test)]

cranelift/object/Cargo.toml

@@ -12,7 +12,7 @@ edition = "2018"
 [dependencies]
 cranelift-module = { path = "../module", version = "0.69.0" }
 cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false, features = ["std"] }
-object = { version = "0.22.0", default-features = false, features = ["write"] }
+object = { version = "0.23.0", default-features = false, features = ["write"] }
 target-lexicon = "0.11"
 anyhow = "1.0"
 log = { version = "0.4.6", default-features = false }

cranelift/peepmatic/Cargo.toml

@@ -15,7 +15,7 @@ peepmatic-macro = { version = "0.69.0", path = "crates/macro" }
 peepmatic-runtime = { version = "0.69.0", path = "crates/runtime", features = ["construct"] }
 peepmatic-traits = { version = "0.69.0", path = "crates/traits" }
 serde = { version = "1.0.105", features = ["derive"] }
-wast = "29.0.0"
+wast = "32.0.0"
 z3 = { version = "0.7.1", features = ["static-link-z3"] }
 
 [dev-dependencies]

cranelift/peepmatic/crates/fuzzing/Cargo.toml

@@ -19,6 +19,6 @@ peepmatic-runtime = { path = "../runtime", features = ["construct"] }
 peepmatic-test = { path = "../test" }
 peepmatic-test-operator = { path = "../test-operator" }
 peepmatic-traits = { path = "../traits" }
-rand = { version = "0.7.3", features = ["small_rng"] }
+rand = { version = "0.8.2", features = ["small_rng"] }
 serde = "1.0.106"
-wast = "29.0.0"
+wast = "32.0.0"

cranelift/peepmatic/crates/runtime/Cargo.toml

@@ -16,7 +16,7 @@ peepmatic-automata = { version = "0.69.0", path = "../automata", features = ["se
 peepmatic-traits = { version = "0.69.0", path = "../traits" }
 serde = { version = "1.0.105", features = ["derive"] }
 thiserror = "1.0.15"
-wast = { version = "29.0.0", optional = true }
+wast = { version = "32.0.0", optional = true }
 
 [dev-dependencies]
 peepmatic-test-operator = { version = "0.69.0", path = "../test-operator" }

cranelift/peepmatic/crates/souper/Cargo.toml

@@ -16,4 +16,4 @@ log = "0.4.8"
 [dev-dependencies]
 peepmatic = { path = "../..", version = "0.69.0" }
 peepmatic-test-operator = { version = "0.69.0", path = "../test-operator" }
-wast = "29.0.0"
+wast = "32.0.0"

cranelift/peepmatic/crates/test-operator/Cargo.toml

@@ -11,4 +11,4 @@ edition = "2018"
 [dependencies]
 peepmatic-traits = { version = "0.69.0", path = "../traits" }
 serde = { version = "1.0.105", features = ["derive"] }
-wast = "29.0.0"
+wast = "32.0.0"

cranelift/src/souper_harvest.rs

@@ -12,7 +12,7 @@ static WASM_MAGIC: &[u8] = &[0x00, 0x61, 0x73, 0x6D];
 /// Harvest candidates for superoptimization from a Wasm or Clif file.
 ///
 /// Candidates are emitted in Souper's text format:
-/// https://github.com/google/souper
+/// <https://github.com/google/souper>
 #[derive(StructOpt)]
 pub struct Options {
     /// Specify an input file to be used. Use '-' for stdin.

cranelift/wasm/Cargo.toml

@@ -12,12 +12,12 @@ keywords = ["webassembly", "wasm"]
 edition = "2018"
 
 [dependencies]
-wasmparser = { version = "0.71", default-features = false }
+wasmparser = { version = "0.73", default-features = false }
 cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false }
 cranelift-entity = { path = "../entity", version = "0.69.0" }
 cranelift-frontend = { path = "../frontend", version = "0.69.0", default-features = false }
 hashbrown = { version = "0.9.1", optional = true }
-itertools = "0.9.0"
+itertools = "0.10.0"
 log = { version = "0.4.6", default-features = false }
 serde = { version = "1.0.94", features = ["derive"], optional = true }
 smallvec = "1.6.1"

cranelift/wasm/src/code_translator.rs

@@ -261,7 +261,7 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
                 .extend_from_slice(builder.block_params(loop_body));
 
             builder.switch_to_block(loop_body);
-            environ.translate_loop_header(builder.cursor())?;
+            environ.translate_loop_header(builder)?;
         }
         Operator::If { ty } => {
             let val = state.pop1();
@@ -1626,7 +1626,7 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
             // operands must match (hence the bitcast).
             state.push1(builder.ins().bitselect(bitcast_c, bitcast_a, bitcast_b))
         }
-        Operator::I8x16AnyTrue | Operator::I16x8AnyTrue | Operator::I32x4AnyTrue => {
+        Operator::V128AnyTrue => {
             let a = pop1_with_bitcast(state, type_of(op), builder);
             let bool_result = builder.ins().vany_true(a);
             state.push1(builder.ins().bint(I32, bool_result))
@@ -1824,7 +1824,21 @@ pub fn translate_operator<FE: FuncEnvironment + ?Sized>(
             let (a, b) = pop2_with_bitcast(state, I16X8, builder);
             state.push1(builder.ins().widening_pairwise_dot_product_s(a, b));
         }
-        Operator::I16x8ExtMulLowI8x16S
+        Operator::I64x2Bitmask
+        | Operator::I64x2WidenLowI32x4S
+        | Operator::I64x2WidenHighI32x4S
+        | Operator::I64x2WidenLowI32x4U
+        | Operator::I64x2WidenHighI32x4U
+        | Operator::V128Load8Lane { .. }
+        | Operator::V128Load16Lane { .. }
+        | Operator::V128Load32Lane { .. }
+        | Operator::V128Load64Lane { .. }
+        | Operator::V128Store8Lane { .. }
+        | Operator::V128Store16Lane { .. }
+        | Operator::V128Store32Lane { .. }
+        | Operator::V128Store64Lane { .. }
+        | Operator::I16x8Q15MulrSatS
+        | Operator::I16x8ExtMulLowI8x16S
         | Operator::I16x8ExtMulHighI8x16S
         | Operator::I16x8ExtMulLowI8x16U
         | Operator::I16x8ExtMulHighI8x16U
@@ -2522,7 +2536,6 @@ fn type_of(operator: &Operator) -> Type {
         | Operator::I8x16GeU
         | Operator::I8x16Neg
         | Operator::I8x16Abs
-        | Operator::I8x16AnyTrue
         | Operator::I8x16AllTrue
         | Operator::I8x16Shl
         | Operator::I8x16ShrS
@@ -2557,7 +2570,6 @@ fn type_of(operator: &Operator) -> Type {
         | Operator::I16x8GeU
         | Operator::I16x8Neg
         | Operator::I16x8Abs
-        | Operator::I16x8AnyTrue
         | Operator::I16x8AllTrue
         | Operator::I16x8Shl
         | Operator::I16x8ShrS
@@ -2592,7 +2604,6 @@ fn type_of(operator: &Operator) -> Type {
         | Operator::I32x4GeU
         | Operator::I32x4Neg
         | Operator::I32x4Abs
-        | Operator::I32x4AnyTrue
         | Operator::I32x4AllTrue
         | Operator::I32x4Shl
         | Operator::I32x4ShrS

cranelift/wasm/src/environ/spec.rs

@@ -205,25 +205,34 @@ pub enum ReturnMode {
 
 /// An entry in the alias section of a wasm module (from the module linking
 /// proposal)
-pub enum Alias {
-    /// A parent's module is being aliased into our own index space.
-    ///
-    /// Note that the index here is in the parent's index space, not our own.
-    ParentModule(ModuleIndex),
+pub enum Alias<'a> {
+    /// An outer module's module is being aliased into our own index space.
+    OuterModule {
+        /// The number of modules above us that we're referencing.
+        relative_depth: u32,
+        /// The module index in the outer module's index space we're referencing.
+        index: ModuleIndex,
+    },
 
-    /// A parent's type is being aliased into our own index space
+    /// An outer module's type is being aliased into our own index space
     ///
-    /// Note that the index here is in the parent's index space, not our own.
-    ParentType(TypeIndex),
+    /// Note that the index here is in the outer module's index space, not our
+    /// own.
+    OuterType {
+        /// The number of modules above us that we're referencing.
+        relative_depth: u32,
+        /// The type index in the outer module's index space we're referencing.
+        index: TypeIndex,
+    },
 
     /// A previously created instance is having one of its exports aliased into
     /// our index space.
-    Child {
+    InstanceExport {
         /// The index we're aliasing.
         instance: InstanceIndex,
         /// The nth export that we're inserting into our own index space
         /// locally.
-        export: usize,
+        export: &'a str,
     },
 }
 
@@ -286,6 +295,12 @@ pub trait FuncEnvironment: TargetEnvironment {
         ReturnMode::NormalReturns
     }
 
+    /// Called after the locals for a function have been parsed, and the number
+    /// of variables defined by this function is provided.
+    fn after_locals(&mut self, num_locals_defined: usize) {
+        drop(num_locals_defined);
+    }
+
     /// Set up the necessary preamble definitions in `func` to access the global variable
     /// identified by `index`.
     ///
@@ -628,7 +643,7 @@ pub trait FuncEnvironment: TargetEnvironment {
     ///
     /// This can be used to insert explicit interrupt or safepoint checking at
     /// the beginnings of loops.
-    fn translate_loop_header(&mut self, _pos: FuncCursor) -> WasmResult<()> {
+    fn translate_loop_header(&mut self, _builder: &mut FunctionBuilder) -> WasmResult<()> {
         // By default, don't emit anything.
         Ok(())
     }
@@ -1014,30 +1029,15 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
         drop(amount);
     }
 
-    /// Declares that a module will come later with the type signature provided.
-    fn declare_module(&mut self, ty: TypeIndex) -> WasmResult<()> {
-        drop(ty);
-        Err(WasmError::Unsupported("module linking".to_string()))
-    }
-
     /// Called at the beginning of translating a module.
     ///
-    /// The `index` argument is a monotonically increasing index which
-    /// corresponds to the nth module that's being translated. This is not the
-    /// 32-bit index in the current module's index space. For example the first
-    /// call to `module_start` will have index 0.
-    ///
     /// Note that for nested modules this may be called multiple times.
-    fn module_start(&mut self, index: usize) {
-        drop(index);
-    }
+    fn module_start(&mut self) {}
 
     /// Called at the end of translating a module.
     ///
     /// Note that for nested modules this may be called multiple times.
-    fn module_end(&mut self, index: usize) {
-        drop(index);
-    }
+    fn module_end(&mut self) {}
 
     /// Indicates that this module will have `amount` instances.
     fn reserve_instances(&mut self, amount: u32) {
@@ -1046,7 +1046,11 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
 
     /// Declares a new instance which this module will instantiate before it's
     /// instantiated.
-    fn declare_instance(&mut self, module: ModuleIndex, args: Vec<EntityIndex>) -> WasmResult<()> {
+    fn declare_instance(
+        &mut self,
+        module: ModuleIndex,
+        args: Vec<(&'data str, EntityIndex)>,
+    ) -> WasmResult<()> {
         drop((module, args));
         Err(WasmError::Unsupported("wasm instance".to_string()))
     }
@@ -1056,7 +1060,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
     /// The alias comes from the `instance` specified (or the parent if `None`
     /// is supplied) and the index is either in the module's own index spaces
     /// for the parent or an index into the exports for nested instances.
-    fn declare_alias(&mut self, alias: Alias) -> WasmResult<()> {
+    fn declare_alias(&mut self, alias: Alias<'data>) -> WasmResult<()> {
         drop(alias);
         Err(WasmError::Unsupported("wasm alias".to_string()))
     }

cranelift/wasm/src/func_translator.rs

@@ -170,6 +170,8 @@ fn parse_local_decls<FE: FuncEnvironment + ?Sized>(
         declare_locals(builder, count, ty, &mut next_local, environ)?;
     }
 
+    environ.after_locals(next_local);
+
     Ok(())
 }
 

cranelift/wasm/src/module_translator.rs

@@ -4,8 +4,8 @@ use crate::environ::{ModuleEnvironment, WasmResult};
 use crate::sections_translator::{
     parse_alias_section, parse_data_section, parse_element_section, parse_event_section,
     parse_export_section, parse_function_section, parse_global_section, parse_import_section,
-    parse_instance_section, parse_memory_section, parse_module_section, parse_name_section,
-    parse_start_section, parse_table_section, parse_type_section,
+    parse_instance_section, parse_memory_section, parse_name_section, parse_start_section,
+    parse_table_section, parse_type_section,
 };
 use crate::state::ModuleTranslationState;
 use cranelift_codegen::timing;
@@ -22,23 +22,16 @@ pub fn translate_module<'data>(
     let mut module_translation_state = ModuleTranslationState::new();
     let mut validator = Validator::new();
     validator.wasm_features(environ.wasm_features());
-    let mut stack = Vec::new();
-    let mut modules = 1;
-    let mut cur_module = 0;
 
     for payload in Parser::new(0).parse_all(data) {
         match payload? {
             Payload::Version { num, range } => {
                 validator.version(num, &range)?;
-                environ.module_start(cur_module);
+                environ.module_start();
             }
             Payload::End => {
                 validator.end()?;
-                environ.module_end(cur_module);
-                if let Some((other, other_index)) = stack.pop() {
-                    validator = other;
-                    cur_module = other_index;
-                }
+                environ.module_end();
             }
 
             Payload::TypeSection(types) => {
@@ -111,10 +104,6 @@ pub fn translate_module<'data>(
                 environ.reserve_passive_data(count)?;
             }
 
-            Payload::ModuleSection(s) => {
-                validator.module_section(&s)?;
-                parse_module_section(s, environ)?;
-            }
             Payload::InstanceSection(s) => {
                 validator.instance_section(&s)?;
                 parse_instance_section(s, environ)?;
@@ -123,20 +112,17 @@ pub fn translate_module<'data>(
                 validator.alias_section(&s)?;
                 parse_alias_section(s, environ)?;
             }
-            Payload::ModuleCodeSectionStart {
+            Payload::ModuleSectionStart {
                 count,
                 range,
                 size: _,
             } => {
-                validator.module_code_section_start(count, &range)?;
+                validator.module_section_start(count, &range)?;
+                environ.reserve_modules(count);
             }
 
-            Payload::ModuleCodeSectionEntry { .. } => {
-                let subvalidator = validator.module_code_section_entry();
-                stack.push((validator, cur_module));
-                validator = subvalidator;
-                cur_module = modules;
-                modules += 1;
+            Payload::ModuleSectionEntry { .. } => {
+                validator.module_section_entry();
             }
 
             Payload::CustomSection {

cranelift/wasm/src/sections_translator.rs

@@ -504,19 +504,6 @@ pub fn parse_name_section<'data>(
     Ok(())
 }
 
-/// Parses the Module section of the wasm module.
-pub fn parse_module_section<'data>(
-    section: wasmparser::ModuleSectionReader<'data>,
-    environ: &mut dyn ModuleEnvironment<'data>,
-) -> WasmResult<()> {
-    environ.reserve_modules(section.get_count());
-
-    for module_ty in section {
-        environ.declare_module(TypeIndex::from_u32(module_ty?))?;
-    }
-    Ok(())
-}
-
 /// Parses the Instance section of the wasm module.
 pub fn parse_instance_section<'data>(
     section: wasmparser::InstanceSectionReader<'data>,
@@ -530,20 +517,23 @@ pub fn parse_instance_section<'data>(
         let args = instance
             .args()?
             .into_iter()
-            .map(|result| {
-                let (kind, idx) = result?;
-                Ok(match kind {
-                    ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(idx)),
-                    ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(idx)),
-                    ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(idx)),
-                    ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(idx)),
-                    ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(idx)),
-                    ExternalKind::Instance => EntityIndex::Instance(InstanceIndex::from_u32(idx)),
+            .map(|arg| {
+                let arg = arg?;
+                let index = match arg.kind {
+                    ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(arg.index)),
+                    ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(arg.index)),
+                    ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(arg.index)),
+                    ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(arg.index)),
+                    ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(arg.index)),
+                    ExternalKind::Instance => {
+                        EntityIndex::Instance(InstanceIndex::from_u32(arg.index))
+                    }
                     ExternalKind::Event => unimplemented!(),
 
                     // this won't pass validation
                     ExternalKind::Type => unreachable!(),
-                })
+                };
+                Ok((arg.name, index))
             })
             .collect::<WasmResult<Vec<_>>>()?;
         environ.declare_instance(module, args)?;
@@ -557,19 +547,28 @@ pub fn parse_alias_section<'data>(
     environ: &mut dyn ModuleEnvironment<'data>,
 ) -> WasmResult<()> {
     for alias in section {
-        let alias = alias?;
-        let alias = match alias.instance {
-            wasmparser::AliasedInstance::Parent => {
-                match alias.kind {
-                    ExternalKind::Module => Alias::ParentModule(ModuleIndex::from_u32(alias.index)),
-                    ExternalKind::Type => Alias::ParentType(TypeIndex::from_u32(alias.index)),
-                    // shouldn't get past validation
-                    _ => unreachable!(),
-                }
-            }
-            wasmparser::AliasedInstance::Child(i) => Alias::Child {
-                instance: InstanceIndex::from_u32(i),
-                export: alias.index as usize,
+        let alias = match alias? {
+            wasmparser::Alias::OuterType {
+                relative_depth,
+                index,
+            } => Alias::OuterType {
+                relative_depth,
+                index: TypeIndex::from_u32(index),
+            },
+            wasmparser::Alias::OuterModule {
+                relative_depth,
+                index,
+            } => Alias::OuterModule {
+                relative_depth,
+                index: ModuleIndex::from_u32(index),
+            },
+            wasmparser::Alias::InstanceExport {
+                instance,
+                export,
+                kind: _,
+            } => Alias::InstanceExport {
+                instance: InstanceIndex::from_u32(instance),
+                export,
             },
         };
         environ.declare_alias(alias)?;

crates/bench-api/Cargo.toml

@@ -16,10 +16,13 @@ crate-type = ["rlib", "cdylib"]
 
 [dependencies]
 anyhow = "1.0"
+shuffling-allocator = { version = "1.1.1", optional = true }
 wasmtime = { path = "../wasmtime", default-features = false }
 wasmtime-wasi = { path = "../wasi" }
 wasi-common = { path = "../wasi-common" }
 
-
 [dev-dependencies]
 wat = "1.0"
+
+[features]
+default = ["shuffling-allocator"]

crates/bench-api/src/lib.rs

@@ -21,10 +21,16 @@
 //! # Example
 //!
 //! ```
+//! use std::ptr;
 //! use wasmtime_bench_api::*;
 //!
-//! let engine = unsafe { wasm_bench_create() };
-//! assert!(!engine.is_null());
+//! let working_dir = std::env::current_dir().unwrap().display().to_string();
+//! let mut bench_api = ptr::null_mut();
+//! unsafe {
+//!     let code = wasm_bench_create(working_dir.as_ptr(), working_dir.len(), &mut bench_api);
+//!     assert_eq!(code, OK);
+//!     assert!(!bench_api.is_null());
+//! };
 //!
 //! let wasm = wat::parse_bytes(br#"
 //!     (module
@@ -42,7 +48,7 @@
 //! "#).unwrap();
 //!
 //! // Start your compilation timer here.
-//! let code = unsafe { wasm_bench_compile(engine, wasm.as_ptr(), wasm.len()) };
+//! let code = unsafe { wasm_bench_compile(bench_api, wasm.as_ptr(), wasm.len()) };
 //! // End your compilation timer here.
 //! assert_eq!(code, OK);
 //!
@@ -57,23 +63,25 @@
 //! }
 //!
 //! // Start your instantiation timer here.
-//! let code = unsafe { wasm_bench_instantiate(engine, bench_start, bench_stop) };
+//! let code = unsafe { wasm_bench_instantiate(bench_api, bench_start, bench_stop) };
 //! // End your instantiation timer here.
 //! assert_eq!(code, OK);
 //!
 //! // No need to start timers for the execution since, by convention, the timer
 //! // functions we passed during instantiation will be called by the benchmark
 //! // at the appropriate time (before and after the benchmarked section).
-//! let code = unsafe { wasm_bench_execute(engine) };
+//! let code = unsafe { wasm_bench_execute(bench_api) };
 //! assert_eq!(code, OK);
 //!
 //! unsafe {
-//!     wasm_bench_free(engine);
+//!     wasm_bench_free(bench_api);
 //! }
 //! ```
 
 use anyhow::{anyhow, Context, Result};
+use std::env;
 use std::os::raw::{c_int, c_void};
+use std::path::Path;
 use std::slice;
 use wasi_common::WasiCtxBuilder;
 use wasmtime::{Config, Engine, Instance, Linker, Module, Store};
@@ -83,15 +91,43 @@ pub type ExitCode = c_int;
 pub const OK: ExitCode = 0;
 pub const ERR: ExitCode = -1;
 
+// Randomize the location of heap objects to avoid accidental locality being an
+// uncontrolled variable that obscures performance evaluation in our
+// experiments.
+#[cfg(feature = "shuffling-allocator")]
+#[global_allocator]
+static ALLOC: shuffling_allocator::ShufflingAllocator<std::alloc::System> =
+    shuffling_allocator::wrap!(&std::alloc::System);
+
 /// Exposes a C-compatible way of creating the engine from the bytes of a single
 /// Wasm module.
 ///
-/// This function returns a pointer to a structure that contains the engine's
-/// initialized state.
+/// On success, the `out_bench_ptr` is initialized to a pointer to a structure
+/// that contains the engine's initialized state, and `0` is returned. On
+/// failure, a non-zero status code is returned and `out_bench_ptr` is left
+/// untouched.
 #[no_mangle]
-pub extern "C" fn wasm_bench_create() -> *mut c_void {
-    let state = Box::new(BenchState::new());
-    Box::into_raw(state) as _
+pub extern "C" fn wasm_bench_create(
+    working_dir_ptr: *const u8,
+    working_dir_len: usize,
+    out_bench_ptr: *mut *mut c_void,
+) -> ExitCode {
+    let result = (|| -> Result<_> {
+        let working_dir = unsafe { std::slice::from_raw_parts(working_dir_ptr, working_dir_len) };
+        let working_dir = std::str::from_utf8(working_dir)
+            .context("given working directory is not valid UTF-8")?;
+        let state = Box::new(BenchState::new(working_dir)?);
+        Ok(Box::into_raw(state) as _)
+    })();
+
+    if let Ok(bench_ptr) = result {
+        unsafe {
+            assert!(!out_bench_ptr.is_null());
+            *out_bench_ptr = bench_ptr;
+        }
+    }
+
+    to_exit_code(result.map(|_| ()))
 }
 
 /// Free the engine state allocated by this library.
@@ -155,27 +191,49 @@ fn to_exit_code<T>(result: impl Into<Result<T>>) -> ExitCode {
 /// to manage the Wasmtime engine between calls.
 struct BenchState {
     engine: Engine,
-    store: Store,
+    linker: Linker,
     module: Option<Module>,
     instance: Option<Instance>,
     did_execute: bool,
 }
 
 impl BenchState {
-    fn new() -> Self {
+    fn new(working_dir: impl AsRef<Path>) -> Result<Self> {
         let mut config = Config::new();
         config.wasm_simd(true);
         // NB: do not configure a code cache.
 
         let engine = Engine::new(&config);
         let store = Store::new(&engine);
-        Self {
+
+        let mut linker = Linker::new(&store);
+
+        // Create a WASI environment.
+
+        let mut cx = WasiCtxBuilder::new();
+        cx.inherit_stdio();
+        // Allow access to the working directory so that the benchmark can read
+        // its input workload(s).
+        let working_dir = wasi_common::preopen_dir(working_dir)
+            .context("failed to preopen the working directory")?;
+        cx.preopened_dir(working_dir, ".");
+        // Pass this env var along so that the benchmark program can use smaller
+        // input workload(s) if it has them and that has been requested.
+        if let Ok(val) = env::var("WASM_BENCH_USE_SMALL_WORKLOAD") {
+            cx.env("WASM_BENCH_USE_SMALL_WORKLOAD", &val);
+        }
+
+        let cx = cx.build()?;
+        let wasi = Wasi::new(&store, cx);
+        wasi.add_to_linker(&mut linker)?;
+
+        Ok(Self {
             engine,
-            store,
+            linker,
             module: None,
             instance: None,
             did_execute: false,
-        }
+        })
     }
 
     fn compile(&mut self, bytes: &[u8]) -> Result<()> {
@@ -201,20 +259,11 @@ impl BenchState {
             .as_mut()
             .expect("compile the module before instantiating it");
 
-        let mut linker = Linker::new(&self.store);
-
-        // Import a very restricted WASI environment.
-        let mut cx = WasiCtxBuilder::new();
-        cx.inherit_stdio();
-        let cx = cx.build()?;
-        let wasi = Wasi::new(linker.store(), cx);
-        wasi.add_to_linker(&mut linker)?;
-
         // Import the specialized benchmarking functions.
-        linker.func("bench", "start", move || bench_start())?;
-        linker.func("bench", "end", move || bench_end())?;
+        self.linker.func("bench", "start", move || bench_start())?;
+        self.linker.func("bench", "end", move || bench_end())?;
 
-        self.instance = Some(linker.instantiate(&module)?);
+        self.instance = Some(self.linker.instantiate(&module)?);
         Ok(())
     }
 

crates/c-api/include/doc-wasm.h

@@ -317,6 +317,12 @@
  * \fn wasm_name_new_new_uninitialized
  * \brief Convenience alias
  * 
+ * \fn wasm_name_new_from_string
+ * \brief Create a new name from a C string.
+ * 
+ * \fn wasm_name_new_from_string_nt
+ * \brief Create a new name from a C string with null terminator.
+ *
  * \fn wasm_name_copy
  * \brief Convenience alias
  *
@@ -407,7 +413,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_valtype_t* wasm_valtype_copy(wasm_valtype_t *)
+ * \fn own wasm_valtype_t* wasm_valtype_copy(const wasm_valtype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -477,7 +483,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_functype_t* wasm_functype_copy(wasm_functype_t *)
+ * \fn own wasm_functype_t* wasm_functype_copy(const wasm_functype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -548,7 +554,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_globaltype_t* wasm_globaltype_copy(wasm_globaltype_t *)
+ * \fn own wasm_globaltype_t* wasm_globaltype_copy(const wasm_globaltype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -625,7 +631,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_tabletype_t* wasm_tabletype_copy(wasm_tabletype_t *)
+ * \fn own wasm_tabletype_t* wasm_tabletype_copy(const wasm_tabletype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -711,7 +717,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_memorytype_t* wasm_memorytype_copy(wasm_memorytype_t *)
+ * \fn own wasm_memorytype_t* wasm_memorytype_copy(const wasm_memorytype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -780,7 +786,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_externtype_t* wasm_externtype_copy(wasm_externtype_t *)
+ * \fn own wasm_externtype_t* wasm_externtype_copy(const wasm_externtype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -957,7 +963,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_importtype_t* wasm_importtype_copy(wasm_importtype_t *)
+ * \fn own wasm_importtype_t* wasm_importtype_copy(const wasm_importtype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -1038,7 +1044,7 @@
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_exporttype_t* wasm_exporttype_copy(wasm_exporttype_t *)
+ * \fn own wasm_exporttype_t* wasm_exporttype_copy(const wasm_exporttype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -1520,8 +1526,6 @@
  * and types of results as the original type signature. It is undefined behavior
  * to return other types or different numbers of values.
  *
- * This function takes ownership of all of the parameters given. It's expected
- * that the caller will invoke `wasm_val_delete` for each one provided.
  * Ownership of the results and the trap returned, if any, is passed to the
  * caller of this function.
  *
@@ -1608,7 +1612,7 @@
  * \fn size_t wasm_func_result_arity(const wasm_func_t *);
  * \brief Returns the number of results returned by this function.
  *
- * \fn own wasm_trap_t *wasm_func_call(const wasm_func_t *, const wasm_val_t args[], const wasm_val_t results[]);
+* \fn own wasm_trap_t *wasm_func_call(const wasm_func_t *, const wasm_val_vec_t *args, wasm_val_vec_t *results);
  * \brief Calls the provided function with the arguments given.
  *
  * This function is used to call WebAssembly from the host. The parameter array
@@ -2164,7 +2168,7 @@
  * \fn wasm_ref_as_instance_const(const wasm_ref_t *);
  * \brief Unimplemented in Wasmtime, aborts the process if called.
  *
- * \fn own wasm_instance_t *wasm_instance_new(wasm_store_t *, const wasm_module_t *, const wasm_extern_t *const[], wasm_trap_t **);
+ * \fn own wasm_instance_t *wasm_instance_new(wasm_store_t *, const wasm_module_t *, const wasm_extern_vec_t *, wasm_trap_t **);
  * \brief Instantiates a module with the provided imports.
  *
  * This function will instantiate the provided #wasm_module_t into the provided
@@ -2194,3 +2198,29 @@
  * the same length as #wasm_module_exports called on the original module. Each
  * element is 1:1 matched with the elements in the list of #wasm_module_exports.
  */
+
+/**
+ * \def WASM_EMPTY_VEC
+ * \brief Used to initialize an empty vector type.
+ *
+ * \def WASM_ARRAY_VEC
+ * \brief Used to initialize a vector type from a C array.
+ *
+ * \def WASM_I32_VAL
+ * \brief Used to initialize a 32-bit integer wasm_val_t value.
+ *
+ * \def WASM_I64_VAL
+ * \brief Used to initialize a 64-bit integer wasm_val_t value.
+ *
+ * \def WASM_F32_VAL
+ * \brief Used to initialize a 32-bit floating point wasm_val_t value.
+ *
+ * \def WASM_F64_VAL
+ * \brief Used to initialize a 64-bit floating point wasm_val_t value.
+ *
+ * \def WASM_REF_VAL
+ * \brief Used to initialize an externref wasm_val_t value.
+ *
+ * \def WASM_INIT_VAL
+ * \brief Used to initialize a null externref wasm_val_t value.
+ */

crates/c-api/include/wasmtime.h

@@ -274,6 +274,14 @@ WASMTIME_CONFIG_PROP(void, static_memory_guard_size, uint64_t)
  */
 WASMTIME_CONFIG_PROP(void, dynamic_memory_guard_size, uint64_t)
 
+/**
+ * \brief Configures the maximum number of instances that can be created.
+ *
+ * For more information see the Rust documentation at
+ * https://bytecodealliance.github.io/wasmtime/api/wasmtime/struct.Config.html#method.max_instances.
+ */
+WASMTIME_CONFIG_PROP(void, max_instances, size_t)
+
 /**
  * \brief Enables Wasmtime's cache and loads configuration from the specified
  * path.
@@ -527,7 +535,7 @@ typedef struct wasmtime_caller_t wasmtime_caller_t;
  * argument is a #wasmtime_caller_t which allows learning information about the
  * caller.
  */
-typedef own wasm_trap_t* (*wasmtime_func_callback_t)(const wasmtime_caller_t* caller, const wasm_val_t args[], wasm_val_t results[]);
+typedef own wasm_trap_t* (*wasmtime_func_callback_t)(const wasmtime_caller_t* caller, const wasm_val_vec_t *args, wasm_val_vec_t *results);
 
 /**
  * \brief Callback signature for #wasmtime_func_new_with_env.
@@ -536,7 +544,7 @@ typedef own wasm_trap_t* (*wasmtime_func_callback_t)(const wasmtime_caller_t* ca
  * first argument is a #wasmtime_caller_t which allows learning information
  * about the caller.
  */
-typedef own wasm_trap_t* (*wasmtime_func_callback_with_env_t)(const wasmtime_caller_t* caller, void* env, const wasm_val_t args[], wasm_val_t results[]);
+typedef own wasm_trap_t* (*wasmtime_func_callback_with_env_t)(const wasmtime_caller_t* caller, void* env, const wasm_val_vec_t *args, wasm_val_vec_t *results);
 
 /**
  * \brief Creates a new host-defined function.
@@ -671,7 +679,6 @@ WASM_API_EXTERN const wasm_name_t *wasmtime_frame_module_name(const wasm_frame_t
  *
  * This function is similar to #wasm_func_call, but with a few tweaks:
  *
- * * `args` and `results` have a size parameter saying how big the arrays are
  * * An error *and* a trap can be returned
  * * Errors are returned if `args` have the wrong types, if the args/results
  *   arrays have the wrong lengths, or if values come from the wrong store.
@@ -697,10 +704,8 @@ WASM_API_EXTERN const wasm_name_t *wasmtime_frame_module_name(const wasm_frame_t
  */
 WASM_API_EXTERN own wasmtime_error_t *wasmtime_func_call(
     wasm_func_t *func,
-    const wasm_val_t *args,
-    size_t num_args,
-    wasm_val_t *results,
-    size_t num_results,
+    const wasm_val_vec_t *args,
+    wasm_val_vec_t *results,
     own wasm_trap_t **trap
 );
 
@@ -741,7 +746,6 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_global_set(
  * This function is similar to #wasm_instance_new, but with a few tweaks:
  *
  * * An error message can be returned from this function.
- * * The number of imports specified is passed as an argument
  * * The `trap` pointer is required to not be NULL.
  *
  * The states of return values from this function are similar to
@@ -759,8 +763,7 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_global_set(
 WASM_API_EXTERN own wasmtime_error_t *wasmtime_instance_new(
     wasm_store_t *store,
     const wasm_module_t *module,
-    const wasm_extern_t* const imports[],
-    size_t num_imports,
+    const wasm_extern_vec_t* imports,
     own wasm_instance_t **instance,
     own wasm_trap_t **trap
 );
@@ -1016,7 +1019,7 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_module_deserialize(
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_instancetype_t* wasm_instancetype_copy(wasm_instancetype_t *)
+ * \fn own wasm_instancetype_t* wasm_instancetype_copy(const wasm_instancetype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.
@@ -1113,7 +1116,7 @@ WASM_API_EXTERN const wasm_instancetype_t* wasm_externtype_as_instancetype_const
  *
  * See #wasm_byte_vec_delete for more information.
  *
- * \fn own wasm_moduletype_t* wasm_moduletype_copy(wasm_moduletype_t *)
+ * \fn own wasm_moduletype_t* wasm_moduletype_copy(const wasm_moduletype_t *)
  * \brief Creates a new value which matches the provided one.
  *
  * The caller is responsible for deleting the returned value.

crates/c-api/src/config.rs

@@ -171,3 +171,8 @@ pub extern "C" fn wasmtime_config_static_memory_guard_size_set(c: &mut wasm_conf
 pub extern "C" fn wasmtime_config_dynamic_memory_guard_size_set(c: &mut wasm_config_t, size: u64) {
     c.config.dynamic_memory_guard_size(size);
 }
+
+#[no_mangle]
+pub extern "C" fn wasmtime_config_max_instances_set(c: &mut wasm_config_t, limit: usize) {
+    c.config.max_instances(limit);
+}

crates/c-api/src/func.rs

@@ -1,4 +1,4 @@
-use crate::{wasm_extern_t, wasm_functype_t, wasm_store_t, wasm_val_t};
+use crate::{wasm_extern_t, wasm_functype_t, wasm_store_t, wasm_val_t, wasm_val_vec_t};
 use crate::{wasm_name_t, wasm_trap_t, wasmtime_error_t};
 use anyhow::anyhow;
 use std::ffi::c_void;
@@ -21,26 +21,28 @@ pub struct wasmtime_caller_t<'a> {
     caller: Caller<'a>,
 }
 
-pub type wasm_func_callback_t =
-    extern "C" fn(args: *const wasm_val_t, results: *mut wasm_val_t) -> Option<Box<wasm_trap_t>>;
+pub type wasm_func_callback_t = extern "C" fn(
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
+) -> Option<Box<wasm_trap_t>>;
 
 pub type wasm_func_callback_with_env_t = extern "C" fn(
     env: *mut std::ffi::c_void,
-    args: *const wasm_val_t,
-    results: *mut wasm_val_t,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
 ) -> Option<Box<wasm_trap_t>>;
 
 pub type wasmtime_func_callback_t = extern "C" fn(
     caller: *const wasmtime_caller_t,
-    args: *const wasm_val_t,
-    results: *mut wasm_val_t,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
 ) -> Option<Box<wasm_trap_t>>;
 
 pub type wasmtime_func_callback_with_env_t = extern "C" fn(
     caller: *const wasmtime_caller_t,
     env: *mut std::ffi::c_void,
-    args: *const wasm_val_t,
-    results: *mut wasm_val_t,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
 ) -> Option<Box<wasm_trap_t>>;
 
 struct Finalizer {
@@ -83,21 +85,25 @@ impl From<Func> for wasm_func_t {
 fn create_function(
     store: &wasm_store_t,
     ty: &wasm_functype_t,
-    func: impl Fn(Caller<'_>, *const wasm_val_t, *mut wasm_val_t) -> Option<Box<wasm_trap_t>> + 'static,
+    func: impl Fn(Caller<'_>, *const wasm_val_vec_t, *mut wasm_val_vec_t) -> Option<Box<wasm_trap_t>>
+        + 'static,
 ) -> Box<wasm_func_t> {
     let store = &store.store;
     let ty = ty.ty().ty.clone();
     let func = Func::new(store, ty, move |caller, params, results| {
-        let params = params
+        let params: wasm_val_vec_t = params
             .iter()
             .cloned()
             .map(|p| wasm_val_t::from_val(p))
-            .collect::<Vec<_>>();
-        let mut out_results = vec![wasm_val_t::default(); results.len()];
-        let out = func(caller, params.as_ptr(), out_results.as_mut_ptr());
+            .collect::<Vec<_>>()
+            .into();
+        let mut out_results: wasm_val_vec_t = vec![wasm_val_t::default(); results.len()].into();
+        let out = func(caller, &params, &mut out_results);
         if let Some(trap) = out {
             return Err(trap.trap.clone());
         }
+
+        let out_results = out_results.as_slice();
         for i in 0..results.len() {
             results[i] = out_results[i].val();
         }
@@ -164,17 +170,14 @@ pub extern "C" fn wasmtime_func_new_with_env(
 #[no_mangle]
 pub unsafe extern "C" fn wasm_func_call(
     wasm_func: &wasm_func_t,
-    args: *const wasm_val_t,
-    results: *mut MaybeUninit<wasm_val_t>,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
 ) -> *mut wasm_trap_t {
-    let func = wasm_func.func();
     let mut trap = ptr::null_mut();
-    let error = wasmtime_func_call(
+    let error = _wasmtime_func_call(
         wasm_func,
-        args,
-        func.param_arity(),
-        results,
-        func.result_arity(),
+        (*args).as_slice(),
+        (*results).as_uninit_slice(),
         &mut trap,
     );
     match error {
@@ -186,16 +189,14 @@ pub unsafe extern "C" fn wasm_func_call(
 #[no_mangle]
 pub unsafe extern "C" fn wasmtime_func_call(
     func: &wasm_func_t,
-    args: *const wasm_val_t,
-    num_args: usize,
-    results: *mut MaybeUninit<wasm_val_t>,
-    num_results: usize,
+    args: *const wasm_val_vec_t,
+    results: *mut wasm_val_vec_t,
     trap_ptr: &mut *mut wasm_trap_t,
 ) -> Option<Box<wasmtime_error_t>> {
     _wasmtime_func_call(
         func,
-        std::slice::from_raw_parts(args, num_args),
-        std::slice::from_raw_parts_mut(results, num_results),
+        (*args).as_slice(),
+        (*results).as_uninit_slice(),
         trap_ptr,
     )
 }

crates/c-api/src/instance.rs

@@ -40,16 +40,15 @@ impl wasm_instance_t {
 pub unsafe extern "C" fn wasm_instance_new(
     store: &wasm_store_t,
     wasm_module: &wasm_module_t,
-    imports: *const Box<wasm_extern_t>,
+    imports: *const wasm_extern_vec_t,
     result: Option<&mut *mut wasm_trap_t>,
 ) -> Option<Box<wasm_instance_t>> {
     let mut instance = ptr::null_mut();
     let mut trap = ptr::null_mut();
-    let err = wasmtime_instance_new(
+    let err = _wasmtime_instance_new(
         store,
         wasm_module,
-        imports,
-        wasm_module.module().imports().len(),
+        (*imports).as_slice(),
         &mut instance,
         &mut trap,
     );
@@ -83,31 +82,27 @@ pub unsafe extern "C" fn wasm_instance_new(
 pub unsafe extern "C" fn wasmtime_instance_new(
     store: &wasm_store_t,
     module: &wasm_module_t,
-    imports: *const Box<wasm_extern_t>,
-    num_imports: usize,
+    imports: *const wasm_extern_vec_t,
     instance_ptr: &mut *mut wasm_instance_t,
     trap_ptr: &mut *mut wasm_trap_t,
 ) -> Option<Box<wasmtime_error_t>> {
-    _wasmtime_instance_new(
-        store,
-        module,
-        std::slice::from_raw_parts(imports, num_imports),
-        instance_ptr,
-        trap_ptr,
-    )
+    _wasmtime_instance_new(store, module, (*imports).as_slice(), instance_ptr, trap_ptr)
 }
 
 fn _wasmtime_instance_new(
     store: &wasm_store_t,
     module: &wasm_module_t,
-    imports: &[Box<wasm_extern_t>],
+    imports: &[Option<Box<wasm_extern_t>>],
     instance_ptr: &mut *mut wasm_instance_t,
     trap_ptr: &mut *mut wasm_trap_t,
 ) -> Option<Box<wasmtime_error_t>> {
     let store = &store.store;
     let imports = imports
         .iter()
-        .map(|import| import.which.clone())
+        .filter_map(|import| match import {
+            Some(i) => Some(i.which.clone()),
+            None => None,
+        })
         .collect::<Vec<_>>();
     handle_instantiate(
         Instance::new(store, module.module(), &imports),

crates/c-api/src/vec.rs

@@ -4,6 +4,7 @@ use crate::{
     wasm_moduletype_t, wasm_tabletype_t, wasm_val_t, wasm_valtype_t,
 };
 use std::mem;
+use std::mem::MaybeUninit;
 use std::ptr;
 use std::slice;
 
@@ -54,6 +55,18 @@ macro_rules! declare_vecs {
                 }
             }
 
+            pub fn as_uninit_slice(&mut self) -> &mut [MaybeUninit<$elem_ty>] {
+                // Note that we're careful to not create a slice with a null
+                // pointer as the data pointer, since that isn't defined
+                // behavior in Rust.
+                if self.size == 0 {
+                    &mut []
+                } else {
+                    assert!(!self.data.is_null());
+                    unsafe { slice::from_raw_parts_mut(self.data as _, self.size) }
+                }
+            }
+
             pub fn take(&mut self) -> Vec<$elem_ty> {
                 if self.data.is_null() {
                     return Vec::new();

crates/cache/Cargo.toml

@@ -18,7 +18,7 @@ log = { version = "0.4.8", default-features = false }
 serde = { version = "1.0.94", features = ["derive"] }
 sha2 = "0.9.0"
 toml = "0.5.5"
-zstd = "0.5"
+zstd = { version = "0.6", default-features = false }
 
 [target.'cfg(target_os = "windows")'.dependencies]
 winapi = "0.3.7"

crates/cranelift/Cargo.toml

@@ -17,3 +17,4 @@ cranelift-wasm = { path = "../../cranelift/wasm", version = "0.69.0" }
 cranelift-codegen = { path = "../../cranelift/codegen", version = "0.69.0" }
 cranelift-frontend = { path = "../../cranelift/frontend", version = "0.69.0" }
 cranelift-entity = { path = "../../cranelift/entity", version = "0.69.0" }
+wasmparser = "0.73.0"

crates/cranelift/src/func_environ.rs

@@ -7,11 +7,14 @@ use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Si
 use cranelift_codegen::isa::{self, TargetFrontendConfig};
 use cranelift_entity::{EntityRef, PrimaryMap};
 use cranelift_frontend::FunctionBuilder;
+use cranelift_frontend::Variable;
 use cranelift_wasm::{
-    self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableIndex,
-    TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType,
+    self, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, MemoryIndex,
+    SignatureIndex, TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType,
 };
 use std::convert::TryFrom;
+use std::mem;
+use wasmparser::Operator;
 use wasmtime_environ::{
     BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, TableStyle, Tunables, VMOffsets,
     INTERRUPTED, WASM_PAGE_SIZE,
@@ -125,6 +128,20 @@ pub struct FuncEnvironment<'module_environment> {
     pub(crate) offsets: VMOffsets,
 
     tunables: &'module_environment Tunables,
+
+    /// A function-local variable which stores the cached value of the amount of
+    /// fuel remaining to execute. If used this is modified frequently so it's
+    /// stored locally as a variable instead of always referenced from the field
+    /// in `*const VMInterrupts`
+    fuel_var: cranelift_frontend::Variable,
+
+    /// A function-local variable which caches the value of `*const
+    /// VMInterrupts` for this function's vmctx argument. This pointer is stored
+    /// in the vmctx itself, but never changes for the lifetime of the function,
+    /// so if we load it up front we can continue to use it throughout.
+    vminterrupts_ptr: cranelift_frontend::Variable,
+
+    fuel_consumed: i64,
 }
 
 impl<'module_environment> FuncEnvironment<'module_environment> {
@@ -151,6 +168,12 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             builtin_function_signatures,
             offsets: VMOffsets::new(target_config.pointer_bytes(), module),
             tunables,
+            fuel_var: Variable::new(0),
+            vminterrupts_ptr: Variable::new(0),
+
+            // Start with at least one fuel being consumed because even empty
+            // functions should consume at least some fuel.
+            fuel_consumed: 1,
         }
     }
 
@@ -418,6 +441,241 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             (global, 0)
         }
     }
+
+    fn declare_vminterrupts_ptr(&mut self, builder: &mut FunctionBuilder<'_>) {
+        // We load the `*const VMInterrupts` value stored within vmctx at the
+        // head of the function and reuse the same value across the entire
+        // function. This is possible since we know that the pointer never
+        // changes for the lifetime of the function.
+        let pointer_type = self.pointer_type();
+        builder.declare_var(self.vminterrupts_ptr, pointer_type);
+        let vmctx = self.vmctx(builder.func);
+        let base = builder.ins().global_value(pointer_type, vmctx);
+        let offset = i32::try_from(self.offsets.vmctx_interrupts()).unwrap();
+        let interrupt_ptr = builder
+            .ins()
+            .load(pointer_type, ir::MemFlags::trusted(), base, offset);
+        builder.def_var(self.vminterrupts_ptr, interrupt_ptr);
+    }
+
+    fn fuel_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) {
+        // On function entry we load the amount of fuel into a function-local
+        // `self.fuel_var` to make fuel modifications fast locally. This cache
+        // is then periodically flushed to the Store-defined location in
+        // `VMInterrupts` later.
+        builder.declare_var(self.fuel_var, ir::types::I64);
+        self.fuel_load_into_var(builder);
+        self.fuel_check(builder);
+    }
+
+    fn fuel_function_exit(&mut self, builder: &mut FunctionBuilder<'_>) {
+        // On exiting the function we need to be sure to save the fuel we have
+        // cached locally in `self.fuel_var` back into the Store-defined
+        // location.
+        self.fuel_save_from_var(builder);
+    }
+
+    fn fuel_before_op(
+        &mut self,
+        op: &Operator<'_>,
+        builder: &mut FunctionBuilder<'_>,
+        reachable: bool,
+    ) {
+        if !reachable {
+            // In unreachable code we shouldn't have any leftover fuel we
+            // haven't accounted for since the reason for us to become
+            // unreachable should have already added it to `self.fuel_var`.
+            debug_assert_eq!(self.fuel_consumed, 0);
+            return;
+        }
+
+        self.fuel_consumed += match op {
+            // Nop and drop generate no code, so don't consume fuel for them.
+            Operator::Nop | Operator::Drop => 0,
+
+            // Control flow may create branches, but is generally cheap and
+            // free, so don't consume fuel. Note the lack of `if` since some
+            // cost is incurred with the conditional check.
+            Operator::Block { .. }
+            | Operator::Loop { .. }
+            | Operator::Unreachable
+            | Operator::Return
+            | Operator::Else
+            | Operator::End => 0,
+
+            // everything else, just call it one operation.
+            _ => 1,
+        };
+
+        match op {
+            // Exiting a function (via a return or unreachable) or otherwise
+            // entering a different function (via a call) means that we need to
+            // update the fuel consumption in `VMInterrupts` because we're
+            // about to move control out of this function itself and the fuel
+            // may need to be read.
+            //
+            // Before this we need to update the fuel counter from our own cost
+            // leading up to this function call, and then we can store
+            // `self.fuel_var` into `VMInterrupts`.
+            Operator::Unreachable
+            | Operator::Return
+            | Operator::CallIndirect { .. }
+            | Operator::Call { .. }
+            | Operator::ReturnCall { .. }
+            | Operator::ReturnCallIndirect { .. } => {
+                self.fuel_increment_var(builder);
+                self.fuel_save_from_var(builder);
+            }
+
+            // To ensure all code preceding a loop is only counted once we
+            // update the fuel variable on entry.
+            Operator::Loop { .. }
+
+            // Entering into an `if` block means that the edge we take isn't
+            // known until runtime, so we need to update our fuel consumption
+            // before we take the branch.
+            | Operator::If { .. }
+
+            // Control-flow instructions mean that we're moving to the end/exit
+            // of a block somewhere else. That means we need to update the fuel
+            // counter since we're effectively terminating our basic block.
+            | Operator::Br { .. }
+            | Operator::BrIf { .. }
+            | Operator::BrTable { .. }
+
+            // Exiting a scope means that we need to update the fuel
+            // consumption because there are multiple ways to exit a scope and
+            // this is the only time we have to account for instructions
+            // executed so far.
+            | Operator::End
+
+            // This is similar to `end`, except that it's only the terminator
+            // for an `if` block. The same reasoning applies though in that we
+            // are terminating a basic block and need to update the fuel
+            // variable.
+            | Operator::Else => self.fuel_increment_var(builder),
+
+            // This is a normal instruction where the fuel is buffered to later
+            // get added to `self.fuel_var`.
+            //
+            // Note that we generally ignore instructions which may trap and
+            // therefore result in exiting a block early. Current usage of fuel
+            // means that it's not too important to account for a precise amount
+            // of fuel consumed but rather "close to the actual amount" is good
+            // enough. For 100% precise counting, however, we'd probably need to
+            // not only increment but also save the fuel amount more often
+            // around trapping instructions. (see the `unreachable` instruction
+            // case above)
+            //
+            // Note that `Block` is specifically omitted from incrementing the
+            // fuel variable. Control flow entering a `block` is unconditional
+            // which means it's effectively executing straight-line code. We'll
+            // update the counter when exiting a block, but we shouldn't need to
+            // do so upon entering a block.
+            _ => {}
+        }
+    }
+
+    fn fuel_after_op(&mut self, op: &Operator<'_>, builder: &mut FunctionBuilder<'_>) {
+        // After a function call we need to reload our fuel value since the
+        // function may have changed it.
+        match op {
+            Operator::Call { .. } | Operator::CallIndirect { .. } => {
+                self.fuel_load_into_var(builder);
+            }
+            _ => {}
+        }
+    }
+
+    /// Adds `self.fuel_consumed` to the `fuel_var`, zero-ing out the amount of
+    /// fuel consumed at that point.
+    fn fuel_increment_var(&mut self, builder: &mut FunctionBuilder<'_>) {
+        let consumption = mem::replace(&mut self.fuel_consumed, 0);
+        if consumption == 0 {
+            return;
+        }
+
+        let fuel = builder.use_var(self.fuel_var);
+        let fuel = builder.ins().iadd_imm(fuel, consumption);
+        builder.def_var(self.fuel_var, fuel);
+    }
+
+    /// Loads the fuel consumption value from `VMInterrupts` into `self.fuel_var`
+    fn fuel_load_into_var(&mut self, builder: &mut FunctionBuilder<'_>) {
+        let (addr, offset) = self.fuel_addr_offset(builder);
+        let fuel = builder
+            .ins()
+            .load(ir::types::I64, ir::MemFlags::trusted(), addr, offset);
+        builder.def_var(self.fuel_var, fuel);
+    }
+
+    /// Stores the fuel consumption value from `self.fuel_var` into
+    /// `VMInterrupts`.
+    fn fuel_save_from_var(&mut self, builder: &mut FunctionBuilder<'_>) {
+        let (addr, offset) = self.fuel_addr_offset(builder);
+        let fuel_consumed = builder.use_var(self.fuel_var);
+        builder
+            .ins()
+            .store(ir::MemFlags::trusted(), fuel_consumed, addr, offset);
+    }
+
+    /// Returns the `(address, offset)` of the fuel consumption within
+    /// `VMInterrupts`, used to perform loads/stores later.
+    fn fuel_addr_offset(
+        &mut self,
+        builder: &mut FunctionBuilder<'_>,
+    ) -> (ir::Value, ir::immediates::Offset32) {
+        (
+            builder.use_var(self.vminterrupts_ptr),
+            i32::from(self.offsets.vminterrupts_fuel_consumed()).into(),
+        )
+    }
+
+    /// Checks the amount of remaining, and if we've run out of fuel we call
+    /// the out-of-fuel function.
+    fn fuel_check(&mut self, builder: &mut FunctionBuilder) {
+        self.fuel_increment_var(builder);
+        let out_of_gas_block = builder.create_block();
+        let continuation_block = builder.create_block();
+
+        // Note that our fuel is encoded as adding positive values to a
+        // negative number. Whenever the negative number goes positive that
+        // means we ran out of fuel.
+        //
+        // Compare to see if our fuel is positive, and if so we ran out of gas.
+        // Otherwise we can continue on like usual.
+        let zero = builder.ins().iconst(ir::types::I64, 0);
+        let fuel = builder.use_var(self.fuel_var);
+        let cmp = builder.ins().ifcmp(fuel, zero);
+        builder
+            .ins()
+            .brif(IntCC::SignedGreaterThanOrEqual, cmp, out_of_gas_block, &[]);
+        builder.ins().jump(continuation_block, &[]);
+        builder.seal_block(out_of_gas_block);
+
+        // If we ran out of gas then we call our out-of-gas intrinsic and it
+        // figures out what to do. Note that this may raise a trap, or do
+        // something like yield to an async runtime. In either case we don't
+        // assume what happens and handle the case the intrinsic returns.
+        //
+        // Note that we save/reload fuel around this since the out-of-gas
+        // intrinsic may alter how much fuel is in the system.
+        builder.switch_to_block(out_of_gas_block);
+        self.fuel_save_from_var(builder);
+        let out_of_gas_sig = self.builtin_function_signatures.out_of_gas(builder.func);
+        let (vmctx, out_of_gas) = self.translate_load_builtin_function_address(
+            &mut builder.cursor(),
+            BuiltinFunctionIndex::out_of_gas(),
+        );
+        builder
+            .ins()
+            .call_indirect(out_of_gas_sig, out_of_gas, &[vmctx]);
+        self.fuel_load_into_var(builder);
+        builder.ins().jump(continuation_block, &[]);
+        builder.seal_block(continuation_block);
+
+        builder.switch_to_block(continuation_block);
+    }
 }
 
 impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environment> {
@@ -437,6 +695,11 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         index >= 2
     }
 
+    fn after_locals(&mut self, num_locals: usize) {
+        self.vminterrupts_ptr = Variable::new(num_locals);
+        self.fuel_var = Variable::new(num_locals + 1);
+    }
+
     fn make_table(&mut self, func: &mut ir::Function, index: TableIndex) -> WasmResult<ir::Table> {
         let pointer_type = self.pointer_type();
 
@@ -1482,24 +1745,16 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
         Ok(*pos.func.dfg.inst_results(call_inst).first().unwrap())
     }
 
-    fn translate_loop_header(&mut self, mut pos: FuncCursor) -> WasmResult<()> {
-        if !self.tunables.interruptable {
-            return Ok(());
-        }
-
-        // Start out each loop with a check to the interupt flag to allow
-        // interruption of long or infinite loops.
+    fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {
+        // If enabled check the interrupt flag to prevent long or infinite
+        // loops.
         //
         // For more information about this see comments in
         // `crates/environ/src/cranelift.rs`
-        let vmctx = self.vmctx(&mut pos.func);
+        if self.tunables.interruptable {
             let pointer_type = self.pointer_type();
-        let base = pos.ins().global_value(pointer_type, vmctx);
-        let offset = i32::try_from(self.offsets.vmctx_interrupts()).unwrap();
-        let interrupt_ptr = pos
-            .ins()
-            .load(pointer_type, ir::MemFlags::trusted(), base, offset);
-        let interrupt = pos.ins().load(
+            let interrupt_ptr = builder.use_var(self.vminterrupts_ptr);
+            let interrupt = builder.ins().load(
                 pointer_type,
                 ir::MemFlags::trusted(),
                 interrupt_ptr,
@@ -1507,11 +1762,73 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
             );
             // Note that the cast to `isize` happens first to allow sign-extension,
             // if necessary, to `i64`.
-        let interrupted_sentinel = pos.ins().iconst(pointer_type, INTERRUPTED as isize as i64);
-        let cmp = pos
+            let interrupted_sentinel = builder
+                .ins()
+                .iconst(pointer_type, INTERRUPTED as isize as i64);
+            let cmp = builder
                 .ins()
                 .icmp(IntCC::Equal, interrupt, interrupted_sentinel);
-        pos.ins().trapnz(cmp, ir::TrapCode::Interrupt);
+            builder.ins().trapnz(cmp, ir::TrapCode::Interrupt);
+        }
+
+        // Additionally if enabled check how much fuel we have remaining to see
+        // if we've run out by this point.
+        if self.tunables.consume_fuel {
+            self.fuel_check(builder);
+        }
+
+        Ok(())
+    }
+
+    fn before_translate_operator(
+        &mut self,
+        op: &Operator,
+        builder: &mut FunctionBuilder,
+        state: &FuncTranslationState,
+    ) -> WasmResult<()> {
+        if self.tunables.consume_fuel {
+            self.fuel_before_op(op, builder, state.reachable());
+        }
+        Ok(())
+    }
+
+    fn after_translate_operator(
+        &mut self,
+        op: &Operator,
+        builder: &mut FunctionBuilder,
+        state: &FuncTranslationState,
+    ) -> WasmResult<()> {
+        if self.tunables.consume_fuel && state.reachable() {
+            self.fuel_after_op(op, builder);
+        }
+        Ok(())
+    }
+
+    fn before_translate_function(
+        &mut self,
+        builder: &mut FunctionBuilder,
+        _state: &FuncTranslationState,
+    ) -> WasmResult<()> {
+        // If the `vminterrupts_ptr` variable will get used then we initialize
+        // it here.
+        if self.tunables.consume_fuel || self.tunables.interruptable {
+            self.declare_vminterrupts_ptr(builder);
+        }
+        // Additionally we initialize `fuel_var` if it will get used.
+        if self.tunables.consume_fuel {
+            self.fuel_function_entry(builder);
+        }
+        Ok(())
+    }
+
+    fn after_translate_function(
+        &mut self,
+        builder: &mut FunctionBuilder,
+        state: &FuncTranslationState,
+    ) -> WasmResult<()> {
+        if self.tunables.consume_fuel && state.reachable() {
+            self.fuel_function_exit(builder);
+        }
         Ok(())
     }
 }

crates/debug/Cargo.toml

@@ -13,8 +13,8 @@ edition = "2018"
 
 [dependencies]
 gimli = "0.23.0"
-wasmparser = "0.71"
-object = { version = "0.22.0", default-features = false, features = ["read_core", "elf", "write"] }
+wasmparser = "0.73"
+object = { version = "0.23.0", default-features = false, features = ["read_core", "elf", "write"] }
 wasmtime-environ = { path = "../environ", version = "0.22.0" }
 target-lexicon = { version = "0.11.0", default-features = false }
 anyhow = "1.0"

crates/debug/src/transform/expression.rs

@@ -7,7 +7,7 @@ use std::collections::{HashMap, HashSet};
 use std::hash::{Hash, Hasher};
 use std::rc::Rc;
 use wasmtime_environ::entity::EntityRef;
-use wasmtime_environ::ir::{StackSlots, ValueLabel, ValueLabelsRanges, ValueLoc};
+use wasmtime_environ::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLabelsRanges, ValueLoc};
 use wasmtime_environ::isa::TargetIsa;
 use wasmtime_environ::wasm::{get_vmctx_value_label, DefinedFuncIndex};
 use wasmtime_environ::ModuleMemoryOffset;
@@ -131,27 +131,24 @@ impl CompiledExpression {
 const X86_64_STACK_OFFSET: i64 = 16;
 
 fn translate_loc(
-    loc: ValueLoc,
+    loc: LabelValueLoc,
     frame_info: Option<&FunctionFrameInfo>,
     isa: &dyn TargetIsa,
     add_stack_value: bool,
 ) -> Result<Option<Vec<u8>>> {
     Ok(match loc {
-        ValueLoc::Reg(reg) if add_stack_value => {
+        LabelValueLoc::ValueLoc(ValueLoc::Reg(reg)) => {
             let machine_reg = isa.map_dwarf_register(reg)?;
             let mut writer = ExpressionWriter::new();
+            if add_stack_value {
                 writer.write_op_reg(machine_reg)?;
-            Some(writer.into_vec())
-        }
-        ValueLoc::Reg(reg) => {
-            assert!(!add_stack_value);
-            let machine_reg = isa.map_dwarf_register(reg)?;
-            let mut writer = ExpressionWriter::new();
+            } else {
                 writer.write_op_breg(machine_reg)?;
                 writer.write_sleb128(0)?;
+            }
             Some(writer.into_vec())
         }
-        ValueLoc::Stack(ss) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => {
             if let Some(frame_info) = frame_info {
                 if let Some(ss_offset) = frame_info.stack_slots[ss].offset {
                     let mut writer = ExpressionWriter::new();
@@ -165,6 +162,27 @@ fn translate_loc(
             }
             None
         }
+        LabelValueLoc::Reg(r) => {
+            let machine_reg = isa.map_regalloc_reg_to_dwarf(r)?;
+            let mut writer = ExpressionWriter::new();
+            if add_stack_value {
+                writer.write_op_reg(machine_reg)?;
+            } else {
+                writer.write_op_breg(machine_reg)?;
+                writer.write_sleb128(0)?;
+            }
+            Some(writer.into_vec())
+        }
+        LabelValueLoc::SPOffset(off) => {
+            let mut writer = ExpressionWriter::new();
+            writer.write_op_breg(X86_64::RSP.0)?;
+            writer.write_sleb128(off)?;
+            if !add_stack_value {
+                writer.write_op(gimli::constants::DW_OP_deref)?;
+            }
+            return Ok(Some(writer.into_vec()));
+        }
+
         _ => None,
     })
 }
@@ -172,13 +190,13 @@ fn translate_loc(
 fn append_memory_deref(
     buf: &mut Vec<u8>,
     frame_info: &FunctionFrameInfo,
-    vmctx_loc: ValueLoc,
+    vmctx_loc: LabelValueLoc,
     isa: &dyn TargetIsa,
 ) -> Result<bool> {
     let mut writer = ExpressionWriter::new();
     // FIXME for imported memory
     match vmctx_loc {
-        ValueLoc::Reg(vmctx_reg) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Reg(vmctx_reg)) => {
             let reg = isa.map_dwarf_register(vmctx_reg)? as u8;
             writer.write_u8(gimli::constants::DW_OP_breg0.0 + reg)?;
             let memory_offset = match frame_info.vmctx_memory_offset() {
@@ -189,7 +207,7 @@ fn append_memory_deref(
             };
             writer.write_sleb128(memory_offset)?;
         }
-        ValueLoc::Stack(ss) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => {
             if let Some(ss_offset) = frame_info.stack_slots[ss].offset {
                 writer.write_op_breg(X86_64::RBP.0)?;
                 writer.write_sleb128(ss_offset as i64 + X86_64_STACK_OFFSET)?;
@@ -207,6 +225,31 @@ fn append_memory_deref(
                 return Ok(false);
             }
         }
+        LabelValueLoc::Reg(r) => {
+            let reg = isa.map_regalloc_reg_to_dwarf(r)?;
+            writer.write_op_breg(reg)?;
+            let memory_offset = match frame_info.vmctx_memory_offset() {
+                Some(offset) => offset,
+                None => {
+                    return Ok(false);
+                }
+            };
+            writer.write_sleb128(memory_offset)?;
+        }
+        LabelValueLoc::SPOffset(off) => {
+            writer.write_op_breg(X86_64::RSP.0)?;
+            writer.write_sleb128(off)?;
+            writer.write_op(gimli::constants::DW_OP_deref)?;
+            writer.write_op(gimli::constants::DW_OP_consts)?;
+            let memory_offset = match frame_info.vmctx_memory_offset() {
+                Some(offset) => offset,
+                None => {
+                    return Ok(false);
+                }
+            };
+            writer.write_sleb128(memory_offset)?;
+            writer.write_op(gimli::constants::DW_OP_plus)?;
+        }
         _ => {
             return Ok(false);
         }
@@ -468,7 +511,7 @@ where
                 let _ = code_chunk; // suppresses warning for final flush
             }
         };
-    };
+    }
     // Find all landing pads by scanning bytes, do not care about
     // false location at this moment.
     // Looks hacky but it is fast; does not need to be really exact.
@@ -653,7 +696,7 @@ struct CachedValueLabelRange {
     func_index: DefinedFuncIndex,
     start: usize,
     end: usize,
-    label_location: HashMap<ValueLabel, ValueLoc>,
+    label_location: HashMap<ValueLabel, LabelValueLoc>,
 }
 
 struct ValueLabelRangesBuilder<'a, 'b> {
@@ -1179,7 +1222,7 @@ mod tests {
     fn create_mock_value_ranges() -> (ValueLabelsRanges, (ValueLabel, ValueLabel, ValueLabel)) {
         use std::collections::HashMap;
         use wasmtime_environ::entity::EntityRef;
-        use wasmtime_environ::ir::{ValueLoc, ValueLocRange};
+        use wasmtime_environ::ir::{LabelValueLoc, ValueLoc, ValueLocRange};
         let mut value_ranges = HashMap::new();
         let value_0 = ValueLabel::new(0);
         let value_1 = ValueLabel::new(1);
@@ -1187,7 +1230,7 @@ mod tests {
         value_ranges.insert(
             value_0,
             vec![ValueLocRange {
-                loc: ValueLoc::Unassigned,
+                loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned),
                 start: 0,
                 end: 25,
             }],
@@ -1195,7 +1238,7 @@ mod tests {
         value_ranges.insert(
             value_1,
             vec![ValueLocRange {
-                loc: ValueLoc::Unassigned,
+                loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned),
                 start: 5,
                 end: 30,
             }],
@@ -1204,12 +1247,12 @@ mod tests {
             value_2,
             vec![
                 ValueLocRange {
-                    loc: ValueLoc::Unassigned,
+                    loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned),
                     start: 0,
                     end: 10,
                 },
                 ValueLocRange {
-                    loc: ValueLoc::Unassigned,
+                    loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned),
                     start: 20,
                     end: 30,
                 },

crates/environ/Cargo.toml

@@ -16,7 +16,7 @@ anyhow = "1.0"
 cranelift-codegen = { path = "../../cranelift/codegen", version = "0.69.0", features = ["enable-serde"] }
 cranelift-entity = { path = "../../cranelift/entity", version = "0.69.0", features = ["enable-serde"] }
 cranelift-wasm = { path = "../../cranelift/wasm", version = "0.69.0", features = ["enable-serde"] }
-wasmparser = "0.71"
+wasmparser = "0.73"
 indexmap = { version = "1.0.2", features = ["serde-1"] }
 thiserror = "1.0.4"
 serde = { version = "1.0.94", features = ["derive"] }

crates/environ/src/builtin.rs

@@ -57,6 +57,8 @@ macro_rules! foreach_builtin_function {
             memory_atomic_wait64(vmctx, i32, i32, i64, i64) -> (i32);
             /// Returns an index for wasm's `memory.atomic.wait64` for imported memories.
             imported_memory_atomic_wait64(vmctx, i32, i32, i64, i64) -> (i32);
+            /// Invoked when fuel has run out while executing a function.
+            out_of_gas(vmctx) -> ();
         }
     };
 }

crates/environ/src/data_structures.rs

@@ -3,8 +3,8 @@
 pub mod ir {
     pub use cranelift_codegen::binemit::{Reloc, StackMap};
     pub use cranelift_codegen::ir::{
-        types, AbiParam, ArgumentPurpose, JumpTableOffsets, LibCall, Signature, SourceLoc,
-        StackSlots, TrapCode, Type, ValueLabel, ValueLoc,
+        types, AbiParam, ArgumentPurpose, JumpTableOffsets, LabelValueLoc, LibCall, Signature,
+        SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc,
     };
     pub use cranelift_codegen::{ValueLabelsRanges, ValueLocRange};
 }

crates/environ/src/module.rs

@@ -142,12 +142,6 @@ impl ModuleType {
 /// memory initializers.
 #[derive(Default, Debug, Clone, Serialize, Deserialize)]
 pub struct Module {
-    /// The parent index of this module, used for the module linking proposal.
-    ///
-    /// This index is into the list of modules returned from compilation of a
-    /// single wasm file with nested modules.
-    pub parent: Option<usize>,
-
     /// The name of this wasm module, often found in the wasm file.
     pub name: Option<String>,
 
@@ -213,25 +207,26 @@ pub struct Module {
 pub enum Initializer {
     /// An imported item is required to be provided.
     Import {
-        /// Module name of this import
-        module: String,
-        /// Optional field name of this import
+        /// Name of this import
+        name: String,
+        /// The field name projection of this import. When module-linking is
+        /// enabled this is always `None`. Otherwise this is always `Some`.
         field: Option<String>,
         /// Where this import will be placed, which also has type information
         /// about the import.
         index: EntityIndex,
     },
 
-    /// A module from the parent's declared modules is inserted into our own
+    /// An export from a previously defined instance is being inserted into our
     /// index space.
-    AliasParentModule(ModuleIndex),
-
-    /// A module from the parent's declared modules is inserted into our own
-    /// index space.
-    #[allow(missing_docs)]
+    ///
+    /// Note that when the module linking proposal is enabled two-level imports
+    /// will implicitly desugar to this initializer.
     AliasInstanceExport {
+        /// The instance that we're referencing.
         instance: InstanceIndex,
-        export: usize,
+        /// Which export is being inserted into our index space.
+        export: String,
     },
 
     /// A module is being instantiated with previously configured intializers
@@ -239,15 +234,36 @@ pub enum Initializer {
     Instantiate {
         /// The module that this instance is instantiating.
         module: ModuleIndex,
-        /// The arguments provided to instantiation.
-        args: Vec<EntityIndex>,
+        /// The arguments provided to instantiation, along with their name in
+        /// the instance being instantiated.
+        args: IndexMap<String, EntityIndex>,
     },
 
-    /// A module is defined into the module index space, and which module is
-    /// being defined is specified by the index payload.
+    /// A module is being created from a set of compiled artifacts.
+    CreateModule {
+        /// The index of the artifact that's being convereted into a module.
+        artifact_index: usize,
+        /// The list of artifacts that this module value will be inheriting.
+        artifacts: Vec<usize>,
+        /// The list of modules that this module value will inherit.
+        modules: Vec<ModuleUpvar>,
+    },
+
+    /// A module is created from a closed-over-module value, defined when this
+    /// module was created.
     DefineModule(usize),
 }
 
+/// Where module values can come from when creating a new module from a compiled
+/// artifact.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum ModuleUpvar {
+    /// A module value is inherited from the module creating the new module.
+    Inherit(usize),
+    /// A module value comes from the instance-to-be-created module index space.
+    Local(ModuleIndex),
+}
+
 impl Module {
     /// Allocates the module data structures.
     pub fn new() -> Self {
@@ -351,11 +367,9 @@ impl Module {
     /// module name, field name, and type that's being imported.
     pub fn imports(&self) -> impl Iterator<Item = (&str, Option<&str>, EntityType)> {
         self.initializers.iter().filter_map(move |i| match i {
-            Initializer::Import {
-                module,
-                field,
-                index,
-            } => Some((module.as_str(), field.as_deref(), self.type_of(*index))),
+            Initializer::Import { name, field, index } => {
+                Some((name.as_str(), field.as_deref(), self.type_of(*index)))
+            }
             _ => None,
         })
     }
@@ -389,16 +403,16 @@ pub struct TypeTables {
 /// The type signature of known modules.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ModuleSignature {
-    /// All imports in this module, listed in order with their module/name and
+    /// All imports in this module, listed in order with their name and
     /// what type they're importing.
-    pub imports: Vec<(String, Option<String>, EntityType)>,
+    pub imports: IndexMap<String, EntityType>,
     /// Exports are what an instance type conveys, so we go through an
     /// indirection over there.
     pub exports: InstanceTypeIndex,
 }
 
 /// The type signature of known instances.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct InstanceSignature {
     /// The name of what's being exported as well as its type signature.
     pub exports: IndexMap<String, EntityType>,

crates/environ/src/module_environ.rs

@@ -1,6 +1,6 @@
 use crate::module::{
-    Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, TableElements,
-    TablePlan, TypeTables,
+    Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, ModuleUpvar,
+    TableElements, TablePlan, TypeTables,
 };
 use crate::tunables::Tunables;
 use cranelift_codegen::ir;
@@ -14,7 +14,7 @@ use cranelift_wasm::{
     WasmError, WasmFuncType, WasmResult,
 };
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{hash_map::Entry, HashMap};
 use std::convert::TryFrom;
 use std::mem;
 use std::path::PathBuf;
@@ -31,6 +31,10 @@ pub struct ModuleEnvironment<'data> {
     /// the module linking proposal.
     results: Vec<ModuleTranslation<'data>>,
 
+    /// Modules which are in-progress being translated, or otherwise also known
+    /// as the outer modules of the current module being processed.
+    in_progress: Vec<ModuleTranslation<'data>>,
+
     /// How many modules that have not yet made their way into `results` which
     /// are coming at some point.
     modules_to_be: usize,
@@ -38,13 +42,11 @@ pub struct ModuleEnvironment<'data> {
     /// Intern'd types for this entire translation, shared by all modules.
     types: TypeTables,
 
-    /// Where our module will get pushed into `results` after it's finished.
-    cur: usize,
-
     // Various bits and pieces of configuration
     features: WasmFeatures,
     target_config: TargetFrontendConfig,
     tunables: Tunables,
+    first_module: bool,
 }
 
 /// The result of translating via `ModuleEnvironment`. Function bodies are not
@@ -72,15 +74,15 @@ pub struct ModuleTranslation<'data> {
     /// which function is currently being defined.
     code_index: u32,
 
-    /// When local modules are declared an entry is pushed onto this list which
-    /// indicates that the initializer at the specified position needs to be
-    /// rewritten with the module's final index in the global list of compiled
-    /// modules.
-    module_initializer_indexes: Vec<usize>,
+    implicit_instances: HashMap<&'data str, InstanceIndex>,
 
-    /// Used as a pointer into the above list as the module code section is
-    /// parsed.
-    num_modules_defined: usize,
+    /// The artifacts which are needed from the parent module when this module
+    /// is created. This is used to insert into `Initializer::CreateModule` when
+    /// this module is defined in the parent.
+    creation_artifacts: Vec<usize>,
+
+    /// Same as `creation_artifacts`, but for modules instead of artifacts.
+    creation_modules: Vec<ModuleUpvar>,
 }
 
 /// Contains function data: byte code and its offset in the module.
@@ -142,12 +144,13 @@ impl<'data> ModuleEnvironment<'data> {
         Self {
             result: ModuleTranslation::default(),
             results: Vec::with_capacity(1),
+            in_progress: Vec::new(),
             modules_to_be: 1,
-            cur: 0,
             types: Default::default(),
             target_config,
             tunables: tunables.clone(),
             features: *features,
+            first_module: true,
         }
     }
 
@@ -163,7 +166,8 @@ impl<'data> ModuleEnvironment<'data> {
     /// Note that for MVP modules this will always be a list with one element,
     /// but with the module linking proposal this may have many elements.
     ///
-    /// For the module linking proposal the top-level module is at index 0.
+    /// For the module linking proposal the top-level module is returned as the
+    /// first return value.
     ///
     /// The `TypeTables` structure returned contains intern'd versions of types
     /// referenced from each module translation. This primarily serves as the
@@ -173,10 +177,10 @@ impl<'data> ModuleEnvironment<'data> {
     pub fn translate(
         mut self,
         data: &'data [u8],
-    ) -> WasmResult<(Vec<ModuleTranslation<'data>>, TypeTables)> {
+    ) -> WasmResult<(usize, Vec<ModuleTranslation<'data>>, TypeTables)> {
         translate_module(data, &mut self)?;
         assert!(self.results.len() > 0);
-        Ok((self.results, self.types))
+        Ok((self.results.len() - 1, self.results, self.types))
     }
 
     fn declare_export(&mut self, export: EntityIndex, name: &str) -> WasmResult<()> {
@@ -224,6 +228,136 @@ impl<'data> ModuleEnvironment<'data> {
         dwarf.ranges = gimli::RangeLists::new(info.debug_ranges, info.debug_rnglists);
         dwarf.locations = gimli::LocationLists::new(info.debug_loc, info.debug_loclists);
     }
+
+    /// Declares a new import with the `module` and `field` names, importing the
+    /// `ty` specified.
+    ///
+    /// Note that this method is somewhat tricky due to the implementation of
+    /// the module linking proposal. In the module linking proposal two-level
+    /// imports are recast as single-level imports of instances. That recasting
+    /// happens here by recording an import of an instance for the first time
+    /// we see a two-level import.
+    ///
+    /// When the module linking proposal is disabled, however, disregard this
+    /// logic and instead work directly with two-level imports since no
+    /// instances are defined.
+    fn declare_import(&mut self, module: &'data str, field: Option<&'data str>, ty: EntityType) {
+        if !self.features.module_linking {
+            assert!(field.is_some());
+            let index = self.push_type(ty);
+            self.result.module.initializers.push(Initializer::Import {
+                name: module.to_owned(),
+                field: field.map(|s| s.to_string()),
+                index,
+            });
+            return;
+        }
+
+        match field {
+            Some(field) => {
+                // If this is a two-level import then this is actually an
+                // implicit import of an instance, where each two-level import
+                // is an alias directive from the original instance. The first
+                // thing we do here is lookup our implicit instance, creating a
+                // blank one if it wasn't already created.
+                let instance = match self.result.implicit_instances.entry(module) {
+                    Entry::Occupied(e) => *e.get(),
+                    Entry::Vacant(v) => {
+                        let ty = self
+                            .types
+                            .instance_signatures
+                            .push(InstanceSignature::default());
+                        let idx = self.result.module.instances.push(ty);
+                        self.result.module.initializers.push(Initializer::Import {
+                            name: module.to_owned(),
+                            field: None,
+                            index: EntityIndex::Instance(idx),
+                        });
+                        *v.insert(idx)
+                    }
+                };
+
+                // Update the implicit instance's type signature with this new
+                // field and its type.
+                self.types.instance_signatures[self.result.module.instances[instance]]
+                    .exports
+                    .insert(field.to_string(), ty.clone());
+
+                // Record our implicit alias annotation which corresponds to
+                // this import that we're processing.
+                self.result
+                    .module
+                    .initializers
+                    .push(Initializer::AliasInstanceExport {
+                        instance,
+                        export: field.to_string(),
+                    });
+
+                // And then record the type information for the item that we're
+                // processing.
+                self.push_type(ty);
+            }
+            None => {
+                // Without a field then this is a single-level import (a feature
+                // of module linking) which means we're simply importing that
+                // name with the specified type. Record the type information and
+                // then the name that we're importing.
+                let index = self.push_type(ty);
+                self.result.module.initializers.push(Initializer::Import {
+                    name: module.to_owned(),
+                    field: None,
+                    index,
+                });
+            }
+        }
+    }
+
+    fn push_type(&mut self, ty: EntityType) -> EntityIndex {
+        match ty {
+            EntityType::Function(ty) => {
+                EntityIndex::Function(self.result.module.functions.push(ty))
+            }
+            EntityType::Table(ty) => {
+                let plan = TablePlan::for_table(ty, &self.tunables);
+                EntityIndex::Table(self.result.module.table_plans.push(plan))
+            }
+            EntityType::Memory(ty) => {
+                let plan = MemoryPlan::for_memory(ty, &self.tunables);
+                EntityIndex::Memory(self.result.module.memory_plans.push(plan))
+            }
+            EntityType::Global(ty) => EntityIndex::Global(self.result.module.globals.push(ty)),
+            EntityType::Instance(ty) => {
+                EntityIndex::Instance(self.result.module.instances.push(ty))
+            }
+            EntityType::Module(ty) => EntityIndex::Module(self.result.module.modules.push(ty)),
+            EntityType::Event(_) => unimplemented!(),
+        }
+    }
+
+    fn gen_type_of_module(&mut self, module: &Module) -> ModuleTypeIndex {
+        let imports = module
+            .imports()
+            .map(|(s, field, ty)| {
+                assert!(field.is_none());
+                (s.to_string(), ty)
+            })
+            .collect();
+        let exports = module
+            .exports
+            .iter()
+            .map(|(name, idx)| (name.clone(), module.type_of(*idx)))
+            .collect();
+
+        // FIXME(#2469): this instance/module signature insertion should likely
+        // be deduplicated.
+        let exports = self
+            .types
+            .instance_signatures
+            .push(InstanceSignature { exports });
+        self.types
+            .module_signatures
+            .push(ModuleSignature { imports, exports })
+    }
 }
 
 impl<'data> TargetEnvironment for ModuleEnvironment<'data> {
@@ -267,13 +401,29 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
 
     fn declare_type_module(
         &mut self,
-        imports: &[(&'data str, Option<&'data str>, EntityType)],
+        declared_imports: &[(&'data str, Option<&'data str>, EntityType)],
         exports: &[(&'data str, EntityType)],
     ) -> WasmResult<()> {
-        let imports = imports
-            .iter()
-            .map(|i| (i.0.to_string(), i.1.map(|s| s.to_string()), i.2.clone()))
-            .collect();
+        let mut imports = indexmap::IndexMap::new();
+        let mut instance_types = HashMap::new();
+        for (module, field, ty) in declared_imports {
+            match field {
+                Some(field) => {
+                    let idx = *instance_types
+                        .entry(module)
+                        .or_insert_with(|| self.types.instance_signatures.push(Default::default()));
+                    self.types.instance_signatures[idx]
+                        .exports
+                        .insert(field.to_string(), ty.clone());
+                    if !imports.contains_key(*module) {
+                        imports.insert(module.to_string(), EntityType::Instance(idx));
+                    }
+                }
+                None => {
+                    imports.insert(module.to_string(), ty.clone());
+                }
+            }
+        }
         let exports = exports
             .iter()
             .map(|e| (e.0.to_string(), e.1.clone()))
@@ -344,8 +494,8 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
     fn declare_func_import(
         &mut self,
         index: TypeIndex,
-        module: &str,
-        field: Option<&str>,
+        module: &'data str,
+        field: Option<&'data str>,
     ) -> WasmResult<()> {
         debug_assert_eq!(
             self.result.module.functions.len(),
@@ -353,12 +503,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
             "Imported functions must be declared first"
         );
         let sig_index = self.result.module.types[index].unwrap_function();
-        let func_index = self.result.module.functions.push(sig_index);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Function(func_index),
-        });
+        self.declare_import(module, field, EntityType::Function(sig_index));
         self.result.module.num_imported_funcs += 1;
         self.result.debuginfo.wasm_file.imported_func_count += 1;
         Ok(())
@@ -367,21 +512,15 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
     fn declare_table_import(
         &mut self,
         table: Table,
-        module: &str,
-        field: Option<&str>,
+        module: &'data str,
+        field: Option<&'data str>,
     ) -> WasmResult<()> {
         debug_assert_eq!(
             self.result.module.table_plans.len(),
             self.result.module.num_imported_tables,
             "Imported tables must be declared first"
         );
-        let plan = TablePlan::for_table(table, &self.tunables);
-        let table_index = self.result.module.table_plans.push(plan);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Table(table_index),
-        });
+        self.declare_import(module, field, EntityType::Table(table));
         self.result.module.num_imported_tables += 1;
         Ok(())
     }
@@ -389,8 +528,8 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
     fn declare_memory_import(
         &mut self,
         memory: Memory,
-        module: &str,
-        field: Option<&str>,
+        module: &'data str,
+        field: Option<&'data str>,
     ) -> WasmResult<()> {
         debug_assert_eq!(
             self.result.module.memory_plans.len(),
@@ -400,13 +539,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
         if memory.shared {
             return Err(WasmError::Unsupported("shared memories".to_owned()));
         }
-        let plan = MemoryPlan::for_memory(memory, &self.tunables);
-        let memory_index = self.result.module.memory_plans.push(plan);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Memory(memory_index),
-        });
+        self.declare_import(module, field, EntityType::Memory(memory));
         self.result.module.num_imported_memories += 1;
         Ok(())
     }
@@ -414,20 +547,15 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
     fn declare_global_import(
         &mut self,
         global: Global,
-        module: &str,
-        field: Option<&str>,
+        module: &'data str,
+        field: Option<&'data str>,
     ) -> WasmResult<()> {
         debug_assert_eq!(
             self.result.module.globals.len(),
             self.result.module.num_imported_globals,
             "Imported globals must be declared first"
         );
-        let global_index = self.result.module.globals.push(global);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Global(global_index),
-        });
+        self.declare_import(module, field, EntityType::Global(global));
         self.result.module.num_imported_globals += 1;
         Ok(())
     }
@@ -439,12 +567,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
         field: Option<&'data str>,
     ) -> WasmResult<()> {
         let signature = self.type_to_module_type(ty_index)?;
-        let module_index = self.result.module.modules.push(signature);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Module(module_index),
-        });
+        self.declare_import(module, field, EntityType::Module(signature));
         Ok(())
     }
 
@@ -455,12 +578,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data
         field: Option<&'data str>,
     ) -> WasmResult<()> {
         let signature = self.type_to_instance_type(ty_index)?;
-        let instance_index = self.result.module.instances.push(signature);
-        self.result.module.initializers.push(Initializer::Import {
-            module: module.to_owned(),
-            field: field.map(|s| s.to_owned()),
-            index: EntityIndex::Instance(instance_index),
-        });
+        self.declare_import(module, field, EntityType::Instance(signature));
         Ok(())
     }
 
@@ -755,62 +873,57 @@ and for re-adding support for interface types you can see this issue:
         self.result.module.initializers.reserve(amount as usize);
     }
 
-    fn declare_module(&mut self, ty: TypeIndex) -> WasmResult<()> {
-        // Record the type signature of this module ...
-        let signature = self.type_to_module_type(ty)?;
-        self.result.module.modules.push(signature);
-
-        // ... and then record that in the initialization steps of this module
-        // we're inserting this module into the module index space. At this
-        // point we don't know the final index of the module we're defining, so
-        // we leave a placeholder to get rewritten later.
-        let loc = self.result.module.initializers.len();
-        self.result
-            .module
-            .initializers
-            .push(Initializer::DefineModule(usize::max_value()));
-        self.result.module_initializer_indexes.push(loc);
-        Ok(())
+    fn module_start(&mut self) {
+        // If this is the first time this method is called, nothing to do.
+        if self.first_module {
+            self.first_module = false;
+            return;
         }
-
-    fn module_start(&mut self, index: usize) {
-        // Reset the contents of `self.result` for a new module that's getting
-        // translataed.
-        let mut prev = mem::replace(&mut self.result, ModuleTranslation::default());
-
-        // If this is a nested submodule then we record the final destination of
-        // the child in parent (we store `index` into `prev`) in the appropriate
-        // initialization slot as dicated by `num_modules_defined` (our index of
-        // iteration through the code section).
-        // Record that the `num_modules_defined`-th module is defined at index
-        // by updating the initializer entry.
-        if index > 0 {
-            let initializer_idx = prev.module_initializer_indexes[prev.num_modules_defined];
-            prev.num_modules_defined += 1;
-            debug_assert!(match &prev.module.initializers[initializer_idx] {
-                Initializer::DefineModule(usize::MAX) => true,
-                _ => false,
-            });
-            prev.module.initializers[initializer_idx] = Initializer::DefineModule(index);
-            self.result.module.parent = Some(self.cur);
-        }
-
-        // Update our current index counter and save our parent's translation
-        // where this current translation will end up, which we'll swap back as
-        // part of `module_end`.
-        self.cur = index;
-        assert_eq!(index, self.results.len());
-        self.results.push(prev);
+        // Reset our internal state for a new module by saving the current
+        // module in `results`.
+        let in_progress = mem::replace(&mut self.result, ModuleTranslation::default());
+        self.in_progress.push(in_progress);
         self.modules_to_be -= 1;
     }
 
-    fn module_end(&mut self, index: usize) {
-        assert!(self.result.num_modules_defined == self.result.module_initializer_indexes.len());
+    fn module_end(&mut self) {
+        self.result.creation_artifacts.shrink_to_fit();
+        self.result.creation_modules.shrink_to_fit();
 
-        // Move our finished module into its final location, swapping it with
-        // what was this module's parent.
-        self.cur = self.result.module.parent.unwrap_or(0);
-        mem::swap(&mut self.result, &mut self.results[index]);
+        let (record_initializer, mut done) = match self.in_progress.pop() {
+            Some(m) => (true, mem::replace(&mut self.result, m)),
+            None => (false, mem::take(&mut self.result)),
+        };
+
+        if record_initializer {
+            // Record the type of the module we just finished in our own
+            // module's list of modules.
+            let sig = self.gen_type_of_module(&done.module);
+            self.result.module.modules.push(sig);
+
+            // The root module will store the artifacts for this finished
+            // module at `artifact_index`. This then needs to be inherited by
+            // all later modules coming down to our now-current `self.result`...
+            let mut artifact_index = self.results.len();
+            for result in self.in_progress.iter_mut().chain(Some(&mut self.result)) {
+                result.creation_artifacts.push(artifact_index);
+                artifact_index = result.creation_artifacts.len() - 1;
+            }
+            // ... and then `self.result` needs to create a new module with
+            // whatever was record to save off as its own artifacts/modules.
+            self.result
+                .module
+                .initializers
+                .push(Initializer::CreateModule {
+                    artifact_index,
+                    artifacts: mem::take(&mut done.creation_artifacts),
+                    modules: mem::take(&mut done.creation_modules),
+                });
+        }
+
+        // And the final step is to insert the module into the list of finished
+        // modules to get returned at the end.
+        self.results.push(done);
     }
 
     fn reserve_instances(&mut self, amt: u32) {
@@ -818,7 +931,12 @@ and for re-adding support for interface types you can see this issue:
         self.result.module.initializers.reserve(amt as usize);
     }
 
-    fn declare_instance(&mut self, module: ModuleIndex, args: Vec<EntityIndex>) -> WasmResult<()> {
+    fn declare_instance(
+        &mut self,
+        module: ModuleIndex,
+        args: Vec<(&'data str, EntityIndex)>,
+    ) -> WasmResult<()> {
+        let args = args.into_iter().map(|(s, i)| (s.to_string(), i)).collect();
         // Record the type of this instance with the type signature of the
         // module we're instantiating and then also add an initializer which
         // records that we'll be adding to the instance index space here.
@@ -839,29 +957,60 @@ and for re-adding support for interface types you can see this issue:
             //
             // Note that we don't add an initializer for this alias because
             // we statically know where all types point to.
-            Alias::ParentType(parent_idx) => {
-                let ty = self.results[self.cur].module.types[parent_idx];
+            Alias::OuterType {
+                relative_depth,
+                index,
+            } => {
+                let module_idx = self.in_progress.len() - 1 - (relative_depth as usize);
+                let ty = self.in_progress[module_idx].module.types[index];
                 self.result.module.types.push(ty);
             }
 
-            // This is similar to types in that it's easy for us to record the
-            // type of the module that's being aliased, but we also need to add
-            // an initializer so during instantiation we can prepare the index
-            // space appropriately.
-            Alias::ParentModule(parent_idx) => {
-                let module_idx = self.results[self.cur].module.modules[parent_idx];
-                self.result.module.modules.push(module_idx);
+            // Modules are a bit trickier since we need to record how to track
+            // the state from the original module down to our own.
+            Alias::OuterModule {
+                relative_depth,
+                index,
+            } => {
+                // First we can copy the type from the parent module into our
+                // own module to record what type our module definition will
+                // have.
+                let module_idx = self.in_progress.len() - 1 - (relative_depth as usize);
+                let module_ty = self.in_progress[module_idx].module.modules[index];
+                self.result.module.modules.push(module_ty);
+
+                // Next we'll be injecting a module value that is closed over,
+                // and that will be used to define the module into the index
+                // space. Record an initializer about where our module is
+                // sourced from (which will be stored within each module value
+                // itself).
+                let module_index = self.result.creation_modules.len();
                 self.result
                     .module
                     .initializers
-                    .push(Initializer::AliasParentModule(parent_idx));
+                    .push(Initializer::DefineModule(module_index));
+
+                // And finally we need to record a breadcrumb trail of how to
+                // get the module value into `module_index`. The module just
+                // after our destination module will use a `ModuleIndex` to
+                // fetch the module value, and everything else inbetween will
+                // inherit that module's closed-over value.
+                let mut upvar = ModuleUpvar::Local(index);
+                for outer in self.in_progress[module_idx + 1..].iter_mut() {
+                    let upvar = mem::replace(
+                        &mut upvar,
+                        ModuleUpvar::Inherit(outer.creation_modules.len()),
+                    );
+                    outer.creation_modules.push(upvar);
+                }
+                self.result.creation_modules.push(upvar);
             }
 
             // This case is slightly more involved, we'll be recording all the
             // type information for each kind of entity, and then we also need
             // to record an initialization step to get the export from the
             // instance.
-            Alias::Child { instance, export } => {
+            Alias::InstanceExport { instance, export } => {
                 let ty = self.result.module.instances[instance];
                 match &self.types.instance_signatures[ty].exports[export] {
                     EntityType::Global(g) => {
@@ -894,7 +1043,10 @@ and for re-adding support for interface types you can see this issue:
                 self.result
                     .module
                     .initializers
-                    .push(Initializer::AliasInstanceExport { instance, export })
+                    .push(Initializer::AliasInstanceExport {
+                        instance,
+                        export: export.to_string(),
+                    })
             }
         }
 

crates/environ/src/tunables.rs

@@ -23,6 +23,10 @@ pub struct Tunables {
     /// calls and interrupts are implemented through the `VMInterrupts`
     /// structure, or `InterruptHandle` in the `wasmtime` crate.
     pub interruptable: bool,
+
+    /// Whether or not fuel is enabled for generated code, meaning that fuel
+    /// will be consumed every time a wasm instruction is executed.
+    pub consume_fuel: bool,
 }
 
 impl Default for Tunables {
@@ -57,6 +61,7 @@ impl Default for Tunables {
             generate_native_debuginfo: false,
             parse_wasm_debuginfo: true,
             interruptable: false,
+            consume_fuel: false,
         }
     }
 }

crates/environ/src/vmoffsets.rs

@@ -258,6 +258,11 @@ impl VMOffsets {
     pub fn vminterrupts_stack_limit(&self) -> u8 {
         0
     }
+
+    /// Return the offset of the `fuel_consumed` field of `VMInterrupts`
+    pub fn vminterrupts_fuel_consumed(&self) -> u8 {
+        self.pointer_size
+    }
 }
 
 /// Offsets for `VMCallerCheckedAnyfunc`.

crates/fuzzing/Cargo.toml

@@ -13,12 +13,12 @@ arbitrary = { version = "0.4.1", features = ["derive"] }
 env_logger = "0.8.1"
 log = "0.4.8"
 rayon = "1.2.1"
-wasmparser = "0.71"
-wasmprinter = "0.2.17"
+wasmparser = "0.73"
+wasmprinter = "0.2.20"
 wasmtime = { path = "../wasmtime" }
 wasmtime-wast = { path = "../wast" }
-wasm-encoder = "0.2"
-wasm-smith = "0.3.0"
+wasm-encoder = "0.4"
+wasm-smith = "0.3.1"
 wasmi = "0.7.0"
 
 [dev-dependencies]

crates/fuzzing/src/generators.rs

@@ -64,6 +64,8 @@ pub struct Config {
     debug_info: bool,
     canonicalize_nans: bool,
     interruptable: bool,
+    #[allow(missing_docs)]
+    pub consume_fuel: bool,
 
     // Note that we use 32-bit values here to avoid blowing the 64-bit address
     // space by requesting ungodly-large sizes/guards.
@@ -75,14 +77,15 @@ pub struct Config {
 impl Config {
     /// Converts this to a `wasmtime::Config` object
     pub fn to_wasmtime(&self) -> wasmtime::Config {
-        let mut cfg = wasmtime::Config::new();
+        let mut cfg = crate::fuzz_default_config(wasmtime::Strategy::Auto).unwrap();
         cfg.debug_info(self.debug_info)
             .static_memory_maximum_size(self.static_memory_maximum_size.unwrap_or(0).into())
             .static_memory_guard_size(self.static_memory_guard_size.unwrap_or(0).into())
             .dynamic_memory_guard_size(self.dynamic_memory_guard_size.unwrap_or(0).into())
             .cranelift_nan_canonicalization(self.canonicalize_nans)
             .cranelift_opt_level(self.opt_level.to_wasmtime())
-            .interruptable(self.interruptable);
+            .interruptable(self.interruptable)
+            .consume_fuel(self.consume_fuel);
         return cfg;
     }
 }