From 32f162aa7811b29a6fc3c20c402fb58e329aa889 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 12 Jan 2021 09:51:09 -0800 Subject: [PATCH 01/55] fix windows flags --- crates/wasi-common/src/sys/windows/fd.rs | 2 +- crates/wasi-common/src/sys/windows/mod.rs | 12 ++++++------ crates/wasi-common/src/sys/windows/path.rs | 18 +++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/wasi-common/src/sys/windows/fd.rs b/crates/wasi-common/src/sys/windows/fd.rs index 4e3e16021c..09659ac22e 100644 --- a/crates/wasi-common/src/sys/windows/fd.rs +++ b/crates/wasi-common/src/sys/windows/fd.rs @@ -86,7 +86,7 @@ fn file_access_mode_from_fdflags(fdflags: Fdflags, read: bool, write: bool) -> A // For append, grant the handle FILE_APPEND_DATA access but *not* FILE_WRITE_DATA. // This makes the handle "append only". // Changes to the file pointer will be ignored (like POSIX's O_APPEND behavior). - if fdflags.contains(&Fdflags::APPEND) { + if fdflags.contains(Fdflags::APPEND) { access_mode.insert(AccessMode::FILE_APPEND_DATA); access_mode.remove(AccessMode::FILE_WRITE_DATA); } diff --git a/crates/wasi-common/src/sys/windows/mod.rs b/crates/wasi-common/src/sys/windows/mod.rs index bd31ea89da..500c16dbfa 100644 --- a/crates/wasi-common/src/sys/windows/mod.rs +++ b/crates/wasi-common/src/sys/windows/mod.rs @@ -147,13 +147,13 @@ impl TryFrom<&File> for Filestat { impl From for CreationDisposition { fn from(oflags: Oflags) -> Self { - if oflags.contains(&Oflags::CREAT) { - if oflags.contains(&Oflags::EXCL) { + if oflags.contains(Oflags::CREAT) { + if oflags.contains(Oflags::EXCL) { CreationDisposition::CREATE_NEW } else { CreationDisposition::CREATE_ALWAYS } - } else if oflags.contains(&Oflags::TRUNC) { + } else if oflags.contains(Oflags::TRUNC) { CreationDisposition::TRUNCATE_EXISTING } else { CreationDisposition::OPEN_EXISTING @@ -171,9 +171,9 @@ impl From for Flags { // treat I/O operations on files as synchronous. WASI might have an async-io API in the future. // Technically, Windows only supports __WASI_FDFLAGS_SYNC, but treat all the flags as the same. - if fdflags.contains(&Fdflags::DSYNC) - || fdflags.contains(&Fdflags::RSYNC) - || fdflags.contains(&Fdflags::SYNC) + if fdflags.contains(Fdflags::DSYNC) + || fdflags.contains(Fdflags::RSYNC) + || fdflags.contains(Fdflags::SYNC) { flags.insert(Flags::FILE_FLAG_WRITE_THROUGH); } diff --git a/crates/wasi-common/src/sys/windows/path.rs b/crates/wasi-common/src/sys/windows/path.rs index 70b9e9b42f..088e59c86f 100644 --- a/crates/wasi-common/src/sys/windows/path.rs +++ b/crates/wasi-common/src/sys/windows/path.rs @@ -72,7 +72,7 @@ fn file_access_mode_from_fdflags(fdflags: Fdflags, read: bool, write: bool) -> A // For append, grant the handle FILE_APPEND_DATA access but *not* FILE_WRITE_DATA. // This makes the handle "append only". // Changes to the file pointer will be ignored (like POSIX's O_APPEND behavior). - if fdflags.contains(&Fdflags::APPEND) { + if fdflags.contains(Fdflags::APPEND) { access_mode.insert(AccessMode::FILE_APPEND_DATA); access_mode.remove(AccessMode::FILE_WRITE_DATA); } @@ -100,16 +100,16 @@ pub(crate) fn open_rights( let mut needed_inheriting = input_rights.base | input_rights.inheriting; // convert open flags - if oflags.contains(&Oflags::CREAT) { + if oflags.contains(Oflags::CREAT) { needed_base |= Rights::PATH_CREATE_FILE; - } else if oflags.contains(&Oflags::TRUNC) { + } else if oflags.contains(Oflags::TRUNC) { needed_base |= Rights::PATH_FILESTAT_SET_SIZE; } // convert file descriptor flags - if fdflags.contains(&Fdflags::DSYNC) - || fdflags.contains(&Fdflags::RSYNC) - || fdflags.contains(&Fdflags::SYNC) + if fdflags.contains(Fdflags::DSYNC) + || fdflags.contains(Fdflags::RSYNC) + || fdflags.contains(Fdflags::SYNC) { needed_inheriting |= Rights::FD_DATASYNC; needed_inheriting |= Rights::FD_SYNC; @@ -211,13 +211,13 @@ pub(crate) fn open( ) -> Result> { use winx::file::{AccessMode, CreationDisposition, Flags}; - let is_trunc = oflags.contains(&Oflags::TRUNC); + let is_trunc = oflags.contains(Oflags::TRUNC); if is_trunc { // Windows does not support append mode when opening for truncation // This is because truncation requires `GENERIC_WRITE` access, which will override the removal // of the `FILE_WRITE_DATA` permission. - if fdflags.contains(&Fdflags::APPEND) { + if fdflags.contains(Fdflags::APPEND) { return Err(Error::Notsup); } } @@ -246,7 +246,7 @@ pub(crate) fn open( return Err(Error::Loop); } // check if we are trying to open a file as a dir - if file_type.is_file() && oflags.contains(&Oflags::DIRECTORY) { + if file_type.is_file() && oflags.contains(Oflags::DIRECTORY) { return Err(Error::Notdir); } } From bc6dc083f0311e2fc3d1cfeccc7331e3cf9a43fd Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 12 Jan 2021 14:02:44 -0800 Subject: [PATCH 02/55] wasmtime-bench-api: Randomize the locations of heap objects This helps us avoid measurement bias due to accidental locality of unrelated heap objects. See *Stabilizer: Statistically Sound Performance Evaluation* by Curtsinger and Berger for details (although Stabilizer deals with much more than just the location of heap allocations): https://people.cs.umass.edu/~emery/pubs/stabilizer-asplos13.pdf --- Cargo.lock | 13 +++++++++++++ crates/bench-api/Cargo.toml | 5 ++++- crates/bench-api/src/lib.rs | 8 ++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c220f496b0..084a2cb0a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1964,6 +1964,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +[[package]] +name = "shuffling-allocator" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "848c0a454373d16ebfaa740c99d4faebe25ea752a35e0c6e341168fe67f987f8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand", + "winapi", +] + [[package]] name = "smallvec" version = "1.6.1" @@ -2430,6 +2442,7 @@ name = "wasmtime-bench-api" version = "0.19.0" dependencies = [ "anyhow", + "shuffling-allocator", "wasi-common", "wasmtime", "wasmtime-wasi", diff --git a/crates/bench-api/Cargo.toml b/crates/bench-api/Cargo.toml index 02dfd986be..c5c70bf6df 100644 --- a/crates/bench-api/Cargo.toml +++ b/crates/bench-api/Cargo.toml @@ -16,10 +16,13 @@ crate-type = ["rlib", "cdylib"] [dependencies] anyhow = "1.0" +shuffling-allocator = { version = "1.1.1", optional = true } wasmtime = { path = "../wasmtime", default-features = false } wasmtime-wasi = { path = "../wasi" } wasi-common = { path = "../wasi-common" } - [dev-dependencies] wat = "1.0" + +[features] +default = ["shuffling-allocator"] diff --git a/crates/bench-api/src/lib.rs b/crates/bench-api/src/lib.rs index a2958f09a8..8daf1197d8 100644 --- a/crates/bench-api/src/lib.rs +++ b/crates/bench-api/src/lib.rs @@ -83,6 +83,14 @@ pub type ExitCode = c_int; pub const OK: ExitCode = 0; pub const ERR: ExitCode = -1; +// Randomize the location of heap objects to avoid accidental locality being an +// uncontrolled variable that obscures performance evaluation in our +// experiments. +#[cfg(feature = "shuffling-allocator")] +#[global_allocator] +static ALLOC: shuffling_allocator::ShufflingAllocator = + shuffling_allocator::wrap!(&std::alloc::System); + /// Exposes a C-compatible way of creating the engine from the bytes of a single /// Wasm module. /// From 4638de673cf43028d22fc6ca58c7867f887704d7 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Tue, 12 Jan 2021 15:37:53 -0800 Subject: [PATCH 03/55] x64 bugfix: prevent load-op fusion of cmp because it could be emitted multiple times. On x64, the new backend generates `cmp` instructions at their use-sites when possible (when the icmp that generates a boolean is known) so that the condition flows directly through flags rather than a materialized boolean. E.g., both `bint` (boolean to int) and `select` (conditional select) instruction lowerings invoke `emit_cmp()` to do so. Load-op fusion in `emit_cmp()` nominally allowed `cmp` to use its `cmp reg, mem` form. However, the mergeable-load condition (load has only single use) was not adequately checked. Consider the sequence: ``` v2 = load.i64 v1 v3 = icmp eq v0, v2 v4 = bint.i64 v3 v5 = select.i64 v3, v0, v1 ``` The load `v2` is only used in the `icmp` at `v3`. However, the cmp will be separately codegen'd twice, once for the `bint` and once for the `select`. Prior to this fix, the above example would result in the load at `v2` sinking to the `cmp` just above the `select`; we then emit another `cmp` for the `bint`, but the load has already been used once so we do not allow merging. We thus (i) expect the register for `v2` to contain the loaded value, but (ii) skip the codegen for the load because it has been sunk. This results in a regalloc error (unexpected livein) as the unfilled register is upward-exposed to the entry point. Because of this, we need to accept only the reg, reg form in `emit_cmp()` (and the FP equivalent). We could get marginally better code by tracking whether the `cmp` we are emitting comes from an `icmp`/`fcmp` with only one use; but IMHO simplicity is a better rule here when subtle interactions occur. --- cranelift/codegen/src/isa/x64/lower.rs | 17 +++++-- .../filetests/isa/x64/cmp-mem-bug.clif | 49 +++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 0b9784d59d..1712d5d172 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -380,11 +380,18 @@ fn emit_cmp>(ctx: &mut C, insn: IRInst) { // TODO Try to commute the operands (and invert the condition) if one is an immediate. let lhs = put_input_in_reg(ctx, inputs[0]); - let rhs = input_to_reg_mem_imm(ctx, inputs[1]); + // We force the RHS into a register, and disallow load-op fusion, because we + // do not have a transitive guarantee that this cmp-site will be the sole + // user of the value. Consider: the icmp might be the only user of a load, + // but there may be multiple users of the icmp (e.g. select or bint + // instructions) that each invoke `emit_cmp()`. If we were to allow a load + // to sink to the *latest* one, but other sites did not permit sinking, then + // we would be missing the load for other cmp-sites. + let rhs = put_input_in_reg(ctx, inputs[1]); // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives // us dst - src at the machine instruction level, so invert operands. - ctx.emit(Inst::cmp_rmi_r(ty.bytes() as u8, rhs, lhs)); + ctx.emit(Inst::cmp_rmi_r(ty.bytes() as u8, RegMemImm::reg(rhs), lhs)); } /// A specification for a fcmp emission. @@ -465,8 +472,10 @@ fn emit_fcmp>( (inputs[0], inputs[1]) }; let lhs = put_input_in_reg(ctx, lhs_input); - let rhs = input_to_reg_mem(ctx, rhs_input); - ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs)); + // See above in `emit_cmp()`. We must only use the reg/reg form of the + // comparison in order to avoid issues with merged loads. + let rhs = put_input_in_reg(ctx, rhs_input); + ctx.emit(Inst::xmm_cmp_rm_r(op, RegMem::reg(rhs), lhs)); let cond_result = match cond_code { FloatCC::Equal => FcmpCondResult::AndConditions(CC::NP, CC::Z), diff --git a/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif b/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif new file mode 100644 index 0000000000..9d05e04b04 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/cmp-mem-bug.clif @@ -0,0 +1,49 @@ +test compile +target x86_64 +feature "experimental_x64" + +function %f0(i64, i64) -> i64, i64 { +block0(v0: i64, v1: i64): + v2 = load.i64 v1 +; check: movq 0(%rsi), %rax + + v3 = icmp eq v0, v2 + + v4 = bint.i64 v3 +; nextln: cmpq %rax, %rdi +; nextln: setz %cl +; nextln: movzbq %cl, %rcx + + v5 = select.i64 v3, v0, v1 +; nextln: cmpq %rax, %rdi +; nextln: cmovzq %rdi, %rsi + + return v4, v5 +; nextln: movq %rcx, %rax +; nextln: movq %rsi, %rdx +} + +function %f1(f64, i64) -> i64, f64 { +block0(v0: f64, v1: i64): + v2 = load.f64 v1 +; check: movsd 0(%rdi), %xmm1 + + v3 = fcmp eq v0, v2 + + v4 = bint.i64 v3 +; nextln: ucomisd %xmm1, %xmm0 +; nextln: setnp %dil +; nextln: setz %sil +; nextln: andl %edi, %esi +; nextln: movzbq %sil, %rsi + + v5 = select.f64 v3, v0, v0 +; nextln: ucomisd %xmm1, %xmm0 +; nextln: movaps %xmm0, %xmm1 +; nextln: jnp $$next; movsd %xmm0, %xmm1; $$next: +; nextln: jz $$next; movsd %xmm0, %xmm1; $$next: + + return v4, v5 +; nextln: movq %rsi, %rax +; nextln: movaps %xmm1, %xmm0 +} From d17815a239545b030aa1b97220d38bcad7aae437 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Wed, 13 Jan 2021 16:50:28 -0800 Subject: [PATCH 04/55] Zero newly allocated registers whose immediate use depends on content not being NaN An intermittent failure during SIMD spectests is described in #2432. This patch corrects code written in a way that assumes comparing fp equality of a register with itself will always return true. This is not true when the register value is NaN as NaN. In this case, and with all ordered comparisons involving NaN, the comparisons will always return false. This patch corrects that assumption for SIMD Fabs and Fneg which seem to be the only instructions generating the failure with #2432. --- cranelift/codegen/src/isa/x64/lower.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 1712d5d172..9293221de5 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -3138,8 +3138,11 @@ fn lower_insn_to_regs>( ctx.emit(Inst::gen_move(dst, src, output_ty)); // Generate an all 1s constant in an XMM register. This uses CMPPS but could - // have used CMPPD with the same effect. + // have used CMPPD with the same effect. Note, we zero the temp we allocate + // because if not, there is a chance that the register we use could be initialized + // with NaN .. in which case the CMPPS would fail since NaN != NaN. let tmp = ctx.alloc_tmp(output_ty).only_reg().unwrap(); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Xorps, RegMem::from(tmp), tmp)); let cond = FcmpImm::from(FloatCC::Equal); let cmpps = Inst::xmm_rm_r_imm( SseOpcode::Cmpps, From cde07b9a79aaab486715e2b3ed966b34a2d605f3 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Wed, 13 Jan 2021 18:01:42 -0800 Subject: [PATCH 05/55] Re-enable spec tests that were disabled for #2432 #2470. Enable new tests Re-enables spec tests that were turned off for #2432 and #2470 while also enabling tests that now work due to patch pushes in the interim. Currently all SIMD spec tests past. Testing to assure this is ok to enable hasn't been super intense so we should monitor but there was an attempt of doing 1000 runs 3 different times to try and reproduce the issue and it did not occur. In the past would have occurred several times with that many runs. --- build.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/build.rs b/build.rs index 35257cdb7b..fe3bf4456b 100644 --- a/build.rs +++ b/build.rs @@ -176,10 +176,52 @@ fn write_testsuite_tests( /// For experimental_x64 backend features that are not supported yet, mark tests as panicking, so /// they stop "passing" once the features are properly implemented. -/// -/// TODO(#2470): removed all tests from this set as we are disabling x64 SIMD tests unconditionally -/// instead until we resolve a nondeterminism bug. Restore when fixed. -fn experimental_x64_should_panic(_testsuite: &str, _testname: &str, _strategy: &str) -> bool { +fn experimental_x64_should_panic(testsuite: &str, testname: &str, strategy: &str) -> bool { + if !cfg!(feature = "experimental_x64") || strategy != "Cranelift" { + return false; + } + + match (testsuite, testname) { + ("simd", "simd_address") => return false, + ("simd", "simd_align") => return false, + ("simd", "simd_bitwise") => return false, + ("simd", "simd_bit_shift") => return false, + ("simd", "simd_boolean") => return false, + ("simd", "simd_const") => return false, + ("simd", "simd_i8x16_arith") => return false, + ("simd", "simd_i8x16_arith2") => return false, + ("simd", "simd_i8x16_cmp") => return false, + ("simd", "simd_i8x16_sat_arith") => return false, + ("simd", "simd_i16x8_arith") => return false, + ("simd", "simd_i16x8_arith2") => return false, + ("simd", "simd_i16x8_cmp") => return false, + ("simd", "simd_i16x8_sat_arith") => return false, + ("simd", "simd_i32x4_arith") => return false, + ("simd", "simd_i32x4_arith2") => return false, + ("simd", "simd_i32x4_cmp") => return false, + ("simd", "simd_i32x4_dot_i16x8") => return false, + ("simd", "simd_i64x2_arith") => return false, + ("simd", "simd_f32x4") => return false, + ("simd", "simd_f32x4_arith") => return false, + ("simd", "simd_f32x4_cmp") => return false, + ("simd", "simd_f32x4_pmin_pmax") => return false, + ("simd", "simd_f64x2") => return false, + ("simd", "simd_f64x2_arith") => return false, + ("simd", "simd_f64x2_cmp") => return false, + ("simd", "simd_f64x2_pmin_pmax") => return false, + ("simd", "simd_lane") => return false, + ("simd", "simd_load") => return false, + ("simd", "simd_load_extend") => return false, + ("simd", "simd_load_splat") => return false, + ("simd", "simd_load_zero") => return false, + ("simd", "simd_splat") => return false, + ("simd", "simd_store") => return false, + ("simd", "simd_conversions") => return false, + ("simd", "simd_f32x4_rounding") => return false, + ("simd", "simd_f64x2_rounding") => return false, + ("simd", _) => return true, + _ => {} + } false } @@ -201,11 +243,6 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "x86_64"; } - // Ignore all x64 SIMD tests for now (#2470). - ("simd", _) if cfg!(feature = "experimental_x64") => { - return env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "x86_64"; - } - // These are only implemented on aarch64 and x64. ("simd", "simd_boolean") | ("simd", "simd_f32x4_pmin_pmax") From f94db6556c3be07ee01ca9b9c4464d8c82ffc01e Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 14 Jan 2021 07:36:12 -0800 Subject: [PATCH 06/55] Update WebAssembly C API submodule to latest commit. (#2579) * Update WebAssembly C API submodule to latest commit. This commit updates the WebAssembly C API submodule (for `wasm.h`) to the latest commit out of master. This fixes the behavior of `wasm_name_new_from_string` such that it no longer copies the null character into the name, which caused unexpected failures when using the Wasmtime linker as imports wouldn't resolve when the null was present. Along with this change were breaking changes to `wasm_func_call`, the host callback signatures, and `wasm_instance_new` to take a vector type instead of a pointer to an unsized array. As a result, Wasmtime language bindings based on the C API will need to be updated once this change is pulled in. Fixes #2211. Fixes #2131. * Update Doxygen comments for wasm.h changes. --- crates/c-api/include/doc-wasm.h | 54 +++++++++++++++++++++++------- crates/c-api/include/wasmtime.h | 15 +++------ crates/c-api/src/func.rs | 57 +++++++++++++++---------------- crates/c-api/src/instance.rs | 25 ++++++-------- crates/c-api/src/vec.rs | 13 ++++++++ crates/c-api/wasm-c-api | 2 +- examples/externref.c | 11 +++--- examples/fib-debug/main.c | 9 +++-- examples/gcd.c | 13 ++++---- examples/hello.c | 11 +++--- examples/hello.cc | 11 +++--- examples/interrupt.c | 8 +++-- examples/linking.c | 4 ++- examples/memory.c | 59 +++++++++++++++++---------------- examples/multi.c | 23 ++++++------- examples/serialize.c | 11 +++--- examples/threads.c | 15 +++++---- examples/wasi-fs/main.c | 5 ++- examples/wasi/main.c | 5 ++- 19 files changed, 207 insertions(+), 144 deletions(-) diff --git a/crates/c-api/include/doc-wasm.h b/crates/c-api/include/doc-wasm.h index 73d57a647b..356db1e1bc 100644 --- a/crates/c-api/include/doc-wasm.h +++ b/crates/c-api/include/doc-wasm.h @@ -316,6 +316,12 @@ * * \fn wasm_name_new_new_uninitialized * \brief Convenience alias + * + * \fn wasm_name_new_from_string + * \brief Create a new name from a C string. + * + * \fn wasm_name_new_from_string_nt + * \brief Create a new name from a C string with null terminator. * * \fn wasm_name_copy * \brief Convenience alias @@ -407,7 +413,7 @@ * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_valtype_t* wasm_valtype_copy(wasm_valtype_t *) + * \fn own wasm_valtype_t* wasm_valtype_copy(const wasm_valtype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. @@ -477,7 +483,7 @@ * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_functype_t* wasm_functype_copy(wasm_functype_t *) + * \fn own wasm_functype_t* wasm_functype_copy(const wasm_functype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. @@ -548,7 +554,7 @@ * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_globaltype_t* wasm_globaltype_copy(wasm_globaltype_t *) + * \fn own wasm_globaltype_t* wasm_globaltype_copy(const wasm_globaltype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. @@ -625,7 +631,7 @@ * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_tabletype_t* wasm_tabletype_copy(wasm_tabletype_t *) + * \fn own wasm_tabletype_t* wasm_tabletype_copy(const wasm_tabletype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. @@ -711,7 +717,7 @@ * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_memorytype_t* wasm_memorytype_copy(wasm_memorytype_t *) + * \fn own wasm_memorytype_t* wasm_memorytype_copy(const wasm_memorytype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. @@ -780,7 +786,7 @@ * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_externtype_t* wasm_externtype_copy(wasm_externtype_t *) + * \fn own wasm_externtype_t* wasm_externtype_copy(const wasm_externtype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. @@ -957,7 +963,7 @@ * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_importtype_t* wasm_importtype_copy(wasm_importtype_t *) + * \fn own wasm_importtype_t* wasm_importtype_copy(const wasm_importtype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. @@ -1038,7 +1044,7 @@ * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_exporttype_t* wasm_exporttype_copy(wasm_exporttype_t *) + * \fn own wasm_exporttype_t* wasm_exporttype_copy(const wasm_exporttype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. @@ -1520,8 +1526,6 @@ * and types of results as the original type signature. It is undefined behavior * to return other types or different numbers of values. * - * This function takes ownership of all of the parameters given. It's expected - * that the caller will invoke `wasm_val_delete` for each one provided. * Ownership of the results and the trap returned, if any, is passed to the * caller of this function. * @@ -1608,7 +1612,7 @@ * \fn size_t wasm_func_result_arity(const wasm_func_t *); * \brief Returns the number of results returned by this function. * - * \fn own wasm_trap_t *wasm_func_call(const wasm_func_t *, const wasm_val_t args[], const wasm_val_t results[]); +* \fn own wasm_trap_t *wasm_func_call(const wasm_func_t *, const wasm_val_vec_t *args, wasm_val_vec_t *results); * \brief Calls the provided function with the arguments given. * * This function is used to call WebAssembly from the host. The parameter array @@ -2164,7 +2168,7 @@ * \fn wasm_ref_as_instance_const(const wasm_ref_t *); * \brief Unimplemented in Wasmtime, aborts the process if called. * - * \fn own wasm_instance_t *wasm_instance_new(wasm_store_t *, const wasm_module_t *, const wasm_extern_t *const[], wasm_trap_t **); + * \fn own wasm_instance_t *wasm_instance_new(wasm_store_t *, const wasm_module_t *, const wasm_extern_vec_t *, wasm_trap_t **); * \brief Instantiates a module with the provided imports. * * This function will instantiate the provided #wasm_module_t into the provided @@ -2194,3 +2198,29 @@ * the same length as #wasm_module_exports called on the original module. Each * element is 1:1 matched with the elements in the list of #wasm_module_exports. */ + +/** + * \def WASM_EMPTY_VEC + * \brief Used to initialize an empty vector type. + * + * \def WASM_ARRAY_VEC + * \brief Used to initialize a vector type from a C array. + * + * \def WASM_I32_VAL + * \brief Used to initialize a 32-bit integer wasm_val_t value. + * + * \def WASM_I64_VAL + * \brief Used to initialize a 64-bit integer wasm_val_t value. + * + * \def WASM_F32_VAL + * \brief Used to initialize a 32-bit floating point wasm_val_t value. + * + * \def WASM_F64_VAL + * \brief Used to initialize a 64-bit floating point wasm_val_t value. + * + * \def WASM_REF_VAL + * \brief Used to initialize an externref wasm_val_t value. + * + * \def WASM_INIT_VAL + * \brief Used to initialize a null externref wasm_val_t value. + */ diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h index 769cfc1268..b2efa3e81c 100644 --- a/crates/c-api/include/wasmtime.h +++ b/crates/c-api/include/wasmtime.h @@ -671,7 +671,6 @@ WASM_API_EXTERN const wasm_name_t *wasmtime_frame_module_name(const wasm_frame_t * * This function is similar to #wasm_func_call, but with a few tweaks: * - * * `args` and `results` have a size parameter saying how big the arrays are * * An error *and* a trap can be returned * * Errors are returned if `args` have the wrong types, if the args/results * arrays have the wrong lengths, or if values come from the wrong store. @@ -697,10 +696,8 @@ WASM_API_EXTERN const wasm_name_t *wasmtime_frame_module_name(const wasm_frame_t */ WASM_API_EXTERN own wasmtime_error_t *wasmtime_func_call( wasm_func_t *func, - const wasm_val_t *args, - size_t num_args, - wasm_val_t *results, - size_t num_results, + const wasm_val_vec_t *args, + wasm_val_vec_t *results, own wasm_trap_t **trap ); @@ -741,7 +738,6 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_global_set( * This function is similar to #wasm_instance_new, but with a few tweaks: * * * An error message can be returned from this function. - * * The number of imports specified is passed as an argument * * The `trap` pointer is required to not be NULL. * * The states of return values from this function are similar to @@ -759,8 +755,7 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_global_set( WASM_API_EXTERN own wasmtime_error_t *wasmtime_instance_new( wasm_store_t *store, const wasm_module_t *module, - const wasm_extern_t* const imports[], - size_t num_imports, + const wasm_extern_vec_t* imports, own wasm_instance_t **instance, own wasm_trap_t **trap ); @@ -1016,7 +1011,7 @@ WASM_API_EXTERN own wasmtime_error_t *wasmtime_module_deserialize( * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_instancetype_t* wasm_instancetype_copy(wasm_instancetype_t *) + * \fn own wasm_instancetype_t* wasm_instancetype_copy(const wasm_instancetype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. @@ -1113,7 +1108,7 @@ WASM_API_EXTERN const wasm_instancetype_t* wasm_externtype_as_instancetype_const * * See #wasm_byte_vec_delete for more information. * - * \fn own wasm_moduletype_t* wasm_moduletype_copy(wasm_moduletype_t *) + * \fn own wasm_moduletype_t* wasm_moduletype_copy(const wasm_moduletype_t *) * \brief Creates a new value which matches the provided one. * * The caller is responsible for deleting the returned value. diff --git a/crates/c-api/src/func.rs b/crates/c-api/src/func.rs index 5c63e9782f..87351e99ca 100644 --- a/crates/c-api/src/func.rs +++ b/crates/c-api/src/func.rs @@ -1,4 +1,4 @@ -use crate::{wasm_extern_t, wasm_functype_t, wasm_store_t, wasm_val_t}; +use crate::{wasm_extern_t, wasm_functype_t, wasm_store_t, wasm_val_t, wasm_val_vec_t}; use crate::{wasm_name_t, wasm_trap_t, wasmtime_error_t}; use anyhow::anyhow; use std::ffi::c_void; @@ -21,26 +21,28 @@ pub struct wasmtime_caller_t<'a> { caller: Caller<'a>, } -pub type wasm_func_callback_t = - extern "C" fn(args: *const wasm_val_t, results: *mut wasm_val_t) -> Option>; +pub type wasm_func_callback_t = extern "C" fn( + args: *const wasm_val_vec_t, + results: *mut wasm_val_vec_t, +) -> Option>; pub type wasm_func_callback_with_env_t = extern "C" fn( env: *mut std::ffi::c_void, - args: *const wasm_val_t, - results: *mut wasm_val_t, + args: *const wasm_val_vec_t, + results: *mut wasm_val_vec_t, ) -> Option>; pub type wasmtime_func_callback_t = extern "C" fn( caller: *const wasmtime_caller_t, - args: *const wasm_val_t, - results: *mut wasm_val_t, + args: *const wasm_val_vec_t, + results: *mut wasm_val_vec_t, ) -> Option>; pub type wasmtime_func_callback_with_env_t = extern "C" fn( caller: *const wasmtime_caller_t, env: *mut std::ffi::c_void, - args: *const wasm_val_t, - results: *mut wasm_val_t, + args: *const wasm_val_vec_t, + results: *mut wasm_val_vec_t, ) -> Option>; struct Finalizer { @@ -83,21 +85,25 @@ impl From for wasm_func_t { fn create_function( store: &wasm_store_t, ty: &wasm_functype_t, - func: impl Fn(Caller<'_>, *const wasm_val_t, *mut wasm_val_t) -> Option> + 'static, + func: impl Fn(Caller<'_>, *const wasm_val_vec_t, *mut wasm_val_vec_t) -> Option> + + 'static, ) -> Box { let store = &store.store; let ty = ty.ty().ty.clone(); let func = Func::new(store, ty, move |caller, params, results| { - let params = params + let params: wasm_val_vec_t = params .iter() .cloned() .map(|p| wasm_val_t::from_val(p)) - .collect::>(); - let mut out_results = vec![wasm_val_t::default(); results.len()]; - let out = func(caller, params.as_ptr(), out_results.as_mut_ptr()); + .collect::>() + .into(); + let mut out_results: wasm_val_vec_t = vec![wasm_val_t::default(); results.len()].into(); + let out = func(caller, ¶ms, &mut out_results); if let Some(trap) = out { return Err(trap.trap.clone()); } + + let out_results = out_results.as_slice(); for i in 0..results.len() { results[i] = out_results[i].val(); } @@ -164,17 +170,14 @@ pub extern "C" fn wasmtime_func_new_with_env( #[no_mangle] pub unsafe extern "C" fn wasm_func_call( wasm_func: &wasm_func_t, - args: *const wasm_val_t, - results: *mut MaybeUninit, + args: *const wasm_val_vec_t, + results: *mut wasm_val_vec_t, ) -> *mut wasm_trap_t { - let func = wasm_func.func(); let mut trap = ptr::null_mut(); - let error = wasmtime_func_call( + let error = _wasmtime_func_call( wasm_func, - args, - func.param_arity(), - results, - func.result_arity(), + (*args).as_slice(), + (*results).as_uninit_slice(), &mut trap, ); match error { @@ -186,16 +189,14 @@ pub unsafe extern "C" fn wasm_func_call( #[no_mangle] pub unsafe extern "C" fn wasmtime_func_call( func: &wasm_func_t, - args: *const wasm_val_t, - num_args: usize, - results: *mut MaybeUninit, - num_results: usize, + args: *const wasm_val_vec_t, + results: *mut wasm_val_vec_t, trap_ptr: &mut *mut wasm_trap_t, ) -> Option> { _wasmtime_func_call( func, - std::slice::from_raw_parts(args, num_args), - std::slice::from_raw_parts_mut(results, num_results), + (*args).as_slice(), + (*results).as_uninit_slice(), trap_ptr, ) } diff --git a/crates/c-api/src/instance.rs b/crates/c-api/src/instance.rs index 5a41d68a1c..9a2dd3b2ca 100644 --- a/crates/c-api/src/instance.rs +++ b/crates/c-api/src/instance.rs @@ -40,16 +40,15 @@ impl wasm_instance_t { pub unsafe extern "C" fn wasm_instance_new( store: &wasm_store_t, wasm_module: &wasm_module_t, - imports: *const Box, + imports: *const wasm_extern_vec_t, result: Option<&mut *mut wasm_trap_t>, ) -> Option> { let mut instance = ptr::null_mut(); let mut trap = ptr::null_mut(); - let err = wasmtime_instance_new( + let err = _wasmtime_instance_new( store, wasm_module, - imports, - wasm_module.module().imports().len(), + (*imports).as_slice(), &mut instance, &mut trap, ); @@ -83,31 +82,27 @@ pub unsafe extern "C" fn wasm_instance_new( pub unsafe extern "C" fn wasmtime_instance_new( store: &wasm_store_t, module: &wasm_module_t, - imports: *const Box, - num_imports: usize, + imports: *const wasm_extern_vec_t, instance_ptr: &mut *mut wasm_instance_t, trap_ptr: &mut *mut wasm_trap_t, ) -> Option> { - _wasmtime_instance_new( - store, - module, - std::slice::from_raw_parts(imports, num_imports), - instance_ptr, - trap_ptr, - ) + _wasmtime_instance_new(store, module, (*imports).as_slice(), instance_ptr, trap_ptr) } fn _wasmtime_instance_new( store: &wasm_store_t, module: &wasm_module_t, - imports: &[Box], + imports: &[Option>], instance_ptr: &mut *mut wasm_instance_t, trap_ptr: &mut *mut wasm_trap_t, ) -> Option> { let store = &store.store; let imports = imports .iter() - .map(|import| import.which.clone()) + .filter_map(|import| match import { + Some(i) => Some(i.which.clone()), + None => None, + }) .collect::>(); handle_instantiate( Instance::new(store, module.module(), &imports), diff --git a/crates/c-api/src/vec.rs b/crates/c-api/src/vec.rs index 21a47f0717..fec8c7dd0d 100644 --- a/crates/c-api/src/vec.rs +++ b/crates/c-api/src/vec.rs @@ -4,6 +4,7 @@ use crate::{ wasm_moduletype_t, wasm_tabletype_t, wasm_val_t, wasm_valtype_t, }; use std::mem; +use std::mem::MaybeUninit; use std::ptr; use std::slice; @@ -54,6 +55,18 @@ macro_rules! declare_vecs { } } + pub fn as_uninit_slice(&mut self) -> &mut [MaybeUninit<$elem_ty>] { + // Note that we're careful to not create a slice with a null + // pointer as the data pointer, since that isn't defined + // behavior in Rust. + if self.size == 0 { + &mut [] + } else { + assert!(!self.data.is_null()); + unsafe { slice::from_raw_parts_mut(self.data as _, self.size) } + } + } + pub fn take(&mut self) -> Vec<$elem_ty> { if self.data.is_null() { return Vec::new(); diff --git a/crates/c-api/wasm-c-api b/crates/c-api/wasm-c-api index d9a80099d4..c9d3128465 160000 --- a/crates/c-api/wasm-c-api +++ b/crates/c-api/wasm-c-api @@ -1 +1 @@ -Subproject commit d9a80099d496b5cdba6f3fe8fc77586e0e505ddc +Subproject commit c9d31284651b975f05ac27cee0bab1377560b87e diff --git a/examples/externref.c b/examples/externref.c index 5828d17665..725e2e0e2f 100644 --- a/examples/externref.c +++ b/examples/externref.c @@ -75,7 +75,8 @@ int main() { printf("Instantiating module...\n"); wasm_trap_t *trap = NULL; wasm_instance_t *instance = NULL; - error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + error = wasmtime_instance_new(store, module, &imports, &instance, &trap); if (instance == NULL) exit_with_error("failed to instantiate", error, trap); @@ -139,10 +140,11 @@ int main() { assert(func != NULL); // And call it! - wasm_val_t args[1]; - wasm_val_copy(&args[0], &externref); + wasm_val_t args[1] = { externref }; wasm_val_t results[1]; - error = wasmtime_func_call(func, args, 1, results, 1, &trap); + wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args); + wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results); + error = wasmtime_func_call(func, &args_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call function", error, trap); @@ -161,7 +163,6 @@ int main() { ret = 0; wasm_val_delete(&results[0]); - wasm_val_delete(&args[0]); wasm_val_delete(&global_val); wasm_val_delete(&elem); wasm_extern_vec_delete(&externs); diff --git a/examples/fib-debug/main.c b/examples/fib-debug/main.c index a4e22dee3c..722ddb01d9 100644 --- a/examples/fib-debug/main.c +++ b/examples/fib-debug/main.c @@ -71,7 +71,8 @@ int main(int argc, const char* argv[]) { printf("Instantiating module...\n"); wasm_instance_t* instance = NULL; wasm_trap_t *trap = NULL; - error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + error = wasmtime_instance_new(store, module, &imports, &instance, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to instantiate", error, trap); wasm_module_delete(module); @@ -95,9 +96,11 @@ int main(int argc, const char* argv[]) { // Call. printf("Calling fib...\n"); - wasm_val_t params[1] = { {.kind = WASM_I32, .of = {.i32 = 6}} }; + wasm_val_t params[1] = { WASM_I32_VAL(6) }; wasm_val_t results[1]; - error = wasmtime_func_call(run_func, params, 1, results, 1, &trap); + wasm_val_vec_t params_vec = WASM_ARRAY_VEC(params); + wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results); + error = wasmtime_func_call(run_func, ¶ms_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call function", error, trap); diff --git a/examples/gcd.c b/examples/gcd.c index 76ca77faf7..d811458bb4 100644 --- a/examples/gcd.c +++ b/examples/gcd.c @@ -65,7 +65,8 @@ int main() { wasm_byte_vec_delete(&wasm); wasm_trap_t *trap = NULL; wasm_instance_t *instance = NULL; - error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + error = wasmtime_instance_new(store, module, &imports, &instance, &trap); if (instance == NULL) exit_with_error("failed to instantiate", error, trap); @@ -79,13 +80,11 @@ int main() { // And call it! int a = 6; int b = 27; - wasm_val_t params[2]; + wasm_val_t params[2] = { WASM_I32_VAL(a), WASM_I32_VAL(b) }; wasm_val_t results[1]; - params[0].kind = WASM_I32; - params[0].of.i32 = a; - params[1].kind = WASM_I32; - params[1].of.i32 = b; - error = wasmtime_func_call(gcd, params, 2, results, 1, &trap); + wasm_val_vec_t params_vec = WASM_ARRAY_VEC(params); + wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results); + error = wasmtime_func_call(gcd, ¶ms_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call gcd", error, trap); assert(results[0].kind == WASM_I32); diff --git a/examples/hello.c b/examples/hello.c index a14ef18aca..7c14612945 100644 --- a/examples/hello.c +++ b/examples/hello.c @@ -26,7 +26,7 @@ to tweak the `-lpthread` and such annotations as well as the name of the static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap); -static wasm_trap_t* hello_callback(const wasm_val_t args[], wasm_val_t results[]) { +static wasm_trap_t* hello_callback(const wasm_val_vec_t* args, wasm_val_vec_t* results) { printf("Calling back...\n"); printf("> Hello World!\n"); return NULL; @@ -86,8 +86,9 @@ int main() { printf("Instantiating module...\n"); wasm_trap_t *trap = NULL; wasm_instance_t *instance = NULL; - const wasm_extern_t *imports[] = { wasm_func_as_extern(hello) }; - error = wasmtime_instance_new(store, module, imports, 1, &instance, &trap); + wasm_extern_t* imports[] = { wasm_func_as_extern(hello) }; + wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports); + error = wasmtime_instance_new(store, module, &imports_vec, &instance, &trap); if (instance == NULL) exit_with_error("failed to instantiate", error, trap); @@ -101,7 +102,9 @@ int main() { // And call it! printf("Calling export...\n"); - error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap); + wasm_val_vec_t args_vec = WASM_EMPTY_VEC; + wasm_val_vec_t results_vec = WASM_EMPTY_VEC; + error = wasmtime_func_call(run, &args_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call function", error, trap); diff --git a/examples/hello.cc b/examples/hello.cc index c83be42576..f44b49c1fc 100644 --- a/examples/hello.cc +++ b/examples/hello.cc @@ -26,7 +26,7 @@ to tweak the `-lpthread` and such annotations as well as the name of the static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap); -static wasm_trap_t* hello_callback(const wasm_val_t args[], wasm_val_t results[]) { +static wasm_trap_t* hello_callback(const wasm_val_vec_t* args, wasm_val_vec_t* results) { printf("Calling back...\n"); printf("> Hello World!\n"); return NULL; @@ -86,8 +86,9 @@ int main() { printf("Instantiating module...\n"); wasm_trap_t *trap = NULL; wasm_instance_t *instance = NULL; - const wasm_extern_t *imports[] = { wasm_func_as_extern(hello) }; - error = wasmtime_instance_new(store, module, imports, 1, &instance, &trap); + wasm_extern_t* imports[] = { wasm_func_as_extern(hello) }; + wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports); + error = wasmtime_instance_new(store, module, &imports_vec, &instance, &trap); if (instance == NULL) exit_with_error("failed to instantiate", error, trap); @@ -101,7 +102,9 @@ int main() { // And call it! printf("Calling export...\n"); - error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap); + wasm_val_vec_t args_vec = WASM_EMPTY_VEC; + wasm_val_vec_t results_vec = WASM_EMPTY_VEC; + error = wasmtime_func_call(run, &args_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call function", error, trap); diff --git a/examples/interrupt.c b/examples/interrupt.c index d8ccd9a8f4..e6540a2d8d 100644 --- a/examples/interrupt.c +++ b/examples/interrupt.c @@ -42,6 +42,7 @@ static void* helper(void *_handle) { printf("Sending an interrupt\n"); wasmtime_interrupt_handle_interrupt(handle); wasmtime_interrupt_handle_delete(handle); + return 0; } static void spawn_interrupt(wasmtime_interrupt_handle_t *handle) { @@ -89,11 +90,12 @@ int main() { wasm_module_t *module = NULL; wasm_trap_t *trap = NULL; wasm_instance_t *instance = NULL; + wasm_extern_vec_t imports = WASM_EMPTY_VEC; error = wasmtime_module_new(engine, &wasm, &module); wasm_byte_vec_delete(&wasm); if (error != NULL) exit_with_error("failed to compile module", error, NULL); - error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap); + error = wasmtime_instance_new(store, module, &imports, &instance, &trap); if (instance == NULL) exit_with_error("failed to instantiate", error, trap); @@ -109,7 +111,9 @@ int main() { // And call it! printf("Entering infinite loop...\n"); - error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap); + wasm_val_vec_t args_vec = WASM_EMPTY_VEC; + wasm_val_vec_t results_vec = WASM_EMPTY_VEC; + error = wasmtime_func_call(run, &args_vec, &results_vec, &trap); assert(error == NULL); assert(trap != NULL); printf("Got a trap!...\n"); diff --git a/examples/linking.c b/examples/linking.c index f4c53b1f5a..d9bc0f93c1 100644 --- a/examples/linking.c +++ b/examples/linking.c @@ -100,7 +100,9 @@ int main() { assert(linking1_externs.size == 1); wasm_func_t *run = wasm_extern_as_func(linking1_externs.data[0]); assert(run != NULL); - error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap); + wasm_val_vec_t args_vec = WASM_EMPTY_VEC; + wasm_val_vec_t results_vec = WASM_EMPTY_VEC; + error = wasmtime_func_call(run, &args_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call run", error, trap); diff --git a/examples/memory.c b/examples/memory.c index a8391e8c20..24e26dbd4c 100644 --- a/examples/memory.c +++ b/examples/memory.c @@ -54,10 +54,11 @@ void check(bool success) { } } -void check_call(wasm_func_t* func, wasm_val_t args[], size_t num_args, int32_t expected) { +void check_call(wasm_func_t* func, const wasm_val_vec_t* args_vec, int32_t expected) { wasm_val_t results[1]; + wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results); wasm_trap_t *trap = NULL; - wasmtime_error_t *error = wasmtime_func_call(func, args, num_args, results, 1, &trap); + wasmtime_error_t *error = wasmtime_func_call(func, args_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call function", error, trap); if (results[0].of.i32 != expected) { @@ -67,42 +68,44 @@ void check_call(wasm_func_t* func, wasm_val_t args[], size_t num_args, int32_t e } void check_call0(wasm_func_t* func, int32_t expected) { - check_call(func, NULL, 0, expected); + wasm_val_vec_t args_vec = WASM_EMPTY_VEC; + check_call(func, &args_vec, expected); } void check_call1(wasm_func_t* func, int32_t arg, int32_t expected) { - wasm_val_t args[] = { {.kind = WASM_I32, .of = {.i32 = arg}} }; - check_call(func, args, 1, expected); + wasm_val_t args[] = { WASM_I32_VAL(arg) }; + wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args); + check_call(func, &args_vec, expected); } void check_call2(wasm_func_t* func, int32_t arg1, int32_t arg2, int32_t expected) { - wasm_val_t args[2] = { - {.kind = WASM_I32, .of = {.i32 = arg1}}, - {.kind = WASM_I32, .of = {.i32 = arg2}} - }; - check_call(func, args, 2, expected); + wasm_val_t args[] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) }; + wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args); + check_call(func, &args_vec, expected); } -void check_ok(wasm_func_t* func, wasm_val_t args[], size_t num_args) { +void check_ok(wasm_func_t* func, const wasm_val_vec_t* args_vec) { wasm_trap_t *trap = NULL; - wasmtime_error_t *error = wasmtime_func_call(func, args, num_args, NULL, 0, &trap); + wasm_val_vec_t results_vec = WASM_EMPTY_VEC; + wasmtime_error_t *error = wasmtime_func_call(func, args_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call function", error, trap); } void check_ok2(wasm_func_t* func, int32_t arg1, int32_t arg2) { - wasm_val_t args[2] = { - {.kind = WASM_I32, .of = {.i32 = arg1}}, - {.kind = WASM_I32, .of = {.i32 = arg2}} - }; - check_ok(func, args, 2); + wasm_val_t args[] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) }; + wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args); + check_ok(func, &args_vec); } -void check_trap(wasm_func_t* func, wasm_val_t args[], size_t num_args, size_t num_results) { - wasm_val_t results[1]; +void check_trap(wasm_func_t* func, const wasm_val_vec_t* args_vec, size_t num_results) { assert(num_results <= 1); + wasm_val_t results[1]; + wasm_val_vec_t results_vec; + results_vec.data = results; + results_vec.size = num_results; wasm_trap_t *trap = NULL; - wasmtime_error_t *error = wasmtime_func_call(func, args, num_args, results, num_results, &trap); + wasmtime_error_t *error = wasmtime_func_call(func, args_vec, &results_vec, &trap); if (error != NULL) exit_with_error("failed to call function", error, NULL); if (trap == NULL) { @@ -113,16 +116,15 @@ void check_trap(wasm_func_t* func, wasm_val_t args[], size_t num_args, size_t nu } void check_trap1(wasm_func_t* func, int32_t arg) { - wasm_val_t args[1] = { {.kind = WASM_I32, .of = {.i32 = arg}} }; - check_trap(func, args, 1, 1); + wasm_val_t args[] = { WASM_I32_VAL(arg) }; + wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args); + check_trap(func, &args_vec, 1); } void check_trap2(wasm_func_t* func, int32_t arg1, int32_t arg2) { - wasm_val_t args[2] = { - {.kind = WASM_I32, .of = {.i32 = arg1}}, - {.kind = WASM_I32, .of = {.i32 = arg2}} - }; - check_trap(func, args, 2, 0); + wasm_val_t args[] = { WASM_I32_VAL(arg1), WASM_I32_VAL(arg2) }; + wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args); + check_trap(func, &args_vec, 0); } int main(int argc, const char* argv[]) { @@ -167,7 +169,8 @@ int main(int argc, const char* argv[]) { printf("Instantiating module...\n"); wasm_instance_t* instance = NULL; wasm_trap_t *trap = NULL; - error = wasmtime_instance_new(store, module, NULL, 0, &instance, &trap); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + error = wasmtime_instance_new(store, module, &imports, &instance, &trap); if (!instance) exit_with_error("failed to instantiate", error, trap); diff --git a/examples/multi.c b/examples/multi.c index a1514b1833..3553706447 100644 --- a/examples/multi.c +++ b/examples/multi.c @@ -32,14 +32,14 @@ static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_t // A function to be called from Wasm code. wasm_trap_t* callback( - const wasm_val_t args[], wasm_val_t results[] + const wasm_val_vec_t* args, wasm_val_vec_t* results ) { printf("Calling back...\n"); - printf("> %"PRIu32" %"PRIu64"\n", args[0].of.i32, args[1].of.i64); + printf("> %"PRIu32" %"PRIu64"\n", args->data[0].of.i32, args->data[1].of.i64); printf("\n"); - wasm_val_copy(&results[0], &args[1]); - wasm_val_copy(&results[1], &args[0]); + wasm_val_copy(&results->data[0], &args->data[1]); + wasm_val_copy(&results->data[1], &args->data[0]); return NULL; } @@ -112,10 +112,11 @@ int main(int argc, const char* argv[]) { // Instantiate. printf("Instantiating module...\n"); - const wasm_extern_t* imports[] = {wasm_func_as_extern(callback_func)}; + wasm_extern_t* imports[] = {wasm_func_as_extern(callback_func)}; + wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports); wasm_instance_t* instance = NULL; wasm_trap_t* trap = NULL; - error = wasmtime_instance_new(store, module, imports, 1, &instance, &trap); + error = wasmtime_instance_new(store, module, &imports_vec, &instance, &trap); if (!instance) exit_with_error("failed to instantiate", error, trap); @@ -140,13 +141,11 @@ int main(int argc, const char* argv[]) { // Call. printf("Calling export...\n"); - wasm_val_t args[2]; - args[0].kind = WASM_I32; - args[0].of.i32 = 1; - args[1].kind = WASM_I64; - args[1].of.i64 = 2; + wasm_val_t args[2] = { WASM_I32_VAL(1), WASM_I64_VAL(2) }; wasm_val_t results[2]; - error = wasmtime_func_call(run_func, args, 2, results, 2, &trap); + wasm_val_vec_t args_vec = WASM_ARRAY_VEC(args); + wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results); + error = wasmtime_func_call(run_func, &args_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call run", error, trap); diff --git a/examples/serialize.c b/examples/serialize.c index c25a1bf401..023e1db30b 100644 --- a/examples/serialize.c +++ b/examples/serialize.c @@ -26,7 +26,7 @@ to tweak the `-lpthread` and such annotations as well as the name of the static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap); -static wasm_trap_t* hello_callback(const wasm_val_t args[], wasm_val_t results[]) { +static wasm_trap_t* hello_callback(const wasm_val_vec_t* args, wasm_val_vec_t* results) { printf("Calling back...\n"); printf("> Hello World!\n"); return NULL; @@ -111,8 +111,9 @@ int deserialize(wasm_byte_vec_t* buffer) { printf("Instantiating module...\n"); wasm_trap_t *trap = NULL; wasm_instance_t *instance = NULL; - const wasm_extern_t *imports[] = { wasm_func_as_extern(hello) }; - error = wasmtime_instance_new(store, module, imports, 1, &instance, &trap); + wasm_extern_t *imports[] = { wasm_func_as_extern(hello) }; + wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports); + error = wasmtime_instance_new(store, module, &imports_vec, &instance, &trap); if (instance == NULL) exit_with_error("failed to instantiate", error, trap); @@ -126,7 +127,9 @@ int deserialize(wasm_byte_vec_t* buffer) { // And call it! printf("Calling export...\n"); - error = wasmtime_func_call(run, NULL, 0, NULL, 0, &trap); + wasm_val_vec_t args_vec = WASM_EMPTY_VEC; + wasm_val_vec_t results_vec = WASM_EMPTY_VEC; + error = wasmtime_func_call(run, &args_vec, &results_vec, &trap); if (error != NULL || trap != NULL) exit_with_error("failed to call function", error, trap); diff --git a/examples/threads.c b/examples/threads.c index bc04ba69c6..06aa1f0aa7 100644 --- a/examples/threads.c +++ b/examples/threads.c @@ -17,9 +17,9 @@ const int N_THREADS = 10; const int N_REPS = 3; // A function to be called from Wasm code. -own wasm_trap_t* callback(const wasm_val_t args[], wasm_val_t results[]) { - assert(args[0].kind == WASM_I32); - printf("> Thread %d running\n", args[0].of.i32); +own wasm_trap_t* callback(const wasm_val_vec_t* args, wasm_val_vec_t* results) { + assert(args->data[0].kind == WASM_I32); + printf("> Thread %d running\n", args->data[0].of.i32); return NULL; } @@ -53,11 +53,12 @@ void* run(void* args_abs) { wasm_globaltype_delete(global_type); // Instantiate. - const wasm_extern_t* imports[] = { + wasm_extern_t* imports[] = { wasm_func_as_extern(func), wasm_global_as_extern(global), }; + wasm_extern_vec_t imports_vec = WASM_ARRAY_VEC(imports); own wasm_instance_t* instance = - wasm_instance_new(store, module, imports, NULL); + wasm_instance_new(store, module, &imports_vec, NULL); if (!instance) { printf("> Error instantiating module!\n"); return NULL; @@ -82,7 +83,9 @@ void* run(void* args_abs) { wasm_instance_delete(instance); // Call. - if (wasm_func_call(run_func, NULL, NULL)) { + wasm_val_vec_t args_vec = WASM_EMPTY_VEC; + wasm_val_vec_t results_vec = WASM_EMPTY_VEC; + if (wasm_func_call(run_func, &args_vec, &results_vec)) { printf("> Error calling function!\n"); return NULL; } diff --git a/examples/wasi-fs/main.c b/examples/wasi-fs/main.c index 6452cb7d41..8633bfbd6c 100644 --- a/examples/wasi-fs/main.c +++ b/examples/wasi-fs/main.c @@ -91,7 +91,10 @@ int main() { wasmtime_linker_get_default(linker, &empty, &func); if (error != NULL) exit_with_error("failed to locate default export for module", error, NULL); - error = wasmtime_func_call(func, NULL, 0, NULL, 0, &trap); + + wasm_val_vec_t args_vec = WASM_EMPTY_VEC; + wasm_val_vec_t results_vec = WASM_EMPTY_VEC; + error = wasmtime_func_call(func, &args_vec, &results_vec, &trap); if (error != NULL) exit_with_error("error calling default export", error, trap); diff --git a/examples/wasi/main.c b/examples/wasi/main.c index a8514c69ca..3bb71ec4de 100644 --- a/examples/wasi/main.c +++ b/examples/wasi/main.c @@ -90,7 +90,10 @@ int main() { wasmtime_linker_get_default(linker, &empty, &func); if (error != NULL) exit_with_error("failed to locate default export for module", error, NULL); - error = wasmtime_func_call(func, NULL, 0, NULL, 0, &trap); + + wasm_val_vec_t args_vec = WASM_EMPTY_VEC; + wasm_val_vec_t results_vec = WASM_EMPTY_VEC; + error = wasmtime_func_call(func, &args_vec, &results_vec, &trap); if (error != NULL) exit_with_error("error calling default export", error, trap); From 55c5424e0eefdc7bf4fcea0c1e415397de87c06c Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 14 Jan 2021 08:57:15 -0800 Subject: [PATCH 07/55] Adjust `wasmtime_func_callback_*` type signature This aligns the C header with what we have in the crate itself, in addition to matching what's in `wasm.h`. --- crates/c-api/include/wasmtime.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h index b2efa3e81c..6752459a4d 100644 --- a/crates/c-api/include/wasmtime.h +++ b/crates/c-api/include/wasmtime.h @@ -527,7 +527,7 @@ typedef struct wasmtime_caller_t wasmtime_caller_t; * argument is a #wasmtime_caller_t which allows learning information about the * caller. */ -typedef own wasm_trap_t* (*wasmtime_func_callback_t)(const wasmtime_caller_t* caller, const wasm_val_t args[], wasm_val_t results[]); +typedef own wasm_trap_t* (*wasmtime_func_callback_t)(const wasmtime_caller_t* caller, const wasm_val_vec_t *args, wasm_val_vec_t *results); /** * \brief Callback signature for #wasmtime_func_new_with_env. @@ -536,7 +536,7 @@ typedef own wasm_trap_t* (*wasmtime_func_callback_t)(const wasmtime_caller_t* ca * first argument is a #wasmtime_caller_t which allows learning information * about the caller. */ -typedef own wasm_trap_t* (*wasmtime_func_callback_with_env_t)(const wasmtime_caller_t* caller, void* env, const wasm_val_t args[], wasm_val_t results[]); +typedef own wasm_trap_t* (*wasmtime_func_callback_with_env_t)(const wasmtime_caller_t* caller, void* env, const wasm_val_vec_t *args, wasm_val_vec_t *results); /** * \brief Creates a new host-defined function. From 703762c49e4bc3f237bb341b92d5ef4e4c699e49 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 12 Jan 2021 10:44:11 -0800 Subject: [PATCH 08/55] Update support for the module linking proposal This commit updates the various tooling used by wasmtime which has new updates to the module linking proposal. This is done primarily to sync with WebAssembly/module-linking#26. The main change implemented here is that wasmtime now supports creating instances from a set of values, nott just from instantiating a module. Additionally subtyping handling of modules with respect to imports is now properly handled by desugaring two-level imports to imports of instances. A number of small refactorings are included here as well, but most of them are in accordance with the changes to `wasmparser` and the updated binary format for module linking. --- Cargo.lock | 50 +-- Cargo.toml | 4 +- cranelift/codegen/Cargo.toml | 2 +- cranelift/peepmatic/Cargo.toml | 2 +- cranelift/peepmatic/crates/fuzzing/Cargo.toml | 2 +- cranelift/peepmatic/crates/runtime/Cargo.toml | 2 +- cranelift/peepmatic/crates/souper/Cargo.toml | 2 +- .../peepmatic/crates/test-operator/Cargo.toml | 2 +- cranelift/wasm/Cargo.toml | 2 +- cranelift/wasm/src/environ/spec.rs | 56 ++- cranelift/wasm/src/module_translator.rs | 32 +- cranelift/wasm/src/sections_translator.rs | 71 ++-- crates/debug/Cargo.toml | 2 +- crates/environ/Cargo.toml | 2 +- crates/environ/src/module.rs | 46 +-- crates/environ/src/module_environ.rs | 366 +++++++++++------- crates/fuzzing/Cargo.toml | 6 +- crates/jit/Cargo.toml | 2 +- crates/jit/src/instantiate.rs | 5 +- crates/lightbeam/Cargo.toml | 2 +- crates/lightbeam/wasmtime/Cargo.toml | 2 +- crates/runtime/src/export.rs | 24 +- crates/runtime/src/imports.rs | 15 - crates/runtime/src/instance.rs | 43 +- crates/runtime/src/lib.rs | 2 +- crates/wasmtime/Cargo.toml | 2 +- crates/wasmtime/src/externals.rs | 59 +-- crates/wasmtime/src/func.rs | 44 ++- crates/wasmtime/src/instance.rs | 219 +++++------ crates/wasmtime/src/linker.rs | 33 +- crates/wasmtime/src/module.rs | 14 +- crates/wasmtime/src/store.rs | 8 +- crates/wasmtime/src/trampoline/global.rs | 4 +- crates/wasmtime/src/trampoline/mod.rs | 16 +- crates/wasmtime/src/trampoline/table.rs | 1 + crates/wasmtime/src/types.rs | 34 +- crates/wasmtime/src/types/matching.rs | 210 +++++++--- crates/wast/Cargo.toml | 2 +- fuzz/Cargo.toml | 2 +- fuzz/fuzz_targets/instantiate-swarm.rs | 3 +- src/obj.rs | 2 +- tests/all/module_linking.rs | 16 +- .../misc_testsuite/module-linking/alias.wast | 38 +- .../module-linking/import-subtyping.wast | 225 +++++++---- .../module-linking/instantiate.wast | 112 +++--- 45 files changed, 1041 insertions(+), 747 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 084a2cb0a2..f6cf7f0f26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -388,7 +388,7 @@ dependencies = [ "souper-ir", "target-lexicon", "thiserror", - "wast 29.0.0", + "wast 31.0.0", ] [[package]] @@ -1393,7 +1393,7 @@ dependencies = [ "peepmatic-test-operator", "peepmatic-traits", "serde", - "wast 29.0.0", + "wast 31.0.0", "z3", ] @@ -1421,7 +1421,7 @@ dependencies = [ "peepmatic-traits", "rand", "serde", - "wast 29.0.0", + "wast 31.0.0", ] [[package]] @@ -1446,7 +1446,7 @@ dependencies = [ "serde", "serde_test", "thiserror", - "wast 29.0.0", + "wast 31.0.0", ] [[package]] @@ -1458,7 +1458,7 @@ dependencies = [ "peepmatic", "peepmatic-test-operator", "souper-ir", - "wast 29.0.0", + "wast 31.0.0", ] [[package]] @@ -1479,7 +1479,7 @@ version = "0.69.0" dependencies = [ "peepmatic-traits", "serde", - "wast 29.0.0", + "wast 31.0.0", ] [[package]] @@ -2350,20 +2350,21 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed89eaf99e08b84f96e477a16588a07dd3b51dc5f07291c3706782f62a10a5e1" +checksum = "c75fa62cf1464aa6655479ae454202a159cc82b7b4d66e8f174409669c0654c5" dependencies = [ "leb128", ] [[package]] name = "wasm-smith" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509904d9c4c4659ac238a3f27c3656dd6d3931697eddd4b0f32e335769c298d0" +checksum = "0b9b9b796bf4da5eb0523b136d6f0cc9a59c16a66ece8e0d5a14a9cdccf2864e" dependencies = [ "arbitrary", + "indexmap", "leb128", "wasm-encoder", ] @@ -2394,15 +2395,15 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.71.0" +version = "0.72.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89a30c99437829ede826802bfcf28500cf58df00e66cb9114df98813bc145ff1" +checksum = "2cdf4d872d407f9fb44956e540582eeaf0dc4fb8142f1f0f64e2c37196bada01" [[package]] name = "wasmprinter" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0515db67c610037f3c53ec36976edfd1eb01bac6b1226914b17ce609480e729f" +checksum = "a0c139586e3b80b899f5aaaa3720c7a236ea9073e315292e01d099da12efacc5" dependencies = [ "anyhow", "wasmparser", @@ -2754,7 +2755,7 @@ version = "0.22.0" dependencies = [ "anyhow", "wasmtime", - "wast 29.0.0", + "wast 31.0.0", ] [[package]] @@ -2790,29 +2791,20 @@ dependencies = [ [[package]] name = "wast" -version = "29.0.0" +version = "31.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf2268937131d63c3d833242bf5e075406f9ed868b4265f3280e15dac29ac18" -dependencies = [ - "leb128", -] - -[[package]] -name = "wast" -version = "30.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b79907b22f740634810e882d8d1d9d0f9563095a8ab94e786e370242bff5cd2" +checksum = "9beb1f6b63f08c523a1e8e76fc70058af4d2a34ef1c504f56cdac7b6970228b9" dependencies = [ "leb128", ] [[package]] name = "wat" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8279a02835bf12e61ed2b3c3cbc6ecf9918762fd97e036917c11a09ec20ca44" +checksum = "2a0b3044da73d3b84a822d955afad356759b2fee454b6882722008dace80b68e" dependencies = [ - "wast 30.0.0", + "wast 31.0.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 54f87e181c..640189732d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,12 +38,12 @@ anyhow = "1.0.19" target-lexicon = { version = "0.11.0", default-features = false } pretty_env_logger = "0.4.0" file-per-thread-logger = "0.1.1" -wat = "1.0.30" +wat = "1.0.32" libc = "0.2.60" log = "0.4.8" rayon = "1.2.1" humantime = "2.0.0" -wasmparser = "0.71.0" +wasmparser = "0.72.0" [dev-dependencies] env_logger = "0.8.1" diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index d8a8baf384..f601965022 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -30,7 +30,7 @@ peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, versi peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.69.0" } regalloc = { version = "0.0.31" } souper-ir = { version = "2.1.0", optional = true } -wast = { version = "29.0.0", optional = true } +wast = { version = "31.0.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary # machine code. Integration tests that need external dependencies can be diff --git a/cranelift/peepmatic/Cargo.toml b/cranelift/peepmatic/Cargo.toml index 9028e7105a..dcb1da1537 100644 --- a/cranelift/peepmatic/Cargo.toml +++ b/cranelift/peepmatic/Cargo.toml @@ -15,7 +15,7 @@ peepmatic-macro = { version = "0.69.0", path = "crates/macro" } peepmatic-runtime = { version = "0.69.0", path = "crates/runtime", features = ["construct"] } peepmatic-traits = { version = "0.69.0", path = "crates/traits" } serde = { version = "1.0.105", features = ["derive"] } -wast = "29.0.0" +wast = "31.0.0" z3 = { version = "0.7.1", features = ["static-link-z3"] } [dev-dependencies] diff --git a/cranelift/peepmatic/crates/fuzzing/Cargo.toml b/cranelift/peepmatic/crates/fuzzing/Cargo.toml index 1611b75c6d..fda6532563 100644 --- a/cranelift/peepmatic/crates/fuzzing/Cargo.toml +++ b/cranelift/peepmatic/crates/fuzzing/Cargo.toml @@ -21,4 +21,4 @@ peepmatic-test-operator = { path = "../test-operator" } peepmatic-traits = { path = "../traits" } rand = { version = "0.7.3", features = ["small_rng"] } serde = "1.0.106" -wast = "29.0.0" +wast = "31.0.0" diff --git a/cranelift/peepmatic/crates/runtime/Cargo.toml b/cranelift/peepmatic/crates/runtime/Cargo.toml index 8e15ad7527..ea1909c282 100644 --- a/cranelift/peepmatic/crates/runtime/Cargo.toml +++ b/cranelift/peepmatic/crates/runtime/Cargo.toml @@ -16,7 +16,7 @@ peepmatic-automata = { version = "0.69.0", path = "../automata", features = ["se peepmatic-traits = { version = "0.69.0", path = "../traits" } serde = { version = "1.0.105", features = ["derive"] } thiserror = "1.0.15" -wast = { version = "29.0.0", optional = true } +wast = { version = "31.0.0", optional = true } [dev-dependencies] peepmatic-test-operator = { version = "0.69.0", path = "../test-operator" } diff --git a/cranelift/peepmatic/crates/souper/Cargo.toml b/cranelift/peepmatic/crates/souper/Cargo.toml index 811363651c..7b512526d6 100644 --- a/cranelift/peepmatic/crates/souper/Cargo.toml +++ b/cranelift/peepmatic/crates/souper/Cargo.toml @@ -16,4 +16,4 @@ log = "0.4.8" [dev-dependencies] peepmatic = { path = "../..", version = "0.69.0" } peepmatic-test-operator = { version = "0.69.0", path = "../test-operator" } -wast = "29.0.0" +wast = "31.0.0" diff --git a/cranelift/peepmatic/crates/test-operator/Cargo.toml b/cranelift/peepmatic/crates/test-operator/Cargo.toml index 295610c1bc..de23fe59fd 100644 --- a/cranelift/peepmatic/crates/test-operator/Cargo.toml +++ b/cranelift/peepmatic/crates/test-operator/Cargo.toml @@ -11,4 +11,4 @@ edition = "2018" [dependencies] peepmatic-traits = { version = "0.69.0", path = "../traits" } serde = { version = "1.0.105", features = ["derive"] } -wast = "29.0.0" +wast = "31.0.0" diff --git a/cranelift/wasm/Cargo.toml b/cranelift/wasm/Cargo.toml index 413d2ee004..ac7866af70 100644 --- a/cranelift/wasm/Cargo.toml +++ b/cranelift/wasm/Cargo.toml @@ -12,7 +12,7 @@ keywords = ["webassembly", "wasm"] edition = "2018" [dependencies] -wasmparser = { version = "0.71", default-features = false } +wasmparser = { version = "0.72", default-features = false } cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false } cranelift-entity = { path = "../entity", version = "0.69.0" } cranelift-frontend = { path = "../frontend", version = "0.69.0", default-features = false } diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index f7577de5d3..84978dc0d8 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -205,25 +205,34 @@ pub enum ReturnMode { /// An entry in the alias section of a wasm module (from the module linking /// proposal) -pub enum Alias { - /// A parent's module is being aliased into our own index space. - /// - /// Note that the index here is in the parent's index space, not our own. - ParentModule(ModuleIndex), +pub enum Alias<'a> { + /// An outer module's module is being aliased into our own index space. + OuterModule { + /// The number of modules above us that we're referencing. + relative_depth: u32, + /// The module index in the outer module's index space we're referencing. + index: ModuleIndex, + }, - /// A parent's type is being aliased into our own index space + /// An outer module's type is being aliased into our own index space /// - /// Note that the index here is in the parent's index space, not our own. - ParentType(TypeIndex), + /// Note that the index here is in the outer module's index space, not our + /// own. + OuterType { + /// The number of modules above us that we're referencing. + relative_depth: u32, + /// The type index in the outer module's index space we're referencing. + index: TypeIndex, + }, /// A previously created instance is having one of its exports aliased into /// our index space. - Child { + InstanceExport { /// The index we're aliasing. instance: InstanceIndex, /// The nth export that we're inserting into our own index space /// locally. - export: usize, + export: &'a str, }, } @@ -1014,30 +1023,15 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { drop(amount); } - /// Declares that a module will come later with the type signature provided. - fn declare_module(&mut self, ty: TypeIndex) -> WasmResult<()> { - drop(ty); - Err(WasmError::Unsupported("module linking".to_string())) - } - /// Called at the beginning of translating a module. /// - /// The `index` argument is a monotonically increasing index which - /// corresponds to the nth module that's being translated. This is not the - /// 32-bit index in the current module's index space. For example the first - /// call to `module_start` will have index 0. - /// /// Note that for nested modules this may be called multiple times. - fn module_start(&mut self, index: usize) { - drop(index); - } + fn module_start(&mut self) {} /// Called at the end of translating a module. /// /// Note that for nested modules this may be called multiple times. - fn module_end(&mut self, index: usize) { - drop(index); - } + fn module_end(&mut self) {} /// Indicates that this module will have `amount` instances. fn reserve_instances(&mut self, amount: u32) { @@ -1046,7 +1040,11 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { /// Declares a new instance which this module will instantiate before it's /// instantiated. - fn declare_instance(&mut self, module: ModuleIndex, args: Vec) -> WasmResult<()> { + fn declare_instance( + &mut self, + module: ModuleIndex, + args: Vec<(&'data str, EntityIndex)>, + ) -> WasmResult<()> { drop((module, args)); Err(WasmError::Unsupported("wasm instance".to_string())) } @@ -1056,7 +1054,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { /// The alias comes from the `instance` specified (or the parent if `None` /// is supplied) and the index is either in the module's own index spaces /// for the parent or an index into the exports for nested instances. - fn declare_alias(&mut self, alias: Alias) -> WasmResult<()> { + fn declare_alias(&mut self, alias: Alias<'data>) -> WasmResult<()> { drop(alias); Err(WasmError::Unsupported("wasm alias".to_string())) } diff --git a/cranelift/wasm/src/module_translator.rs b/cranelift/wasm/src/module_translator.rs index c3e27dd820..ed71d7ce1c 100644 --- a/cranelift/wasm/src/module_translator.rs +++ b/cranelift/wasm/src/module_translator.rs @@ -4,8 +4,8 @@ use crate::environ::{ModuleEnvironment, WasmResult}; use crate::sections_translator::{ parse_alias_section, parse_data_section, parse_element_section, parse_event_section, parse_export_section, parse_function_section, parse_global_section, parse_import_section, - parse_instance_section, parse_memory_section, parse_module_section, parse_name_section, - parse_start_section, parse_table_section, parse_type_section, + parse_instance_section, parse_memory_section, parse_name_section, parse_start_section, + parse_table_section, parse_type_section, }; use crate::state::ModuleTranslationState; use cranelift_codegen::timing; @@ -22,23 +22,16 @@ pub fn translate_module<'data>( let mut module_translation_state = ModuleTranslationState::new(); let mut validator = Validator::new(); validator.wasm_features(environ.wasm_features()); - let mut stack = Vec::new(); - let mut modules = 1; - let mut cur_module = 0; for payload in Parser::new(0).parse_all(data) { match payload? { Payload::Version { num, range } => { validator.version(num, &range)?; - environ.module_start(cur_module); + environ.module_start(); } Payload::End => { validator.end()?; - environ.module_end(cur_module); - if let Some((other, other_index)) = stack.pop() { - validator = other; - cur_module = other_index; - } + environ.module_end(); } Payload::TypeSection(types) => { @@ -111,10 +104,6 @@ pub fn translate_module<'data>( environ.reserve_passive_data(count)?; } - Payload::ModuleSection(s) => { - validator.module_section(&s)?; - parse_module_section(s, environ)?; - } Payload::InstanceSection(s) => { validator.instance_section(&s)?; parse_instance_section(s, environ)?; @@ -123,20 +112,17 @@ pub fn translate_module<'data>( validator.alias_section(&s)?; parse_alias_section(s, environ)?; } - Payload::ModuleCodeSectionStart { + Payload::ModuleSectionStart { count, range, size: _, } => { - validator.module_code_section_start(count, &range)?; + validator.module_section_start(count, &range)?; + environ.reserve_modules(count); } - Payload::ModuleCodeSectionEntry { .. } => { - let subvalidator = validator.module_code_section_entry(); - stack.push((validator, cur_module)); - validator = subvalidator; - cur_module = modules; - modules += 1; + Payload::ModuleSectionEntry { .. } => { + validator.module_section_entry(); } Payload::CustomSection { diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index 0e791c1a43..3906c02393 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -504,19 +504,6 @@ pub fn parse_name_section<'data>( Ok(()) } -/// Parses the Module section of the wasm module. -pub fn parse_module_section<'data>( - section: wasmparser::ModuleSectionReader<'data>, - environ: &mut dyn ModuleEnvironment<'data>, -) -> WasmResult<()> { - environ.reserve_modules(section.get_count()); - - for module_ty in section { - environ.declare_module(TypeIndex::from_u32(module_ty?))?; - } - Ok(()) -} - /// Parses the Instance section of the wasm module. pub fn parse_instance_section<'data>( section: wasmparser::InstanceSectionReader<'data>, @@ -530,20 +517,23 @@ pub fn parse_instance_section<'data>( let args = instance .args()? .into_iter() - .map(|result| { - let (kind, idx) = result?; - Ok(match kind { - ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(idx)), - ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(idx)), - ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(idx)), - ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(idx)), - ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(idx)), - ExternalKind::Instance => EntityIndex::Instance(InstanceIndex::from_u32(idx)), + .map(|arg| { + let arg = arg?; + let index = match arg.kind { + ExternalKind::Function => EntityIndex::Function(FuncIndex::from_u32(arg.index)), + ExternalKind::Table => EntityIndex::Table(TableIndex::from_u32(arg.index)), + ExternalKind::Memory => EntityIndex::Memory(MemoryIndex::from_u32(arg.index)), + ExternalKind::Global => EntityIndex::Global(GlobalIndex::from_u32(arg.index)), + ExternalKind::Module => EntityIndex::Module(ModuleIndex::from_u32(arg.index)), + ExternalKind::Instance => { + EntityIndex::Instance(InstanceIndex::from_u32(arg.index)) + } ExternalKind::Event => unimplemented!(), // this won't pass validation ExternalKind::Type => unreachable!(), - }) + }; + Ok((arg.name, index)) }) .collect::>>()?; environ.declare_instance(module, args)?; @@ -557,19 +547,28 @@ pub fn parse_alias_section<'data>( environ: &mut dyn ModuleEnvironment<'data>, ) -> WasmResult<()> { for alias in section { - let alias = alias?; - let alias = match alias.instance { - wasmparser::AliasedInstance::Parent => { - match alias.kind { - ExternalKind::Module => Alias::ParentModule(ModuleIndex::from_u32(alias.index)), - ExternalKind::Type => Alias::ParentType(TypeIndex::from_u32(alias.index)), - // shouldn't get past validation - _ => unreachable!(), - } - } - wasmparser::AliasedInstance::Child(i) => Alias::Child { - instance: InstanceIndex::from_u32(i), - export: alias.index as usize, + let alias = match alias? { + wasmparser::Alias::OuterType { + relative_depth, + index, + } => Alias::OuterType { + relative_depth, + index: TypeIndex::from_u32(index), + }, + wasmparser::Alias::OuterModule { + relative_depth, + index, + } => Alias::OuterModule { + relative_depth, + index: ModuleIndex::from_u32(index), + }, + wasmparser::Alias::InstanceExport { + instance, + export, + kind: _, + } => Alias::InstanceExport { + instance: InstanceIndex::from_u32(instance), + export, }, }; environ.declare_alias(alias)?; diff --git a/crates/debug/Cargo.toml b/crates/debug/Cargo.toml index 523c136cba..122c22b930 100644 --- a/crates/debug/Cargo.toml +++ b/crates/debug/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] gimli = "0.23.0" -wasmparser = "0.71" +wasmparser = "0.72" object = { version = "0.22.0", default-features = false, features = ["read_core", "elf", "write"] } wasmtime-environ = { path = "../environ", version = "0.22.0" } target-lexicon = { version = "0.11.0", default-features = false } diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index caa6ba3b58..3463bc4d30 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -16,7 +16,7 @@ anyhow = "1.0" cranelift-codegen = { path = "../../cranelift/codegen", version = "0.69.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.69.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.69.0", features = ["enable-serde"] } -wasmparser = "0.71" +wasmparser = "0.72" indexmap = { version = "1.0.2", features = ["serde-1"] } thiserror = "1.0.4" serde = { version = "1.0.94", features = ["derive"] } diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 226d96134d..644f1a2bb3 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -142,12 +142,6 @@ impl ModuleType { /// memory initializers. #[derive(Default, Debug, Clone, Serialize, Deserialize)] pub struct Module { - /// The parent index of this module, used for the module linking proposal. - /// - /// This index is into the list of modules returned from compilation of a - /// single wasm file with nested modules. - pub parent: Option, - /// The name of this wasm module, often found in the wasm file. pub name: Option, @@ -213,25 +207,26 @@ pub struct Module { pub enum Initializer { /// An imported item is required to be provided. Import { - /// Module name of this import - module: String, - /// Optional field name of this import + /// Name of this import + name: String, + /// The field name projection of this import. When module-linking is + /// enabled this is always `None`. Otherwise this is always `Some`. field: Option, /// Where this import will be placed, which also has type information /// about the import. index: EntityIndex, }, - /// A module from the parent's declared modules is inserted into our own + /// An export from a previously defined instance is being inserted into our /// index space. - AliasParentModule(ModuleIndex), - - /// A module from the parent's declared modules is inserted into our own - /// index space. - #[allow(missing_docs)] + /// + /// Note that when the module linking proposal is enabled two-level imports + /// will implicitly desugar to this initializer. AliasInstanceExport { + /// The instance that we're referencing. instance: InstanceIndex, - export: usize, + /// Which export is being inserted into our index space. + export: String, }, /// A module is being instantiated with previously configured intializers @@ -239,8 +234,9 @@ pub enum Initializer { Instantiate { /// The module that this instance is instantiating. module: ModuleIndex, - /// The arguments provided to instantiation. - args: Vec, + /// The arguments provided to instantiation, along with their name in + /// the instance being instantiated. + args: IndexMap, }, /// A module is defined into the module index space, and which module is @@ -351,11 +347,9 @@ impl Module { /// module name, field name, and type that's being imported. pub fn imports(&self) -> impl Iterator, EntityType)> { self.initializers.iter().filter_map(move |i| match i { - Initializer::Import { - module, - field, - index, - } => Some((module.as_str(), field.as_deref(), self.type_of(*index))), + Initializer::Import { name, field, index } => { + Some((name.as_str(), field.as_deref(), self.type_of(*index))) + } _ => None, }) } @@ -389,16 +383,16 @@ pub struct TypeTables { /// The type signature of known modules. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ModuleSignature { - /// All imports in this module, listed in order with their module/name and + /// All imports in this module, listed in order with their name and /// what type they're importing. - pub imports: Vec<(String, Option, EntityType)>, + pub imports: IndexMap, /// Exports are what an instance type conveys, so we go through an /// indirection over there. pub exports: InstanceTypeIndex, } /// The type signature of known instances. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct InstanceSignature { /// The name of what's being exported as well as its type signature. pub exports: IndexMap, diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 7b2cca6110..7bb30ea166 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -14,7 +14,7 @@ use cranelift_wasm::{ WasmError, WasmFuncType, WasmResult, }; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{hash_map::Entry, HashMap}; use std::convert::TryFrom; use std::mem; use std::path::PathBuf; @@ -31,6 +31,10 @@ pub struct ModuleEnvironment<'data> { /// the module linking proposal. results: Vec>, + /// Modules which are in-progress being translated, or otherwise also known + /// as the outer modules of the current module being processed. + in_progress: Vec>, + /// How many modules that have not yet made their way into `results` which /// are coming at some point. modules_to_be: usize, @@ -38,13 +42,11 @@ pub struct ModuleEnvironment<'data> { /// Intern'd types for this entire translation, shared by all modules. types: TypeTables, - /// Where our module will get pushed into `results` after it's finished. - cur: usize, - // Various bits and pieces of configuration features: WasmFeatures, target_config: TargetFrontendConfig, tunables: Tunables, + first_module: bool, } /// The result of translating via `ModuleEnvironment`. Function bodies are not @@ -72,15 +74,7 @@ pub struct ModuleTranslation<'data> { /// which function is currently being defined. code_index: u32, - /// When local modules are declared an entry is pushed onto this list which - /// indicates that the initializer at the specified position needs to be - /// rewritten with the module's final index in the global list of compiled - /// modules. - module_initializer_indexes: Vec, - - /// Used as a pointer into the above list as the module code section is - /// parsed. - num_modules_defined: usize, + implicit_instances: HashMap<&'data str, InstanceIndex>, } /// Contains function data: byte code and its offset in the module. @@ -142,12 +136,13 @@ impl<'data> ModuleEnvironment<'data> { Self { result: ModuleTranslation::default(), results: Vec::with_capacity(1), + in_progress: Vec::new(), modules_to_be: 1, - cur: 0, types: Default::default(), target_config, tunables: tunables.clone(), features: *features, + first_module: true, } } @@ -163,7 +158,8 @@ impl<'data> ModuleEnvironment<'data> { /// Note that for MVP modules this will always be a list with one element, /// but with the module linking proposal this may have many elements. /// - /// For the module linking proposal the top-level module is at index 0. + /// For the module linking proposal the top-level module is returned as the + /// first return value. /// /// The `TypeTables` structure returned contains intern'd versions of types /// referenced from each module translation. This primarily serves as the @@ -173,10 +169,10 @@ impl<'data> ModuleEnvironment<'data> { pub fn translate( mut self, data: &'data [u8], - ) -> WasmResult<(Vec>, TypeTables)> { + ) -> WasmResult<(usize, Vec>, TypeTables)> { translate_module(data, &mut self)?; assert!(self.results.len() > 0); - Ok((self.results, self.types)) + Ok((self.results.len() - 1, self.results, self.types)) } fn declare_export(&mut self, export: EntityIndex, name: &str) -> WasmResult<()> { @@ -224,6 +220,137 @@ impl<'data> ModuleEnvironment<'data> { dwarf.ranges = gimli::RangeLists::new(info.debug_ranges, info.debug_rnglists); dwarf.locations = gimli::LocationLists::new(info.debug_loc, info.debug_loclists); } + + /// Declares a new import with the `module` and `field` names, importing the + /// `ty` specified. + /// + /// Note that this method is somewhat tricky due to the implementation of + /// the module linking proposal. In the module linking proposal two-level + /// imports are recast as single-level imports of instances. That recasting + /// happens here by recording an import of an instance for the first time + /// we see a two-level import. + /// + /// When the module linking proposal is disabled, however, disregard this + /// logic and instead work directly with two-level imports since no + /// instances are defined. + fn declare_import(&mut self, module: &'data str, field: Option<&'data str>, ty: EntityType) { + if !self.features.module_linking { + assert!(field.is_some()); + let index = self.push_type(ty); + self.result.module.initializers.push(Initializer::Import { + name: module.to_owned(), + field: field.map(|s| s.to_string()), + index, + }); + return; + } + + match field { + Some(field) => { + // If this is a two-level import then this is actually an + // implicit import of an instance, where each two-level import + // is an alias directive from the original instance. The first + // thing we do here is lookup our implicit instance, creating a + // blank one if it wasn't already created. + let instance = match self.result.implicit_instances.entry(module) { + Entry::Occupied(e) => *e.get(), + Entry::Vacant(v) => { + let ty = self + .types + .instance_signatures + .push(InstanceSignature::default()); + let idx = self.result.module.instances.push(ty); + self.result.module.initializers.push(Initializer::Import { + name: module.to_owned(), + field: None, + index: EntityIndex::Instance(idx), + }); + *v.insert(idx) + } + }; + + // Update the implicit instance's type signature with this new + // field and its type. + self.types.instance_signatures[self.result.module.instances[instance]] + .exports + .insert(field.to_string(), ty.clone()); + + // Record our implicit alias annotation which corresponds to + // this import that we're processing. + self.result + .module + .initializers + .push(Initializer::AliasInstanceExport { + instance, + export: field.to_string(), + }); + + // And then record the type information for the item that we're + // processing. + self.push_type(ty); + } + None => { + // Without a field then this is a single-level import (a feature + // of module linking) which means we're simply importing that + // name with the specified type. Record the type information and + // then the name that we're importing. + let index = self.push_type(ty); + self.result.module.initializers.push(Initializer::Import { + name: module.to_owned(), + field: None, + index, + }); + } + } + } + + fn push_type(&mut self, ty: EntityType) -> EntityIndex { + match ty { + EntityType::Function(ty) => { + EntityIndex::Function(self.result.module.functions.push(ty)) + } + EntityType::Table(ty) => { + let plan = TablePlan::for_table(ty, &self.tunables); + EntityIndex::Table(self.result.module.table_plans.push(plan)) + } + EntityType::Memory(ty) => { + let plan = MemoryPlan::for_memory(ty, &self.tunables); + EntityIndex::Memory(self.result.module.memory_plans.push(plan)) + } + EntityType::Global(ty) => EntityIndex::Global(self.result.module.globals.push(ty)), + EntityType::Instance(ty) => { + EntityIndex::Instance(self.result.module.instances.push(ty)) + } + EntityType::Module(ty) => EntityIndex::Module(self.result.module.modules.push(ty)), + EntityType::Event(_) => unimplemented!(), + } + } + + fn gen_type_of_module(&mut self, module: usize) -> ModuleTypeIndex { + let module = &self.results[module].module; + let imports = module + .imports() + .map(|(s, field, ty)| { + assert!(field.is_none()); + (s.to_string(), ty) + }) + .collect(); + let exports = module + .exports + .iter() + .map(|(name, idx)| (name.clone(), module.type_of(*idx))) + .collect(); + + // FIXME(#2469): this instance/module signature insertion should likely + // be deduplicated. + let exports = self + .types + .instance_signatures + .push(InstanceSignature { exports }); + self.types + .module_signatures + .push(ModuleSignature { imports, exports }) + } } impl<'data> TargetEnvironment for ModuleEnvironment<'data> { @@ -267,13 +394,29 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn declare_type_module( &mut self, - imports: &[(&'data str, Option<&'data str>, EntityType)], + declared_imports: &[(&'data str, Option<&'data str>, EntityType)], exports: &[(&'data str, EntityType)], ) -> WasmResult<()> { - let imports = imports - .iter() - .map(|i| (i.0.to_string(), i.1.map(|s| s.to_string()), i.2.clone())) - .collect(); + let mut imports = indexmap::IndexMap::new(); + let mut instance_types = HashMap::new(); + for (module, field, ty) in declared_imports { + match field { + Some(field) => { + let idx = *instance_types + .entry(module) + .or_insert_with(|| self.types.instance_signatures.push(Default::default())); + self.types.instance_signatures[idx] + .exports + .insert(field.to_string(), ty.clone()); + if !imports.contains_key(*module) { + imports.insert(module.to_string(), EntityType::Instance(idx)); + } + } + None => { + imports.insert(module.to_string(), ty.clone()); + } + } + } let exports = exports .iter() .map(|e| (e.0.to_string(), e.1.clone())) @@ -344,8 +487,8 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn declare_func_import( &mut self, index: TypeIndex, - module: &str, - field: Option<&str>, + module: &'data str, + field: Option<&'data str>, ) -> WasmResult<()> { debug_assert_eq!( self.result.module.functions.len(), @@ -353,12 +496,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data "Imported functions must be declared first" ); let sig_index = self.result.module.types[index].unwrap_function(); - let func_index = self.result.module.functions.push(sig_index); - self.result.module.initializers.push(Initializer::Import { - module: module.to_owned(), - field: field.map(|s| s.to_owned()), - index: EntityIndex::Function(func_index), - }); + self.declare_import(module, field, EntityType::Function(sig_index)); self.result.module.num_imported_funcs += 1; self.result.debuginfo.wasm_file.imported_func_count += 1; Ok(()) @@ -367,21 +505,15 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn declare_table_import( &mut self, table: Table, - module: &str, - field: Option<&str>, + module: &'data str, + field: Option<&'data str>, ) -> WasmResult<()> { debug_assert_eq!( self.result.module.table_plans.len(), self.result.module.num_imported_tables, "Imported tables must be declared first" ); - let plan = TablePlan::for_table(table, &self.tunables); - let table_index = self.result.module.table_plans.push(plan); - self.result.module.initializers.push(Initializer::Import { - module: module.to_owned(), - field: field.map(|s| s.to_owned()), - index: EntityIndex::Table(table_index), - }); + self.declare_import(module, field, EntityType::Table(table)); self.result.module.num_imported_tables += 1; Ok(()) } @@ -389,8 +521,8 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn declare_memory_import( &mut self, memory: Memory, - module: &str, - field: Option<&str>, + module: &'data str, + field: Option<&'data str>, ) -> WasmResult<()> { debug_assert_eq!( self.result.module.memory_plans.len(), @@ -400,13 +532,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data if memory.shared { return Err(WasmError::Unsupported("shared memories".to_owned())); } - let plan = MemoryPlan::for_memory(memory, &self.tunables); - let memory_index = self.result.module.memory_plans.push(plan); - self.result.module.initializers.push(Initializer::Import { - module: module.to_owned(), - field: field.map(|s| s.to_owned()), - index: EntityIndex::Memory(memory_index), - }); + self.declare_import(module, field, EntityType::Memory(memory)); self.result.module.num_imported_memories += 1; Ok(()) } @@ -414,20 +540,15 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn declare_global_import( &mut self, global: Global, - module: &str, - field: Option<&str>, + module: &'data str, + field: Option<&'data str>, ) -> WasmResult<()> { debug_assert_eq!( self.result.module.globals.len(), self.result.module.num_imported_globals, "Imported globals must be declared first" ); - let global_index = self.result.module.globals.push(global); - self.result.module.initializers.push(Initializer::Import { - module: module.to_owned(), - field: field.map(|s| s.to_owned()), - index: EntityIndex::Global(global_index), - }); + self.declare_import(module, field, EntityType::Global(global)); self.result.module.num_imported_globals += 1; Ok(()) } @@ -439,12 +560,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data field: Option<&'data str>, ) -> WasmResult<()> { let signature = self.type_to_module_type(ty_index)?; - let module_index = self.result.module.modules.push(signature); - self.result.module.initializers.push(Initializer::Import { - module: module.to_owned(), - field: field.map(|s| s.to_owned()), - index: EntityIndex::Module(module_index), - }); + self.declare_import(module, field, EntityType::Module(signature)); Ok(()) } @@ -455,12 +571,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data field: Option<&'data str>, ) -> WasmResult<()> { let signature = self.type_to_instance_type(ty_index)?; - let instance_index = self.result.module.instances.push(signature); - self.result.module.initializers.push(Initializer::Import { - module: module.to_owned(), - field: field.map(|s| s.to_owned()), - index: EntityIndex::Instance(instance_index), - }); + self.declare_import(module, field, EntityType::Instance(signature)); Ok(()) } @@ -755,62 +866,34 @@ and for re-adding support for interface types you can see this issue: self.result.module.initializers.reserve(amount as usize); } - fn declare_module(&mut self, ty: TypeIndex) -> WasmResult<()> { - // Record the type signature of this module ... - let signature = self.type_to_module_type(ty)?; - self.result.module.modules.push(signature); - - // ... and then record that in the initialization steps of this module - // we're inserting this module into the module index space. At this - // point we don't know the final index of the module we're defining, so - // we leave a placeholder to get rewritten later. - let loc = self.result.module.initializers.len(); - self.result - .module - .initializers - .push(Initializer::DefineModule(usize::max_value())); - self.result.module_initializer_indexes.push(loc); - Ok(()) - } - - fn module_start(&mut self, index: usize) { - // Reset the contents of `self.result` for a new module that's getting - // translataed. - let mut prev = mem::replace(&mut self.result, ModuleTranslation::default()); - - // If this is a nested submodule then we record the final destination of - // the child in parent (we store `index` into `prev`) in the appropriate - // initialization slot as dicated by `num_modules_defined` (our index of - // iteration through the code section). - // Record that the `num_modules_defined`-th module is defined at index - // by updating the initializer entry. - if index > 0 { - let initializer_idx = prev.module_initializer_indexes[prev.num_modules_defined]; - prev.num_modules_defined += 1; - debug_assert!(match &prev.module.initializers[initializer_idx] { - Initializer::DefineModule(usize::MAX) => true, - _ => false, - }); - prev.module.initializers[initializer_idx] = Initializer::DefineModule(index); - self.result.module.parent = Some(self.cur); + fn module_start(&mut self) { + // If this is the first time this method is called, nothing to do. + if self.first_module { + self.first_module = false; + return; } - - // Update our current index counter and save our parent's translation - // where this current translation will end up, which we'll swap back as - // part of `module_end`. - self.cur = index; - assert_eq!(index, self.results.len()); - self.results.push(prev); + // Reset our internal state for a new module by saving the current + // module in `results`. + let in_progress = mem::replace(&mut self.result, ModuleTranslation::default()); + self.in_progress.push(in_progress); self.modules_to_be -= 1; } - fn module_end(&mut self, index: usize) { - assert!(self.result.num_modules_defined == self.result.module_initializer_indexes.len()); - - // Move our finished module into its final location, swapping it with - // what was this module's parent. - self.cur = self.result.module.parent.unwrap_or(0); - mem::swap(&mut self.result, &mut self.results[index]); + fn module_end(&mut self) { + let (record_initializer, done) = match self.in_progress.pop() { + Some(m) => (true, mem::replace(&mut self.result, m)), + None => (false, mem::take(&mut self.result)), + }; + self.results.push(done); + if record_initializer { + let index = self.results.len() - 1; + self.result + .module + .initializers + .push(Initializer::DefineModule(index)); + let sig = self.gen_type_of_module(index); + self.result.module.modules.push(sig); + } } fn reserve_instances(&mut self, amt: u32) { @@ -818,7 +901,12 @@ and for re-adding support for interface types you can see this issue: self.result.module.initializers.reserve(amt as usize); } - fn declare_instance(&mut self, module: ModuleIndex, args: Vec) -> WasmResult<()> { + fn declare_instance( + &mut self, + module: ModuleIndex, + args: Vec<(&'data str, EntityIndex)>, + ) -> WasmResult<()> { + let args = args.into_iter().map(|(s, i)| (s.to_string(), i)).collect(); // Record the type of this instance with the type signature of the // module we're instantiating and then also add an initializer which // records that we'll be adding to the instance index space here. @@ -839,29 +927,32 @@ and for re-adding support for interface types you can see this issue: // // Note that we don't add an initializer for this alias because // we statically know where all types point to. - Alias::ParentType(parent_idx) => { - let ty = self.results[self.cur].module.types[parent_idx]; + Alias::OuterType { + relative_depth, + index, + } => { + let module_idx = self.in_progress.len() - 1 - (relative_depth as usize); + let ty = self.in_progress[module_idx].module.types[index]; self.result.module.types.push(ty); } - // This is similar to types in that it's easy for us to record the - // type of the module that's being aliased, but we also need to add - // an initializer so during instantiation we can prepare the index - // space appropriately. - Alias::ParentModule(parent_idx) => { - let module_idx = self.results[self.cur].module.modules[parent_idx]; - self.result.module.modules.push(module_idx); - self.result - .module - .initializers - .push(Initializer::AliasParentModule(parent_idx)); + // FIXME(WebAssembly/module-linking#28) unsure how to implement this + // at this time, if we can alias imported modules it's a lot harder, + // otherwise we'll need to figure out how to translate `index` to a + // `usize` for a defined module (creating Initializer::DefineModule) + Alias::OuterModule { + relative_depth, + index, + } => { + drop((relative_depth, index)); + unimplemented!() } // This case is slightly more involved, we'll be recording all the // type information for each kind of entity, and then we also need // to record an initialization step to get the export from the // instance. - Alias::Child { instance, export } => { + Alias::InstanceExport { instance, export } => { let ty = self.result.module.instances[instance]; match &self.types.instance_signatures[ty].exports[export] { EntityType::Global(g) => { @@ -894,7 +985,10 @@ and for re-adding support for interface types you can see this issue: self.result .module .initializers - .push(Initializer::AliasInstanceExport { instance, export }) + .push(Initializer::AliasInstanceExport { + instance, + export: export.to_string(), + }) } } diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index 2cc157704a..17a0e811c7 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -13,12 +13,12 @@ arbitrary = { version = "0.4.1", features = ["derive"] } env_logger = "0.8.1" log = "0.4.8" rayon = "1.2.1" -wasmparser = "0.71" +wasmparser = "0.72" wasmprinter = "0.2.17" wasmtime = { path = "../wasmtime" } wasmtime-wast = { path = "../wast" } -wasm-encoder = "0.2" -wasm-smith = "0.3.0" +wasm-encoder = "0.4" +wasm-smith = "0.3.1" wasmi = "0.7.0" [dev-dependencies] diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index bebb4d2841..b101c15819 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -28,7 +28,7 @@ rayon = { version = "1.0", optional = true } region = "2.1.0" thiserror = "1.0.4" target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.71" +wasmparser = "0.72" more-asserts = "0.2.1" anyhow = "1.0" cfg-if = "1.0" diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index f739b79d1a..c0738c9c2c 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -105,8 +105,8 @@ impl CompilationArtifacts { pub fn build( compiler: &Compiler, data: &[u8], - ) -> Result<(Vec, TypeTables), SetupError> { - let (translations, types) = ModuleEnvironment::new( + ) -> Result<(usize, Vec, TypeTables), SetupError> { + let (main_module, translations, types) = ModuleEnvironment::new( compiler.frontend_config(), compiler.tunables(), compiler.features(), @@ -166,6 +166,7 @@ impl CompilationArtifacts { }) .collect::, SetupError>>()?; Ok(( + main_module, list, TypeTables { wasm_signatures: types.wasm_signatures, diff --git a/crates/lightbeam/Cargo.toml b/crates/lightbeam/Cargo.toml index 89a18280fa..9b5632384d 100644 --- a/crates/lightbeam/Cargo.toml +++ b/crates/lightbeam/Cargo.toml @@ -24,7 +24,7 @@ more-asserts = "0.2.1" smallvec = "1.6.1" thiserror = "1.0.9" typemap = "0.3" -wasmparser = "0.71" +wasmparser = "0.72" [dev-dependencies] lazy_static = "1.2" diff --git a/crates/lightbeam/wasmtime/Cargo.toml b/crates/lightbeam/wasmtime/Cargo.toml index 71f43c8a3c..08ab2bac17 100644 --- a/crates/lightbeam/wasmtime/Cargo.toml +++ b/crates/lightbeam/wasmtime/Cargo.toml @@ -13,6 +13,6 @@ edition = "2018" [dependencies] lightbeam = { path = "..", version = "0.22.0" } -wasmparser = "0.71" +wasmparser = "0.72" cranelift-codegen = { path = "../../../cranelift/codegen", version = "0.69.0" } wasmtime-environ = { path = "../../environ", version = "0.22.0" } diff --git a/crates/runtime/src/export.rs b/crates/runtime/src/export.rs index 0161e56a68..b9564e571a 100644 --- a/crates/runtime/src/export.rs +++ b/crates/runtime/src/export.rs @@ -1,14 +1,14 @@ use crate::vmcontext::{ VMCallerCheckedAnyfunc, VMContext, VMGlobalDefinition, VMMemoryDefinition, VMTableDefinition, }; -use crate::InstanceHandle; +use crate::RuntimeInstance; use std::any::Any; use std::ptr::NonNull; use wasmtime_environ::wasm::Global; use wasmtime_environ::{MemoryPlan, TablePlan}; /// The value of an export passed from one instance to another. -pub enum Export<'a> { +pub enum Export { /// A function export value. Function(ExportFunction), @@ -22,10 +22,10 @@ pub enum Export<'a> { Global(ExportGlobal), /// An instance - Instance(&'a InstanceHandle), + Instance(RuntimeInstance), /// A module - Module(&'a dyn Any), + Module(Box), } /// A function export value. @@ -38,8 +38,8 @@ pub struct ExportFunction { pub anyfunc: NonNull, } -impl<'a> From for Export<'a> { - fn from(func: ExportFunction) -> Export<'a> { +impl From for Export { + fn from(func: ExportFunction) -> Export { Export::Function(func) } } @@ -55,8 +55,8 @@ pub struct ExportTable { pub table: TablePlan, } -impl<'a> From for Export<'a> { - fn from(func: ExportTable) -> Export<'a> { +impl From for Export { + fn from(func: ExportTable) -> Export { Export::Table(func) } } @@ -72,8 +72,8 @@ pub struct ExportMemory { pub memory: MemoryPlan, } -impl<'a> From for Export<'a> { - fn from(func: ExportMemory) -> Export<'a> { +impl From for Export { + fn from(func: ExportMemory) -> Export { Export::Memory(func) } } @@ -89,8 +89,8 @@ pub struct ExportGlobal { pub global: Global, } -impl<'a> From for Export<'a> { - fn from(func: ExportGlobal) -> Export<'a> { +impl From for Export { + fn from(func: ExportGlobal) -> Export { Export::Global(func) } } diff --git a/crates/runtime/src/imports.rs b/crates/runtime/src/imports.rs index 1969630750..fe6031293e 100644 --- a/crates/runtime/src/imports.rs +++ b/crates/runtime/src/imports.rs @@ -1,8 +1,4 @@ use crate::vmcontext::{VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport}; -use crate::InstanceHandle; -use std::any::Any; -use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::wasm::{InstanceIndex, ModuleIndex}; /// Resolved import pointers. /// @@ -28,15 +24,4 @@ pub struct Imports<'a> { /// Resolved addresses for imported globals. pub globals: &'a [VMGlobalImport], - - /// Resolved imported instances. - pub instances: PrimaryMap, - - /// Resolved imported modules. - /// - /// Note that `Box` here is chosen to allow the embedder of this crate - /// to pick an appropriate representation of what module type should be. For - /// example for the `wasmtime` crate it's `wasmtime::Module` but that's not - /// defined way down here in this low crate. - pub modules: PrimaryMap>, } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 9ff7c4ee6d..1df1cfb640 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -14,6 +14,7 @@ use crate::vmcontext::{ VMSharedSignatureIndex, VMTableDefinition, VMTableImport, }; use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable}; +use indexmap::IndexMap; use memoffset::offset_of; use more_asserts::assert_lt; use std::alloc::{self, Layout}; @@ -22,17 +23,22 @@ use std::cell::RefCell; use std::collections::HashMap; use std::convert::TryFrom; use std::ptr::NonNull; +use std::rc::Rc; use std::sync::Arc; use std::{mem, ptr, slice}; use thiserror::Error; use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap}; use wasmtime_environ::wasm::{ DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, - ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, InstanceIndex, MemoryIndex, - ModuleIndex, SignatureIndex, TableElementType, TableIndex, WasmType, + ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, + TableElementType, TableIndex, WasmType, }; use wasmtime_environ::{ir, DataInitializer, Module, ModuleType, TableElements, VMOffsets}; +/// Runtime representation of an instance value, which erases all `Instance` +/// information since instances are just a collection of values. +pub type RuntimeInstance = Rc>; + /// A WebAssembly instance. /// /// This is repr(C) to ensure that the vmctx field is last. @@ -50,15 +56,6 @@ pub(crate) struct Instance { /// WebAssembly table data. tables: BoxedSlice, - /// Instances our module defined and their handles. - instances: PrimaryMap, - - /// Modules that are located in our index space. - /// - /// For now these are `Box` so the caller can define the type of what a - /// module looks like. - modules: PrimaryMap>, - /// Passive elements in this instantiation. As `elem.drop`s happen, these /// entries get removed. A missing entry is considered equivalent to an /// empty slice. @@ -266,18 +263,8 @@ impl Instance { self.vmctx() as *const VMContext as *mut VMContext } - /// Lookup an export with the given name. - pub fn lookup(&self, field: &str) -> Option { - let export = if let Some(export) = self.module.exports.get(field) { - export.clone() - } else { - return None; - }; - Some(self.lookup_by_declaration(&export)) - } - /// Lookup an export with the given export declaration. - pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export<'_> { + pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export { match export { EntityIndex::Function(index) => { let anyfunc = self.get_caller_checked_anyfunc(*index).unwrap(); @@ -326,8 +313,9 @@ impl Instance { } .into(), - EntityIndex::Instance(index) => Export::Instance(&self.instances[*index]), - EntityIndex::Module(index) => Export::Module(&*self.modules[*index]), + EntityIndex::Instance(_) | EntityIndex::Module(_) => { + panic!("can't use this api for modules/instances") + } } } @@ -855,8 +843,6 @@ impl InstanceHandle { passive_elements: Default::default(), passive_data, host_state, - instances: imports.instances, - modules: imports.modules, vmctx: VMContext {}, }; let layout = instance.alloc_layout(); @@ -1015,11 +1001,6 @@ impl InstanceHandle { self.instance().module() } - /// Lookup an export with the given name. - pub fn lookup(&self, field: &str) -> Option { - self.instance().lookup(field) - } - /// Lookup an export with the given export declaration. pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export { self.instance().lookup_by_declaration(export) diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 97e3934ef0..00539c36b4 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -37,7 +37,7 @@ pub mod libcalls; pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; -pub use crate::instance::{InstanceHandle, InstantiationError, LinkError}; +pub use crate::instance::{InstanceHandle, InstantiationError, LinkError, RuntimeInstance}; pub use crate::jit_int::GdbJitImageRegistration; pub use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; pub use crate::mmap::Mmap; diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 54da1d9873..5cb30d116f 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -16,7 +16,7 @@ wasmtime-jit = { path = "../jit", version = "0.22.0" } wasmtime-cache = { path = "../cache", version = "0.22.0", optional = true } wasmtime-profiling = { path = "../profiling", version = "0.22.0" } target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.71" +wasmparser = "0.72" anyhow = "1.0.19" region = "2.2.0" libc = "0.2" diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index 9a075e308e..a3cede9cb7 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -112,26 +112,25 @@ impl Extern { } } - pub(crate) fn from_wasmtime_export( - wasmtime_export: wasmtime_runtime::Export, - instance: StoreInstanceHandle, + pub(crate) unsafe fn from_wasmtime_export( + wasmtime_export: &wasmtime_runtime::Export, + store: &Store, ) -> Extern { match wasmtime_export { wasmtime_runtime::Export::Function(f) => { - Extern::Func(Func::from_wasmtime_function(f, instance)) + Extern::Func(Func::from_wasmtime_function(f, store)) } wasmtime_runtime::Export::Memory(m) => { - Extern::Memory(Memory::from_wasmtime_memory(m, instance)) + Extern::Memory(Memory::from_wasmtime_memory(m, store)) } wasmtime_runtime::Export::Global(g) => { - Extern::Global(Global::from_wasmtime_global(g, instance)) + Extern::Global(Global::from_wasmtime_global(g, store)) } wasmtime_runtime::Export::Table(t) => { - Extern::Table(Table::from_wasmtime_table(t, instance)) + Extern::Table(Table::from_wasmtime_table(t, store)) } wasmtime_runtime::Export::Instance(i) => { - let handle = unsafe { instance.store.existing_instance_handle(i.clone()) }; - Extern::Instance(Instance::from_wasmtime(handle)) + Extern::Instance(Instance::from_wasmtime(i, store)) } wasmtime_runtime::Export::Module(m) => { Extern::Module(m.downcast_ref::().unwrap().clone()) @@ -335,13 +334,13 @@ impl Global { Ok(()) } - pub(crate) fn from_wasmtime_global( - wasmtime_export: wasmtime_runtime::ExportGlobal, - instance: StoreInstanceHandle, + pub(crate) unsafe fn from_wasmtime_global( + wasmtime_export: &wasmtime_runtime::ExportGlobal, + store: &Store, ) -> Global { Global { - instance, - wasmtime_export, + instance: store.existing_vmctx(wasmtime_export.vmctx), + wasmtime_export: wasmtime_export.clone(), } } @@ -354,6 +353,10 @@ impl Global { from: self.wasmtime_export.definition, } } + + pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportGlobal { + &self.wasmtime_export + } } /// A WebAssembly `table`, or an array of values. @@ -576,13 +579,13 @@ impl Table { Ok(()) } - pub(crate) fn from_wasmtime_table( - wasmtime_export: wasmtime_runtime::ExportTable, - instance: StoreInstanceHandle, + pub(crate) unsafe fn from_wasmtime_table( + wasmtime_export: &wasmtime_runtime::ExportTable, + store: &Store, ) -> Table { Table { - instance, - wasmtime_export, + instance: store.existing_vmctx(wasmtime_export.vmctx), + wasmtime_export: wasmtime_export.clone(), } } @@ -596,6 +599,10 @@ impl Table { vmctx: self.wasmtime_export.vmctx, } } + + pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportTable { + &self.wasmtime_export + } } /// A WebAssembly linear memory. @@ -987,13 +994,13 @@ impl Memory { .ok_or_else(|| anyhow!("failed to grow memory")) } - pub(crate) fn from_wasmtime_memory( - wasmtime_export: wasmtime_runtime::ExportMemory, - instance: StoreInstanceHandle, + pub(crate) unsafe fn from_wasmtime_memory( + wasmtime_export: &wasmtime_runtime::ExportMemory, + store: &Store, ) -> Memory { Memory { - instance, - wasmtime_export, + instance: store.existing_vmctx(wasmtime_export.vmctx), + wasmtime_export: wasmtime_export.clone(), } } @@ -1007,6 +1014,10 @@ impl Memory { vmctx: self.wasmtime_export.vmctx, } } + + pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportMemory { + &self.wasmtime_export + } } /// A linear memory. This trait provides an interface for raw memory buffers which are used diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 630c5f736d..23bf36f73b 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -1,6 +1,6 @@ use crate::store::StoreInner; use crate::trampoline::StoreInstanceHandle; -use crate::{Extern, ExternRef, FuncType, Memory, Store, Trap, Val, ValType}; +use crate::{Extern, ExternRef, FuncType, Store, Trap, Val, ValType}; use anyhow::{bail, ensure, Context as _, Result}; use smallvec::{smallvec, SmallVec}; use std::cmp::max; @@ -9,8 +9,9 @@ use std::mem; use std::panic::{self, AssertUnwindSafe}; use std::ptr::{self, NonNull}; use std::rc::Weak; +use wasmtime_environ::wasm::EntityIndex; use wasmtime_runtime::{ - raise_user_trap, Export, InstanceHandle, VMContext, VMFunctionBody, VMSharedSignatureIndex, + raise_user_trap, InstanceHandle, VMContext, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, }; @@ -331,11 +332,8 @@ impl Func { debug_assert!( anyfunc.as_ref().type_index != wasmtime_runtime::VMSharedSignatureIndex::default() ); - - let instance_handle = wasmtime_runtime::InstanceHandle::from_vmctx(anyfunc.as_ref().vmctx); let export = wasmtime_runtime::ExportFunction { anyfunc }; - let instance = store.existing_instance_handle(instance_handle); - let f = Func::from_wasmtime_function(export, instance); + let f = Func::from_wasmtime_function(&export, store); Some(f) } @@ -649,24 +647,24 @@ impl Func { self.export.anyfunc } - pub(crate) fn from_wasmtime_function( - export: wasmtime_runtime::ExportFunction, - instance: StoreInstanceHandle, + pub(crate) unsafe fn from_wasmtime_function( + export: &wasmtime_runtime::ExportFunction, + store: &Store, ) -> Self { // Each function signature in a module should have a trampoline stored // on that module as well, so unwrap the result here since otherwise // it's a bug in wasmtime. - let trampoline = instance - .store + let anyfunc = export.anyfunc.as_ref(); + let trampoline = store .signatures() .borrow() - .lookup_shared(unsafe { export.anyfunc.as_ref().type_index }) + .lookup_shared(anyfunc.type_index) .expect("failed to retrieve trampoline from module") .1; Func { - instance, - export, + instance: store.existing_vmctx(anyfunc.vmctx), + export: export.clone(), trampoline, } } @@ -807,6 +805,10 @@ impl Func { } } } + + pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportFunction { + &self.export + } } impl fmt::Debug for Func { @@ -1506,10 +1508,16 @@ impl Caller<'_> { debug_assert!(self.store.upgrade().is_some()); let handle = Store::from_inner(self.store.upgrade()?).existing_instance_handle(instance); - let export = handle.lookup(name)?; - match export { - Export::Memory(m) => Some(Extern::Memory(Memory::from_wasmtime_memory(m, handle))), - Export::Function(f) => Some(Extern::Func(Func::from_wasmtime_function(f, handle))), + let index = handle.module().exports.get(name)?; + match index { + // Only allow memory/functions for now to emulate what interface + // types will once provide + EntityIndex::Memory(_) | EntityIndex::Function(_) => { + Some(Extern::from_wasmtime_export( + &handle.lookup_by_declaration(&index), + &handle.store, + )) + } _ => None, } } diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 0dff161052..34f87e5c0a 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -1,21 +1,17 @@ -use crate::trampoline::StoreInstanceHandle; use crate::types::matching; use crate::{ - Engine, Export, Extern, ExternType, Func, Global, InstanceType, Memory, Module, Store, Table, - Trap, + Engine, Export, Extern, Func, Global, InstanceType, Memory, Module, Store, Table, Trap, }; use anyhow::{bail, Context, Error, Result}; use std::mem; -use std::sync::Arc; +use std::rc::Rc; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::{ - EntityIndex, EntityType, FuncIndex, GlobalIndex, InstanceIndex, MemoryIndex, ModuleIndex, - TableIndex, + EntityIndex, FuncIndex, GlobalIndex, InstanceIndex, MemoryIndex, ModuleIndex, TableIndex, }; use wasmtime_environ::Initializer; -use wasmtime_jit::TypeTables; use wasmtime_runtime::{ - Imports, InstanceHandle, InstantiationError, StackMapRegistry, VMContext, + Imports, InstantiationError, RuntimeInstance, StackMapRegistry, VMContext, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport, }; @@ -36,8 +32,6 @@ use wasmtime_runtime::{ /// itself. /// * `type` - the type tables produced during compilation which /// `compiled_module`'s metadata references. -/// * `parent_modules` - this is the list of compiled modules the parent has. -/// This is only applicable on recursive instantiations. /// * `define_import` - this function, like the name implies, defines an import /// into the provided builder. The expected entity that it's defining is also /// passed in for the top-level case where type-checking is performed. This is @@ -45,9 +39,13 @@ use wasmtime_runtime::{ fn instantiate( store: &Store, module: &Module, - parent_modules: &PrimaryMap, - define_import: &mut dyn FnMut(&EntityIndex, &mut ImportsBuilder<'_>) -> Result<()>, -) -> Result { + define_import: &mut dyn FnMut( + &str, + Option<&str>, + &EntityIndex, + &mut ImportsBuilder<'_>, + ) -> Result<()>, +) -> Result { let compiled_module = module.compiled_module(); let env_module = compiled_module.module(); @@ -59,21 +57,9 @@ fn instantiate( // to fetching from the import list for the top-level module and // otherwise fetching from each nested instance's argument list for // submodules. - Initializer::Import { - index, - module, - field, - } => { - define_import(index, &mut imports).with_context(|| match field { - Some(name) => format!("incompatible import type for `{}::{}`", module, name), - None => format!("incompatible import type for `{}`", module), - })?; - } - - // This one's pretty easy, we're just picking up our parent's module - // and putting it into our own index space. - Initializer::AliasParentModule(idx) => { - imports.modules.push(parent_modules[*idx].clone()); + Initializer::Import { index, name, field } => { + define_import(name, field.as_deref(), index, &mut imports) + .with_context(|| format!("incompatible import type for `{}`", name))?; } // Turns out defining any kind of module is pretty easy, we're just @@ -95,18 +81,8 @@ fn instantiate( // handle comes from our same store, but this should be true because // we acquired the handle from an instance in the store. Initializer::AliasInstanceExport { instance, export } => { - let instance_ty = env_module.instances[*instance]; - let export_name = module.types().instance_signatures[instance_ty] - .exports - .get_index(*export) - .expect("validation bug - should be valid") - .0; - let handle = &imports.instances[*instance]; - let entity_index = &handle.module().exports[export_name]; - let item = Extern::from_wasmtime_export( - handle.lookup_by_declaration(entity_index), - unsafe { store.existing_instance_handle(handle.clone()) }, - ); + let export = &imports.instances[*instance][export]; + let item = unsafe { Extern::from_wasmtime_export(export, store) }; imports.push_extern(&item); } @@ -127,13 +103,13 @@ fn instantiate( // we're doing all of this in the context of our `Store` argument // above so we should be safe here. Initializer::Instantiate { module, args } => { - let mut args = args.iter(); let handle = instantiate( store, &imports.modules[*module], - &imports.modules, - &mut |_, builder| { - match *args.next().unwrap() { + &mut |name, field, _, builder| { + debug_assert!(field.is_none()); + let index = args.get(name).expect("should be present after validation"); + match *index { EntityIndex::Global(i) => { builder.globals.push(imports.globals[i]); } @@ -150,24 +126,17 @@ fn instantiate( builder.modules.push(imports.modules[i].clone()); } EntityIndex::Instance(i) => { - builder - .instances - .push(unsafe { imports.instances[i].clone() }); + builder.instances.push(imports.instances[i].clone()); } } Ok(()) }, )?; - imports.instances.push(unsafe { (*handle).clone() }); + imports.instances.push(handle); } } } - // With the above initialization done we've now acquired the final set of - // imports in all the right index spaces and everything. Time to carry on - // with the creation of our own instance. - let imports = imports.build(); - // Register the module just before instantiation to ensure we have a // trampoline registered for every signature and to preserve the module's // compiled JIT code within the `Store`. @@ -176,11 +145,11 @@ fn instantiate( let config = store.engine().config(); let instance = unsafe { let instance = compiled_module.instantiate( - imports, + imports.build(), &store.lookup_shared_signature(module.types()), config.memory_creator.as_ref().map(|a| a as _), store.interrupts(), - Box::new(module.types().clone()), + Box::new(()), store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, store.stack_map_registry() as *const StackMapRegistry as *mut _, )?; @@ -233,7 +202,29 @@ fn instantiate( } } - Ok(instance) + let exports = instance + .handle + .module() + .exports + .iter() + .map(|(name, index)| { + // Note that instances and modules are not handled by + // `wasmtime_runtime`, they're handled by us in this crate. That + // means we need to handle that here, otherwise we defer to the + // instance to load the values. + let item = match index { + EntityIndex::Instance(i) => { + wasmtime_runtime::Export::Instance(imports.instances[*i].clone()) + } + EntityIndex::Module(i) => { + wasmtime_runtime::Export::Module(Box::new(imports.modules[*i].clone())) + } + index => instance.handle.lookup_by_declaration(index), + }; + (name.clone(), item) + }) + .collect(); + Ok(Rc::new(exports)) } /// An instantiated WebAssembly module. @@ -254,7 +245,8 @@ fn instantiate( /// call any code or execute anything! #[derive(Clone)] pub struct Instance { - pub(crate) handle: StoreInstanceHandle, + pub(crate) store: Store, + pub(crate) items: RuntimeInstance, } impl Instance { @@ -316,7 +308,7 @@ impl Instance { bail!("cross-`Engine` instantiation is not currently supported"); } - // Perform some pre-flight checks before we get into the meat of + // Perform some pre-flight checks before we geet into the meat of // instantiation. let expected = module.compiled_module().module().imports().count(); if expected != imports.len() { @@ -329,32 +321,26 @@ impl Instance { } let mut imports = imports.iter(); - let handle = instantiate(store, module, &PrimaryMap::new(), &mut |idx, builder| { + let items = instantiate(store, module, &mut |_name, _field, idx, builder| { let import = imports.next().expect("already checked the length"); - builder.define_extern(idx, import) + builder.define_extern(idx, &import) })?; - Ok(Instance { handle }) + Ok(Instance::from_wasmtime(&items, store)) } - pub(crate) fn from_wasmtime(handle: StoreInstanceHandle) -> Instance { - Instance { handle } + pub(crate) fn from_wasmtime(handle: &RuntimeInstance, store: &Store) -> Instance { + Instance { + items: handle.clone(), + store: store.clone(), + } } /// Returns the type signature of this instance. pub fn ty(&self) -> InstanceType { let mut ty = InstanceType::new(); - let module = self.handle.module(); - let types = self - .handle - .host_state() - .downcast_ref::>() - .unwrap(); - for (name, index) in module.exports.iter() { - ty.add_named_export( - name, - ExternType::from_wasmtime(types, &module.type_of(*index)), - ); + for export in self.exports() { + ty.add_named_export(export.name(), export.ty()); } ty } @@ -364,16 +350,15 @@ impl Instance { /// This is the [`Store`] that generally serves as a sort of global cache /// for various instance-related things. pub fn store(&self) -> &Store { - &self.handle.store + &self.store } /// Returns the list of exported items from this [`Instance`]. pub fn exports<'instance>( &'instance self, ) -> impl ExactSizeIterator> + 'instance { - self.handle.exports().map(move |(name, entity_index)| { - let export = self.handle.lookup_by_declaration(entity_index); - let extern_ = Extern::from_wasmtime_export(export, self.handle.clone()); + self.items.iter().map(move |(name, item)| { + let extern_ = unsafe { Extern::from_wasmtime_export(item, &self.store) }; Export::new(name, extern_) }) } @@ -385,8 +370,8 @@ impl Instance { /// /// Returns `None` if there was no export named `name`. pub fn get_export(&self, name: &str) -> Option { - let export = self.handle.lookup(&name)?; - Some(Extern::from_wasmtime_export(export, self.handle.clone())) + let export = self.items.get(name)?; + Some(unsafe { Extern::from_wasmtime_export(export, &self.store) }) } /// Looks up an exported [`Func`] value by name. @@ -427,7 +412,7 @@ struct ImportsBuilder<'a> { tables: PrimaryMap, memories: PrimaryMap, globals: PrimaryMap, - instances: PrimaryMap, + instances: PrimaryMap, modules: PrimaryMap, module: &'a wasmtime_environ::Module, @@ -452,36 +437,7 @@ impl<'a> ImportsBuilder<'a> { fn define_extern(&mut self, expected: &EntityIndex, actual: &Extern) -> Result<()> { let expected_ty = self.module.type_of(*expected); - let compatible = match &expected_ty { - EntityType::Table(i) => match actual { - Extern::Table(e) => self.matcher.table(i, e), - _ => bail!("expected table, but found {}", actual.desc()), - }, - EntityType::Memory(i) => match actual { - Extern::Memory(e) => self.matcher.memory(i, e), - _ => bail!("expected memory, but found {}", actual.desc()), - }, - EntityType::Global(i) => match actual { - Extern::Global(e) => self.matcher.global(i, e), - _ => bail!("expected global, but found {}", actual.desc()), - }, - EntityType::Function(i) => match actual { - Extern::Func(e) => self.matcher.func(*i, e), - _ => bail!("expected func, but found {}", actual.desc()), - }, - EntityType::Instance(i) => match actual { - Extern::Instance(e) => self.matcher.instance(*i, e), - _ => bail!("expected instance, but found {}", actual.desc()), - }, - EntityType::Module(i) => match actual { - Extern::Module(e) => self.matcher.module(*i, e), - _ => bail!("expected module, but found {}", actual.desc()), - }, - EntityType::Event(_) => unimplemented!(), - }; - if !compatible { - bail!("{} types incompatible", actual.desc()); - } + self.matcher.extern_(&expected_ty, actual)?; self.push_extern(actual); Ok(()) } @@ -502,7 +458,7 @@ impl<'a> ImportsBuilder<'a> { } Extern::Instance(i) => { debug_assert!(Store::same(i.store(), self.matcher.store)); - self.instances.push(unsafe { (*i.handle).clone() }); + self.instances.push(i.items.clone()); } Extern::Module(m) => { self.modules.push(m.clone()); @@ -516,11 +472,40 @@ impl<'a> ImportsBuilder<'a> { globals: self.globals.values().as_slice(), memories: self.memories.values().as_slice(), functions: self.functions.values().as_slice(), - instances: mem::take(&mut self.instances), - modules: mem::take(&mut self.modules) - .into_iter() - .map(|(_, m)| Box::new(m) as Box<_>) - .collect(), + } + } +} + +/// An internal structure to this crate to build an `Instance` from a list of +/// items with names. This is intended to stay private for now, it'll need an +/// audit of APIs if publicly exported. +#[derive(Default)] +pub(crate) struct InstanceBuilder { + items: RuntimeInstance, +} + +impl InstanceBuilder { + pub(crate) fn new() -> InstanceBuilder { + InstanceBuilder::default() + } + + pub(crate) fn insert(&mut self, name: &str, item: impl Into) { + let items = Rc::get_mut(&mut self.items).unwrap(); + let export = match item.into() { + Extern::Func(i) => wasmtime_runtime::Export::Function(i.wasmtime_export().clone()), + Extern::Memory(i) => wasmtime_runtime::Export::Memory(i.wasmtime_export().clone()), + Extern::Table(i) => wasmtime_runtime::Export::Table(i.wasmtime_export().clone()), + Extern::Global(i) => wasmtime_runtime::Export::Global(i.wasmtime_export().clone()), + Extern::Instance(i) => wasmtime_runtime::Export::Instance(i.items.clone()), + Extern::Module(i) => wasmtime_runtime::Export::Module(Box::new(i.clone())), + }; + items.insert(name.to_string(), export); + } + + pub(crate) fn finish(self, store: &Store) -> Instance { + Instance { + store: store.clone(), + items: self.items, } } } diff --git a/crates/wasmtime/src/linker.rs b/crates/wasmtime/src/linker.rs index 763d0fa3cb..396327b5d5 100644 --- a/crates/wasmtime/src/linker.rs +++ b/crates/wasmtime/src/linker.rs @@ -1,3 +1,4 @@ +use crate::instance::InstanceBuilder; use crate::{ Extern, ExternType, Func, FuncType, GlobalType, ImportType, Instance, IntoFunc, Module, Store, Trap, @@ -654,7 +655,37 @@ impl Linker { }, kind: self.import_kind(import.ty()), }; - self.map.get(&key).cloned() + if let Some(result) = self.map.get(&key).cloned() { + return Some(result); + } + + // This is a key location where the module linking proposal is + // implemented. This logic allows single-level imports of an instance to + // get satisfied by multiple definitions of items within this `Linker`. + // + // The instance being import is iterated over to load the names from + // this `Linker` (recursively calling `get`). If anything isn't defined + // we return `None` since the entire value isn't defined. Otherwise when + // all values are loaded it's assembled into an `Instance` and + // returned`. + // + // Note that this isn't exactly the speediest implementation in the + // world. Ideally we would pre-create the `Instance` instead of creating + // it each time a module is instantiated. For now though while the + // module linking proposal is under development this should hopefully + // suffice. + if let ExternType::Instance(t) = import.ty() { + if import.name().is_none() { + let mut builder = InstanceBuilder::new(); + for export in t.exports() { + let item = self.get(&export.as_import(import.module()))?; + builder.insert(export.name(), item); + } + return Some(builder.finish(&self.store).into()); + } + } + + None } /// Returns all items defined for the `module` and `name` pair. diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index d30309c5fc..c78ae8a7f2 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -245,12 +245,14 @@ impl Module { /// ``` pub fn from_binary(engine: &Engine, binary: &[u8]) -> Result { #[cfg(feature = "cache")] - let (artifacts, types) = ModuleCacheEntry::new("wasmtime", engine.cache_config()) - .get_data((engine.compiler(), binary), |(compiler, binary)| { - CompilationArtifacts::build(compiler, binary) - })?; + let (main_module, artifacts, types) = + ModuleCacheEntry::new("wasmtime", engine.cache_config()) + .get_data((engine.compiler(), binary), |(compiler, binary)| { + CompilationArtifacts::build(compiler, binary) + })?; #[cfg(not(feature = "cache"))] - let (artifacts, types) = CompilationArtifacts::build(engine.compiler(), binary)?; + let (main_module, artifacts, types) = + CompilationArtifacts::build(engine.compiler(), binary)?; let modules = CompiledModule::from_artifacts_list( artifacts, @@ -261,7 +263,7 @@ impl Module { let types = Arc::new(types); Ok(Module { engine: engine.clone(), - index: 0, + index: main_module, data: Arc::new(ModuleData { types, modules }), }) } diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 6b314f6ac1..9c622faadf 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -13,8 +13,8 @@ use std::sync::Arc; use wasmtime_environ::wasm; use wasmtime_jit::{CompiledModule, ModuleCode, TypeTables}; use wasmtime_runtime::{ - InstanceHandle, RuntimeMemoryCreator, SignalHandler, StackMapRegistry, TrapInfo, VMExternRef, - VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, + InstanceHandle, RuntimeMemoryCreator, SignalHandler, StackMapRegistry, TrapInfo, VMContext, + VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, }; /// A `Store` is a collection of WebAssembly instances and host-defined items. @@ -234,6 +234,10 @@ impl Store { } } + pub(crate) unsafe fn existing_vmctx(&self, cx: *mut VMContext) -> StoreInstanceHandle { + self.existing_instance_handle(InstanceHandle::from_vmctx(cx)) + } + pub(crate) fn weak(&self) -> Weak { Rc::downgrade(&self.inner) } diff --git a/crates/wasmtime/src/trampoline/global.rs b/crates/wasmtime/src/trampoline/global.rs index 00a91bdb91..bb20bb1569 100644 --- a/crates/wasmtime/src/trampoline/global.rs +++ b/crates/wasmtime/src/trampoline/global.rs @@ -48,7 +48,7 @@ pub fn create_global(store: &Store, gt: &GlobalType, val: Val) -> Result Result unsafe { *(*g.definition).as_externref_mut() = Some(x.inner); }, diff --git a/crates/wasmtime/src/trampoline/mod.rs b/crates/wasmtime/src/trampoline/mod.rs index e8c836b2ec..4772d7481a 100644 --- a/crates/wasmtime/src/trampoline/mod.rs +++ b/crates/wasmtime/src/trampoline/mod.rs @@ -16,6 +16,7 @@ use crate::{FuncType, GlobalType, MemoryType, Store, TableType, Trap, Val}; use anyhow::Result; use std::any::Any; use std::ops::Deref; +use wasmtime_environ::wasm; use wasmtime_runtime::{InstanceHandle, VMContext, VMFunctionBody, VMTrampoline}; /// A wrapper around `wasmtime_runtime::InstanceHandle` which pairs it with the @@ -55,7 +56,8 @@ pub fn generate_func_export( VMTrampoline, )> { let (instance, trampoline) = create_handle_with_function(ft, func, store)?; - match instance.lookup("").expect("trampoline export") { + let idx = wasm::EntityIndex::Function(wasm::FuncIndex::from_u32(0)); + match instance.lookup_by_declaration(&idx) { wasmtime_runtime::Export::Function(f) => Ok((instance, f, trampoline)), _ => unreachable!(), } @@ -72,7 +74,8 @@ pub unsafe fn generate_raw_func_export( state: Box, ) -> Result<(StoreInstanceHandle, wasmtime_runtime::ExportFunction)> { let instance = func::create_handle_with_raw_function(ft, func, trampoline, store, state)?; - match instance.lookup("").expect("trampoline export") { + let idx = wasm::EntityIndex::Function(wasm::FuncIndex::from_u32(0)); + match instance.lookup_by_declaration(&idx) { wasmtime_runtime::Export::Function(f) => Ok((instance, f)), _ => unreachable!(), } @@ -84,7 +87,8 @@ pub fn generate_global_export( val: Val, ) -> Result<(StoreInstanceHandle, wasmtime_runtime::ExportGlobal)> { let instance = create_global(store, gt, val)?; - match instance.lookup("").expect("global export") { + let idx = wasm::EntityIndex::Global(wasm::GlobalIndex::from_u32(0)); + match instance.lookup_by_declaration(&idx) { wasmtime_runtime::Export::Global(g) => Ok((instance, g)), _ => unreachable!(), } @@ -95,7 +99,8 @@ pub fn generate_memory_export( m: &MemoryType, ) -> Result<(StoreInstanceHandle, wasmtime_runtime::ExportMemory)> { let instance = create_handle_with_memory(store, m)?; - match instance.lookup("").expect("memory export") { + let idx = wasm::EntityIndex::Memory(wasm::MemoryIndex::from_u32(0)); + match instance.lookup_by_declaration(&idx) { wasmtime_runtime::Export::Memory(m) => Ok((instance, m)), _ => unreachable!(), } @@ -106,7 +111,8 @@ pub fn generate_table_export( t: &TableType, ) -> Result<(StoreInstanceHandle, wasmtime_runtime::ExportTable)> { let instance = create_handle_with_table(store, t)?; - match instance.lookup("").expect("table export") { + let idx = wasm::EntityIndex::Table(wasm::TableIndex::from_u32(0)); + match instance.lookup_by_declaration(&idx) { wasmtime_runtime::Export::Table(t) => Ok((instance, t)), _ => unreachable!(), } diff --git a/crates/wasmtime/src/trampoline/table.rs b/crates/wasmtime/src/trampoline/table.rs index 7c451f5cb0..c151fca51d 100644 --- a/crates/wasmtime/src/trampoline/table.rs +++ b/crates/wasmtime/src/trampoline/table.rs @@ -23,6 +23,7 @@ pub fn create_handle_with_table(store: &Store, table: &TableType) -> Result, ty: ExternType) { + pub(crate) fn add_named_import(&mut self, module: &str, field: Option<&str>, ty: ExternType) { self.imports .push((module.to_string(), field.map(|f| f.to_string()), ty)); } /// Returns the list of imports associated with this module type. pub fn imports(&self) -> impl ExactSizeIterator> { - self.imports.iter().map(|(module, name, ty)| ImportType { - module, - name: name.as_deref(), + self.imports.iter().map(|(name, field, ty)| ImportType { + module: name, + name: field.as_deref(), ty: EntityOrExtern::Extern(ty), }) } @@ -506,13 +506,7 @@ impl ModuleType { imports: ty .imports .iter() - .map(|(m, name, ty)| { - ( - m.to_string(), - name.as_ref().map(|n| n.to_string()), - ExternType::from_wasmtime(types, ty), - ) - }) + .map(|(m, ty)| (m.to_string(), None, ExternType::from_wasmtime(types, ty))) .collect(), } } @@ -615,8 +609,9 @@ impl<'module> ImportType<'module> { /// Returns the field name of the module that this import is expected to /// come from. /// - /// Note that the name can be `None` for the module linking proposal. If the - /// module linking proposal is not enabled it's safe to unwrap this. + /// Note that this is optional due to the module linking proposal. If the + /// module linking proposal is enabled this is always `None`, otherwise this + /// is always `Some`. pub fn name(&self) -> Option<&'module str> { self.name } @@ -683,6 +678,17 @@ impl<'module> ExportType<'module> { EntityOrExtern::Extern(e) => (*e).clone(), } } + + pub(crate) fn as_import<'a>(&self, module: &'a str) -> ImportType<'a> + where + 'module: 'a, + { + ImportType { + module, + name: Some(self.name), + ty: self.ty.clone(), + } + } } impl<'module> fmt::Debug for ExportType<'module> { diff --git a/crates/wasmtime/src/types/matching.rs b/crates/wasmtime/src/types/matching.rs index 018ab35fd9..a767bbb7e1 100644 --- a/crates/wasmtime/src/types/matching.rs +++ b/crates/wasmtime/src/types/matching.rs @@ -1,5 +1,5 @@ -use crate::Store; -use std::sync::Arc; +use crate::{Extern, Store}; +use anyhow::{bail, Context, Result}; use wasmtime_environ::wasm::{ EntityType, Global, InstanceTypeIndex, Memory, ModuleTypeIndex, SignatureIndex, Table, }; @@ -11,22 +11,27 @@ pub struct MatchCx<'a> { } impl MatchCx<'_> { - pub fn global(&self, expected: &Global, actual: &crate::Global) -> bool { + pub fn global(&self, expected: &Global, actual: &crate::Global) -> Result<()> { self.global_ty(expected, actual.wasmtime_ty()) } - fn global_ty(&self, expected: &Global, actual: &Global) -> bool { - expected.ty == actual.ty + fn global_ty(&self, expected: &Global, actual: &Global) -> Result<()> { + if expected.ty == actual.ty && expected.wasm_ty == actual.wasm_ty && expected.mutability == actual.mutability + { + Ok(()) + } else { + bail!("global types incompatible") + } } - pub fn table(&self, expected: &Table, actual: &crate::Table) -> bool { + pub fn table(&self, expected: &Table, actual: &crate::Table) -> Result<()> { self.table_ty(expected, actual.wasmtime_ty()) } - fn table_ty(&self, expected: &Table, actual: &Table) -> bool { - expected.wasm_ty == actual.wasm_ty + fn table_ty(&self, expected: &Table, actual: &Table) -> Result<()> { + if expected.wasm_ty == actual.wasm_ty && expected.ty == actual.ty && expected.minimum <= actual.minimum && match expected.maximum { @@ -36,14 +41,19 @@ impl MatchCx<'_> { }, None => true, } + { + Ok(()) + } else { + bail!("table types incompatible") + } } - pub fn memory(&self, expected: &Memory, actual: &crate::Memory) -> bool { + pub fn memory(&self, expected: &Memory, actual: &crate::Memory) -> Result<()> { self.memory_ty(expected, actual.wasmtime_ty()) } - fn memory_ty(&self, expected: &Memory, actual: &Memory) -> bool { - expected.shared == actual.shared + fn memory_ty(&self, expected: &Memory, actual: &Memory) -> Result<()> { + if expected.shared == actual.shared && expected.minimum <= actual.minimum && match expected.maximum { Some(expected) => match actual.maximum { @@ -52,10 +62,15 @@ impl MatchCx<'_> { }, None => true, } + { + Ok(()) + } else { + bail!("memory types incompatible") + } } - pub fn func(&self, expected: SignatureIndex, actual: &crate::Func) -> bool { - match self + pub fn func(&self, expected: SignatureIndex, actual: &crate::Func) -> Result<()> { + let matches = match self .store .signatures() .borrow() @@ -65,31 +80,50 @@ impl MatchCx<'_> { // If our expected signature isn't registered, then there's no way // that `actual` can match it. None => false, + }; + if matches { + Ok(()) + } else { + bail!("function types incompatible") } } - pub fn instance(&self, expected: InstanceTypeIndex, actual: &crate::Instance) -> bool { - let module = actual.handle.module(); - self.exports_match( - expected, - actual - .handle - .host_state() - .downcast_ref::>() - .unwrap(), - |name| module.exports.get(name).map(|idx| module.type_of(*idx)), - ) + pub fn instance(&self, expected: InstanceTypeIndex, actual: &crate::Instance) -> Result<()> { + for (name, expected) in self.types.instance_signatures[expected].exports.iter() { + match actual.items.get(name) { + Some(item) => { + let item = unsafe { Extern::from_wasmtime_export(item, self.store) }; + self.extern_(expected, &item) + .with_context(|| format!("instance export {:?} incompatible", name))?; + } + None => bail!("instance type missing export {:?}", name), + } + } + Ok(()) } /// Validates that the type signature of `actual` matches the `expected` /// module type signature. - pub fn module(&self, expected: ModuleTypeIndex, actual: &crate::Module) -> bool { + pub fn module(&self, expected: ModuleTypeIndex, actual: &crate::Module) -> Result<()> { + // This should only ever be invoked with module linking, and this is an + // early check that our `field` assertion below should always work as + // well. + assert!(self.store.engine().config().features.module_linking); + let expected_sig = &self.types.module_signatures[expected]; let module = actual.compiled_module().module(); - self.imports_match(expected, actual.types(), module.imports()) - && self.exports_match(expected_sig.exports, actual.types(), |name| { - module.exports.get(name).map(|idx| module.type_of(*idx)) - }) + self.imports_match( + expected, + actual.types(), + module.imports().map(|(name, field, ty)| { + assert!(field.is_none()); // should be true if module linking is enabled + (name, ty) + }), + )?; + self.exports_match(expected_sig.exports, actual.types(), |name| { + module.exports.get(name).map(|idx| module.type_of(*idx)) + })?; + Ok(()) } /// Validates that the `actual_imports` list of module imports matches the @@ -100,19 +134,25 @@ impl MatchCx<'_> { &self, expected: ModuleTypeIndex, actual_types: &TypeTables, - mut actual_imports: impl Iterator, EntityType)>, - ) -> bool { + actual_imports: impl Iterator, + ) -> Result<()> { + // Imports match if all of the actual imports are satisfied by the + // expected set of imports. Note that we're reversing the order of the + // subtytpe matching here too. let expected_sig = &self.types.module_signatures[expected]; - for (_, _, expected) in expected_sig.imports.iter() { - let (_, _, ty) = match actual_imports.next() { - Some(e) => e, - None => return false, + for (name, actual_ty) in actual_imports { + let expected_ty = match expected_sig.imports.get(name) { + Some(ty) => ty, + None => bail!("expected type doesn't import {:?}", name), }; - if !self.extern_ty_matches(expected, &ty, actual_types) { - return false; + MatchCx { + types: actual_types, + store: self.store, } + .extern_ty_matches(&actual_ty, expected_ty, self.types) + .with_context(|| format!("module import {:?} incompatible", name))?; } - actual_imports.next().is_none() + Ok(()) } /// Validates that all exports in `expected` are defined by `lookup` within @@ -122,16 +162,19 @@ impl MatchCx<'_> { expected: InstanceTypeIndex, actual_types: &TypeTables, lookup: impl Fn(&str) -> Option, - ) -> bool { + ) -> Result<()> { // The `expected` type must be a subset of `actual`, meaning that all // names in `expected` must be present in `actual`. Note that we do // name-based lookup here instead of index-based lookup. - self.types.instance_signatures[expected].exports.iter().all( - |(name, expected)| match lookup(name) { - Some(ty) => self.extern_ty_matches(expected, &ty, actual_types), - None => false, - }, - ) + for (name, expected) in self.types.instance_signatures[expected].exports.iter() { + match lookup(name) { + Some(ty) => self + .extern_ty_matches(expected, &ty, actual_types) + .with_context(|| format!("export {:?} incompatible", name))?, + None => bail!("failed to find export {:?}", name), + } + } + Ok(()) } /// Validates that the `expected` entity matches the `actual_ty` defined @@ -141,34 +184,49 @@ impl MatchCx<'_> { expected: &EntityType, actual_ty: &EntityType, actual_types: &TypeTables, - ) -> bool { + ) -> Result<()> { + let actual_desc = match actual_ty { + EntityType::Global(_) => "global", + EntityType::Module(_) => "module", + EntityType::Memory(_) => "memory", + EntityType::Event(_) => "event", + EntityType::Instance(_) => "instance", + EntityType::Table(_) => "table", + EntityType::Function(_) => "function", + }; match expected { EntityType::Global(expected) => match actual_ty { EntityType::Global(actual) => self.global_ty(expected, actual), - _ => false, + _ => bail!("expected global, but found {}", actual_desc), }, EntityType::Table(expected) => match actual_ty { EntityType::Table(actual) => self.table_ty(expected, actual), - _ => false, + _ => bail!("expected table, but found {}", actual_desc), }, EntityType::Memory(expected) => match actual_ty { EntityType::Memory(actual) => self.memory_ty(expected, actual), - _ => false, + _ => bail!("expected memory, but found {}", actual_desc), }, EntityType::Function(expected) => match *actual_ty { EntityType::Function(actual) => { - self.types.wasm_signatures[*expected] == actual_types.wasm_signatures[actual] + if self.types.wasm_signatures[*expected] == actual_types.wasm_signatures[actual] + { + Ok(()) + } else { + bail!("function types incompatible") + } } - _ => false, + _ => bail!("expected function, but found {}", actual_desc), }, EntityType::Instance(expected) => match actual_ty { EntityType::Instance(actual) => { let sig = &actual_types.instance_signatures[*actual]; self.exports_match(*expected, actual_types, |name| { sig.exports.get(name).cloned() - }) + })?; + Ok(()) } - _ => false, + _ => bail!("expected instance, but found {}", actual_desc), }, EntityType::Module(expected) => match actual_ty { EntityType::Module(actual) => { @@ -180,14 +238,48 @@ impl MatchCx<'_> { self.imports_match( *expected, actual_types, - actual_module_sig.imports.iter().map(|(module, field, ty)| { - (module.as_str(), field.as_deref(), ty.clone()) - }), - ) && self.exports_match(expected_module_sig.exports, actual_types, |name| { + actual_module_sig + .imports + .iter() + .map(|(module, ty)| (module.as_str(), ty.clone())), + )?; + self.exports_match(expected_module_sig.exports, actual_types, |name| { actual_instance_sig.exports.get(name).cloned() - }) + })?; + Ok(()) } - _ => false, + _ => bail!("expected module, but found {}", actual_desc), + }, + EntityType::Event(_) => unimplemented!(), + } + } + + /// Validates that the `expected` type matches the type of `actual` + pub fn extern_(&self, expected: &EntityType, actual: &Extern) -> Result<()> { + match expected { + EntityType::Global(expected) => match actual { + Extern::Global(actual) => self.global(expected, actual), + _ => bail!("expected global, but found {}", actual.desc()), + }, + EntityType::Table(expected) => match actual { + Extern::Table(actual) => self.table(expected, actual), + _ => bail!("expected table, but found {}", actual.desc()), + }, + EntityType::Memory(expected) => match actual { + Extern::Memory(actual) => self.memory(expected, actual), + _ => bail!("expected memory, but found {}", actual.desc()), + }, + EntityType::Function(expected) => match actual { + Extern::Func(actual) => self.func(*expected, actual), + _ => bail!("expected func, but found {}", actual.desc()), + }, + EntityType::Instance(expected) => match actual { + Extern::Instance(actual) => self.instance(*expected, actual), + _ => bail!("expected instance, but found {}", actual.desc()), + }, + EntityType::Module(expected) => match actual { + Extern::Module(actual) => self.module(*expected, actual), + _ => bail!("expected module, but found {}", actual.desc()), }, EntityType::Event(_) => unimplemented!(), } diff --git a/crates/wast/Cargo.toml b/crates/wast/Cargo.toml index bbe1277f4c..97e660d0d1 100644 --- a/crates/wast/Cargo.toml +++ b/crates/wast/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] anyhow = "1.0.19" wasmtime = { path = "../wasmtime", version = "0.22.0", default-features = false } -wast = "29.0.0" +wast = "31.0.0" [badges] maintenance = { status = "actively-developed" } diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 73bd003e1e..df6905a23f 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -17,7 +17,7 @@ target-lexicon = "0.11" peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing", optional = true } wasmtime = { path = "../crates/wasmtime" } wasmtime-fuzzing = { path = "../crates/fuzzing" } -wasm-smith = "0.3.0" +wasm-smith = "0.3.1" [features] experimental_x64 = ["wasmtime-fuzzing/experimental_x64"] diff --git a/fuzz/fuzz_targets/instantiate-swarm.rs b/fuzz/fuzz_targets/instantiate-swarm.rs index 092a32e925..d9c5bdb645 100644 --- a/fuzz/fuzz_targets/instantiate-swarm.rs +++ b/fuzz/fuzz_targets/instantiate-swarm.rs @@ -2,12 +2,13 @@ use libfuzzer_sys::fuzz_target; use std::time::Duration; -use wasm_smith::{ConfiguredModule, SwarmConfig}; +use wasm_smith::{Config, ConfiguredModule, SwarmConfig}; use wasmtime::Strategy; use wasmtime_fuzzing::oracles; fuzz_target!(|module: ConfiguredModule| { let mut cfg = wasmtime_fuzzing::fuzz_default_config(Strategy::Auto).unwrap(); cfg.wasm_multi_memory(true); + cfg.wasm_module_linking(module.config().module_linking_enabled()); oracles::instantiate_with_config(&module.to_bytes(), true, cfg, Some(Duration::from_secs(20))); }); diff --git a/src/obj.rs b/src/obj.rs index e957fb6afd..529d1c3c6d 100644 --- a/src/obj.rs +++ b/src/obj.rs @@ -65,7 +65,7 @@ pub fn compile_to_obj( ); let environ = ModuleEnvironment::new(compiler.isa().frontend_config(), &tunables, &features); - let (mut translation, types) = environ + let (_main_module, mut translation, types) = environ .translate(wasm) .context("failed to translate module")?; assert_eq!(translation.len(), 1); diff --git a/tests/all/module_linking.rs b/tests/all/module_linking.rs index f352eb7f54..10ccfc098d 100644 --- a/tests/all/module_linking.rs +++ b/tests/all/module_linking.rs @@ -105,8 +105,13 @@ fn imports_exports() -> Result<()> { assert_eq!(i.len(), 1); let import = i.next().unwrap(); assert_eq!(import.module(), ""); - assert_eq!(import.name(), Some("a")); - let module_ty = match import.ty() { + assert_eq!(import.name(), None); + let instance_ty = match import.ty() { + ExternType::Instance(t) => t, + _ => panic!("unexpected type"), + }; + assert_eq!(instance_ty.exports().len(), 1); + let module_ty = match instance_ty.exports().next().unwrap().ty() { ExternType::Module(m) => m, _ => panic!("unexpected type"), }; @@ -148,8 +153,13 @@ fn imports_exports() -> Result<()> { assert_eq!(i.len(), 1); let import = i.next().unwrap(); assert_eq!(import.module(), ""); - assert_eq!(import.name(), Some("b")); + assert_eq!(import.name(), None); let instance_ty = match import.ty() { + ExternType::Instance(t) => t, + _ => panic!("unexpected type"), + }; + assert_eq!(instance_ty.exports().len(), 1); + let instance_ty = match instance_ty.exports().next().unwrap().ty() { ExternType::Instance(m) => m, _ => panic!("unexpected type"), }; diff --git a/tests/misc_testsuite/module-linking/alias.wast b/tests/misc_testsuite/module-linking/alias.wast index 0c69b4084c..f9d2b2f3fd 100644 --- a/tests/misc_testsuite/module-linking/alias.wast +++ b/tests/misc_testsuite/module-linking/alias.wast @@ -7,7 +7,7 @@ (instance $a (instantiate $m)) (func (export "get") (result i32) - call $a.$foo) + call (func $a "foo")) ) (assert_return (invoke "get") (i32.const 1)) @@ -19,7 +19,7 @@ (instance $a (instantiate $m)) (func (export "get") (result i32) - global.get $a.$g) + global.get (global $a "g")) ) (assert_return (invoke "get") (i32.const 2)) @@ -30,7 +30,7 @@ (data (i32.const 0) "\03\00\00\00") ) (instance $a (instantiate $m)) - (alias (instance $a) (memory $m)) + (alias $m (memory $a "m")) (func (export "get") (result i32) i32.const 0 @@ -50,7 +50,7 @@ (func (export "get") (result i32) i32.const 0 - call_indirect $a.$t (result i32)) + call_indirect (table $a "t") (result i32)) ) (assert_return (invoke "get") (i32.const 4)) @@ -62,11 +62,10 @@ i32.const 5)) ) (instance $a (instantiate $m)) - (instance $b (instantiate $a.$sub)) - (alias $b.$f (instance $b) (func 0)) + (instance $b (instantiate (module $a "module"))) (func (export "get") (result i32) - call $b.$f) + call (func $b "")) ) (assert_return (invoke "get") (i32.const 5)) @@ -79,11 +78,9 @@ (instance $i (export "") (instantiate $sub)) ) (instance $a (instantiate $m)) - (alias $a.$i (instance $a) (instance 0)) - (alias $a.$i.$f (instance $a.$i) (func 0)) (func (export "get") (result i32) - call $a.$i.$f) + call (func $a "" "")) ) (assert_return (invoke "get") (i32.const 6)) @@ -91,48 +88,51 @@ (module (type $t (func)) (module $m - (func $f (type $t)) + (func $f (type outer 0 $t)) ) (instance $a (instantiate $m)) ) ;; alias parent -- module +(; TODO (module (module $a) (module $m - (instance (instantiate $a)) + (instance (instantiate (module outer 0 $a))) ) (instance (instantiate $m)) ) +;) ;; The alias, import, type, module, and instance sections can all be interleaved -(module +(module $ROOT (module $a) (type $t (func)) (module $m ;; alias - (alias $thunk parent (type $t)) + (alias $thunk (type outer 0 $t)) ;; import (import "" "" (func (type $thunk))) ;; module (referencing parent type) (module - (func (type $thunk)) + (func (type outer $m $thunk)) + (func (type outer $ROOT $t)) ) ;; type (type $thunk2 (func)) ;; module (referencing previous alias) (module $m2 - (func (export "") (type $thunk2)) + (func (export "") (type outer $m $thunk2)) ) ;; instance (instance $i (instantiate $m2)) ;; alias that instance - (alias $my_f (instance $i) (func 0)) + (alias $my_f (func $i "")) ;; module (module $m3 (import "" (func))) ;; use our aliased function to create the module - (instance $i2 (instantiate $m3 (func $my_f))) + (instance $i2 (instantiate $m3 "" (func $my_f))) ;; module (module $m4 (import "" (func))) @@ -141,5 +141,5 @@ ;; instantiate the above module (module $smol (func $f (export ""))) (instance $smol (instantiate $smol)) - (instance (instantiate $m (func $smol.$f))) + (instance (instantiate $m "" (instance $smol))) ) diff --git a/tests/misc_testsuite/module-linking/import-subtyping.wast b/tests/misc_testsuite/module-linking/import-subtyping.wast index 4ac1658070..2055f848f3 100644 --- a/tests/misc_testsuite/module-linking/import-subtyping.wast +++ b/tests/misc_testsuite/module-linking/import-subtyping.wast @@ -9,22 +9,36 @@ (module (import "a" "m" (module)) +) +(module (import "a" "m" (module (export "" (func)))) +) +(module (import "a" "m" (module (export "a" (func)))) +) +(module (import "a" "m" (module (export "b" (global i32)))) +) +(module (import "a" "m" (module (export "" (func)) (export "a" (func)) )) +) +(module (import "a" "m" (module (export "a" (func)) (export "" (func)) )) +) +(module (import "a" "m" (module (export "a" (func)) (export "" (func)) (export "b" (global i32)) )) +) +(module (import "a" "m" (module (export "b" (global i32)) (export "a" (func)) @@ -37,64 +51,60 @@ (module (export "m") (func (export "")))) -(module - (import "a" "m" (module)) - (import "a" "m" (module (export "" (func)))) -) +(module (import "a" "m" (module))) +(module (import "a" "m" (module (export "" (func))))) (assert_unlinkable (module (import "a" "m" (module (export "" (func (param i32)))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (func (result i32)))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (global i32))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (table 1 funcref))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (memory 1))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (module))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (instance))))) - "module types incompatible") + "incompatible import type for `a`") (module $a (module (export "m") (global (export "") i32 (i32.const 0)))) ;; globals -(module - (import "a" "m" (module)) - (import "a" "m" (module (export "" (global i32)))) -) +(module (import "a" "m" (module))) +(module (import "a" "m" (module (export "" (global i32))))) (assert_unlinkable (module (import "a" "m" (module (export "" (global (mut i32))))) ) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (global f32))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (func))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (table 1 funcref))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (memory 1))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (module))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (instance))))) - "module types incompatible") + "incompatible import type for `a`") ;; tables (module $a @@ -105,40 +115,52 @@ ) (module (import "a" "m" (module)) +) +(module (import "a" "m" (module (export "" (table 1 funcref)))) +) +(module (import "a" "m" (module (export "" (table 0 funcref)))) +) +(module (import "a" "m" (module (export "max" (table 1 10 funcref)))) +) +(module (import "a" "m" (module (export "max" (table 0 10 funcref)))) +) +(module (import "a" "m" (module (export "max" (table 0 11 funcref)))) +) +(module (import "a" "m" (module (export "max" (table 0 funcref)))) ) (assert_unlinkable (module (import "a" "m" (module (export "" (global f32))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (func))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (table 2 funcref))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (table 1 10 funcref))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "max" (table 2 10 funcref))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "max" (table 1 9 funcref))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (memory 1))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (module))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (instance))))) - "module types incompatible") + "incompatible import type for `a`") ;; memories (module $a @@ -149,40 +171,52 @@ ) (module (import "a" "m" (module)) +) +(module (import "a" "m" (module (export "" (memory 1)))) +) +(module (import "a" "m" (module (export "" (memory 0)))) +) +(module (import "a" "m" (module (export "max" (memory 1 10)))) +) +(module (import "a" "m" (module (export "max" (memory 0 10)))) +) +(module (import "a" "m" (module (export "max" (memory 0 11)))) +) +(module (import "a" "m" (module (export "max" (memory 0)))) ) (assert_unlinkable (module (import "a" "m" (module (export "" (global f32))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (func))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (table 1 funcref))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (memory 2))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (memory 1 10))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "max" (memory 2 10))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "max" (memory 2))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (module))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (instance))))) - "module types incompatible") + "incompatible import type for `a`") ;; modules (module $a @@ -206,72 +240,102 @@ ) ;; import a mixture (module (export "e") - (import "" (func)) - (import "" (func)) - (import "" (global i32)) + (import "a" (func)) + (import "b" (func)) + (import "c" (global i32)) ) ) ) (module (import "a" "m" (module)) +) +(module (import "a" "m" (module (export "a" (module)))) +) +(module (import "a" "m" (module (export "b" (module)))) +) +(module (import "a" "m" (module (export "b" (module (export "" (func)))))) +) +(module (import "a" "m" (module (export "c" (module)))) +) +(module (import "a" "m" (module (export "c" (module (export "a" (func)) )))) +) +(module (import "a" "m" (module (export "c" (module (export "a" (func)) (export "b" (func (result i32))) )))) +) +(module (import "a" "m" (module (export "c" (module (export "c" (global i32)) )))) +) +(module (import "a" "m" (module (export "c" (module (export "c" (global i32)) (export "a" (func)) )))) - - ;; for now import strings aren't matched at all, imports must simply pairwise - ;; line up - (import "a" "m" (module (export "d" (module (import "" (func)))))) - (import "a" "m" (module (export "d" (module (import "x" (func)))))) - (import "a" "m" (module (export "d" (module (import "x" "y" (func)))))) - - (import "a" "m" (module (export "e" (module - (import "x" "y" (func)) +) +(module + (import "a" "m" (module (export "d" (module + (import "" (func)) (import "a" (func)) - (import "z" (global i32)) + )))) +) +(module + (import "a" "m" (module (export "d" (module (import "" (func)))))) +) +(assert_unlinkable + (module + (import "a" "m" (module (export "d" (module (import "x" (func)))))) + ) + "incompatible import type for `a`") +(assert_unlinkable + (module + (import "a" "m" (module (export "d" (module (import "x" "y" (func)))))) + ) + "incompatible import type for `a`") +(module + (import "a" "m" (module (export "e" (module + (import "a" (func)) + (import "b" (func)) + (import "c" (global i32)) )))) ) (assert_unlinkable (module (import "a" "m" (module (export "" (module (export "a" (func))))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "d" (module))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "d" (module (import "" (module))))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (global f32))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (func))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (table 1 funcref))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (memory 2))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (module (export "foo" (func))))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (instance))))) - "module types incompatible") + "incompatible import type for `a`") ;; instances (module $a @@ -303,46 +367,65 @@ ) (module (import "a" "a" (instance)) +) +(module (import "a" "b" (instance)) +) +(module (import "a" "b" (instance (export "" (func)))) +) +(module (import "a" "c" (instance)) +) +(module (import "a" "c" (instance (export "a" (func)))) +) +(module (import "a" "c" (instance (export "b" (func (result i32))))) +) +(module (import "a" "c" (instance (export "c" (global i32)))) +) +(module (import "a" "c" (instance (export "a" (func)) (export "b" (func (result i32))) (export "c" (global i32)) )) +) +(module (import "a" "c" (instance (export "c" (global i32)) (export "a" (func)) )) - +) +(module (import "a" "m" (module (export "i" (instance)))) +) +(module (import "a" "m" (module (export "i" (instance (export "" (func)))))) ) (assert_unlinkable (module (import "a" "a" (instance (export "" (global f32))))) - "instance types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "i" (instance (export "x" (func))))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (func))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (table 1 funcref))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (memory 2))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (memory 1 10))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "max" (memory 2 10))))) - "module types incompatible") + "incompatible import type for `a`") (assert_unlinkable (module (import "a" "m" (module (export "" (module))))) - "module types incompatible") + "incompatible import type for `a`") diff --git a/tests/misc_testsuite/module-linking/instantiate.wast b/tests/misc_testsuite/module-linking/instantiate.wast index 8a61684438..257aeb90dc 100644 --- a/tests/misc_testsuite/module-linking/instantiate.wast +++ b/tests/misc_testsuite/module-linking/instantiate.wast @@ -34,7 +34,7 @@ (module (import "" (func)) (start 0)) - (instance $a (instantiate 0 (func $set))) + (instance $a (instantiate 0 "" (func $set))) ) (assert_return (invoke $a "get") (i32.const 1)) @@ -49,7 +49,7 @@ global.set 0) (start 0)) - (instance $a (instantiate 0 (global $g))) + (instance $a (instantiate 0 "" (global $g))) ) (assert_return (invoke $a "get") (i32.const 2)) @@ -63,7 +63,7 @@ call_indirect) (start 0)) - (instance $a (instantiate 0 (table $t))) + (instance $a (instantiate 0 "" (table $t))) ) (assert_return (invoke $a "get") (i32.const 3)) @@ -78,7 +78,7 @@ i32.store) (start 0)) - (instance $a (instantiate 0 (memory $m))) + (instance $a (instantiate 0 "" (memory $m))) ) (assert_return (invoke $a "load") (i32.const 100)) @@ -88,13 +88,13 @@ (module $m1 (import "" (instance (export "" (func)))) - (alias (instance 0) (func 0)) + (alias (func 0 "")) (start 0)) (module $m2 (func (export "") (import ""))) - (instance $i (instantiate $m2 (func $set))) - (instance (instantiate $m1 (instance $i))) + (instance $i (instantiate $m2 "" (func $set))) + (instance (instantiate $m1 "" (instance $i))) ) (assert_return (invoke $a "get") (i32.const 4)) @@ -106,14 +106,14 @@ (import "" (module $m (export "" (func $f (result i32))))) (instance $i (instantiate $m)) (func $get (export "") (result i32) - call $i.$f)) + call (func $i ""))) (module $m2 (func (export "") (result i32) i32.const 5)) - (instance $i (instantiate $m1 (module $m2))) + (instance $i (instantiate $m1 "" (module $m2))) (func (export "get") (result i32) - call $i.$get) + call (func $i "")) ) (assert_return (invoke "get") (i32.const 5)) @@ -122,16 +122,16 @@ (module $m (import "" (module $m (export "get" (func (result i32))))) (instance $i (instantiate $m)) - (alias $f (instance $i) (func 0)) + (alias $f (func $i "get")) (export "" (func $f)) ) (module $m2 (func (export "get") (result i32) i32.const 6)) - (instance $a (instantiate $m (module $m2))) + (instance $a (instantiate $m "" (module $m2))) (func (export "get") (result i32) - call $a.$f) + call (func $a "")) ) (assert_return (invoke "get") (i32.const 6)) @@ -143,10 +143,10 @@ (import "a" "memory" (memory $m 1)) (module - (import "" (memory 1)) - (import "" (global (mut i32))) - (import "" (table 1 funcref)) - (import "" (func)) + (import "m" (memory 1)) + (import "g" (global (mut i32))) + (import "t" (table 1 funcref)) + (import "f" (func)) (func $start call 0 @@ -163,10 +163,10 @@ (instance $a (instantiate 0 - (memory $m) - (global $g) - (table $t) - (func $f) + "m" (memory $m) + "g" (global $g) + "t" (table $t) + "f" (func $f) ) ) ) @@ -183,10 +183,10 @@ (module $mt (import "" (table 1 funcref))) (module $mg (import "" (global (mut i32)))) - (instance (instantiate $mm (memory $m))) - (instance (instantiate $mf (func $f))) - (instance (instantiate $mt (table $t))) - (instance (instantiate $mg (global $g))) + (instance (instantiate $mm "" (memory $m))) + (instance (instantiate $mf "" (func $f))) + (instance (instantiate $mt "" (table $t))) + (instance (instantiate $mg "" (global $g))) ) ;; instantiate nested @@ -204,13 +204,13 @@ (import "" (func)) (start 0) ) - (instance (instantiate 0 (func 0))) + (instance (instantiate 0 "" (func 0))) ) - (instance (instantiate 0 (func 0))) + (instance (instantiate 0 "" (func 0))) ) - (instance (instantiate 0 (func 0))) + (instance (instantiate 0 "" (func 0))) ) - (instance (instantiate 0 (func 0))) + (instance (instantiate 0 "" (func 0))) ) (assert_return (invoke $a "get") (i32.const 1)) @@ -219,20 +219,14 @@ (module (export "m")) (instance (export "i") (instantiate 0)) ) -(module - (import "b" "m" (module)) - (import "b" "i" (instance)) -) -(assert_unlinkable - (module - (import "b" "m" (module (import "" (func)))) - ) - "module types incompatible") +(module (import "b" "m" (module))) +(module (import "b" "m" (module (import "" (func))))) +(module (import "b" "i" (instance))) (assert_unlinkable (module (import "b" "i" (instance (export "" (func)))) ) - "instance types incompatible") + "incompatible import type") ;; ensure we ignore other exported items (module $b @@ -250,7 +244,7 @@ )) (func (export "get") (result i32) - global.get $i.$g) + global.get (global $i "g")) ) (assert_return (invoke "get") (i32.const 0xfeed)) @@ -270,15 +264,45 @@ (module (import "b" "i" (instance $i ;; notice that this order is swapped - (export "g" (func $g (param i32) (result i32))) - (export "f" (func $f (result i32))) + (export "g" (func (param i32) (result i32))) + (export "f" (func (result i32))) )) (func (export "f") (result i32) - call $i.$f) + call (func $i "f")) (func (export "g") (param i32) (result i32) local.get 0 - call $i.$g) + call (func $i "g")) ) (assert_return (invoke "f") (i32.const 300)) (assert_return (invoke "g" (i32.const 3000)) (i32.const 3100)) + +(module $a + (func (export "f"))) + +(module + (import "a" "f" (func)) + + (module $m1 + (import "a" "f" (func))) + (instance (instantiate $m1 "a" (instance 0))) +) + +(module + (import "a" "f" (func)) + + ;; this module provides nothing + (module $m1) + + ;; this module imports a module which it says imports something + (module $m2 + (module $a + (func (export ""))) + (instance $i (instantiate $a)) + (import "m" (module $b (import "" (func)))) + (instance $b (instantiate $b "" (func $i "")))) + + ;; we should be able to instantiate m2 with m1 because m1 doesn't actually + ;; import anything (always safe to remove imports!) + (instance (instantiate $m2 "m" (module $m1))) +) From 71ead6e31d5736cb35cd2f5d979f5dd3b22b5470 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Sat, 12 Dec 2020 22:21:39 -0800 Subject: [PATCH 09/55] x64 backend: implement 128-bit ops and misc fixes. This implements all of the ops on I128 that are implemented by the legacy x86 backend, and includes all that are required by at least one major use-case (cg_clif rustc backend). The sequences are open-coded where necessary; for e.g. the bit operations, this can be somewhat complex, but these sequences have been tested carefully. This PR also includes a drive-by fix of clz/ctz for 8- and 16-bit cases where they were incorrect previously. Also includes ridealong fixes developed while bringing up cg_clif support, because they are difficult to completely separate due to other refactors that occurred in this PR: - fix REX prefix logic for some 8-bit instructions. When using an 8-bit register in 64-bit mode on x86-64, the REX prefix semantics are somewhat subtle: without the REX prefix, register numbers 4--7 correspond to the second-to-lowest byte of the first four registers (AH, CH, BH, DH), whereas with the REX prefix, these register numbers correspond to the usual encoding (SPL, BPL, SIL, DIL). We could always emit a REX byte for instructions with 8-bit cases (this is harmless even if unneeded), but this would unnecessarily inflate code size; instead, the usual approach is to emit it only for these registers. This logic was present in some cases but missing for some other instructions: divide, not, negate, shifts. Fixes #2508. - avoid unaligned SSE loads on some f64 ops. The implementations of several FP ops, such as fabs/fneg, used SSE instructions. This is not a problem per-se, except that load-op merging did not take *alignment* into account. Specifically, if an op on an f64 loaded from memory happened to merge that load, and the instruction into which it was merged was an SSE instruction, then the SSE instruction imposes stricter (128-bit) alignment requirements than the load.f64 did. This PR simply forces any instruction lowerings that could use SSE instructions to implement non-SIMD operations to take inputs in registers only, and avoid load-op merging. Fixes #2507. - two bugfixes exposed by cg_clif: urem/srem.i8, select.b1. - urem/srem.i8: the 8-bit form of the DIV instruction on x86-64 places the remainder in AH, not RDX, different from all the other width-forms of this instruction. - select.b1: we were not recognizing selects of boolean values as integer-typed operations, so we were generating XMM moves instead (!). --- cranelift/codegen/src/isa/x64/abi.rs | 103 +- cranelift/codegen/src/isa/x64/inst/args.rs | 24 +- cranelift/codegen/src/isa/x64/inst/emit.rs | 94 +- .../codegen/src/isa/x64/inst/emit_tests.rs | 87 +- cranelift/codegen/src/isa/x64/inst/mod.rs | 182 +- cranelift/codegen/src/isa/x64/lower.rs | 2081 +++++++++++++---- .../filetests/isa/x64/bitops-i128-run.clif | 27 + .../filetests/isa/x64/bitrev-i128-run.clif | 47 + .../filetests/isa/x64/floating-point.clif | 26 + .../filetests/filetests/isa/x64/i128.clif | 1082 +++++++++ .../filetests/isa/x64/select-i128.clif | 29 + .../filetests/isa/x64/shift-i128-run.clif | 106 + 12 files changed, 3213 insertions(+), 675 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/bitops-i128-run.clif create mode 100644 cranelift/filetests/filetests/isa/x64/bitrev-i128-run.clif create mode 100644 cranelift/filetests/filetests/isa/x64/floating-point.clif create mode 100644 cranelift/filetests/filetests/isa/x64/i128.clif create mode 100644 cranelift/filetests/filetests/isa/x64/select-i128.clif create mode 100644 cranelift/filetests/filetests/isa/x64/shift-i128-run.clif diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 74dca6c3ec..aa757392e3 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -138,42 +138,62 @@ impl ABIMachineSpec for X64ABIMachineSpec { ), } - let intreg = in_int_reg(param.value_type); - let vecreg = in_vec_reg(param.value_type); - debug_assert!(intreg || vecreg); - debug_assert!(!(intreg && vecreg)); - - let (next_reg, candidate) = if intreg { - let candidate = match args_or_rets { - ArgsOrRets::Args => get_intreg_for_arg_systemv(&call_conv, next_gpr), - ArgsOrRets::Rets => get_intreg_for_retval_systemv(&call_conv, next_gpr, i), - }; - debug_assert!(candidate - .map(|r| r.get_class() == RegClass::I64) - .unwrap_or(true)); - (&mut next_gpr, candidate) - } else { - let candidate = match args_or_rets { - ArgsOrRets::Args => get_fltreg_for_arg_systemv(&call_conv, next_vreg), - ArgsOrRets::Rets => get_fltreg_for_retval_systemv(&call_conv, next_vreg, i), - }; - debug_assert!(candidate - .map(|r| r.get_class() == RegClass::V128) - .unwrap_or(true)); - (&mut next_vreg, candidate) - }; - if let Some(param) = try_fill_baldrdash_reg(call_conv, param) { - assert!(intreg); ret.push(param); - } else if let Some(reg) = candidate { + continue; + } + + // Find regclass(es) of the register(s) used to store a value of this type. + let (rcs, _) = Inst::rc_for_type(param.value_type)?; + let intreg = rcs[0] == RegClass::I64; + let num_regs = rcs.len(); + assert!(num_regs <= 2); + if num_regs == 2 { + assert_eq!(rcs[0], rcs[1]); + } + + let mut regs: SmallVec<[RealReg; 2]> = smallvec![]; + for j in 0..num_regs { + let nextreg = if intreg { + match args_or_rets { + ArgsOrRets::Args => get_intreg_for_arg_systemv(&call_conv, next_gpr + j), + ArgsOrRets::Rets => { + get_intreg_for_retval_systemv(&call_conv, next_gpr + j, i + j) + } + } + } else { + match args_or_rets { + ArgsOrRets::Args => get_fltreg_for_arg_systemv(&call_conv, next_vreg + j), + ArgsOrRets::Rets => { + get_fltreg_for_retval_systemv(&call_conv, next_vreg + j, i + j) + } + } + }; + if let Some(reg) = nextreg { + regs.push(reg.to_real_reg()); + } else { + regs.clear(); + break; + } + } + + if regs.len() > 0 { + let regs = match num_regs { + 1 => ValueRegs::one(regs[0]), + 2 => ValueRegs::two(regs[0], regs[1]), + _ => panic!("More than two registers unexpected"), + }; ret.push(ABIArg::Reg( - ValueRegs::one(reg.to_real_reg()), + regs, param.value_type, param.extension, param.purpose, )); - *next_reg += 1; + if intreg { + next_gpr += num_regs; + } else { + next_vreg += num_regs; + } } else { // Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte // stack alignment happens separately after all args.) @@ -658,31 +678,6 @@ impl From for SyntheticAmode { } } -fn in_int_reg(ty: types::Type) -> bool { - match ty { - types::I8 - | types::I16 - | types::I32 - | types::I64 - | types::B1 - | types::B8 - | types::B16 - | types::B32 - | types::B64 - | types::R64 => true, - types::R32 => panic!("unexpected 32-bits refs on x64!"), - _ => false, - } -} - -fn in_vec_reg(ty: types::Type) -> bool { - match ty { - types::F32 | types::F64 => true, - _ if ty.is_vector() => true, - _ => false, - } -} - fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option { match call_conv { CallConv::Fast diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 898134644f..39ca25d060 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -346,23 +346,35 @@ impl PrettyPrintSized for RegMem { #[derive(Copy, Clone, PartialEq)] pub enum AluRmiROpcode { Add, + Adc, Sub, + Sbb, And, Or, Xor, /// The signless, non-extending (N x N -> N, for N in {32,64}) variant. Mul, + /// 8-bit form of And. Handled separately as we don't have full 8-bit op + /// support (we just use wider instructions). Used only with some sequences + /// with SETcc. + And8, + /// 8-bit form of Or. + Or8, } impl fmt::Debug for AluRmiROpcode { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { let name = match self { AluRmiROpcode::Add => "add", + AluRmiROpcode::Adc => "adc", AluRmiROpcode::Sub => "sub", + AluRmiROpcode::Sbb => "sbb", AluRmiROpcode::And => "and", AluRmiROpcode::Or => "or", AluRmiROpcode::Xor => "xor", AluRmiROpcode::Mul => "imul", + AluRmiROpcode::And8 => "and", + AluRmiROpcode::Or8 => "or", }; write!(fmt, "{}", name) } @@ -374,6 +386,16 @@ impl fmt::Display for AluRmiROpcode { } } +impl AluRmiROpcode { + /// Is this a special-cased 8-bit ALU op? + pub fn is_8bit(self) -> bool { + match self { + AluRmiROpcode::And8 | AluRmiROpcode::Or8 => true, + _ => false, + } + } +} + #[derive(Clone, PartialEq)] pub enum UnaryRmROpcode { /// Bit-scan reverse. @@ -1010,7 +1032,7 @@ impl fmt::Display for ExtMode { } /// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right. -#[derive(Clone)] +#[derive(Clone, Copy)] pub enum ShiftKind { ShiftLeft, /// Inserts zeros in the most significant bits. diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 580d469b8d..075724d493 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -83,6 +83,14 @@ impl RexFlags { self } + #[inline(always)] + fn always_emit_if_8bit_needed(&mut self, reg: u8) -> &mut Self { + if reg >= 4 && reg <= 7 { + self.always_emit(); + } + self + } + #[inline(always)] fn must_clear_w(&self) -> bool { (self.0 & 1) != 0 @@ -527,7 +535,7 @@ pub(crate) fn emit( src, dst: reg_g, } => { - let rex = if *is_64 { + let mut rex = if *is_64 { RexFlags::set_w() } else { RexFlags::clear_w() @@ -581,17 +589,26 @@ pub(crate) fn emit( } } } else { - let (opcode_r, opcode_m, subopcode_i) = match op { - AluRmiROpcode::Add => (0x01, 0x03, 0), - AluRmiROpcode::Sub => (0x29, 0x2B, 5), - AluRmiROpcode::And => (0x21, 0x23, 4), - AluRmiROpcode::Or => (0x09, 0x0B, 1), - AluRmiROpcode::Xor => (0x31, 0x33, 6), + let (opcode_r, opcode_m, subopcode_i, is_8bit) = match op { + AluRmiROpcode::Add => (0x01, 0x03, 0, false), + AluRmiROpcode::Adc => (0x11, 0x03, 0, false), + AluRmiROpcode::Sub => (0x29, 0x2B, 5, false), + AluRmiROpcode::Sbb => (0x19, 0x2B, 5, false), + AluRmiROpcode::And => (0x21, 0x23, 4, false), + AluRmiROpcode::Or => (0x09, 0x0B, 1, false), + AluRmiROpcode::Xor => (0x31, 0x33, 6, false), + AluRmiROpcode::And8 => (0x20, 0x22, 4, true), + AluRmiROpcode::Or8 => (0x08, 0x0A, 1, true), AluRmiROpcode::Mul => panic!("unreachable"), }; + assert!(!(is_8bit && *is_64)); match src { RegMemImm::Reg { reg: reg_e } => { + if is_8bit { + rex.always_emit_if_8bit_needed(int_reg_enc(*reg_e)); + rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg())); + } // GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R // duality). Do this too, so as to be able to compare generated machine // code easily. @@ -604,11 +621,12 @@ pub(crate) fn emit( reg_g.to_reg(), rex, ); - // NB: if this is ever extended to handle byte size ops, be sure to retain - // redundant REX prefixes. } RegMemImm::Mem { addr } => { + if is_8bit { + rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg())); + } // Here we revert to the "normal" G-E ordering. let amode = addr.finalize(state, sink); emit_std_reg_mem( @@ -625,6 +643,7 @@ pub(crate) fn emit( } RegMemImm::Imm { simm32 } => { + assert!(!is_8bit); let use_imm8 = low8_will_sign_extend_to_32(*simm32); let opcode = if use_imm8 { 0x83 } else { 0x81 }; // And also here we use the "normal" G-E ordering. @@ -685,8 +704,13 @@ pub(crate) fn emit( } Inst::Not { size, src } => { + let src = int_reg_enc(src.to_reg()); let (opcode, prefix, rex_flags) = match size { - 1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()), + 1 => ( + 0xF6, + LegacyPrefixes::None, + *RexFlags::clear_w().always_emit_if_8bit_needed(src), + ), 2 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()), 4 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()), 8 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()), @@ -694,13 +718,17 @@ pub(crate) fn emit( }; let subopcode = 2; - let src = int_reg_enc(src.to_reg()); emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) } Inst::Neg { size, src } => { + let src = int_reg_enc(src.to_reg()); let (opcode, prefix, rex_flags) = match size { - 1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()), + 1 => ( + 0xF6, + LegacyPrefixes::None, + *RexFlags::clear_w().always_emit_if_8bit_needed(src), + ), 2 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()), 4 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()), 8 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()), @@ -708,7 +736,6 @@ pub(crate) fn emit( }; let subopcode = 3; - let src = int_reg_enc(src.to_reg()); emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) } @@ -717,7 +744,7 @@ pub(crate) fn emit( signed, divisor, } => { - let (opcode, prefix, rex_flags) = match size { + let (opcode, prefix, mut rex_flags) = match size { 1 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()), 2 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()), 4 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()), @@ -732,6 +759,9 @@ pub(crate) fn emit( match divisor { RegMem::Reg { reg } => { let src = int_reg_enc(*reg); + if *size == 1 { + rex_flags.always_emit_if_8bit_needed(src); + } emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) } RegMem::Mem { addr: src } => { @@ -987,9 +1017,7 @@ pub(crate) fn emit( ExtMode::BL | ExtMode::BQ => { // A redundant REX prefix must be emitted for certain register inputs. let enc_src = int_reg_enc(*src); - if enc_src >= 4 && enc_src <= 7 { - rex_flags.always_emit(); - }; + rex_flags.always_emit_if_8bit_needed(enc_src); } _ => {} } @@ -1084,9 +1112,7 @@ pub(crate) fn emit( ExtMode::BL | ExtMode::BQ => { // A redundant REX prefix must be emitted for certain register inputs. let enc_src = int_reg_enc(*src); - if enc_src >= 4 && enc_src <= 7 { - rex_flags.always_emit(); - }; + rex_flags.always_emit_if_8bit_needed(enc_src); } _ => {} } @@ -1130,9 +1156,7 @@ pub(crate) fn emit( let mut rex = RexFlags::clear_w(); let enc_src = int_reg_enc(*src); - if enc_src >= 4 && enc_src <= 7 { - rex.always_emit(); - }; + rex.always_emit_if_8bit_needed(enc_src); // MOV r8, r/m8 is (REX.W==0) 88 /r emit_std_reg_mem( @@ -1215,7 +1239,11 @@ pub(crate) fn emit( match num_bits { None => { let (opcode, prefix, rex_flags) = match size { - 1 => (0xD2, LegacyPrefixes::None, RexFlags::clear_w()), + 1 => ( + 0xD2, + LegacyPrefixes::None, + *RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst), + ), 2 => (0xD3, LegacyPrefixes::_66, RexFlags::clear_w()), 4 => (0xD3, LegacyPrefixes::None, RexFlags::clear_w()), 8 => (0xD3, LegacyPrefixes::None, RexFlags::set_w()), @@ -1231,7 +1259,11 @@ pub(crate) fn emit( Some(num_bits) => { let (opcode, prefix, rex_flags) = match size { - 1 => (0xC0, LegacyPrefixes::None, RexFlags::clear_w()), + 1 => ( + 0xC0, + LegacyPrefixes::None, + *RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst), + ), 2 => (0xC1, LegacyPrefixes::_66, RexFlags::clear_w()), 4 => (0xC1, LegacyPrefixes::None, RexFlags::clear_w()), 8 => (0xC1, LegacyPrefixes::None, RexFlags::set_w()), @@ -1330,9 +1362,7 @@ pub(crate) fn emit( let mut rex = RexFlags::clear_w(); // Here, a redundant REX prefix changes the meaning of the instruction. let enc_g = int_reg_enc(*reg_g); - if enc_g >= 4 && enc_g <= 7 { - rex.always_emit(); - } + rex.always_emit_if_8bit_needed(enc_g); rex } _ => panic!("x64::Inst::Cmp_RMI_R::emit: unreachable"), @@ -1343,9 +1373,7 @@ pub(crate) fn emit( if *size == 1 { // Check whether the E register forces the use of a redundant REX. let enc_e = int_reg_enc(*reg_e); - if enc_e >= 4 && enc_e <= 7 { - rex.always_emit(); - } + rex.always_emit_if_8bit_needed(enc_e); } // Use the swapped operands encoding for CMP, to stay consistent with the output of @@ -2761,9 +2789,7 @@ pub(crate) fn emit( types::I8 => { let mut rex_flags = RexFlags::clear_w(); let enc_src = int_reg_enc(*src); - if enc_src >= 4 && enc_src <= 7 { - rex_flags.always_emit(); - }; + rex_flags.always_emit_if_8bit_needed(enc_src); (LegacyPrefixes::_F0, rex_flags, 0x0FB0) } types::I16 => (LegacyPrefixes::_66F0, RexFlags::clear_w(), 0x0FB1), diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index c3489089b9..42e38c9cd5 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -1025,6 +1025,56 @@ fn test_x64_emit() { "4C09FA", "orq %r15, %rdx", )); + insns.push(( + Inst::alu_rmi_r(false, AluRmiROpcode::And8, RegMemImm::reg(r15), w_rdx), + "4420FA", + "andb %r15b, %dl", + )); + insns.push(( + Inst::alu_rmi_r(false, AluRmiROpcode::And8, RegMemImm::reg(rax), w_rsi), + "4020C6", + "andb %al, %sil", + )); + insns.push(( + Inst::alu_rmi_r(false, AluRmiROpcode::And8, RegMemImm::reg(rax), w_rbx), + "20C3", + "andb %al, %bl", + )); + insns.push(( + Inst::alu_rmi_r( + false, + AluRmiROpcode::And8, + RegMemImm::mem(Amode::imm_reg(0, rax)), + w_rbx, + ), + "2218", + "andb 0(%rax), %bl", + )); + insns.push(( + Inst::alu_rmi_r(false, AluRmiROpcode::Or8, RegMemImm::reg(r15), w_rdx), + "4408FA", + "orb %r15b, %dl", + )); + insns.push(( + Inst::alu_rmi_r(false, AluRmiROpcode::Or8, RegMemImm::reg(rax), w_rsi), + "4008C6", + "orb %al, %sil", + )); + insns.push(( + Inst::alu_rmi_r(false, AluRmiROpcode::Or8, RegMemImm::reg(rax), w_rbx), + "08C3", + "orb %al, %bl", + )); + insns.push(( + Inst::alu_rmi_r( + false, + AluRmiROpcode::Or8, + RegMemImm::mem(Amode::imm_reg(0, rax)), + w_rbx, + ), + "0A18", + "orb 0(%rax), %bl", + )); insns.push(( Inst::alu_rmi_r(true, AluRmiROpcode::Xor, RegMemImm::reg(r15), w_rdx), "4C31FA", @@ -1193,6 +1243,16 @@ fn test_x64_emit() { "66F7D7", "notw %di", )); + insns.push(( + Inst::not(1, Writable::from_reg(regs::rdi())), + "40F6D7", + "notb %dil", + )); + insns.push(( + Inst::not(1, Writable::from_reg(regs::rax())), + "F6D0", + "notb %al", + )); // ======================================================== // Neg @@ -1216,6 +1276,16 @@ fn test_x64_emit() { "66F7DF", "negw %di", )); + insns.push(( + Inst::neg(1, Writable::from_reg(regs::rdi())), + "40F6DF", + "negb %dil", + )); + insns.push(( + Inst::neg(1, Writable::from_reg(regs::rax())), + "F6D8", + "negb %al", + )); // ======================================================== // Div @@ -1239,6 +1309,16 @@ fn test_x64_emit() { "48F7F7", "div %rdi", )); + insns.push(( + Inst::div(1, false, RegMem::reg(regs::rax())), + "F6F0", + "div %al", + )); + insns.push(( + Inst::div(1, false, RegMem::reg(regs::rsi())), + "40F6F6", + "div %sil", + )); // ======================================================== // MulHi @@ -2352,9 +2432,14 @@ fn test_x64_emit() { )); insns.push(( Inst::shift_r(1, ShiftKind::RotateRight, None, w_rsi), - "D2CE", + "40D2CE", "rorb %cl, %sil", )); + insns.push(( + Inst::shift_r(1, ShiftKind::RotateRight, None, w_rax), + "D2C8", + "rorb %cl, %al", + )); insns.push(( Inst::shift_r(1, ShiftKind::RotateRight, Some(5), w_r15), "41C0CF05", diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 09c469498d..979c264231 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1243,6 +1243,14 @@ impl PrettyPrint for Inst { (if is_64 { "q" } else { "l" }).to_string() } + fn suffix_lqb(is_64: bool, is_8: bool) -> String { + match (is_64, is_8) { + (_, true) => "b".to_string(), + (true, false) => "q".to_string(), + (false, false) => "l".to_string(), + } + } + fn size_lq(is_64: bool) -> u8 { if is_64 { 8 @@ -1251,6 +1259,16 @@ impl PrettyPrint for Inst { } } + fn size_lqb(is_64: bool, is_8: bool) -> u8 { + if is_8 { + 1 + } else if is_64 { + 8 + } else { + 4 + } + } + fn suffix_bwlq(size: u8) -> String { match size { 1 => "b".to_string(), @@ -1271,9 +1289,9 @@ impl PrettyPrint for Inst { dst, } => format!( "{} {}, {}", - ljustify2(op.to_string(), suffix_lq(*is_64)), - src.show_rru_sized(mb_rru, size_lq(*is_64)), - show_ireg_sized(dst.to_reg(), mb_rru, size_lq(*is_64)), + ljustify2(op.to_string(), suffix_lqb(*is_64, op.is_8bit())), + src.show_rru_sized(mb_rru, size_lqb(*is_64, op.is_8bit())), + show_ireg_sized(dst.to_reg(), mb_rru, size_lqb(*is_64, op.is_8bit())), ), Inst::UnaryRmR { src, dst, op, size } => format!( @@ -2065,6 +2083,17 @@ impl Amode { } } } + + /// Offset the amode by a fixed offset. + pub(crate) fn offset(&self, offset: u32) -> Self { + let mut ret = self.clone(); + match &mut ret { + &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset, + &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset, + _ => panic!("Cannot offset amode: {:?}", self), + } + ret + } } impl RegMemImm { @@ -2548,77 +2577,88 @@ impl MachInst for Inst { ty: Type, mut alloc_tmp: F, ) -> SmallVec<[Self; 4]> { - // We don't support 128-bit constants. - assert!(value <= u64::MAX as u128); let mut ret = SmallVec::new(); - let to_reg = to_regs - .only_reg() - .expect("multi-reg values not supported on x64"); - if ty == types::F32 { - if value == 0 { - ret.push(Inst::xmm_rm_r( - SseOpcode::Xorps, - RegMem::reg(to_reg.to_reg()), - to_reg, - )); - } else { - let tmp = alloc_tmp(types::I32); - ret.push(Inst::imm(OperandSize::Size32, value as u64, tmp)); - - ret.push(Inst::gpr_to_xmm( - SseOpcode::Movd, - RegMem::reg(tmp.to_reg()), - OperandSize::Size32, - to_reg, - )); - } - } else if ty == types::F64 { - if value == 0 { - ret.push(Inst::xmm_rm_r( - SseOpcode::Xorpd, - RegMem::reg(to_reg.to_reg()), - to_reg, - )); - } else { - let tmp = alloc_tmp(types::I64); - ret.push(Inst::imm(OperandSize::Size64, value as u64, tmp)); - - ret.push(Inst::gpr_to_xmm( - SseOpcode::Movq, - RegMem::reg(tmp.to_reg()), - OperandSize::Size64, - to_reg, - )); - } + if ty == types::I128 { + ret.push(Inst::imm( + OperandSize::Size64, + value as u64, + to_regs.regs()[0], + )); + ret.push(Inst::imm( + OperandSize::Size64, + (value >> 64) as u64, + to_regs.regs()[1], + )); } else { - // Must be an integer type. - debug_assert!( - ty == types::B1 - || ty == types::I8 - || ty == types::B8 - || ty == types::I16 - || ty == types::B16 - || ty == types::I32 - || ty == types::B32 - || ty == types::I64 - || ty == types::B64 - || ty == types::R32 - || ty == types::R64 - ); - if value == 0 { - ret.push(Inst::alu_rmi_r( - ty == types::I64, - AluRmiROpcode::Xor, - RegMemImm::reg(to_reg.to_reg()), - to_reg, - )); + let to_reg = to_regs + .only_reg() + .expect("multi-reg values not supported on x64"); + if ty == types::F32 { + if value == 0 { + ret.push(Inst::xmm_rm_r( + SseOpcode::Xorps, + RegMem::reg(to_reg.to_reg()), + to_reg, + )); + } else { + let tmp = alloc_tmp(types::I32); + ret.push(Inst::imm(OperandSize::Size32, value as u64, tmp)); + + ret.push(Inst::gpr_to_xmm( + SseOpcode::Movd, + RegMem::reg(tmp.to_reg()), + OperandSize::Size32, + to_reg, + )); + } + } else if ty == types::F64 { + if value == 0 { + ret.push(Inst::xmm_rm_r( + SseOpcode::Xorpd, + RegMem::reg(to_reg.to_reg()), + to_reg, + )); + } else { + let tmp = alloc_tmp(types::I64); + ret.push(Inst::imm(OperandSize::Size64, value as u64, tmp)); + + ret.push(Inst::gpr_to_xmm( + SseOpcode::Movq, + RegMem::reg(tmp.to_reg()), + OperandSize::Size64, + to_reg, + )); + } } else { - let value = value as u64; - ret.push(Inst::imm( - OperandSize::from_bytes(ty.bytes()), - value.into(), - to_reg, - )); + // Must be an integer type. + debug_assert!( + ty == types::B1 + || ty == types::I8 + || ty == types::B8 + || ty == types::I16 + || ty == types::B16 + || ty == types::I32 + || ty == types::B32 + || ty == types::I64 + || ty == types::B64 + || ty == types::R32 + || ty == types::R64 + ); + if value == 0 { + ret.push(Inst::alu_rmi_r( + ty == types::I64, + AluRmiROpcode::Xor, + RegMemImm::reg(to_reg.to_reg()), + to_reg, + )); + } else { + let value = value as u64; + ret.push(Inst::imm( + OperandSize::from_bytes(ty.bytes()), + value.into(), + to_reg, + )); + } } } ret diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 9293221de5..a25da666b3 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -18,7 +18,7 @@ use alloc::vec::Vec; use cranelift_codegen_shared::condcodes::CondCode; use log::trace; use regalloc::{Reg, RegClass, Writable}; -use smallvec::SmallVec; +use smallvec::{smallvec, SmallVec}; use std::convert::TryFrom; use target_lexicon::Triple; @@ -28,6 +28,7 @@ use target_lexicon::Triple; fn is_int_or_ref_ty(ty: Type) -> bool { match ty { types::I8 | types::I16 | types::I32 | types::I64 | types::R64 => true, + types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true, types::R32 => panic!("shouldn't have 32-bits refs on x64"), _ => false, } @@ -107,23 +108,26 @@ fn generate_constant>(ctx: &mut C, ty: Type, c: u64) -> Va non_writable_value_regs(cst_copy) } -/// Put the given input into a register, and mark it as used (side-effect). -fn put_input_in_reg>(ctx: &mut C, spec: InsnInput) -> Reg { +/// Put the given input into possibly multiple registers, and mark it as used (side-effect). +fn put_input_in_regs>(ctx: &mut C, spec: InsnInput) -> ValueRegs { let ty = ctx.input_ty(spec.insn, spec.input); let input = ctx.get_input_as_source_or_const(spec.insn, spec.input); if let Some(c) = input.constant { // Generate constants fresh at each use to minimize long-range register pressure. generate_constant(ctx, ty, c) - .only_reg() - .expect("multi-reg values not supported yet") } else { ctx.put_input_in_regs(spec.insn, spec.input) - .only_reg() - .expect("multi-reg values not supported yet") } } +/// Put the given input into a register, and mark it as used (side-effect). +fn put_input_in_reg>(ctx: &mut C, spec: InsnInput) -> Reg { + put_input_in_regs(ctx, spec) + .only_reg() + .expect("Multi-register value not expected") +} + /// Determines whether a load operation (indicated by `src_insn`) can be merged /// into the current lowering point. If so, returns the address-base source (as /// an `InsnInput`) and an offset from that address from which to perform the @@ -373,25 +377,120 @@ fn emit_extract_lane>( /// /// Note: make sure that there are no instructions modifying the flags between a call to this /// function and the use of the flags! -fn emit_cmp>(ctx: &mut C, insn: IRInst) { +/// +/// Takes the condition code that will be tested, and returns +/// the condition code that should be used. This allows us to +/// synthesize comparisons out of multiple instructions for +/// special cases (e.g., 128-bit integers). +fn emit_cmp>(ctx: &mut C, insn: IRInst, cc: IntCC) -> IntCC { let ty = ctx.input_ty(insn, 0); let inputs = [InsnInput { insn, input: 0 }, InsnInput { insn, input: 1 }]; - // TODO Try to commute the operands (and invert the condition) if one is an immediate. - let lhs = put_input_in_reg(ctx, inputs[0]); - // We force the RHS into a register, and disallow load-op fusion, because we - // do not have a transitive guarantee that this cmp-site will be the sole - // user of the value. Consider: the icmp might be the only user of a load, - // but there may be multiple users of the icmp (e.g. select or bint - // instructions) that each invoke `emit_cmp()`. If we were to allow a load - // to sink to the *latest* one, but other sites did not permit sinking, then - // we would be missing the load for other cmp-sites. - let rhs = put_input_in_reg(ctx, inputs[1]); + if ty == types::I128 { + // We need to compare both halves and combine the results appropriately. + let cmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let cmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let lhs = put_input_in_regs(ctx, inputs[0]); + let lhs_lo = lhs.regs()[0]; + let lhs_hi = lhs.regs()[1]; + let rhs = put_input_in_regs(ctx, inputs[1]); + let rhs_lo = RegMemImm::reg(rhs.regs()[0]); + let rhs_hi = RegMemImm::reg(rhs.regs()[1]); + match cc { + IntCC::Equal => { + ctx.emit(Inst::cmp_rmi_r(8, rhs_hi, lhs_hi)); + ctx.emit(Inst::setcc(CC::Z, cmp1)); + ctx.emit(Inst::cmp_rmi_r(8, rhs_lo, lhs_lo)); + ctx.emit(Inst::setcc(CC::Z, cmp2)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(cmp1.to_reg()), + cmp2, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::imm(1), + cmp2, + )); + IntCC::NotEqual + } + IntCC::NotEqual => { + ctx.emit(Inst::cmp_rmi_r(8, rhs_hi, lhs_hi)); + ctx.emit(Inst::setcc(CC::NZ, cmp1)); + ctx.emit(Inst::cmp_rmi_r(8, rhs_lo, lhs_lo)); + ctx.emit(Inst::setcc(CC::NZ, cmp2)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(cmp1.to_reg()), + cmp2, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::imm(1), + cmp2, + )); + IntCC::NotEqual + } + IntCC::SignedLessThan + | IntCC::SignedLessThanOrEqual + | IntCC::SignedGreaterThan + | IntCC::SignedGreaterThanOrEqual + | IntCC::UnsignedLessThan + | IntCC::UnsignedLessThanOrEqual + | IntCC::UnsignedGreaterThan + | IntCC::UnsignedGreaterThanOrEqual => { + // Result = (lhs_hi <> rhs_hi) || + // (lhs_hi == rhs_hi && lhs_lo <> rhs_lo) + let cmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + ctx.emit(Inst::cmp_rmi_r(8, rhs_hi, lhs_hi)); + ctx.emit(Inst::setcc(CC::from_intcc(cc.without_equal()), cmp1)); + ctx.emit(Inst::setcc(CC::Z, cmp2)); + ctx.emit(Inst::cmp_rmi_r(8, rhs_lo, lhs_lo)); + ctx.emit(Inst::setcc(CC::from_intcc(cc.unsigned()), cmp3)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(cmp2.to_reg()), + cmp3, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(cmp1.to_reg()), + cmp3, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::imm(1), + cmp3, + )); + IntCC::NotEqual + } + _ => panic!("Unhandled IntCC in I128 comparison: {:?}", cc), + } + } else { + // TODO Try to commute the operands (and invert the condition) if one is an immediate. + let lhs = put_input_in_reg(ctx, inputs[0]); + // We force the RHS into a register, and disallow load-op fusion, because we + // do not have a transitive guarantee that this cmp-site will be the sole + // user of the value. Consider: the icmp might be the only user of a load, + // but there may be multiple users of the icmp (e.g. select or bint + // instructions) that each invoke `emit_cmp()`. If we were to allow a load + // to sink to the *latest* one, but other sites did not permit sinking, then + // we would be missing the load for other cmp-sites. + let rhs = put_input_in_reg(ctx, inputs[1]); - // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives - // us dst - src at the machine instruction level, so invert operands. - ctx.emit(Inst::cmp_rmi_r(ty.bytes() as u8, RegMemImm::reg(rhs), lhs)); + // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives + // us dst - src at the machine instruction level, so invert operands. + ctx.emit(Inst::cmp_rmi_r(ty.bytes() as u8, RegMemImm::reg(rhs), lhs)); + cc + } } /// A specification for a fcmp emission. @@ -489,6 +588,458 @@ fn emit_fcmp>( cond_result } +fn emit_bitrev>(ctx: &mut C, src: Reg, dst: Writable, ty: Type) { + let bits = ty.bits(); + let const_mask = if bits == 64 { + 0xffff_ffff_ffff_ffff + } else { + (1u64 << bits) - 1 + }; + let tmp0 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + + ctx.emit(Inst::gen_move(tmp0, src, types::I64)); + + // Swap 1-bit units. + // tmp1 = src + ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64)); + // tmp2 = 0b0101.. + ctx.emit(Inst::imm( + OperandSize::Size64, + 0x5555_5555_5555_5555 & const_mask, + tmp2, + )); + // tmp1 = src >> 1 + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); + // tmp1 = (src >> 1) & 0b0101.. + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp2.to_reg()), + tmp1, + )); + // tmp2 = src & 0b0101.. + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp0.to_reg()), + tmp2, + )); + // tmp2 = (src & 0b0101..) << 1 + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(1), tmp2)); + // tmp0 = (src >> 1) & 0b0101.. | (src & 0b0101..) << 1 + ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp1.to_reg()), + tmp0, + )); + + // Swap 2-bit units. + ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64)); + ctx.emit(Inst::imm( + OperandSize::Size64, + 0x3333_3333_3333_3333 & const_mask, + tmp2, + )); + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(2), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp2.to_reg()), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp0.to_reg()), + tmp2, + )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(2), tmp2)); + ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp1.to_reg()), + tmp0, + )); + + // Swap 4-bit units. + ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64)); + ctx.emit(Inst::imm( + OperandSize::Size64, + 0x0f0f_0f0f_0f0f_0f0f & const_mask, + tmp2, + )); + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(4), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp2.to_reg()), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp0.to_reg()), + tmp2, + )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(4), tmp2)); + ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp1.to_reg()), + tmp0, + )); + + if bits > 8 { + // Swap 8-bit units. + ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64)); + ctx.emit(Inst::imm( + OperandSize::Size64, + 0x00ff_00ff_00ff_00ff & const_mask, + tmp2, + )); + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(8), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp2.to_reg()), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp0.to_reg()), + tmp2, + )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(8), tmp2)); + ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp1.to_reg()), + tmp0, + )); + } + + if bits > 16 { + // Swap 16-bit units. + ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64)); + ctx.emit(Inst::imm( + OperandSize::Size64, + 0x0000_ffff_0000_ffff & const_mask, + tmp2, + )); + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(16), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp2.to_reg()), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp0.to_reg()), + tmp2, + )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(16), tmp2)); + ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp1.to_reg()), + tmp0, + )); + } + + if bits > 32 { + // Swap 32-bit units. + ctx.emit(Inst::gen_move(tmp1, tmp0.to_reg(), types::I64)); + ctx.emit(Inst::imm( + OperandSize::Size64, + 0x0000_0000_ffff_ffff & const_mask, + tmp2, + )); + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(32), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp2.to_reg()), + tmp1, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::reg(tmp0.to_reg()), + tmp2, + )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, Some(32), tmp2)); + ctx.emit(Inst::gen_move(tmp0, tmp2.to_reg(), types::I64)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp1.to_reg()), + tmp0, + )); + } + + ctx.emit(Inst::gen_move(dst, tmp0.to_reg(), types::I64)); +} + +fn emit_shl_i128>( + ctx: &mut C, + src: ValueRegs, + dst: ValueRegs>, + amt_src: Reg, +) { + let src_lo = src.regs()[0]; + let src_hi = src.regs()[1]; + let dst_lo = dst.regs()[0]; + let dst_hi = dst.regs()[1]; + + // mov tmp1, src_lo + // shl tmp1, amt_src + // mov tmp2, src_hi + // shl tmp2, amt_src + // mov amt, 64 + // sub amt, amt_src + // mov tmp3, src_lo + // shr tmp3, amt + // or tmp3, tmp2 + // xor dst_lo, dst_lo + // mov amt, amt_src + // and amt, 64 + // cmovz dst_hi, tmp3 + // cmovz dst_lo, tmp1 + // cmovnz dst_hi, tmp1 + + let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + + ctx.emit(Inst::gen_move(tmp1, src_lo, types::I64)); + ctx.emit(Inst::gen_move( + Writable::from_reg(regs::rcx()), + amt_src, + types::I64, + )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, None, tmp1)); + + ctx.emit(Inst::gen_move(tmp2, src_hi, types::I64)); + ctx.emit(Inst::gen_move( + Writable::from_reg(regs::rcx()), + amt_src, + types::I64, + )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, None, tmp2)); + + ctx.emit(Inst::imm(OperandSize::Size64, 64, amt)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Sub, + RegMemImm::reg(amt_src), + amt, + )); + + ctx.emit(Inst::gen_move(tmp3, src_lo, types::I64)); + ctx.emit(Inst::gen_move( + Writable::from_reg(regs::rcx()), + amt.to_reg(), + types::I64, + )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftRightLogical, None, tmp3)); + + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp2.to_reg()), + tmp3, + )); + + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Xor, + RegMemImm::reg(dst_lo.to_reg()), + dst_lo, + )); + // This isn't semantically necessary, but it keeps the + // register allocator happy, because it cannot otherwise + // infer that cmovz + cmovnz always defines dst_hi. + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Xor, + RegMemImm::reg(dst_hi.to_reg()), + dst_hi, + )); + + ctx.emit(Inst::gen_move(amt, amt_src, types::I64)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::imm(64), + amt, + )); + ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp3.to_reg()), dst_hi)); + ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp1.to_reg()), dst_lo)); + ctx.emit(Inst::cmove(8, CC::NZ, RegMem::reg(tmp1.to_reg()), dst_hi)); +} + +fn emit_shr_i128>( + ctx: &mut C, + src: ValueRegs, + dst: ValueRegs>, + amt_src: Reg, + is_signed: bool, +) { + let src_lo = src.regs()[0]; + let src_hi = src.regs()[1]; + let dst_lo = dst.regs()[0]; + let dst_hi = dst.regs()[1]; + + // mov tmp1, src_hi + // {u,s}shr tmp1, amt_src + // mov tmp2, src_lo + // {u,s}shr tmp2, amt_src + // mov amt, 64 + // sub amt, amt_src + // mov tmp3, src_hi + // shl tmp3, amt + // or tmp3, tmp2 + // if is_signed: + // mov dst_hi, src_hi + // sshr dst_hi, 63 // get the sign bit + // else: + // xor dst_hi, dst_hi + // mov amt, amt_src + // and amt, 64 + // cmovz dst_hi, tmp1 + // cmovz dst_lo, tmp3 + // cmovnz dst_lo, tmp1 + + let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + + let shift_kind = if is_signed { + ShiftKind::ShiftRightArithmetic + } else { + ShiftKind::ShiftRightLogical + }; + + ctx.emit(Inst::gen_move(tmp1, src_hi, types::I64)); + ctx.emit(Inst::gen_move( + Writable::from_reg(regs::rcx()), + amt_src, + types::I64, + )); + ctx.emit(Inst::shift_r(8, shift_kind, None, tmp1)); + + ctx.emit(Inst::gen_move(tmp2, src_lo, types::I64)); + ctx.emit(Inst::gen_move( + Writable::from_reg(regs::rcx()), + amt_src, + types::I64, + )); + ctx.emit(Inst::shift_r(8, shift_kind, None, tmp2)); + + ctx.emit(Inst::imm(OperandSize::Size64, 64, amt)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Sub, + RegMemImm::reg(amt_src), + amt, + )); + + ctx.emit(Inst::gen_move(tmp3, src_hi, types::I64)); + ctx.emit(Inst::gen_move( + Writable::from_reg(regs::rcx()), + amt.to_reg(), + types::I64, + )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftLeft, None, tmp3)); + + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp2.to_reg()), + tmp3, + )); + + if is_signed { + ctx.emit(Inst::gen_move(dst_hi, src_hi, types::I64)); + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightArithmetic, + Some(63), + dst_hi, + )); + } else { + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Xor, + RegMemImm::reg(dst_hi.to_reg()), + dst_hi, + )); + } + // This isn't semantically necessary, but it keeps the + // register allocator happy, because it cannot otherwise + // infer that cmovz + cmovnz always defines dst_lo. + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Xor, + RegMemImm::reg(dst_lo.to_reg()), + dst_lo, + )); + + ctx.emit(Inst::gen_move(amt, amt_src, types::I64)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::And, + RegMemImm::imm(64), + amt, + )); + ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp1.to_reg()), dst_hi)); + ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp3.to_reg()), dst_lo)); + ctx.emit(Inst::cmove(8, CC::NZ, RegMem::reg(tmp1.to_reg()), dst_lo)); +} + fn make_libcall_sig>( ctx: &mut C, insn: IRInst, @@ -676,6 +1227,101 @@ fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i Amode::imm_reg(offset as u32, input).with_flags(flags) } +fn emit_moves>( + ctx: &mut C, + dst: ValueRegs>, + src: ValueRegs, + ty: Type, +) { + let (_, tys) = Inst::rc_for_type(ty).unwrap(); + for ((dst, src), ty) in dst.regs().iter().zip(src.regs().iter()).zip(tys.iter()) { + ctx.emit(Inst::gen_move(*dst, *src, *ty)); + } +} + +fn emit_cmoves>( + ctx: &mut C, + size: u8, + cc: CC, + src: ValueRegs, + dst: ValueRegs>, +) { + let size = size / src.len() as u8; + let size = u8::max(size, 4); // at least 32 bits + for (dst, src) in dst.regs().iter().zip(src.regs().iter()) { + ctx.emit(Inst::cmove(size, cc, RegMem::reg(*src), *dst)); + } +} + +fn emit_clz>( + ctx: &mut C, + orig_ty: Type, + ty: Type, + src: Reg, + dst: Writable, +) { + let src = RegMem::reg(src); + let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); + ctx.emit(Inst::imm( + OperandSize::from_bytes(ty.bytes()), + u64::max_value(), + dst, + )); + + ctx.emit(Inst::unary_rm_r( + ty.bytes() as u8, + UnaryRmROpcode::Bsr, + src, + tmp, + )); + + ctx.emit(Inst::cmove( + ty.bytes() as u8, + CC::Z, + RegMem::reg(dst.to_reg()), + tmp, + )); + + ctx.emit(Inst::imm( + OperandSize::from_bytes(ty.bytes()), + orig_ty.bits() as u64 - 1, + dst, + )); + + ctx.emit(Inst::alu_rmi_r( + ty == types::I64, + AluRmiROpcode::Sub, + RegMemImm::reg(tmp.to_reg()), + dst, + )); +} + +fn emit_ctz>( + ctx: &mut C, + orig_ty: Type, + ty: Type, + src: Reg, + dst: Writable, +) { + let src = RegMem::reg(src); + let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); + ctx.emit(Inst::imm(OperandSize::Size32, orig_ty.bits() as u64, tmp)); + + ctx.emit(Inst::unary_rm_r( + ty.bytes() as u8, + UnaryRmROpcode::Bsf, + src, + dst, + )); + + ctx.emit(Inst::cmove( + ty.bytes() as u8, + CC::Z, + RegMem::reg(tmp.to_reg()), + dst, + )); +} + //============================================================================= // Top-level instruction lowering entry point, for one instruction. @@ -898,6 +1544,102 @@ fn lower_insn_to_regs>( // Move the `lhs` to the same register as `dst`. ctx.emit(Inst::gen_move(dst, lhs, ty)); ctx.emit(Inst::xmm_rm_r(sse_op, rhs, dst)); + } else if ty == types::I128 || ty == types::B128 { + let alu_ops = match op { + Opcode::Iadd => (AluRmiROpcode::Add, AluRmiROpcode::Adc), + Opcode::Isub => (AluRmiROpcode::Sub, AluRmiROpcode::Sbb), + // multiply handled specially below + Opcode::Imul => (AluRmiROpcode::Mul, AluRmiROpcode::Mul), + Opcode::Band => (AluRmiROpcode::And, AluRmiROpcode::And), + Opcode::Bor => (AluRmiROpcode::Or, AluRmiROpcode::Or), + Opcode::Bxor => (AluRmiROpcode::Xor, AluRmiROpcode::Xor), + _ => panic!("Unsupported opcode with 128-bit integers: {:?}", op), + }; + let lhs = put_input_in_regs(ctx, inputs[0]); + let rhs = put_input_in_regs(ctx, inputs[1]); + let dst = get_output_reg(ctx, outputs[0]); + assert_eq!(lhs.len(), 2); + assert_eq!(rhs.len(), 2); + assert_eq!(dst.len(), 2); + + if op != Opcode::Imul { + // add, sub, and, or, xor: just do ops on lower then upper half. Carry-flag + // propagation is implicit (add/adc, sub/sbb). + ctx.emit(Inst::gen_move(dst.regs()[0], lhs.regs()[0], types::I64)); + ctx.emit(Inst::gen_move(dst.regs()[1], lhs.regs()[1], types::I64)); + ctx.emit(Inst::alu_rmi_r( + /* is_64 = */ true, + alu_ops.0, + RegMemImm::reg(rhs.regs()[0]), + dst.regs()[0], + )); + ctx.emit(Inst::alu_rmi_r( + /* is_64 = */ true, + alu_ops.1, + RegMemImm::reg(rhs.regs()[1]), + dst.regs()[1], + )); + } else { + // mul: + // dst_lo = lhs_lo * rhs_lo + // dst_hi = umulhi(lhs_lo, rhs_lo) + lhs_lo * rhs_hi + lhs_hi * rhs_lo + // + // so we emit: + // mov dst_lo, lhs_lo + // mul dst_lo, rhs_lo + // mov dst_hi, lhs_lo + // mul dst_hi, rhs_hi + // mov tmp, lhs_hi + // mul tmp, rhs_lo + // add dst_hi, tmp + // mov rax, lhs_lo + // umulhi rhs_lo // implicit rax arg/dst + // add dst_hi, rax + let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + ctx.emit(Inst::gen_move(dst.regs()[0], lhs.regs()[0], types::I64)); + ctx.emit(Inst::alu_rmi_r( + /* is_64 = */ true, + AluRmiROpcode::Mul, + RegMemImm::reg(rhs.regs()[0]), + dst.regs()[0], + )); + ctx.emit(Inst::gen_move(dst.regs()[1], lhs.regs()[0], types::I64)); + ctx.emit(Inst::alu_rmi_r( + /* is_64 = */ true, + AluRmiROpcode::Mul, + RegMemImm::reg(rhs.regs()[1]), + dst.regs()[1], + )); + ctx.emit(Inst::gen_move(tmp, lhs.regs()[1], types::I64)); + ctx.emit(Inst::alu_rmi_r( + /* is_64 = */ true, + AluRmiROpcode::Mul, + RegMemImm::reg(rhs.regs()[0]), + tmp, + )); + ctx.emit(Inst::alu_rmi_r( + /* is_64 = */ true, + AluRmiROpcode::Add, + RegMemImm::reg(tmp.to_reg()), + dst.regs()[1], + )); + ctx.emit(Inst::gen_move( + Writable::from_reg(regs::rax()), + lhs.regs()[0], + types::I64, + )); + ctx.emit(Inst::mul_hi( + /* size = */ 8, + /* signed = */ false, + RegMem::reg(rhs.regs()[0]), + )); + ctx.emit(Inst::alu_rmi_r( + /* is_64 = */ true, + AluRmiROpcode::Add, + RegMemImm::reg(regs::rdx()), + dst.regs()[1], + )); + } } else { let is_64 = ty == types::I64; let alu_op = match op { @@ -1022,17 +1764,27 @@ fn lower_insn_to_regs>( Opcode::Bnot => { let ty = ty.unwrap(); let size = ty.bytes() as u8; - let src = put_input_in_reg(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::gen_move(dst, src, ty)); if ty.is_vector() { + let src = put_input_in_reg(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::gen_move(dst, src, ty)); let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); ctx.emit(Inst::equals(ty, RegMem::from(tmp), tmp)); ctx.emit(Inst::xor(ty, RegMem::from(tmp), dst)); + } else if ty == types::I128 || ty == types::B128 { + let src = put_input_in_regs(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]); + ctx.emit(Inst::gen_move(dst.regs()[0], src.regs()[0], types::I64)); + ctx.emit(Inst::not(8, dst.regs()[0])); + ctx.emit(Inst::gen_move(dst.regs()[1], src.regs()[1], types::I64)); + ctx.emit(Inst::not(8, dst.regs()[1])); } else if ty.is_bool() { unimplemented!("bool bnot") } else { + let src = put_input_in_reg(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::gen_move(dst, src, ty)); ctx.emit(Inst::not(size, dst)); } } @@ -1064,7 +1816,7 @@ fn lower_insn_to_regs>( let dst_ty = ctx.output_ty(insn, 0); debug_assert_eq!(ctx.input_ty(insn, 0), dst_ty); - if !dst_ty.is_vector() { + if !dst_ty.is_vector() && dst_ty.bits() <= 64 { // Scalar shifts on x86 have various encodings: // - shift by one bit, e.g. `SAL r/m8, 1` (not used here) // - shift by an immediate amount, e.g. `SAL r/m8, imm8` @@ -1118,6 +1870,89 @@ fn lower_insn_to_regs>( ctx.emit(Inst::mov_r_r(true, rhs.unwrap(), w_rcx)); } ctx.emit(Inst::shift_r(size, shift_kind, count, dst)); + } else if dst_ty == types::I128 { + let amt_src = put_input_in_reg(ctx, inputs[1]); + let src = put_input_in_regs(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]); + + match op { + Opcode::Ishl => { + emit_shl_i128(ctx, src, dst, amt_src); + } + Opcode::Ushr => { + emit_shr_i128(ctx, src, dst, amt_src, /* is_signed = */ false); + } + Opcode::Sshr => { + emit_shr_i128(ctx, src, dst, amt_src, /* is_signed = */ true); + } + Opcode::Rotl => { + // (mov tmp, src) + // (shl.i128 tmp, amt) + // (mov dst, src) + // (ushr.i128 dst, 128-amt) + // (or dst, tmp) + let tmp = ctx.alloc_tmp(types::I128); + emit_shl_i128(ctx, src, tmp, amt_src); + let inv_amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + ctx.emit(Inst::imm(OperandSize::Size64, 128, inv_amt)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Sub, + RegMemImm::reg(amt_src), + inv_amt, + )); + emit_shr_i128( + ctx, + src, + dst, + inv_amt.to_reg(), + /* is_signed = */ false, + ); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp.regs()[0].to_reg()), + dst.regs()[0], + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp.regs()[1].to_reg()), + dst.regs()[1], + )); + } + Opcode::Rotr => { + // (mov tmp, src) + // (ushr.i128 tmp, amt) + // (mov dst, src) + // (shl.i128 dst, 128-amt) + // (or dst, tmp) + let tmp = ctx.alloc_tmp(types::I128); + emit_shr_i128(ctx, src, tmp, amt_src, /* is_signed = */ false); + let inv_amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + ctx.emit(Inst::imm(OperandSize::Size64, 128, inv_amt)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Sub, + RegMemImm::reg(amt_src), + inv_amt, + )); + emit_shl_i128(ctx, src, dst, inv_amt.to_reg()); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp.regs()[0].to_reg()), + dst.regs()[0], + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Or, + RegMemImm::reg(tmp.regs()[1].to_reg()), + dst.regs()[1], + )); + } + _ => unreachable!(), + } } else if dst_ty == types::I8X16 && (op == Opcode::Ishl || op == Opcode::Ushr) { // Since the x86 instruction set does not have any 8x16 shift instructions (even in higher feature sets // like AVX), we lower the `ishl.i8x16` and `ushr.i8x16` to a sequence of instructions. The basic idea, @@ -1449,52 +2284,50 @@ fn lower_insn_to_regs>( // mov $(size_bits - 1), %dst // sub %tmp, %dst - let (ext_spec, ty) = match ctx.input_ty(insn, 0) { - types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32), - a if a == types::I32 || a == types::I64 => (None, a), - _ => unreachable!(), - }; - - let src = if let Some(ext_spec) = ext_spec { - RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec)) + let orig_ty = ty.unwrap(); + if orig_ty == types::I128 { + // clz upper, tmp1 + // clz lower, dst + // add dst, 64 + // cmp tmp1, 64 + // cmovnz tmp1, dst + let dsts = get_output_reg(ctx, outputs[0]); + let dst = dsts.regs()[0]; + let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let srcs = put_input_in_regs(ctx, inputs[0]); + let src_lo = srcs.regs()[0]; + let src_hi = srcs.regs()[1]; + emit_clz(ctx, types::I64, types::I64, src_hi, tmp1); + emit_clz(ctx, types::I64, types::I64, src_lo, dst); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Add, + RegMemImm::imm(64), + dst, + )); + ctx.emit(Inst::cmp_rmi_r(8, RegMemImm::imm(64), tmp1.to_reg())); + ctx.emit(Inst::cmove(8, CC::NZ, RegMem::reg(tmp1.to_reg()), dst)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Xor, + RegMemImm::reg(dsts.regs()[1].to_reg()), + dsts.regs()[1], + )); } else { - input_to_reg_mem(ctx, inputs[0]) - }; - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + let (ext_spec, ty) = match orig_ty { + types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32), + a if a == types::I32 || a == types::I64 => (None, a), + _ => unreachable!(), + }; + let src = if let Some(ext_spec) = ext_spec { + extend_input_to_reg(ctx, inputs[0], ext_spec) + } else { + put_input_in_reg(ctx, inputs[0]) + }; - let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); - ctx.emit(Inst::imm( - OperandSize::from_bytes(ty.bytes()), - u64::max_value(), - dst, - )); - - ctx.emit(Inst::unary_rm_r( - ty.bytes() as u8, - UnaryRmROpcode::Bsr, - src, - tmp, - )); - - ctx.emit(Inst::cmove( - ty.bytes() as u8, - CC::Z, - RegMem::reg(dst.to_reg()), - tmp, - )); - - ctx.emit(Inst::imm( - OperandSize::from_bytes(ty.bytes()), - ty.bits() as u64 - 1, - dst, - )); - - ctx.emit(Inst::alu_rmi_r( - ty == types::I64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp.to_reg()), - dst, - )); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + emit_clz(ctx, orig_ty, ty, src, dst); + } } Opcode::Ctz => { @@ -1504,29 +2337,47 @@ fn lower_insn_to_regs>( // bsf %src, %dst // mov $(size_bits), %tmp // cmovz %tmp, %dst - let ty = ctx.input_ty(insn, 0); - let ty = if ty.bits() < 32 { types::I32 } else { ty }; - debug_assert!(ty == types::I32 || ty == types::I64); + let orig_ty = ctx.input_ty(insn, 0); + if orig_ty == types::I128 { + // ctz src_lo, dst + // ctz src_hi, tmp1 + // add tmp1, 64 + // cmp dst, 64 + // cmovz tmp1, dst + let dsts = get_output_reg(ctx, outputs[0]); + let dst = dsts.regs()[0]; + let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let srcs = put_input_in_regs(ctx, inputs[0]); + let src_lo = srcs.regs()[0]; + let src_hi = srcs.regs()[1]; + emit_ctz(ctx, types::I64, types::I64, src_lo, dst); + emit_ctz(ctx, types::I64, types::I64, src_hi, tmp1); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Add, + RegMemImm::imm(64), + tmp1, + )); + ctx.emit(Inst::cmp_rmi_r(8, RegMemImm::imm(64), dst.to_reg())); + ctx.emit(Inst::cmove(8, CC::Z, RegMem::reg(tmp1.to_reg()), dst)); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Xor, + RegMemImm::reg(dsts.regs()[1].to_reg()), + dsts.regs()[1], + )); + } else { + let ty = if orig_ty.bits() < 32 { + types::I32 + } else { + orig_ty + }; + debug_assert!(ty == types::I32 || ty == types::I64); - let src = input_to_reg_mem(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); - ctx.emit(Inst::imm(OperandSize::Size32, ty.bits() as u64, tmp)); - - ctx.emit(Inst::unary_rm_r( - ty.bytes() as u8, - UnaryRmROpcode::Bsf, - src, - dst, - )); - - ctx.emit(Inst::cmove( - ty.bytes() as u8, - CC::Z, - RegMem::reg(tmp.to_reg()), - dst, - )); + let src = put_input_in_reg(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + emit_ctz(ctx, orig_ty, ty, src, dst); + } } Opcode::Popcnt => { @@ -1535,272 +2386,329 @@ fn lower_insn_to_regs>( let (ext_spec, ty) = match ctx.input_ty(insn, 0) { types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32), a if a == types::I32 || a == types::I64 => (None, a), + types::I128 => (None, types::I128), _ => unreachable!(), }; - let src = if let Some(ext_spec) = ext_spec { - RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec)) + let (srcs, ty): (SmallVec<[RegMem; 2]>, Type) = if let Some(ext_spec) = ext_spec { + ( + smallvec![RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec))], + ty, + ) + } else if ty == types::I128 { + let regs = put_input_in_regs(ctx, inputs[0]); + ( + smallvec![RegMem::reg(regs.regs()[0]), RegMem::reg(regs.regs()[1])], + types::I64, + ) } else { // N.B.: explicitly put input in a reg here because the width of the instruction // into which this RM op goes may not match the width of the input type (in fact, // it won't for i32.popcnt), and we don't want a larger than necessary load. - RegMem::reg(put_input_in_reg(ctx, inputs[0])) + (smallvec![RegMem::reg(put_input_in_reg(ctx, inputs[0]))], ty) }; - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - if ty == types::I64 { - let is_64 = true; + let mut dsts: SmallVec<[Reg; 2]> = smallvec![]; + for src in srcs { + let dst = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + dsts.push(dst.to_reg()); + if ty == types::I64 { + let is_64 = true; - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let cst = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let cst = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - // mov src, tmp1 - ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1)); + // mov src, tmp1 + ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1)); - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - 8, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); + // shr $1, tmp1 + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); - // mov 0x7777_7777_7777_7777, cst - ctx.emit(Inst::imm(OperandSize::Size64, 0x7777777777777777, cst)); + // mov 0x7777_7777_7777_7777, cst + ctx.emit(Inst::imm(OperandSize::Size64, 0x7777777777777777, cst)); - // andq cst, tmp1 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::And, - RegMemImm::reg(cst.to_reg()), - tmp1, - )); + // andq cst, tmp1 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::And, + RegMemImm::reg(cst.to_reg()), + tmp1, + )); - // mov src, tmp2 - ctx.emit(Inst::mov64_rm_r(src, tmp2)); + // mov src, tmp2 + ctx.emit(Inst::mov64_rm_r(src, tmp2)); - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); + // sub tmp1, tmp2 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Sub, + RegMemImm::reg(tmp1.to_reg()), + tmp2, + )); - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - 8, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); + // shr $1, tmp1 + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); - // and cst, tmp1 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::And, - RegMemImm::reg(cst.to_reg()), - tmp1, - )); + // and cst, tmp1 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::And, + RegMemImm::reg(cst.to_reg()), + tmp1, + )); - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); + // sub tmp1, tmp2 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Sub, + RegMemImm::reg(tmp1.to_reg()), + tmp2, + )); - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - 8, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); + // shr $1, tmp1 + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); - // and cst, tmp1 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::And, - RegMemImm::reg(cst.to_reg()), - tmp1, - )); + // and cst, tmp1 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::And, + RegMemImm::reg(cst.to_reg()), + tmp1, + )); - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); + // sub tmp1, tmp2 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Sub, + RegMemImm::reg(tmp1.to_reg()), + tmp2, + )); - // mov tmp2, dst - ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst)); + // mov tmp2, dst + ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst)); - // shr $4, dst - ctx.emit(Inst::shift_r(8, ShiftKind::ShiftRightLogical, Some(4), dst)); + // shr $4, dst + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftRightLogical, Some(4), dst)); - // add tmp2, dst - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::Add, - RegMemImm::reg(tmp2.to_reg()), - dst, - )); + // add tmp2, dst + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Add, + RegMemImm::reg(tmp2.to_reg()), + dst, + )); - // mov $0x0F0F_0F0F_0F0F_0F0F, cst - ctx.emit(Inst::imm(OperandSize::Size64, 0x0F0F0F0F0F0F0F0F, cst)); + // mov $0x0F0F_0F0F_0F0F_0F0F, cst + ctx.emit(Inst::imm(OperandSize::Size64, 0x0F0F0F0F0F0F0F0F, cst)); - // and cst, dst - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::And, - RegMemImm::reg(cst.to_reg()), - dst, - )); + // and cst, dst + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::And, + RegMemImm::reg(cst.to_reg()), + dst, + )); - // mov $0x0101_0101_0101_0101, cst - ctx.emit(Inst::imm(OperandSize::Size64, 0x0101010101010101, cst)); + // mov $0x0101_0101_0101_0101, cst + ctx.emit(Inst::imm(OperandSize::Size64, 0x0101010101010101, cst)); - // mul cst, dst - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::Mul, - RegMemImm::reg(cst.to_reg()), - dst, - )); + // mul cst, dst + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Mul, + RegMemImm::reg(cst.to_reg()), + dst, + )); - // shr $56, dst - ctx.emit(Inst::shift_r( - 8, - ShiftKind::ShiftRightLogical, - Some(56), - dst, - )); + // shr $56, dst + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(56), + dst, + )); + } else { + assert_eq!(ty, types::I32); + let is_64 = false; + + let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + + // mov src, tmp1 + ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1)); + + // shr $1, tmp1 + ctx.emit(Inst::shift_r( + 4, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); + + // andq $0x7777_7777, tmp1 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::And, + RegMemImm::imm(0x77777777), + tmp1, + )); + + // mov src, tmp2 + ctx.emit(Inst::mov64_rm_r(src, tmp2)); + + // sub tmp1, tmp2 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Sub, + RegMemImm::reg(tmp1.to_reg()), + tmp2, + )); + + // shr $1, tmp1 + ctx.emit(Inst::shift_r( + 4, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); + + // and 0x7777_7777, tmp1 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::And, + RegMemImm::imm(0x77777777), + tmp1, + )); + + // sub tmp1, tmp2 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Sub, + RegMemImm::reg(tmp1.to_reg()), + tmp2, + )); + + // shr $1, tmp1 + ctx.emit(Inst::shift_r( + 4, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); + + // and $0x7777_7777, tmp1 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::And, + RegMemImm::imm(0x77777777), + tmp1, + )); + + // sub tmp1, tmp2 + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Sub, + RegMemImm::reg(tmp1.to_reg()), + tmp2, + )); + + // mov tmp2, dst + ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst)); + + // shr $4, dst + ctx.emit(Inst::shift_r(4, ShiftKind::ShiftRightLogical, Some(4), dst)); + + // add tmp2, dst + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Add, + RegMemImm::reg(tmp2.to_reg()), + dst, + )); + + // and $0x0F0F_0F0F, dst + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::And, + RegMemImm::imm(0x0F0F0F0F), + dst, + )); + + // mul $0x0101_0101, dst + ctx.emit(Inst::alu_rmi_r( + is_64, + AluRmiROpcode::Mul, + RegMemImm::imm(0x01010101), + dst, + )); + + // shr $24, dst + ctx.emit(Inst::shift_r( + 4, + ShiftKind::ShiftRightLogical, + Some(24), + dst, + )); + } + } + + if dsts.len() == 1 { + let final_dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::gen_move(final_dst, dsts[0], types::I64)); } else { - assert_eq!(ty, types::I32); - let is_64 = false; - - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - - // mov src, tmp1 - ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1)); - - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - 4, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); - - // andq $0x7777_7777, tmp1 + assert!(dsts.len() == 2); + let final_dst = get_output_reg(ctx, outputs[0]); + ctx.emit(Inst::gen_move(final_dst.regs()[0], dsts[0], types::I64)); ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::And, - RegMemImm::imm(0x77777777), - tmp1, - )); - - // mov src, tmp2 - ctx.emit(Inst::mov64_rm_r(src, tmp2)); - - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); - - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - 4, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); - - // and 0x7777_7777, tmp1 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::And, - RegMemImm::imm(0x77777777), - tmp1, - )); - - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); - - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - 4, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); - - // and $0x7777_7777, tmp1 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::And, - RegMemImm::imm(0x77777777), - tmp1, - )); - - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); - - // mov tmp2, dst - ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst)); - - // shr $4, dst - ctx.emit(Inst::shift_r(4, ShiftKind::ShiftRightLogical, Some(4), dst)); - - // add tmp2, dst - ctx.emit(Inst::alu_rmi_r( - is_64, + true, AluRmiROpcode::Add, - RegMemImm::reg(tmp2.to_reg()), - dst, + RegMemImm::reg(dsts[1]), + final_dst.regs()[0], )); - - // and $0x0F0F_0F0F, dst ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::And, - RegMemImm::imm(0x0F0F0F0F), - dst, + true, + AluRmiROpcode::Xor, + RegMemImm::reg(final_dst.regs()[1].to_reg()), + final_dst.regs()[1], )); + } + } - // mul $0x0101_0101, dst - ctx.emit(Inst::alu_rmi_r( - is_64, - AluRmiROpcode::Mul, - RegMemImm::imm(0x01010101), - dst, - )); + Opcode::Bitrev => { + let ty = ctx.input_ty(insn, 0); + assert!( + ty == types::I8 + || ty == types::I16 + || ty == types::I32 + || ty == types::I64 + || ty == types::I128 + ); - // shr $24, dst - ctx.emit(Inst::shift_r( - 4, - ShiftKind::ShiftRightLogical, - Some(24), - dst, - )); + if ty == types::I128 { + let src = put_input_in_regs(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]); + emit_bitrev(ctx, src.regs()[0], dst.regs()[1], types::I64); + emit_bitrev(ctx, src.regs()[1], dst.regs()[0], types::I64); + } else { + let src = put_input_in_reg(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + emit_bitrev(ctx, src, dst, ty); } } @@ -1836,63 +2744,112 @@ fn lower_insn_to_regs>( let src_ty = ctx.input_ty(insn, 0); let dst_ty = ctx.output_ty(insn, 0); - // Sextend requires a sign-extended move, but all the other opcodes are simply a move - // from a zero-extended source. Here is why this works, in each case: - // - // - Bint: Bool-to-int. We always represent a bool as a 0 or 1, so we merely need to - // zero-extend here. - // - // - Breduce, Bextend: changing width of a boolean. We represent a bool as a 0 or 1, so - // again, this is a zero-extend / no-op. - // - // - Ireduce: changing width of an integer. Smaller ints are stored with undefined - // high-order bits, so we can simply do a copy. + if src_ty == types::I128 { + assert!(dst_ty.bits() <= 64); + assert!(op == Opcode::Ireduce); + let src = put_input_in_regs(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::gen_move(dst, src.regs()[0], types::I64)); + } else if dst_ty == types::I128 { + assert!(src_ty.bits() <= 64); + let src = put_input_in_reg(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]); + assert!(op == Opcode::Uextend || op == Opcode::Sextend || op == Opcode::Bint); + // Extend to 64 bits first. - if src_ty == types::I32 && dst_ty == types::I64 && op != Opcode::Sextend { - // As a particular x64 extra-pattern matching opportunity, all the ALU opcodes on - // 32-bits will zero-extend the upper 32-bits, so we can even not generate a - // zero-extended move in this case. - // TODO add loads and shifts here. - if let Some(_) = matches_input_any( - ctx, - inputs[0], - &[ - Opcode::Iadd, - Opcode::IaddIfcout, - Opcode::Isub, - Opcode::Imul, - Opcode::Band, - Opcode::Bor, - Opcode::Bxor, - ], - ) { - let src = put_input_in_reg(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::gen_move(dst, src, types::I64)); - return Ok(()); - } - } - - let src = input_to_reg_mem(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - let ext_mode = ExtMode::new(src_ty.bits(), dst_ty.bits()); - assert_eq!( - src_ty.bits() < dst_ty.bits(), - ext_mode.is_some(), - "unexpected extension: {} -> {}", - src_ty, - dst_ty - ); - - if let Some(ext_mode) = ext_mode { - if op == Opcode::Sextend { - ctx.emit(Inst::movsx_rm_r(ext_mode, src, dst)); + let ext_mode = ExtMode::new(src_ty.bits(), /* dst bits = */ 64); + if let Some(ext_mode) = ext_mode { + if op == Opcode::Sextend { + ctx.emit(Inst::movsx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0])); + } else { + ctx.emit(Inst::movzx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0])); + } } else { - ctx.emit(Inst::movzx_rm_r(ext_mode, src, dst)); + ctx.emit(Inst::mov64_rm_r(RegMem::reg(src), dst.regs()[0])); + } + + // Now generate the top 64 bits. + if op == Opcode::Sextend { + // Sign-extend: move dst[0] into dst[1] and arithmetic-shift right by 63 bits + // to spread the sign bit across all bits. + ctx.emit(Inst::gen_move( + dst.regs()[1], + dst.regs()[0].to_reg(), + types::I64, + )); + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightArithmetic, + Some(63), + dst.regs()[1], + )); + } else { + // Zero-extend: just zero the top word. + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Xor, + RegMemImm::reg(dst.regs()[1].to_reg()), + dst.regs()[1], + )); } } else { - ctx.emit(Inst::mov64_rm_r(src, dst)); + // Sextend requires a sign-extended move, but all the other opcodes are simply a move + // from a zero-extended source. Here is why this works, in each case: + // + // - Bint: Bool-to-int. We always represent a bool as a 0 or 1, so we merely need to + // zero-extend here. + // + // - Breduce, Bextend: changing width of a boolean. We represent a bool as a 0 or 1, so + // again, this is a zero-extend / no-op. + // + // - Ireduce: changing width of an integer. Smaller ints are stored with undefined + // high-order bits, so we can simply do a copy. + if src_ty == types::I32 && dst_ty == types::I64 && op != Opcode::Sextend { + // As a particular x64 extra-pattern matching opportunity, all the ALU opcodes on + // 32-bits will zero-extend the upper 32-bits, so we can even not generate a + // zero-extended move in this case. + // TODO add loads and shifts here. + if let Some(_) = matches_input_any( + ctx, + inputs[0], + &[ + Opcode::Iadd, + Opcode::IaddIfcout, + Opcode::Isub, + Opcode::Imul, + Opcode::Band, + Opcode::Bor, + Opcode::Bxor, + ], + ) { + let src = put_input_in_reg(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::gen_move(dst, src, types::I64)); + return Ok(()); + } + } + + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + + let ext_mode = ExtMode::new(src_ty.bits(), dst_ty.bits()); + assert_eq!( + src_ty.bits() < dst_ty.bits(), + ext_mode.is_some(), + "unexpected extension: {} -> {}", + src_ty, + dst_ty + ); + + if let Some(ext_mode) = ext_mode { + if op == Opcode::Sextend { + ctx.emit(Inst::movsx_rm_r(ext_mode, src, dst)); + } else { + ctx.emit(Inst::movzx_rm_r(ext_mode, src, dst)); + } + } else { + ctx.emit(Inst::mov64_rm_r(src, dst)); + } } } @@ -1901,7 +2858,7 @@ fn lower_insn_to_regs>( let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); let ty = ctx.input_ty(insn, 0); if !ty.is_vector() { - emit_cmp(ctx, insn); + let condcode = emit_cmp(ctx, insn, condcode); let cc = CC::from_intcc(condcode); ctx.emit(Inst::setcc(cc, dst)); } else { @@ -2108,10 +3065,19 @@ fn lower_insn_to_regs>( Opcode::FallthroughReturn | Opcode::Return => { for i in 0..ctx.num_inputs(insn) { - let src_reg = put_input_in_reg(ctx, inputs[i]); + let src_reg = put_input_in_regs(ctx, inputs[i]); let retval_reg = ctx.retval(i); let ty = ctx.input_ty(insn, i); - ctx.emit(Inst::gen_move(retval_reg.only_reg().unwrap(), src_reg, ty)); + assert!(src_reg.len() == retval_reg.len()); + let (_, tys) = Inst::rc_for_type(ty)?; + for ((&src, &dst), &ty) in src_reg + .regs() + .iter() + .zip(retval_reg.regs().iter()) + .zip(tys.iter()) + { + ctx.emit(Inst::gen_move(dst, src, ty)); + } } // N.B.: the Ret itself is generated by the ABI. } @@ -2147,13 +3113,13 @@ fn lower_insn_to_regs>( abi.emit_stack_pre_adjust(ctx); assert_eq!(inputs.len(), abi.num_args()); for (i, input) in inputs.iter().enumerate() { - let arg_reg = put_input_in_reg(ctx, *input); - abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg)); + let arg_regs = put_input_in_regs(ctx, *input); + abi.emit_copy_regs_to_arg(ctx, i, arg_regs); } abi.emit_call(ctx); for (i, output) in outputs.iter().enumerate() { - let retval_reg = get_output_reg(ctx, *output).only_reg().unwrap(); - abi.emit_copy_retval_to_regs(ctx, i, ValueRegs::one(retval_reg)); + let retval_regs = get_output_reg(ctx, *output); + abi.emit_copy_retval_to_regs(ctx, i, retval_regs); } abi.emit_stack_post_adjust(ctx); } @@ -2180,11 +3146,11 @@ fn lower_insn_to_regs>( ctx.emit_safepoint(Inst::TrapIf { trap_code, cc }); } else if op == Opcode::Trapif { let cond_code = ctx.data(insn).cond_code().unwrap(); - let cc = CC::from_intcc(cond_code); // Verification ensures that the input is always a single-def ifcmp. let ifcmp = matches_input(ctx, inputs[0], Opcode::Ifcmp).unwrap(); - emit_cmp(ctx, ifcmp); + let cond_code = emit_cmp(ctx, ifcmp, cond_code); + let cc = CC::from_intcc(cond_code); ctx.emit_safepoint(Inst::TrapIf { trap_code, cc }); } else { @@ -2266,7 +3232,9 @@ fn lower_insn_to_regs>( Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv => { let lhs = put_input_in_reg(ctx, inputs[0]); - let rhs = input_to_reg_mem(ctx, inputs[1]); + // We can't guarantee the RHS (if a load) is 128-bit aligned, so we + // must avoid merging a load here. + let rhs = RegMem::reg(put_input_in_reg(ctx, inputs[1])); let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); let ty = ty.unwrap(); @@ -2523,7 +3491,9 @@ fn lower_insn_to_regs>( } Opcode::FminPseudo | Opcode::FmaxPseudo => { - let lhs = input_to_reg_mem(ctx, inputs[0]); + // We can't guarantee the RHS (if a load) is 128-bit aligned, so we + // must avoid merging a load here. + let lhs = RegMem::reg(put_input_in_reg(ctx, inputs[0])); let rhs = put_input_in_reg(ctx, inputs[1]); let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); let ty = ty.unwrap(); @@ -2539,7 +3509,9 @@ fn lower_insn_to_regs>( } Opcode::Sqrt => { - let src = input_to_reg_mem(ctx, inputs[0]); + // We can't guarantee the RHS (if a load) is 128-bit aligned, so we + // must avoid merging a load here. + let src = RegMem::reg(put_input_in_reg(ctx, inputs[0])); let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); let ty = ty.unwrap(); @@ -2558,13 +3530,17 @@ fn lower_insn_to_regs>( } Opcode::Fpromote => { - let src = input_to_reg_mem(ctx, inputs[0]); + // We can't guarantee the RHS (if a load) is 128-bit aligned, so we + // must avoid merging a load here. + let src = RegMem::reg(put_input_in_reg(ctx, inputs[0])); let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtss2sd, src, dst)); } Opcode::Fdemote => { - let src = input_to_reg_mem(ctx, inputs[0]); + // We can't guarantee the RHS (if a load) is 128-bit aligned, so we + // must avoid merging a load here. + let src = RegMem::reg(put_input_in_reg(ctx, inputs[0])); let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); ctx.emit(Inst::xmm_unary_rm_r(SseOpcode::Cvtsd2ss, src, dst)); } @@ -2581,7 +3557,7 @@ fn lower_insn_to_regs>( let src = match ext_spec { Some(ext_spec) => RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec)), - None => input_to_reg_mem(ctx, inputs[0]), + None => RegMem::reg(put_input_in_reg(ctx, inputs[0])), }; let opcode = if output_ty == types::F32 { @@ -3096,7 +4072,7 @@ fn lower_insn_to_regs>( } Opcode::Fabs | Opcode::Fneg => { - let src = input_to_reg_mem(ctx, inputs[0]); + let src = RegMem::reg(put_input_in_reg(ctx, inputs[0])); let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); // In both cases, generate a constant and apply a single binary instruction: @@ -3392,59 +4368,64 @@ fn lower_insn_to_regs>( _ => unreachable!(), }; - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let is_xmm = elem_ty.is_float() || elem_ty.is_vector(); - - match (sign_extend, is_xmm) { - (true, false) => { - // The load is sign-extended only when the output size is lower than 64 bits, - // so ext-mode is defined in this case. - ctx.emit(Inst::movsx_rm_r(ext_mode.unwrap(), RegMem::mem(amode), dst)); - } - (false, false) => { - if elem_ty.bytes() == 8 { - // Use a plain load. - ctx.emit(Inst::mov64_m_r(amode, dst)) - } else { - // Use a zero-extended load. - ctx.emit(Inst::movzx_rm_r(ext_mode.unwrap(), RegMem::mem(amode), dst)) + if elem_ty == types::I128 { + let dsts = get_output_reg(ctx, outputs[0]); + ctx.emit(Inst::mov64_m_r(amode.clone(), dsts.regs()[0])); + ctx.emit(Inst::mov64_m_r(amode.offset(8), dsts.regs()[1])); + } else { + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + let is_xmm = elem_ty.is_float() || elem_ty.is_vector(); + match (sign_extend, is_xmm) { + (true, false) => { + // The load is sign-extended only when the output size is lower than 64 bits, + // so ext-mode is defined in this case. + ctx.emit(Inst::movsx_rm_r(ext_mode.unwrap(), RegMem::mem(amode), dst)); } - } - (_, true) => { - ctx.emit(match elem_ty { - types::F32 => Inst::xmm_mov(SseOpcode::Movss, RegMem::mem(amode), dst), - types::F64 => Inst::xmm_mov(SseOpcode::Movsd, RegMem::mem(amode), dst), - types::I8X8 => { - if sign_extend == true { - Inst::xmm_mov(SseOpcode::Pmovsxbw, RegMem::mem(amode), dst) - } else { - Inst::xmm_mov(SseOpcode::Pmovzxbw, RegMem::mem(amode), dst) + (false, false) => { + if elem_ty.bytes() == 8 { + // Use a plain load. + ctx.emit(Inst::mov64_m_r(amode, dst)) + } else { + // Use a zero-extended load. + ctx.emit(Inst::movzx_rm_r(ext_mode.unwrap(), RegMem::mem(amode), dst)) + } + } + (_, true) => { + ctx.emit(match elem_ty { + types::F32 => Inst::xmm_mov(SseOpcode::Movss, RegMem::mem(amode), dst), + types::F64 => Inst::xmm_mov(SseOpcode::Movsd, RegMem::mem(amode), dst), + types::I8X8 => { + if sign_extend == true { + Inst::xmm_mov(SseOpcode::Pmovsxbw, RegMem::mem(amode), dst) + } else { + Inst::xmm_mov(SseOpcode::Pmovzxbw, RegMem::mem(amode), dst) + } } - } - types::I16X4 => { - if sign_extend == true { - Inst::xmm_mov(SseOpcode::Pmovsxwd, RegMem::mem(amode), dst) - } else { - Inst::xmm_mov(SseOpcode::Pmovzxwd, RegMem::mem(amode), dst) + types::I16X4 => { + if sign_extend == true { + Inst::xmm_mov(SseOpcode::Pmovsxwd, RegMem::mem(amode), dst) + } else { + Inst::xmm_mov(SseOpcode::Pmovzxwd, RegMem::mem(amode), dst) + } } - } - types::I32X2 => { - if sign_extend == true { - Inst::xmm_mov(SseOpcode::Pmovsxdq, RegMem::mem(amode), dst) - } else { - Inst::xmm_mov(SseOpcode::Pmovzxdq, RegMem::mem(amode), dst) + types::I32X2 => { + if sign_extend == true { + Inst::xmm_mov(SseOpcode::Pmovsxdq, RegMem::mem(amode), dst) + } else { + Inst::xmm_mov(SseOpcode::Pmovzxdq, RegMem::mem(amode), dst) + } } - } - _ if elem_ty.is_vector() && elem_ty.bits() == 128 => { - Inst::xmm_mov(SseOpcode::Movups, RegMem::mem(amode), dst) - } - // TODO Specialize for different types: MOVUPD, MOVDQU - _ => unreachable!( - "unexpected type for load: {:?} - {:?}", - elem_ty, - elem_ty.bits() - ), - }); + _ if elem_ty.is_vector() && elem_ty.bits() == 128 => { + Inst::xmm_mov(SseOpcode::Movups, RegMem::mem(amode), dst) + } + // TODO Specialize for different types: MOVUPD, MOVDQU + _ => unreachable!( + "unexpected type for load: {:?} - {:?}", + elem_ty, + elem_ty.bits() + ), + }); + } } } } @@ -3491,17 +4472,23 @@ fn lower_insn_to_regs>( _ => unreachable!(), }; - let src = put_input_in_reg(ctx, inputs[0]); + if elem_ty == types::I128 { + let srcs = put_input_in_regs(ctx, inputs[0]); + ctx.emit(Inst::mov_r_m(8, srcs.regs()[0], addr.clone())); + ctx.emit(Inst::mov_r_m(8, srcs.regs()[1], addr.offset(8))); + } else { + let src = put_input_in_reg(ctx, inputs[0]); - ctx.emit(match elem_ty { - types::F32 => Inst::xmm_mov_r_m(SseOpcode::Movss, src, addr), - types::F64 => Inst::xmm_mov_r_m(SseOpcode::Movsd, src, addr), - _ if elem_ty.is_vector() && elem_ty.bits() == 128 => { - // TODO Specialize for different types: MOVUPD, MOVDQU, etc. - Inst::xmm_mov_r_m(SseOpcode::Movups, src, addr) - } - _ => Inst::mov_r_m(elem_ty.bytes() as u8, src, addr), - }); + ctx.emit(match elem_ty { + types::F32 => Inst::xmm_mov_r_m(SseOpcode::Movss, src, addr), + types::F64 => Inst::xmm_mov_r_m(SseOpcode::Movsd, src, addr), + _ if elem_ty.is_vector() && elem_ty.bits() == 128 => { + // TODO Specialize for different types: MOVUPD, MOVDQU, etc. + Inst::xmm_mov_r_m(SseOpcode::Movups, src, addr) + } + _ => Inst::mov_r_m(elem_ty.bytes() as u8, src, addr), + }); + } } Opcode::AtomicRmw => { @@ -3668,17 +4655,9 @@ fn lower_insn_to_regs>( }; let ty = ctx.output_ty(insn, 0); - let rhs = put_input_in_reg(ctx, rhs_input); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - let lhs = if is_int_or_ref_ty(ty) && ty.bytes() < 4 { - // Special case: since the higher bits are undefined per CLIF semantics, we - // can just apply a 32-bit cmove here. Force inputs into registers, to - // avoid partial spilling out-of-bounds with memory accesses, though. - // Sign-extend operands to 32, then do a cmove of size 4. - RegMem::reg(put_input_in_reg(ctx, lhs_input)) - } else { - input_to_reg_mem(ctx, lhs_input) - }; + let rhs = put_input_in_regs(ctx, rhs_input); + let dst = get_output_reg(ctx, outputs[0]); + let lhs = put_input_in_regs(ctx, lhs_input); // We request inversion of Equal to NotEqual here: taking LHS if equal would mean // take it if both CC::NP and CC::Z are set, the conjunction of which can't be @@ -3691,15 +4670,20 @@ fn lower_insn_to_regs>( assert_eq!(cond_code, FloatCC::Equal); } - ctx.emit(Inst::gen_move(dst, rhs, ty)); + emit_moves(ctx, dst, rhs, ty); match fcmp_results { FcmpCondResult::Condition(cc) => { - if is_int_or_ref_ty(ty) { - let size = u8::max(ty.bytes() as u8, 4); - ctx.emit(Inst::cmove(size, cc, lhs, dst)); + if is_int_or_ref_ty(ty) || ty == types::I128 || ty == types::B128 { + let size = ty.bytes() as u8; + emit_cmoves(ctx, size, cc, lhs, dst); } else { - ctx.emit(Inst::xmm_cmove(ty == types::F64, cc, lhs, dst)); + ctx.emit(Inst::xmm_cmove( + ty == types::F64, + cc, + RegMem::reg(lhs.only_reg().unwrap()), + dst.only_reg().unwrap(), + )); } } FcmpCondResult::AndConditions(_, _) => { @@ -3709,40 +4693,37 @@ fn lower_insn_to_regs>( } FcmpCondResult::InvertedEqualOrConditions(cc1, cc2) | FcmpCondResult::OrConditions(cc1, cc2) => { - if is_int_or_ref_ty(ty) { - let size = u8::max(ty.bytes() as u8, 4); - ctx.emit(Inst::cmove(size, cc1, lhs.clone(), dst)); - ctx.emit(Inst::cmove(size, cc2, lhs, dst)); + if is_int_or_ref_ty(ty) || ty == types::I128 { + let size = ty.bytes() as u8; + emit_cmoves(ctx, size, cc1, lhs.clone(), dst); + emit_cmoves(ctx, size, cc2, lhs, dst); } else { - ctx.emit(Inst::xmm_cmove(ty == types::F64, cc1, lhs.clone(), dst)); - ctx.emit(Inst::xmm_cmove(ty == types::F64, cc2, lhs, dst)); + ctx.emit(Inst::xmm_cmove( + ty == types::F64, + cc1, + RegMem::reg(lhs.only_reg().unwrap()), + dst.only_reg().unwrap(), + )); + ctx.emit(Inst::xmm_cmove( + ty == types::F64, + cc2, + RegMem::reg(lhs.only_reg().unwrap()), + dst.only_reg().unwrap(), + )); } } } } else { let ty = ty.unwrap(); - let mut size = ty.bytes() as u8; - let lhs = if is_int_or_ref_ty(ty) { - if size < 4 { - // Special case: since the higher bits are undefined per CLIF semantics, we - // can just apply a 32-bit cmove here. Force inputs into registers, to - // avoid partial spilling out-of-bounds with memory accesses, though. - size = 4; - RegMem::reg(put_input_in_reg(ctx, inputs[1])) - } else { - input_to_reg_mem(ctx, inputs[1]) - } - } else { - input_to_reg_mem(ctx, inputs[1]) - }; - - let rhs = put_input_in_reg(ctx, inputs[2]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + let size = ty.bytes() as u8; + let lhs = put_input_in_regs(ctx, inputs[1]); + let rhs = put_input_in_regs(ctx, inputs[2]); + let dst = get_output_reg(ctx, outputs[0]); let cc = if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) { - emit_cmp(ctx, icmp); let cond_code = ctx.data(icmp).cond_code().unwrap(); + let cond_code = emit_cmp(ctx, icmp, cond_code); CC::from_intcc(cond_code) } else { let sel_ty = ctx.input_ty(insn, 0); @@ -3768,21 +4749,26 @@ fn lower_insn_to_regs>( }; // This doesn't affect the flags. - ctx.emit(Inst::gen_move(dst, rhs, ty)); + emit_moves(ctx, dst, rhs, ty); - if is_int_or_ref_ty(ty) { - ctx.emit(Inst::cmove(size, cc, lhs, dst)); + if is_int_or_ref_ty(ty) || ty == types::I128 { + emit_cmoves(ctx, size, cc, lhs, dst); } else { debug_assert!(ty == types::F32 || ty == types::F64); - ctx.emit(Inst::xmm_cmove(ty == types::F64, cc, lhs, dst)); + ctx.emit(Inst::xmm_cmove( + ty == types::F64, + cc, + RegMem::reg(lhs.only_reg().unwrap()), + dst.only_reg().unwrap(), + )); } } } Opcode::Selectif | Opcode::SelectifSpectreGuard => { - let lhs = input_to_reg_mem(ctx, inputs[1]); - let rhs = put_input_in_reg(ctx, inputs[2]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + let lhs = put_input_in_regs(ctx, inputs[1]); + let rhs = put_input_in_regs(ctx, inputs[2]); + let dst = get_output_reg(ctx, outputs[0]); let ty = ctx.output_ty(insn, 0); // Verification ensures that the input is always a single-def ifcmp. @@ -3792,26 +4778,24 @@ fn lower_insn_to_regs>( .unwrap() .0; debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp); - emit_cmp(ctx, cmp_insn); + let cond_code = ctx.data(insn).cond_code().unwrap(); + let cond_code = emit_cmp(ctx, cmp_insn, cond_code); - let cc = CC::from_intcc(ctx.data(insn).cond_code().unwrap()); + let cc = CC::from_intcc(cond_code); - if is_int_or_ref_ty(ty) { + if is_int_or_ref_ty(ty) || ty == types::I128 { let size = ty.bytes() as u8; - if size == 1 { - // Sign-extend operands to 32, then do a cmove of size 4. - let lhs_se = ctx.alloc_tmp(types::I32).only_reg().unwrap(); - ctx.emit(Inst::movsx_rm_r(ExtMode::BL, lhs, lhs_se)); - ctx.emit(Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(rhs), dst)); - ctx.emit(Inst::cmove(4, cc, RegMem::reg(lhs_se.to_reg()), dst)); - } else { - ctx.emit(Inst::gen_move(dst, rhs, ty)); - ctx.emit(Inst::cmove(size, cc, lhs, dst)); - } + emit_moves(ctx, dst, rhs, ty); + emit_cmoves(ctx, size, cc, lhs, dst); } else { debug_assert!(ty == types::F32 || ty == types::F64); - ctx.emit(Inst::gen_move(dst, rhs, ty)); - ctx.emit(Inst::xmm_cmove(ty == types::F64, cc, lhs, dst)); + emit_moves(ctx, dst, rhs, ty); + ctx.emit(Inst::xmm_cmove( + ty == types::F64, + cc, + RegMem::reg(lhs.only_reg().unwrap()), + dst.only_reg().unwrap(), + )); } } @@ -3894,8 +4878,19 @@ fn lower_insn_to_regs>( // The quotient is in rax. ctx.emit(Inst::gen_move(dst, regs::rax(), input_ty)); } else { - // The remainder is in rdx. - ctx.emit(Inst::gen_move(dst, regs::rdx(), input_ty)); + if size == 1 { + // The remainder is in AH. Right-shift by 8 bits then move from rax. + ctx.emit(Inst::shift_r( + 8, + ShiftKind::ShiftRightLogical, + Some(8), + Writable::from_reg(regs::rax()), + )); + ctx.emit(Inst::gen_move(dst, regs::rax(), input_ty)); + } else { + // The remainder is in rdx. + ctx.emit(Inst::gen_move(dst, regs::rdx(), input_ty)); + } } } @@ -4297,6 +5292,38 @@ fn lower_insn_to_regs>( } } + Opcode::Iconcat => { + let ty = ctx.output_ty(insn, 0); + assert_eq!( + ty, + types::I128, + "Iconcat not expected to be used for non-128-bit type" + ); + assert_eq!(ctx.input_ty(insn, 0), types::I64); + assert_eq!(ctx.input_ty(insn, 1), types::I64); + let lo = put_input_in_reg(ctx, inputs[0]); + let hi = put_input_in_reg(ctx, inputs[1]); + let dst = get_output_reg(ctx, outputs[0]); + ctx.emit(Inst::gen_move(dst.regs()[0], lo, types::I64)); + ctx.emit(Inst::gen_move(dst.regs()[1], hi, types::I64)); + } + + Opcode::Isplit => { + let ty = ctx.input_ty(insn, 0); + assert_eq!( + ty, + types::I128, + "Iconcat not expected to be used for non-128-bit type" + ); + assert_eq!(ctx.output_ty(insn, 0), types::I64); + assert_eq!(ctx.output_ty(insn, 1), types::I64); + let src = put_input_in_regs(ctx, inputs[0]); + let dst_lo = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + let dst_hi = get_output_reg(ctx, outputs[1]).only_reg().unwrap(); + ctx.emit(Inst::gen_move(dst_lo, src.regs()[0], types::I64)); + ctx.emit(Inst::gen_move(dst_hi, src.regs()[1], types::I64)); + } + Opcode::IaddImm | Opcode::ImulImm | Opcode::UdivImm @@ -4384,9 +5411,9 @@ impl LowerBackend for X64Backend { let src_ty = ctx.input_ty(branches[0], 0); if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) { - emit_cmp(ctx, icmp); - let cond_code = ctx.data(icmp).cond_code().unwrap(); + let cond_code = emit_cmp(ctx, icmp, cond_code); + let cond_code = if op0 == Opcode::Brz { cond_code.inverse() } else { @@ -4416,6 +5443,32 @@ impl LowerBackend for X64Backend { } FcmpCondResult::InvertedEqualOrConditions(_, _) => unreachable!(), } + } else if src_ty == types::I128 { + let src = put_input_in_regs( + ctx, + InsnInput { + insn: branches[0], + input: 0, + }, + ); + let (half_cc, comb_op) = match op0 { + Opcode::Brz => (CC::Z, AluRmiROpcode::And8), + Opcode::Brnz => (CC::NZ, AluRmiROpcode::Or8), + _ => unreachable!(), + }; + let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + ctx.emit(Inst::cmp_rmi_r(8, RegMemImm::imm(0), src.regs()[0])); + ctx.emit(Inst::setcc(half_cc, tmp1)); + ctx.emit(Inst::cmp_rmi_r(8, RegMemImm::imm(0), src.regs()[1])); + ctx.emit(Inst::setcc(half_cc, tmp2)); + ctx.emit(Inst::alu_rmi_r( + false, + comb_op, + RegMemImm::reg(tmp1.to_reg()), + tmp2, + )); + ctx.emit(Inst::jmp_cond(CC::NZ, taken, not_taken)); } else if is_int_or_ref_ty(src_ty) || is_bool_ty(src_ty) { let src = put_input_in_reg( ctx, @@ -4483,8 +5536,8 @@ impl LowerBackend for X64Backend { }; if let Some(ifcmp) = matches_input(ctx, flag_input, Opcode::Ifcmp) { - emit_cmp(ctx, ifcmp); let cond_code = ctx.data(branches[0]).cond_code().unwrap(); + let cond_code = emit_cmp(ctx, ifcmp, cond_code); let cc = CC::from_intcc(cond_code); ctx.emit(Inst::jmp_cond(cc, taken, not_taken)); } else if let Some(ifcmp_sp) = matches_input(ctx, flag_input, Opcode::IfcmpSp) { diff --git a/cranelift/filetests/filetests/isa/x64/bitops-i128-run.clif b/cranelift/filetests/filetests/isa/x64/bitops-i128-run.clif new file mode 100644 index 0000000000..5795900438 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/bitops-i128-run.clif @@ -0,0 +1,27 @@ +test run +target x86_64 +feature "experimental_x64" + +function %ctz(i64, i64) -> i8 { +block0(v0: i64, v1: i64): + v2 = iconcat v0, v1 + v3 = ctz.i128 v2 + v4 = ireduce.i8 v3 + return v4 +} +; run: %ctz(0x00000000_00000000, 0x00000001_00000000) == 96 +; run: %ctz(0x00000000_00010000, 0x00000001_00000000) == 16 +; run: %ctz(0x00000000_00010000, 0x00000000_00000000) == 16 +; run: %ctz(0x00000000_00000000, 0x00000000_00000000) == 128 + +function %clz(i64, i64) -> i8 { +block0(v0: i64, v1: i64): + v2 = iconcat v0, v1 + v3 = clz.i128 v2 + v4 = ireduce.i8 v3 + return v4 +} +; run: %clz(0x00000000_00000000, 0x00000001_00000000) == 31 +; run: %clz(0x00000000_00010000, 0x00000001_00000000) == 31 +; run: %clz(0x00000000_00010000, 0x00000000_00000000) == 111 +; run: %clz(0x00000000_00000000, 0x00000000_00000000) == 128 diff --git a/cranelift/filetests/filetests/isa/x64/bitrev-i128-run.clif b/cranelift/filetests/filetests/isa/x64/bitrev-i128-run.clif new file mode 100644 index 0000000000..64ea96716c --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/bitrev-i128-run.clif @@ -0,0 +1,47 @@ +test run +target x86_64 +feature "experimental_x64" + +function %reverse_bits_zero() -> b1 { +block0: + v0 = iconst.i64 0 + v1 = iconcat v0, v0 + v2 = bitrev.i128 v1 + v3 = icmp eq v2, v1 + return v3 +} +; run + +function %reverse_bits_one() -> b1 { +block0: + v0 = iconst.i64 0 + v1 = iconst.i64 1 + v2 = iconcat v0, v1 + + v3 = bitrev.i128 v2 + + v4 = iconst.i64 0x8000_0000_0000_0000 + v5 = iconst.i64 0 + v6 = iconcat v4, v5 + + v7 = icmp eq v3, v6 + return v7 +} +; run + +function %reverse_bits() -> b1 { +block0: + v0 = iconst.i64 0x06AD_8667_69EC_41BA + v1 = iconst.i64 0x6C83_D81A_6E28_83AB + v2 = iconcat v0, v1 + + v3 = bitrev.i128 v2 + + v4 = iconst.i64 0xD5C11476581BC136 + v5 = iconst.i64 0x5D823796E661B560 + v6 = iconcat v4, v5 + + v7 = icmp eq v3, v6 + return v7 +} +; run diff --git a/cranelift/filetests/filetests/isa/x64/floating-point.clif b/cranelift/filetests/filetests/isa/x64/floating-point.clif new file mode 100644 index 0000000000..b3b5907210 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/floating-point.clif @@ -0,0 +1,26 @@ +test compile +target x86_64 +feature "experimental_x64" + +function %f(f64) -> f64 { +block0(v0: f64): + v1 = fabs.f64 v0 + return v1 +} +; check: movabsq $$9223372036854775807, %rsi +; nextln: movq %rsi, %xmm1 +; nextln: andpd %xmm0, %xmm1 +; nextln: movaps %xmm1, %xmm0 + + +function %f(i64) -> f64 { +block0(v0: i64): + v1 = load.f64 v0 + v2 = fabs.f64 v1 + return v2 +} +; check: movsd 0(%rdi), %xmm0 +; nextln: movabsq $$9223372036854775807, %rsi +; nextln: movq %rsi, %xmm1 +; nextln: andpd %xmm0, %xmm1 +; nextln: movaps %xmm1, %xmm0 diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif new file mode 100644 index 0000000000..e7ee34f283 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -0,0 +1,1082 @@ +test compile +target x86_64 +feature "experimental_x64" + +function %f0(i128, i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i128): + + v2 = iadd v0, v1 +; nextln: addq %rdx, %rdi +; nextln: adcq %rcx, %rsi + + return v2 +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f1(i128, i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i128): + + v2 = isub v0, v1 +; nextln: subq %rdx, %rdi +; nextln: sbbq %rcx, %rsi + + return v2 +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f2(i128, i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i128): + + v2 = band v0, v1 +; nextln: andq %rdx, %rdi +; nextln: andq %rcx, %rsi + + return v2 +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f3(i128, i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i128): + + v2 = bor v0, v1 +; nextln: orq %rdx, %rdi +; nextln: orq %rcx, %rsi + + return v2 +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f4(i128, i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i128): + + v2 = bxor v0, v1 +; nextln: xorq %rdx, %rdi +; nextln: xorq %rcx, %rsi + + return v2 +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f5(i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128): + + v1 = bnot v0 +; nextln: notq %rdi +; nextln: notq %rsi + + return v1 +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f6(i128, i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i128): +; v0 in rdi:rsi, v1 in rdx:rcx + + v2 = imul v0, v1 +; nextln: movq %rsi, %rax +; nextln: movq %rcx, %r8 +; nextln: movq %rdi, %rsi +; nextln: imulq %rdx, %rsi +; nextln: movq %rdi, %rcx +; nextln: imulq %r8, %rcx +; nextln: imulq %rdx, %rax +; nextln: addq %rax, %rcx +; nextln: movq %rdi, %rax +; nextln: mul %rdx +; nextln: addq %rdx, %rcx +; nextln: movq %rsi, %rax +; nextln: movq %rcx, %rdx + + return v2 +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f7(i64, i64) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i64, v1: i64): + v2 = iconcat.i64 v0, v1 +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx + + return v2 +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f8(i128) -> i64, i64 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128): + v1, v2 = isplit.i128 v0 +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx + + return v1, v2 +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f9(i128, i128) -> b1 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i128): + v2 = icmp eq v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setz %al +; nextln: cmpq %rdx, %rdi +; nextln: setz %r8b +; nextln: andq %rax, %r8 +; nextln: andq $$1, %r8 +; nextln: setnz %al + + v3 = icmp ne v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setnz %al +; nextln: cmpq %rdx, %rdi +; nextln: setnz %r8b +; nextln: orq %rax, %r8 +; nextln: andq $$1, %r8 +; nextln: setnz %r8b + + v4 = icmp slt v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setl %r9b +; nextln: setz %al +; nextln: cmpq %rdx, %rdi +; nextln: setb %r10b +; nextln: andq %rax, %r10 +; nextln: orq %r9, %r10 +; nextln: andq $$1, %r10 +; nextln: setnz %r9b + + v5 = icmp sle v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setl %r10b +; nextln: setz %al +; nextln: cmpq %rdx, %rdi +; nextln: setbe %r11b +; nextln: andq %rax, %r11 +; nextln: orq %r10, %r11 +; nextln: andq $$1, %r11 +; nextln: setnz %r10b + + v6 = icmp sgt v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setnle %r11b +; nextln: setz %al +; nextln: cmpq %rdx, %rdi +; nextln: setnbe %r12b +; nextln: andq %rax, %r12 +; nextln: orq %r11, %r12 +; nextln: andq $$1, %r12 +; nextln: setnz %r11b + + v7 = icmp sge v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setnle %r12b +; nextln: setz %al +; nextln: cmpq %rdx, %rdi +; nextln: setnb %r13b +; nextln: andq %rax, %r13 +; nextln: orq %r12, %r13 +; nextln: andq $$1, %r13 +; nextln: setnz %r12b + + v8 = icmp ult v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setb %r13b +; nextln: setz %al +; nextln: cmpq %rdx, %rdi +; nextln: setb %r14b +; nextln: andq %rax, %r14 +; nextln: orq %r13, %r14 +; nextln: andq $$1, %r14 +; nextln: setnz %r13b + + v9 = icmp ule v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setb %r14b +; nextln: setz %al +; nextln: cmpq %rdx, %rdi +; nextln: setbe %bl +; nextln: andq %rax, %rbx +; nextln: orq %r14, %rbx +; nextln: andq $$1, %rbx +; nextln: setnz %r14b + + v10 = icmp ugt v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setnbe %bl +; nextln: setz %r15b +; nextln: cmpq %rdx, %rdi +; nextln: setnbe %al +; nextln: andq %r15, %rax +; nextln: orq %rbx, %rax +; nextln: andq $$1, %rax +; nextln: setnz %bl + + v11 = icmp uge v0, v1 +; check: cmpq %rcx, %rsi +; nextln: setnbe %sil +; nextln: setz %cl +; nextln: cmpq %rdx, %rdi +; nextln: setnb %dil +; nextln: andq %rcx, %rdi +; nextln: orq %rsi, %rdi +; nextln: andq $$1, %rdi +; nextln: setnz %sil + + v12 = band v2, v3 + v13 = band v4, v5 + v14 = band v6, v7 + v15 = band v8, v9 + v16 = band v10, v11 + v17 = band v12, v13 + v18 = band v14, v15 + v19 = band v17, v18 + v20 = band v19, v16 + + return v20 +; check: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f10(i128) -> i32 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128): + brz v0, block1 +; check: cmpq $$0, %rdi +; nextln: setz %dil +; nextln: cmpq $$0, %rsi +; nextln: setz %sil +; nextln: andb %dil, %sil +; nextln: jnz label1; j label2 + + jump block2 + +block1: + v1 = iconst.i32 1 + return v1 + +block2: + v2 = iconst.i32 2 + return v2 + +; check: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f11(i128) -> i32 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128): + brnz v0, block1 +; check: cmpq $$0, %rdi +; nextln: setnz %dil +; nextln: cmpq $$0, %rsi +; nextln: setnz %sil +; nextln: orb %dil, %sil +; nextln: jnz label1; j label2 + jump block2 + +block1: + v1 = iconst.i32 1 + return v1 + +block2: + v2 = iconst.i32 2 + return v2 + +; check: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f12(i64) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i64): + v1 = uextend.i128 v0 + return v1 + +; nextln: movq %rdi, %rsi +; nextln: xorq %rdi, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f13(i64) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i64): + v1 = sextend.i128 v0 + return v1 + +; nextln: movq %rdi, %rsi +; nextln: movq %rsi, %rdi +; nextln: sarq $$63, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f14(i8) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i8): + v1 = sextend.i128 v0 + return v1 + +; nextln: movsbq %dil, %rsi +; nextln: movq %rsi, %rdi +; nextln: sarq $$63, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f15(i8) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i8): + v1 = uextend.i128 v0 + return v1 + +; nextln: movzbq %dil, %rsi +; nextln: xorq %rdi, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +} + +function %f16(i128) -> i64 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128): + v1 = ireduce.i64 v0 + return v1 + +; nextln: movq %rdi, %rax + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f17(i128) -> i8 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128): + v1 = ireduce.i8 v0 + return v1 + +; nextln: movq %rdi, %rax + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f18(b1) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: b1): + v1 = bint.i128 v0 + return v1 + +; check: movzbq %dil, %rsi +; nextln: xorq %rdi, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f19(i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128): + v1 = popcnt.i128 v0 + return v1 + +; check: movq %rsi, %rdx +; nextln: movq %rdi, %rsi +; nextln: shrq $$1, %rsi +; nextln: movabsq $$8608480567731124087, %rcx +; nextln: andq %rcx, %rsi +; nextln: movq %rdi, %rax +; nextln: subq %rsi, %rax +; nextln: shrq $$1, %rsi +; nextln: andq %rcx, %rsi +; nextln: subq %rsi, %rax +; nextln: shrq $$1, %rsi +; nextln: andq %rcx, %rsi +; nextln: subq %rsi, %rax +; nextln: movq %rax, %rsi +; nextln: shrq $$4, %rsi +; nextln: addq %rax, %rsi +; nextln: movabsq $$1085102592571150095, %rdi +; nextln: andq %rdi, %rsi +; nextln: movabsq $$72340172838076673, %rdi +; nextln: imulq %rdi, %rsi +; nextln: shrq $$56, %rsi +; nextln: movq %rdx, %rax +; nextln: shrq $$1, %rax +; nextln: movabsq $$8608480567731124087, %rcx +; nextln: andq %rcx, %rax +; nextln: movq %rdx, %rdi +; nextln: subq %rax, %rdi +; nextln: shrq $$1, %rax +; nextln: andq %rcx, %rax +; nextln: subq %rax, %rdi +; nextln: shrq $$1, %rax +; nextln: andq %rcx, %rax +; nextln: subq %rax, %rdi +; nextln: movq %rdi, %rax +; nextln: shrq $$4, %rax +; nextln: addq %rdi, %rax +; nextln: movabsq $$1085102592571150095, %rdi +; nextln: andq %rdi, %rax +; nextln: movabsq $$72340172838076673, %rdi +; nextln: imulq %rdi, %rax +; nextln: shrq $$56, %rax +; nextln: addq %rax, %rsi +; nextln: xorq %rdi, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx + + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + + +function %f20(i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128): + v1 = bitrev.i128 v0 + return v1 + +; check: movq %rdi, %rcx +; nextln: movq %rcx, %rdi +; nextln: movabsq $$6148914691236517205, %rax +; nextln: shrq $$1, %rdi +; nextln: andq %rax, %rdi +; nextln: andq %rcx, %rax +; nextln: shlq $$1, %rax +; nextln: movq %rax, %rcx +; nextln: orq %rdi, %rcx +; nextln: movq %rcx, %rdi +; nextln: movabsq $$3689348814741910323, %rax +; nextln: shrq $$2, %rdi +; nextln: andq %rax, %rdi +; nextln: andq %rcx, %rax +; nextln: shlq $$2, %rax +; nextln: movq %rax, %rcx +; nextln: orq %rdi, %rcx +; nextln: movq %rcx, %rdi +; nextln: movabsq $$1085102592571150095, %rax +; nextln: shrq $$4, %rdi +; nextln: andq %rax, %rdi +; nextln: andq %rcx, %rax +; nextln: shlq $$4, %rax +; nextln: movq %rax, %rcx +; nextln: orq %rdi, %rcx +; nextln: movq %rcx, %rdi +; nextln: movabsq $$71777214294589695, %rax +; nextln: shrq $$8, %rdi +; nextln: andq %rax, %rdi +; nextln: andq %rcx, %rax +; nextln: shlq $$8, %rax +; nextln: movq %rax, %rcx +; nextln: orq %rdi, %rcx +; nextln: movq %rcx, %rdi +; nextln: movabsq $$281470681808895, %rax +; nextln: shrq $$16, %rdi +; nextln: andq %rax, %rdi +; nextln: andq %rcx, %rax +; nextln: shlq $$16, %rax +; nextln: orq %rdi, %rax +; nextln: movq %rax, %rcx +; nextln: movl $$-1, %edi +; nextln: shrq $$32, %rcx +; nextln: andq %rdi, %rcx +; nextln: andq %rax, %rdi +; nextln: shlq $$32, %rdi +; nextln: orq %rcx, %rdi +; nextln: movq %rsi, %rcx +; nextln: movq %rcx, %rsi +; nextln: movabsq $$6148914691236517205, %rax +; nextln: shrq $$1, %rsi +; nextln: andq %rax, %rsi +; nextln: andq %rcx, %rax +; nextln: shlq $$1, %rax +; nextln: movq %rax, %rcx +; nextln: orq %rsi, %rcx +; nextln: movq %rcx, %rsi +; nextln: movabsq $$3689348814741910323, %rax +; nextln: shrq $$2, %rsi +; nextln: andq %rax, %rsi +; nextln: andq %rcx, %rax +; nextln: shlq $$2, %rax +; nextln: movq %rax, %rcx +; nextln: orq %rsi, %rcx +; nextln: movq %rcx, %rsi +; nextln: movabsq $$1085102592571150095, %rax +; nextln: shrq $$4, %rsi +; nextln: andq %rax, %rsi +; nextln: andq %rcx, %rax +; nextln: shlq $$4, %rax +; nextln: movq %rax, %rcx +; nextln: orq %rsi, %rcx +; nextln: movq %rcx, %rsi +; nextln: movabsq $$71777214294589695, %rax +; nextln: shrq $$8, %rsi +; nextln: andq %rax, %rsi +; nextln: andq %rcx, %rax +; nextln: shlq $$8, %rax +; nextln: movq %rax, %rcx +; nextln: orq %rsi, %rcx +; nextln: movq %rcx, %rsi +; nextln: movabsq $$281470681808895, %rax +; nextln: shrq $$16, %rsi +; nextln: andq %rax, %rsi +; nextln: andq %rcx, %rax +; nextln: shlq $$16, %rax +; nextln: orq %rsi, %rax +; nextln: movq %rax, %rsi +; nextln: movl $$-1, %ecx +; nextln: shrq $$32, %rsi +; nextln: andq %rcx, %rsi +; nextln: andq %rax, %rcx +; nextln: shlq $$32, %rcx +; nextln: orq %rsi, %rcx +; nextln: movq %rcx, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f21(i128, i32) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i32): + v2 = ushr v0, v1 + return v2 + +; check: movq %rdi, %rax +; nextln: movq %rsi, %rdi +; nextln: movq %rdi, %rsi +; nextln: movq %rdx, %rcx +; nextln: shrq %cl, %rsi +; nextln: movq %rdx, %rcx +; nextln: shrq %cl, %rax +; nextln: movl $$64, %ecx +; nextln: subq %rdx, %rcx +; nextln: shlq %cl, %rdi +; nextln: orq %rax, %rdi +; nextln: xorq %rax, %rax +; nextln: xorq %rcx, %rcx +; nextln: andq $$64, %rdx +; nextln: cmovzq %rsi, %rax +; nextln: cmovzq %rdi, %rcx +; nextln: cmovnzq %rsi, %rcx +; nextln: movq %rax, %rdx +; nextln: movq %rcx, %rax + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f22(i128, i32) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i32): + v2 = ishl v0, v1 + return v2 + +; check: movq %rsi, %rax +; nextln: movq %rdi, %rsi +; nextln: movq %rdx, %rcx +; nextln: shlq %cl, %rsi +; nextln: movq %rdx, %rcx +; nextln: shlq %cl, %rax +; nextln: movl $$64, %ecx +; nextln: subq %rdx, %rcx +; nextln: shrq %cl, %rdi +; nextln: orq %rax, %rdi +; nextln: xorq %rax, %rax +; nextln: xorq %rcx, %rcx +; nextln: andq $$64, %rdx +; nextln: cmovzq %rdi, %rcx +; nextln: cmovzq %rsi, %rax +; nextln: cmovnzq %rsi, %rcx +; nextln: movq %rcx, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f23(i128, i32) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i32): + v2 = sshr v0, v1 + return v2 + +; check: movq %rdi, %r8 +; nextln: movq %rsi, %rdi +; nextln: movq %rdi, %rsi +; nextln: movq %rdx, %rcx +; nextln: sarq %cl, %rsi +; nextln: movq %rdx, %rcx +; nextln: sarq %cl, %r8 +; nextln: movl $$64, %ecx +; nextln: subq %rdx, %rcx +; nextln: movq %rdi, %rax +; nextln: shlq %cl, %rax +; nextln: orq %r8, %rax +; nextln: sarq $$63, %rdi +; nextln: xorq %rcx, %rcx +; nextln: andq $$64, %rdx +; nextln: cmovzq %rsi, %rdi +; nextln: cmovzq %rax, %rcx +; nextln: cmovnzq %rsi, %rcx +; nextln: movq %rcx, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f24(i128, i32) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i32): + v2 = rotr.i128 v0, v1 + return v2 + +; check: movq %rsi, %r9 +; nextln: movq %rdx, %rcx +; nextln: shrq %cl, %r9 +; nextln: movq %rdi, %rax +; nextln: movq %rdx, %rcx +; nextln: shrq %cl, %rax +; nextln: movl $$64, %ecx +; nextln: subq %rdx, %rcx +; nextln: movq %rsi, %r10 +; nextln: shlq %cl, %r10 +; nextln: orq %rax, %r10 +; nextln: xorq %r8, %r8 +; nextln: xorq %rax, %rax +; nextln: movq %rdx, %rcx +; nextln: andq $$64, %rcx +; nextln: cmovzq %r9, %r8 +; nextln: cmovzq %r10, %rax +; nextln: cmovnzq %r9, %rax +; nextln: movl $$128, %r9d +; nextln: subq %rdx, %r9 +; nextln: movq %rdi, %rdx +; nextln: movq %r9, %rcx +; nextln: shlq %cl, %rdx +; nextln: movq %r9, %rcx +; nextln: shlq %cl, %rsi +; nextln: movl $$64, %ecx +; nextln: subq %r9, %rcx +; nextln: movq %rdi, %r10 +; nextln: shrq %cl, %r10 +; nextln: orq %rsi, %r10 +; nextln: xorq %rsi, %rsi +; nextln: xorq %rdi, %rdi +; nextln: andq $$64, %r9 +; nextln: cmovzq %r10, %rdi +; nextln: cmovzq %rdx, %rsi +; nextln: cmovnzq %rdx, %rdi +; nextln: orq %rax, %rsi +; nextln: orq %r8, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f25(i128, i32) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i32): + v2 = rotl.i128 v0, v1 + return v2 + +; check: movq %rdi, %r9 +; nextln: movq %rdx, %rcx +; nextln: shlq %cl, %r9 +; nextln: movq %rsi, %rax +; nextln: movq %rdx, %rcx +; nextln: shlq %cl, %rax +; nextln: movl $$64, %ecx +; nextln: subq %rdx, %rcx +; nextln: movq %rdi, %r10 +; nextln: shrq %cl, %r10 +; nextln: orq %rax, %r10 +; nextln: xorq %r8, %r8 +; nextln: xorq %rax, %rax +; nextln: movq %rdx, %rcx +; nextln: andq $$64, %rcx +; nextln: cmovzq %r10, %rax +; nextln: cmovzq %r9, %r8 +; nextln: cmovnzq %r9, %rax +; nextln: movl $$128, %r9d +; nextln: subq %rdx, %r9 +; nextln: movq %rsi, %rdx +; nextln: movq %r9, %rcx +; nextln: shrq %cl, %rdx +; nextln: movq %r9, %rcx +; nextln: shrq %cl, %rdi +; nextln: movl $$64, %ecx +; nextln: subq %r9, %rcx +; nextln: shlq %cl, %rsi +; nextln: orq %rdi, %rsi +; nextln: xorq %rdi, %rdi +; nextln: xorq %rcx, %rcx +; nextln: andq $$64, %r9 +; nextln: cmovzq %rdx, %rdi +; nextln: cmovzq %rsi, %rcx +; nextln: cmovnzq %rdx, %rcx +; nextln: orq %r8, %rcx +; nextln: orq %rax, %rdi +; nextln: movq %rcx, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f26(i128, i64) { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128, v1: i64): + store.i128 v0, v1 + return + +; check: movq %rdi, 0(%rdx) +; nextln: movq %rsi, 8(%rdx) + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f27(i64) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i64): + v1 = load.i128 v0 + return v1 + +; check: movq 0(%rdi), %rsi +; nextln: movq 8(%rdi), %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} + +function %f28(i128, b1) -> i128 { +block0(v0: i128, v1: b1): + v2 = iconst.i128 0 + brnz v1, block1(v2) + jump block2(v2) + +block1(v3: i128): + v4 = iconst.i128 1 + v5 = iadd.i128 v3, v4 + return v5 + +block2(v6: i128): + v7 = iconst.i128 2 + v8 = iadd.i128 v6, v7 + return v8 + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: testb $$1, %dl +; nextln: jnz label1; j label2 +; check: Block 1: +; check: movl $$0, %esi +; nextln: movl $$0, %edi +; nextln: movl $$1, %eax +; nextln: movl $$0, %ecx +; nextln: addq %rax, %rsi +; nextln: adcq %rcx, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +; check: Block 2: +; check: movl $$0, %esi +; nextln: movl $$0, %edi +; nextln: movl $$2, %eax +; nextln: movl $$0, %ecx +; nextln: addq %rax, %rsi +; nextln: adcq %rcx, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +} + +function %f29(i128, i128, i64, i128, i128, i128) -> i128 { + +block0(v0: i128, v1: i128, v2: i64, v3: i128, v4: i128, v5: i128): + v6 = iadd.i128 v0, v1 + v7 = uextend.i128 v2 + v8 = iadd.i128 v3, v7 + v9 = iadd.i128 v4, v5 + v10 = iadd.i128 v6, v8 + v11 = iadd.i128 v9, v10 + return v11 + +; check: movq %rsp, %rbp +; nextln: subq $$16, %rsp +; nextln: movq %r12, 0(%rsp) +; nextln: movq %r13, 8(%rsp) +; nextln: virtual_sp_offset_adjust 16 +; nextln: movq 16(%rbp), %r9 +; nextln: movq 24(%rbp), %r10 +; nextln: movq 32(%rbp), %r12 +; nextln: movq 40(%rbp), %r11 +; nextln: movq 48(%rbp), %rax +; nextln: movq 56(%rbp), %r13 +; nextln: addq %rdx, %rdi +; nextln: adcq %rcx, %rsi +; nextln: xorq %rcx, %rcx +; nextln: addq %r8, %r9 +; nextln: adcq %rcx, %r10 +; nextln: addq %rax, %r12 +; nextln: adcq %r13, %r11 +; nextln: addq %r9, %rdi +; nextln: adcq %r10, %rsi +; nextln: addq %rdi, %r12 +; nextln: adcq %rsi, %r11 +; nextln: movq %r12, %rax +; nextln: movq %r11, %rdx +; nextln: movq 0(%rsp), %r12 +; nextln: movq 8(%rsp), %r13 +; nextln: addq $$16, %rsp +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +} + +function %f30(i128) -> i128, i128, i128, i64, i128, i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i128): + v1 = ireduce.i64 v0 + return v0, v0, v0, v1, v0, v0 + +; likely to change with regalloc -- just check the stores into the retval area: + +; check: movq %r8, 0(%rsi) +; nextln: movq %r9, 8(%rsi) +; nextln: movq %r10, 16(%rsi) +; nextln: movq %r11, 24(%rsi) +; nextln: movq %r12, 32(%rsi) +; nextln: movq %r13, 48(%rsi) +; nextln: movq %r14, 56(%rsi) +; nextln: movq %rdi, 64(%rsi) +; nextln: movq %rbx, 72(%rsi) + +} + +function %f31(i128, i128) -> i128, i128 { + fn0 = %g(i128, i128) -> i128, i128 +block0(v0: i128, v1: i128): + v2, v3 = call fn0(v0, v1) + return v2, v3 + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: subq $$16, %rsp +; nextln: movq %r12, 0(%rsp) +; nextln: virtual_sp_offset_adjust 8 +; nextln: movq %r8, %r12 +; nextln: subq $$16, %rsp +; nextln: virtual_sp_offset_adjust 16 +; nextln: lea 0(%rsp), %r8 +; nextln: load_ext_name %g+0, %rax +; nextln: call *%rax +; nextln: movq 0(%rsp), %rsi +; nextln: movq 8(%rsp), %rdi +; nextln: addq $$16, %rsp +; nextln: virtual_sp_offset_adjust -16 +; nextln: movq %rsi, 0(%r12) +; nextln: movq %rdi, 8(%r12) +; nextln: movq 0(%rsp), %r12 +; nextln: addq $$16, %rsp +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +} + +function %f32(i128) -> i128 { +block0(v0: i128): + v1 = clz.i128 v0 + return v1 + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: movabsq $$-1, %rcx +; nextln: bsrq %rsi, %rax +; nextln: cmovzq %rcx, %rax +; nextln: movl $$63, %esi +; nextln: subq %rax, %rsi +; nextln: movabsq $$-1, %rcx +; nextln: bsrq %rdi, %rax +; nextln: cmovzq %rcx, %rax +; nextln: movl $$63, %edi +; nextln: subq %rax, %rdi +; nextln: addq $$64, %rdi +; nextln: cmpq $$64, %rsi +; nextln: cmovnzq %rsi, %rdi +; nextln: xorq %rsi, %rsi +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +} + +function %f33(i128) -> i128 { +block0(v0: i128): + v1 = ctz.i128 v0 + return v1 +} + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: movq %rsi, %rax +; nextln: movl $$64, %ecx +; nextln: bsfq %rdi, %rsi +; nextln: cmovzq %rcx, %rsi +; nextln: movl $$64, %ecx +; nextln: bsfq %rax, %rdi +; nextln: cmovzq %rcx, %rdi +; nextln: addq $$64, %rdi +; nextln: cmpq $$64, %rsi +; nextln: cmovzq %rdi, %rsi +; nextln: xorq %rdi, %rdi +; nextln: movq %rsi, %rax +; nextln: movq %rdi, %rdx +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret diff --git a/cranelift/filetests/filetests/isa/x64/select-i128.clif b/cranelift/filetests/filetests/isa/x64/select-i128.clif new file mode 100644 index 0000000000..3492a71997 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/select-i128.clif @@ -0,0 +1,29 @@ +test compile +target x86_64 +feature "experimental_x64" + +function %f0(i32, i128, i128) -> i128 { +; check: pushq %rbp +; nextln: movq %rsp, %rbp + +block0(v0: i32, v1: i128, v2: i128): + + v3 = iconst.i32 42 + v4 = icmp.i32 eq v0, v3 +; nextln: movl $$42, %eax +; nextln: cmpl %eax, %edi + + v5 = select.i128 v4, v1, v2 +; nextln: cmovzq %rsi, %rcx +; nextln: cmovzq %rdx, %r8 + + return v5 +; nextln: movq %rcx, %rax +; nextln: movq %r8, %rdx + +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +} + diff --git a/cranelift/filetests/filetests/isa/x64/shift-i128-run.clif b/cranelift/filetests/filetests/isa/x64/shift-i128-run.clif new file mode 100644 index 0000000000..37bc4667e7 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/shift-i128-run.clif @@ -0,0 +1,106 @@ +test run +target x86_64 +feature "experimental_x64" + +function %ishl1() -> b1 { +block0: + v0 = iconst.i64 0x01010101_01010101 + v1 = iconcat v0, v0 + v2 = iconst.i32 2 + v3 = ishl.i128 v1, v2 + v4 = iconst.i64 0x04040404_04040404 + v5 = iconcat v4, v4 + v6 = icmp eq v3, v5 + return v6 +} +; run + +function %ishl2() -> b1 { +block0: + v0 = iconst.i64 0x01010101_01010101 + v1 = iconst.i64 0x01010101_01010101 + v2 = iconcat v0, v1 + v3 = iconst.i32 9 + v4 = ishl.i128 v2, v3 + v5 = iconst.i64 0x02020202_02020200 + v6 = iconst.i64 0x02020202_02020202 + v7 = iconcat v5, v6 + v8 = icmp eq v4, v7 + return v8 +} +; run + +function %ishl3() -> b1 { +block0: + v0 = iconst.i64 0x01010101_01010101 + v1 = iconst.i64 0xffffffff_ffffffff + v2 = iconcat v0, v1 + v3 = iconst.i32 66 + v4 = ishl.i128 v2, v3 + v5 = iconst.i64 0x00000000_00000000 + v6 = iconst.i64 0x04040404_04040404 + v7 = iconcat v5, v6 + v8 = icmp eq v4, v7 + return v8 +} +; run + +function %ushr1() -> b1 { +block0: + v0 = iconst.i64 0x01010101_01010101 + v1 = iconst.i64 0x01010101_01010101 + v2 = iconcat v0, v1 + v3 = iconst.i32 2 + v4 = ushr.i128 v2, v3 + v5 = iconst.i64 0x40404040_40404040 + v6 = iconst.i64 0x00404040_40404040 + v7 = iconcat v5, v6 + v8 = icmp eq v4, v7 + return v8 +} +; run + +function %ushr2() -> b1 { +block0: + v0 = iconst.i64 0x01010101_01010101 + v1 = iconst.i64 0x01010101_01010101 + v2 = iconcat v0, v1 + v3 = iconst.i32 66 + v4 = ushr.i128 v2, v3 + v5 = iconst.i64 0x00404040_40404040 + v6 = iconst.i64 0x00000000_00000000 + v7 = iconcat v5, v6 + v8 = icmp eq v4, v7 + return v8 +} +; run + +function %sshr1() -> b1 { +block0: + v0 = iconst.i64 0x01010101_01010101 + v1 = iconst.i64 0x81010101_01010101 + v2 = iconcat v0, v1 + v3 = iconst.i32 2 + v4 = sshr.i128 v2, v3 + v5 = iconst.i64 0x40404040_40404040 + v6 = iconst.i64 0xe0404040_40404040 + v7 = iconcat v5, v6 + v8 = icmp eq v4, v7 + return v8 +} +; run + +function %sshr2() -> b1 { +block0: + v0 = iconst.i64 0x12345678_9abcdef0 + v1 = iconst.i64 0x80101010_10101010 + v2 = iconcat v0, v1 + v3 = iconst.i32 66 + v4 = sshr.i128 v2, v3 + v5 = iconst.i64 0xe0040404_04040404 + v6 = iconst.i64 0xffffffff_ffffffff + v7 = iconcat v5, v6 + v8 = icmp eq v4, v7 + return v8 +} +; run From bc9f10115d5614c51e652d738285cb36278e7e92 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 14 Jan 2021 14:37:44 -0800 Subject: [PATCH 10/55] Fix a typo --- crates/wasmtime/src/instance.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 34f87e5c0a..e574f38812 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -308,7 +308,7 @@ impl Instance { bail!("cross-`Engine` instantiation is not currently supported"); } - // Perform some pre-flight checks before we geet into the meat of + // Perform some pre-flight checks before we get into the meat of // instantiation. let expected = module.compiled_module().module().imports().count(); if expected != imports.len() { From d66db16d00e3054429a5818d19b097ef935f0ac0 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 14 Jan 2021 14:38:05 -0800 Subject: [PATCH 11/55] Fix a doc link --- crates/wasmtime/src/func.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 23bf36f73b..7f68b97964 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -371,8 +371,8 @@ impl Func { /// /// Finally you can also optionally take [`Caller`] as the first argument of /// your closure. If inserted then you're able to inspect the caller's - /// state, for example the [`Memory`] it has exported so you can read what - /// pointers point to. + /// state, for example the [`Memory`](crate::Memory) it has exported so you + /// can read what pointers point to. /// /// Note that when using this API, the intention is to create as thin of a /// layer as possible for when WebAssembly calls the function provided. With From 0f563f786acc21fbca1f7f9a8b2c2f3a4a6f15b3 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Sun, 13 Dec 2020 18:05:38 -0800 Subject: [PATCH 12/55] Add ELF TLS support in new x64 backend. This follows the implementation in the legacy x86 backend, including hardcoded sequence that is compatible with what the linker expects. We could potentially do better here, but it is likely not necessary. Thanks to @bjorn3 for a bugfix to an earlier version of this. --- cranelift/codegen/src/isa/x64/inst/emit.rs | 28 +++++++++++++++++++ .../codegen/src/isa/x64/inst/emit_tests.rs | 11 ++++++++ cranelift/codegen/src/isa/x64/inst/mod.rs | 26 ++++++++++++++++- cranelift/codegen/src/isa/x64/lower.rs | 18 +++++++++++- .../filetests/filetests/isa/x64/tls_elf.clif | 19 +++++++++++++ 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/tls_elf.clif diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 075724d493..e1253b9597 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1,5 +1,6 @@ use crate::binemit::{Addend, Reloc}; use crate::ir::immediates::{Ieee32, Ieee64}; +use crate::ir::LibCall; use crate::ir::TrapCode; use crate::isa::x64::inst::args::*; use crate::isa::x64::inst::*; @@ -2988,6 +2989,33 @@ pub(crate) fn emit( Inst::EpiloguePlaceholder => { // Generate no code. } + + Inst::ElfTlsGetAddr { ref symbol } => { + // N.B.: Must be exactly this byte sequence; the linker requires it, + // because it must know how to rewrite the bytes. + + // data16 lea gv@tlsgd(%rip),%rdi + sink.put1(0x66); // data16 + sink.put1(0b01001000); // REX.W + sink.put1(0x8d); // LEA + sink.put1(0x3d); // ModRM byte + emit_reloc(sink, state, Reloc::ElfX86_64TlsGd, symbol, -4); + sink.put4(0); // offset + + // data16 data16 callq __tls_get_addr-4 + sink.put1(0x66); // data16 + sink.put1(0x66); // data16 + sink.put1(0b01001000); // REX.W + sink.put1(0xe8); // CALL + emit_reloc( + sink, + state, + Reloc::X86CallPLTRel4, + &ExternalName::LibCall(LibCall::ElfTlsGetAddr), + -4, + ); + sink.put4(0); // offset + } } state.clear_post_insn(); diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 42e38c9cd5..f1e74ae22a 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3904,6 +3904,17 @@ fn test_x64_emit() { let trap_code = TrapCode::UnreachableCodeReached; insns.push((Inst::Ud2 { trap_code }, "0F0B", "ud2 unreachable")); + insns.push(( + Inst::ElfTlsGetAddr { + symbol: ExternalName::User { + namespace: 0, + index: 0, + }, + }, + "66488D3D00000000666648E800000000", + "elf_tls_get_addr User { namespace: 0, index: 0 }", + )); + // ======================================================== // Actually run the tests! let mut flag_builder = settings::builder(); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 979c264231..edc7a65109 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -2,7 +2,9 @@ use crate::binemit::{CodeOffset, StackMap}; use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type}; +use crate::isa::x64::abi::X64ABIMachineSpec; use crate::isa::x64::settings as x64_settings; +use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, settings::Flags, CodegenError, CodegenResult}; use alloc::boxed::Box; @@ -474,6 +476,10 @@ pub enum Inst { /// reports its own `def`s/`use`s/`mod`s; this adds complexity (the instruction list is no /// longer flat) and requires knowledge about semantics and initial-value independence anyway. XmmUninitializedValue { dst: Writable }, + + /// A call to the `ElfTlsGetAddr` libcall. Returns address + /// of TLS symbol in rax. + ElfTlsGetAddr { symbol: ExternalName }, } pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool { @@ -532,7 +538,8 @@ impl Inst { | Inst::XmmCmpRmR { .. } | Inst::XmmLoadConst { .. } | Inst::XmmMinMaxSeq { .. } - | Inst::XmmUninitializedValue { .. } => None, + | Inst::XmmUninitializedValue { .. } + | Inst::ElfTlsGetAddr { .. } => None, // These use dynamic SSE opcodes. Inst::GprToXmm { op, .. } @@ -1780,6 +1787,10 @@ impl PrettyPrint for Inst { Inst::Hlt => "hlt".into(), Inst::Ud2 { trap_code } => format!("ud2 {}", trap_code), + + Inst::ElfTlsGetAddr { ref symbol } => { + format!("elf_tls_get_addr {:?}", symbol) + } } } } @@ -2039,6 +2050,18 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { | Inst::Fence { .. } => { // No registers are used. } + + Inst::ElfTlsGetAddr { .. } => { + // All caller-saves are clobbered. + // + // We use the SysV calling convention here because the + // pseudoinstruction (and relocation that it emits) is specific to + // ELF systems; other x86-64 targets with other conventions (i.e., + // Windows) use different TLS strategies. + for reg in X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV) { + collector.add_def(reg); + } + } } } @@ -2425,6 +2448,7 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { | Inst::Ud2 { .. } | Inst::Hlt | Inst::AtomicRmwSeq { .. } + | Inst::ElfTlsGetAddr { .. } | Inst::Fence { .. } => { // Instruction doesn't explicitly mention any regs, so it can't have any virtual // regs that we'd need to remap. Hence no action required. diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index a25da666b3..da231120bb 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -12,7 +12,7 @@ use crate::isa::{x64::X64Backend, CallConv}; use crate::machinst::lower::*; use crate::machinst::*; use crate::result::CodegenResult; -use crate::settings::Flags; +use crate::settings::{Flags, TlsModel}; use alloc::boxed::Box; use alloc::vec::Vec; use cranelift_codegen_shared::condcodes::CondCode; @@ -5324,6 +5324,22 @@ fn lower_insn_to_regs>( ctx.emit(Inst::gen_move(dst_hi, src.regs()[1], types::I64)); } + Opcode::TlsValue => match flags.tls_model() { + TlsModel::ElfGd => { + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + let (name, _, _) = ctx.symbol_value(insn).unwrap(); + let symbol = name.clone(); + ctx.emit(Inst::ElfTlsGetAddr { symbol }); + ctx.emit(Inst::gen_move(dst, regs::rax(), types::I64)); + } + _ => { + todo!( + "Unimplemented TLS model in x64 backend: {:?}", + flags.tls_model() + ); + } + }, + Opcode::IaddImm | Opcode::ImulImm | Opcode::UdivImm diff --git a/cranelift/filetests/filetests/isa/x64/tls_elf.clif b/cranelift/filetests/filetests/isa/x64/tls_elf.clif new file mode 100644 index 0000000000..6136e98e7d --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/tls_elf.clif @@ -0,0 +1,19 @@ +test compile +set tls_model=elf_gd +target x86_64 +feature "experimental_x64" + +function u0:0(i32) -> i64 { +gv0 = symbol colocated tls u1:0 + +block0(v0: i32): + v1 = global_value.i64 gv0 + return v1 +} + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: elf_tls_get_addr User { namespace: 1, index: 0 } +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret From 456561f43180e8174212f35daadc72ac076e5628 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Sun, 13 Dec 2020 18:50:59 -0800 Subject: [PATCH 13/55] x64 and aarch64: allow StructArgument and StructReturn args. The StructReturn ABI is fairly simple at the codegen/isel level: we only need to take care to return the sret pointer as one of the return values if that wasn't specified in the initial function signature. Struct arguments are a little more complex. A struct argument is stored as a chunk of memory in the stack-args space. However, the CLIF semantics are slightly special: on the caller side, the parameter passed in is a pointer to an arbitrary memory block, and we must memcpy this data to the on-stack struct-argument; and on the callee side, we provide a pointer to the passed-in struct-argument as the CLIF block param value. This is necessary to support various ABIs other than Wasm, such as that of Rust (with the cg_clif codegen backend). --- cranelift/codegen/src/isa/aarch64/abi.rs | 140 +++++++---- cranelift/codegen/src/isa/aarch64/lower.rs | 2 +- .../codegen/src/isa/aarch64/lower_inst.rs | 11 +- cranelift/codegen/src/isa/arm32/abi.rs | 41 ++-- cranelift/codegen/src/isa/arm32/lower.rs | 2 +- cranelift/codegen/src/isa/arm32/lower_inst.rs | 6 +- cranelift/codegen/src/isa/x64/abi.rs | 157 ++++++++---- cranelift/codegen/src/isa/x64/lower.rs | 11 +- cranelift/codegen/src/machinst/abi.rs | 13 +- cranelift/codegen/src/machinst/abi_impl.rs | 231 +++++++++++++++--- cranelift/codegen/src/machinst/lower.rs | 25 +- .../filetests/filetests/isa/x64/i128.clif | 2 +- .../filetests/isa/x64/struct-arg.clif | 147 +++++++++++ .../filetests/isa/x64/struct-ret.clif | 19 ++ 14 files changed, 641 insertions(+), 166 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/struct-arg.clif create mode 100644 cranelift/filetests/filetests/isa/x64/struct-ret.clif diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index d5d88e7770..8b371cb159 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -4,6 +4,8 @@ use crate::ir; use crate::ir::types; use crate::ir::types::*; use crate::ir::MemFlags; +use crate::ir::Opcode; +use crate::ir::{ExternalName, LibCall}; use crate::isa; use crate::isa::aarch64::{inst::EmitState, inst::*}; use crate::machinst::*; @@ -76,41 +78,41 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt match ¶m.purpose { &ir::ArgumentPurpose::VMContext => { // This is SpiderMonkey's `WasmTlsReg`. - Some(ABIArg::Reg( - ValueRegs::one(xreg(BALDRDASH_TLS_REG).to_real_reg()), - ir::types::I64, - param.extension, - param.purpose, - )) + Some(ABIArg::Reg { + regs: ValueRegs::one(xreg(BALDRDASH_TLS_REG).to_real_reg()), + ty: ir::types::I64, + extension: param.extension, + purpose: param.purpose, + }) } &ir::ArgumentPurpose::SignatureId => { // This is SpiderMonkey's `WasmTableCallSigReg`. - Some(ABIArg::Reg( - ValueRegs::one(xreg(BALDRDASH_SIG_REG).to_real_reg()), - ir::types::I64, - param.extension, - param.purpose, - )) + Some(ABIArg::Reg { + regs: ValueRegs::one(xreg(BALDRDASH_SIG_REG).to_real_reg()), + ty: ir::types::I64, + extension: param.extension, + purpose: param.purpose, + }) } &ir::ArgumentPurpose::CalleeTLS => { // This is SpiderMonkey's callee TLS slot in the extended frame of Wasm's ABI-2020. assert!(call_conv == isa::CallConv::Baldrdash2020); - Some(ABIArg::Stack( - BALDRDASH_CALLEE_TLS_OFFSET, - ir::types::I64, - ir::ArgumentExtension::None, - param.purpose, - )) + Some(ABIArg::Stack { + offset: BALDRDASH_CALLEE_TLS_OFFSET, + ty: ir::types::I64, + extension: ir::ArgumentExtension::None, + purpose: param.purpose, + }) } &ir::ArgumentPurpose::CallerTLS => { // This is SpiderMonkey's caller TLS slot in the extended frame of Wasm's ABI-2020. assert!(call_conv == isa::CallConv::Baldrdash2020); - Some(ABIArg::Stack( - BALDRDASH_CALLER_TLS_OFFSET, - ir::types::I64, - ir::ArgumentExtension::None, - param.purpose, - )) + Some(ABIArg::Stack { + offset: BALDRDASH_CALLER_TLS_OFFSET, + ty: ir::types::I64, + extension: ir::ArgumentExtension::None, + purpose: param.purpose, + }) } _ => None, } @@ -208,7 +210,9 @@ impl ABIMachineSpec for AArch64MachineDeps { | &ir::ArgumentPurpose::StackLimit | &ir::ArgumentPurpose::SignatureId | &ir::ArgumentPurpose::CallerTLS - | &ir::ArgumentPurpose::CalleeTLS => {} + | &ir::ArgumentPurpose::CalleeTLS + | &ir::ArgumentPurpose::StructReturn + | &ir::ArgumentPurpose::StructArgument(_) => {} _ => panic!( "Unsupported argument purpose {:?} in signature: {:?}", param.purpose, params @@ -233,18 +237,28 @@ impl ABIMachineSpec for AArch64MachineDeps { if let Some(param) = try_fill_baldrdash_reg(call_conv, param) { assert!(rc == RegClass::I64); ret.push(param); + } else if let ir::ArgumentPurpose::StructArgument(size) = param.purpose { + let offset = next_stack as i64; + let size = size as u64; + assert!(size % 8 == 0, "StructArgument size is not properly aligned"); + next_stack += size; + ret.push(ABIArg::StructArg { + offset, + size, + purpose: param.purpose, + }); } else if *next_reg < max_per_class_reg_vals && remaining_reg_vals > 0 { let reg = match rc { RegClass::I64 => xreg(*next_reg), RegClass::V128 => vreg(*next_reg), _ => unreachable!(), }; - ret.push(ABIArg::Reg( - ValueRegs::one(reg.to_real_reg()), - param.value_type, - param.extension, - param.purpose, - )); + ret.push(ABIArg::Reg { + regs: ValueRegs::one(reg.to_real_reg()), + ty: param.value_type, + extension: param.extension, + purpose: param.purpose, + }); *next_reg += 1; remaining_reg_vals -= 1; } else { @@ -255,12 +269,12 @@ impl ABIMachineSpec for AArch64MachineDeps { // Align. debug_assert!(size.is_power_of_two()); next_stack = (next_stack + size - 1) & !(size - 1); - ret.push(ABIArg::Stack( - next_stack as i64, - param.value_type, - param.extension, - param.purpose, - )); + ret.push(ABIArg::Stack { + offset: next_stack as i64, + ty: param.value_type, + extension: param.extension, + purpose: param.purpose, + }); next_stack += size; } } @@ -272,19 +286,19 @@ impl ABIMachineSpec for AArch64MachineDeps { let extra_arg = if add_ret_area_ptr { debug_assert!(args_or_rets == ArgsOrRets::Args); if next_xreg < max_per_class_reg_vals && remaining_reg_vals > 0 { - ret.push(ABIArg::Reg( - ValueRegs::one(xreg(next_xreg).to_real_reg()), - I64, - ir::ArgumentExtension::None, - ir::ArgumentPurpose::Normal, - )); + ret.push(ABIArg::Reg { + regs: ValueRegs::one(xreg(next_xreg).to_real_reg()), + ty: I64, + extension: ir::ArgumentExtension::None, + purpose: ir::ArgumentPurpose::Normal, + }); } else { - ret.push(ABIArg::Stack( - next_stack as i64, - I64, - ir::ArgumentExtension::None, - ir::ArgumentPurpose::Normal, - )); + ret.push(ABIArg::Stack { + offset: next_stack as i64, + ty: I64, + extension: ir::ArgumentExtension::None, + purpose: ir::ArgumentPurpose::Normal, + }); next_stack += 8; } Some(ret.len() - 1) @@ -708,6 +722,34 @@ impl ABIMachineSpec for AArch64MachineDeps { insts } + fn gen_memcpy( + call_conv: isa::CallConv, + dst: Reg, + src: Reg, + size: usize, + ) -> SmallVec<[Self::I; 8]> { + // Baldrdash should not use struct args. + assert!(!call_conv.extends_baldrdash()); + let mut insts = SmallVec::new(); + let arg0 = writable_xreg(0); + let arg1 = writable_xreg(1); + let arg2 = writable_xreg(2); + insts.push(Inst::gen_move(arg0, dst, I64)); + insts.push(Inst::gen_move(arg1, src, I64)); + insts.extend(Inst::load_constant(arg2, size as u64).into_iter()); + insts.push(Inst::Call { + info: Box::new(CallInfo { + dest: ExternalName::LibCall(LibCall::Memcpy), + uses: vec![arg0.to_reg(), arg1.to_reg(), arg2.to_reg()], + defs: Self::get_regs_clobbered_by_call(call_conv), + opcode: Opcode::Call, + caller_callconv: call_conv, + callee_callconv: call_conv, + }), + }); + insts + } + fn get_number_of_spillslots_for_value(rc: RegClass, ty: Type) -> u32 { // We allocate in terms of 8-byte slots. match (rc, ty) { diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index 37c5e79c8d..0f37bb6123 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -1231,7 +1231,7 @@ impl LowerBackend for AArch64Backend { type MInst = Inst; fn lower>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> { - lower_inst::lower_insn_to_regs(ctx, ir_inst) + lower_inst::lower_insn_to_regs(ctx, ir_inst, &self.flags) } fn lower_branch_group>( diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 1c4e3d7e99..73c11008c4 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -7,6 +7,7 @@ use crate::ir::Inst as IRInst; use crate::ir::{InstructionData, Opcode, TrapCode}; use crate::machinst::lower::*; use crate::machinst::*; +use crate::settings::Flags; use crate::{CodegenError, CodegenResult}; use crate::isa::aarch64::abi::*; @@ -24,6 +25,7 @@ use super::lower::*; pub(crate) fn lower_insn_to_regs>( ctx: &mut C, insn: IRInst, + flags: &Flags, ) -> CodegenResult<()> { let op = ctx.data(insn).opcode(); let inputs = insn_inputs(ctx, insn); @@ -1803,7 +1805,7 @@ pub(crate) fn lower_insn_to_regs>( assert!(inputs.len() == sig.params.len()); assert!(outputs.len() == sig.returns.len()); ( - AArch64ABICaller::from_func(sig, &extname, dist, caller_conv)?, + AArch64ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?, &inputs[..], ) } @@ -1813,7 +1815,7 @@ pub(crate) fn lower_insn_to_regs>( assert!(inputs.len() - 1 == sig.params.len()); assert!(outputs.len() == sig.returns.len()); ( - AArch64ABICaller::from_ptr(sig, ptr, op, caller_conv)?, + AArch64ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?, &inputs[1..], ) } @@ -1822,8 +1824,9 @@ pub(crate) fn lower_insn_to_regs>( abi.emit_stack_pre_adjust(ctx); assert!(inputs.len() == abi.num_args()); - for (i, input) in inputs.iter().enumerate() { - let arg_reg = put_input_in_reg(ctx, *input, NarrowValueMode::None); + for i in abi.get_copy_to_arg_order() { + let input = inputs[i]; + let arg_reg = put_input_in_reg(ctx, input, NarrowValueMode::None); abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg)); } abi.emit_call(ctx); diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs index 9e92a7b7aa..e1a64aeb76 100644 --- a/cranelift/codegen/src/isa/arm32/abi.rs +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -81,12 +81,12 @@ impl ABIMachineSpec for Arm32MachineDeps { if next_rreg < max_reg_val { let reg = rreg(next_rreg); - ret.push(ABIArg::Reg( - ValueRegs::one(reg.to_real_reg()), - param.value_type, - param.extension, - param.purpose, - )); + ret.push(ABIArg::Reg { + regs: ValueRegs::one(reg.to_real_reg()), + ty: param.value_type, + extension: param.extension, + purpose: param.purpose, + }); next_rreg += 1; } else { // Arguments are stored on stack in reversed order. @@ -101,12 +101,12 @@ impl ABIMachineSpec for Arm32MachineDeps { let extra_arg = if add_ret_area_ptr { debug_assert!(args_or_rets == ArgsOrRets::Args); if next_rreg < max_reg_val { - ret.push(ABIArg::Reg( - ValueRegs::one(rreg(next_rreg).to_real_reg()), - I32, - ir::ArgumentExtension::None, - ir::ArgumentPurpose::Normal, - )); + ret.push(ABIArg::Reg { + regs: ValueRegs::one(rreg(next_rreg).to_real_reg()), + ty: I32, + extension: ir::ArgumentExtension::None, + purpose: ir::ArgumentPurpose::Normal, + }); } else { stack_args.push(( I32, @@ -124,12 +124,12 @@ impl ABIMachineSpec for Arm32MachineDeps { let max_stack = next_stack; for (ty, ext, purpose) in stack_args.into_iter().rev() { next_stack -= 4; - ret.push(ABIArg::Stack( - (max_stack - next_stack) as i64, + ret.push(ABIArg::Stack { + offset: (max_stack - next_stack) as i64, ty, - ext, + extension: ext, purpose, - )); + }); } assert_eq!(next_stack, 0); @@ -426,6 +426,15 @@ impl ABIMachineSpec for Arm32MachineDeps { insts } + fn gen_memcpy( + _call_conv: isa::CallConv, + _dst: Reg, + _src: Reg, + _size: usize, + ) -> SmallVec<[Self::I; 8]> { + unimplemented!("StructArgs not implemented for ARM32 yet"); + } + fn get_number_of_spillslots_for_value(rc: RegClass, _ty: Type) -> u32 { match rc { RegClass::I32 => 1, diff --git a/cranelift/codegen/src/isa/arm32/lower.rs b/cranelift/codegen/src/isa/arm32/lower.rs index 372c18b8e9..f2a35f9820 100644 --- a/cranelift/codegen/src/isa/arm32/lower.rs +++ b/cranelift/codegen/src/isa/arm32/lower.rs @@ -224,7 +224,7 @@ impl LowerBackend for Arm32Backend { type MInst = Inst; fn lower>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> { - lower_inst::lower_insn_to_regs(ctx, ir_inst) + lower_inst::lower_insn_to_regs(ctx, ir_inst, &self.flags) } fn lower_branch_group>( diff --git a/cranelift/codegen/src/isa/arm32/lower_inst.rs b/cranelift/codegen/src/isa/arm32/lower_inst.rs index dd453d772a..16fd528c56 100644 --- a/cranelift/codegen/src/isa/arm32/lower_inst.rs +++ b/cranelift/codegen/src/isa/arm32/lower_inst.rs @@ -5,6 +5,7 @@ use crate::ir::Inst as IRInst; use crate::ir::Opcode; use crate::machinst::lower::*; use crate::machinst::*; +use crate::settings::Flags; use crate::CodegenResult; use crate::isa::arm32::abi::*; @@ -18,6 +19,7 @@ use super::lower::*; pub(crate) fn lower_insn_to_regs>( ctx: &mut C, insn: IRInst, + flags: &Flags, ) -> CodegenResult<()> { let op = ctx.data(insn).opcode(); let inputs: SmallVec<[InsnInput; 4]> = (0..ctx.num_inputs(insn)) @@ -502,7 +504,7 @@ pub(crate) fn lower_insn_to_regs>( assert_eq!(inputs.len(), sig.params.len()); assert_eq!(outputs.len(), sig.returns.len()); ( - Arm32ABICaller::from_func(sig, &extname, dist, caller_conv)?, + Arm32ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?, &inputs[..], ) } @@ -512,7 +514,7 @@ pub(crate) fn lower_insn_to_regs>( assert_eq!(inputs.len() - 1, sig.params.len()); assert_eq!(outputs.len(), sig.returns.len()); ( - Arm32ABICaller::from_ptr(sig, ptr, op, caller_conv)?, + Arm32ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?, &inputs[1..], ) } diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index aa757392e3..d4f7d5c60c 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -31,41 +31,41 @@ fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option { // This is SpiderMonkey's `WasmTlsReg`. - Some(ABIArg::Reg( - ValueRegs::one(regs::r14().to_real_reg()), - types::I64, - param.extension, - param.purpose, - )) + Some(ABIArg::Reg { + regs: ValueRegs::one(regs::r14().to_real_reg()), + ty: types::I64, + extension: param.extension, + purpose: param.purpose, + }) } &ir::ArgumentPurpose::SignatureId => { // This is SpiderMonkey's `WasmTableCallSigReg`. - Some(ABIArg::Reg( - ValueRegs::one(regs::r10().to_real_reg()), - types::I64, - param.extension, - param.purpose, - )) + Some(ABIArg::Reg { + regs: ValueRegs::one(regs::r10().to_real_reg()), + ty: types::I64, + extension: param.extension, + purpose: param.purpose, + }) } &ir::ArgumentPurpose::CalleeTLS => { // This is SpiderMonkey's callee TLS slot in the extended frame of Wasm's ABI-2020. assert!(call_conv == isa::CallConv::Baldrdash2020); - Some(ABIArg::Stack( - BALDRDASH_CALLEE_TLS_OFFSET, - ir::types::I64, - ir::ArgumentExtension::None, - param.purpose, - )) + Some(ABIArg::Stack { + offset: BALDRDASH_CALLEE_TLS_OFFSET, + ty: ir::types::I64, + extension: ir::ArgumentExtension::None, + purpose: param.purpose, + }) } &ir::ArgumentPurpose::CallerTLS => { // This is SpiderMonkey's caller TLS slot in the extended frame of Wasm's ABI-2020. assert!(call_conv == isa::CallConv::Baldrdash2020); - Some(ABIArg::Stack( - BALDRDASH_CALLER_TLS_OFFSET, - ir::types::I64, - ir::ArgumentExtension::None, - param.purpose, - )) + Some(ABIArg::Stack { + offset: BALDRDASH_CALLER_TLS_OFFSET, + ty: ir::types::I64, + extension: ir::ArgumentExtension::None, + purpose: param.purpose, + }) } _ => None, } @@ -131,7 +131,9 @@ impl ABIMachineSpec for X64ABIMachineSpec { | &ir::ArgumentPurpose::StackLimit | &ir::ArgumentPurpose::SignatureId | &ir::ArgumentPurpose::CalleeTLS - | &ir::ArgumentPurpose::CallerTLS => {} + | &ir::ArgumentPurpose::CallerTLS + | &ir::ArgumentPurpose::StructReturn + | &ir::ArgumentPurpose::StructArgument(_) => {} _ => panic!( "Unsupported argument purpose {:?} in signature: {:?}", param.purpose, params @@ -143,6 +145,19 @@ impl ABIMachineSpec for X64ABIMachineSpec { continue; } + if let ir::ArgumentPurpose::StructArgument(size) = param.purpose { + let offset = next_stack as i64; + let size = size as u64; + assert!(size % 8 == 0, "StructArgument size is not properly aligned"); + next_stack += size; + ret.push(ABIArg::StructArg { + offset, + size, + purpose: param.purpose, + }); + continue; + } + // Find regclass(es) of the register(s) used to store a value of this type. let (rcs, _) = Inst::rc_for_type(param.value_type)?; let intreg = rcs[0] == RegClass::I64; @@ -183,12 +198,12 @@ impl ABIMachineSpec for X64ABIMachineSpec { 2 => ValueRegs::two(regs[0], regs[1]), _ => panic!("More than two registers unexpected"), }; - ret.push(ABIArg::Reg( + ret.push(ABIArg::Reg { regs, - param.value_type, - param.extension, - param.purpose, - )); + ty: param.value_type, + extension: param.extension, + purpose: param.purpose, + }); if intreg { next_gpr += num_regs; } else { @@ -202,12 +217,12 @@ impl ABIMachineSpec for X64ABIMachineSpec { // Align. debug_assert!(size.is_power_of_two()); next_stack = (next_stack + size - 1) & !(size - 1); - ret.push(ABIArg::Stack( - next_stack as i64, - param.value_type, - param.extension, - param.purpose, - )); + ret.push(ABIArg::Stack { + offset: next_stack as i64, + ty: param.value_type, + extension: param.extension, + purpose: param.purpose, + }); next_stack += size; } } @@ -219,19 +234,19 @@ impl ABIMachineSpec for X64ABIMachineSpec { let extra_arg = if add_ret_area_ptr { debug_assert!(args_or_rets == ArgsOrRets::Args); if let Some(reg) = get_intreg_for_arg_systemv(&call_conv, next_gpr) { - ret.push(ABIArg::Reg( - ValueRegs::one(reg.to_real_reg()), - types::I64, - ir::ArgumentExtension::None, - ir::ArgumentPurpose::Normal, - )); + ret.push(ABIArg::Reg { + regs: ValueRegs::one(reg.to_real_reg()), + ty: types::I64, + extension: ir::ArgumentExtension::None, + purpose: ir::ArgumentPurpose::Normal, + }); } else { - ret.push(ABIArg::Stack( - next_stack as i64, - types::I64, - ir::ArgumentExtension::None, - ir::ArgumentPurpose::Normal, - )); + ret.push(ABIArg::Stack { + offset: next_stack as i64, + ty: types::I64, + extension: ir::ArgumentExtension::None, + purpose: ir::ArgumentPurpose::Normal, + }); next_stack += 8; } Some(ret.len() - 1) @@ -441,6 +456,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { let stack_size = clobbered_size + fixed_frame_storage_size; // Align to 16 bytes. let stack_size = (stack_size + 15) & !15; + let clobbered_size = stack_size - fixed_frame_storage_size; // Adjust the stack pointer downward with one `sub rsp, IMM` // instruction. if stack_size > 0 { @@ -567,6 +583,51 @@ impl ABIMachineSpec for X64ABIMachineSpec { insts } + fn gen_memcpy( + call_conv: isa::CallConv, + dst: Reg, + src: Reg, + size: usize, + ) -> SmallVec<[Self::I; 8]> { + // Baldrdash should not use struct args. + assert!(!call_conv.extends_baldrdash()); + let mut insts = SmallVec::new(); + let arg0 = get_intreg_for_arg_systemv(&call_conv, 0).unwrap(); + let arg1 = get_intreg_for_arg_systemv(&call_conv, 1).unwrap(); + let arg2 = get_intreg_for_arg_systemv(&call_conv, 2).unwrap(); + // We need a register to load the address of `memcpy()` below and we + // don't have a lowering context to allocate a temp here; so just use a + // register we know we are free to mutate as part of this sequence + // (because it is clobbered by the call as per the ABI anyway). + let memcpy_addr = get_intreg_for_arg_systemv(&call_conv, 3).unwrap(); + insts.push(Inst::gen_move(Writable::from_reg(arg0), dst, I64)); + insts.push(Inst::gen_move(Writable::from_reg(arg1), src, I64)); + insts.extend( + Inst::gen_constant( + ValueRegs::one(Writable::from_reg(arg2)), + size as u128, + I64, + |_| panic!("tmp should not be needed"), + ) + .into_iter(), + ); + // We use an indirect call and a full LoadExtName because we do not have + // information about the libcall `RelocDistance` here, so we + // conservatively use the more flexible calling sequence. + insts.push(Inst::LoadExtName { + dst: Writable::from_reg(memcpy_addr), + name: Box::new(ExternalName::LibCall(LibCall::Memcpy)), + offset: 0, + }); + insts.push(Inst::call_unknown( + RegMem::reg(memcpy_addr), + /* uses = */ vec![arg0, arg1, arg2], + /* defs = */ Self::get_regs_clobbered_by_call(call_conv), + Opcode::Call, + )); + insts + } + fn get_number_of_spillslots_for_value(rc: RegClass, ty: Type) -> u32 { // We allocate in terms of 8-byte slots. match (rc, ty) { diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index a25da666b3..17f8ab992c 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1083,7 +1083,7 @@ fn emit_vm_call>( let sig = make_libcall_sig(ctx, insn, call_conv, types::I64); let caller_conv = ctx.abi().call_conv(); - let mut abi = X64ABICaller::from_func(&sig, &extname, dist, caller_conv)?; + let mut abi = X64ABICaller::from_func(&sig, &extname, dist, caller_conv, flags)?; abi.emit_stack_pre_adjust(ctx); @@ -3091,7 +3091,7 @@ fn lower_insn_to_regs>( assert_eq!(inputs.len(), sig.params.len()); assert_eq!(outputs.len(), sig.returns.len()); ( - X64ABICaller::from_func(sig, &extname, dist, caller_conv)?, + X64ABICaller::from_func(sig, &extname, dist, caller_conv, flags)?, &inputs[..], ) } @@ -3102,7 +3102,7 @@ fn lower_insn_to_regs>( assert_eq!(inputs.len() - 1, sig.params.len()); assert_eq!(outputs.len(), sig.returns.len()); ( - X64ABICaller::from_ptr(sig, ptr, op, caller_conv)?, + X64ABICaller::from_ptr(sig, ptr, op, caller_conv, flags)?, &inputs[1..], ) } @@ -3112,8 +3112,9 @@ fn lower_insn_to_regs>( abi.emit_stack_pre_adjust(ctx); assert_eq!(inputs.len(), abi.num_args()); - for (i, input) in inputs.iter().enumerate() { - let arg_regs = put_input_in_regs(ctx, *input); + for i in abi.get_copy_to_arg_order() { + let input = inputs[i]; + let arg_regs = put_input_in_regs(ctx, input); abi.emit_copy_regs_to_arg(ctx, i, arg_regs); } abi.emit_call(ctx); diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index 59738bd3a5..ca4b2414df 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -1,7 +1,7 @@ //! ABI definitions. use crate::binemit::StackMap; -use crate::ir::StackSlot; +use crate::ir::{Signature, StackSlot}; use crate::isa::CallConv; use crate::machinst::*; use crate::settings; @@ -27,6 +27,9 @@ pub trait ABICallee { /// lowering context exists. fn init(&mut self, maybe_tmp: Option>); + /// Access the (possibly legalized) signature. + fn signature(&self) -> &Signature; + /// Accumulate outgoing arguments. This ensures that at least SIZE bytes /// are allocated in the prologue to be available for use in function calls /// to hold arguments and/or return values. If this function is called @@ -215,6 +218,9 @@ pub trait ABICaller { /// Get the number of arguments expected. fn num_args(&self) -> usize; + /// Access the (possibly legalized) signature. + fn signature(&self) -> &Signature; + /// Emit a copy of an argument value from a source register, prior to the call. fn emit_copy_regs_to_arg>( &self, @@ -223,6 +229,11 @@ pub trait ABICaller { from_reg: ValueRegs, ); + /// Specific order for copying into arguments at callsites. We must be + /// careful to copy into StructArgs first, because we need to be able + /// to invoke memcpy() before we've loaded other arg regs (see above). + fn get_copy_to_arg_order(&self) -> SmallVec<[usize; 8]>; + /// Emit a copy a return value into a destination register, after the call returns. fn emit_copy_retval_to_regs>( &self, diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index d315c3defb..439e93e2d9 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -111,7 +111,7 @@ use super::abi::*; use crate::binemit::StackMap; use crate::ir::types::*; -use crate::ir::{ArgumentExtension, StackSlot}; +use crate::ir::{ArgumentExtension, ArgumentPurpose, StackSlot}; use crate::machinst::*; use crate::settings; use crate::CodegenResult; @@ -128,22 +128,58 @@ use std::mem; #[derive(Clone, Copy, Debug)] pub enum ABIArg { /// In a real register (or set of registers). - Reg( - ValueRegs, - ir::Type, - ir::ArgumentExtension, - ir::ArgumentPurpose, - ), + Reg { + /// Register(s) that hold this arg. + regs: ValueRegs, + /// Value type of this arg. + ty: ir::Type, + /// Should this arg be zero- or sign-extended? + extension: ir::ArgumentExtension, + /// Purpose of this arg. + purpose: ir::ArgumentPurpose, + }, /// Arguments only: on stack, at given offset from SP at entry. - Stack(i64, ir::Type, ir::ArgumentExtension, ir::ArgumentPurpose), + Stack { + /// Offset of this arg relative to the base of stack args. + offset: i64, + /// Value type of this arg. + ty: ir::Type, + /// Should this arg be zero- or sign-extended? + extension: ir::ArgumentExtension, + /// Purpose of this arg. + purpose: ir::ArgumentPurpose, + }, + /// Structure argument. We reserve stack space for it, but the CLIF-level + /// semantics are a little weird: the value passed to the call instruction, + /// and received in the corresponding block param, is a *pointer*. On the + /// caller side, we memcpy the data from the passed-in pointer to the stack + /// area; on the callee side, we compute a pointer to this stack area and + /// provide that as the argument's value. + StructArg { + /// Offset of this arg relative to base of stack args. + offset: i64, + /// Size of this arg on the stack. + size: u64, + /// Purpose of this arg. + purpose: ir::ArgumentPurpose, + }, } impl ABIArg { /// Get the purpose of this arg. fn get_purpose(self) -> ir::ArgumentPurpose { match self { - ABIArg::Reg(_, _, _, purpose) => purpose, - ABIArg::Stack(_, _, _, purpose) => purpose, + ABIArg::Reg { purpose, .. } => purpose, + ABIArg::Stack { purpose, .. } => purpose, + ABIArg::StructArg { purpose, .. } => purpose, + } + } + + /// Is this a StructArg? + fn is_struct_arg(self) -> bool { + match self { + ABIArg::StructArg { .. } => true, + _ => false, } } } @@ -371,6 +407,16 @@ pub trait ABIMachineSpec { callee_conv: isa::CallConv, ) -> SmallVec<[(InstIsSafepoint, Self::I); 2]>; + /// Generate a memcpy invocation. Used to set up struct args. May clobber + /// caller-save registers; we only memcpy before we start to set up args for + /// a call. + fn gen_memcpy( + call_conv: isa::CallConv, + dst: Reg, + src: Reg, + size: usize, + ) -> SmallVec<[Self::I; 8]>; + /// Get the number of spillslots required for the given register-class and /// type. fn get_number_of_spillslots_for_value(rc: RegClass, ty: Type) -> u32; @@ -455,6 +501,8 @@ impl ABISig { /// ABI object for a function body. pub struct ABICalleeImpl { + /// CLIF-level signature, possibly normalized. + ir_sig: ir::Signature, /// Signature: arg and retval regs. sig: ABISig, /// Offsets to each stackslot. @@ -510,8 +558,8 @@ fn get_special_purpose_param_register( ) -> Option { let idx = f.signature.special_param_index(purpose)?; match abi.args[idx] { - ABIArg::Reg(regs, ..) => Some(regs.only_reg().unwrap().to_reg()), - ABIArg::Stack(..) => None, + ABIArg::Reg { regs, .. } => Some(regs.only_reg().unwrap().to_reg()), + _ => None, } } @@ -520,7 +568,8 @@ impl ABICalleeImpl { pub fn new(f: &ir::Function, flags: settings::Flags) -> CodegenResult { debug!("ABI: func signature {:?}", f.signature); - let sig = ABISig::from_func_sig::(&f.signature)?; + let ir_sig = ensure_struct_return_ptr_is_returned(&f.signature); + let sig = ABISig::from_func_sig::(&ir_sig)?; let call_conv = f.signature.call_conv; // Only these calling conventions are supported. @@ -567,6 +616,7 @@ impl ABICalleeImpl { }; Ok(Self { + ir_sig, sig, stackslots, stackslots_size: stack_offset, @@ -787,9 +837,30 @@ fn gen_store_base_offset_multi( ret } +fn ensure_struct_return_ptr_is_returned(sig: &ir::Signature) -> ir::Signature { + let params_structret = sig + .params + .iter() + .find(|p| p.purpose == ArgumentPurpose::StructReturn); + let rets_have_structret = sig.returns.len() > 0 + && sig + .returns + .iter() + .any(|arg| arg.purpose == ArgumentPurpose::StructReturn); + let mut sig = sig.clone(); + if params_structret.is_some() && !rets_have_structret { + sig.returns.insert(0, params_structret.unwrap().clone()); + } + sig +} + impl ABICallee for ABICalleeImpl { type I = M::I; + fn signature(&self) -> &ir::Signature { + &self.ir_sig + } + fn temp_needed(&self) -> Option { if self.sig.stack_ret_arg.is_some() { Some(M::word_type()) @@ -822,7 +893,7 @@ impl ABICallee for ABICalleeImpl { fn liveins(&self) -> Set { let mut set: Set = Set::empty(); for &arg in &self.sig.args { - if let ABIArg::Reg(regs, ..) = arg { + if let ABIArg::Reg { regs, .. } = arg { for &r in regs.regs() { set.insert(r); } @@ -834,7 +905,7 @@ impl ABICallee for ABICalleeImpl { fn liveouts(&self) -> Set { let mut set: Set = Set::empty(); for &ret in &self.sig.rets { - if let ABIArg::Reg(regs, ..) = ret { + if let ABIArg::Reg { regs, .. } = ret { for &r in regs.regs() { set.insert(r); } @@ -863,14 +934,25 @@ impl ABICallee for ABICalleeImpl { match &self.sig.args[idx] { // Extension mode doesn't matter (we're copying out, not in; we // ignore high bits by convention). - &ABIArg::Reg(regs, ty, ..) => { + &ABIArg::Reg { regs, ty, .. } => { gen_move_multi::(into_regs, regs.map(|r| r.to_reg()), ty) } - &ABIArg::Stack(off, ty, ..) => gen_load_stack_multi::( - StackAMode::FPOffset(M::fp_to_arg_offset(self.call_conv, &self.flags) + off, ty), + &ABIArg::Stack { offset, ty, .. } => gen_load_stack_multi::( + StackAMode::FPOffset( + M::fp_to_arg_offset(self.call_conv, &self.flags) + offset, + ty, + ), into_regs, ty, ), + &ABIArg::StructArg { offset, .. } => smallvec![M::gen_get_stack_addr( + StackAMode::FPOffset( + M::fp_to_arg_offset(self.call_conv, &self.flags) + offset, + I8, + ), + into_regs.only_reg().unwrap(), + I8, + )], } } @@ -892,10 +974,15 @@ impl ABICallee for ABICalleeImpl { let mut ret = smallvec![]; let word_bits = M::word_bits() as u8; match &self.sig.rets[idx] { - &ABIArg::Reg(regs, ty, ext, ..) => { + &ABIArg::Reg { + regs, + ty, + extension, + .. + } => { let from_bits = ty_bits(ty) as u8; let dest_regs = writable_value_regs(regs.map(|r| r.to_reg())); - let ext = M::get_ext_mode(self.sig.call_conv, ext); + let ext = M::get_ext_mode(self.sig.call_conv, extension); match (ext, from_bits) { (ArgumentExtension::Uext, n) | (ArgumentExtension::Sext, n) if n < word_bits => @@ -921,14 +1008,20 @@ impl ABICallee for ABICalleeImpl { ), }; } - &ABIArg::Stack(off, mut ty, ext, ..) => { + &ABIArg::Stack { + offset, + ty, + extension, + .. + } => { + let mut ty = ty; let from_bits = ty_bits(ty) as u8; // A machine ABI implementation should ensure that stack frames // have "reasonable" size. All current ABIs for machinst // backends (aarch64 and x64) enforce a 128MB limit. - let off = i32::try_from(off) + let off = i32::try_from(offset) .expect("Argument stack offset greater than 2GB; should hit impl limit first"); - let ext = M::get_ext_mode(self.sig.call_conv, ext); + let ext = M::get_ext_mode(self.sig.call_conv, extension); // Trash the from_reg; it should be its last use. match (ext, from_bits) { (ArgumentExtension::Uext, n) | (ArgumentExtension::Sext, n) @@ -961,6 +1054,7 @@ impl ABICallee for ABICalleeImpl { .into_iter(), ); } + &ABIArg::StructArg { .. } => panic!("Unexpected StructArg location for return value"), } ret } @@ -1248,7 +1342,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec, Vec uses.extend(regs.regs().iter().map(|r| r.to_reg())), + &ABIArg::Reg { regs, .. } => uses.extend(regs.regs().iter().map(|r| r.to_reg())), _ => {} } } @@ -1257,7 +1351,7 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec, Vec { + &ABIArg::Reg { regs, .. } => { defs.extend(regs.regs().iter().map(|r| Writable::from_reg(r.to_reg()))) } _ => {} @@ -1269,6 +1363,8 @@ fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec, Vec { + /// CLIF-level signature, possibly normalized. + ir_sig: ir::Signature, /// The called function's signature. sig: ABISig, /// All uses for the callsite, i.e., function args. @@ -1281,6 +1377,8 @@ pub struct ABICallerImpl { opcode: ir::Opcode, /// Caller's calling convention. caller_conv: isa::CallConv, + /// The settings controlling this compilation. + flags: settings::Flags, _mach: PhantomData, } @@ -1301,16 +1399,20 @@ impl ABICallerImpl { extname: &ir::ExternalName, dist: RelocDistance, caller_conv: isa::CallConv, + flags: &settings::Flags, ) -> CodegenResult> { - let sig = ABISig::from_func_sig::(sig)?; + let ir_sig = ensure_struct_return_ptr_is_returned(sig); + let sig = ABISig::from_func_sig::(&ir_sig)?; let (uses, defs) = abisig_to_uses_and_defs::(&sig); Ok(ABICallerImpl { + ir_sig, sig, uses, defs, dest: CallDest::ExtName(extname.clone(), dist), opcode: ir::Opcode::Call, caller_conv, + flags: flags.clone(), _mach: PhantomData, }) } @@ -1322,16 +1424,20 @@ impl ABICallerImpl { ptr: Reg, opcode: ir::Opcode, caller_conv: isa::CallConv, + flags: &settings::Flags, ) -> CodegenResult> { - let sig = ABISig::from_func_sig::(sig)?; + let ir_sig = ensure_struct_return_ptr_is_returned(sig); + let sig = ABISig::from_func_sig::(&ir_sig)?; let (uses, defs) = abisig_to_uses_and_defs::(&sig); Ok(ABICallerImpl { + ir_sig, sig, uses, defs, dest: CallDest::Reg(ptr), opcode, caller_conv, + flags: flags.clone(), _mach: PhantomData, }) } @@ -1355,6 +1461,10 @@ fn adjust_stack_and_nominal_sp>( impl ABICaller for ABICallerImpl { type I = M::I; + fn signature(&self) -> &ir::Signature { + &self.ir_sig + } + fn num_args(&self) -> usize { if self.sig.stack_ret_arg.is_some() { self.sig.args.len() - 1 @@ -1387,8 +1497,13 @@ impl ABICaller for ABICallerImpl { let word_rc = M::word_reg_class(); let word_bits = M::word_bits() as usize; match &self.sig.args[idx] { - &ABIArg::Reg(regs, ty, ext, _) => { - let ext = M::get_ext_mode(self.sig.call_conv, ext); + &ABIArg::Reg { + regs, + ty, + extension, + .. + } => { + let ext = M::get_ext_mode(self.sig.call_conv, extension); if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits { let reg = regs.only_reg().unwrap(); assert_eq!(word_rc, reg.get_class()); @@ -1414,8 +1529,14 @@ impl ABICaller for ABICallerImpl { } } } - &ABIArg::Stack(off, mut ty, ext, _) => { - let ext = M::get_ext_mode(self.sig.call_conv, ext); + &ABIArg::Stack { + offset, + ty, + extension, + .. + } => { + let mut ty = ty; + let ext = M::get_ext_mode(self.sig.call_conv, extension); if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits { let from_reg = from_regs .only_reg() @@ -1439,7 +1560,28 @@ impl ABICaller for ABICallerImpl { // Store the extended version. ty = M::word_type(); } - for insn in gen_store_stack_multi::(StackAMode::SPOffset(off, ty), from_regs, ty) + for insn in + gen_store_stack_multi::(StackAMode::SPOffset(offset, ty), from_regs, ty) + { + ctx.emit(insn); + } + } + &ABIArg::StructArg { offset, size, .. } => { + let src_ptr = from_regs.only_reg().unwrap(); + let dst_ptr = ctx.alloc_tmp(M::word_type()).only_reg().unwrap(); + ctx.emit(M::gen_get_stack_addr( + StackAMode::SPOffset(offset, I8), + dst_ptr, + I8, + )); + // Emit a memcpy from `src_ptr` to `dst_ptr` of `size` bytes. + // N.B.: because we process StructArg params *first*, this is + // safe w.r.t. clobbers: we have not yet filled in any other + // arg regs. + let memcpy_call_conv = isa::CallConv::for_libcall(&self.flags, self.sig.call_conv); + for insn in + M::gen_memcpy(memcpy_call_conv, dst_ptr.to_reg(), src_ptr, size as usize) + .into_iter() { ctx.emit(insn); } @@ -1447,6 +1589,24 @@ impl ABICaller for ABICallerImpl { } } + fn get_copy_to_arg_order(&self) -> SmallVec<[usize; 8]> { + let mut ret = SmallVec::new(); + for (i, arg) in self.sig.args.iter().enumerate() { + // Struct args. + if arg.is_struct_arg() { + ret.push(i); + } + } + for (i, arg) in self.sig.args.iter().enumerate() { + // Non-struct args. Skip an appended return-area arg for multivalue + // returns, if any. + if !arg.is_struct_arg() && i < self.ir_sig.params.len() { + ret.push(i); + } + } + ret + } + fn emit_copy_retval_to_regs>( &self, ctx: &mut C, @@ -1456,21 +1616,22 @@ impl ABICaller for ABICallerImpl { match &self.sig.rets[idx] { // Extension mode doesn't matter because we're copying out, not in, // and we ignore high bits in our own registers by convention. - &ABIArg::Reg(regs, ty, _, _) => { + &ABIArg::Reg { regs, ty, .. } => { for insn in gen_move_multi::(into_regs, regs.map(|r| r.to_reg()), ty) { ctx.emit(insn); } } - &ABIArg::Stack(off, ty, _, _) => { + &ABIArg::Stack { offset, ty, .. } => { let ret_area_base = self.sig.stack_arg_space; for insn in gen_load_stack_multi::( - StackAMode::SPOffset(off + ret_area_base, ty), + StackAMode::SPOffset(offset + ret_area_base, ty), into_regs, ty, ) { ctx.emit(insn); } } + &ABIArg::StructArg { .. } => panic!("Unexpected StructArg location for return value"), } } diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 28e4edd0c7..e35e3b068e 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -375,8 +375,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } } - let vm_context = f - .signature + let vm_context = vcode + .abi() + .signature() .special_param_index(ArgumentPurpose::VMContext) .map(|vm_context_index| { let entry_block = f.layout.entry_block().unwrap(); @@ -386,7 +387,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { // Assign vreg(s) to each return value. let mut retval_regs = vec![]; - for ret in &f.signature.returns { + for ret in &vcode.abi().signature().returns.clone() { let regs = alloc_vregs(ret.value_type, &mut next_vreg, &mut vcode)?; retval_regs.push(regs); debug!("retval gets regs {:?}", regs); @@ -465,6 +466,24 @@ impl<'func, I: VCodeInst> Lower<'func, I> { for insn in self.vcode.abi().gen_copy_arg_to_regs(i, regs).into_iter() { self.emit(insn); } + if self.abi().signature().params[i].purpose == ArgumentPurpose::StructReturn { + assert!(regs.len() == 1); + let ty = self.abi().signature().params[i].value_type; + // The ABI implementation must have ensured that a StructReturn + // arg is present in the return values. + let struct_ret_idx = self + .abi() + .signature() + .returns + .iter() + .position(|ret| ret.purpose == ArgumentPurpose::StructReturn) + .expect("StructReturn return value not present!"); + self.emit(I::gen_move( + Writable::from_reg(self.retval_regs[struct_ret_idx].regs()[0]), + regs.regs()[0].to_reg(), + ty, + )); + } } if let Some(insn) = self.vcode.abi().gen_retval_area_setup() { self.emit(insn); diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index e7ee34f283..5e5d2ffb86 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -1006,7 +1006,7 @@ block0(v0: i128, v1: i128): ; nextln: movq %rsp, %rbp ; nextln: subq $$16, %rsp ; nextln: movq %r12, 0(%rsp) -; nextln: virtual_sp_offset_adjust 8 +; nextln: virtual_sp_offset_adjust 16 ; nextln: movq %r8, %r12 ; nextln: subq $$16, %rsp ; nextln: virtual_sp_offset_adjust 16 diff --git a/cranelift/filetests/filetests/isa/x64/struct-arg.clif b/cranelift/filetests/filetests/isa/x64/struct-arg.clif new file mode 100644 index 0000000000..3c867038a7 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/struct-arg.clif @@ -0,0 +1,147 @@ +test compile +target x86_64 +feature "experimental_x64" + +function u0:0(i64 sarg(64)) -> i8 system_v { +block0(v0: i64): + v1 = load.i8 v0 + return v1 +} + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: lea 16(%rbp), %rsi +; nextln: movzbq 0(%rsi), %rsi +; nextln: movq %rsi, %rax +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +function u0:1(i64 sarg(64), i64) -> i8 system_v { +block0(v0: i64, v1: i64): + v2 = load.i8 v1 + v3 = load.i8 v0 + v4 = iadd.i8 v2, v3 + return v4 +} + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: lea 16(%rbp), %rsi +; nextln: movzbq 0(%rdi), %rdi +; nextln: movzbq 0(%rsi), %rsi +; nextln: addl %esi, %edi +; nextln: movq %rdi, %rax +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +function u0:2(i64) -> i8 system_v { +fn1 = colocated u0:0(i64 sarg(64)) -> i8 system_v + +block0(v0: i64): + v1 = call fn1(v0) + return v1 +} + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: movq %rdi, %rsi +; nextln: subq $$64, %rsp +; nextln: virtual_sp_offset_adjust 64 +; nextln: lea 0(%rsp), %rdi +; nextln: movl $$64, %edx +; nextln: load_ext_name %Memcpy+0, %rcx +; nextln: call *%rcx +; nextln: call User { namespace: 0, index: 0 } +; nextln: addq $$64, %rsp +; nextln: virtual_sp_offset_adjust -64 +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +function u0:3(i64, i64) -> i8 system_v { +fn1 = colocated u0:0(i64, i64 sarg(64)) -> i8 system_v + +block0(v0: i64, v1: i64): + v2 = call fn1(v0, v1) + return v2 +} + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: subq $$16, %rsp +; nextln: movq %r12, 0(%rsp) +; nextln: virtual_sp_offset_adjust 16 +; nextln: movq %rdi, %r12 +; nextln: subq $$64, %rsp +; nextln: virtual_sp_offset_adjust 64 +; nextln: lea 0(%rsp), %rdi +; nextln: movl $$64, %edx +; nextln: load_ext_name %Memcpy+0, %rcx +; nextln: call *%rcx +; nextln: movq %r12, %rdi +; nextln: call User { namespace: 0, index: 0 } +; nextln: addq $$64, %rsp +; nextln: virtual_sp_offset_adjust -64 +; nextln: movq 0(%rsp), %r12 +; nextln: addq $$16, %rsp +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +function u0:4(i64 sarg(128), i64 sarg(64)) -> i8 system_v { +block0(v0: i64, v1: i64): + v2 = load.i8 v0 + v3 = load.i8 v1 + v4 = iadd.i8 v2, v3 + return v4 +} + +; check: movq %rsp, %rbp +; nextln: lea 16(%rbp), %rsi +; nextln: lea 144(%rbp), %rdi +; nextln: movzbq 0(%rsi), %rsi +; nextln: movzbq 0(%rdi), %rdi +; nextln: addl %edi, %esi +; nextln: movq %rsi, %rax +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret + +function u0:5(i64, i64, i64) -> i8 system_v { +fn1 = colocated u0:0(i64, i64 sarg(128), i64 sarg(64)) -> i8 system_v + +block0(v0: i64, v1: i64, v2: i64): + v3 = call fn1(v0, v1, v2) + return v3 +} + +; check: movq %rsp, %rbp +; nextln: subq $$16, %rsp +; nextln: movq %r12, 0(%rsp) +; nextln: movq %r13, 8(%rsp) +; nextln: virtual_sp_offset_adjust 16 +; nextln: movq %rdi, %r12 +; nextln: movq %rdx, %r13 +; nextln: subq $$192, %rsp +; nextln: virtual_sp_offset_adjust 192 +; nextln: lea 0(%rsp), %rdi +; nextln: movl $$128, %edx +; nextln: load_ext_name %Memcpy+0, %rcx +; nextln: call *%rcx +; nextln: lea 128(%rsp), %rdi +; nextln: movq %r13, %rsi +; nextln: movl $$64, %edx +; nextln: load_ext_name %Memcpy+0, %rcx +; nextln: call *%rcx +; nextln: movq %r12, %rdi +; nextln: call User { namespace: 0, index: 0 } +; nextln: addq $$192, %rsp +; nextln: virtual_sp_offset_adjust -192 +; nextln: movq 0(%rsp), %r12 +; nextln: movq 8(%rsp), %r13 +; nextln: addq $$16, %rsp +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret diff --git a/cranelift/filetests/filetests/isa/x64/struct-ret.clif b/cranelift/filetests/filetests/isa/x64/struct-ret.clif new file mode 100644 index 0000000000..05ebbd100b --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/struct-ret.clif @@ -0,0 +1,19 @@ +test compile +target x86_64 +feature "experimental_x64" + +function %f0(i64 sret) { +block0(v0: i64): + v1 = iconst.i64 42 + store v1, v0 + return +} + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: movq %rdi, %rax +; nextln: movl $$42, %esi +; nextln: movq %rsi, 0(%rdi) +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret From f8268b2139460a9e113b5a7b6a4f41b254b33a90 Mon Sep 17 00:00:00 2001 From: Han Zhao Date: Tue, 19 Jan 2021 17:08:11 +0100 Subject: [PATCH 14/55] Bugfix of issue #2575 Bugfix of issue #2575. Use libc 0.2.82 on aarch64-apple-darwin Apple Silicon, and local test passes. --- crates/runtime/src/traphandlers.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 948a24d530..942bdef0f9 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -164,9 +164,15 @@ cfg_if::cfg_if! { } else if #[cfg(all(any(target_os = "linux", target_os = "android"), target_arch = "aarch64"))] { let cx = &*(cx as *const libc::ucontext_t); cx.uc_mcontext.pc as *const u8 - } else if #[cfg(target_os = "macos")] { + } else if #[cfg(all(target_os = "macos", target_arch = "x86_64"))] { let cx = &*(cx as *const libc::ucontext_t); (*cx.uc_mcontext).__ss.__rip as *const u8 + } else if #[cfg(all(target_os = "macos", target_arch = "x86"))] { + let cx = &*(cx as *const libc::ucontext_t); + (*cx.uc_mcontext).__ss.__eip as *const u8 + } else if #[cfg(all(target_os = "macos", target_arch = "aarch64"))] { + let cx = &*(cx as *const libc::ucontext_t); + (*cx.uc_mcontext).__ss.__pc as *const u8 } else if #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] { let cx = &*(cx as *const libc::ucontext_t); cx.uc_mcontext.mc_rip as *const u8 From 043a8434d2e79e4b0ba2a179bf30e4965d26186e Mon Sep 17 00:00:00 2001 From: Anton Kirilov Date: Mon, 11 Jan 2021 18:23:03 +0000 Subject: [PATCH 15/55] Cranelift AArch64: Improve the Popcnt implementation Now the backend uses the CNT instruction, which results into a major simplification. Copyright (c) 2021, Arm Limited. --- .../codegen/src/isa/aarch64/inst/args.rs | 8 + .../codegen/src/isa/aarch64/inst/emit.rs | 6 + .../src/isa/aarch64/inst/emit_tests.rs | 33 ++++ cranelift/codegen/src/isa/aarch64/inst/mod.rs | 3 + .../codegen/src/isa/aarch64/lower_inst.rs | 168 +++++------------- .../filetests/isa/aarch64/bitops.clif | 70 ++------ 6 files changed, 106 insertions(+), 182 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs index 738495714a..1a55029b32 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/args.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs @@ -601,6 +601,14 @@ impl ScalarSize { } } + /// Convert from an integer operand size. + pub fn from_operand_size(size: OperandSize) -> ScalarSize { + match size { + OperandSize::Size32 => ScalarSize::Size32, + OperandSize::Size64 => ScalarSize::Size64, + } + } + /// Convert from a type into the smallest size that fits. pub fn from_ty(ty: Type) -> ScalarSize { Self::from_bits(ty_bits(ty)) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 432bbc19dd..1195e879bb 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -1463,12 +1463,18 @@ impl MachInstEmit for Inst { debug_assert!(size == VectorSize::Size32x4 || size == VectorSize::Size64x2); (0b0, 0b11000, enc_size | 0b10) } + VecMisc2::Cnt => { + debug_assert!(size == VectorSize::Size8x8 || size == VectorSize::Size8x16); + (0b0, 0b00101, enc_size) + } }; sink.put4(enc_vec_rr_misc((q << 1) | u, size, bits_12_16, rd, rn)); } &Inst::VecLanes { op, rd, rn, size } => { let (q, size) = match size { + VectorSize::Size8x8 => (0b0, 0b00), VectorSize::Size8x16 => (0b1, 0b00), + VectorSize::Size16x4 => (0b0, 0b01), VectorSize::Size16x8 => (0b1, 0b01), VectorSize::Size32x4 => (0b1, 0b10), _ => unreachable!(), diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index f01fbf43f0..63232d58a4 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -3792,6 +3792,28 @@ fn test_aarch64_binemit() { "frintp v12.2d, v17.2d", )); + insns.push(( + Inst::VecMisc { + op: VecMisc2::Cnt, + rd: writable_vreg(23), + rn: vreg(5), + size: VectorSize::Size8x8, + }, + "B758200E", + "cnt v23.8b, v5.8b", + )); + + insns.push(( + Inst::VecLanes { + op: VecLanesOp::Uminv, + rd: writable_vreg(0), + rn: vreg(31), + size: VectorSize::Size8x8, + }, + "E0AB312E", + "uminv b0, v31.8b", + )); + insns.push(( Inst::VecLanes { op: VecLanesOp::Uminv, @@ -3836,6 +3858,17 @@ fn test_aarch64_binemit() { "addv b2, v29.16b", )); + insns.push(( + Inst::VecLanes { + op: VecLanesOp::Addv, + rd: writable_vreg(15), + rn: vreg(7), + size: VectorSize::Size16x4, + }, + "EFB8710E", + "addv h15, v7.4h", + )); + insns.push(( Inst::VecLanes { op: VecLanesOp::Addv, diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 38b6e29ce4..5e88a783f5 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -331,6 +331,8 @@ pub enum VecMisc2 { Frintm, /// Floating point round to integral, rounding towards plus infinity Frintp, + /// Population count per byte + Cnt, } /// A Vector narrowing operation with two registers. @@ -3752,6 +3754,7 @@ impl Inst { VecMisc2::Frintz => ("frintz", size), VecMisc2::Frintm => ("frintm", size), VecMisc2::Frintp => ("frintp", size), + VecMisc2::Cnt => ("cnt", size), }; let rd_size = if is_shll { size.widen() } else { size }; diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 73c11008c4..93c2385098 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -962,143 +962,57 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Popcnt => { - // Lower popcount using the following algorithm: - // - // x -= (x >> 1) & 0x5555555555555555 - // x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333) - // x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f - // x += x << 8 - // x += x << 16 - // x += x << 32 - // x >> 56 - let ty = ty.unwrap(); let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - // FIXME(#1537): zero-extend 8/16/32-bit operands only to 32 bits, - // and fix the sequence below to work properly for this. - let narrow_mode = NarrowValueMode::ZeroExtend64; - let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); - let tmp = ctx.alloc_tmp(I64).only_reg().unwrap(); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let ty = ty.unwrap(); + let size = ScalarSize::from_operand_size(OperandSize::from_ty(ty)); + let tmp = ctx.alloc_tmp(I8X16).only_reg().unwrap(); - // If this is a 32-bit Popcnt, use Lsr32 to clear the top 32 bits of the register, then - // the rest of the code is identical to the 64-bit version. - // lsr [wx]d, [wx]n, #1 - ctx.emit(Inst::AluRRImmShift { - alu_op: choose_32_64(ty, ALUOp::Lsr32, ALUOp::Lsr64), - rd: rd, + // fmov tmp, rn + // cnt tmp.8b, tmp.8b + // addp tmp.8b, tmp.8b, tmp.8b / addv tmp, tmp.8b / (no instruction for 8-bit inputs) + // umov rd, tmp.b[0] + + ctx.emit(Inst::MovToFpu { + rd: tmp, rn: rn, - immshift: ImmShift::maybe_from_u64(1).unwrap(), + size, }); - - // and xd, xd, #0x5555555555555555 - ctx.emit(Inst::AluRRImmLogic { - alu_op: ALUOp::And64, - rd: rd, - rn: rd.to_reg(), - imml: ImmLogic::maybe_from_u64(0x5555555555555555, I64).unwrap(), - }); - - // sub xd, xn, xd - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Sub64, - rd: rd, - rn: rn, - rm: rd.to_reg(), - }); - - // and xt, xd, #0x3333333333333333 - ctx.emit(Inst::AluRRImmLogic { - alu_op: ALUOp::And64, - rd: tmp, - rn: rd.to_reg(), - imml: ImmLogic::maybe_from_u64(0x3333333333333333, I64).unwrap(), - }); - - // lsr xd, xd, #2 - ctx.emit(Inst::AluRRImmShift { - alu_op: ALUOp::Lsr64, - rd: rd, - rn: rd.to_reg(), - immshift: ImmShift::maybe_from_u64(2).unwrap(), - }); - - // and xd, xd, #0x3333333333333333 - ctx.emit(Inst::AluRRImmLogic { - alu_op: ALUOp::And64, - rd: rd, - rn: rd.to_reg(), - imml: ImmLogic::maybe_from_u64(0x3333333333333333, I64).unwrap(), - }); - - // add xt, xd, xt - ctx.emit(Inst::AluRRR { - alu_op: ALUOp::Add64, - rd: tmp, - rn: rd.to_reg(), - rm: tmp.to_reg(), - }); - - // add xt, xt, xt LSR #4 - ctx.emit(Inst::AluRRRShift { - alu_op: ALUOp::Add64, + ctx.emit(Inst::VecMisc { + op: VecMisc2::Cnt, rd: tmp, rn: tmp.to_reg(), - rm: tmp.to_reg(), - shiftop: ShiftOpAndAmt::new( - ShiftOp::LSR, - ShiftOpShiftImm::maybe_from_shift(4).unwrap(), - ), + size: VectorSize::Size8x8, }); - // and xt, xt, #0x0f0f0f0f0f0f0f0f - ctx.emit(Inst::AluRRImmLogic { - alu_op: ALUOp::And64, - rd: tmp, - rn: tmp.to_reg(), - imml: ImmLogic::maybe_from_u64(0x0f0f0f0f0f0f0f0f, I64).unwrap(), - }); + match ScalarSize::from_ty(ty) { + ScalarSize::Size8 => {} + ScalarSize::Size16 => { + // ADDP is usually cheaper than ADDV. + ctx.emit(Inst::VecRRR { + alu_op: VecALUOp::Addp, + rd: tmp, + rn: tmp.to_reg(), + rm: tmp.to_reg(), + size: VectorSize::Size8x8, + }); + } + ScalarSize::Size32 | ScalarSize::Size64 => { + ctx.emit(Inst::VecLanes { + op: VecLanesOp::Addv, + rd: tmp, + rn: tmp.to_reg(), + size: VectorSize::Size8x8, + }); + } + sz => panic!("Unexpected scalar FP operand size: {:?}", sz), + } - // add xt, xt, xt, LSL #8 - ctx.emit(Inst::AluRRRShift { - alu_op: ALUOp::Add64, - rd: tmp, + ctx.emit(Inst::MovFromVec { + rd, rn: tmp.to_reg(), - rm: tmp.to_reg(), - shiftop: ShiftOpAndAmt::new( - ShiftOp::LSL, - ShiftOpShiftImm::maybe_from_shift(8).unwrap(), - ), - }); - - // add xt, xt, xt, LSL #16 - ctx.emit(Inst::AluRRRShift { - alu_op: ALUOp::Add64, - rd: tmp, - rn: tmp.to_reg(), - rm: tmp.to_reg(), - shiftop: ShiftOpAndAmt::new( - ShiftOp::LSL, - ShiftOpShiftImm::maybe_from_shift(16).unwrap(), - ), - }); - - // add xt, xt, xt, LSL #32 - ctx.emit(Inst::AluRRRShift { - alu_op: ALUOp::Add64, - rd: tmp, - rn: tmp.to_reg(), - rm: tmp.to_reg(), - shiftop: ShiftOpAndAmt::new( - ShiftOp::LSL, - ShiftOpShiftImm::maybe_from_shift(32).unwrap(), - ), - }); - - // lsr xd, xt, #56 - ctx.emit(Inst::AluRRImmShift { - alu_op: ALUOp::Lsr64, - rd: rd, - rn: tmp.to_reg(), - immshift: ImmShift::maybe_from_u64(56).unwrap(), + idx: 0, + size: VectorSize::Size8x16, }); } diff --git a/cranelift/filetests/filetests/isa/aarch64/bitops.clif b/cranelift/filetests/filetests/isa/aarch64/bitops.clif index ab1c113104..8730128cc5 100644 --- a/cranelift/filetests/filetests/isa/aarch64/bitops.clif +++ b/cranelift/filetests/filetests/isa/aarch64/bitops.clif @@ -230,19 +230,10 @@ block0(v0: i64): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp -; nextln: lsr x1, x0, #1 -; nextln: and x1, x1, #6148914691236517205 -; nextln: sub x1, x0, x1 -; nextln: and x0, x1, #3689348814741910323 -; nextln: lsr x1, x1, #2 -; nextln: and x1, x1, #3689348814741910323 -; nextln: add x0, x1, x0 -; nextln: add x0, x0, x0, LSR 4 -; nextln: and x0, x0, #1085102592571150095 -; nextln: add x0, x0, x0, LSL 8 -; nextln: add x0, x0, x0, LSL 16 -; nextln: add x0, x0, x0, LSL 32 -; nextln: lsr x0, x0, #56 +; nextln: fmov d0, x0 +; nextln: cnt v0.8b, v0.8b +; nextln: addv b0, v0.8b +; nextln: umov w0, v0.b[0] ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret @@ -255,20 +246,10 @@ block0(v0: i32): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp -; nextln: mov w0, w0 -; nextln: lsr w1, w0, #1 -; nextln: and x1, x1, #6148914691236517205 -; nextln: sub x1, x0, x1 -; nextln: and x0, x1, #3689348814741910323 -; nextln: lsr x1, x1, #2 -; nextln: and x1, x1, #3689348814741910323 -; nextln: add x0, x1, x0 -; nextln: add x0, x0, x0, LSR 4 -; nextln: and x0, x0, #1085102592571150095 -; nextln: add x0, x0, x0, LSL 8 -; nextln: add x0, x0, x0, LSL 16 -; nextln: add x0, x0, x0, LSL 32 -; nextln: lsr x0, x0, #56 +; nextln: fmov s0, w0 +; nextln: cnt v0.8b, v0.8b +; nextln: addv b0, v0.8b +; nextln: umov w0, v0.b[0] ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret @@ -281,20 +262,10 @@ block0(v0: i16): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp -; nextln: uxth w0, w0 -; nextln: lsr w1, w0, #1 -; nextln: and x1, x1, #6148914691236517205 -; nextln: sub x1, x0, x1 -; nextln: and x0, x1, #3689348814741910323 -; nextln: lsr x1, x1, #2 -; nextln: and x1, x1, #3689348814741910323 -; nextln: add x0, x1, x0 -; nextln: add x0, x0, x0, LSR 4 -; nextln: and x0, x0, #1085102592571150095 -; nextln: add x0, x0, x0, LSL 8 -; nextln: add x0, x0, x0, LSL 16 -; nextln: add x0, x0, x0, LSL 32 -; nextln: lsr x0, x0, #56 +; nextln: fmov s0, w0 +; nextln: cnt v0.8b, v0.8b +; nextln: addp v0.8b, v0.8b, v0.8b +; nextln: umov w0, v0.b[0] ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret @@ -307,20 +278,9 @@ block0(v0: i8): ; check: stp fp, lr, [sp, #-16]! ; nextln: mov fp, sp -; nextln: uxtb w0, w0 -; nextln: lsr w1, w0, #1 -; nextln: and x1, x1, #6148914691236517205 -; nextln: sub x1, x0, x1 -; nextln: and x0, x1, #3689348814741910323 -; nextln: lsr x1, x1, #2 -; nextln: and x1, x1, #3689348814741910323 -; nextln: add x0, x1, x0 -; nextln: add x0, x0, x0, LSR 4 -; nextln: and x0, x0, #1085102592571150095 -; nextln: add x0, x0, x0, LSL 8 -; nextln: add x0, x0, x0, LSL 16 -; nextln: add x0, x0, x0, LSL 32 -; nextln: lsr x0, x0, #56 +; nextln: fmov s0, w0 +; nextln: cnt v0.8b, v0.8b +; nextln: umov w0, v0.b[0] ; nextln: mov sp, fp ; nextln: ldp fp, lr, [sp], #16 ; nextln: ret From d37e2d53c75ece555014197b7f0ef2bfbbddc2a9 Mon Sep 17 00:00:00 2001 From: Han Zhao Date: Tue, 19 Jan 2021 19:11:32 +0100 Subject: [PATCH 16/55] Updated libc version to 0.2.82 in dependencies --- crates/runtime/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 78564d6faf..40266d3793 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -14,7 +14,7 @@ edition = "2018" [dependencies] wasmtime-environ = { path = "../environ", version = "0.22.0" } region = "2.1.0" -libc = { version = "0.2.70", default-features = false } +libc = { version = "0.2.82", default-features = false } log = "0.4.8" memoffset = "0.6.0" indexmap = "1.0.2" From 32343a70184b32dfdeccb90ab286783bad4f1d18 Mon Sep 17 00:00:00 2001 From: Han Zhao Date: Tue, 19 Jan 2021 19:27:20 +0100 Subject: [PATCH 17/55] Updated Cargo.lock for libc --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f6cf7f0f26..4a3f6b0913 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1115,9 +1115,9 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" [[package]] name = "libc" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" +checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" [[package]] name = "libfuzzer-sys" From 207f60a18e40511fbd4b8e08933067b83dad3c0a Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 21 Jan 2021 09:21:30 -0600 Subject: [PATCH 18/55] module-linking: Implement outer module aliases (#2590) This commit fully implements outer aliases of the module linking proposal. Outer aliases can now handle multiple-level-up aliases and now properly also handle closed-over-values of modules that are either imported or defined. The structure of `wasmtime::Module` was altered as part of this commit. It is now a compiled module plus two lists of "upvars", or closed over values used when instantiating the module. One list of upvars is compiled artifacts which are submodules that could be used. Another is module values that are injected via outer aliases. Serialization and such have been updated as appropriate to handle this. --- crates/environ/src/module.rs | 24 +- crates/environ/src/module_environ.rs | 90 ++++-- crates/jit/src/instantiate.rs | 8 +- crates/wasmtime/src/instance.rs | 68 +++-- crates/wasmtime/src/module.rs | 267 ++++++++++++++---- .../module-linking/alias-outer.wast | 67 +++++ .../misc_testsuite/module-linking/alias.wast | 4 +- 7 files changed, 420 insertions(+), 108 deletions(-) create mode 100644 tests/misc_testsuite/module-linking/alias-outer.wast diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 644f1a2bb3..b25682acf2 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -239,11 +239,31 @@ pub enum Initializer { args: IndexMap, }, - /// A module is defined into the module index space, and which module is - /// being defined is specified by the index payload. + /// A module is being created from a set of compiled artifacts. + CreateModule { + /// The index of the artifact that's being convereted into a module. + artifact_index: usize, + /// The list of artifacts that this module value will be inheriting. + artifacts: Vec, + /// The list of modules that this module value will inherit. + modules: Vec, + }, + + /// A module is created from a closed-over-module value, defined when this + /// module was created. DefineModule(usize), } +/// Where module values can come from when creating a new module from a compiled +/// artifact. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ModuleUpvar { + /// A module value is inherited from the module creating the new module. + Inherit(usize), + /// A module value comes from the instance-to-be-created module index space. + Local(ModuleIndex), +} + impl Module { /// Allocates the module data structures. pub fn new() -> Self { diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 7bb30ea166..9d1f89cb1f 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -1,6 +1,6 @@ use crate::module::{ - Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, TableElements, - TablePlan, TypeTables, + Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, ModuleUpvar, + TableElements, TablePlan, TypeTables, }; use crate::tunables::Tunables; use cranelift_codegen::ir; @@ -75,6 +75,14 @@ pub struct ModuleTranslation<'data> { code_index: u32, implicit_instances: HashMap<&'data str, InstanceIndex>, + + /// The artifacts which are needed from the parent module when this module + /// is created. This is used to insert into `Initializer::CreateModule` when + /// this module is defined in the parent. + creation_artifacts: Vec, + + /// Same as `creation_artifacts`, but for modules instead of artifacts. + creation_modules: Vec, } /// Contains function data: byte code and its offset in the module. @@ -326,8 +334,7 @@ impl<'data> ModuleEnvironment<'data> { } } - fn gen_type_of_module(&mut self, module: usize) -> ModuleTypeIndex { - let module = &self.results[module].module; + fn gen_type_of_module(&mut self, module: &Module) -> ModuleTypeIndex { let imports = module .imports() .map(|(s, field, ty)| { @@ -880,20 +887,43 @@ and for re-adding support for interface types you can see this issue: } fn module_end(&mut self) { - let (record_initializer, done) = match self.in_progress.pop() { + self.result.creation_artifacts.shrink_to_fit(); + self.result.creation_modules.shrink_to_fit(); + + let (record_initializer, mut done) = match self.in_progress.pop() { Some(m) => (true, mem::replace(&mut self.result, m)), None => (false, mem::take(&mut self.result)), }; - self.results.push(done); + if record_initializer { - let index = self.results.len() - 1; + // Record the type of the module we just finished in our own + // module's list of modules. + let sig = self.gen_type_of_module(&done.module); + self.result.module.modules.push(sig); + + // The root module will store the artifacts for this finished + // module at `artifact_index`. This then needs to be inherited by + // all later modules coming down to our now-current `self.result`... + let mut artifact_index = self.results.len(); + for result in self.in_progress.iter_mut().chain(Some(&mut self.result)) { + result.creation_artifacts.push(artifact_index); + artifact_index = result.creation_artifacts.len() - 1; + } + // ... and then `self.result` needs to create a new module with + // whatever was record to save off as its own artifacts/modules. self.result .module .initializers - .push(Initializer::DefineModule(index)); - let sig = self.gen_type_of_module(index); - self.result.module.modules.push(sig); + .push(Initializer::CreateModule { + artifact_index, + artifacts: mem::take(&mut done.creation_artifacts), + modules: mem::take(&mut done.creation_modules), + }); } + + // And the final step is to insert the module into the list of finished + // modules to get returned at the end. + self.results.push(done); } fn reserve_instances(&mut self, amt: u32) { @@ -936,16 +966,44 @@ and for re-adding support for interface types you can see this issue: self.result.module.types.push(ty); } - // FIXME(WebAssembly/module-linking#28) unsure how to implement this - // at this time, if we can alias imported modules it's a lot harder, - // otherwise we'll need to figure out how to translate `index` to a - // `usize` for a defined module (creating Initializer::DefineModule) + // Modules are a bit trickier since we need to record how to track + // the state from the original module down to our own. Alias::OuterModule { relative_depth, index, } => { - drop((relative_depth, index)); - unimplemented!() + // First we can copy the type from the parent module into our + // own module to record what type our module definition will + // have. + let module_idx = self.in_progress.len() - 1 - (relative_depth as usize); + let module_ty = self.in_progress[module_idx].module.modules[index]; + self.result.module.modules.push(module_ty); + + // Next we'll be injecting a module value that is closed over, + // and that will be used to define the module into the index + // space. Record an initializer about where our module is + // sourced from (which will be stored within each module value + // itself). + let module_index = self.result.creation_modules.len(); + self.result + .module + .initializers + .push(Initializer::DefineModule(module_index)); + + // And finally we need to record a breadcrumb trail of how to + // get the module value into `module_index`. The module just + // after our destination module will use a `ModuleIndex` to + // fetch the module value, and everything else inbetween will + // inherit that module's closed-over value. + let mut upvar = ModuleUpvar::Local(index); + for outer in self.in_progress[module_idx + 1..].iter_mut() { + let upvar = mem::replace( + &mut upvar, + ModuleUpvar::Inherit(outer.creation_modules.len()), + ); + outer.creation_modules.push(upvar); + } + self.result.creation_modules.push(upvar); } // This case is slightly more involved, we'll be recording all the diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index c0738c9c2c..655432e00f 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -221,7 +221,7 @@ impl CompiledModule { artifacts: Vec, isa: &dyn TargetIsa, profiler: &dyn ProfilingAgent, - ) -> Result, SetupError> { + ) -> Result>, SetupError> { maybe_parallel!(artifacts.(into_iter | into_par_iter)) .map(|a| CompiledModule::from_artifacts(a, isa, profiler)) .collect() @@ -232,7 +232,7 @@ impl CompiledModule { artifacts: CompilationArtifacts, isa: &dyn TargetIsa, profiler: &dyn ProfilingAgent, - ) -> Result { + ) -> Result, SetupError> { // Allocate all of the compiled functions into executable memory, // copying over their contents. let (code_memory, code_range, finished_functions, trampolines) = build_code_memory( @@ -266,7 +266,7 @@ impl CompiledModule { let finished_functions = FinishedFunctions(finished_functions); - Ok(Self { + Ok(Arc::new(Self { module: Arc::new(artifacts.module.clone()), artifacts, code: Arc::new(ModuleCode { @@ -275,7 +275,7 @@ impl CompiledModule { }), finished_functions, trampolines, - }) + })) } /// Crate an `Instance` from this `CompiledModule`. diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index e574f38812..60c52d6271 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -62,46 +62,27 @@ fn instantiate( .with_context(|| format!("incompatible import type for `{}`", name))?; } - // Turns out defining any kind of module is pretty easy, we're just - // slinging around pointers. - Initializer::DefineModule(idx) => { - imports.modules.push(module.submodule(*idx)); - } - // Here we lookup our instance handle, find the right export, // and then push that item into our own index space. We eschew - // type-checking since only valid modules reach this point. - // - // Note that export lookup here needs to happen by name. The - // `export` index is an index into our local type definition of the - // type of the instance to figure out what name it was assigned. - // This is where the subtyping happens! - // - // Note that the unsafety here is because we're asserting that the - // handle comes from our same store, but this should be true because - // we acquired the handle from an instance in the store. + // type-checking since only valid modules should reach this point. Initializer::AliasInstanceExport { instance, export } => { let export = &imports.instances[*instance][export]; let item = unsafe { Extern::from_wasmtime_export(export, store) }; imports.push_extern(&item); } - // Oh boy a recursive instantiation! The recursive arguments here - // are pretty simple, and the only slightly-meaty one is how - // arguments are pulled from `args` and pushed directly into the - // builder specified, which should be an easy enough - // copy-the-pointer operation in all cases. + // Oh boy a recursive instantiation! // - // Note that this recursive call shouldn't result in an infinite - // loop because of wasm module validation which requires everything - // to be a DAG. Additionally the recursion should also be bounded - // due to validation. We may one day need to make this an iterative - // loop, however. + // We use our local index space of modules to find the module to + // instantiate and argument lookup is defined as looking inside of + // `args`. Like above with aliases all type checking is eschewed + // because only valid modules should reach this point. // - // Also note that there's some unsafety here around cloning - // `InstanceHandle` because the handle may not live long enough, but - // we're doing all of this in the context of our `Store` argument - // above so we should be safe here. + // Note that it's thought that due to the acyclic nature of + // instantiation this can't loop to blow the native stack, and + // validation should in theory ensure this has a bounded depth. + // Despite this we may need to change this to a loop instead of + // recursion one day. Initializer::Instantiate { module, args } => { let handle = instantiate( store, @@ -134,6 +115,33 @@ fn instantiate( )?; imports.instances.push(handle); } + + // A new module is being defined, and the source of this module is + // our module's list of closed-over-modules. + // + // This is used for outer aliases. + Initializer::DefineModule(upvar_index) => { + imports + .modules + .push(module.module_upvar(*upvar_index).clone()); + } + + // A new module is defined, created from a set of compiled + // artifacts. The new module value will be created with the + // specified artifacts being closed over as well as the specified + // set of module values in our index/upvar index spaces being closed + // over. + // + // This is used for defining submodules. + Initializer::CreateModule { + artifact_index, + artifacts, + modules, + } => { + let submodule = + module.create_submodule(*artifact_index, artifacts, modules, &imports.modules); + imports.modules.push(submodule); + } } } diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index c78ae8a7f2..8fcd94a3bd 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -2,12 +2,16 @@ use crate::types::{ExportType, ExternType, ImportType}; use crate::{Engine, ModuleType}; use anyhow::{bail, Context, Result}; use bincode::Options; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::hash::Hash; use std::path::Path; use std::sync::Arc; use wasmparser::Validator; #[cfg(feature = "cache")] use wasmtime_cache::ModuleCacheEntry; +use wasmtime_environ::entity::PrimaryMap; +use wasmtime_environ::wasm::ModuleIndex; use wasmtime_jit::{CompilationArtifacts, CompiledModule, TypeTables}; /// A compiled WebAssembly module, ready to be instantiated. @@ -80,14 +84,72 @@ use wasmtime_jit::{CompilationArtifacts, CompiledModule, TypeTables}; /// [`Config`]: crate::Config #[derive(Clone)] pub struct Module { - engine: Engine, - data: Arc, - index: usize, + inner: Arc, } -pub(crate) struct ModuleData { - pub(crate) types: Arc, - pub(crate) modules: Vec, +struct ModuleInner { + engine: Engine, + /// The compiled artifacts for this module that will be instantiated and + /// executed. + module: Arc, + /// Closed-over compilation artifacts used to create submodules when this + /// module is instantiated. + artifact_upvars: Vec>, + /// Closed-over module values which are used when this module is + /// instantiated. + module_upvars: Vec, + /// Type information of this module and all `artifact_upvars` compiled + /// modules. + types: Arc, +} + +/// A small helper struct which defines modules are serialized. +#[derive(serde::Serialize, serde::Deserialize)] +struct ModuleSerialized<'a> { + /// All compiled artifacts neeeded by this module, where the last entry in + /// this list is the artifacts for the module itself. + artifacts: Vec>, + /// Closed-over module values that are also needed for this module. + modules: Vec>, + /// The index into the list of type tables that are used for this module's + /// type tables. + type_tables: usize, +} + +// This is like `std::borrow::Cow` but it doesn't have a `Clone` bound on `T` +enum MyCow<'a, T> { + Borrowed(&'a T), + Owned(T), +} + +impl<'a, T> MyCow<'a, T> { + fn unwrap_owned(self) -> T { + match self { + MyCow::Owned(val) => val, + MyCow::Borrowed(_) => unreachable!(), + } + } +} + +impl<'a, T: Serialize> Serialize for MyCow<'a, T> { + fn serialize(&self, dst: S) -> Result + where + S: serde::ser::Serializer, + { + match self { + MyCow::Borrowed(val) => val.serialize(dst), + MyCow::Owned(val) => val.serialize(dst), + } + } +} + +impl<'a, 'b, T: Deserialize<'a>> Deserialize<'a> for MyCow<'b, T> { + fn deserialize(src: D) -> Result + where + D: serde::de::Deserializer<'a>, + { + Ok(MyCow::Owned(T::deserialize(src)?)) + } } impl Module { @@ -169,7 +231,8 @@ impl Module { /// See [`Module::new`] for other details. pub fn new_with_name(engine: &Engine, bytes: impl AsRef<[u8]>, name: &str) -> Result { let mut module = Module::new(engine, bytes.as_ref())?; - Arc::get_mut(&mut module.data).unwrap().modules[module.index] + Arc::get_mut(&mut Arc::get_mut(&mut module.inner).unwrap().module) + .unwrap() .module_mut() .expect("mutable module") .name = Some(name.to_string()); @@ -254,17 +317,21 @@ impl Module { let (main_module, artifacts, types) = CompilationArtifacts::build(engine.compiler(), binary)?; - let modules = CompiledModule::from_artifacts_list( + let mut modules = CompiledModule::from_artifacts_list( artifacts, engine.compiler().isa(), &*engine.config().profiler, )?; + let module = modules.remove(main_module); - let types = Arc::new(types); Ok(Module { - engine: engine.clone(), - index: main_module, - data: Arc::new(ModuleData { types, modules }), + inner: Arc::new(ModuleInner { + engine: engine.clone(), + module, + types: Arc::new(types), + artifact_upvars: modules, + module_upvars: Vec::new(), + }), }) } @@ -313,21 +380,46 @@ impl Module { /// Serialize compilation artifacts to the buffer. See also `deseriaize`. pub fn serialize(&self) -> Result> { - let artifacts = ( - compiler_fingerprint(&self.engine), - self.data - .modules - .iter() - .map(|i| i.compilation_artifacts()) - .collect::>(), - &*self.data.types, - self.index, - ); - + let mut pushed = HashMap::new(); + let mut tables = Vec::new(); + let module = self.serialized_module(&mut pushed, &mut tables); + let artifacts = (compiler_fingerprint(self.engine()), tables, module); let buffer = bincode_options().serialize(&artifacts)?; Ok(buffer) } + fn serialized_module<'a>( + &'a self, + type_tables_pushed: &mut HashMap, + type_tables: &mut Vec<&'a TypeTables>, + ) -> ModuleSerialized<'a> { + // Deduplicate `Arc` using our two parameters to ensure we + // serialize type tables as little as possible. + let ptr = Arc::as_ptr(self.types()); + let type_tables_idx = *type_tables_pushed.entry(ptr as usize).or_insert_with(|| { + type_tables.push(self.types()); + type_tables.len() - 1 + }); + ModuleSerialized { + artifacts: self + .inner + .artifact_upvars + .iter() + .map(|i| MyCow::Borrowed(i.compilation_artifacts())) + .chain(Some(MyCow::Borrowed( + self.compiled_module().compilation_artifacts(), + ))) + .collect(), + modules: self + .inner + .module_upvars + .iter() + .map(|i| i.serialized_module(type_tables_pushed, type_tables)) + .collect(), + type_tables: type_tables_idx, + } + } + /// Deserializes and creates a module from the compilation artifacts. /// The `serialize` saves the compilation artifacts along with the host /// fingerprint, which consists of target, compiler flags, and wasmtime @@ -338,44 +430,113 @@ impl Module { /// for modifications or curruptions. All responsibily of signing and its /// verification falls on the embedder. pub fn deserialize(engine: &Engine, serialized: &[u8]) -> Result { - let expected_fingerprint = compiler_fingerprint(engine); - - let (fingerprint, artifacts, types, index) = bincode_options() - .deserialize::<(u64, _, _, _)>(serialized) + let (fingerprint, types, serialized) = bincode_options() + .deserialize::<(u64, Vec, _)>(serialized) .context("Deserialize compilation artifacts")?; - if fingerprint != expected_fingerprint { + + if fingerprint != compiler_fingerprint(engine) { bail!("Incompatible compilation artifact"); } - let modules = CompiledModule::from_artifacts_list( - artifacts, - engine.compiler().isa(), - &*engine.config().profiler, - )?; + let types = types.into_iter().map(Arc::new).collect::>(); + return mk(engine, &types, serialized); - let types = Arc::new(types); - Ok(Module { - engine: engine.clone(), - index, - data: Arc::new(ModuleData { modules, types }), - }) - } - - pub(crate) fn compiled_module(&self) -> &CompiledModule { - &self.data.modules[self.index] - } - - pub(crate) fn submodule(&self, index: usize) -> Module { - assert!(index < self.data.modules.len()); - Module { - engine: self.engine.clone(), - data: self.data.clone(), - index, + fn mk( + engine: &Engine, + types: &Vec>, + module: ModuleSerialized<'_>, + ) -> Result { + let mut artifacts = CompiledModule::from_artifacts_list( + module + .artifacts + .into_iter() + .map(|i| i.unwrap_owned()) + .collect(), + engine.compiler().isa(), + &*engine.config().profiler, + )?; + let inner = ModuleInner { + engine: engine.clone(), + types: types[module.type_tables].clone(), + module: artifacts.pop().unwrap(), + artifact_upvars: artifacts, + module_upvars: module + .modules + .into_iter() + .map(|m| mk(engine, types, m)) + .collect::>>()?, + }; + Ok(Module { + inner: Arc::new(inner), + }) } } + /// Creates a submodule `Module` value from the specified parameters. + /// + /// This is used for creating submodules as part of module instantiation. + /// + /// * `artifact_index` - the index in `artifact_upvars` that we're creating + /// a module for + /// * `artifact_upvars` - the mapping of indices of what artifact upvars are + /// needed for the submodule. The length of this array is the length of + /// the upvars array in the submodule to be created, and each element of + /// this array is an index into this module's upvar array. + /// * `module_upvars` - similar to `artifact_upvars` this is a mapping of + /// how to create the e`module_upvars` of the submodule being created. + /// Each entry in this array is either an index into this module's own + /// module upvars array or it's an index into `modules`, the list of + /// modules so far for the instance where this submodule is being + /// created. + /// * `modules` - array indexed by `module_upvars`. + /// + /// Note that the real meat of this happens in `ModuleEnvironment` + /// translation inside of `wasmtime_environ`. This just does the easy thing + /// of handling all the indices, over there is where the indices are + /// actually calculated and such. + pub(crate) fn create_submodule( + &self, + artifact_index: usize, + artifact_upvars: &[usize], + module_upvars: &[wasmtime_environ::ModuleUpvar], + modules: &PrimaryMap, + ) -> Module { + Module { + inner: Arc::new(ModuleInner { + types: self.types().clone(), + engine: self.engine().clone(), + module: self.inner.artifact_upvars[artifact_index].clone(), + artifact_upvars: artifact_upvars + .iter() + .map(|i| self.inner.artifact_upvars[*i].clone()) + .collect(), + module_upvars: module_upvars + .iter() + .map(|i| match *i { + wasmtime_environ::ModuleUpvar::Inherit(i) => { + self.inner.module_upvars[i].clone() + } + wasmtime_environ::ModuleUpvar::Local(i) => modules[i].clone(), + }) + .collect(), + }), + } + } + + pub(crate) fn compiled_module(&self) -> &CompiledModule { + &self.inner.module + } + pub(crate) fn types(&self) -> &Arc { - &self.data.types + &self.inner.types + } + + /// Looks up the module upvar value at the `index` specified. + /// + /// Note that this panics if `index` is out of bounds since this should + /// only be called for valid indices as part of instantiation. + pub(crate) fn module_upvar(&self, index: usize) -> &Module { + &self.inner.module_upvars[index] } /// Returns identifier/name that this [`Module`] has. This name @@ -585,7 +746,7 @@ impl Module { /// Returns the [`Engine`] that this [`Module`] was compiled by. pub fn engine(&self) -> &Engine { - &self.engine + &self.inner.engine } } diff --git a/tests/misc_testsuite/module-linking/alias-outer.wast b/tests/misc_testsuite/module-linking/alias-outer.wast new file mode 100644 index 0000000000..e6dc6255a3 --- /dev/null +++ b/tests/misc_testsuite/module-linking/alias-outer.wast @@ -0,0 +1,67 @@ +(module $a + (module $m1) + (module $b + (module $m2) + (module $c + (instance (instantiate (module outer $a $m1))) + (instance (instantiate (module outer $b $m2))) + ) + (instance (instantiate $c)) + ) + (instance (instantiate $b)) +) + +(module $a + (module (export "m")) +) + +(module $PARENT + (import "a" "m" (module $b)) + (module $c + (module $d + (instance (instantiate (module outer $PARENT $b))) + ) + (instance (instantiate $d)) + ) + (instance (instantiate $c)) +) + +;; Instantiate `$b` here below twice with two different imports. Ensure the +;; exported modules close over the captured state correctly to ensure that we +;; get the right functions. +(module $a + (module $b (export "close_over_imports") + (import "m" (module $m (export "f" (func (result i32))))) + (module (export "m") + (instance $a (instantiate (module outer $b $m))) + (func (export "f") (result i32) + call (func $a "f")) + ) + ) +) + +(module + (import "a" "close_over_imports" (module $m0 + (import "m" (module (export "f" (func (result i32))))) + (export "m" (module (export "f" (func (result i32))))) + )) + + (module $m1 + (func (export "f") (result i32) + i32.const 0)) + (instance $m_g1 (instantiate $m0 "m" (module $m1))) + (instance $g1 (instantiate (module $m_g1 "m"))) + (module $m2 + (func (export "f") (result i32) + i32.const 1)) + (instance $m_g2 (instantiate $m0 "m" (module $m2))) + (instance $g2 (instantiate (module $m_g2 "m"))) + + (func (export "get1") (result i32) + call (func $g1 "f")) + (func (export "get2") (result i32) + call (func $g2 "f")) +) + +(assert_return (invoke "get1") (i32.const 0)) +(assert_return (invoke "get2") (i32.const 1)) diff --git a/tests/misc_testsuite/module-linking/alias.wast b/tests/misc_testsuite/module-linking/alias.wast index f9d2b2f3fd..2fcad01777 100644 --- a/tests/misc_testsuite/module-linking/alias.wast +++ b/tests/misc_testsuite/module-linking/alias.wast @@ -93,8 +93,7 @@ (instance $a (instantiate $m)) ) -;; alias parent -- module -(; TODO +;; alias outer -- module (module (module $a) (module $m @@ -102,7 +101,6 @@ ) (instance (instantiate $m)) ) -;) ;; The alias, import, type, module, and instance sections can all be interleaved (module $ROOT From 2d5037d84c6c16be4bac6ac42e74f3eeacb190df Mon Sep 17 00:00:00 2001 From: Andronik Ordian Date: Thu, 21 Jan 2021 17:16:07 +0100 Subject: [PATCH 19/55] cache: update zstd to 0.6, disable legacy feature (#2592) --- Cargo.lock | 12 ++++++------ crates/cache/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a3f6b0913..612d9d3c20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2959,18 +2959,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.5.4+zstd.1.4.7" +version = "0.6.0+zstd.1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69996ebdb1ba8b1517f61387a883857818a66c8a295f487b1ffd8fd9d2c82910" +checksum = "d4e44664feba7f2f1a9f300c1f6157f2d1bfc3c15c6f3cf4beabf3f5abe9c237" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "2.0.6+zstd.1.4.7" +version = "3.0.0+zstd.1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98aa931fb69ecee256d44589d19754e61851ae4769bf963b385119b1cc37a49e" +checksum = "d9447afcd795693ad59918c7bbffe42fdd6e467d708f3537e3dc14dc598c573f" dependencies = [ "libc", "zstd-sys", @@ -2978,9 +2978,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.4.18+zstd.1.4.7" +version = "1.4.19+zstd.1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" +checksum = "ec24a9273d24437afb8e71b16f3d9a5d569193cccdb7896213b59f552f387674" dependencies = [ "cc", "glob", diff --git a/crates/cache/Cargo.toml b/crates/cache/Cargo.toml index 970ea91b93..9995dbfb60 100644 --- a/crates/cache/Cargo.toml +++ b/crates/cache/Cargo.toml @@ -18,7 +18,7 @@ log = { version = "0.4.8", default-features = false } serde = { version = "1.0.94", features = ["derive"] } sha2 = "0.9.0" toml = "0.5.5" -zstd = "0.5" +zstd = { version = "0.6", default-features = false } [target.'cfg(target_os = "windows")'.dependencies] winapi = "0.3.7" From 81d248c057d9cb5d137ea3f9347c9622f55e07bf Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 21 Jan 2021 18:05:37 +0100 Subject: [PATCH 20/55] Implement Mach-O TLS access for x64 newBE --- cranelift/codegen/src/isa/x64/inst/emit.rs | 13 +++++++++++++ cranelift/codegen/src/isa/x64/inst/emit_tests.rs | 11 +++++++++++ cranelift/codegen/src/isa/x64/inst/mod.rs | 14 ++++++++++++-- cranelift/codegen/src/isa/x64/lower.rs | 7 +++++++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index e1253b9597..6622903452 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -3016,6 +3016,19 @@ pub(crate) fn emit( ); sink.put4(0); // offset } + + Inst::MachOTlsGetAddr { ref symbol } => { + // movq gv@tlv(%rip), %rdi + sink.put1(0x48); // REX.w + sink.put1(0x8b); // MOV + sink.put1(0x3d); // ModRM byte + emit_reloc(sink, state, Reloc::MachOX86_64Tlv, symbol, -4); + sink.put4(0); // offset + + // callq *(%rdi) + sink.put1(0xff); + sink.put1(0x17); + } } state.clear_post_insn(); diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index f1e74ae22a..a3e1c9b658 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3915,6 +3915,17 @@ fn test_x64_emit() { "elf_tls_get_addr User { namespace: 0, index: 0 }", )); + insns.push(( + Inst::MachOTlsGetAddr { + symbol: ExternalName::User { + namespace: 0, + index: 0, + }, + }, + "488B3D00000000FF17", + "macho_tls_get_addr User { namespace: 0, index: 0 }", + )); + // ======================================================== // Actually run the tests! let mut flag_builder = settings::builder(); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index edc7a65109..d1302cacbc 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -480,6 +480,10 @@ pub enum Inst { /// A call to the `ElfTlsGetAddr` libcall. Returns address /// of TLS symbol in rax. ElfTlsGetAddr { symbol: ExternalName }, + + /// A Mach-O TLS symbol access. Returns address of the TLS + /// symbol in rax. + MachOTlsGetAddr { symbol: ExternalName }, } pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool { @@ -539,7 +543,8 @@ impl Inst { | Inst::XmmLoadConst { .. } | Inst::XmmMinMaxSeq { .. } | Inst::XmmUninitializedValue { .. } - | Inst::ElfTlsGetAddr { .. } => None, + | Inst::ElfTlsGetAddr { .. } + | Inst::MachOTlsGetAddr { .. } => None, // These use dynamic SSE opcodes. Inst::GprToXmm { op, .. } @@ -1791,6 +1796,10 @@ impl PrettyPrint for Inst { Inst::ElfTlsGetAddr { ref symbol } => { format!("elf_tls_get_addr {:?}", symbol) } + + Inst::MachOTlsGetAddr { ref symbol } => { + format!("macho_tls_get_addr {:?}", symbol) + } } } } @@ -2051,7 +2060,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { // No registers are used. } - Inst::ElfTlsGetAddr { .. } => { + Inst::ElfTlsGetAddr { .. } | Inst::MachOTlsGetAddr { .. } => { // All caller-saves are clobbered. // // We use the SysV calling convention here because the @@ -2449,6 +2458,7 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { | Inst::Hlt | Inst::AtomicRmwSeq { .. } | Inst::ElfTlsGetAddr { .. } + | Inst::MachOTlsGetAddr { .. } | Inst::Fence { .. } => { // Instruction doesn't explicitly mention any regs, so it can't have any virtual // regs that we'd need to remap. Hence no action required. diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 3339f21b24..45014fca17 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -5333,6 +5333,13 @@ fn lower_insn_to_regs>( ctx.emit(Inst::ElfTlsGetAddr { symbol }); ctx.emit(Inst::gen_move(dst, regs::rax(), types::I64)); } + TlsModel::Macho => { + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + let (name, _, _) = ctx.symbol_value(insn).unwrap(); + let symbol = name.clone(); + ctx.emit(Inst::MachOTlsGetAddr { symbol }); + ctx.emit(Inst::gen_move(dst, regs::rax(), types::I64)); + } _ => { todo!( "Unimplemented TLS model in x64 backend: {:?}", From 8748cf5bd3f7b7ef3b2b04271683c75c6500e4e1 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 21 Jan 2021 11:59:30 -0600 Subject: [PATCH 21/55] Add an instance limit to `Config` (#2593) * Add an instance limit to `Config` This commit adds a new parameter to `Config` which limits the number of instances that can be created within a store connected to that `Config`. The intention here is to provide a default safeguard against module-linking modules that recursively create too many instances. * Update crates/c-api/include/wasmtime.h Co-authored-by: Peter Huene Co-authored-by: Peter Huene --- crates/c-api/include/wasmtime.h | 8 ++++++++ crates/c-api/src/config.rs | 5 +++++ crates/wasmtime/src/config.rs | 11 ++++++++++ crates/wasmtime/src/instance.rs | 2 ++ crates/wasmtime/src/store.rs | 14 ++++++++++++- tests/all/module_linking.rs | 36 +++++++++++++++++++++++++++++++++ 6 files changed, 75 insertions(+), 1 deletion(-) diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h index 6752459a4d..948552b633 100644 --- a/crates/c-api/include/wasmtime.h +++ b/crates/c-api/include/wasmtime.h @@ -274,6 +274,14 @@ WASMTIME_CONFIG_PROP(void, static_memory_guard_size, uint64_t) */ WASMTIME_CONFIG_PROP(void, dynamic_memory_guard_size, uint64_t) +/** + * \brief Configures the maximum number of instances that can be created. + * + * For more information see the Rust documentation at + * https://bytecodealliance.github.io/wasmtime/api/wasmtime/struct.Config.html#method.max_instances. + */ +WASMTIME_CONFIG_PROP(void, max_instances, size_t) + /** * \brief Enables Wasmtime's cache and loads configuration from the specified * path. diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index 08da598ec1..ae0cb6abdc 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -171,3 +171,8 @@ pub extern "C" fn wasmtime_config_static_memory_guard_size_set(c: &mut wasm_conf pub extern "C" fn wasmtime_config_dynamic_memory_guard_size_set(c: &mut wasm_config_t, size: u64) { c.config.dynamic_memory_guard_size(size); } + +#[no_mangle] +pub extern "C" fn wasmtime_config_max_instances(c: &mut wasm_config_t, limit: usize) { + c.config.max_instances(limit); +} diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 5a0686de91..15b5ad58cb 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -33,6 +33,7 @@ pub struct Config { pub(crate) max_wasm_stack: usize, pub(crate) features: WasmFeatures, pub(crate) wasm_backtrace_details_env_used: bool, + pub(crate) max_instances: usize, } impl Config { @@ -79,6 +80,7 @@ impl Config { multi_value: true, ..WasmFeatures::default() }, + max_instances: 10_000, }; ret.wasm_backtrace_details(WasmBacktraceDetails::Environment); return ret; @@ -635,6 +637,15 @@ impl Config { self } + /// Configures the maximum number of instances which can be created within + /// this `Store`. + /// + /// Instantiation will fail with an error if this limit is exceeded. + pub fn max_instances(&mut self, instances: usize) -> &mut Self { + self.max_instances = instances; + self + } + pub(crate) fn target_isa(&self) -> Box { self.isa_flags .clone() diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 60c52d6271..fed2bb29ce 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -46,6 +46,8 @@ fn instantiate( &mut ImportsBuilder<'_>, ) -> Result<()>, ) -> Result { + store.bump_instance_count()?; + let compiled_module = module.compiled_module(); let env_module = compiled_module.module(); diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 9c622faadf..be4aaf1563 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -4,7 +4,7 @@ use crate::trampoline::StoreInstanceHandle; use crate::{Engine, Module}; use anyhow::{bail, Result}; use std::any::Any; -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::collections::HashSet; use std::fmt; use std::hash::{Hash, Hasher}; @@ -67,6 +67,8 @@ pub(crate) struct StoreInner { /// Set of all compiled modules that we're holding a strong reference to /// the module's code for. This includes JIT functions, trampolines, etc. modules: RefCell>, + /// The number of instantiated instances in this store. + instance_count: Cell, } struct HostInfoKey(VMExternRef); @@ -109,6 +111,7 @@ impl Store { stack_map_registry: StackMapRegistry::default(), frame_info: Default::default(), modules: Default::default(), + instance_count: Default::default(), }), } } @@ -213,6 +216,15 @@ impl Store { } } + pub(crate) fn bump_instance_count(&self) -> Result<()> { + let n = self.inner.instance_count.get(); + self.inner.instance_count.set(n + 1); + if n >= self.engine().config().max_instances { + bail!("instance limit of {} exceeded", n); + } + Ok(()) + } + pub(crate) unsafe fn add_instance(&self, handle: InstanceHandle) -> StoreInstanceHandle { self.inner.instances.borrow_mut().push(handle.clone()); StoreInstanceHandle { diff --git a/tests/all/module_linking.rs b/tests/all/module_linking.rs index 10ccfc098d..b81a9deb33 100644 --- a/tests/all/module_linking.rs +++ b/tests/all/module_linking.rs @@ -185,3 +185,39 @@ fn imports_exports() -> Result<()> { } Ok(()) } + +#[test] +fn limit_instances() -> Result<()> { + let mut config = Config::new(); + config.wasm_module_linking(true); + config.max_instances(10); + let engine = Engine::new(&config); + let module = Module::new( + &engine, + r#" + (module $PARENT + (module $m0) + (module $m1 + (instance (instantiate (module outer $PARENT $m0))) + (instance (instantiate (module outer $PARENT $m0)))) + (module $m2 + (instance (instantiate (module outer $PARENT $m1))) + (instance (instantiate (module outer $PARENT $m1)))) + (module $m3 + (instance (instantiate (module outer $PARENT $m2))) + (instance (instantiate (module outer $PARENT $m2)))) + (module $m4 + (instance (instantiate (module outer $PARENT $m3))) + (instance (instantiate (module outer $PARENT $m3)))) + (module $m5 + (instance (instantiate (module outer $PARENT $m4))) + (instance (instantiate (module outer $PARENT $m4)))) + (instance (instantiate $m5)) + ) + "#, + )?; + let store = Store::new(&engine); + let err = Instance::new(&store, &module, &[]).err().unwrap(); + assert!(err.to_string().contains("instance limit of 10 exceeded")); + Ok(()) +} From 57c686d49d58cbc5af7dda14e8dce4470f138f20 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 21 Jan 2021 15:04:17 -0600 Subject: [PATCH 22/55] Refactor instantiation to be more async-friendly (#2596) Instantiation right now uses a recursive `instantiate` function since it was relatively easy to write that way, but this is unfortunately not factored in a way friendly to the async implementation in #2434. This commit refactors the function to instead use an iterative loop and refactors code in such a way that it should be easy to rebase #2434 on top of this change. The main goal is to make the body of `Instance::new` as small as possible since it needs to be duplicated with `Instance::new_async`. --- crates/wasmtime/src/instance.rs | 619 ++++++++++++++++++-------------- crates/wasmtime/src/module.rs | 4 + 2 files changed, 358 insertions(+), 265 deletions(-) diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index fed2bb29ce..02ad1a1b98 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -1,3 +1,4 @@ +use crate::trampoline::StoreInstanceHandle; use crate::types::matching; use crate::{ Engine, Export, Extern, Func, Global, InstanceType, Memory, Module, Store, Table, Trap, @@ -16,227 +17,6 @@ use wasmtime_runtime::{ VMTableImport, }; -/// Performs all low-level steps necessary for instantiation. -/// -/// This function will take all the arguments and attempt to do everything -/// necessary to instantiate the referenced instance. The trickiness of this -/// function stems from the implementation of the module-linking proposal where -/// we're handling nested instances, interleaved imports/aliases, etc. That's -/// all an internal implementation here ideally though! -/// -/// * `store` - the store we're instantiating into -/// * `compiled_module` - the module that we're instantiating -/// * `all_modules` - the list of all modules that were part of the compilation -/// of `compiled_module`. This is only applicable in the module linking -/// proposal, otherwise this will just be a list containing `compiled_module` -/// itself. -/// * `type` - the type tables produced during compilation which -/// `compiled_module`'s metadata references. -/// * `define_import` - this function, like the name implies, defines an import -/// into the provided builder. The expected entity that it's defining is also -/// passed in for the top-level case where type-checking is performed. This is -/// fallible because type checks may fail. -fn instantiate( - store: &Store, - module: &Module, - define_import: &mut dyn FnMut( - &str, - Option<&str>, - &EntityIndex, - &mut ImportsBuilder<'_>, - ) -> Result<()>, -) -> Result { - store.bump_instance_count()?; - - let compiled_module = module.compiled_module(); - let env_module = compiled_module.module(); - - let mut imports = ImportsBuilder::new(store, module); - for initializer in env_module.initializers.iter() { - match initializer { - // Definition of an import depends on how our parent is providing - // imports, so we delegate to our custom closure. This will resolve - // to fetching from the import list for the top-level module and - // otherwise fetching from each nested instance's argument list for - // submodules. - Initializer::Import { index, name, field } => { - define_import(name, field.as_deref(), index, &mut imports) - .with_context(|| format!("incompatible import type for `{}`", name))?; - } - - // Here we lookup our instance handle, find the right export, - // and then push that item into our own index space. We eschew - // type-checking since only valid modules should reach this point. - Initializer::AliasInstanceExport { instance, export } => { - let export = &imports.instances[*instance][export]; - let item = unsafe { Extern::from_wasmtime_export(export, store) }; - imports.push_extern(&item); - } - - // Oh boy a recursive instantiation! - // - // We use our local index space of modules to find the module to - // instantiate and argument lookup is defined as looking inside of - // `args`. Like above with aliases all type checking is eschewed - // because only valid modules should reach this point. - // - // Note that it's thought that due to the acyclic nature of - // instantiation this can't loop to blow the native stack, and - // validation should in theory ensure this has a bounded depth. - // Despite this we may need to change this to a loop instead of - // recursion one day. - Initializer::Instantiate { module, args } => { - let handle = instantiate( - store, - &imports.modules[*module], - &mut |name, field, _, builder| { - debug_assert!(field.is_none()); - let index = args.get(name).expect("should be present after validation"); - match *index { - EntityIndex::Global(i) => { - builder.globals.push(imports.globals[i]); - } - EntityIndex::Function(i) => { - builder.functions.push(imports.functions[i]); - } - EntityIndex::Table(i) => { - builder.tables.push(imports.tables[i]); - } - EntityIndex::Memory(i) => { - builder.memories.push(imports.memories[i]); - } - EntityIndex::Module(i) => { - builder.modules.push(imports.modules[i].clone()); - } - EntityIndex::Instance(i) => { - builder.instances.push(imports.instances[i].clone()); - } - } - Ok(()) - }, - )?; - imports.instances.push(handle); - } - - // A new module is being defined, and the source of this module is - // our module's list of closed-over-modules. - // - // This is used for outer aliases. - Initializer::DefineModule(upvar_index) => { - imports - .modules - .push(module.module_upvar(*upvar_index).clone()); - } - - // A new module is defined, created from a set of compiled - // artifacts. The new module value will be created with the - // specified artifacts being closed over as well as the specified - // set of module values in our index/upvar index spaces being closed - // over. - // - // This is used for defining submodules. - Initializer::CreateModule { - artifact_index, - artifacts, - modules, - } => { - let submodule = - module.create_submodule(*artifact_index, artifacts, modules, &imports.modules); - imports.modules.push(submodule); - } - } - } - - // Register the module just before instantiation to ensure we have a - // trampoline registered for every signature and to preserve the module's - // compiled JIT code within the `Store`. - store.register_module(module); - - let config = store.engine().config(); - let instance = unsafe { - let instance = compiled_module.instantiate( - imports.build(), - &store.lookup_shared_signature(module.types()), - config.memory_creator.as_ref().map(|a| a as _), - store.interrupts(), - Box::new(()), - store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - store.stack_map_registry() as *const StackMapRegistry as *mut _, - )?; - - // After we've created the `InstanceHandle` we still need to run - // initialization to set up data/elements/etc. We do this after adding - // the `InstanceHandle` to the store though. This is required for safety - // because the start function (for example) may trap, but element - // initializers may have run which placed elements into other instance's - // tables. This means that from this point on, regardless of whether - // initialization is successful, we need to keep the instance alive. - let instance = store.add_instance(instance); - instance - .initialize( - config.features.bulk_memory, - &compiled_module.data_initializers(), - ) - .map_err(|e| -> Error { - match e { - InstantiationError::Trap(trap) => Trap::from_runtime(store, trap).into(), - other => other.into(), - } - })?; - - instance - }; - - let start_func = instance.handle.module().start_func; - - // If a start function is present, invoke it. Make sure we use all the - // trap-handling configuration in `store` as well. - if let Some(start) = start_func { - let f = match instance - .handle - .lookup_by_declaration(&EntityIndex::Function(start)) - { - wasmtime_runtime::Export::Function(f) => f, - _ => unreachable!(), // valid modules shouldn't hit this - }; - let vmctx_ptr = instance.handle.vmctx_ptr(); - unsafe { - super::func::invoke_wasm_and_catch_traps(vmctx_ptr, store, || { - mem::transmute::< - *const VMFunctionBody, - unsafe extern "C" fn(*mut VMContext, *mut VMContext), - >(f.anyfunc.as_ref().func_ptr.as_ptr())( - f.anyfunc.as_ref().vmctx, vmctx_ptr - ) - })?; - } - } - - let exports = instance - .handle - .module() - .exports - .iter() - .map(|(name, index)| { - // Note that instances and modules are not handled by - // `wasmtime_runtime`, they're handled by us in this crate. That - // means we need to handle that here, otherwise we defer to the - // instance to load the values. - let item = match index { - EntityIndex::Instance(i) => { - wasmtime_runtime::Export::Instance(imports.instances[*i].clone()) - } - EntityIndex::Module(i) => { - wasmtime_runtime::Export::Module(Box::new(imports.modules[*i].clone())) - } - index => instance.handle.lookup_by_declaration(index), - }; - (name.clone(), item) - }) - .collect(); - Ok(Rc::new(exports)) -} - /// An instantiated WebAssembly module. /// /// This type represents the instantiation of a [`Module`]. Once instantiated @@ -314,29 +94,15 @@ impl Instance { /// [issue]: https://github.com/bytecodealliance/wasmtime/issues/727 /// [`ExternType`]: crate::ExternType pub fn new(store: &Store, module: &Module, imports: &[Extern]) -> Result { - if !Engine::same(store.engine(), module.engine()) { - bail!("cross-`Engine` instantiation is not currently supported"); - } - - // Perform some pre-flight checks before we get into the meat of - // instantiation. - let expected = module.compiled_module().module().imports().count(); - if expected != imports.len() { - bail!("expected {} imports, found {}", expected, imports.len()); - } - for import in imports { - if !import.comes_from_same_store(store) { - bail!("cross-`Store` instantiation is not currently supported"); + let mut i = Instantiator::new(store, module, imports)?; + loop { + if let Some((instance, items)) = i.step()? { + Instantiator::start_raw(&instance)?; + if let Some(items) = items { + break Ok(Instance::from_wasmtime(&items, store)); + } } } - - let mut imports = imports.iter(); - let items = instantiate(store, module, &mut |_name, _field, idx, builder| { - let import = imports.next().expect("already checked the length"); - builder.define_extern(idx, &import) - })?; - - Ok(Instance::from_wasmtime(&items, store)) } pub(crate) fn from_wasmtime(handle: &RuntimeInstance, store: &Store) -> Instance { @@ -417,42 +183,366 @@ impl Instance { } } +struct Instantiator<'a> { + in_progress: Vec>, + cur: ImportsBuilder<'a>, + store: &'a Store, +} + struct ImportsBuilder<'a> { + src: ImportSource<'a>, functions: PrimaryMap, tables: PrimaryMap, memories: PrimaryMap, globals: PrimaryMap, instances: PrimaryMap, modules: PrimaryMap, - - module: &'a wasmtime_environ::Module, - matcher: matching::MatchCx<'a>, + initializer: usize, + module: Module, } -impl<'a> ImportsBuilder<'a> { - fn new(store: &'a Store, module: &'a Module) -> ImportsBuilder<'a> { - let types = module.types(); - let module = module.compiled_module().module(); - ImportsBuilder { - module, - matcher: matching::MatchCx { store, types }, - functions: PrimaryMap::with_capacity(module.num_imported_funcs), - tables: PrimaryMap::with_capacity(module.num_imported_tables), - memories: PrimaryMap::with_capacity(module.num_imported_memories), - globals: PrimaryMap::with_capacity(module.num_imported_globals), - instances: PrimaryMap::with_capacity(module.instances.len()), - modules: PrimaryMap::with_capacity(module.modules.len()), +enum ImportSource<'a> { + Runtime(&'a [Extern]), + Outer { initializer: usize }, +} + +impl<'a> Instantiator<'a> { + /// Creates a new instantiation context used to process all the initializer + /// directives of a module. + /// + /// This doesn't do much work itself beyond setting things up. + fn new(store: &'a Store, module: &Module, imports: &'a [Extern]) -> Result> { + if !Engine::same(store.engine(), module.engine()) { + bail!("cross-`Engine` instantiation is not currently supported"); + } + + // Perform some pre-flight checks before we get into the meat of + // instantiation. + let expected = module.compiled_module().module().imports().count(); + if expected != imports.len() { + bail!("expected {} imports, found {}", expected, imports.len()); + } + for import in imports { + if !import.comes_from_same_store(store) { + bail!("cross-`Store` instantiation is not currently supported"); + } + } + + Ok(Instantiator { + in_progress: Vec::new(), + cur: ImportsBuilder::new(module, ImportSource::Runtime(imports)), + store, + }) + } + + /// Processes the next initializer for the next instance being created + /// without running any wasm code. + /// + /// This function will process module initializers, handling recursive + /// instantiations of modules for module linking if necessary as well. This + /// does not actually execute any WebAssembly code, which means that it + /// will return whenever an instance is created (because its `start` + /// function may need to be executed). + /// + /// If this function returns `None`, then it simply needs to be called + /// again to execute the next initializer. Otherwise this function has two + /// return values: + /// + /// * The first is the raw handle to the instance that was just created. + /// This instance must have its start function executed by the caller. + /// * The second is an optional list of items to get wrapped up in an + /// `Instance`. This is only `Some` for the outermost instance that was + /// created. If this is `None` callers need to keep calling this function + /// since the instance created was simply for a recursive instance + /// defined here. + fn step(&mut self) -> Result)>> { + if self.cur.initializer == 0 { + self.store.bump_instance_count()?; + } + + // Read the current module's initializer and move forward the + // initializer pointer as well. + self.cur.initializer += 1; + match self + .cur + .module + .env_module() + .initializers + .get(self.cur.initializer - 1) + { + Some(Initializer::Import { index, name, field }) => { + match &mut self.cur.src { + // If imports are coming from the runtime-provided list + // (e.g. the root module being instantiated) then we + // need to typecheck each item here before recording it. + // + // Note the `unwrap` here should be ok given the validation + // above in `Instantiation::new`. + ImportSource::Runtime(list) => { + let (head, remaining) = list.split_first().unwrap(); + *list = remaining; + let expected_ty = + self.cur.module.compiled_module().module().type_of(*index); + matching::MatchCx { + types: self.cur.module.types(), + store: self.store, + } + .extern_(&expected_ty, head) + .with_context(|| { + let extra = match field { + Some(name) => format!("::{}", name), + None => String::new(), + }; + format!("incompatible import type for `{}{}`", name, extra) + })?; + self.cur.push(head); + } + + // Otherwise if arguments are coming from our outer + // instance due to a recursive instantiation then we + // look in the previous initializer's mapping of + // arguments to figure out where to load the item from. + // Note that no typechecking is necessary here due to + // validation. + ImportSource::Outer { initializer } => { + debug_assert!(field.is_none()); + let outer = self.in_progress.last().unwrap(); + let args = match &outer.module.env_module().initializers[*initializer] { + Initializer::Instantiate { args, .. } => args, + _ => unreachable!(), + }; + let index = args.get(name).expect("should be present after validation"); + match *index { + EntityIndex::Global(i) => { + self.cur.globals.push(outer.globals[i]); + } + EntityIndex::Function(i) => { + self.cur.functions.push(outer.functions[i]); + } + EntityIndex::Table(i) => { + self.cur.tables.push(outer.tables[i]); + } + EntityIndex::Memory(i) => { + self.cur.memories.push(outer.memories[i]); + } + EntityIndex::Module(i) => { + self.cur.modules.push(outer.modules[i].clone()); + } + EntityIndex::Instance(i) => { + self.cur.instances.push(outer.instances[i].clone()); + } + } + } + } + } + + // Here we lookup our instance handle, find the right export, + // and then push that item into our own index space. We eschew + // type-checking since only valid modules should reach this point. + Some(Initializer::AliasInstanceExport { instance, export }) => { + let export = &self.cur.instances[*instance][export]; + let item = unsafe { Extern::from_wasmtime_export(export, self.store) }; + self.cur.push(&item); + } + + // A recursive instantiation of an instance. + // + // The `module` argument is used to create an import builder + // object, and we specify that the source of imports for the builder is + // this initializer's position so we can look at the `args` payload + // later. + // + // Once that's set up we save off `self.cur` into + // `self.in_progress` and start the instantiation of the child + // instance on the next execution of this function. + Some(Initializer::Instantiate { module, args: _ }) => { + let module = &self.cur.modules[*module]; + let imports = ImportsBuilder::new( + module, + ImportSource::Outer { + initializer: self.cur.initializer - 1, + }, + ); + let prev = mem::replace(&mut self.cur, imports); + self.in_progress.push(prev); + } + + // A new module is being defined, and the source of this module is + // our module's list of closed-over-modules. + // + // This is used for outer aliases. + Some(Initializer::DefineModule(upvar_index)) => { + self.cur + .modules + .push(self.cur.module.module_upvar(*upvar_index).clone()); + } + + // A new module is defined, created from a set of compiled + // artifacts. The new module value will be created with the + // specified artifacts being closed over as well as the specified + // set of module values in our index/upvar index spaces being closed + // over. + // + // This is used for defining submodules. + Some(Initializer::CreateModule { + artifact_index, + artifacts, + modules, + }) => { + let submodule = self.cur.module.create_submodule( + *artifact_index, + artifacts, + modules, + &self.cur.modules, + ); + self.cur.modules.push(submodule); + } + + // All initializers have been processed, which means we're ready to + // perform the actual raw instantiation with the raw import values. + // Once that's done if there's an in-progress module we record the + // instance in the index space. Otherwise this is the final module + // and we return the items out. + // + // Note that in all cases we return the raw instance handle to get + // the start function executed by the outer context. + None => { + let instance = self.instantiate_raw()?; + let items = self.runtime_instance(&instance); + let items = match self.in_progress.pop() { + Some(imports) => { + self.cur = imports; + self.cur.instances.push(items); + None + } + None => Some(items), + }; + return Ok(Some((instance, items))); + } + } + + Ok(None) + } + + fn instantiate_raw(&self) -> Result { + let compiled_module = self.cur.module.compiled_module(); + + // Register the module just before instantiation to ensure we have a + // trampoline registered for every signature and to preserve the module's + // compiled JIT code within the `Store`. + self.store.register_module(&self.cur.module); + + let config = self.store.engine().config(); + unsafe { + let instance = compiled_module.instantiate( + self.cur.build(), + &self.store.lookup_shared_signature(self.cur.module.types()), + config.memory_creator.as_ref().map(|a| a as _), + self.store.interrupts(), + Box::new(()), + self.store.externref_activations_table() as *const VMExternRefActivationsTable + as *mut _, + self.store.stack_map_registry() as *const StackMapRegistry as *mut _, + )?; + + // After we've created the `InstanceHandle` we still need to run + // initialization to set up data/elements/etc. We do this after adding + // the `InstanceHandle` to the store though. This is required for safety + // because the start function (for example) may trap, but element + // initializers may have run which placed elements into other instance's + // tables. This means that from this point on, regardless of whether + // initialization is successful, we need to keep the instance alive. + let instance = self.store.add_instance(instance); + instance + .initialize( + config.features.bulk_memory, + &compiled_module.data_initializers(), + ) + .map_err(|e| -> Error { + match e { + InstantiationError::Trap(trap) => { + Trap::from_runtime(self.store, trap).into() + } + other => other.into(), + } + })?; + + Ok(instance) } } - fn define_extern(&mut self, expected: &EntityIndex, actual: &Extern) -> Result<()> { - let expected_ty = self.module.type_of(*expected); - self.matcher.extern_(&expected_ty, actual)?; - self.push_extern(actual); + fn start_raw(instance: &StoreInstanceHandle) -> Result<()> { + let start_func = instance.handle.module().start_func; + + // If a start function is present, invoke it. Make sure we use all the + // trap-handling configuration in `store` as well. + if let Some(start) = start_func { + let f = match instance + .handle + .lookup_by_declaration(&EntityIndex::Function(start)) + { + wasmtime_runtime::Export::Function(f) => f, + _ => unreachable!(), // valid modules shouldn't hit this + }; + let vmctx_ptr = instance.handle.vmctx_ptr(); + unsafe { + super::func::invoke_wasm_and_catch_traps(vmctx_ptr, &instance.store, || { + mem::transmute::< + *const VMFunctionBody, + unsafe extern "C" fn(*mut VMContext, *mut VMContext), + >(f.anyfunc.as_ref().func_ptr.as_ptr())( + f.anyfunc.as_ref().vmctx, vmctx_ptr + ) + })?; + } + } Ok(()) } - fn push_extern(&mut self, item: &Extern) { + fn runtime_instance(&self, instance: &StoreInstanceHandle) -> RuntimeInstance { + let exports = instance + .handle + .module() + .exports + .iter() + .map(|(name, index)| { + // Note that instances and modules are not handled by + // `wasmtime_runtime`, they're handled by us in this crate. That + // means we need to handle that here, otherwise we defer to the + // instance to load the values. + let item = match index { + EntityIndex::Instance(i) => { + wasmtime_runtime::Export::Instance(self.cur.instances[*i].clone()) + } + EntityIndex::Module(i) => { + wasmtime_runtime::Export::Module(Box::new(self.cur.modules[*i].clone())) + } + index => instance.handle.lookup_by_declaration(index), + }; + (name.clone(), item) + }) + .collect(); + Rc::new(exports) + } +} + +impl<'a> ImportsBuilder<'a> { + fn new(module: &Module, src: ImportSource<'a>) -> ImportsBuilder<'a> { + let raw = module.compiled_module().module(); + ImportsBuilder { + src, + functions: PrimaryMap::with_capacity(raw.num_imported_funcs), + tables: PrimaryMap::with_capacity(raw.num_imported_tables), + memories: PrimaryMap::with_capacity(raw.num_imported_memories), + globals: PrimaryMap::with_capacity(raw.num_imported_globals), + instances: PrimaryMap::with_capacity(raw.instances.len()), + modules: PrimaryMap::with_capacity(raw.modules.len()), + module: module.clone(), + initializer: 0, + } + } + + fn push(&mut self, item: &Extern) { match item { Extern::Func(i) => { self.functions.push(i.vmimport()); @@ -467,7 +557,6 @@ impl<'a> ImportsBuilder<'a> { self.memories.push(i.vmimport()); } Extern::Instance(i) => { - debug_assert!(Store::same(i.store(), self.matcher.store)); self.instances.push(i.items.clone()); } Extern::Module(m) => { @@ -476,7 +565,7 @@ impl<'a> ImportsBuilder<'a> { } } - fn build(&mut self) -> Imports<'_> { + fn build(&self) -> Imports<'_> { Imports { tables: self.tables.values().as_slice(), globals: self.globals.values().as_slice(), diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 8fcd94a3bd..0bcf8f5686 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -527,6 +527,10 @@ impl Module { &self.inner.module } + pub(crate) fn env_module(&self) -> &wasmtime_environ::Module { + self.compiled_module().module() + } + pub(crate) fn types(&self) -> &Arc { &self.inner.types } From 4a351ab7fe71b6a4379ea9fdc2a7e4c8382e9528 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 21 Jan 2021 15:49:13 -0600 Subject: [PATCH 23/55] Update a number of dependencies (#2594) This commit goes through the dependencies that wasmtime has and updates versions where possible. This notably brings in a wasmparser/wast update which has some simd spec changes with new instructions. Otherwise most of these are just routine updates. --- Cargo.lock | 208 ++++++++++-------- Cargo.toml | 6 +- build.rs | 7 +- cranelift/codegen/Cargo.toml | 2 +- cranelift/object/Cargo.toml | 2 +- cranelift/peepmatic/Cargo.toml | 2 +- cranelift/peepmatic/crates/fuzzing/Cargo.toml | 4 +- cranelift/peepmatic/crates/runtime/Cargo.toml | 2 +- cranelift/peepmatic/crates/souper/Cargo.toml | 2 +- .../peepmatic/crates/test-operator/Cargo.toml | 2 +- cranelift/wasm/Cargo.toml | 4 +- cranelift/wasm/src/code_translator.rs | 21 +- crates/debug/Cargo.toml | 4 +- crates/environ/Cargo.toml | 2 +- crates/fuzzing/Cargo.toml | 4 +- crates/jit/Cargo.toml | 4 +- crates/lightbeam/Cargo.toml | 6 +- crates/lightbeam/wasmtime/Cargo.toml | 2 +- crates/obj/Cargo.toml | 2 +- crates/profiling/Cargo.toml | 2 +- crates/wasmtime/Cargo.toml | 2 +- crates/wast/Cargo.toml | 2 +- deny.toml | 4 + tests/misc_testsuite/multi-memory/simple.wast | 24 +- 24 files changed, 176 insertions(+), 144 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 612d9d3c20..98c5577566 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "addr2line" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" dependencies = [ "gimli", ] @@ -94,9 +94,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.55" +version = "0.3.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" +checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc" dependencies = [ "addr2line", "cfg-if 1.0.0", @@ -388,7 +388,7 @@ dependencies = [ "souper-ir", "target-lexicon", "thiserror", - "wast 31.0.0", + "wast 32.0.0", ] [[package]] @@ -585,7 +585,7 @@ dependencies = [ "cranelift-entity", "cranelift-frontend", "hashbrown", - "itertools", + "itertools 0.10.0", "log", "serde", "smallvec", @@ -890,13 +890,13 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" +checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.2.4", "winapi", ] @@ -918,19 +918,6 @@ version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" -[[package]] -name = "generator" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc" -dependencies = [ - "cc", - "libc", - "log", - "rustc_version", - "winapi", -] - [[package]] name = "generic-array" version = "0.14.4" @@ -1071,6 +1058,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -1151,7 +1147,7 @@ dependencies = [ "dynasm", "dynasmrt", "iter-enum", - "itertools", + "itertools 0.10.0", "lazy_static", "memoffset", "more-asserts", @@ -1172,19 +1168,6 @@ dependencies = [ "cfg-if 0.1.10", ] -[[package]] -name = "loom" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e8460f2f2121162705187214720353c517b97bdfb3494c0b1e33d83ebe4bed" -dependencies = [ - "cfg-if 0.1.10", - "generator", - "scoped-tls", - "serde", - "serde_json", -] - [[package]] name = "mach" version = "0.3.2" @@ -1320,9 +1303,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" [[package]] name = "object" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" +checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" dependencies = [ "crc32fast", "indexmap", @@ -1393,7 +1376,7 @@ dependencies = [ "peepmatic-test-operator", "peepmatic-traits", "serde", - "wast 31.0.0", + "wast 32.0.0", "z3", ] @@ -1419,9 +1402,9 @@ dependencies = [ "peepmatic-test", "peepmatic-test-operator", "peepmatic-traits", - "rand", + "rand 0.8.2", "serde", - "wast 31.0.0", + "wast 32.0.0", ] [[package]] @@ -1446,7 +1429,7 @@ dependencies = [ "serde", "serde_test", "thiserror", - "wast 31.0.0", + "wast 32.0.0", ] [[package]] @@ -1458,7 +1441,7 @@ dependencies = [ "peepmatic", "peepmatic-test-operator", "souper-ir", - "wast 31.0.0", + "wast 32.0.0", ] [[package]] @@ -1479,7 +1462,7 @@ version = "0.69.0" dependencies = [ "peepmatic-traits", "serde", - "wast 31.0.0", + "wast 32.0.0", ] [[package]] @@ -1553,8 +1536,8 @@ dependencies = [ "lazy_static", "num-traits", "quick-error", - "rand", - "rand_chacha", + "rand 0.7.3", + "rand_chacha 0.2.2", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -1578,21 +1561,20 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quickcheck" -version = "0.9.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" +checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ - "env_logger 0.7.1", + "env_logger 0.8.2", "log", - "rand", - "rand_core", + "rand 0.8.2", ] [[package]] name = "quote" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" dependencies = [ "proc-macro2", ] @@ -1605,10 +1587,21 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom 0.1.15", "libc", - "rand_chacha", - "rand_core", - "rand_hc", - "rand_pcg", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18519b42a40024d661e1714153e9ad0c3de27cd495760ceb09710920f1098b1e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.1", + "rand_hc 0.3.0", ] [[package]] @@ -1618,7 +1611,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.1", ] [[package]] @@ -1630,22 +1633,31 @@ dependencies = [ "getrandom 0.1.15", ] +[[package]] +name = "rand_core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" +dependencies = [ + "getrandom 0.2.0", +] + [[package]] name = "rand_hc" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core", + "rand_core 0.5.1", ] [[package]] -name = "rand_pcg" -version = "0.2.1" +name = "rand_hc" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" dependencies = [ - "rand_core", + "rand_core 0.6.1", ] [[package]] @@ -1654,7 +1666,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" dependencies = [ - "rand_core", + "rand_core 0.5.1", ] [[package]] @@ -1699,6 +1711,15 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "redox_syscall" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.3.5" @@ -1706,7 +1727,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" dependencies = [ "getrandom 0.1.15", - "redox_syscall", + "redox_syscall 0.1.57", "rust-argon2", ] @@ -1839,12 +1860,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "scoped-tls" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" - [[package]] name = "scopeguard" version = "1.1.0" @@ -1888,18 +1903,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.118" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +checksum = "166b2349061381baf54a58e4b13c89369feb0ef2eaa57198899e2312aac30aab" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.118" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +checksum = "0ca2a8cb5805ce9e3b95435e3765b7b553cecc762d938d409434338386cb5775" dependencies = [ "proc-macro2", "quote", @@ -1941,12 +1956,11 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127" +checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3" dependencies = [ "lazy_static", - "loom", ] [[package]] @@ -1972,7 +1986,7 @@ checksum = "848c0a454373d16ebfaa740c99d4faebe25ea752a35e0c6e341168fe67f987f8" dependencies = [ "cfg-if 1.0.0", "libc", - "rand", + "rand 0.7.3", "winapi", ] @@ -2029,9 +2043,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.54" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44" +checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" dependencies = [ "proc-macro2", "quote", @@ -2046,14 +2060,14 @@ checksum = "4ee5a98e506fb7231a304c3a1bd7c132a55016cf65001e0282480665870dfcb9" [[package]] name = "tempfile" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "rand", - "redox_syscall", + "rand 0.8.2", + "redox_syscall 0.2.4", "remove_dir_all", "winapi", ] @@ -2395,15 +2409,15 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.72.0" +version = "0.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cdf4d872d407f9fb44956e540582eeaf0dc4fb8142f1f0f64e2c37196bada01" +checksum = "a15011eb23c404cdc1f6f60124a15921a11b060cca195486e36f4dc62e83c61d" [[package]] name = "wasmprinter" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c139586e3b80b899f5aaaa3720c7a236ea9073e315292e01d099da12efacc5" +checksum = "c467ab13e60cc347a17ed2c60bf761501eb45314d8916cad0d53ccc31078b20d" dependencies = [ "anyhow", "wasmparser", @@ -2755,7 +2769,7 @@ version = "0.22.0" dependencies = [ "anyhow", "wasmtime", - "wast 31.0.0", + "wast 32.0.0", ] [[package]] @@ -2791,20 +2805,20 @@ dependencies = [ [[package]] name = "wast" -version = "31.0.0" +version = "32.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9beb1f6b63f08c523a1e8e76fc70058af4d2a34ef1c504f56cdac7b6970228b9" +checksum = "c24a3ee360d01d60ed0a0f960ab76a6acce64348cdb0bf8699c2a866fad57c7c" dependencies = [ "leb128", ] [[package]] name = "wat" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b3044da73d3b84a822d955afad356759b2fee454b6882722008dace80b68e" +checksum = "5e8f7f34773fa6318e8897283abf7941c1f250faae4e1a52f82df09c3bad7cce" dependencies = [ - "wast 31.0.0", + "wast 32.0.0", ] [[package]] @@ -2984,6 +2998,6 @@ checksum = "ec24a9273d24437afb8e71b16f3d9a5d569193cccdb7896213b59f552f387674" dependencies = [ "cc", "glob", - "itertools", + "itertools 0.9.0", "libc", ] diff --git a/Cargo.toml b/Cargo.toml index 640189732d..3221971b18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,17 +33,17 @@ wasmtime-wasi = { path = "crates/wasi", version = "0.22.0" } wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "0.22.0", optional = true } wasi-common = { path = "crates/wasi-common", version = "0.22.0" } structopt = { version = "0.3.5", features = ["color", "suggestions"] } -object = { version = "0.22.0", default-features = false, features = ["write"] } +object = { version = "0.23.0", default-features = false, features = ["write"] } anyhow = "1.0.19" target-lexicon = { version = "0.11.0", default-features = false } pretty_env_logger = "0.4.0" file-per-thread-logger = "0.1.1" -wat = "1.0.32" +wat = "1.0.33" libc = "0.2.60" log = "0.4.8" rayon = "1.2.1" humantime = "2.0.0" -wasmparser = "0.72.0" +wasmparser = "0.73.0" [dev-dependencies] env_logger = "0.8.1" diff --git a/build.rs b/build.rs index fe3bf4456b..9827fd2b0b 100644 --- a/build.rs +++ b/build.rs @@ -243,9 +243,12 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { return env::var("CARGO_CFG_TARGET_ARCH").unwrap() != "x86_64"; } + // Waiting for an update to the spec testsuite to not use old + // instruction names. + ("simd", "simd_boolean") | ("simd", "simd_lane") => return true, + // These are only implemented on aarch64 and x64. - ("simd", "simd_boolean") - | ("simd", "simd_f32x4_pmin_pmax") + ("simd", "simd_f32x4_pmin_pmax") | ("simd", "simd_f64x2_pmin_pmax") | ("simd", "simd_f32x4_rounding") | ("simd", "simd_f64x2_rounding") diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index f601965022..15b592772a 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -30,7 +30,7 @@ peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, versi peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.69.0" } regalloc = { version = "0.0.31" } souper-ir = { version = "2.1.0", optional = true } -wast = { version = "31.0.0", optional = true } +wast = { version = "32.0.0", optional = true } # It is a goal of the cranelift-codegen crate to have minimal external dependencies. # Please don't add any unless they are essential to the task of creating binary # machine code. Integration tests that need external dependencies can be diff --git a/cranelift/object/Cargo.toml b/cranelift/object/Cargo.toml index 95c5b07447..bb27e3ee6c 100644 --- a/cranelift/object/Cargo.toml +++ b/cranelift/object/Cargo.toml @@ -12,7 +12,7 @@ edition = "2018" [dependencies] cranelift-module = { path = "../module", version = "0.69.0" } cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false, features = ["std"] } -object = { version = "0.22.0", default-features = false, features = ["write"] } +object = { version = "0.23.0", default-features = false, features = ["write"] } target-lexicon = "0.11" anyhow = "1.0" log = { version = "0.4.6", default-features = false } diff --git a/cranelift/peepmatic/Cargo.toml b/cranelift/peepmatic/Cargo.toml index dcb1da1537..9bd6540673 100644 --- a/cranelift/peepmatic/Cargo.toml +++ b/cranelift/peepmatic/Cargo.toml @@ -15,7 +15,7 @@ peepmatic-macro = { version = "0.69.0", path = "crates/macro" } peepmatic-runtime = { version = "0.69.0", path = "crates/runtime", features = ["construct"] } peepmatic-traits = { version = "0.69.0", path = "crates/traits" } serde = { version = "1.0.105", features = ["derive"] } -wast = "31.0.0" +wast = "32.0.0" z3 = { version = "0.7.1", features = ["static-link-z3"] } [dev-dependencies] diff --git a/cranelift/peepmatic/crates/fuzzing/Cargo.toml b/cranelift/peepmatic/crates/fuzzing/Cargo.toml index fda6532563..b84b8aef3c 100644 --- a/cranelift/peepmatic/crates/fuzzing/Cargo.toml +++ b/cranelift/peepmatic/crates/fuzzing/Cargo.toml @@ -19,6 +19,6 @@ peepmatic-runtime = { path = "../runtime", features = ["construct"] } peepmatic-test = { path = "../test" } peepmatic-test-operator = { path = "../test-operator" } peepmatic-traits = { path = "../traits" } -rand = { version = "0.7.3", features = ["small_rng"] } +rand = { version = "0.8.2", features = ["small_rng"] } serde = "1.0.106" -wast = "31.0.0" +wast = "32.0.0" diff --git a/cranelift/peepmatic/crates/runtime/Cargo.toml b/cranelift/peepmatic/crates/runtime/Cargo.toml index ea1909c282..729c3678c3 100644 --- a/cranelift/peepmatic/crates/runtime/Cargo.toml +++ b/cranelift/peepmatic/crates/runtime/Cargo.toml @@ -16,7 +16,7 @@ peepmatic-automata = { version = "0.69.0", path = "../automata", features = ["se peepmatic-traits = { version = "0.69.0", path = "../traits" } serde = { version = "1.0.105", features = ["derive"] } thiserror = "1.0.15" -wast = { version = "31.0.0", optional = true } +wast = { version = "32.0.0", optional = true } [dev-dependencies] peepmatic-test-operator = { version = "0.69.0", path = "../test-operator" } diff --git a/cranelift/peepmatic/crates/souper/Cargo.toml b/cranelift/peepmatic/crates/souper/Cargo.toml index 7b512526d6..1bec224d06 100644 --- a/cranelift/peepmatic/crates/souper/Cargo.toml +++ b/cranelift/peepmatic/crates/souper/Cargo.toml @@ -16,4 +16,4 @@ log = "0.4.8" [dev-dependencies] peepmatic = { path = "../..", version = "0.69.0" } peepmatic-test-operator = { version = "0.69.0", path = "../test-operator" } -wast = "31.0.0" +wast = "32.0.0" diff --git a/cranelift/peepmatic/crates/test-operator/Cargo.toml b/cranelift/peepmatic/crates/test-operator/Cargo.toml index de23fe59fd..6574827a44 100644 --- a/cranelift/peepmatic/crates/test-operator/Cargo.toml +++ b/cranelift/peepmatic/crates/test-operator/Cargo.toml @@ -11,4 +11,4 @@ edition = "2018" [dependencies] peepmatic-traits = { version = "0.69.0", path = "../traits" } serde = { version = "1.0.105", features = ["derive"] } -wast = "31.0.0" +wast = "32.0.0" diff --git a/cranelift/wasm/Cargo.toml b/cranelift/wasm/Cargo.toml index ac7866af70..b5549453f8 100644 --- a/cranelift/wasm/Cargo.toml +++ b/cranelift/wasm/Cargo.toml @@ -12,12 +12,12 @@ keywords = ["webassembly", "wasm"] edition = "2018" [dependencies] -wasmparser = { version = "0.72", default-features = false } +wasmparser = { version = "0.73", default-features = false } cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false } cranelift-entity = { path = "../entity", version = "0.69.0" } cranelift-frontend = { path = "../frontend", version = "0.69.0", default-features = false } hashbrown = { version = "0.9.1", optional = true } -itertools = "0.9.0" +itertools = "0.10.0" log = { version = "0.4.6", default-features = false } serde = { version = "1.0.94", features = ["derive"], optional = true } smallvec = "1.6.1" diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs index bb22a333ca..266e002ae4 100644 --- a/cranelift/wasm/src/code_translator.rs +++ b/cranelift/wasm/src/code_translator.rs @@ -1626,7 +1626,7 @@ pub fn translate_operator( // operands must match (hence the bitcast). state.push1(builder.ins().bitselect(bitcast_c, bitcast_a, bitcast_b)) } - Operator::I8x16AnyTrue | Operator::I16x8AnyTrue | Operator::I32x4AnyTrue => { + Operator::V128AnyTrue => { let a = pop1_with_bitcast(state, type_of(op), builder); let bool_result = builder.ins().vany_true(a); state.push1(builder.ins().bint(I32, bool_result)) @@ -1824,7 +1824,21 @@ pub fn translate_operator( let (a, b) = pop2_with_bitcast(state, I16X8, builder); state.push1(builder.ins().widening_pairwise_dot_product_s(a, b)); } - Operator::I16x8ExtMulLowI8x16S + Operator::I64x2Bitmask + | Operator::I64x2WidenLowI32x4S + | Operator::I64x2WidenHighI32x4S + | Operator::I64x2WidenLowI32x4U + | Operator::I64x2WidenHighI32x4U + | Operator::V128Load8Lane { .. } + | Operator::V128Load16Lane { .. } + | Operator::V128Load32Lane { .. } + | Operator::V128Load64Lane { .. } + | Operator::V128Store8Lane { .. } + | Operator::V128Store16Lane { .. } + | Operator::V128Store32Lane { .. } + | Operator::V128Store64Lane { .. } + | Operator::I16x8Q15MulrSatS + | Operator::I16x8ExtMulLowI8x16S | Operator::I16x8ExtMulHighI8x16S | Operator::I16x8ExtMulLowI8x16U | Operator::I16x8ExtMulHighI8x16U @@ -2522,7 +2536,6 @@ fn type_of(operator: &Operator) -> Type { | Operator::I8x16GeU | Operator::I8x16Neg | Operator::I8x16Abs - | Operator::I8x16AnyTrue | Operator::I8x16AllTrue | Operator::I8x16Shl | Operator::I8x16ShrS @@ -2557,7 +2570,6 @@ fn type_of(operator: &Operator) -> Type { | Operator::I16x8GeU | Operator::I16x8Neg | Operator::I16x8Abs - | Operator::I16x8AnyTrue | Operator::I16x8AllTrue | Operator::I16x8Shl | Operator::I16x8ShrS @@ -2592,7 +2604,6 @@ fn type_of(operator: &Operator) -> Type { | Operator::I32x4GeU | Operator::I32x4Neg | Operator::I32x4Abs - | Operator::I32x4AnyTrue | Operator::I32x4AllTrue | Operator::I32x4Shl | Operator::I32x4ShrS diff --git a/crates/debug/Cargo.toml b/crates/debug/Cargo.toml index 122c22b930..28e1297ea8 100644 --- a/crates/debug/Cargo.toml +++ b/crates/debug/Cargo.toml @@ -13,8 +13,8 @@ edition = "2018" [dependencies] gimli = "0.23.0" -wasmparser = "0.72" -object = { version = "0.22.0", default-features = false, features = ["read_core", "elf", "write"] } +wasmparser = "0.73" +object = { version = "0.23.0", default-features = false, features = ["read_core", "elf", "write"] } wasmtime-environ = { path = "../environ", version = "0.22.0" } target-lexicon = { version = "0.11.0", default-features = false } anyhow = "1.0" diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 3463bc4d30..81b775402c 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -16,7 +16,7 @@ anyhow = "1.0" cranelift-codegen = { path = "../../cranelift/codegen", version = "0.69.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.69.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.69.0", features = ["enable-serde"] } -wasmparser = "0.72" +wasmparser = "0.73" indexmap = { version = "1.0.2", features = ["serde-1"] } thiserror = "1.0.4" serde = { version = "1.0.94", features = ["derive"] } diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index 17a0e811c7..140922809e 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -13,8 +13,8 @@ arbitrary = { version = "0.4.1", features = ["derive"] } env_logger = "0.8.1" log = "0.4.8" rayon = "1.2.1" -wasmparser = "0.72" -wasmprinter = "0.2.17" +wasmparser = "0.73" +wasmprinter = "0.2.20" wasmtime = { path = "../wasmtime" } wasmtime-wast = { path = "../wast" } wasm-encoder = "0.4" diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index b101c15819..2353d98ed5 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -28,13 +28,13 @@ rayon = { version = "1.0", optional = true } region = "2.1.0" thiserror = "1.0.4" target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.72" +wasmparser = "0.73" more-asserts = "0.2.1" anyhow = "1.0" cfg-if = "1.0" log = "0.4" gimli = { version = "0.23.0", default-features = false, features = ["write"] } -object = { version = "0.22.0", default-features = false, features = ["write"] } +object = { version = "0.23.0", default-features = false, features = ["write"] } serde = { version = "1.0.94", features = ["derive"] } addr2line = { version = "0.14", default-features = false } diff --git a/crates/lightbeam/Cargo.toml b/crates/lightbeam/Cargo.toml index 9b5632384d..2cde3bd067 100644 --- a/crates/lightbeam/Cargo.toml +++ b/crates/lightbeam/Cargo.toml @@ -18,18 +18,18 @@ derive_more = "0.99" dynasm = "1.0.0" dynasmrt = "1.0.0" iter-enum = "0.2" -itertools = "0.9.0" +itertools = "0.10.0" memoffset = "0.6.0" more-asserts = "0.2.1" smallvec = "1.6.1" thiserror = "1.0.9" typemap = "0.3" -wasmparser = "0.72" +wasmparser = "0.73" [dev-dependencies] lazy_static = "1.2" wat = "1.0.23" -quickcheck = "0.9.0" +quickcheck = "1.0.0" anyhow = "1.0" [badges] diff --git a/crates/lightbeam/wasmtime/Cargo.toml b/crates/lightbeam/wasmtime/Cargo.toml index 08ab2bac17..a2b8b06e9d 100644 --- a/crates/lightbeam/wasmtime/Cargo.toml +++ b/crates/lightbeam/wasmtime/Cargo.toml @@ -13,6 +13,6 @@ edition = "2018" [dependencies] lightbeam = { path = "..", version = "0.22.0" } -wasmparser = "0.72" +wasmparser = "0.73" cranelift-codegen = { path = "../../../cranelift/codegen", version = "0.69.0" } wasmtime-environ = { path = "../../environ", version = "0.22.0" } diff --git a/crates/obj/Cargo.toml b/crates/obj/Cargo.toml index 917ea45236..d88d388fc6 100644 --- a/crates/obj/Cargo.toml +++ b/crates/obj/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] anyhow = "1.0" wasmtime-environ = { path = "../environ", version = "0.22.0" } -object = { version = "0.22.0", default-features = false, features = ["write"] } +object = { version = "0.23.0", default-features = false, features = ["write"] } more-asserts = "0.2.1" target-lexicon = { version = "0.11.0", default-features = false } wasmtime-debug = { path = "../debug", version = "0.22.0" } diff --git a/crates/profiling/Cargo.toml b/crates/profiling/Cargo.toml index dfc853b3d7..072cb0fbb5 100644 --- a/crates/profiling/Cargo.toml +++ b/crates/profiling/Cargo.toml @@ -24,7 +24,7 @@ wasmtime-runtime = { path = "../runtime", version = "0.22.0" } ittapi-rs = { version = "0.1.5", optional = true } [dependencies.object] -version = "0.22.0" +version = "0.23.0" optional = true default-features = false features = ['read_core', 'elf', 'std'] diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 5cb30d116f..52b36464cf 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -16,7 +16,7 @@ wasmtime-jit = { path = "../jit", version = "0.22.0" } wasmtime-cache = { path = "../cache", version = "0.22.0", optional = true } wasmtime-profiling = { path = "../profiling", version = "0.22.0" } target-lexicon = { version = "0.11.0", default-features = false } -wasmparser = "0.72" +wasmparser = "0.73" anyhow = "1.0.19" region = "2.2.0" libc = "0.2" diff --git a/crates/wast/Cargo.toml b/crates/wast/Cargo.toml index 97e660d0d1..8c1dea6331 100644 --- a/crates/wast/Cargo.toml +++ b/crates/wast/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] anyhow = "1.0.19" wasmtime = { path = "../wasmtime", version = "0.22.0", default-features = false } -wast = "31.0.0" +wast = "32.0.0" [badges] maintenance = { status = "actively-developed" } diff --git a/deny.toml b/deny.toml index 37e5a3a0f8..5fbdd14a0e 100644 --- a/deny.toml +++ b/deny.toml @@ -42,4 +42,8 @@ skip = [ { name = "humantime" }, # caused by env_logger { name = "getrandom" }, # rand not updates to 0.2 yet { name = "wast" }, # old one pulled in by witx + { name = "rand" }, # 0.7 pulled in by transitive deps + { name = "rand_core" }, # 0.7 pulled in by transitive deps + { name = "rand_chacha" }, # 0.7 pulled in by transitive deps + { name = "itertools" }, # 0.9 pulled in by zstd-sys ] diff --git a/tests/misc_testsuite/multi-memory/simple.wast b/tests/misc_testsuite/multi-memory/simple.wast index c06c0ff420..14b203b1be 100644 --- a/tests/misc_testsuite/multi-memory/simple.wast +++ b/tests/misc_testsuite/multi-memory/simple.wast @@ -5,20 +5,20 @@ (func (export "store1") (param i32 i64) local.get 0 local.get 1 - i64.store $m1) + i64.store (memory $m1)) (func (export "store2") (param i32 i64) local.get 0 local.get 1 - i64.store $m2) + i64.store (memory $m2)) (func (export "load1") (param i32) (result i64) local.get 0 - i64.load $m1) + i64.load (memory $m1)) (func (export "load2") (param i32) (result i64) local.get 0 - i64.load $m2) + i64.load (memory $m2)) ) (invoke "store1" (i32.const 0) (i64.const 1)) @@ -64,20 +64,20 @@ (func (export "store1") (param i32 i64) local.get 0 local.get 1 - i64.store $m1) + i64.store (memory $m1)) (func (export "store2") (param i32 i64) local.get 0 local.get 1 - i64.store $m2) + i64.store (memory $m2)) (func (export "load1") (param i32) (result i64) local.get 0 - i64.load $m1) + i64.load (memory $m1)) (func (export "load2") (param i32) (result i64) local.get 0 - i64.load $m2) + i64.load (memory $m2)) ) (invoke "store1" (i32.const 0) (i64.const 1)) @@ -92,11 +92,11 @@ (func (export "grow1") (param i32) (result i32) local.get 0 - memory.grow $m1) + memory.grow (memory $m1)) (func (export "grow2") (param i32) (result i32) local.get 0 - memory.grow $m2) + memory.grow (memory $m2)) (func (export "size1") (result i32) memory.size $m1) (func (export "size2") (result i32) memory.size $m2) @@ -128,7 +128,7 @@ i32.const 4 memory.init $d $m2 i32.const 1 - i32.load $m2) + i32.load (memory $m2)) (data $d "\01\00\00\00" "\02\00\00\00") ) @@ -154,7 +154,7 @@ i32.const 2 memory.fill $m2 i32.const 1 - i32.load $m2) + i32.load (memory $m2)) ) (assert_return (invoke "fill1") (i32.const 0x01010101)) From c84d6be6f46c6b0d9308f22fa463125eb0348c36 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Sat, 9 Jan 2021 02:53:26 -0800 Subject: [PATCH 24/55] Detailed debug-info (DWARF) support in new backends (initially x64). This PR propagates "value labels" all the way from CLIF to DWARF metadata on the emitted machine code. The key idea is as follows: - Translate value-label metadata on the input into "value_label" pseudo-instructions when lowering into VCode. These pseudo-instructions take a register as input, denote a value label, and semantically are like a "move into value label" -- i.e., they update the current value (as seen by debugging tools) of the given local. These pseudo-instructions emit no machine code. - Perform a dataflow analysis *at the machine-code level*, tracking value-labels that propagate into registers and into [SP+constant] stack storage. This is a forward dataflow fixpoint analysis where each storage location can contain a *set* of value labels, and each value label can reside in a *set* of storage locations. (Meet function is pairwise intersection by storage location.) This analysis traces value labels symbolically through loads and stores and reg-to-reg moves, so it will naturally handle spills and reloads without knowing anything special about them. - When this analysis converges, we have, at each machine-code offset, a mapping from value labels to some number of storage locations; for each offset for each label, we choose the best location (prefer registers). Note that we can choose any location, as the symbolic dataflow analysis is sound and guarantees that the value at the value_label instruction propagates to all of the named locations. - Then we can convert this mapping into a format that the DWARF generation code (wasmtime's debug crate) can use. This PR also adds the new-backend variant to the gdb tests on CI. --- .github/workflows/main.yml | 8 + cranelift/codegen/Cargo.toml | 2 +- cranelift/codegen/src/context.rs | 1 + cranelift/codegen/src/ir/mod.rs | 1 + .../codegen/src/isa/aarch64/inst/emit.rs | 3 + cranelift/codegen/src/isa/aarch64/inst/mod.rs | 29 +- cranelift/codegen/src/isa/aarch64/mod.rs | 1 + cranelift/codegen/src/isa/arm32/mod.rs | 1 + cranelift/codegen/src/isa/mod.rs | 6 + cranelift/codegen/src/isa/x64/inst/emit.rs | 4 + cranelift/codegen/src/isa/x64/inst/mod.rs | 48 +- cranelift/codegen/src/isa/x64/mod.rs | 10 +- cranelift/codegen/src/machinst/adapter.rs | 8 + cranelift/codegen/src/machinst/debug.rs | 414 ++++++++++++++++++ cranelift/codegen/src/machinst/lower.rs | 81 +++- cranelift/codegen/src/machinst/mod.rs | 115 +++-- cranelift/codegen/src/machinst/vcode.rs | 62 ++- cranelift/codegen/src/value_label.rs | 35 +- cranelift/codegen/src/write.rs | 12 +- crates/debug/src/transform/expression.rs | 89 +++- crates/environ/src/data_structures.rs | 4 +- 21 files changed, 842 insertions(+), 92 deletions(-) create mode 100644 cranelift/codegen/src/machinst/debug.rs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3885cd9f09..c954cf6595 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -292,6 +292,14 @@ jobs: env: RUST_BACKTRACE: 1 + # Test debug (DWARF) related functionality on new backend. + - run: | + sudo apt-get update && sudo apt-get install -y gdb + cargo test --features experimental_x64 test_debug_dwarf -- --ignored --test-threads 1 + if: matrix.target == 'x86_64-unknown-linux-gnu' + env: + RUST_BACKTRACE: 1 + # Build and test lightbeam. Note that # Lightbeam tests fail right now, but we don't want to block on that. - run: cargo build --package lightbeam diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 15b592772a..e7a618ba46 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -74,7 +74,7 @@ all-arch = [ ] # For dependent crates that want to serialize some parts of cranelift -enable-serde = ["serde"] +enable-serde = ["serde", "regalloc/enable-serde"] # Allow snapshotting regalloc test cases. Useful only to report bad register # allocation failures, or for regalloc.rs developers. diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index ef092fb818..b831f9966a 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -473,6 +473,7 @@ impl Context { Ok(build_value_labels_ranges::( &self.func, &self.regalloc, + self.mach_compile_result.as_ref(), isa, )) } diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index c5e827db3d..c78dde81de 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -58,6 +58,7 @@ pub use crate::ir::table::TableData; pub use crate::ir::trapcode::TrapCode; pub use crate::ir::types::Type; pub use crate::ir::valueloc::{ArgumentLoc, ValueLoc}; +pub use crate::value_label::LabelValueLoc; pub use cranelift_codegen_shared::condcodes; use crate::binemit; diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 1195e879bb..599a8edacd 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -2371,6 +2371,9 @@ impl MachInstEmit for Inst { sink.bind_label(jump_around_label); } } + &Inst::ValueLabelMarker { .. } => { + // Nothing; this is only used to compute debug info. + } } let end_off = sink.cur_offset(); diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 5e88a783f5..b85928edcc 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -7,7 +7,7 @@ use crate::binemit::CodeOffset; use crate::ir::types::{ B1, B128, B16, B32, B64, B8, F32, F64, FFLAGS, I128, I16, I32, I64, I8, I8X16, IFLAGS, R32, R64, }; -use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type}; +use crate::ir::{ExternalName, MemFlags, Opcode, SourceLoc, TrapCode, Type, ValueLabel}; use crate::isa::CallConv; use crate::machinst::*; use crate::{settings, CodegenError, CodegenResult}; @@ -1210,6 +1210,12 @@ pub enum Inst { /// The needed space before the next deadline. needed_space: CodeOffset, }, + + /// A definition of a value label. + ValueLabelMarker { + reg: Reg, + label: ValueLabel, + }, } fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize { @@ -2017,6 +2023,9 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { memarg_regs(mem, collector); } &Inst::VirtualSPOffsetAdj { .. } => {} + &Inst::ValueLabelMarker { reg, .. } => { + collector.add_use(reg); + } &Inst::EmitIsland { .. } => {} } } @@ -2767,6 +2776,9 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { } &mut Inst::VirtualSPOffsetAdj { .. } => {} &mut Inst::EmitIsland { .. } => {} + &mut Inst::ValueLabelMarker { ref mut reg, .. } => { + map_use(mapper, reg); + } } } @@ -2962,6 +2974,17 @@ impl MachInst for Inst { fn ref_type_regclass(_: &settings::Flags) -> RegClass { RegClass::I64 } + + fn gen_value_label_marker(label: ValueLabel, reg: Reg) -> Self { + Inst::ValueLabelMarker { label, reg } + } + + fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> { + match self { + Inst::ValueLabelMarker { label, reg } => Some((*label, *reg)), + _ => None, + } + } } //============================================================================= @@ -4071,6 +4094,10 @@ impl Inst { format!("virtual_sp_offset_adjust {}", offset) } &Inst::EmitIsland { needed_space } => format!("emit_island {}", needed_space), + + &Inst::ValueLabelMarker { label, reg } => { + format!("value_label {:?}, {}", label, reg.show_rru(mb_rru)) + } } } } diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs index c3c56632d3..11eb0a6ea6 100644 --- a/cranelift/codegen/src/isa/aarch64/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/mod.rs @@ -79,6 +79,7 @@ impl MachBackend for AArch64Backend { frame_size, disasm, unwind_info, + value_labels_ranges: None, }) } diff --git a/cranelift/codegen/src/isa/arm32/mod.rs b/cranelift/codegen/src/isa/arm32/mod.rs index 4b9701fd1d..6ab0f9c57c 100644 --- a/cranelift/codegen/src/isa/arm32/mod.rs +++ b/cranelift/codegen/src/isa/arm32/mod.rs @@ -74,6 +74,7 @@ impl MachBackend for Arm32Backend { frame_size, disasm, unwind_info: None, + value_labels_ranges: None, }) } diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 900dfd9ddd..73a83dda34 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -325,6 +325,12 @@ pub trait TargetIsa: fmt::Display + Send + Sync { Err(RegisterMappingError::UnsupportedArchitecture) } + #[cfg(feature = "unwind")] + /// Map a regalloc::Reg to its corresponding DWARF register. + fn map_regalloc_reg_to_dwarf(&self, _: ::regalloc::Reg) -> Result { + Err(RegisterMappingError::UnsupportedArchitecture) + } + /// Returns an iterator over legal encodings for the instruction. fn legal_encodings<'a>( &'a self, diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 6622903452..095256ab49 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -3029,6 +3029,10 @@ pub(crate) fn emit( sink.put1(0xff); sink.put1(0x17); } + + Inst::ValueLabelMarker { .. } => { + // Nothing; this is only used to compute debug info. + } } state.clear_post_insn(); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index d1302cacbc..bab28f2aa0 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1,7 +1,7 @@ //! This module defines x86_64-specific machine instruction types. use crate::binemit::{CodeOffset, StackMap}; -use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type}; +use crate::ir::{types, ExternalName, Opcode, SourceLoc, TrapCode, Type, ValueLabel}; use crate::isa::x64::abi::X64ABIMachineSpec; use crate::isa::x64::settings as x64_settings; use crate::isa::CallConv; @@ -484,6 +484,9 @@ pub enum Inst { /// A Mach-O TLS symbol access. Returns address of the TLS /// symbol in rax. MachOTlsGetAddr { symbol: ExternalName }, + + /// A definition of a value label. + ValueLabelMarker { reg: Reg, label: ValueLabel }, } pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool { @@ -544,7 +547,8 @@ impl Inst { | Inst::XmmMinMaxSeq { .. } | Inst::XmmUninitializedValue { .. } | Inst::ElfTlsGetAddr { .. } - | Inst::MachOTlsGetAddr { .. } => None, + | Inst::MachOTlsGetAddr { .. } + | Inst::ValueLabelMarker { .. } => None, // These use dynamic SSE opcodes. Inst::GprToXmm { op, .. } @@ -1800,6 +1804,10 @@ impl PrettyPrint for Inst { Inst::MachOTlsGetAddr { ref symbol } => { format!("macho_tls_get_addr {:?}", symbol) } + + Inst::ValueLabelMarker { label, reg } => { + format!("value_label {:?}, {}", label, reg.show_rru(mb_rru)) + } } } } @@ -2071,6 +2079,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(reg); } } + + Inst::ValueLabelMarker { reg, .. } => { + collector.add_use(*reg); + } } } @@ -2446,6 +2458,8 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { dst.map_uses(mapper); } + Inst::ValueLabelMarker { ref mut reg, .. } => map_use(mapper, reg), + Inst::Ret | Inst::EpiloguePlaceholder | Inst::JmpKnown { .. } @@ -2536,6 +2550,25 @@ impl MachInst for Inst { } } + fn stack_op_info(&self) -> Option { + match self { + Self::VirtualSPOffsetAdj { offset } => Some(MachInstStackOpInfo::NomSPAdj(*offset)), + Self::MovRM { + size: 8, + src, + dst: SyntheticAmode::NominalSPOffset { simm32 }, + } => Some(MachInstStackOpInfo::StoreNomSPOff(*src, *simm32 as i64)), + Self::Mov64MR { + src: SyntheticAmode::NominalSPOffset { simm32 }, + dst, + } => Some(MachInstStackOpInfo::LoadNomSPOff( + dst.to_reg(), + *simm32 as i64, + )), + _ => None, + } + } + fn gen_move(dst_reg: Writable, src_reg: Reg, ty: Type) -> Inst { let rc_dst = dst_reg.to_reg().get_class(); let rc_src = src_reg.get_class(); @@ -2710,6 +2743,17 @@ impl MachInst for Inst { RegClass::I64 } + fn gen_value_label_marker(label: ValueLabel, reg: Reg) -> Self { + Inst::ValueLabelMarker { label, reg } + } + + fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> { + match self { + Inst::ValueLabelMarker { label, reg } => Some((*label, *reg)), + _ => None, + } + } + type LabelUse = LabelUse; } diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index 73183f79e8..ca809e2d55 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -4,13 +4,14 @@ use self::inst::EmitInfo; use super::TargetIsa; use crate::ir::{condcodes::IntCC, Function}; +use crate::isa::unwind::systemv::RegisterMappingError; use crate::isa::x64::{inst::regs::create_reg_universe_systemv, settings as x64_settings}; use crate::isa::Builder as IsaBuilder; use crate::machinst::{compile, MachBackend, MachCompileResult, TargetIsaAdapter, VCode}; use crate::result::CodegenResult; use crate::settings::{self as shared_settings, Flags}; use alloc::boxed::Box; -use regalloc::{PrettyPrint, RealRegUniverse}; +use regalloc::{PrettyPrint, RealRegUniverse, Reg}; use target_lexicon::Triple; mod abi; @@ -60,6 +61,7 @@ impl MachBackend for X64Backend { let buffer = buffer.finish(); let frame_size = vcode.frame_size(); let unwind_info = vcode.unwind_info()?; + let value_labels_ranges = vcode.value_labels_ranges()?; let disasm = if want_disasm { Some(vcode.show_rru(Some(&create_reg_universe_systemv(flags)))) @@ -72,6 +74,7 @@ impl MachBackend for X64Backend { frame_size, disasm, unwind_info, + value_labels_ranges, }) } @@ -127,6 +130,11 @@ impl MachBackend for X64Backend { fn create_systemv_cie(&self) -> Option { Some(inst::unwind::systemv::create_cie()) } + + #[cfg(feature = "unwind")] + fn map_reg_to_dwarf(&self, reg: Reg) -> Result { + inst::unwind::systemv::map_reg(reg).map(|reg| reg.0) + } } /// Create a new `isa::Builder`. diff --git a/cranelift/codegen/src/machinst/adapter.rs b/cranelift/codegen/src/machinst/adapter.rs index 43ff9e51a4..4b3be0f3c0 100644 --- a/cranelift/codegen/src/machinst/adapter.rs +++ b/cranelift/codegen/src/machinst/adapter.rs @@ -10,6 +10,9 @@ use crate::settings::Flags; #[cfg(feature = "testing_hooks")] use crate::regalloc::RegDiversions; +#[cfg(feature = "unwind")] +use crate::isa::unwind::systemv::RegisterMappingError; + use core::any::Any; use std::borrow::Cow; use std::fmt; @@ -134,6 +137,11 @@ impl TargetIsa for TargetIsaAdapter { self.backend.create_systemv_cie() } + #[cfg(feature = "unwind")] + fn map_regalloc_reg_to_dwarf(&self, r: Reg) -> Result { + self.backend.map_reg_to_dwarf(r) + } + fn as_any(&self) -> &dyn Any { self as &dyn Any } diff --git a/cranelift/codegen/src/machinst/debug.rs b/cranelift/codegen/src/machinst/debug.rs new file mode 100644 index 0000000000..be0d0a8ec7 --- /dev/null +++ b/cranelift/codegen/src/machinst/debug.rs @@ -0,0 +1,414 @@ +//! Debug info analysis: computes value-label ranges from value-label markers in +//! generated VCode. +//! +//! We "reverse-engineer" debug info like this because it is far more reliable +//! than generating it while emitting code and keeping it in sync. +//! +//! This works by (i) observing "value-label marker" instructions, which are +//! semantically just an assignment from a register to a "value label" (which +//! one can think of as another register; they represent, e.g., Wasm locals) at +//! a certain point in the code, and (ii) observing loads and stores to the +//! stack and register moves. +//! +//! We track, at every program point, the correspondence between each value +//! label and *all* locations in which it resides. E.g., if it is stored to the +//! stack, we remember that it is in both a register and the stack slot; but if +//! the register is later overridden, then we have it just in the stack slot. +//! This allows us to avoid false-positives observing loads/stores that we think +//! are spillslots but really aren't. +//! +//! We do a standard forward dataflow analysis to compute this info. + +use crate::ir::ValueLabel; +use crate::machinst::*; +use crate::value_label::{LabelValueLoc, ValueLabelsRanges, ValueLocRange}; +use log::trace; +use regalloc::{Reg, RegUsageCollector}; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::hash::Hash; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum ValueLoc { + Reg(Reg), + /// Nominal-SP offset. + Stack(i64), +} + +impl From for LabelValueLoc { + fn from(v: ValueLoc) -> Self { + match v { + ValueLoc::Reg(r) => LabelValueLoc::Reg(r), + ValueLoc::Stack(off) => LabelValueLoc::SPOffset(off), + } + } +} + +impl ValueLoc { + fn is_reg(self) -> bool { + match self { + ValueLoc::Reg(_) => true, + _ => false, + } + } + fn is_stack(self) -> bool { + match self { + ValueLoc::Stack(_) => true, + _ => false, + } + } +} + +/// Mappings at one program point. +#[derive(Clone, Debug)] +struct AnalysisInfo { + // nominal SP relative to real SP. + nominal_sp_offset: Option, + label_to_locs: HashMap>, + reg_to_label: HashMap, + stack_to_label: HashMap, +} + +fn get_inst_writes(m: &M) -> Vec { + // TODO: expose this part of regalloc.rs's interface publicly. + let mut vecs = RegUsageCollector::get_empty_reg_vecs_test_framework_only(false); + let mut coll = RegUsageCollector::new(&mut vecs); + m.get_regs(&mut coll); + vecs.defs.extend(vecs.mods.into_iter()); + vecs.defs +} + +impl AnalysisInfo { + fn new() -> Self { + AnalysisInfo { + nominal_sp_offset: Some(0), + label_to_locs: HashMap::new(), + reg_to_label: HashMap::new(), + stack_to_label: HashMap::new(), + } + } + + fn clear_label(&mut self, label: ValueLabel) { + if let Some(locs) = self.label_to_locs.remove(&label) { + for loc in locs { + match loc { + ValueLoc::Reg(r) => { + self.reg_to_label.remove(&r); + } + ValueLoc::Stack(off) => { + self.stack_to_label.remove(&off); + } + } + } + } + } + fn clear_reg(&mut self, reg: Reg) { + if let Some(label) = self.reg_to_label.remove(®) { + if let Some(locs) = self.label_to_locs.get_mut(&label) { + locs.remove(&ValueLoc::Reg(reg)); + } + } + } + fn clear_stack_off(&mut self, off: i64) { + if let Some(label) = self.stack_to_label.remove(&off) { + if let Some(locs) = self.label_to_locs.get_mut(&label) { + locs.remove(&ValueLoc::Stack(off)); + } + } + } + fn def_label_at_reg(&mut self, label: ValueLabel, reg: Reg) { + self.clear_label(label); + self.label_to_locs + .entry(label) + .or_insert_with(|| HashSet::new()) + .insert(ValueLoc::Reg(reg)); + self.reg_to_label.insert(reg, label); + } + fn store_reg(&mut self, reg: Reg, off: i64) { + self.clear_stack_off(off); + if let Some(label) = self.reg_to_label.get(®) { + if let Some(locs) = self.label_to_locs.get_mut(label) { + locs.insert(ValueLoc::Stack(off)); + } + self.stack_to_label.insert(off, *label); + } + } + fn load_reg(&mut self, reg: Reg, off: i64) { + self.clear_reg(reg); + if let Some(&label) = self.stack_to_label.get(&off) { + if let Some(locs) = self.label_to_locs.get_mut(&label) { + locs.insert(ValueLoc::Reg(reg)); + } + self.reg_to_label.insert(reg, label); + } + } + fn move_reg(&mut self, to: Reg, from: Reg) { + self.clear_reg(to); + if let Some(&label) = self.reg_to_label.get(&from) { + if let Some(locs) = self.label_to_locs.get_mut(&label) { + locs.insert(ValueLoc::Reg(to)); + } + self.reg_to_label.insert(to, label); + } + } + + fn step(&mut self, inst: &M) { + for write in get_inst_writes(inst) { + self.clear_reg(write); + } + if let Some((label, reg)) = inst.defines_value_label() { + self.def_label_at_reg(label, reg); + } + match inst.stack_op_info() { + Some(MachInstStackOpInfo::LoadNomSPOff(reg, offset)) => { + self.load_reg(reg, offset + self.nominal_sp_offset.unwrap()); + } + Some(MachInstStackOpInfo::StoreNomSPOff(reg, offset)) => { + self.store_reg(reg, offset + self.nominal_sp_offset.unwrap()); + } + Some(MachInstStackOpInfo::NomSPAdj(offset)) => { + if self.nominal_sp_offset.is_some() { + self.nominal_sp_offset = Some(self.nominal_sp_offset.unwrap() + offset); + } + } + _ => {} + } + if let Some((to, from)) = inst.is_move() { + let to = to.to_reg(); + self.move_reg(to, from); + } + } +} + +trait IntersectFrom { + fn intersect_from(&mut self, other: &Self) -> IntersectResult; +} +struct IntersectResult { + changed: bool, + remove: bool, +} + +impl IntersectFrom for AnalysisInfo { + fn intersect_from(&mut self, other: &Self) -> IntersectResult { + let mut changed = false; + changed |= self + .nominal_sp_offset + .intersect_from(&other.nominal_sp_offset) + .changed; + changed |= self + .label_to_locs + .intersect_from(&other.label_to_locs) + .changed; + changed |= self + .reg_to_label + .intersect_from(&other.reg_to_label) + .changed; + changed |= self + .stack_to_label + .intersect_from(&other.stack_to_label) + .changed; + IntersectResult { + changed, + remove: false, + } + } +} + +impl IntersectFrom for HashMap +where + K: Copy + Eq + Hash, + V: IntersectFrom, +{ + fn intersect_from(&mut self, other: &Self) -> IntersectResult { + let mut changed = false; + let mut remove_keys = vec![]; + for k in self.keys() { + if !other.contains_key(k) { + remove_keys.push(*k); + } + } + for k in remove_keys { + changed = true; + self.remove(&k); + } + + let mut remove_keys = vec![]; + for k in other.keys() { + if let Some(v) = self.get_mut(k) { + let result = v.intersect_from(other.get(k).unwrap()); + changed |= result.changed; + if result.remove { + remove_keys.push(*k); + } + } + } + for k in remove_keys { + changed = true; + self.remove(&k); + } + + IntersectResult { + changed, + remove: self.len() == 0, + } + } +} +impl IntersectFrom for HashSet +where + T: Copy + Eq + Hash, +{ + fn intersect_from(&mut self, other: &Self) -> IntersectResult { + let mut changed = false; + let mut remove = vec![]; + for val in self.iter() { + if !other.contains(val) { + remove.push(*val); + } + } + for val in remove { + changed = true; + self.remove(&val); + } + + IntersectResult { + changed, + remove: self.len() == 0, + } + } +} +impl IntersectFrom for ValueLabel { + fn intersect_from(&mut self, other: &Self) -> IntersectResult { + // Remove if not equal (simple top -> values -> bottom lattice) + IntersectResult { + changed: false, + remove: *self != *other, + } + } +} +impl IntersectFrom for Option +where + T: Copy + Eq, +{ + fn intersect_from(&mut self, other: &Self) -> IntersectResult { + let mut changed = false; + if !(self.is_some() && other.is_some() && self == other) { + changed = true; + *self = None; + } + IntersectResult { + changed, + remove: self.is_none(), + } + } +} + +pub(crate) fn compute( + insts: &[I], + inst_ends: &[u32], + label_insn_iix: &[u32], +) -> ValueLabelsRanges { + let inst_start = |idx: usize| if idx == 0 { 0 } else { inst_ends[idx - 1] }; + + trace!("compute: insts ="); + for i in 0..insts.len() { + trace!(" #{} end: {} -> {:?}", i, inst_ends[i], insts[i]); + } + trace!("label_insn_iix: {:?}", label_insn_iix); + + // Info at each block head, indexed by label. + let mut block_starts: HashMap = HashMap::new(); + + // Initialize state at entry. + block_starts.insert(0, AnalysisInfo::new()); + + // Worklist: basic-block start offset. + let mut worklist = VecDeque::new(); + let mut worklist_set = HashSet::new(); + worklist.push_back(0); + worklist_set.insert(0); + + while !worklist.is_empty() { + let block = worklist.pop_front().unwrap(); + worklist_set.remove(&block); + + let mut state = block_starts.get(&block).unwrap().clone(); + trace!("at block {} -> state: {:?}", block, state); + let mut iix = label_insn_iix[block as usize]; + while iix < insts.len() as u32 { + state.step(&insts[iix as usize]); + trace!(" -> inst #{}: {:?}", iix, insts[iix as usize]); + trace!(" --> state: {:?}", state); + + let term = insts[iix as usize].is_term(); + if term.is_term() { + for succ in term.get_succs() { + trace!(" SUCCESSOR block {}", succ.get()); + if let Some(succ_state) = block_starts.get_mut(&succ.get()) { + trace!(" orig state: {:?}", succ_state); + if succ_state.intersect_from(&state).changed { + if worklist_set.insert(succ.get()) { + worklist.push_back(succ.get()); + } + trace!(" (changed)"); + } + trace!(" new state: {:?}", succ_state); + } else { + // First time seeing this block + block_starts.insert(succ.get(), state.clone()); + worklist.push_back(succ.get()); + worklist_set.insert(succ.get()); + } + } + break; + } + + iix += 1; + } + } + + // Now iterate over blocks one last time, collecting + // value-label locations. + + let mut value_labels_ranges: ValueLabelsRanges = HashMap::new(); + for block in 0..label_insn_iix.len() { + let start_iix = label_insn_iix[block]; + let end_iix = if block == label_insn_iix.len() - 1 { + insts.len() as u32 + } else { + label_insn_iix[block + 1] + }; + let block = block as u32; + let mut state = block_starts.get(&block).unwrap().clone(); + for iix in start_iix..end_iix { + let offset = inst_start(iix as usize); + let end = inst_ends[iix as usize]; + state.step(&insts[iix as usize]); + + for (label, locs) in &state.label_to_locs { + trace!(" inst {} has label {:?} -> locs {:?}", iix, label, locs); + // Find an appropriate loc: a register if possible, otherwise pick the first stack + // loc. + let reg = locs.iter().cloned().find(|l| l.is_reg()); + let stack = locs.iter().cloned().find(|l| l.is_stack()); + if let Some(loc) = reg.or(stack) { + let loc = LabelValueLoc::from(loc); + let list = value_labels_ranges.entry(*label).or_insert_with(|| vec![]); + if list.is_empty() + || list.last().unwrap().end <= offset + || list.last().unwrap().loc != loc + { + list.push(ValueLocRange { + loc, + start: end, + end: end + 1, + }); + } else { + list.last_mut().unwrap().end = end + 1; + } + } + } + } + } + + trace!("ret: {:?}", value_labels_ranges); + value_labels_ranges +} diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index e35e3b068e..7cb1c8d769 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -13,6 +13,7 @@ use crate::ir::instructions::BranchInfo; use crate::ir::{ ArgumentPurpose, Block, Constant, ConstantData, ExternalName, Function, GlobalValueData, Inst, InstructionData, MemFlags, Opcode, Signature, SourceLoc, Type, Value, ValueDef, + ValueLabelAssignments, ValueLabelStart, }; use crate::machinst::{ writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder, LoweredBlock, MachLabel, VCode, @@ -24,7 +25,7 @@ use alloc::vec::Vec; use core::convert::TryInto; use log::debug; use regalloc::{Reg, StackmapRequestInfo, Writable}; -use smallvec::SmallVec; +use smallvec::{smallvec, SmallVec}; use std::fmt::Debug; /// An "instruction color" partitions CLIF instructions by side-effecting ops. @@ -492,6 +493,14 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } fn gen_retval_setup(&mut self, gen_ret_inst: GenerateReturn) { + // Hack: to keep `vmctx` alive, if it exists, we emit a value label here + // for it if debug info is requested. This ensures that it exists either + // in a register or spillslot throughout the entire function body, and + // allows for a better debugging experience. + if let Some(vmctx_val) = self.f.special_param(ArgumentPurpose::VMContext) { + self.emit_value_label_marks_for_value(vmctx_val); + } + let retval_regs = self.retval_regs.clone(); for (i, regs) in retval_regs.into_iter().enumerate() { let regs = writable_value_regs(regs); @@ -725,6 +734,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> { if has_side_effect || value_needed { debug!("lowering: inst {}: {:?}", inst, self.f.dfg[inst]); backend.lower(self, inst)?; + // Emit value-label markers if needed, to later recover debug + // mappings. + self.emit_value_label_markers_for_inst(inst); } if data.opcode().is_return() { // Return: handle specially, using ABI-appropriate sequence. @@ -744,6 +756,72 @@ impl<'func, I: VCodeInst> Lower<'func, I> { Ok(()) } + fn get_value_labels<'a>(&'a self, val: Value, depth: usize) -> Option<&'a [ValueLabelStart]> { + if let Some(ref values_labels) = self.f.dfg.values_labels { + debug!( + "get_value_labels: val {} -> {} -> {:?}", + val, + self.f.dfg.resolve_aliases(val), + values_labels.get(&self.f.dfg.resolve_aliases(val)) + ); + let val = self.f.dfg.resolve_aliases(val); + match values_labels.get(&val) { + Some(&ValueLabelAssignments::Starts(ref list)) => Some(&list[..]), + Some(&ValueLabelAssignments::Alias { value, .. }) if depth < 10 => { + self.get_value_labels(value, depth + 1) + } + _ => None, + } + } else { + None + } + } + + fn emit_value_label_marks_for_value(&mut self, val: Value) { + let mut markers: SmallVec<[I; 4]> = smallvec![]; + let regs = self.value_regs[val]; + if regs.len() > 1 { + return; + } + let reg = regs.only_reg().unwrap(); + + if let Some(label_starts) = self.get_value_labels(val, 0) { + let labels = label_starts + .iter() + .map(|&ValueLabelStart { label, .. }| label) + .collect::>(); + for label in labels { + debug!( + "value labeling: defines val {:?} -> reg {:?} -> label {:?}", + val, reg, label, + ); + markers.push(I::gen_value_label_marker(label, reg)); + } + } + for marker in markers { + self.emit(marker); + } + } + + fn emit_value_label_markers_for_inst(&mut self, inst: Inst) { + debug!( + "value labeling: srcloc {}: inst {}", + self.srcloc(inst), + inst + ); + for &val in self.f.dfg.inst_results(inst) { + self.emit_value_label_marks_for_value(val); + } + } + + fn emit_value_label_markers_for_block_args(&mut self, block: Block) { + debug!("value labeling: block {}", block); + for &arg in self.f.dfg.block_params(block) { + self.emit_value_label_marks_for_value(arg); + } + self.finish_ir_inst(SourceLoc::default()); + } + fn finish_ir_inst(&mut self, loc: SourceLoc) { // `bb_insts` is kept in reverse order, so emit the instructions in // reverse order. @@ -885,6 +963,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { // Original block body. if let Some(bb) = lb.orig_block() { self.lower_clif_block(backend, bb)?; + self.emit_value_label_markers_for_block_args(bb); } // In-edge phi moves. if let Some((pred, inst, succ)) = lb.in_edge() { diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index 764531d54f..7ed2661dda 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -52,45 +52,9 @@ //! | - all symbolic stack references to //! | stackslots and spillslots are resolved //! | to concrete FP-offset mem addresses.) -//! | [block/insn ordering] //! | -//! VCode (machine instructions: -//! | - vcode.final_block_order is filled in. -//! | - new insn sequence from regalloc is -//! | placed back into vcode and block -//! | boundaries are updated.) -//! | [redundant branch/block -//! | removal] -//! | -//! VCode (machine instructions: -//! | - all blocks that were just an -//! | unconditional branch are removed.) -//! | -//! | [branch finalization -//! | (fallthroughs)] -//! | -//! VCode (machine instructions: -//! | - all branches are in lowered one- -//! | target form, but targets are still -//! | block indices.) -//! | -//! | [branch finalization -//! | (offsets)] -//! | -//! VCode (machine instructions: -//! | - all branch offsets from start of -//! | function are known, and all branches -//! | have resolved-offset targets.) -//! | -//! | [MemArg finalization] -//! | -//! VCode (machine instructions: -//! | - all MemArg references to the constant -//! | pool are replaced with offsets. -//! | - all constant-pool data is collected -//! | in the VCode.) -//! | -//! | [binary emission] +//! | [binary emission via MachBuffer +//! | with streaming branch resolution/simplification] //! | //! Vec (machine code!) //! @@ -98,11 +62,11 @@ use crate::binemit::{CodeInfo, CodeOffset, StackMap}; use crate::ir::condcodes::IntCC; -use crate::ir::{Function, SourceLoc, Type}; +use crate::ir::{Function, SourceLoc, Type, ValueLabel}; use crate::isa::unwind::input as unwind_input; use crate::result::CodegenResult; use crate::settings::Flags; - +use crate::value_label::ValueLabelsRanges; use alloc::boxed::Box; use alloc::vec::Vec; use core::fmt::Debug; @@ -111,10 +75,13 @@ use regalloc::RegUsageCollector; use regalloc::{ RealReg, RealRegUniverse, Reg, RegClass, RegUsageMapper, SpillSlot, VirtualReg, Writable, }; -use smallvec::SmallVec; +use smallvec::{smallvec, SmallVec}; use std::string::String; use target_lexicon::Triple; +#[cfg(feature = "unwind")] +use crate::isa::unwind::systemv::RegisterMappingError; + pub mod lower; pub use lower::*; pub mod vcode; @@ -137,6 +104,7 @@ pub mod inst_common; pub use inst_common::*; pub mod valueregs; pub use valueregs::*; +pub mod debug; /// A machine instruction. pub trait MachInst: Clone + Debug { @@ -163,6 +131,11 @@ pub trait MachInst: Clone + Debug { true } + /// If this is a load or store to the stack, return that info. + fn stack_op_info(&self) -> Option { + None + } + /// Generate a move. fn gen_move(to_reg: Writable, from_reg: Reg, ty: Type) -> Self; @@ -223,6 +196,17 @@ pub trait MachInst: Clone + Debug { /// be dependent on compilation flags. fn ref_type_regclass(_flags: &Flags) -> RegClass; + /// Does this instruction define a ValueLabel? Returns the `Reg` whose value + /// becomes the new value of the `ValueLabel` after this instruction. + fn defines_value_label(&self) -> Option<(ValueLabel, Reg)> { + None + } + + /// Create a marker instruction that defines a value label. + fn gen_value_label_marker(_label: ValueLabel, _reg: Reg) -> Self { + Self::gen_nop(0) + } + /// A label-use kind: a type that describes the types of label references that /// can occur in an instruction. type LabelUse: MachInstLabelUse; @@ -285,6 +269,35 @@ pub enum MachTerminator<'a> { Indirect(&'a [MachLabel]), } +impl<'a> MachTerminator<'a> { + /// Get the successor labels named in a `MachTerminator`. + pub fn get_succs(&self) -> SmallVec<[MachLabel; 2]> { + let mut ret = smallvec![]; + match self { + &MachTerminator::Uncond(l) => { + ret.push(l); + } + &MachTerminator::Cond(l1, l2) => { + ret.push(l1); + ret.push(l2); + } + &MachTerminator::Indirect(ls) => { + ret.extend(ls.iter().cloned()); + } + _ => {} + } + ret + } + + /// Is this a terminator? + pub fn is_term(&self) -> bool { + match self { + MachTerminator::None => false, + _ => true, + } + } +} + /// A trait describing the ability to encode a MachInst into binary machine code. pub trait MachInstEmit: MachInst { /// Persistent state carried across `emit` invocations. @@ -330,6 +343,8 @@ pub struct MachCompileResult { pub disasm: Option, /// Unwind info. pub unwind_info: Option>, + /// Debug info: value labels to registers/stackslots at code offsets. + pub value_labels_ranges: Option, } impl MachCompileResult { @@ -386,13 +401,17 @@ pub trait MachBackend { Ok(None) } - /// Machine-specific condcode info needed by TargetIsa. /// Creates a new System V Common Information Entry for the ISA. #[cfg(feature = "unwind")] fn create_systemv_cie(&self) -> Option { // By default, an ISA cannot create a System V CIE None } + /// Maps a regalloc::Reg to a DWARF register number. + #[cfg(feature = "unwind")] + fn map_reg_to_dwarf(&self, _: Reg) -> Result { + Err(RegisterMappingError::UnsupportedArchitecture) + } } /// Expected unwind info type. @@ -431,3 +450,15 @@ pub trait UnwindInfoGenerator { context: UnwindInfoContext, ) -> CodegenResult>>; } + +/// Info about an operation that loads or stores from/to the stack. +#[derive(Clone, Copy, Debug)] +pub enum MachInstStackOpInfo { + /// Load from an offset from the nominal stack pointer into the given reg. + LoadNomSPOff(Reg, i64), + /// Store to an offset from the nominal stack pointer from the given reg. + StoreNomSPOff(Reg, i64), + /// Adjustment of nominal-SP up or down. This value is added to subsequent + /// offsets in loads/stores above to produce real-SP offsets. + NomSPAdj(i64), +} diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index c57f018e35..6b554359b6 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -21,7 +21,6 @@ use crate::ir::{self, types, Constant, ConstantData, SourceLoc}; use crate::machinst::*; use crate::settings; use crate::timing; - use regalloc::Function as RegallocFunction; use regalloc::Set as RegallocSet; use regalloc::{ @@ -110,8 +109,12 @@ pub struct VCode { /// Ranges for prologue and epilogue instructions. prologue_epilogue_ranges: Option<(InsnRange, Box<[InsnRange]>)>, - /// Instruction end offsets - insts_layout: RefCell<(Vec, u32)>, + /// Do we generate debug info? + generate_debug_info: bool, + + /// Instruction end offsets, instruction indices at each label, and total + /// buffer size. Only present if `generate_debug_info` is set. + insts_layout: RefCell<(Vec, Vec, u32)>, /// Constants. constants: VCodeConstants, @@ -157,7 +160,13 @@ impl VCodeBuilder { constants: VCodeConstants, ) -> VCodeBuilder { let reftype_class = I::ref_type_regclass(abi.flags()); - let vcode = VCode::new(abi, emit_info, block_order, constants); + let vcode = VCode::new( + abi, + emit_info, + block_order, + constants, + /* generate_debug_info = */ true, + ); let stack_map_info = StackmapRequestInfo { reftype_class, reftyped_vregs: vec![], @@ -296,6 +305,7 @@ impl VCode { emit_info: I::Info, block_order: BlockLoweringOrder, constants: VCodeConstants, + generate_debug_info: bool, ) -> VCode { VCode { liveins: abi.liveins(), @@ -314,7 +324,8 @@ impl VCode { safepoint_insns: vec![], safepoint_slots: vec![], prologue_epilogue_ranges: None, - insts_layout: RefCell::new((vec![], 0)), + generate_debug_info, + insts_layout: RefCell::new((vec![], vec![], 0)), constants, } } @@ -484,7 +495,8 @@ impl VCode { buffer.reserve_labels_for_blocks(self.num_blocks() as BlockIndex); buffer.reserve_labels_for_constants(&self.constants); - let mut insts_layout = vec![0; self.insts.len()]; + let mut inst_ends = vec![0; self.insts.len()]; + let mut label_insn_iix = vec![0; self.num_blocks()]; let mut safepoint_idx = 0; let mut cur_srcloc = None; @@ -500,6 +512,7 @@ impl VCode { let (start, end) = self.block_ranges[block as usize]; buffer.bind_label(MachLabel::from_block(block)); + label_insn_iix[block as usize] = start; for iix in start..end { let srcloc = self.srclocs[iix as usize]; if cur_srcloc != Some(srcloc) { @@ -526,7 +539,19 @@ impl VCode { self.insts[iix as usize].emit(&mut buffer, &self.emit_info, &mut state); - insts_layout[iix as usize] = buffer.cur_offset(); + if self.generate_debug_info { + // Buffer truncation may have happened since last inst append; trim inst-end + // layout info as appropriate. + let l = &mut inst_ends[0..iix as usize]; + for end in l.iter_mut().rev() { + if *end > buffer.cur_offset() { + *end = buffer.cur_offset(); + } else { + break; + } + } + inst_ends[iix as usize] = buffer.cur_offset(); + } } if cur_srcloc.is_some() { @@ -553,7 +578,16 @@ impl VCode { buffer.defer_constant(label, data.alignment(), data.as_slice(), u32::max_value()); } - *self.insts_layout.borrow_mut() = (insts_layout, buffer.cur_offset()); + if self.generate_debug_info { + for end in inst_ends.iter_mut().rev() { + if *end > buffer.cur_offset() { + *end = buffer.cur_offset(); + } else { + break; + } + } + *self.insts_layout.borrow_mut() = (inst_ends, label_insn_iix, buffer.cur_offset()); + } buffer } @@ -567,13 +601,23 @@ impl VCode { let context = UnwindInfoContext { insts: &self.insts, insts_layout: &layout.0, - len: layout.1, + len: layout.2, prologue: prologue.clone(), epilogues, }; I::UnwindInfo::create_unwind_info(context) } + /// Generates value-label ranges. + pub fn value_labels_ranges(&self) -> crate::result::CodegenResult> { + let layout = &self.insts_layout.borrow(); + Ok(Some(debug::compute( + &self.insts, + &layout.0[..], + &layout.1[..], + ))) + } + /// Get the IR block for a BlockIndex, if one exists. pub fn bindex_to_bb(&self, block: BlockIndex) -> Option { self.block_order.lowered_order()[block as usize].orig_block() diff --git a/cranelift/codegen/src/value_label.rs b/cranelift/codegen/src/value_label.rs index e3daeb0f7a..3d3ca2ea99 100644 --- a/cranelift/codegen/src/value_label.rs +++ b/cranelift/codegen/src/value_label.rs @@ -1,13 +1,16 @@ use crate::ir::{Function, SourceLoc, Value, ValueLabel, ValueLabelAssignments, ValueLoc}; use crate::isa::TargetIsa; +use crate::machinst::MachCompileResult; use crate::regalloc::{Context, RegDiversions}; use crate::HashMap; use alloc::collections::BTreeMap; use alloc::vec::Vec; use core::cmp::Ordering; +use core::convert::From; use core::iter::Iterator; use core::ops::Bound::*; use core::ops::Deref; +use regalloc::Reg; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -17,13 +20,31 @@ use serde::{Deserialize, Serialize}; #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ValueLocRange { /// The ValueLoc containing a ValueLabel during this range. - pub loc: ValueLoc, + pub loc: LabelValueLoc, /// The start of the range. It is an offset in the generated code. pub start: u32, /// The end of the range. It is an offset in the generated code. pub end: u32, } +/// The particular location for a value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub enum LabelValueLoc { + /// Old-backend location: RegUnit, StackSlot, or Unassigned. + ValueLoc(ValueLoc), + /// New-backend Reg. + Reg(Reg), + /// New-backend offset from stack pointer. + SPOffset(i64), +} + +impl From for LabelValueLoc { + fn from(v: ValueLoc) -> Self { + LabelValueLoc::ValueLoc(v) + } +} + /// Resulting map of Value labels and their ranges/locations. pub type ValueLabelsRanges = HashMap>; @@ -86,14 +107,18 @@ where pub fn build_value_labels_ranges( func: &Function, regalloc: &Context, + mach_compile_result: Option<&MachCompileResult>, isa: &dyn TargetIsa, ) -> ValueLabelsRanges where T: From + Deref + Ord + Copy, { - // FIXME(#1523): New-style backend does not yet have debug info. - if isa.get_mach_backend().is_some() { - return HashMap::new(); + if mach_compile_result.is_some() && mach_compile_result.unwrap().value_labels_ranges.is_some() { + return mach_compile_result + .unwrap() + .value_labels_ranges + .clone() + .unwrap(); } let values_labels = build_value_labels_index::(func); @@ -113,7 +138,7 @@ where .entry(label) .or_insert_with(Vec::new) .push(ValueLocRange { - loc, + loc: loc.into(), start: range.0, end: range.1, }); diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs index 8d73e2d1e4..d7528beef4 100644 --- a/cranelift/codegen/src/write.rs +++ b/cranelift/codegen/src/write.rs @@ -11,7 +11,7 @@ use crate::ir::{ }; use crate::isa::{RegInfo, TargetIsa}; use crate::packed_option::ReservedValue; -use crate::value_label::ValueLabelsRanges; +use crate::value_label::{LabelValueLoc, ValueLabelsRanges}; use crate::HashSet; use alloc::string::String; use alloc::vec::Vec; @@ -278,11 +278,13 @@ pub fn write_block_header( writeln!(w, "):") } -fn write_valueloc(w: &mut dyn Write, loc: ValueLoc, regs: &RegInfo) -> fmt::Result { +fn write_valueloc(w: &mut dyn Write, loc: LabelValueLoc, regs: &RegInfo) -> fmt::Result { match loc { - ValueLoc::Reg(r) => write!(w, "{}", regs.display_regunit(r)), - ValueLoc::Stack(ss) => write!(w, "{}", ss), - ValueLoc::Unassigned => write!(w, "?"), + LabelValueLoc::ValueLoc(ValueLoc::Reg(r)) => write!(w, "{}", regs.display_regunit(r)), + LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => write!(w, "{}", ss), + LabelValueLoc::ValueLoc(ValueLoc::Unassigned) => write!(w, "?"), + LabelValueLoc::Reg(r) => write!(w, "{:?}", r), + LabelValueLoc::SPOffset(off) => write!(w, "[sp+{}]", off), } } diff --git a/crates/debug/src/transform/expression.rs b/crates/debug/src/transform/expression.rs index 0a286aa616..44d2299aaf 100644 --- a/crates/debug/src/transform/expression.rs +++ b/crates/debug/src/transform/expression.rs @@ -7,7 +7,7 @@ use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; use std::rc::Rc; use wasmtime_environ::entity::EntityRef; -use wasmtime_environ::ir::{StackSlots, ValueLabel, ValueLabelsRanges, ValueLoc}; +use wasmtime_environ::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLabelsRanges, ValueLoc}; use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::{get_vmctx_value_label, DefinedFuncIndex}; use wasmtime_environ::ModuleMemoryOffset; @@ -131,27 +131,24 @@ impl CompiledExpression { const X86_64_STACK_OFFSET: i64 = 16; fn translate_loc( - loc: ValueLoc, + loc: LabelValueLoc, frame_info: Option<&FunctionFrameInfo>, isa: &dyn TargetIsa, add_stack_value: bool, ) -> Result>> { Ok(match loc { - ValueLoc::Reg(reg) if add_stack_value => { + LabelValueLoc::ValueLoc(ValueLoc::Reg(reg)) => { let machine_reg = isa.map_dwarf_register(reg)?; let mut writer = ExpressionWriter::new(); - writer.write_op_reg(machine_reg)?; + if add_stack_value { + writer.write_op_reg(machine_reg)?; + } else { + writer.write_op_breg(machine_reg)?; + writer.write_sleb128(0)?; + } Some(writer.into_vec()) } - ValueLoc::Reg(reg) => { - assert!(!add_stack_value); - let machine_reg = isa.map_dwarf_register(reg)?; - let mut writer = ExpressionWriter::new(); - writer.write_op_breg(machine_reg)?; - writer.write_sleb128(0)?; - Some(writer.into_vec()) - } - ValueLoc::Stack(ss) => { + LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => { if let Some(frame_info) = frame_info { if let Some(ss_offset) = frame_info.stack_slots[ss].offset { let mut writer = ExpressionWriter::new(); @@ -165,6 +162,27 @@ fn translate_loc( } None } + LabelValueLoc::Reg(r) => { + let machine_reg = isa.map_regalloc_reg_to_dwarf(r)?; + let mut writer = ExpressionWriter::new(); + if add_stack_value { + writer.write_op_reg(machine_reg)?; + } else { + writer.write_op_breg(machine_reg)?; + writer.write_sleb128(0)?; + } + Some(writer.into_vec()) + } + LabelValueLoc::SPOffset(off) => { + let mut writer = ExpressionWriter::new(); + writer.write_op_breg(X86_64::RSP.0)?; + writer.write_sleb128(off)?; + if !add_stack_value { + writer.write_op(gimli::constants::DW_OP_deref)?; + } + return Ok(Some(writer.into_vec())); + } + _ => None, }) } @@ -172,13 +190,13 @@ fn translate_loc( fn append_memory_deref( buf: &mut Vec, frame_info: &FunctionFrameInfo, - vmctx_loc: ValueLoc, + vmctx_loc: LabelValueLoc, isa: &dyn TargetIsa, ) -> Result { let mut writer = ExpressionWriter::new(); // FIXME for imported memory match vmctx_loc { - ValueLoc::Reg(vmctx_reg) => { + LabelValueLoc::ValueLoc(ValueLoc::Reg(vmctx_reg)) => { let reg = isa.map_dwarf_register(vmctx_reg)? as u8; writer.write_u8(gimli::constants::DW_OP_breg0.0 + reg)?; let memory_offset = match frame_info.vmctx_memory_offset() { @@ -189,7 +207,7 @@ fn append_memory_deref( }; writer.write_sleb128(memory_offset)?; } - ValueLoc::Stack(ss) => { + LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => { if let Some(ss_offset) = frame_info.stack_slots[ss].offset { writer.write_op_breg(X86_64::RBP.0)?; writer.write_sleb128(ss_offset as i64 + X86_64_STACK_OFFSET)?; @@ -207,6 +225,31 @@ fn append_memory_deref( return Ok(false); } } + LabelValueLoc::Reg(r) => { + let reg = isa.map_regalloc_reg_to_dwarf(r)? as u8; + writer.write_u8(gimli::constants::DW_OP_breg0.0 + reg)?; + let memory_offset = match frame_info.vmctx_memory_offset() { + Some(offset) => offset, + None => { + return Ok(false); + } + }; + writer.write_sleb128(memory_offset)?; + } + LabelValueLoc::SPOffset(off) => { + writer.write_op_breg(X86_64::RSP.0)?; + writer.write_sleb128(off)?; + writer.write_op(gimli::constants::DW_OP_deref)?; + writer.write_op(gimli::constants::DW_OP_consts)?; + let memory_offset = match frame_info.vmctx_memory_offset() { + Some(offset) => offset, + None => { + return Ok(false); + } + }; + writer.write_sleb128(memory_offset)?; + writer.write_op(gimli::constants::DW_OP_plus)?; + } _ => { return Ok(false); } @@ -468,7 +511,7 @@ where let _ = code_chunk; // suppresses warning for final flush } }; - }; + } // Find all landing pads by scanning bytes, do not care about // false location at this moment. // Looks hacky but it is fast; does not need to be really exact. @@ -653,7 +696,7 @@ struct CachedValueLabelRange { func_index: DefinedFuncIndex, start: usize, end: usize, - label_location: HashMap, + label_location: HashMap, } struct ValueLabelRangesBuilder<'a, 'b> { @@ -1179,7 +1222,7 @@ mod tests { fn create_mock_value_ranges() -> (ValueLabelsRanges, (ValueLabel, ValueLabel, ValueLabel)) { use std::collections::HashMap; use wasmtime_environ::entity::EntityRef; - use wasmtime_environ::ir::{ValueLoc, ValueLocRange}; + use wasmtime_environ::ir::{LabelValueLoc, ValueLoc, ValueLocRange}; let mut value_ranges = HashMap::new(); let value_0 = ValueLabel::new(0); let value_1 = ValueLabel::new(1); @@ -1187,7 +1230,7 @@ mod tests { value_ranges.insert( value_0, vec![ValueLocRange { - loc: ValueLoc::Unassigned, + loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned), start: 0, end: 25, }], @@ -1195,7 +1238,7 @@ mod tests { value_ranges.insert( value_1, vec![ValueLocRange { - loc: ValueLoc::Unassigned, + loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned), start: 5, end: 30, }], @@ -1204,12 +1247,12 @@ mod tests { value_2, vec![ ValueLocRange { - loc: ValueLoc::Unassigned, + loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned), start: 0, end: 10, }, ValueLocRange { - loc: ValueLoc::Unassigned, + loc: LabelValueLoc::ValueLoc(ValueLoc::Unassigned), start: 20, end: 30, }, diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 9e1a164675..2fc9e9ecd0 100644 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -3,8 +3,8 @@ pub mod ir { pub use cranelift_codegen::binemit::{Reloc, StackMap}; pub use cranelift_codegen::ir::{ - types, AbiParam, ArgumentPurpose, JumpTableOffsets, LibCall, Signature, SourceLoc, - StackSlots, TrapCode, Type, ValueLabel, ValueLoc, + types, AbiParam, ArgumentPurpose, JumpTableOffsets, LabelValueLoc, LibCall, Signature, + SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc, }; pub use cranelift_codegen::{ValueLabelsRanges, ValueLocRange}; } From 997fab55d5e15aa7a91ac2463057c0d21bc7672c Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Sat, 9 Jan 2021 19:21:06 -0800 Subject: [PATCH 25/55] Skip value-label analysis if no value labels are present. --- cranelift/codegen/src/machinst/lower.rs | 8 ++++++++ cranelift/codegen/src/machinst/vcode.rs | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 7cb1c8d769..89e184fd4e 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -804,6 +804,10 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } fn emit_value_label_markers_for_inst(&mut self, inst: Inst) { + if self.f.dfg.values_labels.is_none() { + return; + } + debug!( "value labeling: srcloc {}: inst {}", self.srcloc(inst), @@ -815,6 +819,10 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } fn emit_value_label_markers_for_block_args(&mut self, block: Block) { + if self.f.dfg.values_labels.is_none() { + return; + } + debug!("value labeling: block {}", block); for &arg in self.f.dfg.block_params(block) { self.emit_value_label_marks_for_value(arg); diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index 6b554359b6..9fc46d9655 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -118,6 +118,10 @@ pub struct VCode { /// Constants. constants: VCodeConstants, + + /// Are any debug value-labels present? If not, we can skip the + /// post-emission analysis. + has_value_labels: bool, } /// A builder for a VCode function body. This builder is designed for the @@ -251,6 +255,9 @@ impl VCodeBuilder { } } } + if insn.defines_value_label().is_some() { + self.vcode.has_value_labels = true; + } self.vcode.insts.push(insn); self.vcode.srclocs.push(self.cur_srcloc); if is_safepoint { @@ -327,6 +334,7 @@ impl VCode { generate_debug_info, insts_layout: RefCell::new((vec![], vec![], 0)), constants, + has_value_labels: false, } } @@ -610,6 +618,10 @@ impl VCode { /// Generates value-label ranges. pub fn value_labels_ranges(&self) -> crate::result::CodegenResult> { + if !self.has_value_labels { + return Ok(None); + } + let layout = &self.insts_layout.borrow(); Ok(Some(debug::compute( &self.insts, From 7e12abce71e54c390efb81d7b19c0dcbdac9295a Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 21 Jan 2021 16:01:46 -0800 Subject: [PATCH 26/55] Fix a few comment typos and add a clarifying comment. --- cranelift/codegen/src/machinst/debug.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cranelift/codegen/src/machinst/debug.rs b/cranelift/codegen/src/machinst/debug.rs index be0d0a8ec7..79a7467c56 100644 --- a/cranelift/codegen/src/machinst/debug.rs +++ b/cranelift/codegen/src/machinst/debug.rs @@ -13,7 +13,7 @@ //! We track, at every program point, the correspondence between each value //! label and *all* locations in which it resides. E.g., if it is stored to the //! stack, we remember that it is in both a register and the stack slot; but if -//! the register is later overridden, then we have it just in the stack slot. +//! the register is later overwritten, then we have it just in the stack slot. //! This allows us to avoid false-positives observing loads/stores that we think //! are spillslots but really aren't. //! @@ -320,7 +320,7 @@ pub(crate) fn compute( // Initialize state at entry. block_starts.insert(0, AnalysisInfo::new()); - // Worklist: basic-block start offset. + // Worklist: label indices for basic blocks. let mut worklist = VecDeque::new(); let mut worklist_set = HashSet::new(); worklist.push_back(0); @@ -332,6 +332,8 @@ pub(crate) fn compute( let mut state = block_starts.get(&block).unwrap().clone(); trace!("at block {} -> state: {:?}", block, state); + // Iterate for each instruction in the block (we break at the first + // terminator we see). let mut iix = label_insn_iix[block as usize]; while iix < insts.len() as u32 { state.step(&insts[iix as usize]); From 55b0e8b9e528d185f085539aaef01e16c29a2d69 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 22 Jan 2021 09:55:28 -0600 Subject: [PATCH 27/55] Fix C API function name of setting max instances (#2598) Forgot the trailing `_set` at the end... --- crates/c-api/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index ae0cb6abdc..5ca7e7a3ba 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -173,6 +173,6 @@ pub extern "C" fn wasmtime_config_dynamic_memory_guard_size_set(c: &mut wasm_con } #[no_mangle] -pub extern "C" fn wasmtime_config_max_instances(c: &mut wasm_config_t, limit: usize) { +pub extern "C" fn wasmtime_config_max_instances_set(c: &mut wasm_config_t, limit: usize) { c.config.max_instances(limit); } From f54d0d05c7295e1c928a8bd2aa508ba7e68e22f5 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Fri, 22 Jan 2021 16:02:29 -0800 Subject: [PATCH 28/55] Address review comments. --- cranelift/codegen/src/machinst/debug.rs | 168 +++++++++++++++++------ crates/debug/src/transform/expression.rs | 4 +- 2 files changed, 128 insertions(+), 44 deletions(-) diff --git a/cranelift/codegen/src/machinst/debug.rs b/cranelift/codegen/src/machinst/debug.rs index 79a7467c56..4768b8d06e 100644 --- a/cranelift/codegen/src/machinst/debug.rs +++ b/cranelift/codegen/src/machinst/debug.rs @@ -24,9 +24,12 @@ use crate::machinst::*; use crate::value_label::{LabelValueLoc, ValueLabelsRanges, ValueLocRange}; use log::trace; use regalloc::{Reg, RegUsageCollector}; -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{HashMap, HashSet}; use std::hash::Hash; +/// Location of a labeled value: in a register or in a stack slot. Note that a +/// value may live in more than one location; `AnalysisInfo` maps each +/// value-label to multiple `ValueLoc`s. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] enum ValueLoc { Reg(Reg), @@ -61,13 +64,19 @@ impl ValueLoc { /// Mappings at one program point. #[derive(Clone, Debug)] struct AnalysisInfo { - // nominal SP relative to real SP. + /// Nominal SP relative to real SP. If `None`, then the offset is + /// indeterminate (i.e., we merged to the lattice 'bottom' element). This + /// should not happen in well-formed code. nominal_sp_offset: Option, + /// Forward map from labeled values to sets of locations. label_to_locs: HashMap>, + /// Reverse map for each register indicating the value it holds, if any. reg_to_label: HashMap, + /// Reverse map for each stack offset indicating the value it holds, if any. stack_to_label: HashMap, } +/// Get the registers written (mod'd or def'd) by a machine instruction. fn get_inst_writes(m: &M) -> Vec { // TODO: expose this part of regalloc.rs's interface publicly. let mut vecs = RegUsageCollector::get_empty_reg_vecs_test_framework_only(false); @@ -78,6 +87,8 @@ fn get_inst_writes(m: &M) -> Vec { } impl AnalysisInfo { + /// Create a new analysis state. This is the "top" lattice element at which + /// the fixpoint dataflow analysis starts. fn new() -> Self { AnalysisInfo { nominal_sp_offset: Some(0), @@ -87,6 +98,8 @@ impl AnalysisInfo { } } + /// Remove all locations for a given labeled value. Used when the labeled + /// value is redefined (so old values become stale). fn clear_label(&mut self, label: ValueLabel) { if let Some(locs) = self.label_to_locs.remove(&label) { for loc in locs { @@ -101,6 +114,9 @@ impl AnalysisInfo { } } } + + /// Remove a label from a register, if any. Used, e.g., if the register is + /// overwritten. fn clear_reg(&mut self, reg: Reg) { if let Some(label) = self.reg_to_label.remove(®) { if let Some(locs) = self.label_to_locs.get_mut(&label) { @@ -108,6 +124,9 @@ impl AnalysisInfo { } } } + + /// Remove a label from a stack offset, if any. Used, e.g., when the stack + /// slot is overwritten. fn clear_stack_off(&mut self, off: i64) { if let Some(label) = self.stack_to_label.remove(&off) { if let Some(locs) = self.label_to_locs.get_mut(&label) { @@ -115,6 +134,9 @@ impl AnalysisInfo { } } } + + /// Indicate that a labeled value is newly defined and its new value is in + /// `reg`. fn def_label_at_reg(&mut self, label: ValueLabel, reg: Reg) { self.clear_label(label); self.label_to_locs @@ -123,6 +145,8 @@ impl AnalysisInfo { .insert(ValueLoc::Reg(reg)); self.reg_to_label.insert(reg, label); } + + /// Process a store from a register to a stack slot (offset). fn store_reg(&mut self, reg: Reg, off: i64) { self.clear_stack_off(off); if let Some(label) = self.reg_to_label.get(®) { @@ -132,6 +156,8 @@ impl AnalysisInfo { self.stack_to_label.insert(off, *label); } } + + /// Process a load from a stack slot (offset) to a register. fn load_reg(&mut self, reg: Reg, off: i64) { self.clear_reg(reg); if let Some(&label) = self.stack_to_label.get(&off) { @@ -141,6 +167,8 @@ impl AnalysisInfo { self.reg_to_label.insert(reg, label); } } + + /// Process a move from one register to another. fn move_reg(&mut self, to: Reg, from: Reg) { self.clear_reg(to); if let Some(&label) = self.reg_to_label.get(&from) { @@ -151,6 +179,9 @@ impl AnalysisInfo { } } + /// Update the analysis state w.r.t. an instruction's effects. Given the + /// state just before `inst`, this method updates `self` to be the state + /// just after `inst`. fn step(&mut self, inst: &M) { for write in get_inst_writes(inst) { self.clear_reg(write); @@ -179,12 +210,29 @@ impl AnalysisInfo { } } +/// Trait used to implement the dataflow analysis' meet (intersect) function +/// onthe `AnalysisInfo` components. For efficiency, this is implemented as a +/// mutation on the LHS, rather than a pure functional operation. trait IntersectFrom { fn intersect_from(&mut self, other: &Self) -> IntersectResult; } + +/// Result of an intersection operation. Indicates whether the mutated LHS +/// (which becomes the intersection result) differs from the original LHS. Also +/// indicates if the value has become "empty" and should be removed from a +/// parent container, if any. struct IntersectResult { + /// Did the intersection change the LHS input (the one that was mutated into + /// the result)? This is needed to drive the fixpoint loop; when no more + /// changes occur, then we have converted. changed: bool, - remove: bool, + /// Is the resulting value "empty"? This can be used when a container, such + /// as a map, holds values of this (intersection result) type; when + /// `is_empty` is true for the merge of the values at a particular key, we + /// can remove that key from the merged (intersected) result. This is not + /// necessary for analysis correctness but reduces the memory and runtime + /// cost of the fixpoint loop. + is_empty: bool, } impl IntersectFrom for AnalysisInfo { @@ -208,7 +256,7 @@ impl IntersectFrom for AnalysisInfo { .changed; IntersectResult { changed, - remove: false, + is_empty: false, } } } @@ -218,6 +266,8 @@ where K: Copy + Eq + Hash, V: IntersectFrom, { + /// Intersection for hashmap: remove keys that are not in both inputs; + /// recursively intersect values for keys in common. fn intersect_from(&mut self, other: &Self) -> IntersectResult { let mut changed = false; let mut remove_keys = vec![]; @@ -226,29 +276,29 @@ where remove_keys.push(*k); } } - for k in remove_keys { + for k in &remove_keys { changed = true; - self.remove(&k); + self.remove(k); } - let mut remove_keys = vec![]; + remove_keys.clear(); for k in other.keys() { if let Some(v) = self.get_mut(k) { let result = v.intersect_from(other.get(k).unwrap()); changed |= result.changed; - if result.remove { + if result.is_empty { remove_keys.push(*k); } } } - for k in remove_keys { + for k in &remove_keys { changed = true; - self.remove(&k); + self.remove(k); } IntersectResult { changed, - remove: self.len() == 0, + is_empty: self.len() == 0, } } } @@ -256,6 +306,7 @@ impl IntersectFrom for HashSet where T: Copy + Eq + Hash, { + /// Intersection for hashset: just take the set intersection. fn intersect_from(&mut self, other: &Self) -> IntersectResult { let mut changed = false; let mut remove = vec![]; @@ -271,16 +322,18 @@ where IntersectResult { changed, - remove: self.len() == 0, + is_empty: self.len() == 0, } } } impl IntersectFrom for ValueLabel { + // Intersection for labeled value: remove if not equal. This is equivalent + // to a three-level lattice with top, bottom, and unordered set of + // individual labels in between. fn intersect_from(&mut self, other: &Self) -> IntersectResult { - // Remove if not equal (simple top -> values -> bottom lattice) IntersectResult { changed: false, - remove: *self != *other, + is_empty: *self != *other, } } } @@ -288,6 +341,8 @@ impl IntersectFrom for Option where T: Copy + Eq, { + /// Intersectino for Option: recursively intersect if both `Some`, else + /// `None`. fn intersect_from(&mut self, other: &Self) -> IntersectResult { let mut changed = false; if !(self.is_some() && other.is_some() && self == other) { @@ -296,15 +351,35 @@ where } IntersectResult { changed, - remove: self.is_none(), + is_empty: self.is_none(), } } } +/// Compute the value-label ranges (locations for program-point ranges for +/// labeled values) from a given `VCode` compilation result. +/// +/// In order to compute this information, we perform a dataflow analysis on the +/// machine code. To do so, and translate the results into a form usable by the +/// debug-info consumers, we need to know two additional things: +/// +/// - The machine-code layout (code offsets) of the instructions. DWARF is +/// encoded in terms of instruction *ends* (and we reason about value +/// locations at program points *after* instructions, to match this), so we +/// take an array `inst_ends`, giving us code offsets for each instruction's +/// end-point. (Note that this is one *past* the last byte; so a 4-byte +/// instruction at offset 0 has an end offset of 4.) +/// +/// - The locations of the labels to which branches will jump. Branches can tell +/// us about their targets in terms of `MachLabel`s, but we don't know where +/// those `MachLabel`s will be placed in the linear array of instructions. We +/// take the array `label_insn_index` to provide this info: for a label with +/// index `l`, `label_insn_index[l]` is the index of the instruction before +/// which that label is bound. pub(crate) fn compute( insts: &[I], inst_ends: &[u32], - label_insn_iix: &[u32], + label_insn_index: &[u32], ) -> ValueLabelsRanges { let inst_start = |idx: usize| if idx == 0 { 0 } else { inst_ends[idx - 1] }; @@ -312,7 +387,7 @@ pub(crate) fn compute( for i in 0..insts.len() { trace!(" #{} end: {} -> {:?}", i, inst_ends[i], insts[i]); } - trace!("label_insn_iix: {:?}", label_insn_iix); + trace!("label_insn_index: {:?}", label_insn_index); // Info at each block head, indexed by label. let mut block_starts: HashMap = HashMap::new(); @@ -321,26 +396,26 @@ pub(crate) fn compute( block_starts.insert(0, AnalysisInfo::new()); // Worklist: label indices for basic blocks. - let mut worklist = VecDeque::new(); + let mut worklist = Vec::new(); let mut worklist_set = HashSet::new(); - worklist.push_back(0); + worklist.push(0); worklist_set.insert(0); while !worklist.is_empty() { - let block = worklist.pop_front().unwrap(); + let block = worklist.pop().unwrap(); worklist_set.remove(&block); let mut state = block_starts.get(&block).unwrap().clone(); trace!("at block {} -> state: {:?}", block, state); // Iterate for each instruction in the block (we break at the first // terminator we see). - let mut iix = label_insn_iix[block as usize]; - while iix < insts.len() as u32 { - state.step(&insts[iix as usize]); - trace!(" -> inst #{}: {:?}", iix, insts[iix as usize]); + let mut index = label_insn_index[block as usize]; + while index < insts.len() as u32 { + state.step(&insts[index as usize]); + trace!(" -> inst #{}: {:?}", index, insts[index as usize]); trace!(" --> state: {:?}", state); - let term = insts[iix as usize].is_term(); + let term = insts[index as usize].is_term(); if term.is_term() { for succ in term.get_succs() { trace!(" SUCCESSOR block {}", succ.get()); @@ -348,7 +423,7 @@ pub(crate) fn compute( trace!(" orig state: {:?}", succ_state); if succ_state.intersect_from(&state).changed { if worklist_set.insert(succ.get()) { - worklist.push_back(succ.get()); + worklist.push(succ.get()); } trace!(" (changed)"); } @@ -356,14 +431,14 @@ pub(crate) fn compute( } else { // First time seeing this block block_starts.insert(succ.get(), state.clone()); - worklist.push_back(succ.get()); + worklist.push(succ.get()); worklist_set.insert(succ.get()); } } break; } - iix += 1; + index += 1; } } @@ -371,32 +446,41 @@ pub(crate) fn compute( // value-label locations. let mut value_labels_ranges: ValueLabelsRanges = HashMap::new(); - for block in 0..label_insn_iix.len() { - let start_iix = label_insn_iix[block]; - let end_iix = if block == label_insn_iix.len() - 1 { + for block in 0..label_insn_index.len() { + let start_index = label_insn_index[block]; + let end_index = if block == label_insn_index.len() - 1 { insts.len() as u32 } else { - label_insn_iix[block + 1] + label_insn_index[block + 1] }; let block = block as u32; let mut state = block_starts.get(&block).unwrap().clone(); - for iix in start_iix..end_iix { - let offset = inst_start(iix as usize); - let end = inst_ends[iix as usize]; - state.step(&insts[iix as usize]); + for index in start_index..end_index { + let offset = inst_start(index as usize); + let end = inst_ends[index as usize]; + state.step(&insts[index as usize]); for (label, locs) in &state.label_to_locs { - trace!(" inst {} has label {:?} -> locs {:?}", iix, label, locs); + trace!(" inst {} has label {:?} -> locs {:?}", index, label, locs); // Find an appropriate loc: a register if possible, otherwise pick the first stack // loc. let reg = locs.iter().cloned().find(|l| l.is_reg()); - let stack = locs.iter().cloned().find(|l| l.is_stack()); - if let Some(loc) = reg.or(stack) { + let loc = reg.or_else(|| locs.iter().cloned().find(|l| l.is_stack())); + if let Some(loc) = loc { let loc = LabelValueLoc::from(loc); let list = value_labels_ranges.entry(*label).or_insert_with(|| vec![]); - if list.is_empty() - || list.last().unwrap().end <= offset - || list.last().unwrap().loc != loc + // If the existing location list for this value-label is + // either empty, or has an end location that does not extend + // to the current offset, then we have to append a new + // entry. Otherwise, we can extend the current entry. + // + // Note that `end` is one past the end of the instruction; + // it appears that `end` is exclusive, so a mapping valid at + // offset 5 will have start = 5, end = 6. + if list + .last() + .map(|last| last.end <= offset || last.loc != loc) + .unwrap_or(true) { list.push(ValueLocRange { loc, diff --git a/crates/debug/src/transform/expression.rs b/crates/debug/src/transform/expression.rs index 44d2299aaf..57f3cc7a21 100644 --- a/crates/debug/src/transform/expression.rs +++ b/crates/debug/src/transform/expression.rs @@ -226,8 +226,8 @@ fn append_memory_deref( } } LabelValueLoc::Reg(r) => { - let reg = isa.map_regalloc_reg_to_dwarf(r)? as u8; - writer.write_u8(gimli::constants::DW_OP_breg0.0 + reg)?; + let reg = isa.map_regalloc_reg_to_dwarf(r)?; + writer.write_op_breg(reg)?; let memory_offset = match frame_info.vmctx_memory_offset() { Some(offset) => offset, None => { From 557a932757fd54f070a3bab0d39f578ae0d8f759 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Fri, 22 Jan 2021 16:42:35 -0800 Subject: [PATCH 29/55] Fix GitHub Actions config (actually run gdb tests on new backend). I had missed that the CI config didn't actually run the tests, because (I think) `matrix.target` is not set by default (?). All of our hosts are native x86-64, so we can just gate on OS (Ubuntu) instead. I also discovered that while I had been testing with the gdb tests locally, when *all* `debug::*` tests are run, there are two that do not pass on the new backend because of specific differences in compiled code. One is a value-lifetime issue (the value is "optimized out" at the point the breakpoint is set) and the other has to do with basic-block order (it is trying to match against hardcoded machine-code offsets which have changed). --- .github/workflows/main.yml | 4 ++-- tests/all/debug/lldb.rs | 6 +++++- tests/all/debug/translate.rs | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c954cf6595..936fd61788 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -295,8 +295,8 @@ jobs: # Test debug (DWARF) related functionality on new backend. - run: | sudo apt-get update && sudo apt-get install -y gdb - cargo test --features experimental_x64 test_debug_dwarf -- --ignored --test-threads 1 - if: matrix.target == 'x86_64-unknown-linux-gnu' + cargo test --features experimental_x64 test_debug_dwarf -- --ignored --test-threads 1 --test debug:: + if: matrix.os == 'ubuntu-latest' env: RUST_BACKTRACE: 1 diff --git a/tests/all/debug/lldb.rs b/tests/all/debug/lldb.rs index c29e757b31..7807732700 100644 --- a/tests/all/debug/lldb.rs +++ b/tests/all/debug/lldb.rs @@ -137,7 +137,11 @@ check: exited with status #[ignore] #[cfg(all( any(target_os = "linux", target_os = "macos"), - target_pointer_width = "64" + target_pointer_width = "64", + // Ignore test on new backend. The value this is looking for is + // not available at the point that the breakpoint is set when + // compiled by the new backend. + not(feature = "experimental_x64"), ))] pub fn test_debug_dwarf_ptr() -> Result<()> { let output = lldb_with_script( diff --git a/tests/all/debug/translate.rs b/tests/all/debug/translate.rs index e8ceb7bd9e..7253989d57 100644 --- a/tests/all/debug/translate.rs +++ b/tests/all/debug/translate.rs @@ -114,7 +114,11 @@ check: DW_AT_decl_line (10) #[cfg(all( any(target_os = "linux", target_os = "macos"), target_arch = "x86_64", - target_pointer_width = "64" + target_pointer_width = "64", + // Ignore test on new backend. This is a specific test with hardcoded + // offsets and the new backend compiles the return basic-block at a different + // offset, causing mismatches. + not(feature = "experimental_x64"), ))] fn test_debug_dwarf5_translate_lines() -> Result<()> { check_line_program( From a0fad6065ac3e0802a1ac26affcb53d4334d8e5b Mon Sep 17 00:00:00 2001 From: Frank Denis <124872+jedisct1@users.noreply.github.com> Date: Mon, 25 Jan 2021 16:32:58 +0100 Subject: [PATCH 30/55] Add support for the experimental wasi-crypto APIs (#2597) * Add support for the experimental wasi-crypto APIs The sole purpose of the implementation is to allow bindings and application developers to test the proposed APIs. Rust and AssemblyScript bindings are also available as examples. Like `wasi-nn`, it is currently disabled by default, and requires the `wasi-crypto` feature flag to be compiled in. * Rename the wasi-crypto/spec submodule * Add a path dependency into the submodule for wasi-crypto * Tell the publish script to vendor wasi-crypto --- .github/workflows/main.yml | 17 + .gitmodules | 3 + Cargo.lock | 869 ++++++++++++++++-- Cargo.toml | 2 + ci/run-experimental-x64-ci.sh | 1 + ci/run-wasi-crypto-example.sh | 10 + crates/misc/run-examples/src/main.rs | 3 +- crates/wasi-crypto/Cargo.toml | 22 + crates/wasi-crypto/LICENSE | 220 +++++ crates/wasi-crypto/README.md | 56 ++ crates/wasi-crypto/spec | 1 + crates/wasi-crypto/src/lib.rs | 31 + .../wiggle_interfaces/asymmetric_common.rs | 292 ++++++ .../src/wiggle_interfaces/common.rs | 150 +++ .../src/wiggle_interfaces/error.rs | 67 ++ .../src/wiggle_interfaces/key_exchange.rs | 40 + .../wasi-crypto/src/wiggle_interfaces/mod.rs | 38 + .../src/wiggle_interfaces/signatures.rs | 129 +++ .../src/wiggle_interfaces/symmetric.rs | 384 ++++++++ scripts/publish.rs | 10 +- src/commands/run.rs | 15 + 21 files changed, 2263 insertions(+), 97 deletions(-) create mode 100755 ci/run-wasi-crypto-example.sh create mode 100644 crates/wasi-crypto/Cargo.toml create mode 100644 crates/wasi-crypto/LICENSE create mode 100644 crates/wasi-crypto/README.md create mode 160000 crates/wasi-crypto/spec create mode 100644 crates/wasi-crypto/src/lib.rs create mode 100644 crates/wasi-crypto/src/wiggle_interfaces/asymmetric_common.rs create mode 100644 crates/wasi-crypto/src/wiggle_interfaces/common.rs create mode 100644 crates/wasi-crypto/src/wiggle_interfaces/error.rs create mode 100644 crates/wasi-crypto/src/wiggle_interfaces/key_exchange.rs create mode 100644 crates/wasi-crypto/src/wiggle_interfaces/mod.rs create mode 100644 crates/wasi-crypto/src/wiggle_interfaces/signatures.rs create mode 100644 crates/wasi-crypto/src/wiggle_interfaces/symmetric.rs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 936fd61788..9ce263c5cb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -274,6 +274,7 @@ jobs: --exclude lightbeam \ --exclude wasmtime-lightbeam \ --exclude wasmtime-wasi-nn \ + --exclude wasmtime-wasi-crypto \ --exclude peepmatic \ --exclude peepmatic-automata \ --exclude peepmatic-fuzzing \ @@ -351,6 +352,21 @@ jobs: env: RUST_BACKTRACE: 1 + # Build and test the wasi-crypto module. + test_wasi_crypto: + name: Test wasi-crypto module + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - run: rustup target add wasm32-wasi + - name: Install Rust + run: rustup update stable && rustup default stable + - run: ./ci/run-wasi-crypto-example.sh + env: + RUST_BACKTRACE: 1 + # Verify that cranelift's code generation is deterministic meta_determinist_check: name: Meta deterministic check @@ -459,6 +475,7 @@ jobs: --exclude lightbeam \ --exclude wasmtime-lightbeam \ --exclude wasmtime-wasi-nn \ + --exclude wasmtime-wasi-crypto \ --exclude peepmatic \ --exclude peepmatic-automata \ --exclude peepmatic-fuzzing \ diff --git a/.gitmodules b/.gitmodules index 0eb1df4e0f..ee264b99c4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "crates/wasi-nn/spec"] path = crates/wasi-nn/spec url = https://github.com/WebAssembly/wasi-nn +[submodule "crates/wasi-crypto/spec"] + path = crates/wasi-crypto/spec + url = https://github.com/WebAssembly/wasi-crypto.git diff --git a/Cargo.lock b/Cargo.lock index 98c5577566..9e4057f5b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,60 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +[[package]] +name = "aead" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" +dependencies = [ + "aes-soft", + "aesni", + "cipher", +] + +[[package]] +name = "aes-gcm" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aes-soft" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" +dependencies = [ + "cipher", + "opaque-debug", +] + +[[package]] +name = "aesni" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" +dependencies = [ + "cipher", + "opaque-debug", +] + [[package]] name = "ahash" version = "0.4.7" @@ -50,9 +104,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.35" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0df63cb2955042487fad3aefd2c6e3ae7389ac5dc1beb28921de0b69f779d4" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" [[package]] name = "arbitrary" @@ -86,6 +140,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + [[package]] name = "autocfg" version = "1.0.1" @@ -167,6 +227,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bitvec" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2838fdd79e8776dbe07a106c784b0f8dda571a21b2750a092cc4cbaa653c8e" +dependencies = [ + "funty", + "radium", + "wyz", +] + [[package]] name = "blake2b_simd" version = "0.5.11" @@ -195,9 +266,9 @@ checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" [[package]] name = "byteorder" -version = "1.3.4" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" [[package]] name = "capstone" @@ -249,6 +320,29 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chacha20" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed8738f14471a99f0e316c327e68fc82a3611cc2895fcb604b89eedaf8f39d95" +dependencies = [ + "cipher", + "zeroize", +] + +[[package]] +name = "chacha20poly1305" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1fc18e6d90c40164bf6c317476f2a98f04661e310e79830366b7e914c58a8e" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "chrono" version = "0.4.19" @@ -262,6 +356,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + [[package]] name = "clang-sys" version = "1.0.3" @@ -299,9 +402,9 @@ dependencies = [ [[package]] name = "console" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50aab2529019abfabfa93f1e6c41ef392f91fbf179b347a7e96abb524884a08" +checksum = "7cc80946b3480f421c2f17ed1cb841753a371c7c5104f51d507e13f532c856aa" dependencies = [ "encode_unicode", "lazy_static", @@ -310,14 +413,19 @@ dependencies = [ "terminal_size", "unicode-width", "winapi", - "winapi-util", ] [[package]] -name = "const_fn" -version = "0.4.4" +name = "const-oid" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" +checksum = "c5d82796b70971fbb603900a5edc797a4d9be0f9ec1257f83a1dba0aa374e3e9" + +[[package]] +name = "const_fn" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" [[package]] name = "constant_time_eq" @@ -351,6 +459,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +[[package]] +name = "cpuid-bool" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" + [[package]] name = "cranelift" version = "0.69.0" @@ -645,11 +759,43 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" dependencies = [ - "autocfg", + "autocfg 1.0.1", "cfg-if 1.0.0", "lazy_static", ] +[[package]] +name = "crypto-mac" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "ctr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" +dependencies = [ + "cipher", +] + +[[package]] +name = "curve25519-dalek" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f627126b946c25a4638eec0ea634fc52506dea98db118aae985118ce7c3d723f" +dependencies = [ + "byteorder", + "digest", + "rand_core 0.5.1", + "subtle", + "zeroize", +] + [[package]] name = "cvt" version = "0.1.1" @@ -659,6 +805,26 @@ dependencies = [ "cfg-if 0.1.10", ] +[[package]] +name = "der" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f59c66c30bb7445c8320a5f9233e437e3572368099f25532a59054328899b4" +dependencies = [ + "const-oid", +] + +[[package]] +name = "derivative" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaed5874effa6cde088c644ddcdcb4ffd1511391c5be4fdd7a5ccd02c7e4a183" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive_arbitrary" version = "0.4.7" @@ -683,9 +849,9 @@ dependencies = [ [[package]] name = "derive_utils" -version = "0.11.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64196eb9f551916167225134f1e8a90f0b5774331d3c900d6328fd94bafe3544" +checksum = "532b4c15dccee12c7044f1fcad956e98410860b22231e44a3b827464797ca7bf" dependencies = [ "proc-macro2", "quote", @@ -744,18 +910,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" dependencies = [ "libc", - "redox_users", + "redox_users 0.3.5", "winapi", ] [[package]] name = "dirs-sys-next" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99de365f605554ae33f115102a02057d4fc18b01f3284d6870be0938743cfe7d" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.0", "winapi", ] @@ -767,9 +933,9 @@ checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" [[package]] name = "dynasm" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62a59fbab09460c1569eeea9b5e4cf62f13f5198b1c2ba0e5196dd7fdd17cd42" +checksum = "3d7d1242462849390bb2ad38aeed769499f1afc7383affa2ab0c1baa894c0200" dependencies = [ "bitflags", "byteorder", @@ -782,13 +948,47 @@ dependencies = [ [[package]] name = "dynasmrt" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bec3edae2841d37b1c3dc7f3fd403c9061f26e9ffeeee97a3ea909b1bb2ef1" +checksum = "c1dd4d1d5ca12258cef339a57a7643e8b233a42dea9bb849630ddd9dd7726aa9" dependencies = [ "byteorder", "dynasm", - "memmap", + "memmap2", +] + +[[package]] +name = "ecdsa" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fbdb4ff710acb4db8ca29f93b897529ea6d6a45626d5183b47e012aa6ae7e4" +dependencies = [ + "elliptic-curve", + "hmac", + "signature", +] + +[[package]] +name = "ed25519" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37c66a534cbb46ab4ea03477eae19d5c22c01da8258030280b7bd9d8433fb6ef" +dependencies = [ + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand 0.7.3", + "serde", + "sha2", + "zeroize", ] [[package]] @@ -797,6 +997,23 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "elliptic-curve" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "592b1c857559479c056b73a3053c717108a70e4dce320ad28c79c63f5c2e62ba" +dependencies = [ + "bitvec", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.5.1", + "subtle", + "zeroize", +] + [[package]] name = "encode_unicode" version = "0.3.6" @@ -823,7 +1040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" dependencies = [ "atty", - "humantime 2.0.1", + "humantime 2.1.0", "log", "regex", "termcolor", @@ -868,6 +1085,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "ff" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01646e077d4ebda82b73f1bca002ea1e91561a77df2431a9e79729bcc31950ef" +dependencies = [ + "bitvec", + "rand_core 0.5.1", + "subtle", +] + [[package]] name = "file-per-thread-logger" version = "0.1.4" @@ -912,6 +1140,12 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d79238883cf0307100b90aba4a755d8051a3182305dfe7f649a1e9dc0517006f" +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + [[package]] name = "gcc" version = "0.3.55" @@ -930,24 +1164,34 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] [[package]] name = "getrandom" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee8025cf36f917e6a52cce185b7c7177689b838b7ec138364e50cc2277a56cf4" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "wasi 0.9.0+wasi-snapshot-preview1", + "wasi 0.10.1+wasi-snapshot-preview1", +] + +[[package]] +name = "ghash" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" +dependencies = [ + "opaque-debug", + "polyval", ] [[package]] @@ -967,6 +1211,17 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "group" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc11f9f5fbf1943b48ae7c2bf6846e7d827a512d1be4f23af708f5ca5d01dde1" +dependencies = [ + "ff", + "rand_core 0.5.1", + "subtle", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -978,22 +1233,42 @@ dependencies = [ [[package]] name = "heck" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" dependencies = [ "unicode-segmentation", ] [[package]] name = "hermit-abi" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] +[[package]] +name = "hkdf" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "hmac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +dependencies = [ + "crypto-mac", + "digest", +] + [[package]] name = "humantime" version = "1.3.0" @@ -1005,9 +1280,9 @@ dependencies = [ [[package]] name = "humantime" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "id-arena" @@ -1021,7 +1296,7 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" dependencies = [ - "autocfg", + "autocfg 1.0.1", "hashbrown", "serde", ] @@ -1039,10 +1314,19 @@ dependencies = [ ] [[package]] -name = "iter-enum" -version = "0.2.5" +name = "instant" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86a94bc12a53bf84b705acee29eb8697a5ea7b4587d836152499e5db0a6d52b9" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iter-enum" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad34f24d3b48ceffdff38af2df5ce1b7d1d9cc113e503d8e86fe8cdb889c871" dependencies = [ "derive_utils", "quote", @@ -1069,9 +1353,9 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "ittapi-rs" @@ -1091,11 +1375,25 @@ dependencies = [ "libc", ] +[[package]] +name = "k256" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf02ecc966e1b7e8db1c81ac8f321ba24d1cfab5b634961fab10111f015858e1" +dependencies = [ + "cfg-if 1.0.0", + "ecdsa", + "elliptic-curve", +] + [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin", +] [[package]] name = "lazycell" @@ -1127,14 +1425,20 @@ dependencies = [ [[package]] name = "libloading" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9367bdfa836b7e3cf895867f7a570283444da90562980ec2263d6e1569b16bc" +checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883" dependencies = [ "cfg-if 1.0.0", "winapi", ] +[[package]] +name = "libm" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" + [[package]] name = "lightbeam" version = "0.22.0" @@ -1160,10 +1464,19 @@ dependencies = [ ] [[package]] -name = "log" -version = "0.4.11" +name = "lock_api" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" dependencies = [ "cfg-if 0.1.10", ] @@ -1202,13 +1515,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "memmap2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e73be3b7d04a0123e933fea1d50d126cc7196bbc0362c0ce426694f777194eee" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" dependencies = [ - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -1224,7 +1546,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" dependencies = [ "adler", - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -1249,18 +1571,59 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ - "autocfg", + "autocfg 1.0.1", "num-integer", "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e9a41747ae4633fce5adffb4d2e81ffc5e89593cb19917f8fb2cc5ff76507bf" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d51546d704f52ef14b3c962b5776e53d5b862e5790e40a350d366c209bd7f7a" +dependencies = [ + "autocfg 0.1.7", + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.7.3", + "serde", + "smallvec", + "zeroize", +] + [[package]] name = "num-integer" version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg", + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg 1.0.1", + "num-integer", "num-traits", ] @@ -1270,8 +1633,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" dependencies = [ - "autocfg", - "num-bigint", + "autocfg 1.0.1", + "num-bigint 0.2.6", "num-integer", "num-traits", ] @@ -1282,7 +1645,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -1325,9 +1688,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openvino" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87a74f90f07f134153e3ad2ffa724a3ebda92cdc6e099f7fe7d9185cf960f028" +checksum = "43eeb44285b7ce8e2012b92bec32968622e1dad452e812e6edea9e001e5e9410" dependencies = [ "openvino-sys", "thiserror", @@ -1335,9 +1698,9 @@ dependencies = [ [[package]] name = "openvino-sys" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72a2e5bd353bd3cf39b2663767e0ae0325a7588c47fd496cbf9a09237ef7ca8" +checksum = "8fb64bef270a1ff665b0b2e28ebfa213e6205a007ce88223d020730225d6008f" dependencies = [ "bindgen", "cmake", @@ -1353,12 +1716,48 @@ dependencies = [ "winapi", ] +[[package]] +name = "p256" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca0196a204bb3f33305ba4a48b38f6e6e621cba8603a4e0650e6532e0949de4" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "parity-wasm" version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.1.57", + "smallvec", + "winapi", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -1470,10 +1869,53 @@ name = "peepmatic-traits" version = "0.69.0" [[package]] -name = "pin-project-lite" -version = "0.2.0" +name = "pem" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +checksum = "f4c220d01f863d13d96ca82359d1e81e64a7c6bf0637bcde7b2349630addf0c6" +dependencies = [ + "base64", + "once_cell", + "regex", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" + +[[package]] +name = "pkcs8" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4839a901843f3942576e65857f0ebf2e190ef7024d3c62a94099ba3f819ad1d" +dependencies = [ + "der", + "subtle-encoding", + "zeroize", +] + +[[package]] +name = "poly1305" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b7456bc1ad2d4cf82b3a016be4c2ac48daf11bf990c1603ebd447fe6f30fca8" +dependencies = [ + "cpuid-bool 0.2.0", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" +dependencies = [ + "cpuid-bool 0.2.0", + "opaque-debug", + "universal-hash", +] [[package]] name = "ppv-lite86" @@ -1481,6 +1923,34 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "pqcrypto" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3874384bf37d988b83f806d632e2f7fca69a8cd0338efaa64e8e7664573052" +dependencies = [ + "pqcrypto-kyber", + "pqcrypto-traits", +] + +[[package]] +name = "pqcrypto-kyber" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33550a5b6e0844d1b2363f67e15e4ca64586bb4fb2363a83af762e6c2d092bff" +dependencies = [ + "cc", + "glob", + "libc", + "pqcrypto-traits", +] + +[[package]] +name = "pqcrypto-traits" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e1563eff60a9ae869cacee0a33fa5c4ba27861fec6e3e23de95eb0ae805e4b" + [[package]] name = "pretty_env_logger" version = "0.4.0" @@ -1579,13 +2049,19 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64de9a0c5361e034f1aefc9f71a86871ec870e766fe31a009734a989b329286a" + [[package]] name = "rand" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -1630,7 +2106,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", ] [[package]] @@ -1639,7 +2115,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" dependencies = [ - "getrandom 0.2.0", + "getrandom 0.2.2", ] [[package]] @@ -1680,13 +2156,19 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "rawbytes" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26d81f4c222fd11ad63bf56cbda89d1810aecf1a720a423ff7eb2020475d8bb" + [[package]] name = "rayon" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" dependencies = [ - "autocfg", + "autocfg 1.0.1", "crossbeam-deque", "either", "rayon-core", @@ -1726,11 +2208,21 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", "redox_syscall 0.1.57", "rust-argon2", ] +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom 0.2.2", + "redox_syscall 0.2.4", +] + [[package]] name = "regalloc" version = "0.0.31" @@ -1745,9 +2237,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ "aho-corasick", "memchr", @@ -1767,9 +2259,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" [[package]] name = "region" @@ -1792,6 +2284,40 @@ dependencies = [ "winapi", ] +[[package]] +name = "rsa" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3648b669b10afeab18972c105e284a7b953a669b0be3514c27f9b17acab2f9cd" +dependencies = [ + "byteorder", + "digest", + "lazy_static", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pem", + "rand 0.7.3", + "sha2", + "simple_asn1 0.4.1", + "subtle", + "thiserror", + "zeroize", +] + +[[package]] +name = "rsa-export" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "358de25c89a5a71597ebc85f7ad222e2c67ed553e0ce31170104c3a77296a01c" +dependencies = [ + "num-bigint-dig", + "pem", + "rsa", + "simple_asn1 0.5.1", +] + [[package]] name = "run-examples" version = "0.19.0" @@ -1923,9 +2449,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" +checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" dependencies = [ "itoa", "ryu", @@ -1934,9 +2460,9 @@ dependencies = [ [[package]] name = "serde_test" -version = "1.0.118" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f3f8714511d29f60be0ea965bc784df1b6903da5bbac801df36b1bbc7b4880" +checksum = "3dd7d96489b14fa2f4a89be299ac117c8023d1ead9aaee963a2dde72dad4d14b" dependencies = [ "serde", ] @@ -1949,7 +2475,7 @@ checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" dependencies = [ "block-buffer", "cfg-if 1.0.0", - "cpuid-bool", + "cpuid-bool 0.1.2", "digest", "opaque-debug", ] @@ -1980,16 +2506,49 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] name = "shuffling-allocator" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848c0a454373d16ebfaa740c99d4faebe25ea752a35e0c6e341168fe67f987f8" +checksum = "4ee9977fa98489d9006f4ab26fc5cbe2a139985baed09d2ec08dee6e506fc496" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.7.3", + "rand 0.8.2", "winapi", ] +[[package]] +name = "signature" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29f060a7d147e33490ec10da418795238fd7545bba241504d6b31a409f2e6210" +dependencies = [ + "digest", + "rand_core 0.5.1", +] + +[[package]] +name = "simple_asn1" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b" +dependencies = [ + "chrono", + "num-bigint 0.2.6", + "num-traits", +] + +[[package]] +name = "simple_asn1" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8d597fce66eb0f19dd129b9956e4054cba21aeaf97d4116595027b670fac50" +dependencies = [ + "chrono", + "num-bigint 0.3.1", + "num-traits", + "thiserror", +] + [[package]] name = "smallvec" version = "1.6.1" @@ -2005,6 +2564,12 @@ dependencies = [ "id-arena", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2041,6 +2606,21 @@ dependencies = [ "syn", ] +[[package]] +name = "subtle" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" + +[[package]] +name = "subtle-encoding" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcb1ed7b8330c5eed5441052651dd7a12c75e2ed88f2ec024ae1fa3a5e59945" +dependencies = [ + "zeroize", +] + [[package]] name = "syn" version = "1.0.58" @@ -2052,6 +2632,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + [[package]] name = "target-lexicon" version = "0.11.1" @@ -2093,9 +2685,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd2d183bd3fac5f5fe38ddbeb4dc9aec4a39a9d7d59e7491d900302da01cbe1" +checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406" dependencies = [ "libc", "winapi", @@ -2128,18 +2720,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" dependencies = [ "proc-macro2", "quote", @@ -2148,29 +2740,28 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +checksum = "bb9bc092d0d51e76b2b19d9d85534ffc9ec2db959a2523cdae0697e2972cd447" dependencies = [ "lazy_static", ] [[package]] name = "time" -version = "0.1.44" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] [[package]] name = "toml" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cf45bb0bef80604d001caaec0d09da99611b3c0fd39d3080468875cdb65645" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" dependencies = [ "serde", ] @@ -2290,6 +2881,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "universal-hash" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "unsafe-any" version = "0.4.2" @@ -2339,9 +2940,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +version = "0.10.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "93c6c3420963c5c64bca373b25e77acb562081b9bb4dd5bb864187742186cea9" [[package]] name = "wasi-common" @@ -2351,7 +2952,7 @@ dependencies = [ "cfg-if 1.0.0", "cpu-time", "filetime", - "getrandom 0.2.0", + "getrandom 0.2.2", "lazy_static", "libc", "thiserror", @@ -2362,6 +2963,35 @@ dependencies = [ "yanix", ] +[[package]] +name = "wasi-crypto" +version = "0.1.4" +dependencies = [ + "aes-gcm", + "anyhow", + "bincode", + "byteorder", + "chacha20poly1305", + "curve25519-dalek", + "derivative", + "ed25519-dalek", + "hkdf", + "hmac", + "k256", + "p256", + "parking_lot", + "pqcrypto", + "rand_core 0.5.1", + "rsa", + "rsa-export", + "serde", + "sha2", + "subtle", + "thiserror", + "xoodyak", + "zeroize", +] + [[package]] name = "wasm-encoder" version = "0.4.0" @@ -2518,7 +3148,7 @@ dependencies = [ "env_logger 0.8.2", "file-per-thread-logger", "filecheck", - "humantime 2.0.1", + "humantime 2.1.0", "libc", "log", "more-asserts", @@ -2541,6 +3171,7 @@ dependencies = [ "wasmtime-obj", "wasmtime-runtime", "wasmtime-wasi", + "wasmtime-wasi-crypto", "wasmtime-wasi-nn", "wasmtime-wast", "wat", @@ -2747,6 +3378,17 @@ dependencies = [ "wiggle", ] +[[package]] +name = "wasmtime-wasi-crypto" +version = "0.22.0" +dependencies = [ + "anyhow", + "wasi-crypto", + "wasmtime", + "wasmtime-wiggle", + "wiggle", +] + [[package]] name = "wasmtime-wasi-nn" version = "0.22.0" @@ -2940,6 +3582,22 @@ dependencies = [ "wast 22.0.0", ] +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + +[[package]] +name = "xoodyak" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f9c85605c3a376cec858899f7d284f453359743adaeddf09c7d6ef18474a481" +dependencies = [ + "rawbytes", + "zeroize", +] + [[package]] name = "yanix" version = "0.22.0" @@ -2971,6 +3629,27 @@ dependencies = [ "cmake", ] +[[package]] +name = "zeroize" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a974bcdd357f0dca4d41677db03436324d45a4c9ed2d0b873a5a360ce41c36" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f369ddb18862aba61aa49bf31e74d29f0f162dec753063200e1dc084345d16" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zstd" version = "0.6.0+zstd.1.4.8" diff --git a/Cargo.toml b/Cargo.toml index 3221971b18..c1d0cc8ed7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ wasmtime-jit = { path = "crates/jit", version = "0.22.0" } wasmtime-obj = { path = "crates/obj", version = "0.22.0" } wasmtime-wast = { path = "crates/wast", version = "0.22.0" } wasmtime-wasi = { path = "crates/wasi", version = "0.22.0" } +wasmtime-wasi-crypto = { path = "crates/wasi-crypto", version = "0.22.0", optional = true } wasmtime-wasi-nn = { path = "crates/wasi-nn", version = "0.22.0", optional = true } wasi-common = { path = "crates/wasi-common", version = "0.22.0" } structopt = { version = "0.3.5", features = ["color", "suggestions"] } @@ -82,6 +83,7 @@ default = ["jitdump", "wasmtime/wat", "wasmtime/parallel-compilation"] lightbeam = ["wasmtime/lightbeam"] jitdump = ["wasmtime/jitdump"] vtune = ["wasmtime/vtune"] +wasi-crypto = ["wasmtime-wasi-crypto"] wasi-nn = ["wasmtime-wasi-nn"] # Try the experimental, work-in-progress new x86_64 backend. This is not stable diff --git a/ci/run-experimental-x64-ci.sh b/ci/run-experimental-x64-ci.sh index 8b41831b15..6b11352d47 100755 --- a/ci/run-experimental-x64-ci.sh +++ b/ci/run-experimental-x64-ci.sh @@ -14,6 +14,7 @@ cargo $CARGO_VERSION \ --all \ --exclude wasmtime-lightbeam \ --exclude wasmtime-wasi-nn \ + --exclude wasmtime-wasi-crypto \ --exclude peepmatic \ --exclude peepmatic-automata \ --exclude peepmatic-fuzzing \ diff --git a/ci/run-wasi-crypto-example.sh b/ci/run-wasi-crypto-example.sh new file mode 100755 index 0000000000..d2582c71b1 --- /dev/null +++ b/ci/run-wasi-crypto-example.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +set -e + +RUST_BINDINGS="crates/wasi-crypto/spec/implementations/bindings/rust" +pushd "$RUST_BINDINGS" +cargo build --release --target=wasm32-wasi +popd + +cargo run --features wasi-crypto -- run "$RUST_BINDINGS/target/wasm32-wasi/release/wasi-crypto-guest.wasm" diff --git a/crates/misc/run-examples/src/main.rs b/crates/misc/run-examples/src/main.rs index 94e5b56dc9..12746b2a38 100644 --- a/crates/misc/run-examples/src/main.rs +++ b/crates/misc/run-examples/src/main.rs @@ -83,7 +83,8 @@ fn main() -> anyhow::Result<()> { .arg("userenv.lib") .arg("ntdll.lib") .arg("shell32.lib") - .arg("ole32.lib"); + .arg("ole32.lib") + .arg("bcrypt.lib"); if is_dir { "main.exe".to_string() } else { diff --git a/crates/wasi-crypto/Cargo.toml b/crates/wasi-crypto/Cargo.toml new file mode 100644 index 0000000000..a0df36082d --- /dev/null +++ b/crates/wasi-crypto/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "wasmtime-wasi-crypto" +version = "0.22.0" +authors = ["The Wasmtime Project Developers"] +description = "Wasmtime implementation of the wasi-crypto API" +documentation = "https://docs.rs/wasmtime-wasi-crypto" +license = "Apache-2.0 WITH LLVM-exception" +categories = ["wasm", "cryptography"] +keywords = ["webassembly", "wasm", "crypto"] +repository = "https://github.com/bytecodealliance/wasmtime" +readme = "README.md" +edition = "2018" + +[dependencies] +anyhow = "1.0" +wasi-crypto = { path = "spec/implementations/hostcalls/rust", version = "0.1.4" } +wasmtime = { path = "../wasmtime", version = "0.22.0", default-features = false } +wasmtime-wiggle = { path = "../wiggle/wasmtime", version = "0.22.0" } +wiggle = { path = "../wiggle", version = "0.22.0" } + +[badges] +maintenance = { status = "experimental" } diff --git a/crates/wasi-crypto/LICENSE b/crates/wasi-crypto/LICENSE new file mode 100644 index 0000000000..f9d81955f4 --- /dev/null +++ b/crates/wasi-crypto/LICENSE @@ -0,0 +1,220 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + diff --git a/crates/wasi-crypto/README.md b/crates/wasi-crypto/README.md new file mode 100644 index 0000000000..ca8766e1a0 --- /dev/null +++ b/crates/wasi-crypto/README.md @@ -0,0 +1,56 @@ +# wasmtime-wasi-crypto + +This crate enables support for the [wasi-crypto] APIs in Wasmtime. + +The sole purpose of the implementation is to allow bindings and +application developers to test the proposed APIs. This implementation +is not meant to be used in production. Like the specification, it is +currently experimental and its functionality can quickly change. + +Since the [wasi-crypto] API is expected to be an optional feature of +WASI, this crate is currently separate from the [wasi-common] crate. + +* [documentation] +* [interfaces reference] +* [interfaces reference (compact)] + +[wasi-crypto]: https://github.com/WebAssembly/wasi-crypto +[wasi-common]: ../../wasi-common +[documentation]: ../spec/docs/wasi-crypto.md +[interfaces reference]: ../spec/witx/wasi_ephemeral_crypto.md +[interfaces reference (compact)]: ../spec/witx/wasi_ephemeral_crypto.txt + +## Wasmtime integration + +Use the Wasmtime APIs to instantiate a Wasm module and link the +`wasi-crypto` modules as follows: + +```rust +use wasmtime_wasi_crypto::{ + WasiCryptoAsymmetricCommon, WasiCryptoCommon, WasiCryptoCtx, WasiCryptoSignatures, + WasiCryptoSymmetric, +}; + +let cx_crypto = WasiCryptoCtx::new(); +WasiCryptoCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; +WasiCryptoAsymmetricCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; +WasiCryptoSignatures::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; +WasiCryptoSymmetric::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; + +let wasi = wasmtime_wasi::old::snapshot_0::Wasi::new(linker.store(), mk_cx()?); +wasi.add_to_linker(linker)?; +``` + +## Building Wasmtime + +Wasmtime must be compiled with the `wasi-crypto` feature flag +(disabled by default) in order to include the crypto APIs. + +## Examples + +Example [rust bindings] and [assemblyscript bindings] are provided to +demonstrate how these APIs can be used and exposed to applications in +an idiomatic way. + +[rust bindings]: ../spec/implementations/bindings/rust +[assemblyscript bindings]: ../spec/implementations/bindings/assemblyscript diff --git a/crates/wasi-crypto/spec b/crates/wasi-crypto/spec new file mode 160000 index 0000000000..6d7821dec3 --- /dev/null +++ b/crates/wasi-crypto/spec @@ -0,0 +1 @@ +Subproject commit 6d7821dec301a11dcf3c0d50e5a51af5169eaee3 diff --git a/crates/wasi-crypto/src/lib.rs b/crates/wasi-crypto/src/lib.rs new file mode 100644 index 0000000000..42a21f36d4 --- /dev/null +++ b/crates/wasi-crypto/src/lib.rs @@ -0,0 +1,31 @@ +mod wiggle_interfaces; + +pub use wiggle_interfaces::WasiCryptoCtx; + +wasmtime_wiggle::wasmtime_integration!({ + target: wiggle_interfaces::wasi_modules, + witx: ["$CARGO_MANIFEST_DIR/spec/witx/wasi_ephemeral_crypto.witx"], + ctx: WasiCryptoCtx, + modules: { + wasi_ephemeral_crypto_common => + { + name: WasiCryptoCommon, + docs: "wasi-crypto - Common module." + }, + wasi_ephemeral_crypto_asymmetric_common => + { + name: WasiCryptoAsymmetricCommon, + docs: "wasi-crypto - Common module for asymmetric operations." + }, + wasi_ephemeral_crypto_signatures => + { + name: WasiCryptoSignatures, + docs: "wasi-crypto - Signature module." + }, + wasi_ephemeral_crypto_symmetric => + { + name: WasiCryptoSymmetric, + docs: "wasi-crypto - Symmetric cryptography module." + } + } +}); diff --git a/crates/wasi-crypto/src/wiggle_interfaces/asymmetric_common.rs b/crates/wasi-crypto/src/wiggle_interfaces/asymmetric_common.rs new file mode 100644 index 0000000000..8de31ee429 --- /dev/null +++ b/crates/wasi-crypto/src/wiggle_interfaces/asymmetric_common.rs @@ -0,0 +1,292 @@ +use super::{guest_types, WasiCryptoCtx}; + +use std::convert::TryInto; +use wasi_crypto::{ensure, CryptoError, KeyPairEncoding, PublicKeyEncoding, SecretKeyEncoding}; + +impl super::wasi_ephemeral_crypto_asymmetric_common::WasiEphemeralCryptoAsymmetricCommon + for WasiCryptoCtx +{ + // --- keypair_manager + + fn keypair_generate_managed( + &self, + secrets_manager_handle: guest_types::SecretsManager, + alg_type: guest_types::AlgorithmType, + alg_str: &wiggle::GuestPtr<'_, str>, + options_handle: &guest_types::OptOptions, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let options_handle = match *options_handle { + guest_types::OptOptions::Some(options_handle) => Some(options_handle), + guest_types::OptOptions::None => None, + }; + Ok(self + .ctx + .keypair_generate_managed( + secrets_manager_handle.into(), + alg_type.into(), + alg_str, + options_handle.map(Into::into), + )? + .into()) + } + + fn keypair_store_managed( + &self, + secrets_manager_handle: guest_types::SecretsManager, + kp_handle: guest_types::Keypair, + kp_id_ptr: &wiggle::GuestPtr<'_, u8>, + kp_id_max_len: guest_types::Size, + ) -> Result<(), guest_types::CryptoErrno> { + let key_id_buf = &mut *kp_id_ptr.as_array(kp_id_max_len).as_slice_mut()?; + Ok(self.ctx.keypair_store_managed( + secrets_manager_handle.into(), + kp_handle.into(), + key_id_buf, + )?) + } + + fn keypair_replace_managed( + &self, + secrets_manager_handle: guest_types::SecretsManager, + kp_old_handle: guest_types::Keypair, + kp_new_handle: guest_types::Keypair, + ) -> Result { + Ok(self + .ctx + .keypair_replace_managed( + secrets_manager_handle.into(), + kp_old_handle.into(), + kp_new_handle.into(), + )? + .into()) + } + + fn keypair_from_id( + &self, + secrets_manager_handle: guest_types::SecretsManager, + kp_id_ptr: &wiggle::GuestPtr<'_, u8>, + kp_id_len: guest_types::Size, + kp_version: guest_types::Version, + ) -> Result { + let kp_id = &*kp_id_ptr.as_array(kp_id_len).as_slice()?; + Ok(self + .ctx + .keypair_from_id(secrets_manager_handle.into(), kp_id, kp_version.into())? + .into()) + } + + // --- keypair + + fn keypair_generate( + &self, + alg_type: guest_types::AlgorithmType, + alg_str: &wiggle::GuestPtr<'_, str>, + options_handle: &guest_types::OptOptions, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let options_handle = match *options_handle { + guest_types::OptOptions::Some(options_handle) => Some(options_handle), + guest_types::OptOptions::None => None, + }; + Ok(self + .ctx + .keypair_generate(alg_type.into(), alg_str, options_handle.map(Into::into))? + .into()) + } + + fn keypair_import( + &self, + alg_type: guest_types::AlgorithmType, + alg_str: &wiggle::GuestPtr<'_, str>, + encoded_ptr: &wiggle::GuestPtr<'_, u8>, + encoded_len: guest_types::Size, + encoding: guest_types::KeypairEncoding, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let encoded = &*encoded_ptr.as_array(encoded_len).as_slice()?; + Ok(self + .ctx + .keypair_import(alg_type.into(), alg_str, encoded, encoding.into())? + .into()) + } + + fn keypair_id( + &self, + kp_handle: guest_types::Keypair, + kp_id_ptr: &wiggle::GuestPtr<'_, u8>, + kp_id_max_len: guest_types::Size, + ) -> Result<(guest_types::Size, guest_types::Version), guest_types::CryptoErrno> { + let kp_id_buf = &mut *kp_id_ptr.as_array(kp_id_max_len as _).as_slice_mut()?; + let (kp_id, version) = self.ctx.keypair_id(kp_handle.into())?; + ensure!(kp_id.len() <= kp_id_buf.len(), CryptoError::Overflow.into()); + kp_id_buf.copy_from_slice(&kp_id); + Ok((kp_id.len().try_into()?, version.into())) + } + + fn keypair_export( + &self, + kp_handle: guest_types::Keypair, + encoding: guest_types::KeypairEncoding, + ) -> Result { + Ok(self + .ctx + .keypair_export(kp_handle.into(), encoding.into())? + .into()) + } + + fn keypair_publickey( + &self, + kp_handle: guest_types::Keypair, + ) -> Result { + Ok(self.ctx.keypair_publickey(kp_handle.into())?.into()) + } + + fn keypair_close( + &self, + kp_handle: guest_types::Keypair, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self.ctx.keypair_close(kp_handle.into())?) + } + + // --- publickey + + fn publickey_import( + &self, + alg_type: guest_types::AlgorithmType, + alg_str: &wiggle::GuestPtr<'_, str>, + encoded_ptr: &wiggle::GuestPtr<'_, u8>, + encoded_len: guest_types::Size, + encoding: guest_types::PublickeyEncoding, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let encoded = &*encoded_ptr.as_array(encoded_len).as_slice()?; + Ok(self + .ctx + .publickey_import(alg_type.into(), alg_str, encoded, encoding.into())? + .into()) + } + + fn publickey_export( + &self, + pk_handle: guest_types::Publickey, + encoding: guest_types::PublickeyEncoding, + ) -> Result { + Ok(self + .ctx + .publickey_export(pk_handle.into(), encoding.into())? + .into()) + } + + fn publickey_from_secretkey( + &self, + sk_handle: guest_types::Secretkey, + ) -> Result { + Ok(self.ctx.keypair_publickey(sk_handle.into())?.into()) + } + + fn publickey_verify( + &self, + pk_handle: guest_types::Publickey, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self.ctx.publickey_verify(pk_handle.into())?) + } + + fn publickey_close( + &self, + pk_handle: guest_types::Publickey, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self.ctx.publickey_close(pk_handle.into())?) + } + + // --- secretkey + + fn secretkey_import( + &self, + alg_type: guest_types::AlgorithmType, + alg_str: &wiggle::GuestPtr<'_, str>, + encoded_ptr: &wiggle::GuestPtr<'_, u8>, + encoded_len: guest_types::Size, + encoding: guest_types::SecretkeyEncoding, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let encoded = &*encoded_ptr.as_array(encoded_len).as_slice()?; + Ok(self + .ctx + .secretkey_import(alg_type.into(), alg_str, encoded, encoding.into())? + .into()) + } + + fn secretkey_export( + &self, + sk_handle: guest_types::Secretkey, + encoding: guest_types::SecretkeyEncoding, + ) -> Result { + Ok(self + .ctx + .secretkey_export(sk_handle.into(), encoding.into())? + .into()) + } + + fn secretkey_close( + &self, + sk_handle: guest_types::Secretkey, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self.ctx.secretkey_close(sk_handle.into())?) + } + + fn keypair_from_pk_and_sk( + &self, + pk_handle: guest_types::Publickey, + sk_handle: guest_types::Secretkey, + ) -> Result { + Ok(self + .ctx + .keypair_from_pk_and_sk(pk_handle.into(), sk_handle.into())? + .into()) + } + + fn keypair_secretkey( + &self, + kp_handle: guest_types::Keypair, + ) -> Result { + Ok(self.ctx.keypair_secretkey(kp_handle.into())?.into()) + } +} + +impl From for KeyPairEncoding { + fn from(encoding: guest_types::KeypairEncoding) -> Self { + match encoding { + guest_types::KeypairEncoding::Raw => KeyPairEncoding::Raw, + guest_types::KeypairEncoding::Pkcs8 => KeyPairEncoding::Pkcs8, + guest_types::KeypairEncoding::Pem => KeyPairEncoding::Pem, + guest_types::KeypairEncoding::Local => KeyPairEncoding::Local, + } + } +} + +impl From for PublicKeyEncoding { + fn from(encoding: guest_types::PublickeyEncoding) -> Self { + match encoding { + guest_types::PublickeyEncoding::Raw => PublicKeyEncoding::Raw, + guest_types::PublickeyEncoding::Pkcs8 => PublicKeyEncoding::Pkcs8, + guest_types::PublickeyEncoding::Pem => PublicKeyEncoding::Pem, + guest_types::PublickeyEncoding::Sec => PublicKeyEncoding::Sec, + guest_types::PublickeyEncoding::CompressedSec => PublicKeyEncoding::CompressedSec, + guest_types::PublickeyEncoding::Local => PublicKeyEncoding::Local, + } + } +} + +impl From for SecretKeyEncoding { + fn from(encoding: guest_types::SecretkeyEncoding) -> Self { + match encoding { + guest_types::SecretkeyEncoding::Raw => SecretKeyEncoding::Raw, + guest_types::SecretkeyEncoding::Pkcs8 => SecretKeyEncoding::Pkcs8, + guest_types::SecretkeyEncoding::Pem => SecretKeyEncoding::Pem, + guest_types::SecretkeyEncoding::Sec => SecretKeyEncoding::Sec, + guest_types::SecretkeyEncoding::CompressedSec => SecretKeyEncoding::CompressedSec, + guest_types::SecretkeyEncoding::Local => SecretKeyEncoding::Local, + } + } +} diff --git a/crates/wasi-crypto/src/wiggle_interfaces/common.rs b/crates/wasi-crypto/src/wiggle_interfaces/common.rs new file mode 100644 index 0000000000..f0d5a32ff8 --- /dev/null +++ b/crates/wasi-crypto/src/wiggle_interfaces/common.rs @@ -0,0 +1,150 @@ +use super::{guest_types, WasiCryptoCtx}; + +use std::convert::TryInto; +use wasi_crypto::{AlgorithmType, Version}; + +impl super::wasi_ephemeral_crypto_common::WasiEphemeralCryptoCommon for WasiCryptoCtx { + // --- options + + fn options_open( + &self, + options_type: guest_types::AlgorithmType, + ) -> Result { + Ok(self.ctx.options_open(options_type.into())?.into()) + } + + fn options_close( + &self, + options_handle: guest_types::Options, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self.ctx.options_close(options_handle.into())?) + } + + fn options_set( + &self, + options_handle: guest_types::Options, + name_str: &wiggle::GuestPtr<'_, str>, + value_ptr: &wiggle::GuestPtr<'_, u8>, + value_len: guest_types::Size, + ) -> Result<(), guest_types::CryptoErrno> { + let name_str: &str = &*name_str.as_str()?; + let value: &[u8] = { &*value_ptr.as_array(value_len).as_slice()? }; + Ok(self + .ctx + .options_set(options_handle.into(), name_str, value)?) + } + + fn options_set_guest_buffer( + &self, + options_handle: guest_types::Options, + name_str: &wiggle::GuestPtr<'_, str>, + buffer_ptr: &wiggle::GuestPtr<'_, u8>, + buffer_len: guest_types::Size, + ) -> Result<(), guest_types::CryptoErrno> { + let name_str: &str = &*name_str.as_str()?; + let buffer: &'static mut [u8] = + unsafe { std::mem::transmute(&mut *buffer_ptr.as_array(buffer_len).as_slice_mut()?) }; + Ok(self + .ctx + .options_set_guest_buffer(options_handle.into(), name_str, buffer)?) + } + + fn options_set_u64( + &self, + options_handle: guest_types::Options, + name_str: &wiggle::GuestPtr<'_, str>, + value: u64, + ) -> Result<(), guest_types::CryptoErrno> { + let name_str: &str = &*name_str.as_str()?; + Ok(self + .ctx + .options_set_u64(options_handle.into(), name_str, value)?) + } + + // --- array + + fn array_output_len( + &self, + array_output_handle: guest_types::ArrayOutput, + ) -> Result { + Ok(self + .ctx + .array_output_len(array_output_handle.into())? + .try_into()?) + } + + fn array_output_pull( + &self, + array_output_handle: guest_types::ArrayOutput, + buf_ptr: &wiggle::GuestPtr<'_, u8>, + buf_len: guest_types::Size, + ) -> Result { + let buf: &mut [u8] = { &mut *buf_ptr.as_array(buf_len).as_slice_mut()? }; + Ok(self + .ctx + .array_output_pull(array_output_handle.into(), buf)? + .try_into()?) + } + + // --- secrets_manager + + fn secrets_manager_open( + &self, + options_handle: &guest_types::OptOptions, + ) -> Result { + let options_handle = match *options_handle { + guest_types::OptOptions::Some(options_handle) => Some(options_handle), + guest_types::OptOptions::None => None, + }; + Ok(self + .ctx + .secrets_manager_open(options_handle.map(Into::into))? + .into()) + } + + fn secrets_manager_close( + &self, + secrets_manager_handle: guest_types::SecretsManager, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self + .ctx + .secrets_manager_close(secrets_manager_handle.into())?) + } + + fn secrets_manager_invalidate( + &self, + secrets_manager_handle: guest_types::SecretsManager, + key_id_ptr: &wiggle::GuestPtr<'_, u8>, + key_id_len: guest_types::Size, + key_version: guest_types::Version, + ) -> Result<(), guest_types::CryptoErrno> { + let key_id: &[u8] = { &*key_id_ptr.as_array(key_id_len).as_slice()? }; + Ok(self.ctx.secrets_manager_invalidate( + secrets_manager_handle.into(), + key_id, + key_version.into(), + )?) + } +} + +impl From for AlgorithmType { + fn from(options_type: guest_types::AlgorithmType) -> Self { + match options_type { + guest_types::AlgorithmType::Signatures => AlgorithmType::Signatures, + guest_types::AlgorithmType::Symmetric => AlgorithmType::Symmetric, + guest_types::AlgorithmType::KeyExchange => AlgorithmType::KeyExchange, + } + } +} + +impl From for Version { + fn from(version: guest_types::Version) -> Self { + Version(version.into()) + } +} + +impl From for guest_types::Version { + fn from(version: Version) -> Self { + version.into() + } +} diff --git a/crates/wasi-crypto/src/wiggle_interfaces/error.rs b/crates/wasi-crypto/src/wiggle_interfaces/error.rs new file mode 100644 index 0000000000..e3651f6db9 --- /dev/null +++ b/crates/wasi-crypto/src/wiggle_interfaces/error.rs @@ -0,0 +1,67 @@ +use super::{guest_types, WasiCryptoCtx}; + +use std::num::TryFromIntError; +use wasi_crypto::CryptoError; + +impl From for guest_types::CryptoErrno { + fn from(e: CryptoError) -> Self { + match e { + CryptoError::Success => guest_types::CryptoErrno::Success, + CryptoError::GuestError(_wiggle_error) => guest_types::CryptoErrno::GuestError, + CryptoError::NotImplemented => guest_types::CryptoErrno::NotImplemented, + CryptoError::UnsupportedFeature => guest_types::CryptoErrno::UnsupportedFeature, + CryptoError::ProhibitedOperation => guest_types::CryptoErrno::ProhibitedOperation, + CryptoError::UnsupportedEncoding => guest_types::CryptoErrno::UnsupportedEncoding, + CryptoError::UnsupportedAlgorithm => guest_types::CryptoErrno::UnsupportedAlgorithm, + CryptoError::UnsupportedOption => guest_types::CryptoErrno::UnsupportedOption, + CryptoError::InvalidKey => guest_types::CryptoErrno::InvalidKey, + CryptoError::InvalidLength => guest_types::CryptoErrno::InvalidLength, + CryptoError::VerificationFailed => guest_types::CryptoErrno::VerificationFailed, + CryptoError::RNGError => guest_types::CryptoErrno::RngError, + CryptoError::AlgorithmFailure => guest_types::CryptoErrno::AlgorithmFailure, + CryptoError::InvalidSignature => guest_types::CryptoErrno::InvalidSignature, + CryptoError::Closed => guest_types::CryptoErrno::Closed, + CryptoError::InvalidHandle => guest_types::CryptoErrno::InvalidHandle, + CryptoError::Overflow => guest_types::CryptoErrno::Overflow, + CryptoError::InternalError => guest_types::CryptoErrno::InternalError, + CryptoError::TooManyHandles => guest_types::CryptoErrno::TooManyHandles, + CryptoError::KeyNotSupported => guest_types::CryptoErrno::KeyNotSupported, + CryptoError::KeyRequired => guest_types::CryptoErrno::KeyRequired, + CryptoError::InvalidTag => guest_types::CryptoErrno::InvalidTag, + CryptoError::InvalidOperation => guest_types::CryptoErrno::InvalidOperation, + CryptoError::NonceRequired => guest_types::CryptoErrno::NonceRequired, + CryptoError::InvalidNonce => guest_types::CryptoErrno::InvalidNonce, + CryptoError::OptionNotSet => guest_types::CryptoErrno::OptionNotSet, + CryptoError::NotFound => guest_types::CryptoErrno::NotFound, + CryptoError::ParametersMissing => guest_types::CryptoErrno::ParametersMissing, + CryptoError::IncompatibleKeys => guest_types::CryptoErrno::IncompatibleKeys, + CryptoError::Expired => guest_types::CryptoErrno::Expired, + } + } +} + +impl From for guest_types::CryptoErrno { + fn from(_: TryFromIntError) -> Self { + CryptoError::Overflow.into() + } +} + +impl<'a> wiggle::GuestErrorType for guest_types::CryptoErrno { + fn success() -> Self { + guest_types::CryptoErrno::Success + } +} + +impl guest_types::GuestErrorConversion for WasiCryptoCtx { + fn into_crypto_errno(&self, e: wiggle::GuestError) -> guest_types::CryptoErrno { + eprintln!("GuestError (witx) {:?}", e); + guest_types::CryptoErrno::GuestError + } +} + +impl From for guest_types::CryptoErrno { + fn from(e: wiggle::GuestError) -> Self { + eprintln!("GuestError (impl) {:?}", e); + guest_types::CryptoErrno::GuestError + } +} diff --git a/crates/wasi-crypto/src/wiggle_interfaces/key_exchange.rs b/crates/wasi-crypto/src/wiggle_interfaces/key_exchange.rs new file mode 100644 index 0000000000..a1685de116 --- /dev/null +++ b/crates/wasi-crypto/src/wiggle_interfaces/key_exchange.rs @@ -0,0 +1,40 @@ +use super::{guest_types, WasiCryptoCtx}; + +impl super::wasi_ephemeral_crypto_kx::WasiEphemeralCryptoKx for WasiCryptoCtx { + // --- key exchange + + fn kx_dh( + &self, + pk_handle: guest_types::Publickey, + sk_handle: guest_types::Secretkey, + ) -> Result { + Ok(self.ctx.kx_dh(pk_handle.into(), sk_handle.into())?.into()) + } + + // --- Key encapsulation + + fn kx_encapsulate( + &self, + pk_handle: guest_types::Publickey, + ) -> Result<(guest_types::ArrayOutput, guest_types::ArrayOutput), guest_types::CryptoErrno> + { + let (secret_handle, encapsulated_secret_handle) = + self.ctx.kx_encapsulate(pk_handle.into())?; + Ok((secret_handle.into(), encapsulated_secret_handle.into())) + } + + fn kx_decapsulate( + &self, + sk_handle: guest_types::Secretkey, + encapsulated_secret_ptr: &wiggle::GuestPtr<'_, u8>, + encapsulated_secret_len: guest_types::Size, + ) -> Result { + let encapsulated_secret = &*encapsulated_secret_ptr + .as_array(encapsulated_secret_len) + .as_slice()?; + Ok(self + .ctx + .kx_decapsulate(sk_handle.into(), encapsulated_secret)? + .into()) + } +} diff --git a/crates/wasi-crypto/src/wiggle_interfaces/mod.rs b/crates/wasi-crypto/src/wiggle_interfaces/mod.rs new file mode 100644 index 0000000000..8784a10bd1 --- /dev/null +++ b/crates/wasi-crypto/src/wiggle_interfaces/mod.rs @@ -0,0 +1,38 @@ +use std::rc::Rc; + +use wasi_crypto::CryptoCtx; + +wiggle::from_witx!({ + witx: ["$CARGO_MANIFEST_DIR/spec/witx/wasi_ephemeral_crypto.witx"], + ctx: WasiCryptoCtx +}); + +pub mod wasi_modules { + pub use super::{ + wasi_ephemeral_crypto_asymmetric_common, wasi_ephemeral_crypto_common, + wasi_ephemeral_crypto_kx, wasi_ephemeral_crypto_signatures, + wasi_ephemeral_crypto_symmetric, + }; +} + +pub use types as guest_types; + +#[derive(Clone)] +pub struct WasiCryptoCtx { + ctx: Rc, +} + +impl WasiCryptoCtx { + pub fn new() -> Self { + WasiCryptoCtx { + ctx: Rc::new(CryptoCtx::new()), + } + } +} + +mod asymmetric_common; +mod common; +mod error; +mod key_exchange; +mod signatures; +mod symmetric; diff --git a/crates/wasi-crypto/src/wiggle_interfaces/signatures.rs b/crates/wasi-crypto/src/wiggle_interfaces/signatures.rs new file mode 100644 index 0000000000..64fc56ed2e --- /dev/null +++ b/crates/wasi-crypto/src/wiggle_interfaces/signatures.rs @@ -0,0 +1,129 @@ +use super::{guest_types, WasiCryptoCtx}; + +use wasi_crypto::SignatureEncoding; + +impl super::wasi_ephemeral_crypto_signatures::WasiEphemeralCryptoSignatures for WasiCryptoCtx { + // --- signature + + fn signature_export( + &self, + signature_handle: guest_types::Signature, + encoding: guest_types::SignatureEncoding, + ) -> Result { + Ok(self + .ctx + .signature_export(signature_handle.into(), encoding.into())? + .into()) + } + + fn signature_import( + &self, + alg_str: &wiggle::GuestPtr<'_, str>, + encoded_ptr: &wiggle::GuestPtr<'_, u8>, + encoded_len: guest_types::Size, + encoding: guest_types::SignatureEncoding, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let encoded = &*encoded_ptr.as_array(encoded_len).as_slice()?; + Ok(self + .ctx + .signature_import(alg_str, encoded, encoding.into())? + .into()) + } + + fn signature_state_open( + &self, + kp_handle: guest_types::Keypair, + ) -> Result { + Ok(self.ctx.signature_state_open(kp_handle.into())?.into()) + } + + fn signature_state_update( + &self, + state_handle: guest_types::SignatureState, + input_ptr: &wiggle::GuestPtr<'_, u8>, + input_len: guest_types::Size, + ) -> Result<(), guest_types::CryptoErrno> { + let input = &*input_ptr.as_array(input_len).as_slice()?; + Ok(self + .ctx + .signature_state_update(state_handle.into(), input)?) + } + + fn signature_state_sign( + &self, + signature_state_handle: guest_types::SignatureState, + ) -> Result { + Ok(self + .ctx + .signature_state_sign(signature_state_handle.into())? + .into()) + } + + fn signature_state_close( + &self, + signature_state_handle: guest_types::SignatureState, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self + .ctx + .signature_state_close(signature_state_handle.into())?) + } + + fn signature_verification_state_open( + &self, + pk_handle: guest_types::Publickey, + ) -> Result { + Ok(self + .ctx + .signature_verification_state_open(pk_handle.into())? + .into()) + } + + fn signature_verification_state_update( + &self, + verification_state_handle: guest_types::SignatureVerificationState, + input_ptr: &wiggle::GuestPtr<'_, u8>, + input_len: guest_types::Size, + ) -> Result<(), guest_types::CryptoErrno> { + let input: &[u8] = &*input_ptr.as_array(input_len).as_slice()?; + Ok(self + .ctx + .signature_verification_state_update(verification_state_handle.into(), input)?) + } + + fn signature_verification_state_verify( + &self, + verification_state_handle: guest_types::SignatureVerificationState, + signature_handle: guest_types::Signature, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self.ctx.signature_verification_state_verify( + verification_state_handle.into(), + signature_handle.into(), + )?) + } + + fn signature_verification_state_close( + &self, + verification_state_handle: guest_types::SignatureVerificationState, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self + .ctx + .signature_verification_state_close(verification_state_handle.into())?) + } + + fn signature_close( + &self, + signature_handle: guest_types::Signature, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self.ctx.signature_close(signature_handle.into())?) + } +} + +impl From for SignatureEncoding { + fn from(encoding: guest_types::SignatureEncoding) -> Self { + match encoding { + guest_types::SignatureEncoding::Raw => SignatureEncoding::Raw, + guest_types::SignatureEncoding::Der => SignatureEncoding::Der, + } + } +} diff --git a/crates/wasi-crypto/src/wiggle_interfaces/symmetric.rs b/crates/wasi-crypto/src/wiggle_interfaces/symmetric.rs new file mode 100644 index 0000000000..c4f1d8f32b --- /dev/null +++ b/crates/wasi-crypto/src/wiggle_interfaces/symmetric.rs @@ -0,0 +1,384 @@ +use super::{guest_types, WasiCryptoCtx}; + +use std::convert::TryInto; +use wasi_crypto::{ensure, CryptoError}; + +impl super::wasi_ephemeral_crypto_symmetric::WasiEphemeralCryptoSymmetric for WasiCryptoCtx { + // --- secrets_manager + + fn symmetric_key_generate_managed( + &self, + secrets_manager_handle: guest_types::SecretsManager, + alg_str: &wiggle::GuestPtr<'_, str>, + options_handle: &guest_types::OptOptions, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let options_handle = match *options_handle { + guest_types::OptOptions::Some(options_handle) => Some(options_handle), + guest_types::OptOptions::None => None, + }; + Ok(self + .ctx + .symmetric_key_generate_managed( + secrets_manager_handle.into(), + alg_str, + options_handle.map(Into::into), + )? + .into()) + } + + fn symmetric_key_store_managed( + &self, + secrets_manager_handle: guest_types::SecretsManager, + symmetric_key_handle: guest_types::SymmetricKey, + symmetric_key_id_ptr: &wiggle::GuestPtr<'_, u8>, + symmetric_key_id_max_len: guest_types::Size, + ) -> Result<(), guest_types::CryptoErrno> { + let key_id_buf = &mut *symmetric_key_id_ptr + .as_array(symmetric_key_id_max_len) + .as_slice_mut()?; + Ok(self.ctx.symmetric_key_store_managed( + secrets_manager_handle.into(), + symmetric_key_handle.into(), + key_id_buf, + )?) + } + + fn symmetric_key_replace_managed( + &self, + secrets_manager_handle: guest_types::SecretsManager, + symmetric_key_old_handle: guest_types::SymmetricKey, + symmetric_key_new_handle: guest_types::SymmetricKey, + ) -> Result { + Ok(self + .ctx + .symmetric_key_replace_managed( + secrets_manager_handle.into(), + symmetric_key_old_handle.into(), + symmetric_key_new_handle.into(), + )? + .into()) + } + + fn symmetric_key_from_id( + &self, + secrets_manager_handle: guest_types::SecretsManager, + symmetric_key_id_ptr: &wiggle::GuestPtr<'_, u8>, + symmetric_key_id_len: guest_types::Size, + symmetric_key_version: guest_types::Version, + ) -> Result { + let symmetric_key_id = &*symmetric_key_id_ptr + .as_array(symmetric_key_id_len) + .as_slice()?; + Ok(self + .ctx + .symmetric_key_from_id( + secrets_manager_handle.into(), + symmetric_key_id, + symmetric_key_version.into(), + )? + .into()) + } + + // --- key + + fn symmetric_key_generate( + &self, + alg_str: &wiggle::GuestPtr<'_, str>, + options_handle: &guest_types::OptOptions, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let options_handle = match *options_handle { + guest_types::OptOptions::Some(options_handle) => Some(options_handle), + guest_types::OptOptions::None => None, + }; + Ok(self + .ctx + .symmetric_key_generate(alg_str, options_handle.map(Into::into))? + .into()) + } + + fn symmetric_key_import( + &self, + alg_str: &wiggle::GuestPtr<'_, str>, + raw_ptr: &wiggle::GuestPtr<'_, u8>, + raw_len: guest_types::Size, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let raw = &*raw_ptr.as_array(raw_len).as_slice()?; + Ok(self.ctx.symmetric_key_import(alg_str, raw)?.into()) + } + + fn symmetric_key_export( + &self, + symmetric_key_handle: guest_types::SymmetricKey, + ) -> Result { + Ok(self + .ctx + .symmetric_key_export(symmetric_key_handle.into())? + .into()) + } + + fn symmetric_key_id( + &self, + symmetric_key_handle: guest_types::SymmetricKey, + symmetric_key_id_ptr: &wiggle::GuestPtr<'_, u8>, + symmetric_key_id_max_len: guest_types::Size, + ) -> Result<(guest_types::Size, guest_types::Version), guest_types::CryptoErrno> { + let key_id_buf = &mut *symmetric_key_id_ptr + .as_array(symmetric_key_id_max_len) + .as_slice_mut()?; + let (key_id, version) = self.ctx.symmetric_key_id(symmetric_key_handle.into())?; + ensure!( + key_id.len() <= key_id_buf.len(), + CryptoError::Overflow.into() + ); + key_id_buf.copy_from_slice(&key_id); + Ok((key_id.len().try_into()?, version.into())) + } + + fn symmetric_key_close( + &self, + key_handle: guest_types::SymmetricKey, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self.ctx.symmetric_key_close(key_handle.into())?) + } + + // --- state + + fn symmetric_state_open( + &self, + alg_str: &wiggle::GuestPtr<'_, str>, + key_handle: &guest_types::OptSymmetricKey, + options_handle: &guest_types::OptOptions, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + let key_handle = match *key_handle { + guest_types::OptSymmetricKey::Some(key_handle) => Some(key_handle), + guest_types::OptSymmetricKey::None => None, + }; + let options_handle = match *options_handle { + guest_types::OptOptions::Some(options_handle) => Some(options_handle), + guest_types::OptOptions::None => None, + }; + Ok(self + .ctx + .symmetric_state_open( + alg_str, + key_handle.map(Into::into), + options_handle.map(Into::into), + )? + .into()) + } + + fn symmetric_state_options_get( + &self, + symmetric_state_handle: guest_types::SymmetricState, + name_str: &wiggle::GuestPtr<'_, str>, + value_ptr: &wiggle::GuestPtr<'_, u8>, + value_max_len: guest_types::Size, + ) -> Result { + let name_str: &str = &*name_str.as_str()?; + let value = &mut *value_ptr.as_array(value_max_len).as_slice_mut()?; + Ok(self + .ctx + .options_get(symmetric_state_handle.into(), name_str, value)? + .try_into()?) + } + + fn symmetric_state_options_get_u64( + &self, + symmetric_state_handle: guest_types::SymmetricState, + name_str: &wiggle::GuestPtr<'_, str>, + ) -> Result { + let name_str: &str = &*name_str.as_str()?; + Ok(self + .ctx + .options_get_u64(symmetric_state_handle.into(), name_str)?) + } + + fn symmetric_state_close( + &self, + symmetric_state_handle: guest_types::SymmetricState, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self + .ctx + .symmetric_state_close(symmetric_state_handle.into())?) + } + + fn symmetric_state_absorb( + &self, + symmetric_state_handle: guest_types::SymmetricState, + data_ptr: &wiggle::GuestPtr<'_, u8>, + data_len: guest_types::Size, + ) -> Result<(), guest_types::CryptoErrno> { + let data = &*data_ptr.as_array(data_len).as_slice()?; + Ok(self + .ctx + .symmetric_state_absorb(symmetric_state_handle.into(), data)?) + } + + fn symmetric_state_squeeze( + &self, + symmetric_state_handle: guest_types::SymmetricState, + out_ptr: &wiggle::GuestPtr<'_, u8>, + out_len: guest_types::Size, + ) -> Result<(), guest_types::CryptoErrno> { + let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?; + Ok(self + .ctx + .symmetric_state_squeeze(symmetric_state_handle.into(), out)?) + } + + fn symmetric_state_squeeze_tag( + &self, + symmetric_state_handle: guest_types::SymmetricState, + ) -> Result { + Ok(self + .ctx + .symmetric_state_squeeze_tag(symmetric_state_handle.into())? + .into()) + } + + fn symmetric_state_squeeze_key( + &self, + symmetric_state_handle: guest_types::SymmetricState, + alg_str: &wiggle::GuestPtr<'_, str>, + ) -> Result { + let alg_str = &*alg_str.as_str()?; + Ok(self + .ctx + .symmetric_state_squeeze_key(symmetric_state_handle.into(), alg_str)? + .into()) + } + + fn symmetric_state_max_tag_len( + &self, + symmetric_state_handle: guest_types::SymmetricState, + ) -> Result { + Ok(self + .ctx + .symmetric_state_max_tag_len(symmetric_state_handle.into())? + .try_into()?) + } + + fn symmetric_state_encrypt( + &self, + symmetric_state_handle: guest_types::SymmetricState, + out_ptr: &wiggle::GuestPtr<'_, u8>, + out_len: guest_types::Size, + data_ptr: &wiggle::GuestPtr<'_, u8>, + data_len: guest_types::Size, + ) -> Result { + let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?; + let data = &*data_ptr.as_array(data_len).as_slice()?; + Ok(self + .ctx + .symmetric_state_encrypt(symmetric_state_handle.into(), out, data)? + .try_into()?) + } + + fn symmetric_state_encrypt_detached( + &self, + symmetric_state_handle: guest_types::SymmetricState, + out_ptr: &wiggle::GuestPtr<'_, u8>, + out_len: guest_types::Size, + data_ptr: &wiggle::GuestPtr<'_, u8>, + data_len: guest_types::Size, + ) -> Result { + let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?; + let data = &*data_ptr.as_array(data_len).as_slice()?; + Ok(self + .ctx + .symmetric_state_encrypt_detached(symmetric_state_handle.into(), out, data)? + .into()) + } + + fn symmetric_state_decrypt( + &self, + symmetric_state_handle: guest_types::SymmetricState, + out_ptr: &wiggle::GuestPtr<'_, u8>, + out_len: guest_types::Size, + data_ptr: &wiggle::GuestPtr<'_, u8>, + data_len: guest_types::Size, + ) -> Result { + let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?; + let data = &*data_ptr.as_array(data_len).as_slice()?; + Ok(self + .ctx + .symmetric_state_decrypt(symmetric_state_handle.into(), out, data)? + .try_into()?) + } + + fn symmetric_state_decrypt_detached( + &self, + symmetric_state_handle: guest_types::SymmetricState, + out_ptr: &wiggle::GuestPtr<'_, u8>, + out_len: guest_types::Size, + data_ptr: &wiggle::GuestPtr<'_, u8>, + data_len: guest_types::Size, + raw_tag_ptr: &wiggle::GuestPtr<'_, u8>, + raw_tag_len: guest_types::Size, + ) -> Result { + let out = &mut *out_ptr.as_array(out_len).as_slice_mut()?; + let data = &*data_ptr.as_array(data_len).as_slice()?; + let raw_tag: &[u8] = &*raw_tag_ptr.as_array(raw_tag_len).as_slice()?; + Ok(self + .ctx + .symmetric_state_decrypt_detached(symmetric_state_handle.into(), out, data, raw_tag)? + .try_into()?) + } + + fn symmetric_state_ratchet( + &self, + symmetric_state_handle: guest_types::SymmetricState, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self + .ctx + .symmetric_state_ratchet(symmetric_state_handle.into())?) + } + + // --- tag + + fn symmetric_tag_len( + &self, + symmetric_tag_handle: guest_types::SymmetricTag, + ) -> Result { + Ok(self + .ctx + .symmetric_tag_len(symmetric_tag_handle.into())? + .try_into()?) + } + + fn symmetric_tag_pull( + &self, + symmetric_tag_handle: guest_types::SymmetricTag, + buf_ptr: &wiggle::GuestPtr<'_, u8>, + buf_len: guest_types::Size, + ) -> Result { + let buf = &mut *buf_ptr.as_array(buf_len).as_slice_mut()?; + Ok(self + .ctx + .symmetric_tag_pull(symmetric_tag_handle.into(), buf)? + .try_into()?) + } + + fn symmetric_tag_verify( + &self, + symmetric_tag_handle: guest_types::SymmetricTag, + expected_raw_ptr: &wiggle::GuestPtr<'_, u8>, + expected_raw_len: guest_types::Size, + ) -> Result<(), guest_types::CryptoErrno> { + let expected_raw = &*expected_raw_ptr.as_array(expected_raw_len).as_slice()?; + Ok(self + .ctx + .symmetric_tag_verify(symmetric_tag_handle.into(), expected_raw)?) + } + + fn symmetric_tag_close( + &self, + symmetric_tag_handle: guest_types::SymmetricTag, + ) -> Result<(), guest_types::CryptoErrno> { + Ok(self.ctx.symmetric_tag_close(symmetric_tag_handle.into())?) + } +} diff --git a/scripts/publish.rs b/scripts/publish.rs index 330a7e0dfb..75a8d652cd 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -65,6 +65,7 @@ const CRATES_TO_PUBLISH: &[&str] = &[ "wasmtime-wiggle", "wasmtime-wasi", "wasmtime-wasi-nn", + "wasmtime-wasi-crypto", "wasmtime-rust-macro", "wasmtime-rust", "wasmtime-wast", @@ -176,7 +177,7 @@ fn read_crate(manifest: &Path) -> Crate { } else { version.clone() }; - if name == "witx" { + if ["witx", "wasi-crypto"].contains(&&name[..]) { publish = false; } Crate { @@ -299,6 +300,13 @@ fn verify(crates: &[Crate]) { .unwrap(); verify_and_vendor(&witx); + // Vendor wasi-crypto which is also a path dependency + let wasi_crypto = crates + .iter() + .find(|c| c.name == "wasi-crypto") + .unwrap(); + verify_and_vendor(&wasi_crypto); + for krate in crates { if !krate.publish { continue; diff --git a/src/commands/run.rs b/src/commands/run.rs index e1b891d598..a7f26bc42b 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -18,6 +18,12 @@ use wasmtime_wasi::Wasi; #[cfg(feature = "wasi-nn")] use wasmtime_wasi_nn::{WasiNn, WasiNnCtx}; +#[cfg(feature = "wasi-crypto")] +use wasmtime_wasi_crypto::{ + WasiCryptoAsymmetricCommon, WasiCryptoCommon, WasiCryptoCtx, WasiCryptoSignatures, + WasiCryptoSymmetric, +}; + fn parse_module(s: &OsStr) -> Result { // Do not accept wasmtime subcommand names as the module name match s.to_str() { @@ -364,6 +370,15 @@ fn populate_with_wasi( wasi_nn.add_to_linker(linker)?; } + #[cfg(feature = "wasi-crypto")] + { + let cx_crypto = WasiCryptoCtx::new(); + WasiCryptoCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; + WasiCryptoAsymmetricCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; + WasiCryptoSignatures::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; + WasiCryptoSymmetric::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; + } + let wasi = wasmtime_wasi::old::snapshot_0::Wasi::new(linker.store(), mk_cx()?); wasi.add_to_linker(linker)?; From 3c5416446c47eec5266f758ac8ff260c6c56430e Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Mon, 25 Jan 2021 08:32:06 -0800 Subject: [PATCH 31/55] Fix cargo-deny issue with raw-cpuid advisory. cargo-deny tells us that we should upgrade raw-cpuid to v9.0.0. This new version also seems to lack the `nightly` feature (perhaps it has been incorporated into the base functionality) so I had to remove this feature selector to build. --- Cargo.lock | 30 ++---------------------------- cranelift/native/Cargo.toml | 7 ++----- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e4057f5b5..86cbcb9f6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2147,13 +2147,11 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "8.1.2" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdf7d9dbd43f3d81d94a49c1c3df73cc2b3827995147e6cf7f89d4ec5483e73" +checksum = "c27cb5785b85bd05d4eb171556c9a1a514552e26123aeae6bb7d811353148026" dependencies = [ "bitflags", - "cc", - "rustc_version", ] [[package]] @@ -2350,15 +2348,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver", -] - [[package]] name = "rusty-fork" version = "0.3.0" @@ -2412,21 +2401,6 @@ dependencies = [ "syn", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "serde" version = "1.0.120" diff --git a/cranelift/native/Cargo.toml b/cranelift/native/Cargo.toml index e578bccf12..047d21261a 100644 --- a/cranelift/native/Cargo.toml +++ b/cranelift/native/Cargo.toml @@ -15,15 +15,12 @@ cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features target-lexicon = "0.11" [target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies] -raw-cpuid = "8.1.2" +raw-cpuid = "9.0.0" [features] default = ["std"] std = ["cranelift-codegen/std"] -# when compiling with the "core" feature, nightly must be enabled -# enabling the "nightly" feature for raw-cpuid allows avoiding -# linking in a c-library. -core = ["cranelift-codegen/core", "raw-cpuid/nightly"] +core = ["cranelift-codegen/core"] [badges] maintenance = { status = "experimental" } From c55c5e0506f78038ababe5fbf94ff114a9913516 Mon Sep 17 00:00:00 2001 From: Kasey Carrothers Date: Sat, 23 Jan 2021 18:03:06 -0800 Subject: [PATCH 32/55] Add additional tests for icmp-i128. Fixes #1136. Tests added: * eq with nonzero values * gt with nonzero values * ge with nonzero values --- .../filetests/isa/x86/icmp-i128.clif | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/cranelift/filetests/filetests/isa/x86/icmp-i128.clif b/cranelift/filetests/filetests/isa/x86/icmp-i128.clif index dce0e1db87..789fcc6ea3 100644 --- a/cranelift/filetests/filetests/isa/x86/icmp-i128.clif +++ b/cranelift/filetests/filetests/isa/x86/icmp-i128.clif @@ -50,3 +50,45 @@ block0: } ; run + +function %test_icmp_nz_eq_i128() -> b1 { +block0: + v11 = iconst.i64 0x1 + v12 = iconst.i64 0x1 + v1 = iconcat v11, v12 + v21 = iconst.i64 0x1 + v22 = iconst.i64 0x1 + v2 = iconcat v21, v22 + v10 = icmp.i128 eq v1, v2 + return v10 +} + +; run + +function %test_icmp_nz_gt_i128() -> b1 { +block0: + v11 = iconst.i64 0x1 + v12 = iconst.i64 0x1 + v1 = iconcat v11, v12 + v21 = iconst.i64 0x1 + v22 = iconst.i64 0x2 + v2 = iconcat v21, v22 + v10 = icmp.i128 ugt v2, v1 + return v10 +} + +; run + +function %test_icmp_nz_ge_i128() -> b1 { +block0: + v11 = iconst.i64 0x1 + v12 = iconst.i64 0x1 + v1 = iconcat v11, v12 + v21 = iconst.i64 0x1 + v22 = iconst.i64 0x1 + v2 = iconcat v21, v22 + v10 = icmp.i128 uge v1, v2 + return v10 +} + +; run From c6c5fe48b6209f61c0aecc6b618027899b0809db Mon Sep 17 00:00:00 2001 From: Kasey Carrothers Date: Sun, 24 Jan 2021 22:18:21 -0800 Subject: [PATCH 33/55] Add i128.icmp run tests for the x64 backend. --- .../filetests/isa/x64/icmp-i128-run.clif | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 cranelift/filetests/filetests/isa/x64/icmp-i128-run.clif diff --git a/cranelift/filetests/filetests/isa/x64/icmp-i128-run.clif b/cranelift/filetests/filetests/isa/x64/icmp-i128-run.clif new file mode 100644 index 0000000000..2f9e2d5331 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/icmp-i128-run.clif @@ -0,0 +1,95 @@ +test run +target x86_64 +feature "experimental_x64" + +function %test_icmp_eq_i128() -> b1 { +block0: + v11 = iconst.i64 0x0 + v12 = iconst.i64 0x0 + v1 = iconcat v11, v12 + v21 = iconst.i64 0x0 + v22 = iconst.i64 0x0 + v2 = iconcat v21, v22 + v10 = icmp.i128 eq v1, v2 + return v10 +} + +; run + +function %test_icmp_imm_eq_i128() -> b1 { +block0: + v11 = iconst.i64 0x0 + v12 = iconst.i64 0x0 + v1 = iconcat v11, v12 + v10 = icmp_imm.i128 eq v1, 0x0 + return v10 +} + +; run + +function %test_icmp_ne_i128() -> b1 { +block0: + v11 = iconst.i64 0x0 + v12 = iconst.i64 0x0 + v1 = iconcat v11, v12 + v21 = iconst.i64 0x0 + v22 = iconst.i64 0x1 + v2 = iconcat v21, v22 + v10 = icmp.i128 ne v1, v2 + return v10 +} + +; run + +function %test_icmp_imm_ne_i128() -> b1 { +block0: + v11 = iconst.i64 0x0 + v12 = iconst.i64 0x0 + v1 = iconcat v11, v12 + v10 = icmp_imm.i128 ne v1, 0x1 + return v10 +} + +; run + +function %test_icmp_nz_eq_i128() -> b1 { +block0: + v11 = iconst.i64 0x1 + v12 = iconst.i64 0x1 + v1 = iconcat v11, v12 + v21 = iconst.i64 0x1 + v22 = iconst.i64 0x1 + v2 = iconcat v21, v22 + v10 = icmp.i128 eq v1, v2 + return v10 +} + +; run + +function %test_icmp_nz_gt_i128() -> b1 { +block0: + v11 = iconst.i64 0x1 + v12 = iconst.i64 0x1 + v1 = iconcat v11, v12 + v21 = iconst.i64 0x1 + v22 = iconst.i64 0x2 + v2 = iconcat v21, v22 + v10 = icmp.i128 ugt v2, v1 + return v10 +} + +; run + +function %test_icmp_nz_ge_i128() -> b1 { +block0: + v11 = iconst.i64 0x1 + v12 = iconst.i64 0x1 + v1 = iconcat v11, v12 + v21 = iconst.i64 0x1 + v22 = iconst.i64 0x1 + v2 = iconcat v21, v22 + v10 = icmp.i128 uge v1, v2 + return v10 +} + +; run From 79649a15f62000a7487468d5b7041656c0fcabf4 Mon Sep 17 00:00:00 2001 From: omar Date: Wed, 6 Jan 2021 09:44:59 -0500 Subject: [PATCH 34/55] Update README.md --- cranelift/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cranelift/README.md b/cranelift/README.md index 52d4e0cf8f..b083122b37 100644 --- a/cranelift/README.md +++ b/cranelift/README.md @@ -19,7 +19,7 @@ For more information, see [the documentation](docs/index.md). For an example of how to use the JIT, see the [JIT Demo], which implements a toy language. -[JIT Demo]: https://github.com/bytecodealliance/simplejit-demo +[JIT Demo]: https://github.com/bytecodealliance/cranelift-jit-demo For an example of how to use Cranelift to run WebAssembly code, see [Wasmtime], which implements a standalone, embeddable, VM using Cranelift. From f4faa04dca6ef9b822764d78a0bdd4bd5f07fb7d Mon Sep 17 00:00:00 2001 From: theduke Date: Tue, 26 Jan 2021 16:09:20 +0100 Subject: [PATCH 35/55] Safe Memory read/write API (#2528) This commit introduces two new methods on `Memory` that enable reading and writing memory contents without requiring `unsafe`. The methods return a new `MemoryError` if the memory access fails. --- crates/wasmtime/src/externals.rs | 68 +++++++++++++++++++++++++++++++- tests/all/externals.rs | 37 +++++++++++++++++ 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index a3cede9cb7..890471b1a2 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -605,6 +605,22 @@ impl Table { } } +/// Error for out of bounds [`Memory`] access. +#[derive(Debug)] +#[non_exhaustive] +pub struct MemoryAccessError { + // Keep struct internals private for future extensibility. + _private: (), +} + +impl std::fmt::Display for MemoryAccessError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "out of bounds memory access") + } +} + +impl std::error::Error for MemoryAccessError {} + /// A WebAssembly linear memory. /// /// WebAssembly memories represent a contiguous array of bytes that have a size @@ -661,9 +677,24 @@ impl Table { /// Let's run through a few safe examples first of how you can use a `Memory`. /// /// ```rust -/// use wasmtime::Memory; +/// use wasmtime::{Memory, MemoryAccessError}; /// -/// fn safe_examples(mem: &Memory) { +/// // Memory can be read and written safely with the `Memory::read` and +/// // `Memory::write` methods. +/// // An error is returned if the copy did not succeed. +/// fn safe_examples(mem: &Memory) -> Result<(), MemoryAccessError> { +/// let offset = 5; +/// mem.write(offset, b"hello")?; +/// let mut buffer = [0u8; 5]; +/// mem.read(offset, &mut buffer)?; +/// assert_eq!(b"hello", &buffer); +/// Ok(()) +/// } +/// +/// // You can also get direct, unsafe access to the memory, but must manually +/// // ensure that safety invariants are upheld. +/// +/// fn correct_unsafe_examples(mem: &Memory) { /// // Just like wasm, it's safe to read memory almost at any time. The /// // gotcha here is that we need to be sure to load from the correct base /// // pointer and perform the bounds check correctly. So long as this is @@ -871,6 +902,39 @@ impl Memory { MemoryType::from_wasmtime_memory(&self.wasmtime_export.memory.memory) } + /// Safely reads memory contents at the given offset into a buffer. + /// + /// The entire buffer will be filled. + /// + /// If offset + buffer length exceed the current memory capacity, + /// a [`MemoryAccessError`] is returned. + pub fn read(&self, offset: usize, buffer: &mut [u8]) -> Result<(), MemoryAccessError> { + unsafe { + let slice = self + .data_unchecked() + .get(offset..) + .and_then(|s| s.get(..buffer.len())) + .ok_or(MemoryAccessError { _private: () })?; + buffer.copy_from_slice(slice); + Ok(()) + } + } + + /// Safely writes contents of a buffer to this memory at the given offset. + /// + /// If the offset + buffer length exceed current memory capacity, a + /// [`MemoryAccessError`] is returned. + pub fn write(&self, offset: usize, buffer: &[u8]) -> Result<(), MemoryAccessError> { + unsafe { + self.data_unchecked_mut() + .get_mut(offset..) + .and_then(|s| s.get_mut(..buffer.len())) + .ok_or(MemoryAccessError { _private: () })? + .copy_from_slice(buffer); + Ok(()) + } + } + /// Returns this memory as a slice view that can be read natively in Rust. /// /// # Safety diff --git a/tests/all/externals.rs b/tests/all/externals.rs index 365d5b2611..6c13aabc17 100644 --- a/tests/all/externals.rs +++ b/tests/all/externals.rs @@ -340,3 +340,40 @@ fn grow_externref_tables_via_api() -> anyhow::Result<()> { Ok(()) } + +#[test] +fn read_write_memory_via_api() { + let cfg = Config::new(); + let store = Store::new(&Engine::new(&cfg)); + let ty = MemoryType::new(Limits::new(1, None)); + let mem = Memory::new(&store, ty); + mem.grow(1).unwrap(); + + let value = b"hello wasm"; + mem.write(mem.data_size() - value.len(), value).unwrap(); + + let mut buffer = [0u8; 10]; + mem.read(mem.data_size() - buffer.len(), &mut buffer) + .unwrap(); + assert_eq!(value, &buffer); + + // Error conditions. + + // Out of bounds write. + + let res = mem.write(mem.data_size() - value.len() + 1, value); + assert!(res.is_err()); + + // Out of bounds read. + + let res = mem.read(mem.data_size() - buffer.len() + 1, &mut buffer); + assert!(res.is_err()); + + // Read offset overflow. + let res = mem.read(usize::MAX, &mut buffer); + assert!(res.is_err()); + + // Write offset overflow. + let res = mem.write(usize::MAX, &mut buffer); + assert!(res.is_err()); +} From c7c6e76f9b93e10bfb56a5a4e96edd165b1af334 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 26 Jan 2021 07:11:24 -0800 Subject: [PATCH 36/55] fuzzing: Add tests for dummy import generation (#2604) --- crates/fuzzing/src/oracles/dummy.rs | 229 ++++++++++++++++++++++++++++ crates/wasmtime/src/types.rs | 4 +- 2 files changed, 231 insertions(+), 2 deletions(-) diff --git a/crates/fuzzing/src/oracles/dummy.rs b/crates/fuzzing/src/oracles/dummy.rs index 561a8fb329..f49cee8c5b 100644 --- a/crates/fuzzing/src/oracles/dummy.rs +++ b/crates/fuzzing/src/oracles/dummy.rs @@ -349,3 +349,232 @@ fn wat_ty(ty: &ValType) -> &'static str { ValType::FuncRef => "funcref", } } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + fn store() -> Store { + let mut config = Config::default(); + config.wasm_module_linking(true); + config.wasm_multi_memory(true); + let engine = wasmtime::Engine::new(&config); + Store::new(&engine) + } + + #[test] + fn dummy_table_import() { + let store = store(); + let table = dummy_table( + &store, + TableType::new(ValType::ExternRef, Limits::at_least(10)), + ); + assert_eq!(table.size(), 10); + for i in 0..10 { + assert!(table.get(i).unwrap().unwrap_externref().is_none()); + } + } + + #[test] + fn dummy_global_import() { + let store = store(); + let global = dummy_global(&store, GlobalType::new(ValType::I32, Mutability::Const)); + assert_eq!(global.val_type(), ValType::I32); + assert_eq!(global.mutability(), Mutability::Const); + } + + #[test] + fn dummy_memory_import() { + let store = store(); + let memory = dummy_memory(&store, MemoryType::new(Limits::at_least(1))); + assert_eq!(memory.size(), 1); + } + + #[test] + fn dummy_function_import() { + let store = store(); + let func_ty = FuncType::new(vec![ValType::I32], vec![ValType::I64]); + let func = dummy_func(&store, func_ty.clone()); + assert_eq!(func.ty(), func_ty); + } + + #[test] + fn dummy_instance_import() { + let store = store(); + + let mut instance_ty = InstanceType::new(); + + // Functions. + instance_ty.add_named_export("func0", FuncType::new(vec![ValType::I32], vec![]).into()); + instance_ty.add_named_export("func1", FuncType::new(vec![], vec![ValType::I64]).into()); + + // Globals. + instance_ty.add_named_export( + "global0", + GlobalType::new(ValType::I32, Mutability::Const).into(), + ); + instance_ty.add_named_export( + "global1", + GlobalType::new(ValType::I64, Mutability::Var).into(), + ); + + // Tables. + instance_ty.add_named_export( + "table0", + TableType::new(ValType::ExternRef, Limits::at_least(1)).into(), + ); + instance_ty.add_named_export( + "table1", + TableType::new(ValType::ExternRef, Limits::at_least(1)).into(), + ); + + // Memories. + instance_ty.add_named_export("memory0", MemoryType::new(Limits::at_least(1)).into()); + instance_ty.add_named_export("memory1", MemoryType::new(Limits::at_least(1)).into()); + + // Modules. + instance_ty.add_named_export("module0", ModuleType::new().into()); + instance_ty.add_named_export("module1", ModuleType::new().into()); + + // Instances. + instance_ty.add_named_export("instance0", InstanceType::new().into()); + instance_ty.add_named_export("instance1", InstanceType::new().into()); + + let instance = dummy_instance(&store, instance_ty.clone()); + + let mut expected_exports = vec![ + "func0", + "func1", + "global0", + "global1", + "table0", + "table1", + "memory0", + "memory1", + "module0", + "module1", + "instance0", + "instance1", + ] + .into_iter() + .collect::>(); + for exp in instance.ty().exports() { + let was_expected = expected_exports.remove(exp.name()); + assert!(was_expected); + } + assert!(expected_exports.is_empty()); + } + + #[test] + fn dummy_module_import() { + let store = store(); + + let mut module_ty = ModuleType::new(); + + // Multiple exported and imported functions. + module_ty.add_named_export("func0", FuncType::new(vec![ValType::I32], vec![]).into()); + module_ty.add_named_export("func1", FuncType::new(vec![], vec![ValType::I64]).into()); + module_ty.add_named_import( + "func2", + None, + FuncType::new(vec![ValType::I64], vec![]).into(), + ); + module_ty.add_named_import( + "func3", + None, + FuncType::new(vec![], vec![ValType::I32]).into(), + ); + + // Multiple exported and imported globals. + module_ty.add_named_export( + "global0", + GlobalType::new(ValType::I32, Mutability::Const).into(), + ); + module_ty.add_named_export( + "global1", + GlobalType::new(ValType::I64, Mutability::Var).into(), + ); + module_ty.add_named_import( + "global2", + None, + GlobalType::new(ValType::I32, Mutability::Var).into(), + ); + module_ty.add_named_import( + "global3", + None, + GlobalType::new(ValType::I64, Mutability::Const).into(), + ); + + // Multiple exported and imported tables. + module_ty.add_named_export( + "table0", + TableType::new(ValType::ExternRef, Limits::at_least(1)).into(), + ); + module_ty.add_named_export( + "table1", + TableType::new(ValType::ExternRef, Limits::at_least(1)).into(), + ); + module_ty.add_named_import( + "table2", + None, + TableType::new(ValType::ExternRef, Limits::at_least(1)).into(), + ); + module_ty.add_named_import( + "table3", + None, + TableType::new(ValType::ExternRef, Limits::at_least(1)).into(), + ); + + // Multiple exported and imported memories. + module_ty.add_named_export("memory0", MemoryType::new(Limits::at_least(1)).into()); + module_ty.add_named_export("memory1", MemoryType::new(Limits::at_least(1)).into()); + module_ty.add_named_import("memory2", None, MemoryType::new(Limits::at_least(1)).into()); + module_ty.add_named_import("memory3", None, MemoryType::new(Limits::at_least(1)).into()); + + // An exported and an imported module. + module_ty.add_named_export("module0", ModuleType::new().into()); + module_ty.add_named_import("module1", None, ModuleType::new().into()); + + // An exported and an imported instance. + module_ty.add_named_export("instance0", InstanceType::new().into()); + module_ty.add_named_import("instance1", None, InstanceType::new().into()); + + // Create the module. + let module = dummy_module(&store, module_ty); + + // Check that we have the expected exports. + assert!(module.get_export("func0").is_some()); + assert!(module.get_export("func1").is_some()); + assert!(module.get_export("global0").is_some()); + assert!(module.get_export("global1").is_some()); + assert!(module.get_export("table0").is_some()); + assert!(module.get_export("table1").is_some()); + assert!(module.get_export("memory0").is_some()); + assert!(module.get_export("memory1").is_some()); + assert!(module.get_export("instance0").is_some()); + assert!(module.get_export("module0").is_some()); + + // Check that we have the exported imports. + let mut expected_imports = vec![ + "func2", + "func3", + "global2", + "global3", + "table2", + "table3", + "memory2", + "memory3", + "instance1", + "module1", + ] + .into_iter() + .collect::>(); + for imp in module.imports() { + assert!(imp.name().is_none()); + let was_expected = expected_imports.remove(imp.module()); + assert!(was_expected); + } + assert!(expected_imports.is_empty()); + } +} diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs index fe2915402b..5a090d794c 100644 --- a/crates/wasmtime/src/types.rs +++ b/crates/wasmtime/src/types.rs @@ -466,12 +466,12 @@ impl ModuleType { } /// Adds a new export to this `ModuleType`. - pub(crate) fn add_named_export(&mut self, name: &str, ty: ExternType) { + pub fn add_named_export(&mut self, name: &str, ty: ExternType) { self.exports.push((name.to_string(), ty)); } /// Adds a new import to this `ModuleType`. - pub(crate) fn add_named_import(&mut self, module: &str, field: Option<&str>, ty: ExternType) { + pub fn add_named_import(&mut self, module: &str, field: Option<&str>, ty: ExternType) { self.imports .push((module.to_string(), field.map(|f| f.to_string()), ty)); } From 8d84482153238bce22bd7e89f084ec4b7cfa1264 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 22 Jan 2021 17:30:42 -0800 Subject: [PATCH 37/55] bench-api: Allow access to files in the current directory And pass through the `WASM_BENCH_USE_SMALL_WORKLOAD` env var. Part of https://github.com/bytecodealliance/sightglass/issues/70 --- crates/bench-api/src/lib.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/bench-api/src/lib.rs b/crates/bench-api/src/lib.rs index 8daf1197d8..1d0e2dcd64 100644 --- a/crates/bench-api/src/lib.rs +++ b/crates/bench-api/src/lib.rs @@ -73,6 +73,7 @@ //! ``` use anyhow::{anyhow, Context, Result}; +use std::env; use std::os::raw::{c_int, c_void}; use std::slice; use wasi_common::WasiCtxBuilder; @@ -211,9 +212,23 @@ impl BenchState { let mut linker = Linker::new(&self.store); - // Import a very restricted WASI environment. + // Create a WASI environment. + let mut cx = WasiCtxBuilder::new(); cx.inherit_stdio(); + // Allow access to the current working directory so that the benchmark + // can read its input workload(s). The sightglass benchmark runner will + // make sure that this process is spawned with its current directory set + // to the current benchmark's directory. + let cwd = wasi_common::preopen_dir(".") + .context("failed to open the current working directory")?; + cx.preopened_dir(cwd, "."); + // Pass this env var along so that the benchmark program can use smaller + // input workload(s) if it has them and that has been requested. + if let Ok(val) = env::var("WASM_BENCH_USE_SMALL_WORKLOAD") { + cx.env("WASM_BENCH_USE_SMALL_WORKLOAD", &val); + } + let cx = cx.build()?; let wasi = Wasi::new(linker.store(), cx); wasi.add_to_linker(&mut linker)?; From 3b7f3e0c238ba13a44dd5fc5d309c08691324d73 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Mon, 25 Jan 2021 12:37:49 -0800 Subject: [PATCH 38/55] Remove semicolon to quiet a rustc warning --- crates/debug/src/transform/expression.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/debug/src/transform/expression.rs b/crates/debug/src/transform/expression.rs index 0a286aa616..a9460f9c2d 100644 --- a/crates/debug/src/transform/expression.rs +++ b/crates/debug/src/transform/expression.rs @@ -468,7 +468,7 @@ where let _ = code_chunk; // suppresses warning for final flush } }; - }; + } // Find all landing pads by scanning bytes, do not care about // false location at this moment. // Looks hacky but it is fast; does not need to be really exact. From 0cdc80fbf8603fe88416084ba46fe3c76e5e3a74 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 26 Jan 2021 10:04:48 -0800 Subject: [PATCH 39/55] wasmtime: move `Memory` to its own module Purely code motion and fixing up imports. --- crates/wasmtime/src/config.rs | 2 +- crates/wasmtime/src/externals.rs | 583 +---------------------- crates/wasmtime/src/lib.rs | 2 + crates/wasmtime/src/memory.rs | 577 ++++++++++++++++++++++ crates/wasmtime/src/trampoline/memory.rs | 2 +- 5 files changed, 585 insertions(+), 581 deletions(-) create mode 100644 crates/wasmtime/src/memory.rs diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 15b5ad58cb..e6e3f68952 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -1,4 +1,4 @@ -use crate::externals::MemoryCreator; +use crate::memory::MemoryCreator; use crate::trampoline::MemoryCreatorProxy; use anyhow::{bail, Result}; use std::cmp; diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index 890471b1a2..00bcaa499b 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -1,15 +1,13 @@ -use crate::trampoline::{ - generate_global_export, generate_memory_export, generate_table_export, StoreInstanceHandle, -}; +use crate::memory::Memory; +use crate::trampoline::{generate_global_export, generate_table_export, StoreInstanceHandle}; use crate::values::{from_checked_anyfunc, into_checked_anyfunc, Val}; use crate::{ - ExternRef, ExternType, Func, GlobalType, Instance, MemoryType, Module, Mutability, Store, - TableType, Trap, ValType, + ExternRef, ExternType, Func, GlobalType, Instance, Module, Mutability, Store, TableType, Trap, + ValType, }; use anyhow::{anyhow, bail, Result}; use std::mem; use std::ptr; -use std::slice; use wasmtime_environ::wasm; use wasmtime_runtime::{self as runtime, InstanceHandle}; @@ -605,579 +603,6 @@ impl Table { } } -/// Error for out of bounds [`Memory`] access. -#[derive(Debug)] -#[non_exhaustive] -pub struct MemoryAccessError { - // Keep struct internals private for future extensibility. - _private: (), -} - -impl std::fmt::Display for MemoryAccessError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "out of bounds memory access") - } -} - -impl std::error::Error for MemoryAccessError {} - -/// A WebAssembly linear memory. -/// -/// WebAssembly memories represent a contiguous array of bytes that have a size -/// that is always a multiple of the WebAssembly page size, currently 64 -/// kilobytes. -/// -/// WebAssembly memory is used for global data, statics in C/C++/Rust, shadow -/// stack memory, etc. Accessing wasm memory is generally quite fast! -/// -/// # `Memory` and `Clone` -/// -/// Memories are internally reference counted so you can `clone` a `Memory`. The -/// cloning process only performs a shallow clone, so two cloned `Memory` -/// instances are equivalent in their functionality. -/// -/// # `Memory` and threads -/// -/// It is intended that `Memory` is safe to share between threads. At this time -/// this is not implemented in `wasmtime`, however. This is planned to be -/// implemented though! -/// -/// # `Memory` and Safety -/// -/// Linear memory is a lynchpin of safety for WebAssembly, but it turns out -/// there are very few ways to safely inspect the contents of a memory from the -/// host (Rust). This is because memory safety is quite tricky when working with -/// a `Memory` and we're still working out the best idioms to encapsulate -/// everything safely where it's efficient and ergonomic. This section of -/// documentation, however, is intended to help educate a bit what is and isn't -/// safe when working with `Memory`. -/// -/// For safety purposes you can think of a `Memory` as a glorified -/// `Rc>>`. There are a few consequences of this -/// interpretation: -/// -/// * At any time someone else may have access to the memory (hence the `Rc`). -/// This could be a wasm instance, other host code, or a set of wasm instances -/// which all reference a `Memory`. When in doubt assume someone else has a -/// handle to your `Memory`. -/// -/// * At any time, memory can be read from or written to (hence the -/// `UnsafeCell`). Anyone with a handle to a wasm memory can read/write to it. -/// Primarily other instances can execute the `load` and `store` family of -/// instructions, as well as any other which modifies or reads memory. -/// -/// * At any time memory may grow (hence the `Vec<..>`). Growth may relocate the -/// base memory pointer (similar to how `vec.push(...)` can change the result -/// of `.as_ptr()`) -/// -/// So given that we're working roughly with `Rc>>` that's a -/// lot to keep in mind! It's hopefully though sort of setting the stage as to -/// what you can safely do with memories. -/// -/// Let's run through a few safe examples first of how you can use a `Memory`. -/// -/// ```rust -/// use wasmtime::{Memory, MemoryAccessError}; -/// -/// // Memory can be read and written safely with the `Memory::read` and -/// // `Memory::write` methods. -/// // An error is returned if the copy did not succeed. -/// fn safe_examples(mem: &Memory) -> Result<(), MemoryAccessError> { -/// let offset = 5; -/// mem.write(offset, b"hello")?; -/// let mut buffer = [0u8; 5]; -/// mem.read(offset, &mut buffer)?; -/// assert_eq!(b"hello", &buffer); -/// Ok(()) -/// } -/// -/// // You can also get direct, unsafe access to the memory, but must manually -/// // ensure that safety invariants are upheld. -/// -/// fn correct_unsafe_examples(mem: &Memory) { -/// // Just like wasm, it's safe to read memory almost at any time. The -/// // gotcha here is that we need to be sure to load from the correct base -/// // pointer and perform the bounds check correctly. So long as this is -/// // all self contained here (e.g. not arbitrary code in the middle) we're -/// // good to go. -/// let byte = unsafe { mem.data_unchecked()[0x123] }; -/// -/// // Short-lived borrows of memory are safe, but they must be scoped and -/// // not have code which modifies/etc `Memory` while the borrow is active. -/// // For example if you want to read a string from memory it is safe to do -/// // so: -/// let string_base = 0xdead; -/// let string_len = 0xbeef; -/// let string = unsafe { -/// let bytes = &mem.data_unchecked()[string_base..][..string_len]; -/// match std::str::from_utf8(bytes) { -/// Ok(s) => s.to_string(), // copy out of wasm memory -/// Err(_) => panic!("not valid utf-8"), -/// } -/// }; -/// -/// // Additionally like wasm you can write to memory at any point in time, -/// // again making sure that after you get the unchecked slice you don't -/// // execute code which could read/write/modify `Memory`: -/// unsafe { -/// mem.data_unchecked_mut()[0x123] = 3; -/// } -/// -/// // When working with *borrows* that point directly into wasm memory you -/// // need to be extremely careful. Any functionality that operates on a -/// // borrow into wasm memory needs to be thoroughly audited to effectively -/// // not touch the `Memory` at all -/// let data_base = 0xfeed; -/// let data_len = 0xface; -/// unsafe { -/// let data = &mem.data_unchecked()[data_base..][..data_len]; -/// host_function_that_doesnt_touch_memory(data); -/// -/// // effectively the same rules apply to mutable borrows -/// let data_mut = &mut mem.data_unchecked_mut()[data_base..][..data_len]; -/// host_function_that_doesnt_touch_memory(data); -/// } -/// } -/// # fn host_function_that_doesnt_touch_memory(_: &[u8]){} -/// ``` -/// -/// It's worth also, however, covering some examples of **incorrect**, -/// **unsafe** usages of `Memory`. Do not do these things! -/// -/// ```rust -/// # use anyhow::Result; -/// use wasmtime::Memory; -/// -/// // NOTE: All code in this function is not safe to execute and may cause -/// // segfaults/undefined behavior at runtime. Do not copy/paste these examples -/// // into production code! -/// unsafe fn unsafe_examples(mem: &Memory) -> Result<()> { -/// // First and foremost, any borrow can be invalidated at any time via the -/// // `Memory::grow` function. This can relocate memory which causes any -/// // previous pointer to be possibly invalid now. -/// let pointer: &u8 = &mem.data_unchecked()[0x100]; -/// mem.grow(1)?; // invalidates `pointer`! -/// // println!("{}", *pointer); // FATAL: use-after-free -/// -/// // Note that the use-after-free also applies to slices, whether they're -/// // slices of bytes or strings. -/// let slice: &[u8] = &mem.data_unchecked()[0x100..0x102]; -/// mem.grow(1)?; // invalidates `slice`! -/// // println!("{:?}", slice); // FATAL: use-after-free -/// -/// // Due to the reference-counted nature of `Memory` note that literal -/// // calls to `Memory::grow` are not sufficient to audit for. You'll need -/// // to be careful that any mutation of `Memory` doesn't happen while -/// // you're holding an active borrow. -/// let slice: &[u8] = &mem.data_unchecked()[0x100..0x102]; -/// some_other_function(); // may invalidate `slice` through another `mem` reference -/// // println!("{:?}", slice); // FATAL: maybe a use-after-free -/// -/// // An especially subtle aspect of accessing a wasm instance's memory is -/// // that you need to be extremely careful about aliasing. Anyone at any -/// // time can call `data_unchecked()` or `data_unchecked_mut()`, which -/// // means you can easily have aliasing mutable references: -/// let ref1: &u8 = &mem.data_unchecked()[0x100]; -/// let ref2: &mut u8 = &mut mem.data_unchecked_mut()[0x100]; -/// // *ref2 = *ref1; // FATAL: violates Rust's aliasing rules -/// -/// // Note that aliasing applies to strings as well, for example this is -/// // not valid because the slices overlap. -/// let slice1: &mut [u8] = &mut mem.data_unchecked_mut()[0x100..][..3]; -/// let slice2: &mut [u8] = &mut mem.data_unchecked_mut()[0x102..][..4]; -/// // println!("{:?} {:?}", slice1, slice2); // FATAL: aliasing mutable pointers -/// -/// Ok(()) -/// } -/// # fn some_other_function() {} -/// ``` -/// -/// Overall there's some general rules of thumb when working with `Memory` and -/// getting raw pointers inside of it: -/// -/// * If you never have a "long lived" pointer into memory, you're likely in the -/// clear. Care still needs to be taken in threaded scenarios or when/where -/// data is read, but you'll be shielded from many classes of issues. -/// * Long-lived pointers must always respect Rust'a aliasing rules. It's ok for -/// shared borrows to overlap with each other, but mutable borrows must -/// overlap with nothing. -/// * Long-lived pointers are only valid if `Memory` isn't used in an unsafe way -/// while the pointer is valid. This includes both aliasing and growth. -/// -/// At this point it's worth reiterating again that working with `Memory` is -/// pretty tricky and that's not great! Proposals such as [interface types] are -/// intended to prevent wasm modules from even needing to import/export memory -/// in the first place, which obviates the need for all of these safety caveats! -/// Additionally over time we're still working out the best idioms to expose in -/// `wasmtime`, so if you've got ideas or questions please feel free to [open an -/// issue]! -/// -/// ## `Memory` Safety and Threads -/// -/// Currently the `wasmtime` crate does not implement the wasm threads proposal, -/// but it is planned to do so. It's additionally worthwhile discussing how this -/// affects memory safety and what was previously just discussed as well. -/// -/// Once threads are added into the mix, all of the above rules still apply. -/// There's an additional, rule, however, that all reads and writes can -/// happen *concurrently*. This effectively means that long-lived borrows into -/// wasm memory are virtually never safe to have. -/// -/// Mutable pointers are fundamentally unsafe to have in a concurrent scenario -/// in the face of arbitrary wasm code. Only if you dynamically know for sure -/// that wasm won't access a region would it be safe to construct a mutable -/// pointer. Additionally even shared pointers are largely unsafe because their -/// underlying contents may change, so unless `UnsafeCell` in one form or -/// another is used everywhere there's no safety. -/// -/// One important point about concurrency is that `Memory::grow` can indeed -/// happen concurrently. This, however, will never relocate the base pointer. -/// Shared memories must always have a maximum size and they will be -/// preallocated such that growth will never relocate the base pointer. The -/// maximum length of the memory, however, will change over time. -/// -/// Overall the general rule of thumb for shared memories is that you must -/// atomically read and write everything. Nothing can be borrowed and everything -/// must be eagerly copied out. -/// -/// [interface types]: https://github.com/webassembly/interface-types -/// [open an issue]: https://github.com/bytecodealliance/wasmtime/issues/new -#[derive(Clone)] -pub struct Memory { - instance: StoreInstanceHandle, - wasmtime_export: wasmtime_runtime::ExportMemory, -} - -impl Memory { - /// Creates a new WebAssembly memory given the configuration of `ty`. - /// - /// The `store` argument is a general location for cache information, and - /// otherwise the memory will immediately be allocated according to the - /// type's configuration. All WebAssembly memory is initialized to zero. - /// - /// # Examples - /// - /// ``` - /// # use wasmtime::*; - /// # fn main() -> anyhow::Result<()> { - /// let engine = Engine::default(); - /// let store = Store::new(&engine); - /// - /// let memory_ty = MemoryType::new(Limits::new(1, None)); - /// let memory = Memory::new(&store, memory_ty); - /// - /// let module = Module::new(&engine, "(module (memory (import \"\" \"\") 1))")?; - /// let instance = Instance::new(&store, &module, &[memory.into()])?; - /// // ... - /// # Ok(()) - /// # } - /// ``` - pub fn new(store: &Store, ty: MemoryType) -> Memory { - let (instance, wasmtime_export) = - generate_memory_export(store, &ty).expect("generated memory"); - Memory { - instance, - wasmtime_export, - } - } - - /// Returns the underlying type of this memory. - /// - /// # Examples - /// - /// ``` - /// # use wasmtime::*; - /// # fn main() -> anyhow::Result<()> { - /// let engine = Engine::default(); - /// let store = Store::new(&engine); - /// let module = Module::new(&engine, "(module (memory (export \"mem\") 1))")?; - /// let instance = Instance::new(&store, &module, &[])?; - /// let memory = instance.get_memory("mem").unwrap(); - /// let ty = memory.ty(); - /// assert_eq!(ty.limits().min(), 1); - /// # Ok(()) - /// # } - /// ``` - pub fn ty(&self) -> MemoryType { - MemoryType::from_wasmtime_memory(&self.wasmtime_export.memory.memory) - } - - /// Safely reads memory contents at the given offset into a buffer. - /// - /// The entire buffer will be filled. - /// - /// If offset + buffer length exceed the current memory capacity, - /// a [`MemoryAccessError`] is returned. - pub fn read(&self, offset: usize, buffer: &mut [u8]) -> Result<(), MemoryAccessError> { - unsafe { - let slice = self - .data_unchecked() - .get(offset..) - .and_then(|s| s.get(..buffer.len())) - .ok_or(MemoryAccessError { _private: () })?; - buffer.copy_from_slice(slice); - Ok(()) - } - } - - /// Safely writes contents of a buffer to this memory at the given offset. - /// - /// If the offset + buffer length exceed current memory capacity, a - /// [`MemoryAccessError`] is returned. - pub fn write(&self, offset: usize, buffer: &[u8]) -> Result<(), MemoryAccessError> { - unsafe { - self.data_unchecked_mut() - .get_mut(offset..) - .and_then(|s| s.get_mut(..buffer.len())) - .ok_or(MemoryAccessError { _private: () })? - .copy_from_slice(buffer); - Ok(()) - } - } - - /// Returns this memory as a slice view that can be read natively in Rust. - /// - /// # Safety - /// - /// This is an unsafe operation because there is no guarantee that the - /// following operations do not happen concurrently while the slice is in - /// use: - /// - /// * Data could be modified by calling into a wasm module. - /// * Memory could be relocated through growth by calling into a wasm - /// module. - /// * When threads are supported, non-atomic reads will race with other - /// writes. - /// - /// Extreme care need be taken when the data of a `Memory` is read. The - /// above invariants all need to be upheld at a bare minimum, and in - /// general you'll need to ensure that while you're looking at slice you're - /// the only one who can possibly look at the slice and read/write it. - /// - /// Be sure to keep in mind that `Memory` is reference counted, meaning - /// that there may be other users of this `Memory` instance elsewhere in - /// your program. Additionally `Memory` can be shared and used in any number - /// of wasm instances, so calling any wasm code should be considered - /// dangerous while you're holding a slice of memory. - /// - /// For more information and examples see the documentation on the - /// [`Memory`] type. - pub unsafe fn data_unchecked(&self) -> &[u8] { - self.data_unchecked_mut() - } - - /// Returns this memory as a slice view that can be read and written - /// natively in Rust. - /// - /// # Safety - /// - /// All of the same safety caveats of [`Memory::data_unchecked`] apply - /// here, doubly so because this is returning a mutable slice! As a - /// double-extra reminder, remember that `Memory` is reference counted, so - /// you can very easily acquire two mutable slices by simply calling this - /// function twice. Extreme caution should be used when using this method, - /// and in general you probably want to result to unsafe accessors and the - /// `data` methods below. - /// - /// For more information and examples see the documentation on the - /// [`Memory`] type. - pub unsafe fn data_unchecked_mut(&self) -> &mut [u8] { - let definition = &*self.wasmtime_export.definition; - slice::from_raw_parts_mut(definition.base, definition.current_length) - } - - /// Returns the base pointer, in the host's address space, that the memory - /// is located at. - /// - /// When reading and manipulating memory be sure to read up on the caveats - /// of [`Memory::data_unchecked`] to make sure that you can safely - /// read/write the memory. - /// - /// For more information and examples see the documentation on the - /// [`Memory`] type. - pub fn data_ptr(&self) -> *mut u8 { - unsafe { (*self.wasmtime_export.definition).base } - } - - /// Returns the byte length of this memory. - /// - /// The returned value will be a multiple of the wasm page size, 64k. - /// - /// For more information and examples see the documentation on the - /// [`Memory`] type. - pub fn data_size(&self) -> usize { - unsafe { (*self.wasmtime_export.definition).current_length } - } - - /// Returns the size, in pages, of this wasm memory. - pub fn size(&self) -> u32 { - (self.data_size() / wasmtime_environ::WASM_PAGE_SIZE as usize) as u32 - } - - /// Grows this WebAssembly memory by `delta` pages. - /// - /// This will attempt to add `delta` more pages of memory on to the end of - /// this `Memory` instance. If successful this may relocate the memory and - /// cause [`Memory::data_ptr`] to return a new value. Additionally previous - /// slices into this memory may no longer be valid. - /// - /// On success returns the number of pages this memory previously had - /// before the growth succeeded. - /// - /// # Errors - /// - /// Returns an error if memory could not be grown, for example if it exceeds - /// the maximum limits of this memory. - /// - /// # Examples - /// - /// ``` - /// # use wasmtime::*; - /// # fn main() -> anyhow::Result<()> { - /// let engine = Engine::default(); - /// let store = Store::new(&engine); - /// let module = Module::new(&engine, "(module (memory (export \"mem\") 1 2))")?; - /// let instance = Instance::new(&store, &module, &[])?; - /// let memory = instance.get_memory("mem").unwrap(); - /// - /// assert_eq!(memory.size(), 1); - /// assert_eq!(memory.grow(1)?, 1); - /// assert_eq!(memory.size(), 2); - /// assert!(memory.grow(1).is_err()); - /// assert_eq!(memory.size(), 2); - /// assert_eq!(memory.grow(0)?, 2); - /// # Ok(()) - /// # } - /// ``` - pub fn grow(&self, delta: u32) -> Result { - let index = self - .instance - .memory_index(unsafe { &*self.wasmtime_export.definition }); - self.instance - .memory_grow(index, delta) - .ok_or_else(|| anyhow!("failed to grow memory")) - } - - pub(crate) unsafe fn from_wasmtime_memory( - wasmtime_export: &wasmtime_runtime::ExportMemory, - store: &Store, - ) -> Memory { - Memory { - instance: store.existing_vmctx(wasmtime_export.vmctx), - wasmtime_export: wasmtime_export.clone(), - } - } - - pub(crate) fn wasmtime_ty(&self) -> &wasmtime_environ::wasm::Memory { - &self.wasmtime_export.memory.memory - } - - pub(crate) fn vmimport(&self) -> wasmtime_runtime::VMMemoryImport { - wasmtime_runtime::VMMemoryImport { - from: self.wasmtime_export.definition, - vmctx: self.wasmtime_export.vmctx, - } - } - - pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportMemory { - &self.wasmtime_export - } -} - -/// A linear memory. This trait provides an interface for raw memory buffers which are used -/// by wasmtime, e.g. inside ['Memory']. Such buffers are in principle not thread safe. -/// By implementing this trait together with MemoryCreator, -/// one can supply wasmtime with custom allocated host managed memory. -/// -/// # Safety -/// The memory should be page aligned and a multiple of page size. -/// To prevent possible silent overflows, the memory should be protected by a guard page. -/// Additionally the safety concerns explained in ['Memory'], for accessing the memory -/// apply here as well. -/// -/// Note that this is a relatively new and experimental feature and it is recommended -/// to be familiar with wasmtime runtime code to use it. -pub unsafe trait LinearMemory { - /// Returns the number of allocated wasm pages. - fn size(&self) -> u32; - - /// Grow memory by the specified amount of wasm pages. - /// - /// Returns `None` if memory can't be grown by the specified amount - /// of wasm pages. - fn grow(&self, delta: u32) -> Option; - - /// Return the allocated memory as a mutable pointer to u8. - fn as_ptr(&self) -> *mut u8; -} - -/// A memory creator. Can be used to provide a memory creator -/// to wasmtime which supplies host managed memory. -/// -/// # Safety -/// This trait is unsafe, as the memory safety depends on proper implementation of -/// memory management. Memories created by the MemoryCreator should always be treated -/// as owned by wasmtime instance, and any modification of them outside of wasmtime -/// invoked routines is unsafe and may lead to corruption. -/// -/// Note that this is a relatively new and experimental feature and it is recommended -/// to be familiar with wasmtime runtime code to use it. -pub unsafe trait MemoryCreator: Send + Sync { - /// Create a new `LinearMemory` object from the specified parameters. - /// - /// The type of memory being created is specified by `ty` which indicates - /// both the minimum and maximum size, in wasm pages. - /// - /// The `reserved_size_in_bytes` value indicates the expected size of the - /// reservation that is to be made for this memory. If this value is `None` - /// than the implementation is free to allocate memory as it sees fit. If - /// the value is `Some`, however, then the implementation is expected to - /// reserve that many bytes for the memory's allocation, plus the guard - /// size at the end. Note that this reservation need only be a virtual - /// memory reservation, physical memory does not need to be allocated - /// immediately. In this case `grow` should never move the base pointer and - /// the maximum size of `ty` is guaranteed to fit within `reserved_size_in_bytes`. - /// - /// The `guard_size_in_bytes` parameter indicates how many bytes of space, after the - /// memory allocation, is expected to be unmapped. JIT code will elide - /// bounds checks based on the `guard_size_in_bytes` provided, so for JIT code to - /// work correctly the memory returned will need to be properly guarded with - /// `guard_size_in_bytes` bytes left unmapped after the base allocation. - /// - /// Note that the `reserved_size_in_bytes` and `guard_size_in_bytes` options are tuned from - /// the various [`Config`](crate::Config) methods about memory - /// sizes/guards. Additionally these two values are guaranteed to be - /// multiples of the system page size. - fn new_memory( - &self, - ty: MemoryType, - reserved_size_in_bytes: Option, - guard_size_in_bytes: u64, - ) -> Result, String>; -} - -#[cfg(test)] -mod tests { - use crate::*; - - // Assert that creating a memory via `Memory::new` respects the limits/tunables - // in `Config`. - #[test] - fn respect_tunables() { - let mut cfg = Config::new(); - cfg.static_memory_maximum_size(0) - .dynamic_memory_guard_size(0); - let store = Store::new(&Engine::new(&cfg)); - let ty = MemoryType::new(Limits::new(1, None)); - let mem = Memory::new(&store, ty); - assert_eq!(mem.wasmtime_export.memory.offset_guard_size, 0); - match mem.wasmtime_export.memory.style { - wasmtime_environ::MemoryStyle::Dynamic => {} - other => panic!("unexpected style {:?}", other), - } - } -} - // Exports /// An exported WebAssembly value. diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index 2248eec5a7..9deb3638d0 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -241,6 +241,7 @@ mod frame_info; mod func; mod instance; mod linker; +mod memory; mod module; mod r#ref; mod sig_registry; @@ -257,6 +258,7 @@ pub use crate::frame_info::{FrameInfo, FrameSymbol}; pub use crate::func::*; pub use crate::instance::Instance; pub use crate::linker::*; +pub use crate::memory::*; pub use crate::module::Module; pub use crate::r#ref::ExternRef; pub use crate::store::*; diff --git a/crates/wasmtime/src/memory.rs b/crates/wasmtime/src/memory.rs new file mode 100644 index 0000000000..f5c82292a2 --- /dev/null +++ b/crates/wasmtime/src/memory.rs @@ -0,0 +1,577 @@ +use crate::trampoline::{generate_memory_export, StoreInstanceHandle}; +use crate::{MemoryType, Store}; +use anyhow::{anyhow, Result}; +use std::slice; + +/// Error for out of bounds [`Memory`] access. +#[derive(Debug)] +#[non_exhaustive] +pub struct MemoryAccessError { + // Keep struct internals private for future extensibility. + _private: (), +} + +impl std::fmt::Display for MemoryAccessError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "out of bounds memory access") + } +} + +impl std::error::Error for MemoryAccessError {} + +/// A WebAssembly linear memory. +/// +/// WebAssembly memories represent a contiguous array of bytes that have a size +/// that is always a multiple of the WebAssembly page size, currently 64 +/// kilobytes. +/// +/// WebAssembly memory is used for global data, statics in C/C++/Rust, shadow +/// stack memory, etc. Accessing wasm memory is generally quite fast! +/// +/// # `Memory` and `Clone` +/// +/// Memories are internally reference counted so you can `clone` a `Memory`. The +/// cloning process only performs a shallow clone, so two cloned `Memory` +/// instances are equivalent in their functionality. +/// +/// # `Memory` and threads +/// +/// It is intended that `Memory` is safe to share between threads. At this time +/// this is not implemented in `wasmtime`, however. This is planned to be +/// implemented though! +/// +/// # `Memory` and Safety +/// +/// Linear memory is a lynchpin of safety for WebAssembly, but it turns out +/// there are very few ways to safely inspect the contents of a memory from the +/// host (Rust). This is because memory safety is quite tricky when working with +/// a `Memory` and we're still working out the best idioms to encapsulate +/// everything safely where it's efficient and ergonomic. This section of +/// documentation, however, is intended to help educate a bit what is and isn't +/// safe when working with `Memory`. +/// +/// For safety purposes you can think of a `Memory` as a glorified +/// `Rc>>`. There are a few consequences of this +/// interpretation: +/// +/// * At any time someone else may have access to the memory (hence the `Rc`). +/// This could be a wasm instance, other host code, or a set of wasm instances +/// which all reference a `Memory`. When in doubt assume someone else has a +/// handle to your `Memory`. +/// +/// * At any time, memory can be read from or written to (hence the +/// `UnsafeCell`). Anyone with a handle to a wasm memory can read/write to it. +/// Primarily other instances can execute the `load` and `store` family of +/// instructions, as well as any other which modifies or reads memory. +/// +/// * At any time memory may grow (hence the `Vec<..>`). Growth may relocate the +/// base memory pointer (similar to how `vec.push(...)` can change the result +/// of `.as_ptr()`) +/// +/// So given that we're working roughly with `Rc>>` that's a +/// lot to keep in mind! It's hopefully though sort of setting the stage as to +/// what you can safely do with memories. +/// +/// Let's run through a few safe examples first of how you can use a `Memory`. +/// +/// ```rust +/// use wasmtime::{Memory, MemoryAccessError}; +/// +/// // Memory can be read and written safely with the `Memory::read` and +/// // `Memory::write` methods. +/// // An error is returned if the copy did not succeed. +/// fn safe_examples(mem: &Memory) -> Result<(), MemoryAccessError> { +/// let offset = 5; +/// mem.write(offset, b"hello")?; +/// let mut buffer = [0u8; 5]; +/// mem.read(offset, &mut buffer)?; +/// assert_eq!(b"hello", &buffer); +/// Ok(()) +/// } +/// +/// // You can also get direct, unsafe access to the memory, but must manually +/// // ensure that safety invariants are upheld. +/// +/// fn correct_unsafe_examples(mem: &Memory) { +/// // Just like wasm, it's safe to read memory almost at any time. The +/// // gotcha here is that we need to be sure to load from the correct base +/// // pointer and perform the bounds check correctly. So long as this is +/// // all self contained here (e.g. not arbitrary code in the middle) we're +/// // good to go. +/// let byte = unsafe { mem.data_unchecked()[0x123] }; +/// +/// // Short-lived borrows of memory are safe, but they must be scoped and +/// // not have code which modifies/etc `Memory` while the borrow is active. +/// // For example if you want to read a string from memory it is safe to do +/// // so: +/// let string_base = 0xdead; +/// let string_len = 0xbeef; +/// let string = unsafe { +/// let bytes = &mem.data_unchecked()[string_base..][..string_len]; +/// match std::str::from_utf8(bytes) { +/// Ok(s) => s.to_string(), // copy out of wasm memory +/// Err(_) => panic!("not valid utf-8"), +/// } +/// }; +/// +/// // Additionally like wasm you can write to memory at any point in time, +/// // again making sure that after you get the unchecked slice you don't +/// // execute code which could read/write/modify `Memory`: +/// unsafe { +/// mem.data_unchecked_mut()[0x123] = 3; +/// } +/// +/// // When working with *borrows* that point directly into wasm memory you +/// // need to be extremely careful. Any functionality that operates on a +/// // borrow into wasm memory needs to be thoroughly audited to effectively +/// // not touch the `Memory` at all +/// let data_base = 0xfeed; +/// let data_len = 0xface; +/// unsafe { +/// let data = &mem.data_unchecked()[data_base..][..data_len]; +/// host_function_that_doesnt_touch_memory(data); +/// +/// // effectively the same rules apply to mutable borrows +/// let data_mut = &mut mem.data_unchecked_mut()[data_base..][..data_len]; +/// host_function_that_doesnt_touch_memory(data); +/// } +/// } +/// # fn host_function_that_doesnt_touch_memory(_: &[u8]){} +/// ``` +/// +/// It's worth also, however, covering some examples of **incorrect**, +/// **unsafe** usages of `Memory`. Do not do these things! +/// +/// ```rust +/// # use anyhow::Result; +/// use wasmtime::Memory; +/// +/// // NOTE: All code in this function is not safe to execute and may cause +/// // segfaults/undefined behavior at runtime. Do not copy/paste these examples +/// // into production code! +/// unsafe fn unsafe_examples(mem: &Memory) -> Result<()> { +/// // First and foremost, any borrow can be invalidated at any time via the +/// // `Memory::grow` function. This can relocate memory which causes any +/// // previous pointer to be possibly invalid now. +/// let pointer: &u8 = &mem.data_unchecked()[0x100]; +/// mem.grow(1)?; // invalidates `pointer`! +/// // println!("{}", *pointer); // FATAL: use-after-free +/// +/// // Note that the use-after-free also applies to slices, whether they're +/// // slices of bytes or strings. +/// let slice: &[u8] = &mem.data_unchecked()[0x100..0x102]; +/// mem.grow(1)?; // invalidates `slice`! +/// // println!("{:?}", slice); // FATAL: use-after-free +/// +/// // Due to the reference-counted nature of `Memory` note that literal +/// // calls to `Memory::grow` are not sufficient to audit for. You'll need +/// // to be careful that any mutation of `Memory` doesn't happen while +/// // you're holding an active borrow. +/// let slice: &[u8] = &mem.data_unchecked()[0x100..0x102]; +/// some_other_function(); // may invalidate `slice` through another `mem` reference +/// // println!("{:?}", slice); // FATAL: maybe a use-after-free +/// +/// // An especially subtle aspect of accessing a wasm instance's memory is +/// // that you need to be extremely careful about aliasing. Anyone at any +/// // time can call `data_unchecked()` or `data_unchecked_mut()`, which +/// // means you can easily have aliasing mutable references: +/// let ref1: &u8 = &mem.data_unchecked()[0x100]; +/// let ref2: &mut u8 = &mut mem.data_unchecked_mut()[0x100]; +/// // *ref2 = *ref1; // FATAL: violates Rust's aliasing rules +/// +/// // Note that aliasing applies to strings as well, for example this is +/// // not valid because the slices overlap. +/// let slice1: &mut [u8] = &mut mem.data_unchecked_mut()[0x100..][..3]; +/// let slice2: &mut [u8] = &mut mem.data_unchecked_mut()[0x102..][..4]; +/// // println!("{:?} {:?}", slice1, slice2); // FATAL: aliasing mutable pointers +/// +/// Ok(()) +/// } +/// # fn some_other_function() {} +/// ``` +/// +/// Overall there's some general rules of thumb when working with `Memory` and +/// getting raw pointers inside of it: +/// +/// * If you never have a "long lived" pointer into memory, you're likely in the +/// clear. Care still needs to be taken in threaded scenarios or when/where +/// data is read, but you'll be shielded from many classes of issues. +/// * Long-lived pointers must always respect Rust'a aliasing rules. It's ok for +/// shared borrows to overlap with each other, but mutable borrows must +/// overlap with nothing. +/// * Long-lived pointers are only valid if `Memory` isn't used in an unsafe way +/// while the pointer is valid. This includes both aliasing and growth. +/// +/// At this point it's worth reiterating again that working with `Memory` is +/// pretty tricky and that's not great! Proposals such as [interface types] are +/// intended to prevent wasm modules from even needing to import/export memory +/// in the first place, which obviates the need for all of these safety caveats! +/// Additionally over time we're still working out the best idioms to expose in +/// `wasmtime`, so if you've got ideas or questions please feel free to [open an +/// issue]! +/// +/// ## `Memory` Safety and Threads +/// +/// Currently the `wasmtime` crate does not implement the wasm threads proposal, +/// but it is planned to do so. It's additionally worthwhile discussing how this +/// affects memory safety and what was previously just discussed as well. +/// +/// Once threads are added into the mix, all of the above rules still apply. +/// There's an additional, rule, however, that all reads and writes can +/// happen *concurrently*. This effectively means that long-lived borrows into +/// wasm memory are virtually never safe to have. +/// +/// Mutable pointers are fundamentally unsafe to have in a concurrent scenario +/// in the face of arbitrary wasm code. Only if you dynamically know for sure +/// that wasm won't access a region would it be safe to construct a mutable +/// pointer. Additionally even shared pointers are largely unsafe because their +/// underlying contents may change, so unless `UnsafeCell` in one form or +/// another is used everywhere there's no safety. +/// +/// One important point about concurrency is that `Memory::grow` can indeed +/// happen concurrently. This, however, will never relocate the base pointer. +/// Shared memories must always have a maximum size and they will be +/// preallocated such that growth will never relocate the base pointer. The +/// maximum length of the memory, however, will change over time. +/// +/// Overall the general rule of thumb for shared memories is that you must +/// atomically read and write everything. Nothing can be borrowed and everything +/// must be eagerly copied out. +/// +/// [interface types]: https://github.com/webassembly/interface-types +/// [open an issue]: https://github.com/bytecodealliance/wasmtime/issues/new +#[derive(Clone)] +pub struct Memory { + pub(crate) instance: StoreInstanceHandle, + wasmtime_export: wasmtime_runtime::ExportMemory, +} + +impl Memory { + /// Creates a new WebAssembly memory given the configuration of `ty`. + /// + /// The `store` argument is a general location for cache information, and + /// otherwise the memory will immediately be allocated according to the + /// type's configuration. All WebAssembly memory is initialized to zero. + /// + /// # Examples + /// + /// ``` + /// # use wasmtime::*; + /// # fn main() -> anyhow::Result<()> { + /// let engine = Engine::default(); + /// let store = Store::new(&engine); + /// + /// let memory_ty = MemoryType::new(Limits::new(1, None)); + /// let memory = Memory::new(&store, memory_ty); + /// + /// let module = Module::new(&engine, "(module (memory (import \"\" \"\") 1))")?; + /// let instance = Instance::new(&store, &module, &[memory.into()])?; + /// // ... + /// # Ok(()) + /// # } + /// ``` + pub fn new(store: &Store, ty: MemoryType) -> Memory { + let (instance, wasmtime_export) = + generate_memory_export(store, &ty).expect("generated memory"); + Memory { + instance, + wasmtime_export, + } + } + + /// Returns the underlying type of this memory. + /// + /// # Examples + /// + /// ``` + /// # use wasmtime::*; + /// # fn main() -> anyhow::Result<()> { + /// let engine = Engine::default(); + /// let store = Store::new(&engine); + /// let module = Module::new(&engine, "(module (memory (export \"mem\") 1))")?; + /// let instance = Instance::new(&store, &module, &[])?; + /// let memory = instance.get_memory("mem").unwrap(); + /// let ty = memory.ty(); + /// assert_eq!(ty.limits().min(), 1); + /// # Ok(()) + /// # } + /// ``` + pub fn ty(&self) -> MemoryType { + MemoryType::from_wasmtime_memory(&self.wasmtime_export.memory.memory) + } + + /// Safely reads memory contents at the given offset into a buffer. + /// + /// The entire buffer will be filled. + /// + /// If offset + buffer length exceed the current memory capacity, + /// a [`MemoryAccessError`] is returned. + pub fn read(&self, offset: usize, buffer: &mut [u8]) -> Result<(), MemoryAccessError> { + unsafe { + let slice = self + .data_unchecked() + .get(offset..) + .and_then(|s| s.get(..buffer.len())) + .ok_or(MemoryAccessError { _private: () })?; + buffer.copy_from_slice(slice); + Ok(()) + } + } + + /// Safely writes contents of a buffer to this memory at the given offset. + /// + /// If the offset + buffer length exceed current memory capacity, a + /// [`MemoryAccessError`] is returned. + pub fn write(&self, offset: usize, buffer: &[u8]) -> Result<(), MemoryAccessError> { + unsafe { + self.data_unchecked_mut() + .get_mut(offset..) + .and_then(|s| s.get_mut(..buffer.len())) + .ok_or(MemoryAccessError { _private: () })? + .copy_from_slice(buffer); + Ok(()) + } + } + + /// Returns this memory as a slice view that can be read natively in Rust. + /// + /// # Safety + /// + /// This is an unsafe operation because there is no guarantee that the + /// following operations do not happen concurrently while the slice is in + /// use: + /// + /// * Data could be modified by calling into a wasm module. + /// * Memory could be relocated through growth by calling into a wasm + /// module. + /// * When threads are supported, non-atomic reads will race with other + /// writes. + /// + /// Extreme care need be taken when the data of a `Memory` is read. The + /// above invariants all need to be upheld at a bare minimum, and in + /// general you'll need to ensure that while you're looking at slice you're + /// the only one who can possibly look at the slice and read/write it. + /// + /// Be sure to keep in mind that `Memory` is reference counted, meaning + /// that there may be other users of this `Memory` instance elsewhere in + /// your program. Additionally `Memory` can be shared and used in any number + /// of wasm instances, so calling any wasm code should be considered + /// dangerous while you're holding a slice of memory. + /// + /// For more information and examples see the documentation on the + /// [`Memory`] type. + pub unsafe fn data_unchecked(&self) -> &[u8] { + self.data_unchecked_mut() + } + + /// Returns this memory as a slice view that can be read and written + /// natively in Rust. + /// + /// # Safety + /// + /// All of the same safety caveats of [`Memory::data_unchecked`] apply + /// here, doubly so because this is returning a mutable slice! As a + /// double-extra reminder, remember that `Memory` is reference counted, so + /// you can very easily acquire two mutable slices by simply calling this + /// function twice. Extreme caution should be used when using this method, + /// and in general you probably want to result to unsafe accessors and the + /// `data` methods below. + /// + /// For more information and examples see the documentation on the + /// [`Memory`] type. + pub unsafe fn data_unchecked_mut(&self) -> &mut [u8] { + let definition = &*self.wasmtime_export.definition; + slice::from_raw_parts_mut(definition.base, definition.current_length) + } + + /// Returns the base pointer, in the host's address space, that the memory + /// is located at. + /// + /// When reading and manipulating memory be sure to read up on the caveats + /// of [`Memory::data_unchecked`] to make sure that you can safely + /// read/write the memory. + /// + /// For more information and examples see the documentation on the + /// [`Memory`] type. + pub fn data_ptr(&self) -> *mut u8 { + unsafe { (*self.wasmtime_export.definition).base } + } + + /// Returns the byte length of this memory. + /// + /// The returned value will be a multiple of the wasm page size, 64k. + /// + /// For more information and examples see the documentation on the + /// [`Memory`] type. + pub fn data_size(&self) -> usize { + unsafe { (*self.wasmtime_export.definition).current_length } + } + + /// Returns the size, in pages, of this wasm memory. + pub fn size(&self) -> u32 { + (self.data_size() / wasmtime_environ::WASM_PAGE_SIZE as usize) as u32 + } + + /// Grows this WebAssembly memory by `delta` pages. + /// + /// This will attempt to add `delta` more pages of memory on to the end of + /// this `Memory` instance. If successful this may relocate the memory and + /// cause [`Memory::data_ptr`] to return a new value. Additionally previous + /// slices into this memory may no longer be valid. + /// + /// On success returns the number of pages this memory previously had + /// before the growth succeeded. + /// + /// # Errors + /// + /// Returns an error if memory could not be grown, for example if it exceeds + /// the maximum limits of this memory. + /// + /// # Examples + /// + /// ``` + /// # use wasmtime::*; + /// # fn main() -> anyhow::Result<()> { + /// let engine = Engine::default(); + /// let store = Store::new(&engine); + /// let module = Module::new(&engine, "(module (memory (export \"mem\") 1 2))")?; + /// let instance = Instance::new(&store, &module, &[])?; + /// let memory = instance.get_memory("mem").unwrap(); + /// + /// assert_eq!(memory.size(), 1); + /// assert_eq!(memory.grow(1)?, 1); + /// assert_eq!(memory.size(), 2); + /// assert!(memory.grow(1).is_err()); + /// assert_eq!(memory.size(), 2); + /// assert_eq!(memory.grow(0)?, 2); + /// # Ok(()) + /// # } + /// ``` + pub fn grow(&self, delta: u32) -> Result { + let index = self + .instance + .memory_index(unsafe { &*self.wasmtime_export.definition }); + self.instance + .memory_grow(index, delta) + .ok_or_else(|| anyhow!("failed to grow memory")) + } + + pub(crate) unsafe fn from_wasmtime_memory( + wasmtime_export: &wasmtime_runtime::ExportMemory, + store: &Store, + ) -> Memory { + Memory { + instance: store.existing_vmctx(wasmtime_export.vmctx), + wasmtime_export: wasmtime_export.clone(), + } + } + + pub(crate) fn wasmtime_ty(&self) -> &wasmtime_environ::wasm::Memory { + &self.wasmtime_export.memory.memory + } + + pub(crate) fn vmimport(&self) -> wasmtime_runtime::VMMemoryImport { + wasmtime_runtime::VMMemoryImport { + from: self.wasmtime_export.definition, + vmctx: self.wasmtime_export.vmctx, + } + } + + pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::ExportMemory { + &self.wasmtime_export + } +} + +/// A linear memory. This trait provides an interface for raw memory buffers which are used +/// by wasmtime, e.g. inside ['Memory']. Such buffers are in principle not thread safe. +/// By implementing this trait together with MemoryCreator, +/// one can supply wasmtime with custom allocated host managed memory. +/// +/// # Safety +/// The memory should be page aligned and a multiple of page size. +/// To prevent possible silent overflows, the memory should be protected by a guard page. +/// Additionally the safety concerns explained in ['Memory'], for accessing the memory +/// apply here as well. +/// +/// Note that this is a relatively new and experimental feature and it is recommended +/// to be familiar with wasmtime runtime code to use it. +pub unsafe trait LinearMemory { + /// Returns the number of allocated wasm pages. + fn size(&self) -> u32; + + /// Grow memory by the specified amount of wasm pages. + /// + /// Returns `None` if memory can't be grown by the specified amount + /// of wasm pages. + fn grow(&self, delta: u32) -> Option; + + /// Return the allocated memory as a mutable pointer to u8. + fn as_ptr(&self) -> *mut u8; +} + +/// A memory creator. Can be used to provide a memory creator +/// to wasmtime which supplies host managed memory. +/// +/// # Safety +/// This trait is unsafe, as the memory safety depends on proper implementation of +/// memory management. Memories created by the MemoryCreator should always be treated +/// as owned by wasmtime instance, and any modification of them outside of wasmtime +/// invoked routines is unsafe and may lead to corruption. +/// +/// Note that this is a relatively new and experimental feature and it is recommended +/// to be familiar with wasmtime runtime code to use it. +pub unsafe trait MemoryCreator: Send + Sync { + /// Create a new `LinearMemory` object from the specified parameters. + /// + /// The type of memory being created is specified by `ty` which indicates + /// both the minimum and maximum size, in wasm pages. + /// + /// The `reserved_size_in_bytes` value indicates the expected size of the + /// reservation that is to be made for this memory. If this value is `None` + /// than the implementation is free to allocate memory as it sees fit. If + /// the value is `Some`, however, then the implementation is expected to + /// reserve that many bytes for the memory's allocation, plus the guard + /// size at the end. Note that this reservation need only be a virtual + /// memory reservation, physical memory does not need to be allocated + /// immediately. In this case `grow` should never move the base pointer and + /// the maximum size of `ty` is guaranteed to fit within `reserved_size_in_bytes`. + /// + /// The `guard_size_in_bytes` parameter indicates how many bytes of space, after the + /// memory allocation, is expected to be unmapped. JIT code will elide + /// bounds checks based on the `guard_size_in_bytes` provided, so for JIT code to + /// work correctly the memory returned will need to be properly guarded with + /// `guard_size_in_bytes` bytes left unmapped after the base allocation. + /// + /// Note that the `reserved_size_in_bytes` and `guard_size_in_bytes` options are tuned from + /// the various [`Config`](crate::Config) methods about memory + /// sizes/guards. Additionally these two values are guaranteed to be + /// multiples of the system page size. + fn new_memory( + &self, + ty: MemoryType, + reserved_size_in_bytes: Option, + guard_size_in_bytes: u64, + ) -> Result, String>; +} + +#[cfg(test)] +mod tests { + use crate::*; + + // Assert that creating a memory via `Memory::new` respects the limits/tunables + // in `Config`. + #[test] + fn respect_tunables() { + let mut cfg = Config::new(); + cfg.static_memory_maximum_size(0) + .dynamic_memory_guard_size(0); + let store = Store::new(&Engine::new(&cfg)); + let ty = MemoryType::new(Limits::new(1, None)); + let mem = Memory::new(&store, ty); + assert_eq!(mem.wasmtime_export.memory.offset_guard_size, 0); + match mem.wasmtime_export.memory.style { + wasmtime_environ::MemoryStyle::Dynamic => {} + other => panic!("unexpected style {:?}", other), + } + } +} diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs index 7993c8e87c..0bfb2bfff9 100644 --- a/crates/wasmtime/src/trampoline/memory.rs +++ b/crates/wasmtime/src/trampoline/memory.rs @@ -1,5 +1,5 @@ use super::create_handle::create_handle; -use crate::externals::{LinearMemory, MemoryCreator}; +use crate::memory::{LinearMemory, MemoryCreator}; use crate::trampoline::StoreInstanceHandle; use crate::Store; use crate::{Limits, MemoryType}; From 4f3bc1d5d48a958e185fb9c3ac7f6c4b238c24ee Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Tue, 26 Jan 2021 10:11:29 -0800 Subject: [PATCH 40/55] wasmtime: clarify `Memory::{read,write}` behavior with out-of-bounds ranges This documents that we will never do partial reads/writes, and expands our existing tests to assert this. --- crates/wasmtime/src/memory.rs | 9 +++++---- tests/all/externals.rs | 7 +++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/wasmtime/src/memory.rs b/crates/wasmtime/src/memory.rs index f5c82292a2..0b2d21e2b6 100644 --- a/crates/wasmtime/src/memory.rs +++ b/crates/wasmtime/src/memory.rs @@ -304,8 +304,8 @@ impl Memory { /// /// The entire buffer will be filled. /// - /// If offset + buffer length exceed the current memory capacity, - /// a [`MemoryAccessError`] is returned. + /// If offset + buffer length exceed the current memory capacity, then the + /// buffer is left untouched and a [`MemoryAccessError`] is returned. pub fn read(&self, offset: usize, buffer: &mut [u8]) -> Result<(), MemoryAccessError> { unsafe { let slice = self @@ -320,8 +320,9 @@ impl Memory { /// Safely writes contents of a buffer to this memory at the given offset. /// - /// If the offset + buffer length exceed current memory capacity, a - /// [`MemoryAccessError`] is returned. + /// If the offset + buffer length exceed current memory capacity, then none + /// of the buffer is written to memory and a [`MemoryAccessError`] is + /// returned. pub fn write(&self, offset: usize, buffer: &[u8]) -> Result<(), MemoryAccessError> { unsafe { self.data_unchecked_mut() diff --git a/tests/all/externals.rs b/tests/all/externals.rs index 6c13aabc17..db70f7e3cb 100644 --- a/tests/all/externals.rs +++ b/tests/all/externals.rs @@ -363,11 +363,18 @@ fn read_write_memory_via_api() { let res = mem.write(mem.data_size() - value.len() + 1, value); assert!(res.is_err()); + assert_ne!( + unsafe { mem.data_unchecked()[mem.data_size() - value.len() + 1] }, + value[0], + "no data is written", + ); // Out of bounds read. + buffer[0] = 0x42; let res = mem.read(mem.data_size() - buffer.len() + 1, &mut buffer); assert!(res.is_err()); + assert_eq!(buffer[0], 0x42, "no data is read"); // Read offset overflow. let res = mem.read(usize::MAX, &mut buffer); From 503129ad9103028cd0b56ae969848ec1bbf25bc7 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 26 Jan 2021 15:59:12 -0600 Subject: [PATCH 41/55] Add a method to share `Config` across machines (#2608) With `Module::{serialize,deserialize}` it should be possible to share wasmtime modules across machines or CPUs. Serialization, however, embeds a hash of all configuration values, including cranelift compilation settings. By default wasmtime's selection of the native ISA would enable ISA flags according to CPU features available on the host, but the same CPU features may not be available across two machines. This commit adds a `Config::cranelift_clear_cpu_flags` method which allows clearing the target-specific ISA flags that are automatically inferred by default for the native CPU. Options can then be incrementally built back up as-desired with teh `cranelift_other_flag` method. --- cranelift/codegen/meta/src/gen_settings.rs | 2 +- cranelift/codegen/src/isa/aarch64/mod.rs | 5 ++ cranelift/codegen/src/isa/arm32/mod.rs | 5 ++ cranelift/codegen/src/isa/mod.rs | 5 ++ cranelift/codegen/src/isa/riscv/mod.rs | 6 +++ cranelift/codegen/src/isa/x64/mod.rs | 6 +++ cranelift/codegen/src/isa/x86/mod.rs | 6 +++ cranelift/codegen/src/machinst/adapter.rs | 5 ++ cranelift/codegen/src/machinst/mod.rs | 5 ++ cranelift/codegen/src/settings.rs | 2 +- cranelift/native/src/lib.rs | 7 +-- crates/jit/src/compiler.rs | 8 +--- crates/jit/src/native.rs | 5 ++ crates/wasmtime/src/config.rs | 14 ++++++ tests/all/main.rs | 1 + tests/all/module.rs | 54 ++++++++++++++++++++++ 16 files changed, 125 insertions(+), 11 deletions(-) create mode 100644 tests/all/module.rs diff --git a/cranelift/codegen/meta/src/gen_settings.rs b/cranelift/codegen/meta/src/gen_settings.rs index 2ed5941b80..a70ddccfe1 100644 --- a/cranelift/codegen/meta/src/gen_settings.rs +++ b/cranelift/codegen/meta/src/gen_settings.rs @@ -418,7 +418,7 @@ fn gen_display(group: &SettingGroup, fmt: &mut Formatter) { fn gen_group(group: &SettingGroup, parent: ParentGroup, fmt: &mut Formatter) { // Generate struct. - fmtln!(fmt, "#[derive(Clone)]"); + fmtln!(fmt, "#[derive(Clone, Hash)]"); fmt.doc_comment(format!("Flags group `{}`.", group.name)); fmtln!(fmt, "pub struct Flags {"); fmt.indent(|fmt| { diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs index 11eb0a6ea6..af13cb70c0 100644 --- a/cranelift/codegen/src/isa/aarch64/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/mod.rs @@ -8,6 +8,7 @@ use crate::result::CodegenResult; use crate::settings; use alloc::boxed::Box; +use core::hash::{Hash, Hasher}; use regalloc::{PrettyPrint, RealRegUniverse}; use target_lexicon::{Aarch64Architecture, Architecture, Triple}; @@ -95,6 +96,10 @@ impl MachBackend for AArch64Backend { &self.flags } + fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) { + self.flags.hash(&mut hasher); + } + fn reg_universe(&self) -> &RealRegUniverse { &self.reg_universe } diff --git a/cranelift/codegen/src/isa/arm32/mod.rs b/cranelift/codegen/src/isa/arm32/mod.rs index 6ab0f9c57c..3976d74ba6 100644 --- a/cranelift/codegen/src/isa/arm32/mod.rs +++ b/cranelift/codegen/src/isa/arm32/mod.rs @@ -8,6 +8,7 @@ use crate::result::CodegenResult; use crate::settings; use alloc::boxed::Box; +use core::hash::{Hash, Hasher}; use regalloc::{PrettyPrint, RealRegUniverse}; use target_lexicon::{Architecture, ArmArchitecture, Triple}; @@ -90,6 +91,10 @@ impl MachBackend for Arm32Backend { &self.flags } + fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) { + self.flags.hash(&mut hasher); + } + fn reg_universe(&self) -> &RealRegUniverse { &self.reg_universe } diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 73a83dda34..bfc4e0d0d0 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -69,6 +69,7 @@ use alloc::boxed::Box; use core::any::Any; use core::fmt; use core::fmt::{Debug, Formatter}; +use core::hash::Hasher; use target_lexicon::{triple, Architecture, PointerWidth, Triple}; use thiserror::Error; @@ -265,6 +266,10 @@ pub trait TargetIsa: fmt::Display + Send + Sync { /// Get the ISA-independent flags that were used to make this trait object. fn flags(&self) -> &settings::Flags; + /// Hashes all flags, both ISA-independent and ISA-specific, into the + /// specified hasher. + fn hash_all_flags(&self, hasher: &mut dyn Hasher); + /// Get the default calling convention of this target. fn default_call_conv(&self) -> CallConv { CallConv::triple_default(self.triple()) diff --git a/cranelift/codegen/src/isa/riscv/mod.rs b/cranelift/codegen/src/isa/riscv/mod.rs index e69a3a0e12..500451c72e 100644 --- a/cranelift/codegen/src/isa/riscv/mod.rs +++ b/cranelift/codegen/src/isa/riscv/mod.rs @@ -19,6 +19,7 @@ use alloc::borrow::Cow; use alloc::boxed::Box; use core::any::Any; use core::fmt; +use core::hash::{Hash, Hasher}; use target_lexicon::{PointerWidth, Triple}; #[allow(dead_code)] @@ -69,6 +70,11 @@ impl TargetIsa for Isa { &self.shared_flags } + fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) { + self.shared_flags.hash(&mut hasher); + self.isa_flags.hash(&mut hasher); + } + fn register_info(&self) -> RegInfo { registers::INFO.clone() } diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index ca809e2d55..28cd503615 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -11,6 +11,7 @@ use crate::machinst::{compile, MachBackend, MachCompileResult, TargetIsaAdapter, use crate::result::CodegenResult; use crate::settings::{self as shared_settings, Flags}; use alloc::boxed::Box; +use core::hash::{Hash, Hasher}; use regalloc::{PrettyPrint, RealRegUniverse, Reg}; use target_lexicon::Triple; @@ -82,6 +83,11 @@ impl MachBackend for X64Backend { &self.flags } + fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) { + self.flags.hash(&mut hasher); + self.x64_flags.hash(&mut hasher); + } + fn name(&self) -> &'static str { "x64" } diff --git a/cranelift/codegen/src/isa/x86/mod.rs b/cranelift/codegen/src/isa/x86/mod.rs index cbdeb3069d..272c3dfe5d 100644 --- a/cranelift/codegen/src/isa/x86/mod.rs +++ b/cranelift/codegen/src/isa/x86/mod.rs @@ -25,6 +25,7 @@ use alloc::borrow::Cow; use alloc::boxed::Box; use core::any::Any; use core::fmt; +use core::hash::{Hash, Hasher}; use target_lexicon::{PointerWidth, Triple}; #[allow(dead_code)] @@ -78,6 +79,11 @@ impl TargetIsa for Isa { &self.shared_flags } + fn hash_all_flags(&self, mut hasher: &mut dyn Hasher) { + self.shared_flags.hash(&mut hasher); + self.isa_flags.hash(&mut hasher); + } + fn uses_cpu_flags(&self) -> bool { true } diff --git a/cranelift/codegen/src/machinst/adapter.rs b/cranelift/codegen/src/machinst/adapter.rs index 4b3be0f3c0..eb4760fae5 100644 --- a/cranelift/codegen/src/machinst/adapter.rs +++ b/cranelift/codegen/src/machinst/adapter.rs @@ -14,6 +14,7 @@ use crate::regalloc::RegDiversions; use crate::isa::unwind::systemv::RegisterMappingError; use core::any::Any; +use core::hash::Hasher; use std::borrow::Cow; use std::fmt; use target_lexicon::Triple; @@ -58,6 +59,10 @@ impl TargetIsa for TargetIsaAdapter { self.backend.flags() } + fn hash_all_flags(&self, hasher: &mut dyn Hasher) { + self.backend.hash_all_flags(hasher) + } + fn register_info(&self) -> RegInfo { // Called from function's Display impl, so we need a stub here. RegInfo { diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index 7ed2661dda..297d531955 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -76,6 +76,7 @@ use regalloc::{ RealReg, RealRegUniverse, Reg, RegClass, RegUsageMapper, SpillSlot, VirtualReg, Writable, }; use smallvec::{smallvec, SmallVec}; +use std::hash::Hasher; use std::string::String; use target_lexicon::Triple; @@ -373,6 +374,10 @@ pub trait MachBackend { /// Return flags for this backend. fn flags(&self) -> &Flags; + /// Hashes all flags, both ISA-independent and ISA-specific, into the + /// specified hasher. + fn hash_all_flags(&self, hasher: &mut dyn Hasher); + /// Return triple for this backend. fn triple(&self) -> Triple; diff --git a/cranelift/codegen/src/settings.rs b/cranelift/codegen/src/settings.rs index 6f25b134af..a1bc954c54 100644 --- a/cranelift/codegen/src/settings.rs +++ b/cranelift/codegen/src/settings.rs @@ -188,7 +188,7 @@ pub type SetResult = Result; /// The settings objects themselves are generated and appear in the `isa/*/settings.rs` modules. /// Each settings object provides a `predicate_view()` method that makes it possible to query /// ISA predicates by number. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Hash)] pub struct PredicateView<'a>(&'a [u8]); impl<'a> PredicateView<'a> { diff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rs index 3bd5716dac..8dffd8ff79 100644 --- a/cranelift/native/src/lib.rs +++ b/cranelift/native/src/lib.rs @@ -34,7 +34,7 @@ use raw_cpuid::CpuId; /// machine, or `Err(())` if the host machine is not supported /// in the current configuration. pub fn builder() -> Result { - builder_with_backend_variant(isa::BackendVariant::Any) + builder_with_options(isa::BackendVariant::Any, true) } /// Return an `isa` builder configured for the current host @@ -44,8 +44,9 @@ pub fn builder() -> Result { /// Selects the given backend variant specifically; this is /// useful when more than oen backend exists for a given target /// (e.g., on x86-64). -pub fn builder_with_backend_variant( +pub fn builder_with_options( variant: isa::BackendVariant, + infer_native_flags: bool, ) -> Result { let mut isa_builder = isa::lookup_variant(Triple::host(), variant).map_err(|err| match err { @@ -55,7 +56,7 @@ pub fn builder_with_backend_variant( isa::LookupError::Unsupported => "unsupported architecture", })?; - if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { + if infer_native_flags && cfg!(any(target_arch = "x86", target_arch = "x86_64")) { parse_x86_cpuid(&mut isa_builder)?; } diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index 5adb0e8312..fe94c27c02 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -182,14 +182,10 @@ impl Hash for Compiler { // misc tunables. strategy.hash(hasher); isa.triple().hash(hasher); - features.hash(hasher); - // TODO: if this `to_string()` is too expensive then we should upstream - // a native hashing ability of flags into cranelift itself, but - // compilation and/or cache loading is relatively expensive so seems - // unlikely. - isa.flags().to_string().hash(hasher); + isa.hash_all_flags(hasher); isa.frontend_config().hash(hasher); tunables.hash(hasher); + features.hash(hasher); // Catch accidental bugs of reusing across crate versions. env!("CARGO_PKG_VERSION").hash(hasher); diff --git a/crates/jit/src/native.rs b/crates/jit/src/native.rs index 9d1fdd7b66..afcf83d3cc 100644 --- a/crates/jit/src/native.rs +++ b/crates/jit/src/native.rs @@ -6,6 +6,11 @@ pub fn builder() -> cranelift_codegen::isa::Builder { cranelift_native::builder().expect("host machine is not a supported target") } +pub fn builder_without_flags() -> cranelift_codegen::isa::Builder { + cranelift_native::builder_with_options(cranelift_codegen::isa::BackendVariant::Any, false) + .expect("host machine is not a supported target") +} + pub fn call_conv() -> cranelift_codegen::isa::CallConv { use target_lexicon::HOST; cranelift_codegen::isa::CallConv::triple_default(&HOST) diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index e6e3f68952..80b10548ed 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -385,6 +385,20 @@ impl Config { self } + /// Clears native CPU flags inferred from the host. + /// + /// By default Wasmtime will tune generated code for the host that Wasmtime + /// itself is running on. If you're compiling on one host, however, and + /// shipping artifacts to another host then this behavior may not be + /// desired. This function will clear all inferred native CPU features. + /// + /// To enable CPU features afterwards it's recommended to use the + /// [`Config::cranelift_other_flag`] method. + pub fn cranelift_clear_cpu_flags(&mut self) -> &mut Self { + self.isa_flags = native::builder_without_flags(); + self + } + /// Allows settings another Cranelift flag defined by a flag name and value. This allows /// fine-tuning of Cranelift settings. /// diff --git a/tests/all/main.rs b/tests/all/main.rs index 414708144f..1f6cb80459 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -12,6 +12,7 @@ mod instance; mod invoke_func_via_table; mod linker; mod memory_creator; +mod module; mod module_linking; mod module_serialize; mod name; diff --git a/tests/all/module.rs b/tests/all/module.rs new file mode 100644 index 0000000000..64384e0de6 --- /dev/null +++ b/tests/all/module.rs @@ -0,0 +1,54 @@ +use wasmtime::*; + +#[test] +fn caches_across_engines() { + let mut c = Config::new(); + c.cranelift_clear_cpu_flags(); + + let bytes = Module::new(&Engine::new(&c), "(module)") + .unwrap() + .serialize() + .unwrap(); + + let res = Module::deserialize( + &Engine::new(&Config::new().cranelift_clear_cpu_flags()), + &bytes, + ); + assert!(res.is_ok()); + + // differ in shared cranelift flags + let res = Module::deserialize( + &Engine::new( + &Config::new() + .cranelift_clear_cpu_flags() + .cranelift_nan_canonicalization(true), + ), + &bytes, + ); + assert!(res.is_err()); + + // differ in cranelift settings + let res = Module::deserialize( + &Engine::new( + &Config::new() + .cranelift_clear_cpu_flags() + .cranelift_opt_level(OptLevel::None), + ), + &bytes, + ); + assert!(res.is_err()); + + // differ in cpu-specific flags + if cfg!(target_arch = "x86_64") { + let res = Module::deserialize( + &Engine::new(unsafe { + &Config::new() + .cranelift_clear_cpu_flags() + .cranelift_other_flag("has_sse3", "true") + .unwrap() + }), + &bytes, + ); + assert!(res.is_err()); + } +} From 7f840870c727ca7037111600768b057baa2387a0 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 26 Jan 2021 16:42:11 -0600 Subject: [PATCH 42/55] cranelift-native: Use libstd feature detection (#2607) This commit switches cranelift-native to useing the `is_x86_feature_detected!` macro in the standard library instead of the `raw-cpuid` crate. --- Cargo.lock | 10 ----- cranelift/native/Cargo.toml | 3 -- cranelift/native/src/lib.rs | 74 ++++++++++++++++--------------------- 3 files changed, 31 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86cbcb9f6b..ead0ab8413 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -607,7 +607,6 @@ name = "cranelift-native" version = "0.69.0" dependencies = [ "cranelift-codegen", - "raw-cpuid", "target-lexicon", ] @@ -2145,15 +2144,6 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "raw-cpuid" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27cb5785b85bd05d4eb171556c9a1a514552e26123aeae6bb7d811353148026" -dependencies = [ - "bitflags", -] - [[package]] name = "rawbytes" version = "0.1.1" diff --git a/cranelift/native/Cargo.toml b/cranelift/native/Cargo.toml index 047d21261a..3bcd72d699 100644 --- a/cranelift/native/Cargo.toml +++ b/cranelift/native/Cargo.toml @@ -14,9 +14,6 @@ edition = "2018" cranelift-codegen = { path = "../codegen", version = "0.69.0", default-features = false } target-lexicon = "0.11" -[target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies] -raw-cpuid = "9.0.0" - [features] default = ["std"] std = ["cranelift-codegen/std"] diff --git a/cranelift/native/src/lib.rs b/cranelift/native/src/lib.rs index 8dffd8ff79..43938bd97e 100644 --- a/cranelift/native/src/lib.rs +++ b/cranelift/native/src/lib.rs @@ -22,14 +22,10 @@ clippy::use_self ) )] -#![no_std] use cranelift_codegen::isa; use target_lexicon::Triple; -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -use raw_cpuid::CpuId; - /// Return an `isa` builder configured for the current host /// machine, or `Err(())` if the host machine is not supported /// in the current configuration. @@ -56,72 +52,64 @@ pub fn builder_with_options( isa::LookupError::Unsupported => "unsupported architecture", })?; - if infer_native_flags && cfg!(any(target_arch = "x86", target_arch = "x86_64")) { - parse_x86_cpuid(&mut isa_builder)?; - } + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + use cranelift_codegen::settings::Configurable; - Ok(isa_builder) -} - -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -fn parse_x86_cpuid(isa_builder: &mut isa::Builder) -> Result<(), &'static str> { - use cranelift_codegen::settings::Configurable; - let cpuid = CpuId::new(); - - if let Some(info) = cpuid.get_feature_info() { - if !info.has_sse2() { + if !std::is_x86_feature_detected!("sse2") { return Err("x86 support requires SSE2"); } - if info.has_sse3() { + + if !infer_native_flags { + return Ok(isa_builder); + } + + if std::is_x86_feature_detected!("sse3") { isa_builder.enable("has_sse3").unwrap(); } - if info.has_ssse3() { + if std::is_x86_feature_detected!("ssse3") { isa_builder.enable("has_ssse3").unwrap(); } - if info.has_sse41() { + if std::is_x86_feature_detected!("sse4.1") { isa_builder.enable("has_sse41").unwrap(); } - if info.has_sse42() { + if std::is_x86_feature_detected!("sse4.2") { isa_builder.enable("has_sse42").unwrap(); } - if info.has_popcnt() { + if std::is_x86_feature_detected!("popcnt") { isa_builder.enable("has_popcnt").unwrap(); } - if info.has_avx() { + if std::is_x86_feature_detected!("avx") { isa_builder.enable("has_avx").unwrap(); } - } - if let Some(info) = cpuid.get_extended_feature_info() { - if info.has_bmi1() { - isa_builder.enable("has_bmi1").unwrap(); - } - if info.has_bmi2() { - isa_builder.enable("has_bmi2").unwrap(); - } - if info.has_avx2() { + if std::is_x86_feature_detected!("avx2") { isa_builder.enable("has_avx2").unwrap(); } - if info.has_avx512dq() { + if std::is_x86_feature_detected!("bmi1") { + isa_builder.enable("has_bmi1").unwrap(); + } + if std::is_x86_feature_detected!("bmi2") { + isa_builder.enable("has_bmi2").unwrap(); + } + if std::is_x86_feature_detected!("avx512dq") { isa_builder.enable("has_avx512dq").unwrap(); } - if info.has_avx512vl() { + if std::is_x86_feature_detected!("avx512vl") { isa_builder.enable("has_avx512vl").unwrap(); } - if info.has_avx512f() { + if std::is_x86_feature_detected!("avx512f") { isa_builder.enable("has_avx512f").unwrap(); } - } - if let Some(info) = cpuid.get_extended_function_info() { - if info.has_lzcnt() { + if std::is_x86_feature_detected!("lzcnt") { isa_builder.enable("has_lzcnt").unwrap(); } } - Ok(()) -} -#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] -fn parse_x86_cpuid(_isa_builder: &mut isa::Builder) -> Result<(), &'static str> { - unreachable!(); + // squelch warnings about unused mut/variables on some platforms. + drop(&mut isa_builder); + drop(infer_native_flags); + + Ok(isa_builder) } #[cfg(test)] From 39f677d2ddfc357468f08efcd16878588f8a681e Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 28 Jan 2021 07:44:06 -0800 Subject: [PATCH 43/55] Only handle signals at pcs with trap information Previously wasmtime would handle any signal originating from wasm JIT code. This would, however, handle bugs in JIT code as-if they were wasm traps. Instead this commit switches signal handling to specifically check for whether the precise program counter is expected to be a trap. This way if a program counter traps and it's not expected to trap the signal isn't handled and the process is aborted (presumably leading to further debugging of whomever happens to work on the JIT at that time). --- crates/runtime/src/traphandlers.rs | 4 ++-- crates/wasmtime/src/store.rs | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 942bdef0f9..e554e54c33 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -432,7 +432,7 @@ pub unsafe trait TrapInfo { /// Returns whether the given program counter lies within wasm code, /// indicating whether we should handle a trap or not. - fn is_wasm_code(&self, pc: usize) -> bool; + fn is_wasm_trap(&self, pc: usize) -> bool; /// Uses `call` to call a custom signal handler, if one is specified. /// @@ -635,7 +635,7 @@ impl<'a> CallThreadState<'a> { } // If this fault wasn't in wasm code, then it's not our problem - if !self.trap_info.is_wasm_code(pc as usize) { + if !self.trap_info.is_wasm_trap(pc as usize) { return ptr::null(); } diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index be4aaf1563..acd229eb1f 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -191,8 +191,9 @@ impl Store { None => return, }; // Only register this module if it hasn't already been registered. - if !self.is_wasm_code(first_pc) { - self.inner.frame_info.borrow_mut().register(module); + let mut info = self.inner.frame_info.borrow_mut(); + if !info.contains_pc(first_pc) { + info.register(module); } } @@ -400,8 +401,8 @@ unsafe impl TrapInfo for Store { self } - fn is_wasm_code(&self, addr: usize) -> bool { - self.frame_info().borrow().contains_pc(addr) + fn is_wasm_trap(&self, addr: usize) -> bool { + self.frame_info().borrow().lookup_trap_info(addr).is_some() } fn custom_signal_handler(&self, call: &dyn Fn(&SignalHandler) -> bool) -> bool { From cb65c755c583b20bf6f04232f1d13d855dac5d87 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 28 Jan 2021 07:55:04 -0800 Subject: [PATCH 44/55] Disable module-linking in plain instantiate fuzzers We already cover module linking with the `instantiate-swarm` target and otherwise enabling module linking is preventing otherwise-valid modules from being compiled because of the breaking change in the module linking proposal with respect to imports. --- crates/fuzzing/src/oracles.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 6581f9e0dc..14e3b09e35 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -47,12 +47,11 @@ fn log_wasm(wasm: &[u8]) { /// /// You can control which compiler is used via passing a `Strategy`. pub fn instantiate(wasm: &[u8], known_valid: bool, strategy: Strategy) { - instantiate_with_config( - wasm, - known_valid, - crate::fuzz_default_config(strategy).unwrap(), - None, - ); + // Explicitly disable module linking for now since it's a breaking change to + // pre-module-linking modules due to imports + let mut cfg = crate::fuzz_default_config(strategy).unwrap(); + cfg.wasm_module_linking(false); + instantiate_with_config(wasm, known_valid, cfg, None); } /// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected From dccaa649627590943afd270cf96aec7dd2eaf6d3 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 28 Jan 2021 08:44:48 -0800 Subject: [PATCH 45/55] Add knobs to limit memories/tables in a `Store` Fuzzing has turned up that module linking can create large amounts of tables and memories in addition to instances. For example if N instances are allowed and M tables are allowed per-instance, then currently wasmtime allows MxN tables (which is quite a lot). This is causing some wasm-smith-generated modules to exceed resource limits while fuzzing! This commits adds corresponding `max_tables` and `max_memories` functions to sit alongside the `max_instances` configuration. Additionally fuzzing now by default configures all of these to a somewhat low value to avoid too much resource usage while fuzzing. --- crates/fuzzing/src/generators.rs | 2 +- crates/fuzzing/src/lib.rs | 3 ++ crates/fuzzing/src/oracles.rs | 3 ++ crates/wasmtime/src/config.rs | 28 ++++++++++++ crates/wasmtime/src/instance.rs | 2 +- crates/wasmtime/src/store.rs | 45 ++++++++++++++++--- tests/all/module_linking.rs | 77 +++++++++++++++++++++++++++++++- 7 files changed, 151 insertions(+), 9 deletions(-) diff --git a/crates/fuzzing/src/generators.rs b/crates/fuzzing/src/generators.rs index d31eba04cb..a35bd186a9 100644 --- a/crates/fuzzing/src/generators.rs +++ b/crates/fuzzing/src/generators.rs @@ -75,7 +75,7 @@ pub struct Config { impl Config { /// Converts this to a `wasmtime::Config` object pub fn to_wasmtime(&self) -> wasmtime::Config { - let mut cfg = wasmtime::Config::new(); + let mut cfg = crate::fuzz_default_config(wasmtime::Strategy::Auto).unwrap(); cfg.debug_info(self.debug_info) .static_memory_maximum_size(self.static_memory_maximum_size.unwrap_or(0).into()) .static_memory_guard_size(self.static_memory_guard_size.unwrap_or(0).into()) diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs index 7ef3382411..d131149ddb 100644 --- a/crates/fuzzing/src/lib.rs +++ b/crates/fuzzing/src/lib.rs @@ -39,6 +39,9 @@ pub fn fuzz_default_config(strategy: wasmtime::Strategy) -> anyhow::Result {} // Allow traps which can happen normally with `unreachable` Err(e) if e.downcast_ref::().is_some() => {} + // Allow resource exhaustion since this is something that our wasm-smith + // generator doesn't guarantee is forbidden. + Err(e) if e.to_string().contains("resource limit exceeded") => {} Err(e) => panic!("failed to instantiate {}", e), } } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 80b10548ed..3edbfebb2b 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -34,6 +34,8 @@ pub struct Config { pub(crate) features: WasmFeatures, pub(crate) wasm_backtrace_details_env_used: bool, pub(crate) max_instances: usize, + pub(crate) max_tables: usize, + pub(crate) max_memories: usize, } impl Config { @@ -81,6 +83,8 @@ impl Config { ..WasmFeatures::default() }, max_instances: 10_000, + max_tables: 10_000, + max_memories: 10_000, }; ret.wasm_backtrace_details(WasmBacktraceDetails::Environment); return ret; @@ -655,11 +659,35 @@ impl Config { /// this `Store`. /// /// Instantiation will fail with an error if this limit is exceeded. + /// + /// This value defaults to 10,000. pub fn max_instances(&mut self, instances: usize) -> &mut Self { self.max_instances = instances; self } + /// Configures the maximum number of tables which can be created within + /// this `Store`. + /// + /// Instantiation will fail with an error if this limit is exceeded. + /// + /// This value defaults to 10,000. + pub fn max_tables(&mut self, tables: usize) -> &mut Self { + self.max_tables = tables; + self + } + + /// Configures the maximum number of memories which can be created within + /// this `Store`. + /// + /// Instantiation will fail with an error if this limit is exceeded. + /// + /// This value defaults to 10,000. + pub fn max_memories(&mut self, memories: usize) -> &mut Self { + self.max_memories = memories; + self + } + pub(crate) fn target_isa(&self) -> Box { self.isa_flags .clone() diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 02ad1a1b98..47115d90ae 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -257,7 +257,7 @@ impl<'a> Instantiator<'a> { /// defined here. fn step(&mut self) -> Result)>> { if self.cur.initializer == 0 { - self.store.bump_instance_count()?; + self.store.bump_resource_counts(&self.cur.module)?; } // Read the current module's initializer and move forward the diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index be4aaf1563..2a85fa61c2 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -67,8 +67,11 @@ pub(crate) struct StoreInner { /// Set of all compiled modules that we're holding a strong reference to /// the module's code for. This includes JIT functions, trampolines, etc. modules: RefCell>, - /// The number of instantiated instances in this store. + + // Numbers of resources instantiated in this store. instance_count: Cell, + memory_count: Cell, + table_count: Cell, } struct HostInfoKey(VMExternRef); @@ -112,6 +115,8 @@ impl Store { frame_info: Default::default(), modules: Default::default(), instance_count: Default::default(), + memory_count: Default::default(), + table_count: Default::default(), }), } } @@ -216,12 +221,40 @@ impl Store { } } - pub(crate) fn bump_instance_count(&self) -> Result<()> { - let n = self.inner.instance_count.get(); - self.inner.instance_count.set(n + 1); - if n >= self.engine().config().max_instances { - bail!("instance limit of {} exceeded", n); + pub(crate) fn bump_resource_counts(&self, module: &Module) -> Result<()> { + let config = self.engine().config(); + + fn bump(slot: &Cell, max: usize, amt: usize, desc: &str) -> Result<()> { + let new = slot.get().saturating_add(amt); + if new > max { + bail!( + "resource limit exceeded: {} count too high at {}", + desc, + new + ); + } + slot.set(new); + Ok(()) } + + let module = module.env_module(); + let memories = module.memory_plans.len() - module.num_imported_memories; + let tables = module.table_plans.len() - module.num_imported_tables; + + bump( + &self.inner.instance_count, + config.max_instances, + 1, + "instance", + )?; + bump( + &self.inner.memory_count, + config.max_memories, + memories, + "memory", + )?; + bump(&self.inner.table_count, config.max_tables, tables, "table")?; + Ok(()) } diff --git a/tests/all/module_linking.rs b/tests/all/module_linking.rs index b81a9deb33..829ddd9b8e 100644 --- a/tests/all/module_linking.rs +++ b/tests/all/module_linking.rs @@ -218,6 +218,81 @@ fn limit_instances() -> Result<()> { )?; let store = Store::new(&engine); let err = Instance::new(&store, &module, &[]).err().unwrap(); - assert!(err.to_string().contains("instance limit of 10 exceeded")); + assert!( + err.to_string().contains("resource limit exceeded"), + "bad error: {}", + err + ); + Ok(()) +} + +#[test] +fn limit_memories() -> Result<()> { + let mut config = Config::new(); + config.wasm_module_linking(true); + config.wasm_multi_memory(true); + config.max_memories(10); + let engine = Engine::new(&config); + let module = Module::new( + &engine, + r#" + (module + (module $m0 + (memory 1 1) + (memory 1 1) + (memory 1 1) + (memory 1 1) + (memory 1 1) + ) + + (instance (instantiate $m0)) + (instance (instantiate $m0)) + (instance (instantiate $m0)) + (instance (instantiate $m0)) + ) + "#, + )?; + let store = Store::new(&engine); + let err = Instance::new(&store, &module, &[]).err().unwrap(); + assert!( + err.to_string().contains("resource limit exceeded"), + "bad error: {}", + err + ); + Ok(()) +} + +#[test] +fn limit_tables() -> Result<()> { + let mut config = Config::new(); + config.wasm_module_linking(true); + config.max_tables(10); + let engine = Engine::new(&config); + let module = Module::new( + &engine, + r#" + (module + (module $m0 + (table 1 1 funcref) + (table 1 1 funcref) + (table 1 1 funcref) + (table 1 1 funcref) + (table 1 1 funcref) + ) + + (instance (instantiate $m0)) + (instance (instantiate $m0)) + (instance (instantiate $m0)) + (instance (instantiate $m0)) + ) + "#, + )?; + let store = Store::new(&engine); + let err = Instance::new(&store, &module, &[]).err().unwrap(); + assert!( + err.to_string().contains("resource limit exceeded"), + "bad error: {}", + err + ); Ok(()) } From d1c1cb6a25c7256c82811d6b9ddc60c2a60da075 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Wed, 27 Jan 2021 15:20:43 -0800 Subject: [PATCH 46/55] bench-api: receive working directory as an argument Rather than implicitly use the current working directory. --- crates/bench-api/src/lib.rs | 110 ++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 42 deletions(-) diff --git a/crates/bench-api/src/lib.rs b/crates/bench-api/src/lib.rs index 1d0e2dcd64..8923b4c788 100644 --- a/crates/bench-api/src/lib.rs +++ b/crates/bench-api/src/lib.rs @@ -21,10 +21,16 @@ //! # Example //! //! ``` +//! use std::ptr; //! use wasmtime_bench_api::*; //! -//! let engine = unsafe { wasm_bench_create() }; -//! assert!(!engine.is_null()); +//! let working_dir = std::env::current_dir().unwrap().display().to_string(); +//! let mut bench_api = ptr::null_mut(); +//! unsafe { +//! let code = wasm_bench_create(working_dir.as_ptr(), working_dir.len(), &mut bench_api); +//! assert_eq!(code, OK); +//! assert!(!bench_api.is_null()); +//! }; //! //! let wasm = wat::parse_bytes(br#" //! (module @@ -42,7 +48,7 @@ //! "#).unwrap(); //! //! // Start your compilation timer here. -//! let code = unsafe { wasm_bench_compile(engine, wasm.as_ptr(), wasm.len()) }; +//! let code = unsafe { wasm_bench_compile(bench_api, wasm.as_ptr(), wasm.len()) }; //! // End your compilation timer here. //! assert_eq!(code, OK); //! @@ -57,24 +63,25 @@ //! } //! //! // Start your instantiation timer here. -//! let code = unsafe { wasm_bench_instantiate(engine, bench_start, bench_stop) }; +//! let code = unsafe { wasm_bench_instantiate(bench_api, bench_start, bench_stop) }; //! // End your instantiation timer here. //! assert_eq!(code, OK); //! //! // No need to start timers for the execution since, by convention, the timer //! // functions we passed during instantiation will be called by the benchmark //! // at the appropriate time (before and after the benchmarked section). -//! let code = unsafe { wasm_bench_execute(engine) }; +//! let code = unsafe { wasm_bench_execute(bench_api) }; //! assert_eq!(code, OK); //! //! unsafe { -//! wasm_bench_free(engine); +//! wasm_bench_free(bench_api); //! } //! ``` use anyhow::{anyhow, Context, Result}; use std::env; use std::os::raw::{c_int, c_void}; +use std::path::Path; use std::slice; use wasi_common::WasiCtxBuilder; use wasmtime::{Config, Engine, Instance, Linker, Module, Store}; @@ -95,12 +102,32 @@ static ALLOC: shuffling_allocator::ShufflingAllocator = /// Exposes a C-compatible way of creating the engine from the bytes of a single /// Wasm module. /// -/// This function returns a pointer to a structure that contains the engine's -/// initialized state. +/// On success, the `out_bench_ptr` is initialized to a pointer to a structure +/// that contains the engine's initialized state, and `0` is returned. On +/// failure, a non-zero status code is returned and `out_bench_ptr` is left +/// untouched. #[no_mangle] -pub extern "C" fn wasm_bench_create() -> *mut c_void { - let state = Box::new(BenchState::new()); - Box::into_raw(state) as _ +pub extern "C" fn wasm_bench_create( + working_dir_ptr: *const u8, + working_dir_len: usize, + out_bench_ptr: *mut *mut c_void, +) -> ExitCode { + let result = (|| -> Result<_> { + let working_dir = unsafe { std::slice::from_raw_parts(working_dir_ptr, working_dir_len) }; + let working_dir = std::str::from_utf8(working_dir) + .context("given working directory is not valid UTF-8")?; + let state = Box::new(BenchState::new(working_dir)?); + Ok(Box::into_raw(state) as _) + })(); + + if let Ok(bench_ptr) = result { + unsafe { + assert!(!out_bench_ptr.is_null()); + *out_bench_ptr = bench_ptr; + } + } + + to_exit_code(result.map(|_| ())) } /// Free the engine state allocated by this library. @@ -164,27 +191,49 @@ fn to_exit_code(result: impl Into>) -> ExitCode { /// to manage the Wasmtime engine between calls. struct BenchState { engine: Engine, - store: Store, + linker: Linker, module: Option, instance: Option, did_execute: bool, } impl BenchState { - fn new() -> Self { + fn new(working_dir: impl AsRef) -> Result { let mut config = Config::new(); config.wasm_simd(true); // NB: do not configure a code cache. let engine = Engine::new(&config); let store = Store::new(&engine); - Self { + + let mut linker = Linker::new(&store); + + // Create a WASI environment. + + let mut cx = WasiCtxBuilder::new(); + cx.inherit_stdio(); + // Allow access to the working directory so that the benchmark can read + // its input workload(s). + let working_dir = wasi_common::preopen_dir(working_dir) + .context("failed to preopen the working directory")?; + cx.preopened_dir(working_dir, "."); + // Pass this env var along so that the benchmark program can use smaller + // input workload(s) if it has them and that has been requested. + if let Ok(val) = env::var("WASM_BENCH_USE_SMALL_WORKLOAD") { + cx.env("WASM_BENCH_USE_SMALL_WORKLOAD", &val); + } + + let cx = cx.build()?; + let wasi = Wasi::new(&store, cx); + wasi.add_to_linker(&mut linker)?; + + Ok(Self { engine, - store, + linker, module: None, instance: None, did_execute: false, - } + }) } fn compile(&mut self, bytes: &[u8]) -> Result<()> { @@ -210,34 +259,11 @@ impl BenchState { .as_mut() .expect("compile the module before instantiating it"); - let mut linker = Linker::new(&self.store); - - // Create a WASI environment. - - let mut cx = WasiCtxBuilder::new(); - cx.inherit_stdio(); - // Allow access to the current working directory so that the benchmark - // can read its input workload(s). The sightglass benchmark runner will - // make sure that this process is spawned with its current directory set - // to the current benchmark's directory. - let cwd = wasi_common::preopen_dir(".") - .context("failed to open the current working directory")?; - cx.preopened_dir(cwd, "."); - // Pass this env var along so that the benchmark program can use smaller - // input workload(s) if it has them and that has been requested. - if let Ok(val) = env::var("WASM_BENCH_USE_SMALL_WORKLOAD") { - cx.env("WASM_BENCH_USE_SMALL_WORKLOAD", &val); - } - - let cx = cx.build()?; - let wasi = Wasi::new(linker.store(), cx); - wasi.add_to_linker(&mut linker)?; - // Import the specialized benchmarking functions. - linker.func("bench", "start", move || bench_start())?; - linker.func("bench", "end", move || bench_end())?; + self.linker.func("bench", "start", move || bench_start())?; + self.linker.func("bench", "end", move || bench_end())?; - self.instance = Some(linker.instantiate(&module)?); + self.instance = Some(self.linker.instantiate(&module)?); Ok(()) } From cbd7a6a80e577ebc255795c53c3cb2904df29561 Mon Sep 17 00:00:00 2001 From: Johnnie Birch <45402135+jlb6740@users.noreply.github.com> Date: Mon, 25 Jan 2021 00:02:57 -0800 Subject: [PATCH 47/55] Add sse41 lowering for rounding x64 --- cranelift/codegen/src/isa/x64/inst/emit.rs | 2 + cranelift/codegen/src/isa/x64/inst/mod.rs | 10 +++- cranelift/codegen/src/isa/x64/lower.rs | 53 ++++++++++------------ 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 095256ab49..0a029301a6 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -2094,7 +2094,9 @@ pub(crate) fn emit( SseOpcode::Pextrd => (LegacyPrefixes::_66, 0x0F3A16, 3), SseOpcode::Pshufd => (LegacyPrefixes::_66, 0x0F70, 2), SseOpcode::Roundps => (LegacyPrefixes::_66, 0x0F3A08, 3), + SseOpcode::Roundss => (LegacyPrefixes::_66, 0x0F3A0A, 3), SseOpcode::Roundpd => (LegacyPrefixes::_66, 0x0F3A09, 3), + SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3), _ => unimplemented!("Opcode {:?} not implemented", op), }; let rex = if *is64 { diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index bab28f2aa0..b6276c943d 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1823,7 +1823,7 @@ impl fmt::Debug for Inst { fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { // This is a bit subtle. If some register is in the modified set, then it may not be in either // the use or def sets. However, enforcing that directly is somewhat difficult. Instead, - // regalloc.rs will "fix" this for us by removing the the modified set from the use and def + // regalloc.rs will "fix" this for us by removing the modified set from the use and def // sets. match inst { Inst::AluRmiR { src, dst, .. } => { @@ -1895,6 +1895,10 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { || *op == SseOpcode::Pextrw || *op == SseOpcode::Pextrd || *op == SseOpcode::Pshufd + || *op == SseOpcode::Roundss + || *op == SseOpcode::Roundsd + || *op == SseOpcode::Roundps + || *op == SseOpcode::Roundpd { src.get_regs_as_uses(collector); collector.add_def(*dst); @@ -2236,6 +2240,10 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { || *op == SseOpcode::Pextrw || *op == SseOpcode::Pextrd || *op == SseOpcode::Pshufd + || *op == SseOpcode::Roundss + || *op == SseOpcode::Roundsd + || *op == SseOpcode::Roundps + || *op == SseOpcode::Roundpd { src.map_uses(mapper); map_def(mapper, dst); diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 45014fca17..6c002dd6cf 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -8,7 +8,7 @@ use crate::ir::{ use crate::isa::x64::abi::*; use crate::isa::x64::inst::args::*; use crate::isa::x64::inst::*; -use crate::isa::{x64::X64Backend, CallConv}; +use crate::isa::{x64::settings as x64_settings, x64::X64Backend, CallConv}; use crate::machinst::lower::*; use crate::machinst::*; use crate::result::CodegenResult; @@ -1330,6 +1330,7 @@ fn lower_insn_to_regs>( ctx: &mut C, insn: IRInst, flags: &Flags, + isa_flags: &x64_settings::Flags, triple: &Triple, ) -> CodegenResult<()> { let op = ctx.data(insn).opcode(); @@ -4211,11 +4212,29 @@ fn lower_insn_to_regs>( } Opcode::Ceil | Opcode::Floor | Opcode::Nearest | Opcode::Trunc => { - // TODO use ROUNDSS/ROUNDSD after sse4.1. - - // Lower to VM calls when there's no access to SSE4.1. let ty = ty.unwrap(); - if !ty.is_vector() { + if isa_flags.use_sse41() { + let mode = match op { + Opcode::Ceil => RoundImm::RoundUp, + Opcode::Floor => RoundImm::RoundDown, + Opcode::Nearest => RoundImm::RoundNearest, + Opcode::Trunc => RoundImm::RoundZero, + _ => panic!("unexpected opcode {:?} in Ceil/Floor/Nearest/Trunc", op), + }; + let op = match ty { + types::F32 => SseOpcode::Roundss, + types::F64 => SseOpcode::Roundsd, + types::F32X4 => SseOpcode::Roundps, + types::F64X2 => SseOpcode::Roundpd, + _ => panic!("unexpected type {:?} in Ceil/Floor/Nearest/Trunc", ty), + }; + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::xmm_rm_r_imm(op, src, dst, mode.encode(), false)); + } else { + // Lower to VM calls when there's no access to SSE4.1. + // Note, for vector types on platforms that don't support sse41 + // the execution will panic here. let libcall = match (op, ty) { (Opcode::Ceil, types::F32) => LibCall::CeilF32, (Opcode::Ceil, types::F64) => LibCall::CeilF64, @@ -4231,28 +4250,6 @@ fn lower_insn_to_regs>( ), }; emit_vm_call(ctx, flags, triple, libcall, insn, inputs, outputs)?; - } else { - let (op, mode) = match (op, ty) { - (Opcode::Ceil, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundUp), - (Opcode::Ceil, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundUp), - (Opcode::Floor, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundDown), - (Opcode::Floor, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundDown), - (Opcode::Trunc, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundZero), - (Opcode::Trunc, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundZero), - (Opcode::Nearest, types::F32X4) => (SseOpcode::Roundps, RoundImm::RoundNearest), - (Opcode::Nearest, types::F64X2) => (SseOpcode::Roundpd, RoundImm::RoundNearest), - _ => panic!("Unknown op/ty combination (vector){:?}", ty), - }; - let src = put_input_in_reg(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::gen_move(dst, src, ty)); - ctx.emit(Inst::xmm_rm_r_imm( - op, - RegMem::from(dst), - dst, - mode.encode(), - false, - )); } } @@ -5389,7 +5386,7 @@ impl LowerBackend for X64Backend { type MInst = Inst; fn lower>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> { - lower_insn_to_regs(ctx, ir_inst, &self.flags, &self.triple) + lower_insn_to_regs(ctx, ir_inst, &self.flags, &self.x64_flags, &self.triple) } fn lower_branch_group>( From f76a9d436e9f0e153ca705b78afdc7cb4b002ce4 Mon Sep 17 00:00:00 2001 From: Kasey Carrothers Date: Wed, 27 Jan 2021 22:45:35 -0800 Subject: [PATCH 48/55] Clean up handling of NOPs in the x64 backend. 1. Restricts max nop size to 15 instead of 16. 2. Fixes an edge case where gen_nop() would return a zero sized intruction on multiples of 16. 3. Clarifies the documentation of the gen_nop interface to state that returning zero is allowed when preferred_size is zero. --- cranelift/codegen/src/isa/x64/inst/mod.rs | 6 +++--- cranelift/codegen/src/machinst/mod.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index bab28f2aa0..44cedb06ca 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -566,7 +566,7 @@ impl Inst { impl Inst { pub(crate) fn nop(len: u8) -> Self { - debug_assert!(len <= 16); + debug_assert!(len <= 15); Self::Nop { len } } @@ -2594,11 +2594,11 @@ impl MachInst for Inst { } fn gen_zero_len_nop() -> Inst { - Inst::Nop { len: 0 } + Inst::nop(0) } fn gen_nop(preferred_size: usize) -> Inst { - Inst::nop((preferred_size % 16) as u8) + Inst::nop(std::cmp::min(preferred_size, 15) as u8) } fn maybe_direct_reload(&self, _reg: VirtualReg, _slot: SpillSlot) -> Option { diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index 297d531955..d65fd9b3ca 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -178,7 +178,7 @@ pub trait MachInst: Clone + Debug { /// request a NOP of that size, or as close to it as possible. The machine /// backend may return a NOP whose binary encoding is smaller than the /// preferred size, but must not return a NOP that is larger. However, - /// the instruction must have a nonzero size. + /// the instruction must have a nonzero size if preferred_size is nonzero. fn gen_nop(preferred_size: usize) -> Self; /// Get the register universe for this backend. From 99be82c866e73a59b74ea66f4bdb959becaa890d Mon Sep 17 00:00:00 2001 From: Kasey Carrothers Date: Fri, 29 Jan 2021 01:09:32 -0800 Subject: [PATCH 49/55] Replace MachInst::gen_zero_len_nop with gen_nop(0) --- cranelift/codegen/src/isa/aarch64/inst/mod.rs | 7 +++---- cranelift/codegen/src/isa/arm32/inst/mod.rs | 7 +++---- cranelift/codegen/src/isa/x64/inst/mod.rs | 4 ---- cranelift/codegen/src/machinst/mod.rs | 3 --- cranelift/codegen/src/machinst/vcode.rs | 2 +- 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index b85928edcc..aff9b36e0a 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -2907,11 +2907,10 @@ impl MachInst for Inst { } } - fn gen_zero_len_nop() -> Inst { - Inst::Nop0 - } - fn gen_nop(preferred_size: usize) -> Inst { + if preferred_size == 0 { + return Inst::Nop0; + } // We can't give a NOP (or any insn) < 4 bytes. assert!(preferred_size >= 4); Inst::Nop4 diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs index 309aa43102..811e8cf920 100644 --- a/cranelift/codegen/src/isa/arm32/inst/mod.rs +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -831,11 +831,10 @@ impl MachInst for Inst { } } - fn gen_zero_len_nop() -> Inst { - Inst::Nop0 - } - fn gen_nop(preferred_size: usize) -> Inst { + if preferred_size == 0 { + return Inst::Nop0; + } assert!(preferred_size >= 2); Inst::Nop2 } diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 76bde8e139..e6a6ec9486 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -2601,10 +2601,6 @@ impl MachInst for Inst { } } - fn gen_zero_len_nop() -> Inst { - Inst::nop(0) - } - fn gen_nop(preferred_size: usize) -> Inst { Inst::nop(std::cmp::min(preferred_size, 15) as u8) } diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index d65fd9b3ca..c7e86437fa 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -148,9 +148,6 @@ pub trait MachInst: Clone + Debug { alloc_tmp: F, ) -> SmallVec<[Self; 4]>; - /// Generate a zero-length no-op. - fn gen_zero_len_nop() -> Self; - /// Possibly operate on a value directly in a spill-slot rather than a /// register. Useful if the machine has register-memory instruction forms /// (e.g., add directly from or directly to memory), like x86. diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index 9fc46d9655..ab1cca0f47 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -726,7 +726,7 @@ impl RegallocFunction for VCode { } fn gen_zero_len_nop(&self) -> I { - I::gen_zero_len_nop() + I::gen_nop(0) } fn maybe_direct_reload(&self, insn: &I, reg: VirtualReg, slot: SpillSlot) -> Option { From 78f312799e742dcb0a06be3417a8332578f96833 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 28 Jan 2021 16:06:28 +0000 Subject: [PATCH 50/55] Optimize EntityList::extend and add EntityList::from_iter --- cranelift/entity/src/list.rs | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/cranelift/entity/src/list.rs b/cranelift/entity/src/list.rs index 68cab8166b..dfb8fac646 100644 --- a/cranelift/entity/src/list.rs +++ b/cranelift/entity/src/list.rs @@ -91,17 +91,20 @@ type SizeClass = u8; /// Get the size of a given size class. The size includes the length field, so the maximum list /// length is one less than the class size. +#[inline] fn sclass_size(sclass: SizeClass) -> usize { 4 << sclass } /// Get the size class to use for a given list length. /// This always leaves room for the length element in addition to the list elements. +#[inline] fn sclass_for_length(len: usize) -> SizeClass { 30 - (len as u32 | 3).leading_zeros() as SizeClass } /// Is `len` the minimum length in its size class? +#[inline] fn is_sclass_min_length(len: usize) -> bool { len > 3 && len.is_power_of_two() } @@ -387,14 +390,34 @@ impl EntityList { &mut pool.data[block + 1..block + 1 + new_len] } + /// Constructs a list from an iterator. + pub fn from_iter(elements: I, pool: &mut ListPool) -> Self + where + I: IntoIterator, + { + let mut list = Self::new(); + list.extend(elements, pool); + list + } + /// Appends multiple elements to the back of the list. pub fn extend(&mut self, elements: I, pool: &mut ListPool) where I: IntoIterator, { - // TODO: use `size_hint()` to reduce reallocations. - for x in elements { - self.push(x, pool); + let iterator = elements.into_iter(); + let (len, upper) = iterator.size_hint(); + // On most iterators this check is optimized down to `true`. + if upper == Some(len) { + let data = self.grow(len, pool); + let offset = data.len() - len; + for (src, dst) in iterator.zip(data[offset..].iter_mut()) { + *dst = src; + } + } else { + for x in iterator { + self.push(x, pool); + } } } @@ -630,6 +653,10 @@ mod tests { list.as_slice(pool), &[i1, i2, i3, i4, i1, i1, i2, i2, i3, i3, i4, i4] ); + + let list2 = EntityList::from_iter([i1, i1, i2, i2, i3, i3, i4, i4].iter().cloned(), pool); + assert_eq!(list2.len(pool), 8); + assert_eq!(list2.as_slice(pool), &[i1, i1, i2, i2, i3, i3, i4, i4]); } #[test] From 0e41861662bde3f5a6280811406f68facfdca238 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 29 Jan 2021 08:57:17 -0600 Subject: [PATCH 51/55] Implement limiting WebAssembly execution with fuel (#2611) * Consume fuel during function execution This commit adds codegen infrastructure necessary to instrument wasm code to consume fuel as it executes. Currently nothing is really done with the fuel, but that'll come in later commits. The focus of this commit is to implement the codegen infrastructure necessary to consume fuel and account for fuel consumed correctly. * Periodically check remaining fuel in wasm JIT code This commit enables wasm code to periodically check to see if fuel has run out. When fuel runs out an intrinsic is called which can do what it needs to do in the result of fuel running out. For now a trap is thrown to have at least some semantics in synchronous stores, but another planned use for this feature is for asynchronous stores to periodically yield back to the host based on fuel running out. Checks for remaining fuel happen in the same locations as interrupt checks, which is to say the start of the function as well as loop headers. * Improve codegen by caching `*const VMInterrupts` The location of the shared interrupt value and fuel value is through a double-indirection on the vmctx (load through the vmctx and then load through that pointer). The second pointer in this chain, however, never changes, so we can alter codegen to account for this and remove some extraneous load instructions and hopefully reduce some register pressure even maybe. * Add tests fuel can abort infinite loops * More fuzzing with fuel Use fuel to time out modules in addition to time, using fuzz input to figure out which. * Update docs on trapping instructions * Fix doc links * Fix a fuzz test * Change setting fuel to adding fuel * Fix a doc link * Squelch some rustdoc warnings --- Cargo.lock | 2 + Cargo.toml | 1 + cranelift/src/souper_harvest.rs | 2 +- cranelift/wasm/src/code_translator.rs | 2 +- cranelift/wasm/src/environ/spec.rs | 8 +- cranelift/wasm/src/func_translator.rs | 2 + crates/cranelift/Cargo.toml | 1 + crates/cranelift/src/func_environ.rs | 375 ++++++++++++++++-- crates/environ/src/builtin.rs | 2 + crates/environ/src/tunables.rs | 5 + crates/environ/src/vmoffsets.rs | 5 + crates/fuzzing/src/generators.rs | 5 +- crates/fuzzing/src/oracles.rs | 83 +++- crates/profiling/src/jitdump_linux.rs | 2 +- crates/runtime/src/externref.rs | 2 +- crates/runtime/src/libcalls.rs | 5 + crates/runtime/src/traphandlers.rs | 13 + crates/runtime/src/vmcontext.rs | 14 +- crates/wasmtime/src/config.rs | 22 + crates/wasmtime/src/store.rs | 88 ++++ .../fuzz_targets/instantiate-maybe-invalid.rs | 11 +- fuzz/fuzz_targets/instantiate-swarm.rs | 16 +- tests/all/fuel.rs | 124 ++++++ tests/all/fuel.wast | 208 ++++++++++ tests/all/fuzzing.rs | 4 +- tests/all/main.rs | 1 + 26 files changed, 936 insertions(+), 67 deletions(-) create mode 100644 tests/all/fuel.rs create mode 100644 tests/all/fuel.wast diff --git a/Cargo.lock b/Cargo.lock index ead0ab8413..352bcee781 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3138,6 +3138,7 @@ dependencies = [ "wasmtime-wasi-crypto", "wasmtime-wasi-nn", "wasmtime-wast", + "wast 32.0.0", "wat", ] @@ -3149,6 +3150,7 @@ dependencies = [ "cranelift-entity", "cranelift-frontend", "cranelift-wasm", + "wasmparser", "wasmtime-environ", ] diff --git a/Cargo.toml b/Cargo.toml index c1d0cc8ed7..00cd71f526 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ test-programs = { path = "crates/test-programs" } wasmtime-fuzzing = { path = "crates/fuzzing" } wasmtime-runtime = { path = "crates/runtime" } tracing-subscriber = "0.2.0" +wast = "32.0.0" [build-dependencies] anyhow = "1.0.19" diff --git a/cranelift/src/souper_harvest.rs b/cranelift/src/souper_harvest.rs index 5bfa1fe629..e92a502302 100644 --- a/cranelift/src/souper_harvest.rs +++ b/cranelift/src/souper_harvest.rs @@ -12,7 +12,7 @@ static WASM_MAGIC: &[u8] = &[0x00, 0x61, 0x73, 0x6D]; /// Harvest candidates for superoptimization from a Wasm or Clif file. /// /// Candidates are emitted in Souper's text format: -/// https://github.com/google/souper +/// #[derive(StructOpt)] pub struct Options { /// Specify an input file to be used. Use '-' for stdin. diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs index 266e002ae4..a20975b25b 100644 --- a/cranelift/wasm/src/code_translator.rs +++ b/cranelift/wasm/src/code_translator.rs @@ -261,7 +261,7 @@ pub fn translate_operator( .extend_from_slice(builder.block_params(loop_body)); builder.switch_to_block(loop_body); - environ.translate_loop_header(builder.cursor())?; + environ.translate_loop_header(builder)?; } Operator::If { ty } => { let val = state.pop1(); diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index 84978dc0d8..0bd3c48745 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -295,6 +295,12 @@ pub trait FuncEnvironment: TargetEnvironment { ReturnMode::NormalReturns } + /// Called after the locals for a function have been parsed, and the number + /// of variables defined by this function is provided. + fn after_locals(&mut self, num_locals_defined: usize) { + drop(num_locals_defined); + } + /// Set up the necessary preamble definitions in `func` to access the global variable /// identified by `index`. /// @@ -637,7 +643,7 @@ pub trait FuncEnvironment: TargetEnvironment { /// /// This can be used to insert explicit interrupt or safepoint checking at /// the beginnings of loops. - fn translate_loop_header(&mut self, _pos: FuncCursor) -> WasmResult<()> { + fn translate_loop_header(&mut self, _builder: &mut FunctionBuilder) -> WasmResult<()> { // By default, don't emit anything. Ok(()) } diff --git a/cranelift/wasm/src/func_translator.rs b/cranelift/wasm/src/func_translator.rs index 7a8f9c19d2..97e5354c6e 100644 --- a/cranelift/wasm/src/func_translator.rs +++ b/cranelift/wasm/src/func_translator.rs @@ -170,6 +170,8 @@ fn parse_local_decls( declare_locals(builder, count, ty, &mut next_local, environ)?; } + environ.after_locals(next_local); + Ok(()) } diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index 99ccc5a497..9b4e0972bb 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -17,3 +17,4 @@ cranelift-wasm = { path = "../../cranelift/wasm", version = "0.69.0" } cranelift-codegen = { path = "../../cranelift/codegen", version = "0.69.0" } cranelift-frontend = { path = "../../cranelift/frontend", version = "0.69.0" } cranelift-entity = { path = "../../cranelift/entity", version = "0.69.0" } +wasmparser = "0.73.0" diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 6b94824f93..a9481aca93 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -7,11 +7,14 @@ use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Si use cranelift_codegen::isa::{self, TargetFrontendConfig}; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_frontend::FunctionBuilder; +use cranelift_frontend::Variable; use cranelift_wasm::{ - self, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, SignatureIndex, TableIndex, - TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType, + self, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, MemoryIndex, + SignatureIndex, TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType, }; use std::convert::TryFrom; +use std::mem; +use wasmparser::Operator; use wasmtime_environ::{ BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, TableStyle, Tunables, VMOffsets, INTERRUPTED, WASM_PAGE_SIZE, @@ -125,6 +128,20 @@ pub struct FuncEnvironment<'module_environment> { pub(crate) offsets: VMOffsets, tunables: &'module_environment Tunables, + + /// A function-local variable which stores the cached value of the amount of + /// fuel remaining to execute. If used this is modified frequently so it's + /// stored locally as a variable instead of always referenced from the field + /// in `*const VMInterrupts` + fuel_var: cranelift_frontend::Variable, + + /// A function-local variable which caches the value of `*const + /// VMInterrupts` for this function's vmctx argument. This pointer is stored + /// in the vmctx itself, but never changes for the lifetime of the function, + /// so if we load it up front we can continue to use it throughout. + vminterrupts_ptr: cranelift_frontend::Variable, + + fuel_consumed: i64, } impl<'module_environment> FuncEnvironment<'module_environment> { @@ -151,6 +168,12 @@ impl<'module_environment> FuncEnvironment<'module_environment> { builtin_function_signatures, offsets: VMOffsets::new(target_config.pointer_bytes(), module), tunables, + fuel_var: Variable::new(0), + vminterrupts_ptr: Variable::new(0), + + // Start with at least one fuel being consumed because even empty + // functions should consume at least some fuel. + fuel_consumed: 1, } } @@ -418,6 +441,241 @@ impl<'module_environment> FuncEnvironment<'module_environment> { (global, 0) } } + + fn declare_vminterrupts_ptr(&mut self, builder: &mut FunctionBuilder<'_>) { + // We load the `*const VMInterrupts` value stored within vmctx at the + // head of the function and reuse the same value across the entire + // function. This is possible since we know that the pointer never + // changes for the lifetime of the function. + let pointer_type = self.pointer_type(); + builder.declare_var(self.vminterrupts_ptr, pointer_type); + let vmctx = self.vmctx(builder.func); + let base = builder.ins().global_value(pointer_type, vmctx); + let offset = i32::try_from(self.offsets.vmctx_interrupts()).unwrap(); + let interrupt_ptr = builder + .ins() + .load(pointer_type, ir::MemFlags::trusted(), base, offset); + builder.def_var(self.vminterrupts_ptr, interrupt_ptr); + } + + fn fuel_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) { + // On function entry we load the amount of fuel into a function-local + // `self.fuel_var` to make fuel modifications fast locally. This cache + // is then periodically flushed to the Store-defined location in + // `VMInterrupts` later. + builder.declare_var(self.fuel_var, ir::types::I64); + self.fuel_load_into_var(builder); + self.fuel_check(builder); + } + + fn fuel_function_exit(&mut self, builder: &mut FunctionBuilder<'_>) { + // On exiting the function we need to be sure to save the fuel we have + // cached locally in `self.fuel_var` back into the Store-defined + // location. + self.fuel_save_from_var(builder); + } + + fn fuel_before_op( + &mut self, + op: &Operator<'_>, + builder: &mut FunctionBuilder<'_>, + reachable: bool, + ) { + if !reachable { + // In unreachable code we shouldn't have any leftover fuel we + // haven't accounted for since the reason for us to become + // unreachable should have already added it to `self.fuel_var`. + debug_assert_eq!(self.fuel_consumed, 0); + return; + } + + self.fuel_consumed += match op { + // Nop and drop generate no code, so don't consume fuel for them. + Operator::Nop | Operator::Drop => 0, + + // Control flow may create branches, but is generally cheap and + // free, so don't consume fuel. Note the lack of `if` since some + // cost is incurred with the conditional check. + Operator::Block { .. } + | Operator::Loop { .. } + | Operator::Unreachable + | Operator::Return + | Operator::Else + | Operator::End => 0, + + // everything else, just call it one operation. + _ => 1, + }; + + match op { + // Exiting a function (via a return or unreachable) or otherwise + // entering a different function (via a call) means that we need to + // update the fuel consumption in `VMInterrupts` because we're + // about to move control out of this function itself and the fuel + // may need to be read. + // + // Before this we need to update the fuel counter from our own cost + // leading up to this function call, and then we can store + // `self.fuel_var` into `VMInterrupts`. + Operator::Unreachable + | Operator::Return + | Operator::CallIndirect { .. } + | Operator::Call { .. } + | Operator::ReturnCall { .. } + | Operator::ReturnCallIndirect { .. } => { + self.fuel_increment_var(builder); + self.fuel_save_from_var(builder); + } + + // To ensure all code preceding a loop is only counted once we + // update the fuel variable on entry. + Operator::Loop { .. } + + // Entering into an `if` block means that the edge we take isn't + // known until runtime, so we need to update our fuel consumption + // before we take the branch. + | Operator::If { .. } + + // Control-flow instructions mean that we're moving to the end/exit + // of a block somewhere else. That means we need to update the fuel + // counter since we're effectively terminating our basic block. + | Operator::Br { .. } + | Operator::BrIf { .. } + | Operator::BrTable { .. } + + // Exiting a scope means that we need to update the fuel + // consumption because there are multiple ways to exit a scope and + // this is the only time we have to account for instructions + // executed so far. + | Operator::End + + // This is similar to `end`, except that it's only the terminator + // for an `if` block. The same reasoning applies though in that we + // are terminating a basic block and need to update the fuel + // variable. + | Operator::Else => self.fuel_increment_var(builder), + + // This is a normal instruction where the fuel is buffered to later + // get added to `self.fuel_var`. + // + // Note that we generally ignore instructions which may trap and + // therefore result in exiting a block early. Current usage of fuel + // means that it's not too important to account for a precise amount + // of fuel consumed but rather "close to the actual amount" is good + // enough. For 100% precise counting, however, we'd probably need to + // not only increment but also save the fuel amount more often + // around trapping instructions. (see the `unreachable` instruction + // case above) + // + // Note that `Block` is specifically omitted from incrementing the + // fuel variable. Control flow entering a `block` is unconditional + // which means it's effectively executing straight-line code. We'll + // update the counter when exiting a block, but we shouldn't need to + // do so upon entering a block. + _ => {} + } + } + + fn fuel_after_op(&mut self, op: &Operator<'_>, builder: &mut FunctionBuilder<'_>) { + // After a function call we need to reload our fuel value since the + // function may have changed it. + match op { + Operator::Call { .. } | Operator::CallIndirect { .. } => { + self.fuel_load_into_var(builder); + } + _ => {} + } + } + + /// Adds `self.fuel_consumed` to the `fuel_var`, zero-ing out the amount of + /// fuel consumed at that point. + fn fuel_increment_var(&mut self, builder: &mut FunctionBuilder<'_>) { + let consumption = mem::replace(&mut self.fuel_consumed, 0); + if consumption == 0 { + return; + } + + let fuel = builder.use_var(self.fuel_var); + let fuel = builder.ins().iadd_imm(fuel, consumption); + builder.def_var(self.fuel_var, fuel); + } + + /// Loads the fuel consumption value from `VMInterrupts` into `self.fuel_var` + fn fuel_load_into_var(&mut self, builder: &mut FunctionBuilder<'_>) { + let (addr, offset) = self.fuel_addr_offset(builder); + let fuel = builder + .ins() + .load(ir::types::I64, ir::MemFlags::trusted(), addr, offset); + builder.def_var(self.fuel_var, fuel); + } + + /// Stores the fuel consumption value from `self.fuel_var` into + /// `VMInterrupts`. + fn fuel_save_from_var(&mut self, builder: &mut FunctionBuilder<'_>) { + let (addr, offset) = self.fuel_addr_offset(builder); + let fuel_consumed = builder.use_var(self.fuel_var); + builder + .ins() + .store(ir::MemFlags::trusted(), fuel_consumed, addr, offset); + } + + /// Returns the `(address, offset)` of the fuel consumption within + /// `VMInterrupts`, used to perform loads/stores later. + fn fuel_addr_offset( + &mut self, + builder: &mut FunctionBuilder<'_>, + ) -> (ir::Value, ir::immediates::Offset32) { + ( + builder.use_var(self.vminterrupts_ptr), + i32::from(self.offsets.vminterrupts_fuel_consumed()).into(), + ) + } + + /// Checks the amount of remaining, and if we've run out of fuel we call + /// the out-of-fuel function. + fn fuel_check(&mut self, builder: &mut FunctionBuilder) { + self.fuel_increment_var(builder); + let out_of_gas_block = builder.create_block(); + let continuation_block = builder.create_block(); + + // Note that our fuel is encoded as adding positive values to a + // negative number. Whenever the negative number goes positive that + // means we ran out of fuel. + // + // Compare to see if our fuel is positive, and if so we ran out of gas. + // Otherwise we can continue on like usual. + let zero = builder.ins().iconst(ir::types::I64, 0); + let fuel = builder.use_var(self.fuel_var); + let cmp = builder.ins().ifcmp(fuel, zero); + builder + .ins() + .brif(IntCC::SignedGreaterThanOrEqual, cmp, out_of_gas_block, &[]); + builder.ins().jump(continuation_block, &[]); + builder.seal_block(out_of_gas_block); + + // If we ran out of gas then we call our out-of-gas intrinsic and it + // figures out what to do. Note that this may raise a trap, or do + // something like yield to an async runtime. In either case we don't + // assume what happens and handle the case the intrinsic returns. + // + // Note that we save/reload fuel around this since the out-of-gas + // intrinsic may alter how much fuel is in the system. + builder.switch_to_block(out_of_gas_block); + self.fuel_save_from_var(builder); + let out_of_gas_sig = self.builtin_function_signatures.out_of_gas(builder.func); + let (vmctx, out_of_gas) = self.translate_load_builtin_function_address( + &mut builder.cursor(), + BuiltinFunctionIndex::out_of_gas(), + ); + builder + .ins() + .call_indirect(out_of_gas_sig, out_of_gas, &[vmctx]); + self.fuel_load_into_var(builder); + builder.ins().jump(continuation_block, &[]); + builder.seal_block(continuation_block); + + builder.switch_to_block(continuation_block); + } } impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environment> { @@ -437,6 +695,11 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m index >= 2 } + fn after_locals(&mut self, num_locals: usize) { + self.vminterrupts_ptr = Variable::new(num_locals); + self.fuel_var = Variable::new(num_locals + 1); + } + fn make_table(&mut self, func: &mut ir::Function, index: TableIndex) -> WasmResult { let pointer_type = self.pointer_type(); @@ -1482,36 +1745,90 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m Ok(*pos.func.dfg.inst_results(call_inst).first().unwrap()) } - fn translate_loop_header(&mut self, mut pos: FuncCursor) -> WasmResult<()> { - if !self.tunables.interruptable { - return Ok(()); - } - - // Start out each loop with a check to the interupt flag to allow - // interruption of long or infinite loops. + fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> { + // If enabled check the interrupt flag to prevent long or infinite + // loops. // // For more information about this see comments in // `crates/environ/src/cranelift.rs` - let vmctx = self.vmctx(&mut pos.func); - let pointer_type = self.pointer_type(); - let base = pos.ins().global_value(pointer_type, vmctx); - let offset = i32::try_from(self.offsets.vmctx_interrupts()).unwrap(); - let interrupt_ptr = pos - .ins() - .load(pointer_type, ir::MemFlags::trusted(), base, offset); - let interrupt = pos.ins().load( - pointer_type, - ir::MemFlags::trusted(), - interrupt_ptr, - i32::from(self.offsets.vminterrupts_stack_limit()), - ); - // Note that the cast to `isize` happens first to allow sign-extension, - // if necessary, to `i64`. - let interrupted_sentinel = pos.ins().iconst(pointer_type, INTERRUPTED as isize as i64); - let cmp = pos - .ins() - .icmp(IntCC::Equal, interrupt, interrupted_sentinel); - pos.ins().trapnz(cmp, ir::TrapCode::Interrupt); + if self.tunables.interruptable { + let pointer_type = self.pointer_type(); + let interrupt_ptr = builder.use_var(self.vminterrupts_ptr); + let interrupt = builder.ins().load( + pointer_type, + ir::MemFlags::trusted(), + interrupt_ptr, + i32::from(self.offsets.vminterrupts_stack_limit()), + ); + // Note that the cast to `isize` happens first to allow sign-extension, + // if necessary, to `i64`. + let interrupted_sentinel = builder + .ins() + .iconst(pointer_type, INTERRUPTED as isize as i64); + let cmp = builder + .ins() + .icmp(IntCC::Equal, interrupt, interrupted_sentinel); + builder.ins().trapnz(cmp, ir::TrapCode::Interrupt); + } + + // Additionally if enabled check how much fuel we have remaining to see + // if we've run out by this point. + if self.tunables.consume_fuel { + self.fuel_check(builder); + } + + Ok(()) + } + + fn before_translate_operator( + &mut self, + op: &Operator, + builder: &mut FunctionBuilder, + state: &FuncTranslationState, + ) -> WasmResult<()> { + if self.tunables.consume_fuel { + self.fuel_before_op(op, builder, state.reachable()); + } + Ok(()) + } + + fn after_translate_operator( + &mut self, + op: &Operator, + builder: &mut FunctionBuilder, + state: &FuncTranslationState, + ) -> WasmResult<()> { + if self.tunables.consume_fuel && state.reachable() { + self.fuel_after_op(op, builder); + } + Ok(()) + } + + fn before_translate_function( + &mut self, + builder: &mut FunctionBuilder, + _state: &FuncTranslationState, + ) -> WasmResult<()> { + // If the `vminterrupts_ptr` variable will get used then we initialize + // it here. + if self.tunables.consume_fuel || self.tunables.interruptable { + self.declare_vminterrupts_ptr(builder); + } + // Additionally we initialize `fuel_var` if it will get used. + if self.tunables.consume_fuel { + self.fuel_function_entry(builder); + } + Ok(()) + } + + fn after_translate_function( + &mut self, + builder: &mut FunctionBuilder, + state: &FuncTranslationState, + ) -> WasmResult<()> { + if self.tunables.consume_fuel && state.reachable() { + self.fuel_function_exit(builder); + } Ok(()) } } diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index f2bb330217..1bbc064241 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -57,6 +57,8 @@ macro_rules! foreach_builtin_function { memory_atomic_wait64(vmctx, i32, i32, i64, i64) -> (i32); /// Returns an index for wasm's `memory.atomic.wait64` for imported memories. imported_memory_atomic_wait64(vmctx, i32, i32, i64, i64) -> (i32); + /// Invoked when fuel has run out while executing a function. + out_of_gas(vmctx) -> (); } }; } diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs index 939861504d..bd86fef237 100644 --- a/crates/environ/src/tunables.rs +++ b/crates/environ/src/tunables.rs @@ -23,6 +23,10 @@ pub struct Tunables { /// calls and interrupts are implemented through the `VMInterrupts` /// structure, or `InterruptHandle` in the `wasmtime` crate. pub interruptable: bool, + + /// Whether or not fuel is enabled for generated code, meaning that fuel + /// will be consumed every time a wasm instruction is executed. + pub consume_fuel: bool, } impl Default for Tunables { @@ -57,6 +61,7 @@ impl Default for Tunables { generate_native_debuginfo: false, parse_wasm_debuginfo: true, interruptable: false, + consume_fuel: false, } } } diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 8673a38d3c..7f74f46754 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -258,6 +258,11 @@ impl VMOffsets { pub fn vminterrupts_stack_limit(&self) -> u8 { 0 } + + /// Return the offset of the `fuel_consumed` field of `VMInterrupts` + pub fn vminterrupts_fuel_consumed(&self) -> u8 { + self.pointer_size + } } /// Offsets for `VMCallerCheckedAnyfunc`. diff --git a/crates/fuzzing/src/generators.rs b/crates/fuzzing/src/generators.rs index a35bd186a9..b44af8daf5 100644 --- a/crates/fuzzing/src/generators.rs +++ b/crates/fuzzing/src/generators.rs @@ -64,6 +64,8 @@ pub struct Config { debug_info: bool, canonicalize_nans: bool, interruptable: bool, + #[allow(missing_docs)] + pub consume_fuel: bool, // Note that we use 32-bit values here to avoid blowing the 64-bit address // space by requesting ungodly-large sizes/guards. @@ -82,7 +84,8 @@ impl Config { .dynamic_memory_guard_size(self.dynamic_memory_guard_size.unwrap_or(0).into()) .cranelift_nan_canonicalization(self.canonicalize_nans) .cranelift_opt_level(self.opt_level.to_wasmtime()) - .interruptable(self.interruptable); + .interruptable(self.interruptable) + .consume_fuel(self.consume_fuel); return cfg; } } diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 093ffbacd8..2fc4532de4 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -40,6 +40,20 @@ fn log_wasm(wasm: &[u8]) { } } +/// Methods of timing out execution of a WebAssembly module +#[derive(Debug)] +pub enum Timeout { + /// No timeout is used, it should be guaranteed via some other means that + /// the input does not infinite loop. + None, + /// A time-based timeout is used with a sleeping thread sending a signal + /// after the specified duration. + Time(Duration), + /// Fuel-based timeouts are used where the specified fuel is all that the + /// provided wasm module is allowed to consume. + Fuel(u64), +} + /// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected /// panic or segfault or anything else that can be detected "passively". /// @@ -51,7 +65,7 @@ pub fn instantiate(wasm: &[u8], known_valid: bool, strategy: Strategy) { // pre-module-linking modules due to imports let mut cfg = crate::fuzz_default_config(strategy).unwrap(); cfg.wasm_module_linking(false); - instantiate_with_config(wasm, known_valid, cfg, None); + instantiate_with_config(wasm, known_valid, cfg, Timeout::None); } /// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected @@ -64,28 +78,38 @@ pub fn instantiate_with_config( wasm: &[u8], known_valid: bool, mut config: Config, - timeout: Option, + timeout: Timeout, ) { crate::init_fuzzing(); - config.interruptable(timeout.is_some()); + config.interruptable(match &timeout { + Timeout::Time(_) => true, + _ => false, + }); + config.consume_fuel(match &timeout { + Timeout::Fuel(_) => true, + _ => false, + }); let engine = Engine::new(&config); let store = Store::new(&engine); - // If a timeout is requested then we spawn a helper thread to wait for the - // requested time and then send us a signal to get interrupted. We also - // arrange for the thread's sleep to get interrupted if we return early (or - // the wasm returns within the time limit), which allows the thread to get - // torn down. - // - // This prevents us from creating a huge number of sleeping threads if this - // function is executed in a loop, like it does on nightly fuzzing - // infrastructure. - let mut timeout_state = SignalOnDrop::default(); - if let Some(timeout) = timeout { - let handle = store.interrupt_handle().unwrap(); - timeout_state.spawn_timeout(timeout, move || handle.interrupt()); + match timeout { + Timeout::Fuel(fuel) => store.add_fuel(fuel), + // If a timeout is requested then we spawn a helper thread to wait for + // the requested time and then send us a signal to get interrupted. We + // also arrange for the thread's sleep to get interrupted if we return + // early (or the wasm returns within the time limit), which allows the + // thread to get torn down. + // + // This prevents us from creating a huge number of sleeping threads if + // this function is executed in a loop, like it does on nightly fuzzing + // infrastructure. + Timeout::Time(timeout) => { + let handle = store.interrupt_handle().unwrap(); + timeout_state.spawn_timeout(timeout, move || handle.interrupt()); + } + Timeout::None => {} } log_wasm(wasm); @@ -98,11 +122,14 @@ pub fn instantiate_with_config( match Instance::new(&store, &module, &imports) { Ok(_) => {} - // Allow traps which can happen normally with `unreachable` + // Allow traps which can happen normally with `unreachable` or a timeout Err(e) if e.downcast_ref::().is_some() => {} // Allow resource exhaustion since this is something that our wasm-smith // generator doesn't guarantee is forbidden. Err(e) if e.to_string().contains("resource limit exceeded") => {} + // Also allow errors related to fuel consumption + Err(e) if e.to_string().contains("all fuel consumed") => {} + // Everything else should be a bug in the fuzzer Err(e) => panic!("failed to instantiate {}", e), } } @@ -383,13 +410,16 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) { /// Executes the wast `test` spectest with the `config` specified. /// /// Ensures that spec tests pass regardless of the `Config`. -pub fn spectest(config: crate::generators::Config, test: crate::generators::SpecTest) { +pub fn spectest(fuzz_config: crate::generators::Config, test: crate::generators::SpecTest) { crate::init_fuzzing(); - log::debug!("running {:?} with {:?}", test.file, config); - let mut config = config.to_wasmtime(); + log::debug!("running {:?} with {:?}", test.file, fuzz_config); + let mut config = fuzz_config.to_wasmtime(); config.wasm_reference_types(false); config.wasm_bulk_memory(false); let store = Store::new(&Engine::new(&config)); + if fuzz_config.consume_fuel { + store.add_fuel(u64::max_value()); + } let mut wast_context = WastContext::new(store); wast_context.register_spectest().unwrap(); wast_context @@ -398,16 +428,22 @@ pub fn spectest(config: crate::generators::Config, test: crate::generators::Spec } /// Execute a series of `table.get` and `table.set` operations. -pub fn table_ops(config: crate::generators::Config, ops: crate::generators::table_ops::TableOps) { +pub fn table_ops( + fuzz_config: crate::generators::Config, + ops: crate::generators::table_ops::TableOps, +) { let _ = env_logger::try_init(); let num_dropped = Rc::new(Cell::new(0)); { - let mut config = config.to_wasmtime(); + let mut config = fuzz_config.to_wasmtime(); config.wasm_reference_types(true); let engine = Engine::new(&config); let store = Store::new(&engine); + if fuzz_config.consume_fuel { + store.add_fuel(u64::max_value()); + } let wasm = ops.to_wasm_binary(); log_wasm(&wasm); @@ -520,6 +556,9 @@ pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Con wasmtime_config.cranelift_nan_canonicalization(true); let wasmtime_engine = Engine::new(&wasmtime_config); let wasmtime_store = Store::new(&wasmtime_engine); + if config.consume_fuel { + wasmtime_store.add_fuel(u64::max_value()); + } let wasmtime_module = Module::new(&wasmtime_engine, &wasm).expect("Wasmtime can compile module"); let wasmtime_instance = Instance::new(&wasmtime_store, &wasmtime_module, &[]) diff --git a/crates/profiling/src/jitdump_linux.rs b/crates/profiling/src/jitdump_linux.rs index ef6684c95f..0f3a4bb00d 100644 --- a/crates/profiling/src/jitdump_linux.rs +++ b/crates/profiling/src/jitdump_linux.rs @@ -1,6 +1,6 @@ //! Support for jitdump files which can be used by perf for profiling jitted code. //! Spec definitions for the output format is as described here: -//! https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt +//! //! //! Usage Example: //! Record diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index dd0a4f1475..b5a8ef30bb 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -97,7 +97,7 @@ //! //! For more general information on deferred reference counting, see *An //! Examination of Deferred Reference Counting and Cycle Detection* by Quinane: -//! https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf +//! use std::alloc::Layout; use std::any::Any; diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index 61e8ca4a21..026738a8a1 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -581,3 +581,8 @@ pub unsafe extern "C" fn wasmtime_imported_memory_atomic_wait64( "wasm atomics (fn wasmtime_imported_memory_atomic_wait64) unsupported", )))); } + +/// Hook for when an instance runs out of fuel. +pub unsafe extern "C" fn wasmtime_out_of_gas(_vmctx: *mut VMContext) { + crate::traphandlers::out_of_gas() +} diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index e554e54c33..feaf2c4a9a 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -410,6 +410,13 @@ pub fn with_last_info(func: impl FnOnce(Option<&dyn Any>) -> R) -> R { tls::with(|state| func(state.map(|s| s.trap_info.as_any()))) } +/// Invokes the contextually-defined context's out-of-gas function. +/// +/// (basically delegates to `wasmtime::Store::out_of_gas`) +pub fn out_of_gas() { + tls::with(|state| state.unwrap().trap_info.out_of_gas()) +} + /// Temporary state stored on the stack which is registered in the `tls` module /// below for calls into wasm. pub struct CallThreadState<'a> { @@ -442,6 +449,12 @@ pub unsafe trait TrapInfo { /// Returns the maximum size, in bytes, the wasm native stack is allowed to /// grow to. fn max_wasm_stack(&self) -> usize; + + /// Callback invoked whenever WebAssembly has entirely consumed the fuel + /// that it was allotted. + /// + /// This function may return, and it may also `raise_lib_trap`. + fn out_of_gas(&self); } enum UnwindReason { diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index 3eebdabd4e..c20a42b5ed 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -4,6 +4,7 @@ use crate::externref::VMExternRef; use crate::instance::Instance; use std::any::Any; +use std::cell::UnsafeCell; use std::ptr::NonNull; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::u32; @@ -612,6 +613,7 @@ impl VMBuiltinFunctionsArray { wasmtime_memory_atomic_wait64 as usize; ptrs[BuiltinFunctionIndex::imported_memory_atomic_wait64().index() as usize] = wasmtime_imported_memory_atomic_wait64 as usize; + ptrs[BuiltinFunctionIndex::out_of_gas().index() as usize] = wasmtime_out_of_gas as usize; if cfg!(debug_assertions) { for i in 0..ptrs.len() { @@ -658,8 +660,7 @@ impl VMInvokeArgument { } } -/// Structure used to control interrupting wasm code, currently with only one -/// atomic flag internally used. +/// Structure used to control interrupting wasm code. #[derive(Debug)] #[repr(C)] pub struct VMInterrupts { @@ -668,6 +669,14 @@ pub struct VMInterrupts { /// This is used to control both stack overflow as well as interrupting wasm /// modules. For more information see `crates/environ/src/cranelift.rs`. pub stack_limit: AtomicUsize, + + /// Indicator of how much fuel has been consumed and is remaining to + /// WebAssembly. + /// + /// This field is typically negative and increments towards positive. Upon + /// turning positive a wasm trap will be generated. This field is only + /// modified if wasm is configured to consume fuel. + pub fuel_consumed: UnsafeCell, } impl VMInterrupts { @@ -682,6 +691,7 @@ impl Default for VMInterrupts { fn default() -> VMInterrupts { VMInterrupts { stack_limit: AtomicUsize::new(usize::max_value()), + fuel_consumed: UnsafeCell::new(0), } } } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 3edbfebb2b..a662599dc8 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -137,6 +137,28 @@ impl Config { self } + /// Configures whether execution of WebAssembly will "consume fuel" to + /// either halt or yield execution as desired. + /// + /// This option is similar in purpose to [`Config::interruptable`] where + /// you can prevent infinitely-executing WebAssembly code. The difference + /// is that this option allows deterministic execution of WebAssembly code + /// by instrumenting generated code consume fuel as it executes. When fuel + /// runs out the behavior is defined by configuration within a [`Store`], + /// and by default a trap is raised. + /// + /// Note that a [`Store`] starts with no fuel, so if you enable this option + /// you'll have to be sure to pour some fuel into [`Store`] before + /// executing some code. + /// + /// By default this option is `false`. + /// + /// [`Store`]: crate::Store + pub fn consume_fuel(&mut self, enable: bool) -> &mut Self { + self.tunables.consume_fuel = enable; + self + } + /// Configures the maximum amount of native stack space available to /// executing WebAssembly code. /// diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index cc86011fb2..10564d79dd 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -6,6 +6,7 @@ use anyhow::{bail, Result}; use std::any::Any; use std::cell::{Cell, RefCell}; use std::collections::HashSet; +use std::convert::TryFrom; use std::fmt; use std::hash::{Hash, Hasher}; use std::rc::{Rc, Weak}; @@ -72,6 +73,10 @@ pub(crate) struct StoreInner { instance_count: Cell, memory_count: Cell, table_count: Cell, + + /// An adjustment to add to the fuel consumed value in `interrupts` above + /// to get the true amount of fuel consumed. + fuel_adj: Cell, } struct HostInfoKey(VMExternRef); @@ -117,6 +122,7 @@ impl Store { instance_count: Default::default(), memory_count: Default::default(), table_count: Default::default(), + fuel_adj: Cell::new(0), }), } } @@ -427,6 +433,64 @@ impl Store { ); } } + + /// Returns the amount of fuel consumed by this store's execution so far. + /// + /// If fuel consumption is not enabled via + /// [`Config::consume_fuel`](crate::Config::consume_fuel) then this + /// function will return `None`. Also note that fuel, if enabled, must be + /// originally configured via [`Store::add_fuel`]. + pub fn fuel_consumed(&self) -> Option { + if !self.engine().config().tunables.consume_fuel { + return None; + } + let consumed = unsafe { *self.inner.interrupts.fuel_consumed.get() }; + Some(u64::try_from(self.inner.fuel_adj.get() + consumed).unwrap()) + } + + /// Adds fuel to this [`Store`] for wasm to consume while executing. + /// + /// For this method to work fuel consumption must be enabled via + /// [`Config::consume_fuel`](crate::Config::consume_fuel). By default a + /// [`Store`] starts with 0 fuel for wasm to execute with (meaning it will + /// immediately trap). This function must be called for the store to have + /// some fuel to allow WebAssembly to execute. + /// + /// Note that at this time when fuel is entirely consumed it will cause + /// wasm to trap. More usages of fuel are planned for the future. + /// + /// # Panics + /// + /// This function will panic if the store's [`Config`](crate::Config) did + /// not have fuel consumption enabled. + pub fn add_fuel(&self, fuel: u64) { + assert!(self.engine().config().tunables.consume_fuel); + + // Fuel is stored as an i64, so we need to cast it. If the provided fuel + // value overflows that just assume that i64::max will suffice. Wasm + // execution isn't fast enough to burn through i64::max fuel in any + // reasonable amount of time anyway. + let fuel = i64::try_from(fuel).unwrap_or(i64::max_value()); + let adj = self.inner.fuel_adj.get(); + let consumed_ptr = unsafe { &mut *self.inner.interrupts.fuel_consumed.get() }; + + match (consumed_ptr.checked_sub(fuel), adj.checked_add(fuel)) { + // If we succesfully did arithmetic without overflowing then we can + // just update our fields. + (Some(consumed), Some(adj)) => { + self.inner.fuel_adj.set(adj); + *consumed_ptr = consumed; + } + + // Otherwise something overflowed. Make sure that we preserve the + // amount of fuel that's already consumed, but otherwise assume that + // we were given infinite fuel. + _ => { + self.inner.fuel_adj.set(i64::max_value()); + *consumed_ptr = (*consumed_ptr + adj) - i64::max_value(); + } + } + } } unsafe impl TrapInfo for Store { @@ -448,6 +512,23 @@ unsafe impl TrapInfo for Store { fn max_wasm_stack(&self) -> usize { self.engine().config().max_wasm_stack } + + fn out_of_gas(&self) { + #[derive(Debug)] + struct OutOfGas; + + impl fmt::Display for OutOfGas { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("all fuel consumed by WebAssembly") + } + } + + impl std::error::Error for OutOfGas {} + + unsafe { + wasmtime_runtime::raise_lib_trap(wasmtime_runtime::Trap::User(Box::new(OutOfGas))) + } + } } impl Default for Store { @@ -481,6 +562,13 @@ pub struct InterruptHandle { interrupts: Arc, } +// The `VMInterrupts` type is a pod-type with no destructor, and we only access +// `interrupts` from other threads, so add in these trait impls which are +// otherwise not available due to the `fuel_consumed` variable in +// `VMInterrupts`. +unsafe impl Send for InterruptHandle {} +unsafe impl Sync for InterruptHandle {} + impl InterruptHandle { /// Flags that execution within this handle's original [`Store`] should be /// interrupted. diff --git a/fuzz/fuzz_targets/instantiate-maybe-invalid.rs b/fuzz/fuzz_targets/instantiate-maybe-invalid.rs index 219986a25a..46a749f5e8 100644 --- a/fuzz/fuzz_targets/instantiate-maybe-invalid.rs +++ b/fuzz/fuzz_targets/instantiate-maybe-invalid.rs @@ -4,13 +4,18 @@ use libfuzzer_sys::fuzz_target; use std::time::Duration; use wasm_smith::MaybeInvalidModule; use wasmtime::Strategy; -use wasmtime_fuzzing::oracles; +use wasmtime_fuzzing::oracles::{self, Timeout}; -fuzz_target!(|module: MaybeInvalidModule| { +fuzz_target!(|pair: (bool, MaybeInvalidModule)| { + let (timeout_with_time, module) = pair; oracles::instantiate_with_config( &module.to_bytes(), false, wasmtime_fuzzing::fuzz_default_config(Strategy::Auto).unwrap(), - Some(Duration::from_secs(20)), + if timeout_with_time { + Timeout::Time(Duration::from_secs(20)) + } else { + Timeout::Fuel(100_000) + }, ); }); diff --git a/fuzz/fuzz_targets/instantiate-swarm.rs b/fuzz/fuzz_targets/instantiate-swarm.rs index d9c5bdb645..b534cc4970 100644 --- a/fuzz/fuzz_targets/instantiate-swarm.rs +++ b/fuzz/fuzz_targets/instantiate-swarm.rs @@ -4,11 +4,21 @@ use libfuzzer_sys::fuzz_target; use std::time::Duration; use wasm_smith::{Config, ConfiguredModule, SwarmConfig}; use wasmtime::Strategy; -use wasmtime_fuzzing::oracles; +use wasmtime_fuzzing::oracles::{self, Timeout}; -fuzz_target!(|module: ConfiguredModule| { +fuzz_target!(|pair: (bool, ConfiguredModule)| { + let (timeout_with_time, module) = pair; let mut cfg = wasmtime_fuzzing::fuzz_default_config(Strategy::Auto).unwrap(); cfg.wasm_multi_memory(true); cfg.wasm_module_linking(module.config().module_linking_enabled()); - oracles::instantiate_with_config(&module.to_bytes(), true, cfg, Some(Duration::from_secs(20))); + oracles::instantiate_with_config( + &module.to_bytes(), + true, + cfg, + if timeout_with_time { + Timeout::Time(Duration::from_secs(20)) + } else { + Timeout::Fuel(100_000) + }, + ); }); diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs new file mode 100644 index 0000000000..b1b73bc6a8 --- /dev/null +++ b/tests/all/fuel.rs @@ -0,0 +1,124 @@ +use anyhow::Result; +use wasmtime::*; +use wast::parser::{self, Parse, ParseBuffer, Parser}; + +mod kw { + wast::custom_keyword!(assert_fuel); +} + +struct FuelWast<'a> { + assertions: Vec<(wast::Span, u64, wast::Module<'a>)>, +} + +impl<'a> Parse<'a> for FuelWast<'a> { + fn parse(parser: Parser<'a>) -> parser::Result { + let mut assertions = Vec::new(); + while !parser.is_empty() { + assertions.push(parser.parens(|p| { + let span = p.parse::()?.0; + Ok((span, p.parse()?, p.parens(|p| p.parse())?)) + })?); + } + Ok(FuelWast { assertions }) + } +} + +#[test] +fn run() -> Result<()> { + let test = std::fs::read_to_string("tests/all/fuel.wast")?; + let buf = ParseBuffer::new(&test)?; + let mut wast = parser::parse::>(&buf)?; + for (span, fuel, module) in wast.assertions.iter_mut() { + let consumed = fuel_consumed(&module.encode()?); + if consumed == *fuel { + continue; + } + let (line, col) = span.linecol_in(&test); + panic!( + "tests/all/fuel.wast:{}:{} - expected {} fuel, found {}", + line + 1, + col + 1, + fuel, + consumed + ); + } + Ok(()) +} + +fn fuel_consumed(wasm: &[u8]) -> u64 { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config); + let module = Module::new(&engine, wasm).unwrap(); + let store = Store::new(&engine); + store.add_fuel(u64::max_value()); + drop(Instance::new(&store, &module, &[])); + store.fuel_consumed().unwrap() +} + +#[test] +fn iloop() { + iloop_aborts( + r#" + (module + (start 0) + (func loop br 0 end) + ) + "#, + ); + iloop_aborts( + r#" + (module + (start 0) + (func loop i32.const 1 br_if 0 end) + ) + "#, + ); + iloop_aborts( + r#" + (module + (start 0) + (func loop i32.const 0 br_table 0 end) + ) + "#, + ); + iloop_aborts( + r#" + (module + (start 0) + (func $f0 call $f1 call $f1) + (func $f1 call $f2 call $f2) + (func $f2 call $f3 call $f3) + (func $f3 call $f4 call $f4) + (func $f4 call $f5 call $f5) + (func $f5 call $f6 call $f6) + (func $f6 call $f7 call $f7) + (func $f7 call $f8 call $f8) + (func $f8 call $f9 call $f9) + (func $f9 call $f10 call $f10) + (func $f10 call $f11 call $f11) + (func $f11 call $f12 call $f12) + (func $f12 call $f13 call $f13) + (func $f13 call $f14 call $f14) + (func $f14 call $f15 call $f15) + (func $f15 call $f16 call $f16) + (func $f16) + ) + "#, + ); + + fn iloop_aborts(wat: &str) { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config); + let module = Module::new(&engine, wat).unwrap(); + let store = Store::new(&engine); + store.add_fuel(10_000); + let error = Instance::new(&store, &module, &[]).err().unwrap(); + assert!( + error.to_string().contains("all fuel consumed"), + "bad error: {}", + error + ); + } +} diff --git a/tests/all/fuel.wast b/tests/all/fuel.wast new file mode 100644 index 0000000000..bb3450b6cc --- /dev/null +++ b/tests/all/fuel.wast @@ -0,0 +1,208 @@ +(assert_fuel 0 (module)) + +(assert_fuel 1 + (module + (func $f) + (start $f))) + +(assert_fuel 2 + (module + (func $f + i32.const 0 + drop + ) + (start $f))) + +(assert_fuel 1 + (module + (func $f + block + end + ) + (start $f))) + +(assert_fuel 1 + (module + (func $f + unreachable + ) + (start $f))) + +(assert_fuel 7 + (module + (func $f + i32.const 0 + i32.const 0 + i32.const 0 + i32.const 0 + i32.const 0 + i32.const 0 + unreachable + ) + (start $f))) + +(assert_fuel 1 + (module + (func $f + return + i32.const 0 + i32.const 0 + i32.const 0 + i32.const 0 + i32.const 0 + i32.const 0 + unreachable + ) + (start $f))) + +(assert_fuel 3 + (module + (func $f + i32.const 0 + if + call $f + end + ) + (start $f))) + +(assert_fuel 4 + (module + (func $f + i32.const 1 + if + i32.const 0 + drop + end + ) + (start $f))) + +(assert_fuel 4 + (module + (func $f + i32.const 1 + if + i32.const 0 + drop + else + call $f + end + ) + (start $f))) + +(assert_fuel 4 + (module + (func $f + i32.const 0 + if + call $f + else + i32.const 0 + drop + end + ) + (start $f))) + +(assert_fuel 3 + (module + (func $f + block + i32.const 1 + br_if 0 + i32.const 0 + drop + end + ) + (start $f))) + +(assert_fuel 4 + (module + (func $f + block + i32.const 0 + br_if 0 + i32.const 0 + drop + end + ) + (start $f))) + +;; count code before unreachable +(assert_fuel 2 + (module + (func $f + i32.const 0 + unreachable + ) + (start $f))) + +;; count code before return +(assert_fuel 2 + (module + (func $f + i32.const 0 + return + ) + (start $f))) + +;; cross-function fuel works +(assert_fuel 3 + (module + (func $f + call $other + ) + (func $other) + (start $f))) +(assert_fuel 5 + (module + (func $f + i32.const 0 + call $other + i32.const 0 + drop + ) + (func $other (param i32)) + (start $f))) +(assert_fuel 4 + (module + (func $f + call $other + drop + ) + (func $other (result i32) + i32.const 0 + ) + (start $f))) +(assert_fuel 4 + (module + (func $f + i32.const 0 + call_indirect + ) + (func $other) + (table funcref (elem $other)) + (start $f))) + +;; loops! +(assert_fuel 1 + (module + (func $f + loop + end + ) + (start $f))) +(assert_fuel 53 ;; 5 loop instructions, 10 iterations, 2 header instrs, 1 func + (module + (func $f + (local i32) + i32.const 10 + local.set 0 + + loop + local.get 0 + i32.const 1 + i32.sub + local.tee 0 + br_if 0 + end + ) + (start $f))) diff --git a/tests/all/fuzzing.rs b/tests/all/fuzzing.rs index 3e46af2d72..e5c1906886 100644 --- a/tests/all/fuzzing.rs +++ b/tests/all/fuzzing.rs @@ -6,7 +6,7 @@ //! `include_bytes!("./fuzzing/some-descriptive-name.wasm")`. use wasmtime::{Config, Strategy}; -use wasmtime_fuzzing::oracles; +use wasmtime_fuzzing::oracles::{self, Timeout}; #[test] fn instantiate_empty_module() { @@ -26,5 +26,5 @@ fn instantiate_module_that_compiled_to_x64_has_register_32() { let mut config = Config::new(); config.debug_info(true); let data = wat::parse_str(include_str!("./fuzzing/issue694.wat")).unwrap(); - oracles::instantiate_with_config(&data, true, config, None); + oracles::instantiate_with_config(&data, true, config, Timeout::None); } diff --git a/tests/all/main.rs b/tests/all/main.rs index 1f6cb80459..c20b2dddce 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -2,6 +2,7 @@ mod cli_tests; mod custom_signal_handler; mod debug; mod externals; +mod fuel; mod func; mod fuzzing; mod globals; From f1d922dc2dc8e301cec86933f7adac7c2ef85b41 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 29 Jan 2021 10:04:36 -0800 Subject: [PATCH 52/55] Update some wasm-tools dependencies Mostly focused around some small bugfixes and improvements related to module-linking fuzzing. --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 352bcee781..8aa226852f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2967,9 +2967,9 @@ dependencies = [ [[package]] name = "wasm-smith" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9b9b796bf4da5eb0523b136d6f0cc9a59c16a66ece8e0d5a14a9cdccf2864e" +checksum = "5af0d70f17515b1bc412b7727b01304ba2484bc416da72f325d6bf4ab58f4213" dependencies = [ "arbitrary", "indexmap", @@ -3003,15 +3003,15 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.73.0" +version = "0.73.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15011eb23c404cdc1f6f60124a15921a11b060cca195486e36f4dc62e83c61d" +checksum = "b8526ab131cbc49495a483c98954913ae7b83551adacab5e294cf77992e70ee7" [[package]] name = "wasmprinter" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c467ab13e60cc347a17ed2c60bf761501eb45314d8916cad0d53ccc31078b20d" +checksum = "edf67b8f2b3b49a5ca4f8ab6879c1c70057a6b2c14474d6126be912051f2392e" dependencies = [ "anyhow", "wasmparser", From d3acd9a2831436e7296d002fd1cd283333d144cb Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Wed, 27 Jan 2021 18:44:14 +0100 Subject: [PATCH 53/55] cranelift x64: use the LZCNT instruction for Clz when it's available; --- cranelift/codegen/src/isa/x64/inst/args.rs | 3 ++ cranelift/codegen/src/isa/x64/inst/emit.rs | 38 ++++++++++++++----- cranelift/codegen/src/isa/x64/lower.rs | 19 +++++++++- .../filetests/isa/x64/clz-lzcnt.clif | 31 +++++++++++++++ 4 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/clz-lzcnt.clif diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 39ca25d060..d81e1bbf8a 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -402,6 +402,8 @@ pub enum UnaryRmROpcode { Bsr, /// Bit-scan forward. Bsf, + /// Counts leading zeroes (Leading Zero CouNT). + Lzcnt, } impl fmt::Debug for UnaryRmROpcode { @@ -409,6 +411,7 @@ impl fmt::Debug for UnaryRmROpcode { match self { UnaryRmROpcode::Bsr => write!(fmt, "bsr"), UnaryRmROpcode::Bsf => write!(fmt, "bsf"), + UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"), } } } diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 0a029301a6..6ae0dc012e 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -129,18 +129,20 @@ impl RexFlags { /// We may need to include one or more legacy prefix bytes before the REX prefix. This enum /// covers only the small set of possibilities that we actually need. enum LegacyPrefixes { - /// No prefix bytes + /// No prefix bytes. None, - /// Operand Size Override -- here, denoting "16-bit operation" + /// Operand Size Override -- here, denoting "16-bit operation". _66, - /// The Lock prefix + /// The Lock prefix. _F0, - /// Operand size override and Lock + /// Operand size override and Lock. _66F0, - /// REPNE, but no specific meaning here -- is just an opcode extension + /// REPNE, but no specific meaning here -- is just an opcode extension. _F2, - /// REP/REPE, but no specific meaning here -- is just an opcode extension + /// REP/REPE, but no specific meaning here -- is just an opcode extension. _F3, + /// Operand size override and same effect as F3. + _66F3, } impl LegacyPrefixes { @@ -157,6 +159,10 @@ impl LegacyPrefixes { } LegacyPrefixes::_F2 => sink.put1(0xF2), LegacyPrefixes::_F3 => sink.put1(0xF3), + LegacyPrefixes::_66F3 => { + sink.put1(0x66); + sink.put1(0xF3); + } LegacyPrefixes::None => (), } } @@ -665,16 +671,28 @@ pub(crate) fn emit( } Inst::UnaryRmR { size, op, src, dst } => { - let (prefix, rex_flags) = match size { - 2 => (LegacyPrefixes::_66, RexFlags::clear_w()), - 4 => (LegacyPrefixes::None, RexFlags::clear_w()), - 8 => (LegacyPrefixes::None, RexFlags::set_w()), + let rex_flags = match size { + 2 | 4 => RexFlags::clear_w(), + 8 => RexFlags::set_w(), + _ => unreachable!(), + }; + + let prefix = match size { + 2 => match op { + UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => LegacyPrefixes::_66, + UnaryRmROpcode::Lzcnt => LegacyPrefixes::_66F3, + }, + 4 | 8 => match op { + UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => LegacyPrefixes::None, + UnaryRmROpcode::Lzcnt => LegacyPrefixes::_F3, + }, _ => unreachable!(), }; let (opcode, num_opcodes) = match op { UnaryRmROpcode::Bsr => (0x0fbd, 2), UnaryRmROpcode::Bsf => (0x0fbc, 2), + UnaryRmROpcode::Lzcnt => (0x0fbd, 2), }; match src { diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 6c002dd6cf..837d4d1266 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2276,7 +2276,22 @@ fn lower_insn_to_regs>( } Opcode::Clz => { - // TODO when the x86 flags have use_lzcnt, we can use LZCNT. + let orig_ty = ty.unwrap(); + + if isa_flags.use_lzcnt() && (orig_ty == types::I32 || orig_ty == types::I64) { + // We can use a plain lzcnt instruction here. Note no special handling is required + // for zero inputs, because the machine instruction does what the CLIF expects for + // zero, i.e. it returns zero. + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::unary_rm_r( + orig_ty.bytes() as u8, + UnaryRmROpcode::Lzcnt, + src, + dst, + )); + return Ok(()); + } // General formula using bit-scan reverse (BSR): // mov -1, %dst @@ -2285,7 +2300,6 @@ fn lower_insn_to_regs>( // mov $(size_bits - 1), %dst // sub %tmp, %dst - let orig_ty = ty.unwrap(); if orig_ty == types::I128 { // clz upper, tmp1 // clz lower, dst @@ -4427,6 +4441,7 @@ fn lower_insn_to_regs>( } } } + Opcode::Store | Opcode::Istore8 | Opcode::Istore16 diff --git a/cranelift/filetests/filetests/isa/x64/clz-lzcnt.clif b/cranelift/filetests/filetests/isa/x64/clz-lzcnt.clif new file mode 100644 index 0000000000..ac0df03384 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/clz-lzcnt.clif @@ -0,0 +1,31 @@ +test compile +target x86_64 has_lzcnt +feature "experimental_x64" + +function %clz(i64) -> i64 { +block0(v0: i64): + v1 = clz v0 + return v1 +} + +; check: pushq %rbp +; check: movq %rsp, %rbp +; check: lzcntq %rdi, %rsi +; check: movq %rsi, %rax +; check: movq %rbp, %rsp +; check: popq %rbp +; check: ret + +function %clz(i32) -> i32 { +block0(v0: i32): + v1 = clz v0 + return v1 +} + +; check: pushq %rbp +; check: movq %rsp, %rbp +; check: lzcntl %edi, %esi +; check: movq %rsi, %rax +; check: movq %rbp, %rsp +; check: popq %rbp +; check: ret From 6bf6612d96e6cc8709563e327165326fa6aa2174 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 28 Jan 2021 12:02:47 +0100 Subject: [PATCH 54/55] cranelift x64: use the TZCNT instruction for Ctz when it's available; --- cranelift/codegen/src/isa/x64/inst/args.rs | 3 ++ cranelift/codegen/src/isa/x64/inst/emit.rs | 5 +-- cranelift/codegen/src/isa/x64/lower.rs | 18 +++++++++-- .../filetests/filetests/isa/x64/ctz-bmi1.clif | 31 +++++++++++++++++++ 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/ctz-bmi1.clif diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index d81e1bbf8a..a6938ca64d 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -404,6 +404,8 @@ pub enum UnaryRmROpcode { Bsf, /// Counts leading zeroes (Leading Zero CouNT). Lzcnt, + /// Counts trailing zeroes (Trailing Zero CouNT). + Tzcnt, } impl fmt::Debug for UnaryRmROpcode { @@ -412,6 +414,7 @@ impl fmt::Debug for UnaryRmROpcode { UnaryRmROpcode::Bsr => write!(fmt, "bsr"), UnaryRmROpcode::Bsf => write!(fmt, "bsf"), UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"), + UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"), } } } diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 6ae0dc012e..74559272a7 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -680,11 +680,11 @@ pub(crate) fn emit( let prefix = match size { 2 => match op { UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => LegacyPrefixes::_66, - UnaryRmROpcode::Lzcnt => LegacyPrefixes::_66F3, + UnaryRmROpcode::Lzcnt | UnaryRmROpcode::Tzcnt => LegacyPrefixes::_66F3, }, 4 | 8 => match op { UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => LegacyPrefixes::None, - UnaryRmROpcode::Lzcnt => LegacyPrefixes::_F3, + UnaryRmROpcode::Lzcnt | UnaryRmROpcode::Tzcnt => LegacyPrefixes::_F3, }, _ => unreachable!(), }; @@ -693,6 +693,7 @@ pub(crate) fn emit( UnaryRmROpcode::Bsr => (0x0fbd, 2), UnaryRmROpcode::Bsf => (0x0fbc, 2), UnaryRmROpcode::Lzcnt => (0x0fbd, 2), + UnaryRmROpcode::Tzcnt => (0x0fbc, 2), }; match src { diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 837d4d1266..ccf887bb87 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2346,13 +2346,27 @@ fn lower_insn_to_regs>( } Opcode::Ctz => { - // TODO when the x86 flags have use_bmi1, we can use TZCNT. + let orig_ty = ctx.input_ty(insn, 0); + + if isa_flags.use_bmi1() && (orig_ty == types::I32 || orig_ty == types::I64) { + // We can use a plain tzcnt instruction here. Note no special handling is required + // for zero inputs, because the machine instruction does what the CLIF expects for + // zero, i.e. it returns zero. + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::unary_rm_r( + orig_ty.bytes() as u8, + UnaryRmROpcode::Tzcnt, + src, + dst, + )); + return Ok(()); + } // General formula using bit-scan forward (BSF): // bsf %src, %dst // mov $(size_bits), %tmp // cmovz %tmp, %dst - let orig_ty = ctx.input_ty(insn, 0); if orig_ty == types::I128 { // ctz src_lo, dst // ctz src_hi, tmp1 diff --git a/cranelift/filetests/filetests/isa/x64/ctz-bmi1.clif b/cranelift/filetests/filetests/isa/x64/ctz-bmi1.clif new file mode 100644 index 0000000000..b50b10107a --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/ctz-bmi1.clif @@ -0,0 +1,31 @@ +test compile +target x86_64 has_bmi1 +feature "experimental_x64" + +function %ctz(i64) -> i64 { +block0(v0: i64): + v1 = ctz v0 + return v1 +} + +; check: pushq %rbp +; check: movq %rsp, %rbp +; check: tzcntq %rdi, %rsi +; check: movq %rsi, %rax +; check: movq %rbp, %rsp +; check: popq %rbp +; check: ret + +function %ctz(i32) -> i32 { +block0(v0: i32): + v1 = ctz v0 + return v1 +} + +; check: pushq %rbp +; check: movq %rsp, %rbp +; check: tzcntl %edi, %esi +; check: movq %rsi, %rax +; check: movq %rbp, %rsp +; check: popq %rbp +; check: ret From 2275519cb19cacf3361a34fbcc9cb81f35f5ec1d Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 28 Jan 2021 12:16:46 +0100 Subject: [PATCH 55/55] cranelift x64: use the POPCNT instruction for Popcount when it's available; --- cranelift/codegen/src/isa/x64/inst/args.rs | 3 + cranelift/codegen/src/isa/x64/inst/emit.rs | 18 +++--- cranelift/codegen/src/isa/x64/lower.rs | 62 +++++++++++++++++-- .../filetests/isa/x64/popcnt-use-popcnt.clif | 31 ++++++++++ 4 files changed, 102 insertions(+), 12 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/x64/popcnt-use-popcnt.clif diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index a6938ca64d..0e99ff94fa 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -406,6 +406,8 @@ pub enum UnaryRmROpcode { Lzcnt, /// Counts trailing zeroes (Trailing Zero CouNT). Tzcnt, + /// Counts the number of ones (POPulation CouNT). + Popcnt, } impl fmt::Debug for UnaryRmROpcode { @@ -415,6 +417,7 @@ impl fmt::Debug for UnaryRmROpcode { UnaryRmROpcode::Bsf => write!(fmt, "bsf"), UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"), UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"), + UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"), } } } diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 74559272a7..4d6ae596eb 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -677,23 +677,25 @@ pub(crate) fn emit( _ => unreachable!(), }; + use UnaryRmROpcode::*; let prefix = match size { 2 => match op { - UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => LegacyPrefixes::_66, - UnaryRmROpcode::Lzcnt | UnaryRmROpcode::Tzcnt => LegacyPrefixes::_66F3, + Bsr | Bsf => LegacyPrefixes::_66, + Lzcnt | Tzcnt | Popcnt => LegacyPrefixes::_66F3, }, 4 | 8 => match op { - UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => LegacyPrefixes::None, - UnaryRmROpcode::Lzcnt | UnaryRmROpcode::Tzcnt => LegacyPrefixes::_F3, + Bsr | Bsf => LegacyPrefixes::None, + Lzcnt | Tzcnt | Popcnt => LegacyPrefixes::_F3, }, _ => unreachable!(), }; let (opcode, num_opcodes) = match op { - UnaryRmROpcode::Bsr => (0x0fbd, 2), - UnaryRmROpcode::Bsf => (0x0fbc, 2), - UnaryRmROpcode::Lzcnt => (0x0fbd, 2), - UnaryRmROpcode::Tzcnt => (0x0fbc, 2), + Bsr => (0x0fbd, 2), + Bsf => (0x0fbc, 2), + Lzcnt => (0x0fbd, 2), + Tzcnt => (0x0fbc, 2), + Popcnt => (0x0fb8, 2), }; match src { diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index ccf887bb87..1319ac2f94 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2410,15 +2410,69 @@ fn lower_insn_to_regs>( } Opcode::Popcnt => { - // TODO when the x86 flags have use_popcnt, we can use the popcnt instruction. - let (ext_spec, ty) = match ctx.input_ty(insn, 0) { types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32), - a if a == types::I32 || a == types::I64 => (None, a), - types::I128 => (None, types::I128), + a if a == types::I32 || a == types::I64 || a == types::I128 => (None, a), _ => unreachable!(), }; + if isa_flags.use_popcnt() { + match ty { + types::I32 | types::I64 => { + let src = input_to_reg_mem(ctx, inputs[0]); + let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + ctx.emit(Inst::unary_rm_r( + ty.bytes() as u8, + UnaryRmROpcode::Popcnt, + src, + dst, + )); + return Ok(()); + } + + types::I128 => { + // The number of ones in a 128-bits value is the plain sum of the number of + // ones in its low and high parts. No risk of overflow here. + let dsts = get_output_reg(ctx, outputs[0]); + let dst = dsts.regs()[0]; + let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap(); + let srcs = put_input_in_regs(ctx, inputs[0]); + let src_lo = srcs.regs()[0]; + let src_hi = srcs.regs()[1]; + + ctx.emit(Inst::unary_rm_r( + 8, + UnaryRmROpcode::Popcnt, + RegMem::reg(src_lo), + dst, + )); + ctx.emit(Inst::unary_rm_r( + 8, + UnaryRmROpcode::Popcnt, + RegMem::reg(src_hi), + tmp, + )); + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Add, + RegMemImm::reg(tmp.to_reg()), + dst, + )); + + // Zero the result's high component. + ctx.emit(Inst::alu_rmi_r( + true, + AluRmiROpcode::Xor, + RegMemImm::reg(dsts.regs()[1].to_reg()), + dsts.regs()[1], + )); + + return Ok(()); + } + _ => {} + } + } + let (srcs, ty): (SmallVec<[RegMem; 2]>, Type) = if let Some(ext_spec) = ext_spec { ( smallvec![RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec))], diff --git a/cranelift/filetests/filetests/isa/x64/popcnt-use-popcnt.clif b/cranelift/filetests/filetests/isa/x64/popcnt-use-popcnt.clif new file mode 100644 index 0000000000..4e49cd6d4f --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/popcnt-use-popcnt.clif @@ -0,0 +1,31 @@ +test compile +target x86_64 has_popcnt has_sse42 +feature "experimental_x64" + +function %popcnt(i64) -> i64 { +block0(v0: i64): + v1 = popcnt v0 + return v1 +} + +; check: pushq %rbp +; check: movq %rsp, %rbp +; check: popcntq %rdi, %rsi +; check: movq %rsi, %rax +; check: movq %rbp, %rsp +; check: popq %rbp +; check: ret + +function %popcnt(i32) -> i32 { +block0(v0: i32): + v1 = popcnt v0 + return v1 +} + +; check: pushq %rbp +; check: movq %rsp, %rbp +; check: popcntl %edi, %esi +; check: movq %rsi, %rax +; check: movq %rbp, %rsp +; check: popq %rbp +; check: ret