diff --git a/Cargo.lock b/Cargo.lock index 2f98977231..2551a2e8ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,30 +189,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.55.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b13ce559e6433d360c26305643803cb52cfbabbc2b9c47ce04a58493dfb443" -dependencies = [ - "bitflags", - "cexpr", - "cfg-if 0.1.10", - "clang-sys", - "clap", - "env_logger 0.7.1", - "lazy_static", - "lazycell", - "log", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "which", -] - [[package]] name = "bindgen" version = "0.57.0" @@ -293,9 +269,9 @@ checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" [[package]] name = "cap-fs-ext" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee87a3a916d6f051fc6809c39c4627f0c3a73b2a803bcfbb5fdf2bdfa1da0cb" +checksum = "c4a7f9bdd49f3979c659811723041929803c986b3ed5381af726a4082e806c27" dependencies = [ "cap-primitives", "cap-std", @@ -306,9 +282,9 @@ dependencies = [ [[package]] name = "cap-primitives" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3e3ea29994a34f3bc67b5396a43c87597d302d9e2e5e3b3d5ba952d86c7b41" +checksum = "0e206e688244a8f8158f91d77d3d8ed5891eb27aef2542bb26ceb98b970d5597" dependencies = [ "errno", "fs-set-times", @@ -326,18 +302,18 @@ dependencies = [ [[package]] name = "cap-rand" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0418058b38db7efc6021c5ce012e3a39c57e1a4d7bf2ddcd3789771de505d2f" +checksum = "0842ea6c9a2e078ab901b3b805608af65e11fcbfb87024d37fb129b2b74cb1ac" dependencies = [ "rand 0.8.3", ] [[package]] name = "cap-std" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5f20cbb3055e9c72b16ba45913fe9f92836d2aa7a880e1ffacb8d244f454319" +checksum = "36d8cb4b761a2fb1ccf0e92d795c717bba9dcb1e54636ee0579fca628abfa305" dependencies = [ "cap-primitives", "posish", @@ -358,9 +334,9 @@ dependencies = [ [[package]] name = "cap-time-ext" -version = "0.13.7" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b684f9db089b0558520076b4eeda2b719a5c4c06f329be96c9497f2b48c3944" +checksum = "bc5c279d213dfe8f71c02c4cd740bd130d1dce39f45fdcdb745ae3de8a777fbd" dependencies = [ "cap-primitives", "once_cell", @@ -561,7 +537,6 @@ name = "cranelift-codegen" version = "0.73.0" dependencies = [ "bincode", - "byteorder", "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", @@ -577,7 +552,6 @@ dependencies = [ "smallvec", "souper-ir", "target-lexicon", - "thiserror", "wast 35.0.2", ] @@ -608,7 +582,6 @@ name = "cranelift-filetests" version = "0.73.0" dependencies = [ "anyhow", - "byteorder", "cranelift-codegen", "cranelift-frontend", "cranelift-interpreter", @@ -678,7 +651,6 @@ dependencies = [ "cranelift-entity", "hashbrown", "log", - "thiserror", ] [[package]] @@ -718,7 +690,6 @@ dependencies = [ "cranelift-codegen", "smallvec", "target-lexicon", - "thiserror", ] [[package]] @@ -1389,9 +1360,9 @@ checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" [[package]] name = "iter-enum" -version = "0.2.7" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad34f24d3b48ceffdff38af2df5ce1b7d1d9cc113e503d8e86fe8cdb889c871" +checksum = "4f947f0d9df7e69c4df60a950c0a83741455bb9ebd8fd9b5a87994dda4dbb005" dependencies = [ "derive_utils", "quote", @@ -1784,22 +1755,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openvino" -version = "0.1.8" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43eeb44285b7ce8e2012b92bec32968622e1dad452e812e6edea9e001e5e9410" +checksum = "0cb74b3d8c653f7a9928bda494d329e6363ea0b428d3a3e5805b45ebb74ace76" dependencies = [ "openvino-sys", "thiserror", ] [[package]] -name = "openvino-sys" -version = "0.1.8" +name = "openvino-finder" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb64bef270a1ff665b0b2e28ebfa213e6205a007ce88223d020730225d6008f" +checksum = "426587a131841eb1e1111b0fea96cbd4fd0fd5d7b6526fb9c41400587d1c525c" + +[[package]] +name = "openvino-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d5e5d5e913f4e9aa42b2a7ae9c8719aedb4bc0eb443bf92f07d9ee9a05e7b1" dependencies = [ - "bindgen 0.55.1", "cmake", + "lazy_static", + "libloading", + "openvino-finder", ] [[package]] @@ -3066,7 +3045,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ada4f4ae167325015f52cc65f9fb6c251b868d8fb3b6dd0ce2d60e497c4870a" dependencies = [ - "bindgen 0.57.0", + "bindgen", "cc", "cfg-if 0.1.10", ] @@ -3449,7 +3428,6 @@ dependencies = [ name = "wasmtime-environ" version = "0.26.0" dependencies = [ - "anyhow", "cfg-if 1.0.0", "cranelift-codegen", "cranelift-entity", @@ -3458,7 +3436,6 @@ dependencies = [ "indexmap", "log", "more-asserts", - "region", "serde", "thiserror", "wasmparser", @@ -3723,15 +3700,6 @@ dependencies = [ "wast 35.0.2", ] -[[package]] -name = "which" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" -dependencies = [ - "libc", -] - [[package]] name = "wiggle" version = "0.26.0" diff --git a/Cargo.toml b/Cargo.toml index 98c5e65f07..ae79508b1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,7 @@ members = [ ] [features] -default = ["jitdump", "wasmtime/wat", "wasmtime/parallel-compilation"] +default = ["jitdump", "wasmtime/wat", "wasmtime/parallel-compilation", "wasi-nn"] lightbeam = ["wasmtime/lightbeam"] jitdump = ["wasmtime/jitdump"] vtune = ["wasmtime/vtune"] diff --git a/RELEASES.md b/RELEASES.md index 73ebf304c4..acf115f1c5 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -4,6 +4,19 @@ ## Unreleased +### Added + +* Added `Store::with_limits`, `StoreLimits`, and `ResourceLimiter` to the + Wasmtime API to help with enforcing resource limits at runtime. The + `ResourceLimiter` trait can be implemented by custom resource limiters to + decide if linear memories or tables can be grown. + +### Changed + +* Breaking: `Memory::new` has been changed to return `Result` as creating a + host memory object is now a fallible operation when the initial size of + the memory exceeds the store limits. + ## 0.26.0 Released 2021-04-05. diff --git a/build.rs b/build.rs index 411593c505..42d786799f 100644 --- a/build.rs +++ b/build.rs @@ -220,10 +220,6 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { }, "Cranelift" => match (testsuite, testname) { ("simd", _) if cfg!(feature = "old-x86-backend") => return true, // skip all SIMD tests on old backend. - // These are only implemented on x64. - ("simd", "simd_i64x2_arith2") | ("simd", "simd_boolean") => { - return !platform_is_x64() || cfg!(feature = "old-x86-backend") - } // These are new instructions that are not really implemented in any backend. ("simd", "simd_i8x16_arith2") | ("simd", "simd_conversions") diff --git a/ci/run-wasi-crypto-example.sh b/ci/run-wasi-crypto-example.sh index d2582c71b1..b027e48340 100755 --- a/ci/run-wasi-crypto-example.sh +++ b/ci/run-wasi-crypto-example.sh @@ -7,4 +7,4 @@ pushd "$RUST_BINDINGS" cargo build --release --target=wasm32-wasi popd -cargo run --features wasi-crypto -- run "$RUST_BINDINGS/target/wasm32-wasi/release/wasi-crypto-guest.wasm" +cargo run --features wasi-crypto -- run "$RUST_BINDINGS/target/wasm32-wasi/release/wasi-crypto-guest.wasm" --wasi-modules=experimental-wasi-crypto diff --git a/ci/run-wasi-nn-example.sh b/ci/run-wasi-nn-example.sh index 49c472c95e..97b55362c6 100755 --- a/ci/run-wasi-nn-example.sh +++ b/ci/run-wasi-nn-example.sh @@ -37,7 +37,7 @@ cp target/wasm32-wasi/release/wasi-nn-example.wasm $TMP_DIR popd # Run the example in Wasmtime (note that the example uses `fixture` as the expected location of the model/tensor files). -OPENVINO_INSTALL_DIR=/opt/intel/openvino cargo run --features wasi-nn -- run --mapdir fixture::$TMP_DIR $TMP_DIR/wasi-nn-example.wasm +cargo run -- run --mapdir fixture::$TMP_DIR $TMP_DIR/wasi-nn-example.wasm --wasi-modules=experimental-wasi-nn # Clean up the temporary directory only if it was not specified (users may want to keep the directory around). if [[ $REMOVE_TMP_DIR -eq 1 ]]; then diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 90ca998ae8..5ef650f1b3 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -23,8 +23,6 @@ serde = { version = "1.0.94", features = ["derive"], optional = true } bincode = { version = "1.2.1", optional = true } gimli = { version = "0.23.0", default-features = false, features = ["write"], optional = true } smallvec = { version = "1.6.1" } -thiserror = "1.0.4" -byteorder = { version = "1.3.2", default-features = false } peepmatic = { path = "../peepmatic", optional = true, version = "0.73.0" } peepmatic-traits = { path = "../peepmatic/crates/traits", optional = true, version = "0.73.0" } peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.73.0" } diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index b534ec9765..aa3102797e 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -60,6 +60,8 @@ pub enum Reloc { Arm64Call, /// RISC-V call target RiscvCall, + /// s390x PC-relative 4-byte offset + S390xPCRel32Dbl, /// Elf x86_64 32 bit signed PC relative offset to two GOT entries for GD symbol. ElfX86_64TlsGd, @@ -75,6 +77,7 @@ impl fmt::Display for Reloc { match *self { Self::Abs4 => write!(f, "Abs4"), Self::Abs8 => write!(f, "Abs8"), + Self::S390xPCRel32Dbl => write!(f, "PCRel32Dbl"), Self::X86PCRel4 => write!(f, "PCRel4"), Self::X86PCRelRodata4 => write!(f, "PCRelRodata4"), Self::X86CallPCRel4 => write!(f, "CallPCRel4"), diff --git a/cranelift/codegen/src/data_value.rs b/cranelift/codegen/src/data_value.rs index 193607f392..a317c7f394 100644 --- a/cranelift/codegen/src/data_value.rs +++ b/cranelift/codegen/src/data_value.rs @@ -5,7 +5,6 @@ use crate::ir::{types, ConstantData, Type}; use core::convert::TryInto; use core::fmt::{self, Display, Formatter}; use core::ptr; -use thiserror::Error; /// Represent a data value. Where [Value] is an SSA reference, [DataValue] is the type + value /// that would be referred to by a [Value]. @@ -97,15 +96,38 @@ impl DataValue { } /// Record failures to cast [DataValue]. -#[derive(Error, Debug, PartialEq)] +#[derive(Debug, PartialEq)] #[allow(missing_docs)] pub enum DataValueCastFailure { - #[error("unable to cast data value of type {0} to type {1}")] TryInto(Type, Type), - #[error("unable to cast i64({0}) to a data value of type {1}")] FromInteger(i64, Type), } +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. +impl std::error::Error for DataValueCastFailure {} + +impl Display for DataValueCastFailure { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + DataValueCastFailure::TryInto(from, to) => { + write!( + f, + "unable to cast data value of type {} to type {}", + from, to + ) + } + DataValueCastFailure::FromInteger(val, to) => { + write!( + f, + "unable to cast i64({}) to a data value of type {}", + val, to + ) + } + } + } +} + /// Helper for creating conversion implementations for [DataValue]. macro_rules! build_conversion_impl { ( $rust_ty:ty, $data_value_ty:ident, $cranelift_ty:ident ) => { diff --git a/cranelift/codegen/src/ir/entities.rs b/cranelift/codegen/src/ir/entities.rs index 09eaed3bec..d8ca7cef36 100644 --- a/cranelift/codegen/src/ir/entities.rs +++ b/cranelift/codegen/src/ir/entities.rs @@ -146,7 +146,7 @@ impl StackSlot { /// [`VmContext`](super::GlobalValueData::VMContext) using /// [`FuncEnvironment::make_global`](https://docs.rs/cranelift-wasm/*/cranelift_wasm/trait.FuncEnvironment.html#tymethod.make_global). /// - When compiling to native code, you can use it for objects in static memory with -/// [`Module::declare_data_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/struct.Module.html#method.declare_data_in_func). +/// [`Module::declare_data_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/trait.Module.html#method.declare_data_in_func). /// - For any compilation target, it can be registered with /// [`FunctionBuilder::create_global_value`](https://docs.rs/cranelift-frontend/*/cranelift_frontend/struct.FunctionBuilder.html#method.create_global_value). /// @@ -264,9 +264,9 @@ impl JumpTable { /// /// - [`FunctionBuilder::import_function`](https://docs.rs/cranelift-frontend/*/cranelift_frontend/struct.FunctionBuilder.html#method.import_function) /// for external functions -/// - [`Module::declare_func_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/struct.Module.html#method.declare_func_in_func) +/// - [`Module::declare_func_in_func`](https://docs.rs/cranelift-module/*/cranelift_module/trait.Module.html#method.declare_func_in_func) /// for functions declared elsewhere in the same native -/// [`Module`](https://docs.rs/cranelift-module/*/cranelift_module/struct.Module.html) +/// [`Module`](https://docs.rs/cranelift-module/*/cranelift_module/trait.Module.html) /// - [`FuncEnvironment::make_direct_func`](https://docs.rs/cranelift-wasm/*/cranelift_wasm/trait.FuncEnvironment.html#tymethod.make_direct_func) /// for functions declared in the same WebAssembly /// [`FuncEnvironment`](https://docs.rs/cranelift-wasm/*/cranelift_wasm/trait.FuncEnvironment.html#tymethod.make_direct_func) diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 76aaf4ac95..6a264f1604 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -603,6 +603,7 @@ impl ABIMachineSpec for AArch64MachineDeps { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> (u64, SmallVec<[Inst; 16]>) { let mut insts = SmallVec::new(); let (clobbered_int, clobbered_vec) = get_regs_saved_in_prologue(call_conv, clobbers); @@ -819,6 +820,7 @@ impl ABIMachineSpec for AArch64MachineDeps { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> SmallVec<[Inst; 16]> { let mut insts = SmallVec::new(); let (clobbered_int, clobbered_vec) = get_regs_saved_in_prologue(call_conv, clobbers); diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 1e8ca78317..6621e3f409 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -427,6 +427,15 @@ fn enc_vec_rr_misc(qu: u32, size: u32, bits_12_16: u32, rd: Writable, rn: R | machreg_to_vec(rd.to_reg()) } +fn enc_vec_rr_pair(bits_12_16: u32, rd: Writable, rn: Reg) -> u32 { + debug_assert_eq!(bits_12_16 & 0b11111, bits_12_16); + + 0b010_11110_11_11000_11011_10_00000_00000 + | bits_12_16 << 12 + | machreg_to_vec(rn) << 5 + | machreg_to_vec(rd.to_reg()) +} + fn enc_vec_lanes(q: u32, u: u32, size: u32, opcode: u32, rd: Writable, rn: Reg) -> u32 { debug_assert_eq!(q & 0b1, q); debug_assert_eq!(u & 0b1, u); @@ -1628,6 +1637,7 @@ impl MachInstEmit for Inst { debug_assert!(size == VectorSize::Size8x8 || size == VectorSize::Size8x16); (0b0, 0b00101, enc_size) } + VecMisc2::Cmeq0 => (0b0, 0b01001, enc_size), }; sink.put4(enc_vec_rr_misc((q << 1) | u, size, bits_12_16, rd, rn)); } @@ -2054,6 +2064,13 @@ impl MachInstEmit for Inst { | machreg_to_vec(rd.to_reg()), ); } + &Inst::VecRRPair { op, rd, rn } => { + let bits_12_16 = match op { + VecPairOp::Addp => 0b11011, + }; + + sink.put4(enc_vec_rr_pair(bits_12_16, rd, rn)); + } &Inst::VecRRR { rd, rn, diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index 505fd2c86b..9f628fced6 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -2311,6 +2311,16 @@ fn test_aarch64_binemit() { "sqxtun v16.8b, v23.8h", )); + insns.push(( + Inst::VecRRPair { + op: VecPairOp::Addp, + rd: writable_vreg(0), + rn: vreg(30), + }, + "C0BBF15E", + "addp d0, v30.2d", + )); + insns.push(( Inst::VecRRR { alu_op: VecALUOp::Sqadd, @@ -3803,6 +3813,17 @@ fn test_aarch64_binemit() { "cnt v23.8b, v5.8b", )); + insns.push(( + Inst::VecMisc { + op: VecMisc2::Cmeq0, + rd: writable_vreg(12), + rn: vreg(27), + size: VectorSize::Size16x8, + }, + "6C9B604E", + "cmeq v12.8h, v27.8h, #0", + )); + insns.push(( Inst::VecLanes { op: VecLanesOp::Uminv, diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index f6a6aa59d0..35903c18d0 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -334,6 +334,8 @@ pub enum VecMisc2 { Frintp, /// Population count per byte Cnt, + /// Compare bitwise equal to 0 + Cmeq0, } /// A Vector narrowing operation with two registers. @@ -347,6 +349,13 @@ pub enum VecMiscNarrowOp { Sqxtun, } +/// A vector operation on a pair of elements with one register. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum VecPairOp { + /// Add pair of elements + Addp, +} + /// An operation across the lanes of vectors. #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum VecLanesOp { @@ -1011,6 +1020,13 @@ pub enum Inst { high_half: bool, }, + /// 1-operand vector instruction that operates on a pair of elements. + VecRRPair { + op: VecPairOp, + rd: Writable, + rn: Reg, + }, + /// A vector ALU op. VecRRR { alu_op: VecALUOp, @@ -2028,6 +2044,10 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(rd); } } + &Inst::VecRRPair { rd, rn, .. } => { + collector.add_def(rd); + collector.add_use(rn); + } &Inst::VecRRR { alu_op, rd, rn, rm, .. } => { @@ -2816,6 +2836,14 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { map_def(mapper, rd); } } + &mut Inst::VecRRPair { + ref mut rd, + ref mut rn, + .. + } => { + map_def(mapper, rd); + map_use(mapper, rn); + } &mut Inst::VecRRR { alu_op, ref mut rd, @@ -3856,6 +3884,15 @@ impl Inst { }; format!("{} {}, {}", op, rd, rn) } + &Inst::VecRRPair { op, rd, rn } => { + let op = match op { + VecPairOp::Addp => "addp", + }; + let rd = show_vreg_scalar(rd.to_reg(), mb_rru, ScalarSize::Size64); + let rn = show_vreg_vector(rn, mb_rru, VectorSize::Size64x2); + + format!("{} {}, {}", op, rd, rn) + } &Inst::VecRRR { rd, rn, @@ -3919,43 +3956,44 @@ impl Inst { format!("{} {}, {}, {}", op, rd, rn, rm) } &Inst::VecMisc { op, rd, rn, size } => { - let is_shll = op == VecMisc2::Shll; - let suffix = match (is_shll, size) { - (true, VectorSize::Size8x8) => ", #8", - (true, VectorSize::Size16x4) => ", #16", - (true, VectorSize::Size32x2) => ", #32", - _ => "", - }; - - let (op, size) = match op { - VecMisc2::Not => ( - "mvn", - if size.is_128bits() { + let (op, rd_size, size, suffix) = match op { + VecMisc2::Not => { + let size = if size.is_128bits() { VectorSize::Size8x16 } else { VectorSize::Size8x8 + }; + + ("mvn", size, size, "") + } + VecMisc2::Neg => ("neg", size, size, ""), + VecMisc2::Abs => ("abs", size, size, ""), + VecMisc2::Fabs => ("fabs", size, size, ""), + VecMisc2::Fneg => ("fneg", size, size, ""), + VecMisc2::Fsqrt => ("fsqrt", size, size, ""), + VecMisc2::Rev64 => ("rev64", size, size, ""), + VecMisc2::Shll => ( + "shll", + size.widen(), + size, + match size { + VectorSize::Size8x8 => ", #8", + VectorSize::Size16x4 => ", #16", + VectorSize::Size32x2 => ", #32", + _ => panic!("Unexpected vector size: {:?}", size), }, ), - VecMisc2::Neg => ("neg", size), - VecMisc2::Abs => ("abs", size), - VecMisc2::Fabs => ("fabs", size), - VecMisc2::Fneg => ("fneg", size), - VecMisc2::Fsqrt => ("fsqrt", size), - VecMisc2::Rev64 => ("rev64", size), - VecMisc2::Shll => ("shll", size), - VecMisc2::Fcvtzs => ("fcvtzs", size), - VecMisc2::Fcvtzu => ("fcvtzu", size), - VecMisc2::Scvtf => ("scvtf", size), - VecMisc2::Ucvtf => ("ucvtf", size), - VecMisc2::Frintn => ("frintn", size), - VecMisc2::Frintz => ("frintz", size), - VecMisc2::Frintm => ("frintm", size), - VecMisc2::Frintp => ("frintp", size), - VecMisc2::Cnt => ("cnt", size), + VecMisc2::Fcvtzs => ("fcvtzs", size, size, ""), + VecMisc2::Fcvtzu => ("fcvtzu", size, size, ""), + VecMisc2::Scvtf => ("scvtf", size, size, ""), + VecMisc2::Ucvtf => ("ucvtf", size, size, ""), + VecMisc2::Frintn => ("frintn", size, size, ""), + VecMisc2::Frintz => ("frintz", size, size, ""), + VecMisc2::Frintm => ("frintm", size, size, ""), + VecMisc2::Frintp => ("frintp", size, size, ""), + VecMisc2::Cnt => ("cnt", size, size, ""), + VecMisc2::Cmeq0 => ("cmeq", size, size, ", #0"), }; - - let rd_size = if is_shll { size.widen() } else { size }; - let rd = show_vreg_vector(rd.to_reg(), mb_rru, rd_size); let rn = show_vreg_vector(rn, mb_rru, size); format!("{} {}, {}{}", op, rd, rn, suffix) diff --git a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs index 9f2eb741a0..b514dc20b8 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/unwind/systemv.rs @@ -56,8 +56,8 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { fn sp(&self) -> u16 { regs::stack_reg().get_hw_encoding().into() } - fn fp(&self) -> u16 { - regs::fp_reg().get_hw_encoding().into() + fn fp(&self) -> Option { + Some(regs::fp_reg().get_hw_encoding().into()) } fn lr(&self) -> Option { Some(regs::link_reg().get_hw_encoding().into()) diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index be3edd953b..ede66295e9 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -1950,6 +1950,40 @@ pub(crate) fn lower_insn_to_regs>( } } + Opcode::VallTrue if ctx.input_ty(insn, 0) == I64X2 => { + let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); + let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let tmp = ctx.alloc_tmp(I64X2).only_reg().unwrap(); + + // cmeq vtmp.2d, vm.2d, #0 + // addp dtmp, vtmp.2d + // fcmp dtmp, dtmp + // cset xd, eq + // + // Note that after the ADDP the value of the temporary register will + // be either 0 when all input elements are true, i.e. non-zero, or a + // NaN otherwise (either -1 or -2 when represented as an integer); + // NaNs are the only floating-point numbers that compare unequal to + // themselves. + + ctx.emit(Inst::VecMisc { + op: VecMisc2::Cmeq0, + rd: tmp, + rn: rm, + size: VectorSize::Size64x2, + }); + ctx.emit(Inst::VecRRPair { + op: VecPairOp::Addp, + rd: tmp, + rn: tmp.to_reg(), + }); + ctx.emit(Inst::FpuCmp64 { + rn: tmp.to_reg(), + rm: tmp.to_reg(), + }); + materialize_bool_result(ctx, insn, rd, Cond::Eq); + } + Opcode::VanyTrue | Opcode::VallTrue => { let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); @@ -2180,6 +2214,47 @@ pub(crate) fn lower_insn_to_regs>( size: VectorSize::Size32x4, }); } + I64X2 => { + // mov dst_r, src_v.d[0] + // mov tmp_r0, src_v.d[1] + // lsr dst_r, dst_r, #63 + // lsr tmp_r0, tmp_r0, #63 + // add dst_r, dst_r, tmp_r0, lsl #1 + ctx.emit(Inst::MovFromVec { + rd: dst_r, + rn: src_v, + idx: 0, + size: VectorSize::Size64x2, + }); + ctx.emit(Inst::MovFromVec { + rd: tmp_r0, + rn: src_v, + idx: 1, + size: VectorSize::Size64x2, + }); + ctx.emit(Inst::AluRRImmShift { + alu_op: ALUOp::Lsr64, + rd: dst_r, + rn: dst_r.to_reg(), + immshift: ImmShift::maybe_from_u64(63).unwrap(), + }); + ctx.emit(Inst::AluRRImmShift { + alu_op: ALUOp::Lsr64, + rd: tmp_r0, + rn: tmp_r0.to_reg(), + immshift: ImmShift::maybe_from_u64(63).unwrap(), + }); + ctx.emit(Inst::AluRRRShift { + alu_op: ALUOp::Add32, + rd: dst_r, + rn: dst_r.to_reg(), + rm: tmp_r0.to_reg(), + shiftop: ShiftOpAndAmt::new( + ShiftOp::LSL, + ShiftOpShiftImm::maybe_from_shift(1).unwrap(), + ), + }); + } _ => panic!("arm64 isel: VhighBits unhandled, ty = {:?}", ty), } } diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs index 94627a227c..5a4145d8b7 100644 --- a/cranelift/codegen/src/isa/arm32/abi.rs +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -319,6 +319,7 @@ impl ABIMachineSpec for Arm32MachineDeps { _flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> (u64, SmallVec<[Inst; 16]>) { let mut insts = SmallVec::new(); if fixed_frame_storage_size > 0 { @@ -348,6 +349,7 @@ impl ABIMachineSpec for Arm32MachineDeps { _flags: &settings::Flags, clobbers: &Set>, _fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> SmallVec<[Inst; 16]> { let mut insts = SmallVec::new(); let clobbered_vec = get_callee_saves(clobbers); diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index a24f64a256..fccb46f7a6 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -69,7 +69,6 @@ use core::fmt; use core::fmt::{Debug, Formatter}; use core::hash::Hasher; use target_lexicon::{triple, Architecture, OperatingSystem, PointerWidth, Triple}; -use thiserror::Error; #[cfg(feature = "riscv")] mod riscv; @@ -178,17 +177,30 @@ pub fn lookup_by_name(name: &str) -> Result { } /// Describes reason for target lookup failure -#[derive(Error, PartialEq, Eq, Copy, Clone, Debug)] +#[derive(PartialEq, Eq, Copy, Clone, Debug)] pub enum LookupError { /// Support for this target was disabled in the current build. - #[error("Support for this target is disabled")] SupportDisabled, /// Support for this target has not yet been implemented. - #[error("Support for this target has not been implemented yet")] Unsupported, } +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. +impl std::error::Error for LookupError {} + +impl fmt::Display for LookupError { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + LookupError::SupportDisabled => write!(f, "Support for this target is disabled"), + LookupError::Unsupported => { + write!(f, "Support for this target has not been implemented yet") + } + } + } +} + /// Builder for a `TargetIsa`. /// Modify the ISA-specific settings before creating the `TargetIsa` trait object with `finish`. #[derive(Clone)] diff --git a/cranelift/codegen/src/isa/unwind.rs b/cranelift/codegen/src/isa/unwind.rs index 7c9718a570..13397c3266 100644 --- a/cranelift/codegen/src/isa/unwind.rs +++ b/cranelift/codegen/src/isa/unwind.rs @@ -225,6 +225,11 @@ pub enum UnwindInst { /// the clobber area. offset_downward_to_clobbers: u32, }, + /// The stack pointer was adjusted to allocate the stack. + StackAlloc { + /// Size to allocate. + size: u32, + }, /// The stack slot at the given offset from the clobber-area base has been /// used to save the given register. /// diff --git a/cranelift/codegen/src/isa/unwind/systemv.rs b/cranelift/codegen/src/isa/unwind/systemv.rs index 965603d4e1..da3bfea869 100644 --- a/cranelift/codegen/src/isa/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/unwind/systemv.rs @@ -6,7 +6,6 @@ use crate::isa::unwind::UnwindInst; use crate::result::{CodegenError, CodegenResult}; use alloc::vec::Vec; use gimli::write::{Address, FrameDescriptionEntry}; -use thiserror::Error; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -15,16 +14,32 @@ type Register = u16; /// Enumerate the errors possible in mapping Cranelift registers to their DWARF equivalent. #[allow(missing_docs)] -#[derive(Error, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] pub enum RegisterMappingError { - #[error("unable to find bank for register info")] MissingBank, - #[error("register mapping is currently only implemented for x86_64")] UnsupportedArchitecture, - #[error("unsupported register bank: {0}")] UnsupportedRegisterBank(&'static str), } +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. +impl std::error::Error for RegisterMappingError {} + +impl std::fmt::Display for RegisterMappingError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + RegisterMappingError::MissingBank => write!(f, "unable to find bank for register info"), + RegisterMappingError::UnsupportedArchitecture => write!( + f, + "register mapping is currently only implemented for x86_64" + ), + RegisterMappingError::UnsupportedRegisterBank(bank) => { + write!(f, "unsupported register bank: {}", bank) + } + } + } +} + // This mirrors gimli's CallFrameInstruction, but is serializable // This excludes CfaExpression, Expression, ValExpression due to // https://github.com/gimli-rs/gimli/issues/513. @@ -122,8 +137,10 @@ pub(crate) trait RegisterMapper { fn map(&self, reg: Reg) -> Result; /// Gets stack pointer register. fn sp(&self) -> Register; - /// Gets the frame pointer register. - fn fp(&self) -> Register; + /// Gets the frame pointer register, if any. + fn fp(&self) -> Option { + None + } /// Gets the link register, if any. fn lr(&self) -> Option { None @@ -151,6 +168,7 @@ pub(crate) fn create_unwind_info_from_insts>( ) -> CodegenResult { let mut instructions = vec![]; + let mut cfa_offset = 0; let mut clobber_offset_to_cfa = 0; for &(instruction_offset, ref inst) in insts { match inst { @@ -163,10 +181,14 @@ pub(crate) fn create_unwind_info_from_insts>( instruction_offset, CallFrameInstruction::CfaOffset(offset_upward_to_caller_sp as i32), )); - // Note that we saved the old FP value on the stack. + // Note that we saved the old FP value on the stack. Use of this + // operation implies that the target defines a FP register. instructions.push(( instruction_offset, - CallFrameInstruction::Offset(mr.fp(), -(offset_upward_to_caller_sp as i32)), + CallFrameInstruction::Offset( + mr.fp().unwrap(), + -(offset_upward_to_caller_sp as i32), + ), )); // If there is a link register on this architecture, note that // we saved it as well. @@ -188,15 +210,29 @@ pub(crate) fn create_unwind_info_from_insts>( // Define CFA in terms of FP. Note that we assume it was already // defined correctly in terms of the current SP, and FP has just // been set to the current SP, so we do not need to change the - // offset, only the register. - instructions.push(( - instruction_offset, - CallFrameInstruction::CfaRegister(mr.fp()), - )); + // offset, only the register. (This is done only if the target + // defines a frame pointer register.) + if let Some(fp) = mr.fp() { + instructions.push((instruction_offset, CallFrameInstruction::CfaRegister(fp))); + } + // Record initial CFA offset. This will be used with later + // StackAlloc calls if we do not have a frame pointer. + cfa_offset = offset_upward_to_caller_sp; // Record distance from CFA downward to clobber area so we can // express clobber offsets later in terms of CFA. clobber_offset_to_cfa = offset_upward_to_caller_sp + offset_downward_to_clobbers; } + &UnwindInst::StackAlloc { size } => { + // If we do not use a frame pointer, we need to update the + // CFA offset whenever the stack pointer changes. + if mr.fp().is_none() { + cfa_offset += size; + instructions.push(( + instruction_offset, + CallFrameInstruction::CfaOffset(cfa_offset as i32), + )); + } + } &UnwindInst::SaveReg { clobber_offset, reg, diff --git a/cranelift/codegen/src/isa/unwind/winx64.rs b/cranelift/codegen/src/isa/unwind/winx64.rs index f447d341ae..1c232f6855 100644 --- a/cranelift/codegen/src/isa/unwind/winx64.rs +++ b/cranelift/codegen/src/isa/unwind/winx64.rs @@ -3,7 +3,6 @@ use crate::isa::unwind::input; use crate::result::{CodegenError, CodegenResult}; use alloc::vec::Vec; -use byteorder::{ByteOrder, LittleEndian}; use log::warn; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -31,13 +30,13 @@ impl<'a> Writer<'a> { self.offset += 1; } - fn write_u16(&mut self, v: u16) { - T::write_u16(&mut self.buf[self.offset..(self.offset + 2)], v); + fn write_u16_le(&mut self, v: u16) { + self.buf[self.offset..(self.offset + 2)].copy_from_slice(&v.to_le_bytes()); self.offset += 2; } - fn write_u32(&mut self, v: u32) { - T::write_u32(&mut self.buf[self.offset..(self.offset + 4)], v); + fn write_u32_le(&mut self, v: u32) { + self.buf[self.offset..(self.offset + 4)].copy_from_slice(&v.to_le_bytes()); self.offset += 4; } } @@ -121,11 +120,11 @@ impl UnwindCode { let scaled_stack_offset = stack_offset / 16; if scaled_stack_offset <= core::u16::MAX as u32 { writer.write_u8((*reg << 4) | (op_small as u8)); - writer.write_u16::(scaled_stack_offset as u16); + writer.write_u16_le(scaled_stack_offset as u16); } else { writer.write_u8((*reg << 4) | (op_large as u8)); - writer.write_u16::(*stack_offset as u16); - writer.write_u16::((stack_offset >> 16) as u16); + writer.write_u16_le(*stack_offset as u16); + writer.write_u16_le((stack_offset >> 16) as u16); } } Self::StackAlloc { @@ -143,10 +142,10 @@ impl UnwindCode { ); } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { writer.write_u8(UnwindOperation::LargeStackAlloc as u8); - writer.write_u16::((*size / 8) as u16); + writer.write_u16_le((*size / 8) as u16); } else { writer.write_u8((1 << 4) | (UnwindOperation::LargeStackAlloc as u8)); - writer.write_u32::(*size); + writer.write_u32_le(*size); } } Self::SetFPReg { instruction_offset } => { @@ -248,7 +247,7 @@ impl UnwindInfo { // To keep a 32-bit alignment, emit 2 bytes of padding if there's an odd number of 16-bit nodes if (node_count & 1) == 1 { - writer.write_u16::(0); + writer.write_u16_le(0); } // Ensure the correct number of bytes was emitted @@ -356,6 +355,12 @@ pub(crate) fn create_unwind_info_from_insts>( frame_register_offset = ensure_unwind_offset(offset_downward_to_clobbers)?; unwind_codes.push(UnwindCode::SetFPReg { instruction_offset }); } + &UnwindInst::StackAlloc { size } => { + unwind_codes.push(UnwindCode::StackAlloc { + instruction_offset, + size, + }); + } &UnwindInst::SaveReg { clobber_offset, reg, diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 29a0dbd982..63a8af7c9b 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -500,6 +500,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> (u64, SmallVec<[Self::I; 16]>) { let mut insts = SmallVec::new(); // Find all clobbered registers that are callee-save. @@ -574,6 +575,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + _outgoing_args_size: u32, ) -> SmallVec<[Self::I; 16]> { let mut insts = SmallVec::new(); diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 2eebf4140e..b54f1b6126 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -460,9 +460,7 @@ pub(crate) enum InstructionSet { BMI1, #[allow(dead_code)] // never constructed (yet). BMI2, - #[allow(dead_code)] AVX512F, - #[allow(dead_code)] AVX512VL, } @@ -995,13 +993,11 @@ impl fmt::Display for SseOpcode { #[derive(Clone)] pub enum Avx512Opcode { - #[allow(dead_code)] Vpabsq, } impl Avx512Opcode { /// Which `InstructionSet`s support the opcode? - #[allow(dead_code)] pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> { match self { Avx512Opcode::Vpabsq => smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL], diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 3bd6a58e7c..a35bbe0a99 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -6,9 +6,11 @@ use crate::isa::x64::inst::args::*; use crate::isa::x64::inst::*; use crate::machinst::{inst_common, MachBuffer, MachInstEmit, MachLabel}; use core::convert::TryInto; +use encoding::evex::{EvexInstruction, EvexVectorLength}; use encoding::rex::{ emit_simm, emit_std_enc_enc, emit_std_enc_mem, emit_std_reg_mem, emit_std_reg_reg, int_reg_enc, - low8_will_sign_extend_to_32, low8_will_sign_extend_to_64, reg_enc, LegacyPrefixes, RexFlags, + low8_will_sign_extend_to_32, low8_will_sign_extend_to_64, reg_enc, LegacyPrefixes, OpcodeMap, + RexFlags, }; use log::debug; use regalloc::{Reg, Writable}; @@ -1404,6 +1406,24 @@ pub(crate) fn emit( }; } + Inst::XmmUnaryRmREvex { op, src, dst } => { + let opcode = match op { + Avx512Opcode::Vpabsq => 0x1f, + }; + match src { + RegMem::Reg { reg: src } => EvexInstruction::new() + .length(EvexVectorLength::V128) + .prefix(LegacyPrefixes::_66) + .map(OpcodeMap::_0F38) + .w(true) + .opcode(opcode) + .reg(dst.to_reg().get_hw_encoding()) + .rm(src.get_hw_encoding()) + .encode(sink), + _ => todo!(), + }; + } + Inst::XmmRmR { op, src: src_e, diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index f730d25e93..f03762b97b 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -3865,6 +3865,12 @@ fn test_x64_emit() { "cvtdq2pd %xmm2, %xmm8", )); + insns.push(( + Inst::xmm_unary_rm_r_evex(Avx512Opcode::Vpabsq, RegMem::reg(xmm2), w_xmm8), + "6272FD081FC2", + "vpabsq %xmm2, %xmm8", + )); + // Xmm to int conversions, and conversely. insns.push(( @@ -4276,6 +4282,7 @@ fn test_x64_emit() { let mut isa_flag_builder = x64::settings::builder(); isa_flag_builder.enable("has_ssse3").unwrap(); isa_flag_builder.enable("has_sse41").unwrap(); + isa_flag_builder.enable("has_avx512f").unwrap(); let isa_flags = x64::settings::Flags::new(&flags, isa_flag_builder); let rru = regs::create_reg_universe_systemv(&flags); diff --git a/cranelift/codegen/src/isa/x64/inst/encoding/evex.rs b/cranelift/codegen/src/isa/x64/inst/encoding/evex.rs new file mode 100644 index 0000000000..9029b9f43e --- /dev/null +++ b/cranelift/codegen/src/isa/x64/inst/encoding/evex.rs @@ -0,0 +1,396 @@ +//! Encodes EVEX instructions. These instructions are those added by the AVX-512 extensions. The +//! EVEX encoding requires a 4-byte prefix: +//! +//! Byte 0: 0x62 +//! ┌───┬───┬───┬───┬───┬───┬───┬───┐ +//! Byte 1: │ R │ X │ B │ R'│ 0 │ 0 │ m │ m │ +//! ├───┼───┼───┼───┼───┼───┼───┼───┤ +//! Byte 2: │ W │ v │ v │ v │ v │ 1 │ p │ p │ +//! ├───┼───┼───┼───┼───┼───┼───┼───┤ +//! Byte 3: │ z │ L'│ L │ b │ V'│ a │ a │ a │ +//! └───┴───┴───┴───┴───┴───┴───┴───┘ +//! +//! The prefix is then followeded by the opcode byte, the ModR/M byte, and other optional suffixes +//! (e.g. SIB byte, displacements, immediates) based on the instruction (see section 2.6, Intel +//! Software Development Manual, volume 2A). +use super::rex::{encode_modrm, LegacyPrefixes, OpcodeMap}; +use super::ByteSink; +use core::ops::RangeInclusive; + +/// Constructs an EVEX-encoded instruction using a builder pattern. This approach makes it visually +/// easier to transform something the manual's syntax, `EVEX.256.66.0F38.W1 1F /r` to code: +/// `EvexInstruction::new().length(...).prefix(...).map(...).w(true).opcode(0x1F).reg(...).rm(...)`. +pub struct EvexInstruction { + bits: u32, + opcode: u8, + reg: Register, + rm: Register, +} + +/// Because some of the bit flags in the EVEX prefix are reversed and users of `EvexInstruction` may +/// choose to skip setting fields, here we set some sane defaults. Note that: +/// - the first byte is always `0x62` but you will notice it at the end of the default `bits` value +/// implemented--remember the little-endian order +/// - some bits are always set to certain values: bits 10-11 to 0, bit 18 to 1 +/// - the other bits set correspond to reversed bits: R, X, B, R' (byte 1), vvvv (byte 2), V' (byte +/// 3). +/// +/// See the `default_emission` test for what these defaults are equivalent to (e.g. using RAX, +/// unsetting the W bit, etc.) +impl Default for EvexInstruction { + fn default() -> Self { + Self { + bits: 0x08_7C_F0_62, + opcode: 0, + reg: Register::default(), + rm: Register::default(), + } + } +} + +#[allow(non_upper_case_globals)] // This makes it easier to match the bit range names to the manual's names. +impl EvexInstruction { + /// Construct a default EVEX instruction. + pub fn new() -> Self { + Self::default() + } + + /// Set the length of the instruction . Note that there are sets of instructions (i.e. rounding, + /// memory broadcast) that modify the same underlying bits--at some point (TODO) we can add a + /// way to set those context bits and verify that both are not used (e.g. rounding AND length). + /// For now, this method is very convenient. + #[inline(always)] + pub fn length(mut self, length: EvexVectorLength) -> Self { + self.write(Self::LL, EvexContext::Other { length }.bits() as u32); + self + } + + /// Set the legacy prefix byte of the instruction: None | 66 | F0 | F2 | F3. EVEX instructions + /// pack these into the prefix, not as separate bytes. + #[inline(always)] + pub fn prefix(mut self, prefix: LegacyPrefixes) -> Self { + self.write(Self::pp, prefix.bits() as u32); + self + } + + /// Set the opcode map byte of the instruction: None | 0F | 0F38 | 0F3A. EVEX instructions pack + /// these into the prefix, not as separate bytes. + #[inline(always)] + pub fn map(mut self, map: OpcodeMap) -> Self { + self.write(Self::mm, map.bits() as u32); + self + } + + /// Set the W bit, typically used to indicate an instruction using 64 bits of an operand (e.g. + /// 64 bit lanes). EVEX packs this bit in the EVEX prefix; previous encodings used the REX + /// prefix. + #[inline(always)] + pub fn w(mut self, w: bool) -> Self { + self.write(Self::W, w as u32); + self + } + + /// Set the instruction opcode byte. + #[inline(always)] + pub fn opcode(mut self, opcode: u8) -> Self { + self.opcode = opcode; + self + } + + /// Set the register to use for the `reg` bits; many instructions use this as the write operand. + /// Setting this affects both the ModRM byte (`reg` section) and the EVEX prefix (the extension + /// bits for register encodings > 8). + #[inline(always)] + pub fn reg(mut self, reg: impl Into) -> Self { + self.reg = reg.into(); + let r = !(self.reg.0 >> 3) & 1; + let r_ = !(self.reg.0 >> 4) & 1; + self.write(Self::R, r as u32); + self.write(Self::R_, r_ as u32); + self + } + + /// Set the mask to use. See section 2.6 in the Intel Software Developer's Manual, volume 2A for + /// more details. + #[allow(dead_code)] + #[inline(always)] + pub fn mask(mut self, mask: EvexMasking) -> Self { + self.write(Self::aaa, mask.aaa_bits() as u32); + self.write(Self::z, mask.z_bit() as u32); + self + } + + /// Set the `vvvvv` register; some instructions allow using this as a second, non-destructive + /// source register in 3-operand instructions (e.g. 2 read, 1 write). + #[allow(dead_code)] + #[inline(always)] + pub fn vvvvv(mut self, reg: impl Into) -> Self { + let reg = reg.into(); + self.write(Self::vvvv, !(reg.0 as u32) & 0b1111); + self.write(Self::V_, !(reg.0 as u32 >> 4) & 0b1); + self + } + + /// Set the register to use for the `rm` bits; many instructions use this as the "read from + /// register/memory" operand. Currently this does not support memory addressing (TODO).Setting + /// this affects both the ModRM byte (`rm` section) and the EVEX prefix (the extension bits for + /// register encodings > 8). + #[inline(always)] + pub fn rm(mut self, reg: impl Into) -> Self { + self.rm = reg.into(); + let b = !(self.rm.0 >> 3) & 1; + let x = !(self.rm.0 >> 4) & 1; + self.write(Self::X, x as u32); + self.write(Self::B, b as u32); + self + } + + /// Emit the EVEX-encoded instruction to the code sink: + /// - first, the 4-byte EVEX prefix; + /// - then, the opcode byte; + /// - finally, the ModR/M byte. + /// + /// Eventually this method should support encodings of more than just the reg-reg addressing mode (TODO). + pub fn encode(&self, sink: &mut CS) { + sink.put4(self.bits); + sink.put1(self.opcode); + sink.put1(encode_modrm(3, self.reg.0 & 7, self.rm.0 & 7)); + } + + // In order to simplify the encoding of the various bit ranges in the prefix, we specify those + // ranges according to the table below (extracted from the Intel Software Development Manual, + // volume 2A). Remember that, because we pack the 4-byte prefix into a little-endian `u32`, this + // chart should be read from right-to-left, top-to-bottom. Note also that we start ranges at bit + // 8, leaving bits 0-7 for the mandatory `0x62`. + // ┌───┬───┬───┬───┬───┬───┬───┬───┐ + // Byte 1: │ R │ X │ B │ R'│ 0 │ 0 │ m │ m │ + // ├───┼───┼───┼───┼───┼───┼───┼───┤ + // Byte 2: │ W │ v │ v │ v │ v │ 1 │ p │ p │ + // ├───┼───┼───┼───┼───┼───┼───┼───┤ + // Byte 3: │ z │ L'│ L │ b │ V'│ a │ a │ a │ + // └───┴───┴───┴───┴───┴───┴───┴───┘ + + // Byte 1: + const mm: RangeInclusive = 8..=9; + const R_: RangeInclusive = 12..=12; + const B: RangeInclusive = 13..=13; + const X: RangeInclusive = 14..=14; + const R: RangeInclusive = 15..=15; + + // Byte 2: + const pp: RangeInclusive = 16..=17; + const vvvv: RangeInclusive = 19..=22; + const W: RangeInclusive = 23..=23; + + // Byte 3: + const aaa: RangeInclusive = 24..=26; + const V_: RangeInclusive = 27..=27; + #[allow(dead_code)] // Will be used once broadcast and rounding controls are exposed. + const b: RangeInclusive = 28..=28; + const LL: RangeInclusive = 29..=30; + const z: RangeInclusive = 31..=31; + + // A convenience method for writing the `value` bits to the given range in `self.bits`. + #[inline] + fn write(&mut self, range: RangeInclusive, value: u32) { + assert!(ExactSizeIterator::len(&range) > 0); + let size = range.end() - range.start() + 1; // Calculate the number of bits in the range. + let mask: u32 = (1 << size) - 1; // Generate a bit mask. + debug_assert!( + value <= mask, + "The written value should have fewer than {} bits.", + size + ); + let mask_complement = !(mask << *range.start()); // Create the bitwise complement for the clear mask. + self.bits &= mask_complement; // Clear the bits in `range`; otherwise the OR below may allow previously-set bits to slip through. + let value = value << *range.start(); // Place the value in the correct location (assumes `value <= mask`). + self.bits |= value; // Modify the bits in `range`. + } +} + +#[derive(Copy, Clone, Default)] +pub struct Register(u8); +impl From for Register { + fn from(reg: u8) -> Self { + debug_assert!(reg < 16); + Self(reg) + } +} + +/// Defines the EVEX context for the `L'`, `L`, and `b` bits (bits 6:4 of EVEX P2 byte). Table 2-36 in +/// section 2.6.10 (Intel Software Development Manual, volume 2A) describes how these bits can be +/// used together for certain classes of instructions; i.e., special care should be taken to ensure +/// that instructions use an applicable correct `EvexContext`. Table 2-39 contains cases where +/// opcodes can result in an #UD. +#[allow(dead_code)] // Rounding and broadcast modes are not yet used. +pub enum EvexContext { + RoundingRegToRegFP { + rc: EvexRoundingControl, + }, + NoRoundingFP { + sae: bool, + length: EvexVectorLength, + }, + MemoryOp { + broadcast: bool, + length: EvexVectorLength, + }, + Other { + length: EvexVectorLength, + }, +} + +impl Default for EvexContext { + fn default() -> Self { + Self::Other { + length: EvexVectorLength::default(), + } + } +} + +impl EvexContext { + /// Encode the `L'`, `L`, and `b` bits (bits 6:4 of EVEX P2 byte) for merging with the P2 byte. + fn bits(&self) -> u8 { + match self { + Self::RoundingRegToRegFP { rc } => 0b001 | rc.bits() << 1, + Self::NoRoundingFP { sae, length } => (*sae as u8) | length.bits() << 1, + Self::MemoryOp { broadcast, length } => (*broadcast as u8) | length.bits() << 1, + Self::Other { length } => length.bits() << 1, + } + } +} + +/// The EVEX format allows choosing a vector length in the `L'` and `L` bits; see `EvexContext`. +#[allow(dead_code)] // Wider-length vectors are not yet used. +pub enum EvexVectorLength { + V128, + V256, + V512, +} + +impl EvexVectorLength { + /// Encode the `L'` and `L` bits for merging with the P2 byte. + fn bits(&self) -> u8 { + match self { + Self::V128 => 0b00, + Self::V256 => 0b01, + Self::V512 => 0b10, + // 0b11 is reserved (#UD). + } + } +} + +impl Default for EvexVectorLength { + fn default() -> Self { + Self::V128 + } +} + +/// The EVEX format allows defining rounding control in the `L'` and `L` bits; see `EvexContext`. +#[allow(dead_code)] // Rounding controls are not yet used. +pub enum EvexRoundingControl { + RNE, + RD, + RU, + RZ, +} + +impl EvexRoundingControl { + /// Encode the `L'` and `L` bits for merging with the P2 byte. + fn bits(&self) -> u8 { + match self { + Self::RNE => 0b00, + Self::RD => 0b01, + Self::RU => 0b10, + Self::RZ => 0b11, + } + } +} + +/// Defines the EVEX masking behavior; masking support is described in section 2.6.4 of the Intel +/// Software Development Manual, volume 2A. +#[allow(dead_code)] // Masking is not yet used. +pub enum EvexMasking { + None, + Merging { k: u8 }, + Zeroing { k: u8 }, +} + +impl Default for EvexMasking { + fn default() -> Self { + EvexMasking::None + } +} + +impl EvexMasking { + /// Encode the `z` bit for merging with the P2 byte. + fn z_bit(&self) -> u8 { + match self { + Self::None | Self::Merging { .. } => 0, + Self::Zeroing { .. } => 1, + } + } + + /// Encode the `aaa` bits for merging with the P2 byte. + fn aaa_bits(&self) -> u8 { + match self { + Self::None => 0b000, + Self::Merging { k } | Self::Zeroing { k } => { + debug_assert!(*k <= 7); + *k + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::isa::x64::inst::regs; + use std::vec::Vec; + + // As a sanity test, we verify that the output of `xed-asmparse-main 'vpabsq xmm0{k0}, + // xmm1'` matches this EVEX encoding machinery. + #[test] + fn vpabsq() { + let dst = regs::xmm0(); + let src = regs::xmm1(); + let mut sink0 = Vec::new(); + + EvexInstruction::new() + .prefix(LegacyPrefixes::_66) + .map(OpcodeMap::_0F38) + .w(true) + .opcode(0x1F) + .reg(dst.get_hw_encoding()) + .rm(src.get_hw_encoding()) + .length(EvexVectorLength::V128) + .encode(&mut sink0); + + assert_eq!(sink0, vec![0x62, 0xf2, 0xfd, 0x08, 0x1f, 0xc1]); + } + + /// Verify that the defaults are equivalent to an instruction with a `0x00` opcode using the + /// "0" register (i.e. `rax`), with sane defaults for the various configurable parameters. This + /// test is more interesting than it may appear because some of the parameters have flipped-bit + /// representations (e.g. `vvvvv`) so emitting 0s as a default will not work. + #[test] + fn default_emission() { + let mut sink0 = Vec::new(); + EvexInstruction::new().encode(&mut sink0); + + let mut sink1 = Vec::new(); + EvexInstruction::new() + .length(EvexVectorLength::V128) + .prefix(LegacyPrefixes::None) + .map(OpcodeMap::None) + .w(false) + .opcode(0x00) + .reg(regs::rax().get_hw_encoding()) + .rm(regs::rax().get_hw_encoding()) + .mask(EvexMasking::None) + .encode(&mut sink1); + + assert_eq!(sink0, sink1); + } +} diff --git a/cranelift/codegen/src/isa/x64/inst/encoding/mod.rs b/cranelift/codegen/src/isa/x64/inst/encoding/mod.rs index 7fd3aeeae7..a269e58609 100644 --- a/cranelift/codegen/src/isa/x64/inst/encoding/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/encoding/mod.rs @@ -1 +1,57 @@ +use crate::{isa::x64, machinst::MachBuffer}; +use std::vec::Vec; + +pub mod evex; pub mod rex; +pub mod vex; + +pub trait ByteSink { + /// Add 1 byte to the code section. + fn put1(&mut self, _: u8); + + /// Add 2 bytes to the code section. + fn put2(&mut self, _: u16); + + /// Add 4 bytes to the code section. + fn put4(&mut self, _: u32); + + /// Add 8 bytes to the code section. + fn put8(&mut self, _: u64); +} + +impl ByteSink for MachBuffer { + fn put1(&mut self, value: u8) { + self.put1(value) + } + + fn put2(&mut self, value: u16) { + self.put2(value) + } + + fn put4(&mut self, value: u32) { + self.put4(value) + } + + fn put8(&mut self, value: u64) { + self.put8(value) + } +} + +/// Provide a convenient implementation for testing. +impl ByteSink for Vec { + fn put1(&mut self, v: u8) { + self.extend_from_slice(&[v]) + } + + fn put2(&mut self, v: u16) { + self.extend_from_slice(&v.to_le_bytes()) + } + + fn put4(&mut self, v: u32) { + self.extend_from_slice(&v.to_le_bytes()) + } + + fn put8(&mut self, v: u64) { + self.extend_from_slice(&v.to_le_bytes()) + } +} diff --git a/cranelift/codegen/src/isa/x64/inst/encoding/rex.rs b/cranelift/codegen/src/isa/x64/inst/encoding/rex.rs index 923f2e3615..648941790e 100644 --- a/cranelift/codegen/src/isa/x64/inst/encoding/rex.rs +++ b/cranelift/codegen/src/isa/x64/inst/encoding/rex.rs @@ -153,9 +153,37 @@ impl From<(OperandSize, Reg)> for RexFlags { } } +/// Allows using the same opcode byte in different "opcode maps" to allow for more instruction +/// encodings. See appendix A in the Intel Software Developer's Manual, volume 2A, for more details. +pub enum OpcodeMap { + None, + _0F, + _0F38, + _0F3A, +} + +impl OpcodeMap { + /// Normally the opcode map is specified as bytes in the instruction, but some x64 encoding + /// formats pack this information as bits in a prefix (e.g. EVEX). + pub(crate) fn bits(&self) -> u8 { + match self { + OpcodeMap::None => 0b00, + OpcodeMap::_0F => 0b01, + OpcodeMap::_0F38 => 0b10, + OpcodeMap::_0F3A => 0b11, + } + } +} + +impl Default for OpcodeMap { + fn default() -> Self { + Self::None + } +} + /// We may need to include one or more legacy prefix bytes before the REX prefix. This enum /// covers only the small set of possibilities that we actually need. -pub(crate) enum LegacyPrefixes { +pub enum LegacyPrefixes { /// No prefix bytes. None, /// Operand Size Override -- here, denoting "16-bit operation". @@ -173,26 +201,47 @@ pub(crate) enum LegacyPrefixes { } impl LegacyPrefixes { + /// Emit the legacy prefix as bytes (e.g. in REX instructions). #[inline(always)] pub(crate) fn emit(&self, sink: &mut MachBuffer) { match self { - LegacyPrefixes::_66 => sink.put1(0x66), - LegacyPrefixes::_F0 => sink.put1(0xF0), - LegacyPrefixes::_66F0 => { + Self::_66 => sink.put1(0x66), + Self::_F0 => sink.put1(0xF0), + Self::_66F0 => { // I don't think the order matters, but in any case, this is the same order that // the GNU assembler uses. sink.put1(0x66); sink.put1(0xF0); } - LegacyPrefixes::_F2 => sink.put1(0xF2), - LegacyPrefixes::_F3 => sink.put1(0xF3), - LegacyPrefixes::_66F3 => { + Self::_F2 => sink.put1(0xF2), + Self::_F3 => sink.put1(0xF3), + Self::_66F3 => { sink.put1(0x66); sink.put1(0xF3); } - LegacyPrefixes::None => (), + Self::None => (), } } + + /// Emit the legacy prefix as bits (e.g. for EVEX instructions). + #[inline(always)] + pub(crate) fn bits(&self) -> u8 { + match self { + Self::None => 0b00, + Self::_66 => 0b01, + Self::_F3 => 0b10, + Self::_F2 => 0b11, + _ => panic!( + "VEX and EVEX bits can only be extracted from single prefixes: None, 66, F3, F2" + ), + } + } +} + +impl Default for LegacyPrefixes { + fn default() -> Self { + Self::None + } } /// This is the core 'emit' function for instructions that reference memory. diff --git a/cranelift/codegen/src/isa/x64/inst/encoding/vex.rs b/cranelift/codegen/src/isa/x64/inst/encoding/vex.rs new file mode 100644 index 0000000000..f2f3feebba --- /dev/null +++ b/cranelift/codegen/src/isa/x64/inst/encoding/vex.rs @@ -0,0 +1,2 @@ +//! Encodes VEX instructions. These instructions are those added by the Advanced Vector Extensions +//! (AVX). diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 0e8b8d9f17..cfb0351bf3 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -225,6 +225,12 @@ pub enum Inst { dst: Writable, }, + XmmUnaryRmREvex { + op: Avx512Opcode, + src: RegMem, + dst: Writable, + }, + /// XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd, movq XmmMovRM { op: SseOpcode, @@ -571,6 +577,8 @@ impl Inst { | Inst::XmmRmRImm { op, .. } | Inst::XmmToGpr { op, .. } | Inst::XmmUnaryRmR { op, .. } => smallvec![op.available_from()], + + Inst::XmmUnaryRmREvex { op, .. } => op.available_from(), } } } @@ -705,6 +713,12 @@ impl Inst { Inst::XmmUnaryRmR { op, src, dst } } + pub(crate) fn xmm_unary_rm_r_evex(op: Avx512Opcode, src: RegMem, dst: Writable) -> Inst { + src.assert_regclass_is(RegClass::V128); + debug_assert!(dst.to_reg().get_class() == RegClass::V128); + Inst::XmmUnaryRmREvex { op, src, dst } + } + pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { src.assert_regclass_is(RegClass::V128); debug_assert!(dst.to_reg().get_class() == RegClass::V128); @@ -1391,6 +1405,13 @@ impl PrettyPrint for Inst { show_ireg_sized(dst.to_reg(), mb_rru, 8), ), + Inst::XmmUnaryRmREvex { op, src, dst, .. } => format!( + "{} {}, {}", + ljustify(op.to_string()), + src.show_rru_sized(mb_rru, 8), + show_ireg_sized(dst.to_reg(), mb_rru, 8), + ), + Inst::XmmMovRM { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), @@ -1863,7 +1884,9 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(Writable::from_reg(regs::rdx())); } }, - Inst::UnaryRmR { src, dst, .. } | Inst::XmmUnaryRmR { src, dst, .. } => { + Inst::UnaryRmR { src, dst, .. } + | Inst::XmmUnaryRmR { src, dst, .. } + | Inst::XmmUnaryRmREvex { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); } @@ -2210,6 +2233,11 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut dst, .. } + | Inst::XmmUnaryRmREvex { + ref mut src, + ref mut dst, + .. + } | Inst::UnaryRmR { ref mut src, ref mut dst, diff --git a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs index c2a04a5c8e..9115db0671 100644 --- a/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x64/inst/unwind/systemv.rs @@ -89,8 +89,8 @@ impl crate::isa::unwind::systemv::RegisterMapper for RegisterMapper { fn sp(&self) -> u16 { X86_64::RSP.0 } - fn fp(&self) -> u16 { - X86_64::RBP.0 + fn fp(&self) -> Option { + Some(X86_64::RBP.0) } } diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 26c0b89740..6f675b9232 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1855,25 +1855,29 @@ fn lower_insn_to_regs>( let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); let ty = ty.unwrap(); if ty == types::I64X2 { - // This lowering could be a single instruction with AVX512F/VL's VPABSQ instruction. - // Instead, we use a separate register, `tmp`, to contain the results of `0 - src` - // and then blend in those results with `BLENDVPD` if the MSB of `tmp` was set to 1 - // (i.e. if `tmp` was negative or, conversely, if `src` was originally positive). + if isa_flags.use_avx512f_simd() || isa_flags.use_avx512vl_simd() { + ctx.emit(Inst::xmm_unary_rm_r_evex(Avx512Opcode::Vpabsq, src, dst)); + } else { + // If `VPABSQ` from AVX512 is unavailable, we use a separate register, `tmp`, to + // contain the results of `0 - src` and then blend in those results with + // `BLENDVPD` if the MSB of `tmp` was set to 1 (i.e. if `tmp` was negative or, + // conversely, if `src` was originally positive). - // Emit all 0s into the `tmp` register. - let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp), tmp)); - // Subtract the lanes from 0 and set up `dst`. - ctx.emit(Inst::xmm_rm_r(SseOpcode::Psubq, src.clone(), tmp)); - ctx.emit(Inst::gen_move(dst, tmp.to_reg(), ty)); - // Choose the subtracted lanes when `tmp` has an MSB of 1. BLENDVPD's semantics - // require the "choice" mask to be in XMM0. - ctx.emit(Inst::gen_move( - Writable::from_reg(regs::xmm0()), - tmp.to_reg(), - ty, - )); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Blendvpd, src, dst)); + // Emit all 0s into the `tmp` register. + let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp), tmp)); + // Subtract the lanes from 0 and set up `dst`. + ctx.emit(Inst::xmm_rm_r(SseOpcode::Psubq, src.clone(), tmp)); + ctx.emit(Inst::gen_move(dst, tmp.to_reg(), ty)); + // Choose the subtracted lanes when `tmp` has an MSB of 1. BLENDVPD's semantics + // require the "choice" mask to be in XMM0. + ctx.emit(Inst::gen_move( + Writable::from_reg(regs::xmm0()), + tmp.to_reg(), + ty, + )); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Blendvpd, src, dst)); + } } else if ty.is_vector() { let opcode = match ty { types::I8X16 => SseOpcode::Pabsb, diff --git a/cranelift/codegen/src/isa/x86/unwind/systemv.rs b/cranelift/codegen/src/isa/x86/unwind/systemv.rs index 0c3e749762..31fc64c9fb 100644 --- a/cranelift/codegen/src/isa/x86/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x86/unwind/systemv.rs @@ -121,8 +121,8 @@ pub(crate) fn create_unwind_info( fn sp(&self) -> u16 { X86_64::RSP.0 } - fn fp(&self) -> u16 { - X86_64::RBP.0 + fn fp(&self) -> Option { + Some(X86_64::RBP.0) } } let map = RegisterMapper(isa); diff --git a/cranelift/codegen/src/isa/x86/unwind/winx64.rs b/cranelift/codegen/src/isa/x86/unwind/winx64.rs index b2da0bc8b9..33e5463bb8 100644 --- a/cranelift/codegen/src/isa/x86/unwind/winx64.rs +++ b/cranelift/codegen/src/isa/x86/unwind/winx64.rs @@ -2,7 +2,7 @@ use crate::ir::Function; use crate::isa::x86::registers::{FPR, GPR}; -use crate::isa::{unwind::winx64::UnwindInfo, CallConv, RegUnit, TargetIsa}; +use crate::isa::{unwind::winx64::UnwindInfo, RegUnit, TargetIsa}; use crate::result::CodegenResult; pub(crate) fn create_unwind_info( @@ -10,7 +10,7 @@ pub(crate) fn create_unwind_info( isa: &dyn TargetIsa, ) -> CodegenResult> { // Only Windows fastcall is supported for unwind information - if func.signature.call_conv != CallConv::WindowsFastcall || func.prologue_end.is_none() { + if !func.signature.call_conv.extends_windows_fastcall() || func.prologue_end.is_none() { return Ok(None); } diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 5b80073b7f..331c8f81b7 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -97,6 +97,7 @@ mod inst_predicates; mod iterators; mod legalizer; mod licm; +mod log; mod nan_canonicalization; mod partition_slice; mod postopt; diff --git a/cranelift/codegen/src/log.rs b/cranelift/codegen/src/log.rs new file mode 100644 index 0000000000..c5bd59aa58 --- /dev/null +++ b/cranelift/codegen/src/log.rs @@ -0,0 +1,39 @@ +//! This module implements deferred display helpers. +//! +//! These are particularly useful in logging contexts, where the maximum logging level filter might +//! be enabled, but we don't want the arguments to be evaluated early: +//! +//! ``` +//! log::set_max_level(log::LevelFilter::max()); +//! fn expensive_calculation() -> String { +//! "a string that is very slow to generate".into() +//! } +//! log::debug!("{}", expensive_calculation()); +//! ``` +//! +//! If the associated log implementation filters out log debug entries, the expensive calculation +//! would have been spurious. In this case, we can wrap the expensive computation within an +//! `DeferredDisplay`, so that the computation only happens when the actual `fmt` function is +//! called. + +use core::fmt; + +pub(crate) struct DeferredDisplay(F); + +impl T, T: fmt::Display> DeferredDisplay { + pub(crate) fn new(f: F) -> Self { + Self(f) + } +} + +impl T, T: fmt::Display> fmt::Display for DeferredDisplay { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0().fmt(f) + } +} + +impl T, T: fmt::Debug> fmt::Debug for DeferredDisplay { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0().fmt(f) + } +} diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index ac4cb7fad2..7af9b087c8 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -30,6 +30,12 @@ pub trait ABICallee { /// Access the (possibly legalized) signature. fn signature(&self) -> &Signature; + /// Accumulate outgoing arguments. This ensures that at least SIZE bytes + /// are allocated in the prologue to be available for use in function calls + /// to hold arguments and/or return values. If this function is called + /// multiple times, the maximum of all SIZE values will be available. + fn accumulate_outgoing_args_size(&mut self, size: u32); + /// Get the settings controlling this function's compilation. fn flags(&self) -> &settings::Flags; @@ -242,6 +248,13 @@ pub trait ABICaller { /// Emit code to post-adjust the satck, after call return and return-value copies. fn emit_stack_post_adjust>(&self, ctx: &mut C); + /// Accumulate outgoing arguments. This ensures that the caller (as + /// identified via the CTX argument) allocates enough space in the + /// prologue to hold all arguments and return values for this call. + /// There is no code emitted at the call site, everything is done + /// in the caller's function prologue. + fn accumulate_outgoing_args_size>(&self, ctx: &mut C); + /// Emit the call itself. /// /// The returned instruction should have proper use- and def-sets according diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 85a6d6831b..56ebec48f0 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -444,6 +444,7 @@ pub trait ABIMachineSpec { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + outgoing_args_size: u32, ) -> (u64, SmallVec<[Self::I; 16]>); /// Generate a clobber-restore sequence. This sequence should perform the @@ -455,6 +456,7 @@ pub trait ABIMachineSpec { flags: &settings::Flags, clobbers: &Set>, fixed_frame_storage_size: u32, + outgoing_args_size: u32, ) -> SmallVec<[Self::I; 16]>; /// Generate a call instruction/sequence. This method is provided one @@ -576,6 +578,8 @@ pub struct ABICalleeImpl { stackslots: PrimaryMap, /// Total stack size of all stackslots. stackslots_size: u32, + /// Stack size to be reserved for outgoing arguments. + outgoing_args_size: u32, /// Clobbered registers, from regalloc. clobbered: Set>, /// Total number of spillslots, from regalloc. @@ -691,6 +695,7 @@ impl ABICalleeImpl { sig, stackslots, stackslots_size: stack_offset, + outgoing_args_size: 0, clobbered: Set::empty(), spillslots: None, fixed_frame_storage_size: 0, @@ -917,6 +922,12 @@ impl ABICallee for ABICalleeImpl { } } + fn accumulate_outgoing_args_size(&mut self, size: u32) { + if size > self.outgoing_args_size { + self.outgoing_args_size = size; + } + } + fn flags(&self) -> &settings::Flags { &self.flags } @@ -1198,6 +1209,15 @@ impl ABICallee for ABICalleeImpl { let spill_off = islot * M::word_bytes() as i64; let sp_off = self.stackslots_size as i64 + spill_off; trace!("load_spillslot: slot {:?} -> sp_off {}", slot, sp_off); + + // Integer types smaller than word size have been spilled as words below, + // and therefore must be reloaded in the same type. + let ty = if ty.is_int() && ty.bytes() < M::word_bytes() { + M::word_type() + } else { + ty + }; + gen_load_stack_multi::(StackAMode::NominalSPOffset(sp_off, ty), into_regs, ty) } @@ -1213,6 +1233,19 @@ impl ABICallee for ABICalleeImpl { let spill_off = islot * M::word_bytes() as i64; let sp_off = self.stackslots_size as i64 + spill_off; trace!("store_spillslot: slot {:?} -> sp_off {}", slot, sp_off); + + // When reloading from a spill slot, we might have lost information about real integer + // types. For instance, on the x64 backend, a zero-extension can become spurious and + // optimized into a move, causing vregs of types I32 and I64 to share the same coalescing + // equivalency class. As a matter of fact, such a value can be spilled as an I32 and later + // reloaded as an I64; to make sure the high bits are always defined, do a word-sized store + // all the time, in this case. + let ty = if ty.is_int() && ty.bytes() < M::word_bytes() { + M::word_type() + } else { + ty + }; + gen_store_stack_multi::(StackAMode::NominalSPOffset(sp_off, ty), from_regs, ty) } @@ -1285,11 +1318,12 @@ impl ABICallee for ABICalleeImpl { } // Save clobbered registers. - let (_, clobber_insts) = M::gen_clobber_save( + let (clobber_size, clobber_insts) = M::gen_clobber_save( self.call_conv, &self.flags, &self.clobbered, self.fixed_frame_storage_size, + self.outgoing_args_size, ); insts.extend(clobber_insts); @@ -1304,7 +1338,7 @@ impl ABICallee for ABICalleeImpl { // [crate::machinst::abi_impl](this module) for more details // on stackframe layout and nominal SP maintenance. - self.total_frame_size = Some(total_stacksize); + self.total_frame_size = Some(total_stacksize + clobber_size as u32); insts } @@ -1317,6 +1351,7 @@ impl ABICallee for ABICalleeImpl { &self.flags, &self.clobbered, self.fixed_frame_storage_size, + self.outgoing_args_size, )); // N.B.: we do *not* emit a nominal SP adjustment here, because (i) there will be no @@ -1519,6 +1554,11 @@ impl ABICaller for ABICallerImpl { } } + fn accumulate_outgoing_args_size>(&self, ctx: &mut C) { + let off = self.sig.stack_arg_space + self.sig.stack_ret_space; + ctx.abi().accumulate_outgoing_args_size(off as u32); + } + fn emit_stack_pre_adjust>(&self, ctx: &mut C) { let off = self.sig.stack_arg_space + self.sig.stack_ret_space; adjust_stack_and_nominal_sp::(ctx, off as i32, /* is_sub = */ true) diff --git a/cranelift/codegen/src/machinst/compile.rs b/cranelift/codegen/src/machinst/compile.rs index 0aaee5e98f..2dfbb85785 100644 --- a/cranelift/codegen/src/machinst/compile.rs +++ b/cranelift/codegen/src/machinst/compile.rs @@ -1,11 +1,12 @@ //! Compilation backend pipeline: optimized IR to VCode / binemit. use crate::ir::Function; +use crate::log::DeferredDisplay; use crate::machinst::*; use crate::settings; use crate::timing; -use log::{debug, log_enabled, Level}; +use log::debug; use regalloc::{allocate_registers_with_opts, Algorithm, Options, PrettyPrint}; /// Compile the given function down to VCode with allocated registers, ready @@ -29,15 +30,12 @@ where lower.lower(b)? }; - // Creating the vcode string representation may be costly for large functions, so don't do it - // if the Debug level hasn't been statically (through features) or dynamically (through - // RUST_LOG) enabled. - if log_enabled!(Level::Debug) { - debug!( - "vcode from lowering: \n{}", - vcode.show_rru(Some(b.reg_universe())) - ); - } + // Creating the vcode string representation may be costly for large functions, so defer its + // rendering. + debug!( + "vcode from lowering: \n{}", + DeferredDisplay::new(|| vcode.show_rru(Some(b.reg_universe()))) + ); // Perform register allocation. let (run_checker, algorithm) = match vcode.flags().regalloc() { @@ -106,12 +104,10 @@ where vcode.replace_insns_from_regalloc(result); } - if log_enabled!(Level::Debug) { - debug!( - "vcode after regalloc: final version:\n{}", - vcode.show_rru(Some(b.reg_universe())) - ); - } + debug!( + "vcode after regalloc: final version:\n{}", + DeferredDisplay::new(|| vcode.show_rru(Some(b.reg_universe()))) + ); Ok(vcode) } diff --git a/cranelift/codegen/src/result.rs b/cranelift/codegen/src/result.rs index 493545c151..3178cd5ba9 100644 --- a/cranelift/codegen/src/result.rs +++ b/cranelift/codegen/src/result.rs @@ -2,19 +2,17 @@ use crate::verifier::VerifierErrors; use std::string::String; -use thiserror::Error; /// A compilation error. /// /// When Cranelift fails to compile a function, it will return one of these error codes. -#[derive(Error, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] pub enum CodegenError { /// A list of IR verifier errors. /// /// This always represents a bug, either in the code that generated IR for Cranelift, or a bug /// in Cranelift itself. - #[error("Verifier errors")] - Verifier(#[from] VerifierErrors), + Verifier(VerifierErrors), /// An implementation limit was exceeded. /// @@ -22,27 +20,57 @@ pub enum CodegenError { /// limits][limits] that cause compilation to fail when they are exceeded. /// /// [limits]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/docs/ir.md#implementation-limits - #[error("Implementation limit exceeded")] ImplLimitExceeded, /// The code size for the function is too large. /// /// Different target ISAs may impose a limit on the size of a compiled function. If that limit /// is exceeded, compilation fails. - #[error("Code for function is too large")] CodeTooLarge, /// Something is not supported by the code generator. This might be an indication that a /// feature is used without explicitly enabling it, or that something is temporarily /// unsupported by a given target backend. - #[error("Unsupported feature: {0}")] Unsupported(String), /// A failure to map Cranelift register representation to a DWARF register representation. #[cfg(feature = "unwind")] - #[error("Register mapping error")] RegisterMappingError(crate::isa::unwind::systemv::RegisterMappingError), } /// A convenient alias for a `Result` that uses `CodegenError` as the error type. pub type CodegenResult = Result; + +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. +impl std::error::Error for CodegenError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + CodegenError::Verifier(source) => Some(source), + CodegenError::ImplLimitExceeded { .. } + | CodegenError::CodeTooLarge { .. } + | CodegenError::Unsupported { .. } => None, + #[cfg(feature = "unwind")] + CodegenError::RegisterMappingError { .. } => None, + } + } +} + +impl std::fmt::Display for CodegenError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + CodegenError::Verifier(_) => write!(f, "Verifier errors"), + CodegenError::ImplLimitExceeded => write!(f, "Implementation limit exceeded"), + CodegenError::CodeTooLarge => write!(f, "Code for function is too large"), + CodegenError::Unsupported(feature) => write!(f, "Unsupported feature: {}", feature), + #[cfg(feature = "unwind")] + CodegenError::RegisterMappingError(_0) => write!(f, "Register mapping error"), + } + } +} + +impl From for CodegenError { + fn from(source: VerifierErrors) -> Self { + CodegenError::Verifier { 0: source } + } +} diff --git a/cranelift/codegen/src/settings.rs b/cranelift/codegen/src/settings.rs index 88a3c62157..0f36db82a9 100644 --- a/cranelift/codegen/src/settings.rs +++ b/cranelift/codegen/src/settings.rs @@ -26,7 +26,6 @@ use alloc::boxed::Box; use alloc::string::{String, ToString}; use core::fmt; use core::str; -use thiserror::Error; /// A string-based configurator for settings groups. /// @@ -261,21 +260,34 @@ impl Configurable for Builder { } /// An error produced when changing a setting. -#[derive(Error, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] pub enum SetError { /// No setting by this name exists. - #[error("No existing setting named '{0}'")] BadName(String), /// Type mismatch for setting (e.g., setting an enum setting as a bool). - #[error("Trying to set a setting with the wrong type")] BadType, /// This is not a valid value for this setting. - #[error("Unexpected value for a setting, expected {0}")] BadValue(String), } +impl std::error::Error for SetError {} + +impl fmt::Display for SetError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + SetError::BadName(name) => write!(f, "No existing setting named '{}'", name), + SetError::BadType => { + write!(f, "Trying to set a setting with the wrong type") + } + SetError::BadValue(value) => { + write!(f, "Unexpected value for a setting, expected {}", value) + } + } + } +} + /// A result returned when changing a setting. pub type SetResult = Result; diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs index e20570c951..1d1801016b 100644 --- a/cranelift/codegen/src/verifier/mod.rs +++ b/cranelift/codegen/src/verifier/mod.rs @@ -80,7 +80,6 @@ use alloc::vec::Vec; use core::cmp::Ordering; use core::fmt::{self, Display, Formatter, Write}; use log::debug; -use thiserror::Error; pub use self::cssa::verify_cssa; pub use self::liveness::verify_liveness; @@ -92,8 +91,7 @@ mod liveness; mod locations; /// A verifier error. -#[derive(Error, Debug, PartialEq, Eq, Clone)] -#[error("{}{}: {}", .location, format_context(.context), .message)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct VerifierError { /// The entity causing the verifier error. pub location: AnyEntity, @@ -104,11 +102,16 @@ pub struct VerifierError { pub message: String, } -/// Helper for formatting Verifier::Error context. -fn format_context(context: &Option) -> String { - match context { - None => "".to_string(), - Some(c) => format!(" ({})", c), +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. +impl std::error::Error for VerifierError {} + +impl Display for VerifierError { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match &self.context { + None => write!(f, "{}: {}", self.location, self.message), + Some(context) => write!(f, "{} ({}): {}", self.location, context, self.message), + } } } @@ -175,9 +178,13 @@ pub type VerifierStepResult = Result; pub type VerifierResult = Result; /// List of verifier errors. -#[derive(Error, Debug, Default, PartialEq, Eq, Clone)] +#[derive(Debug, Default, PartialEq, Eq, Clone)] pub struct VerifierErrors(pub Vec); +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. +impl std::error::Error for VerifierErrors {} + impl VerifierErrors { /// Return a new `VerifierErrors` struct. #[inline] diff --git a/cranelift/filetests/Cargo.toml b/cranelift/filetests/Cargo.toml index ffeecd7ac6..f25cbcf5ac 100644 --- a/cranelift/filetests/Cargo.toml +++ b/cranelift/filetests/Cargo.toml @@ -16,7 +16,6 @@ cranelift-interpreter = { path = "../interpreter", version = "0.73.0" } cranelift-native = { path = "../native", version = "0.73.0" } cranelift-reader = { path = "../reader", version = "0.73.0" } cranelift-preopt = { path = "../preopt", version = "0.73.0" } -byteorder = { version = "1.3.2", default-features = false } file-per-thread-logger = "0.1.2" filecheck = "0.5.0" gimli = { version = "0.23.0", default-features = false, features = ["read"] } diff --git a/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif b/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif new file mode 100644 index 0000000000..31edd7bdca --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/store-stack-full-width-i32.clif @@ -0,0 +1,127 @@ +test compile +target x86_64 machinst + +;; The goal of this test is to ensure that stack spills of an integer value, +;; which width is less than the machine word's size, cause the full word to be +;; stored, and not only the lower bits. + +;; Because of unsigned extensions which can be transformed into simple moves, +;; the source vreg of the extension operation can be coalesced with its +;; destination vreg, and if it happens to be spill, then the reload may use a +;; reload of a different, larger size. + +function %f0(i32, i32, i32) -> i64 { + fn0 = %g(i32) -> i64 + +; check: pushq %rbp +; nextln: movq %rsp, %rbp +; nextln: subq $$64, %rsp + +;; Stash all the callee-saved registers. + +; nextln: movq %r12, 16(%rsp) +; nextln: movq %r13, 24(%rsp) +; nextln: movq %r14, 32(%rsp) +; nextln: movq %rbx, 40(%rsp) +; nextln: movq %r15, 48(%rsp) + +block0(v0: i32, v1: i32, v2: i32): + ;; First, create enough virtual registers so that the call instructions + ;; causes at least one of them to be spilled onto the stack. + + v3 = iadd.i32 v0, v1 + v4 = iadd.i32 v1, v2 + v5 = iadd.i32 v0, v2 + v6 = iadd.i32 v3, v0 + v7 = iadd.i32 v4, v0 + v8 = iadd.i32 v5, v0 + +; nextln: movq %rdi, %r12 +; nextln: addl %esi, %r12d +; nextln: movq %rsi, %r13 +; nextln: addl %edx, %r13d +; nextln: movq %rdi, %r14 +; nextln: addl %edx, %r14d +; nextln: movq %r12, %rbx +; nextln: addl %edi, %ebx +; nextln: movq %r13, %r15 +; nextln: addl %edi, %r15d +; nextln: movq %r14, %rsi + +;; This should be movq below, not movl. +; nextln: movq %rsi, rsp(0 + virtual offset) + +; nextln: movq rsp(0 + virtual offset), %rsi +; nextln: addl %edi, %esi + + ;; Put an effectful instruction so that the live-ranges of the adds and + ;; uextends are split here, and to prevent the uextend to be emitted + ;; before the call. This will effectively causing the above i32 to be + ;; spilled as an i32, and not a full i64. + + v300 = call fn0(v0) + +;; This should be movq below, not movl. +; nextln: movq %rsi, rsp(0 + virtual offset) + +; nextln: load_ext_name %g+0, %rsi +; nextln: call *%rsi + + v31 = uextend.i64 v3 + v41 = uextend.i64 v4 + v51 = uextend.i64 v5 + v61 = uextend.i64 v6 + v71 = uextend.i64 v7 + v81 = uextend.i64 v8 + + ;; None of the uextends are generated here yet. + + ;; At this point, I'd expect that this second call below would be not + ;; necessary, but if it is removed, the uextend is applied before the call, + ;; and the i64 is spilled (then reloaded), causing the bug to not appear. So + ;; an additional call it is! + + v100 = call fn0(v3) + +; nextln: movq %r12, %rsi +; nextln: movq %rsi, rsp(8 + virtual offset) +; nextln: nop len=0 +; nextln: movq %r12, %rdi +; nextln: load_ext_name %g+0, %rsi +; nextln: call *%rsi + + ;; Cause reloads of all the values. Most are in registers, but one of them + ;; is on the stack. Make sure they're all used in the final computation. + + v101 = iadd.i64 v100, v31 + v102 = iadd.i64 v101, v41 + v103 = iadd.i64 v102, v51 + v104 = iadd.i64 v103, v61 + v105 = iadd.i64 v104, v71 + v200 = iadd.i64 v105, v81 + +; nextln: movq %rax, %rsi +; nextln: movq rsp(8 + virtual offset), %rdi +; nextln: addq %rdi, %rsi +; nextln: addq %r13, %rsi +; nextln: addq %r14, %rsi +; nextln: addq %rbx, %rsi +; nextln: addq %r15, %rsi + +;; The reload operates on a full word, so uses movq. +; nextln: movq rsp(0 + virtual offset), %rdi + +; nextln: addq %rdi, %rsi +; nextln: movq %rsi, %rax +; nextln: movq 16(%rsp), %r12 +; nextln: movq 24(%rsp), %r13 +; nextln: movq 32(%rsp), %r14 +; nextln: movq 40(%rsp), %rbx +; nextln: movq 48(%rsp), %r15 +; nextln: addq $$64, %rsp + + return v200 +; nextln: movq %rbp, %rsp +; nextln: popq %rbp +; nextln: ret +} diff --git a/cranelift/filetests/src/test_unwind.rs b/cranelift/filetests/src/test_unwind.rs index 0c7124e8b2..3d22c4d3d9 100644 --- a/cranelift/filetests/src/test_unwind.rs +++ b/cranelift/filetests/src/test_unwind.rs @@ -72,7 +72,6 @@ impl SubTest for TestUnwind { } mod windowsx64 { - use byteorder::{ByteOrder, LittleEndian}; use std::fmt::Write; pub fn dump(text: &mut W, mem: &[u8]) { @@ -165,23 +164,24 @@ mod windowsx64 { let op_and_info = mem[1]; let op = UnwindOperation::from(op_and_info & 0xF); let info = (op_and_info & 0xF0) >> 4; + let unwind_le_bytes = |bytes| match (bytes, &mem[2..]) { + (2, &[b0, b1, ..]) => UnwindValue::U16(u16::from_le_bytes([b0, b1])), + (4, &[b0, b1, b2, b3, ..]) => { + UnwindValue::U32(u32::from_le_bytes([b0, b1, b2, b3])) + } + (_, _) => panic!("not enough bytes to unwind value"), + }; - let value = match op { - UnwindOperation::LargeStackAlloc => match info { - 0 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), - 1 => UnwindValue::U32(LittleEndian::read_u32(&mem[2..])), - _ => panic!("unexpected stack alloc info value"), - }, - UnwindOperation::SaveNonvolatileRegister => { - UnwindValue::U16(LittleEndian::read_u16(&mem[2..])) - } - UnwindOperation::SaveNonvolatileRegisterFar => { - UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) - } - UnwindOperation::SaveXmm128 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), - UnwindOperation::SaveXmm128Far => { - UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) + let value = match (&op, info) { + (UnwindOperation::LargeStackAlloc, 0) => unwind_le_bytes(2), + (UnwindOperation::LargeStackAlloc, 1) => unwind_le_bytes(4), + (UnwindOperation::LargeStackAlloc, _) => { + panic!("unexpected stack alloc info value") } + (UnwindOperation::SaveNonvolatileRegister, _) => unwind_le_bytes(2), + (UnwindOperation::SaveNonvolatileRegisterFar, _) => unwind_le_bytes(4), + (UnwindOperation::SaveXmm128, _) => unwind_le_bytes(2), + (UnwindOperation::SaveXmm128Far, _) => unwind_le_bytes(4), _ => UnwindValue::None, }; diff --git a/cranelift/frontend/src/frontend.rs b/cranelift/frontend/src/frontend.rs index 365e5aeffb..3ebfc8efa0 100644 --- a/cranelift/frontend/src/frontend.rs +++ b/cranelift/frontend/src/frontend.rs @@ -640,6 +640,7 @@ impl<'a> FunctionBuilder<'a> { dest_align: u8, src_align: u8, non_overlapping: bool, + mut flags: MemFlags, ) { // Currently the result of guess work, not actual profiling. const THRESHOLD: u64 = 4; @@ -676,7 +677,6 @@ impl<'a> FunctionBuilder<'a> { return; } - let mut flags = MemFlags::new(); flags.set_aligned(); // Load all of the memory first. This is necessary in case `dest` overlaps. @@ -732,6 +732,7 @@ impl<'a> FunctionBuilder<'a> { ch: u8, size: u64, buffer_align: u8, + mut flags: MemFlags, ) { // Currently the result of guess work, not actual profiling. const THRESHOLD: u64 = 4; @@ -763,7 +764,6 @@ impl<'a> FunctionBuilder<'a> { let size = self.ins().iconst(config.pointer_type(), size as i64); self.call_memset(config, buffer, ch, size); } else { - let mut flags = MemFlags::new(); flags.set_aligned(); let ch = u64::from(ch); @@ -851,7 +851,9 @@ mod tests { use alloc::string::ToString; use cranelift_codegen::entity::EntityRef; use cranelift_codegen::ir::types::*; - use cranelift_codegen::ir::{AbiParam, ExternalName, Function, InstBuilder, Signature}; + use cranelift_codegen::ir::{ + AbiParam, ExternalName, Function, InstBuilder, MemFlags, Signature, + }; use cranelift_codegen::isa::CallConv; use cranelift_codegen::settings; use cranelift_codegen::verifier::verify_function; @@ -1063,7 +1065,16 @@ block0: let src = builder.use_var(x); let dest = builder.use_var(y); let size = 8; - builder.emit_small_memory_copy(target.frontend_config(), dest, src, size, 8, 8, true); + builder.emit_small_memory_copy( + target.frontend_config(), + dest, + src, + size, + 8, + 8, + true, + MemFlags::new(), + ); builder.ins().return_(&[dest]); builder.seal_all_blocks(); @@ -1121,7 +1132,16 @@ block0: let src = builder.use_var(x); let dest = builder.use_var(y); let size = 8192; - builder.emit_small_memory_copy(target.frontend_config(), dest, src, size, 8, 8, true); + builder.emit_small_memory_copy( + target.frontend_config(), + dest, + src, + size, + 8, + 8, + true, + MemFlags::new(), + ); builder.ins().return_(&[dest]); builder.seal_all_blocks(); @@ -1179,7 +1199,7 @@ block0: let dest = builder.use_var(y); let size = 8; - builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8); + builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8, MemFlags::new()); builder.ins().return_(&[dest]); builder.seal_all_blocks(); @@ -1232,7 +1252,7 @@ block0: let dest = builder.use_var(y); let size = 8192; - builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8); + builder.emit_small_memset(target.frontend_config(), dest, 1, size, 8, MemFlags::new()); builder.ins().return_(&[dest]); builder.seal_all_blocks(); diff --git a/cranelift/interpreter/Cargo.toml b/cranelift/interpreter/Cargo.toml index 0439c964d8..c539f70fe1 100644 --- a/cranelift/interpreter/Cargo.toml +++ b/cranelift/interpreter/Cargo.toml @@ -13,13 +13,13 @@ edition = "2018" [dependencies] cranelift-codegen = { path = "../codegen", version = "0.73.0", features = ["all-arch"] } cranelift-entity = { path = "../entity", version = "0.73.0" } -cranelift-reader = { path = "../reader", version = "0.73.0" } log = { version = "0.4.8", default-features = false } smallvec = "1.6.1" thiserror = "1.0.15" [dev-dependencies] cranelift-frontend = { path = "../frontend", version = "0.73.0" } +cranelift-reader = { path = "../reader", version = "0.73.0" } [badges] maintenance = { status = "experimental" } diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index 26768ffc74..4389455d8d 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -285,6 +285,13 @@ impl JITModule { } } + /// Returns the given function's entry in the Global Offset Table. + /// + /// Panics if there's no entry in the table for the given function. + pub fn read_got_entry(&self, func_id: FuncId) -> *const u8 { + unsafe { *self.function_got_entries[func_id].unwrap().as_ptr() } + } + fn get_got_address(&self, name: &ir::ExternalName) -> *const u8 { match *name { ir::ExternalName::User { .. } => { diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs index d44497ae9e..f00165dbab 100644 --- a/cranelift/jit/src/compiled_blob.rs +++ b/cranelift/jit/src/compiled_blob.rs @@ -72,6 +72,15 @@ impl CompiledBlob { write_unaligned(at as *mut i32, pcrel) }; } + Reloc::S390xPCRel32Dbl => { + let base = get_address(name); + let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; + let pcrel = i32::try_from(((what as isize) - (at as isize)) >> 1).unwrap(); + #[cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] + unsafe { + write_unaligned(at as *mut i32, pcrel) + }; + } _ => unimplemented!(), } } diff --git a/cranelift/module/Cargo.toml b/cranelift/module/Cargo.toml index 6650714cf6..a3fe4a0901 100644 --- a/cranelift/module/Cargo.toml +++ b/cranelift/module/Cargo.toml @@ -15,7 +15,6 @@ cranelift-codegen = { path = "../codegen", version = "0.73.0", default-features cranelift-entity = { path = "../entity", version = "0.73.0" } hashbrown = { version = "0.9.1", optional = true } log = { version = "0.4.6", default-features = false } -thiserror = "1.0.4" anyhow = "1.0" [features] diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index 6047dda103..191d468ed7 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -12,7 +12,6 @@ use cranelift_codegen::entity::{entity_impl, PrimaryMap}; use cranelift_codegen::{ir, isa, CodegenError, Context}; use std::borrow::ToOwned; use std::string::String; -use thiserror::Error; /// A function identifier for use in the `Module` interface. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -168,30 +167,85 @@ impl FunctionDeclaration { } /// Error messages for all `Module` methods -#[derive(Error, Debug)] +#[derive(Debug)] pub enum ModuleError { /// Indicates an identifier was used before it was declared - #[error("Undeclared identifier: {0}")] Undeclared(String), + /// Indicates an identifier was used as data/function first, but then used as the other - #[error("Incompatible declaration of identifier: {0}")] IncompatibleDeclaration(String), + /// Indicates a function identifier was declared with a /// different signature than declared previously - #[error("Function {0} signature {2:?} is incompatible with previous declaration {1:?}")] IncompatibleSignature(String, ir::Signature, ir::Signature), + /// Indicates an identifier was defined more than once - #[error("Duplicate definition of identifier: {0}")] DuplicateDefinition(String), + /// Indicates an identifier was defined, but was declared as an import - #[error("Invalid to define identifier declared as an import: {0}")] InvalidImportDefinition(String), + /// Wraps a `cranelift-codegen` error - #[error("Compilation error: {0}")] - Compilation(#[from] CodegenError), + Compilation(CodegenError), + /// Wraps a generic error from a backend - #[error("Backend error: {0}")] - Backend(#[source] anyhow::Error), + Backend(anyhow::Error), +} + +// This is manually implementing Error and Display instead of using thiserror to reduce the amount +// of dependencies used by Cranelift. +impl std::error::Error for ModuleError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Undeclared { .. } + | Self::IncompatibleDeclaration { .. } + | Self::IncompatibleSignature { .. } + | Self::DuplicateDefinition { .. } + | Self::InvalidImportDefinition { .. } => None, + Self::Compilation(source) => Some(source), + Self::Backend(source) => Some(&**source), + } + } +} + +impl std::fmt::Display for ModuleError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Undeclared(name) => { + write!(f, "Undeclared identifier: {}", name) + } + Self::IncompatibleDeclaration(name) => { + write!(f, "Incompatible declaration of identifier: {}", name,) + } + Self::IncompatibleSignature(name, prev_sig, new_sig) => { + write!( + f, + "Function {} signature {:?} is incompatible with previous declaration {:?}", + name, new_sig, prev_sig, + ) + } + Self::DuplicateDefinition(name) => { + write!(f, "Duplicate definition of identifier: {}", name) + } + Self::InvalidImportDefinition(name) => { + write!( + f, + "Invalid to define identifier declared as an import: {}", + name, + ) + } + Self::Compilation(err) => { + write!(f, "Compilation error: {}", err) + } + Self::Backend(err) => write!(f, "Backend error: {}", err), + } + } +} + +impl std::convert::From for ModuleError { + fn from(source: CodegenError) -> Self { + Self::Compilation { 0: source } + } } /// A convenient alias for a `Result` that uses `ModuleError` as the error type. diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index abfdcf10c9..6cf54fc30a 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -639,8 +639,8 @@ fn translate_linkage(linkage: Linkage) -> (SymbolScope, bool) { (scope, weak) } -/// This is the output of `Module`'s -/// [`finish`](../cranelift_module/struct.Module.html#method.finish) function. +/// This is the output of `ObjectModule`'s +/// [`finish`](../struct.ObjectModule.html#method.finish) function. /// It contains the generated `Object` and other information produced during /// compilation. pub struct ObjectProduct { diff --git a/cranelift/reader/Cargo.toml b/cranelift/reader/Cargo.toml index d1d8a29ae3..2d7e93fee0 100644 --- a/cranelift/reader/Cargo.toml +++ b/cranelift/reader/Cargo.toml @@ -13,7 +13,6 @@ edition = "2018" cranelift-codegen = { path = "../codegen", version = "0.73.0" } smallvec = "1.6.1" target-lexicon = "0.12" -thiserror = "1.0.15" [badges] maintenance = { status = "experimental" } diff --git a/cranelift/src/disasm.rs b/cranelift/src/disasm.rs index a98e867380..35a581d344 100644 --- a/cranelift/src/disasm.rs +++ b/cranelift/src/disasm.rs @@ -153,6 +153,11 @@ cfg_if! { cs.set_skipdata(true).map_err(map_caperr)?; cs } + Architecture::S390x {..} => Capstone::new() + .sysz() + .mode(arch::sysz::ArchMode::Default) + .build() + .map_err(map_caperr)?, _ => anyhow::bail!("Unknown ISA"), }; diff --git a/cranelift/wasm/src/environ/dummy.rs b/cranelift/wasm/src/environ/dummy.rs index ea9157e552..6ac82b73f0 100644 --- a/cranelift/wasm/src/environ/dummy.rs +++ b/cranelift/wasm/src/environ/dummy.rs @@ -747,7 +747,7 @@ impl<'data> ModuleEnvironment<'data> for DummyEnvironment { &mut self, _table_index: TableIndex, _base: Option, - _offset: usize, + _offset: u32, _elements: Box<[FuncIndex]>, ) -> WasmResult<()> { // We do nothing @@ -792,7 +792,7 @@ impl<'data> ModuleEnvironment<'data> for DummyEnvironment { &mut self, _memory_index: MemoryIndex, _base: Option, - _offset: usize, + _offset: u32, _data: &'data [u8], ) -> WasmResult<()> { // We do nothing diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index 3a2389c44d..31c8d86a4e 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -937,7 +937,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { &mut self, table_index: TableIndex, base: Option, - offset: usize, + offset: u32, elements: Box<[FuncIndex]>, ) -> WasmResult<()>; @@ -948,6 +948,13 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { elements: Box<[FuncIndex]>, ) -> WasmResult<()>; + /// Indicates that a declarative element segment was seen in the wasm + /// module. + fn declare_elements(&mut self, elements: Box<[FuncIndex]>) -> WasmResult<()> { + drop(elements); + Ok(()) + } + /// Provides the number of passive data segments up front. /// /// By default this does nothing, but implementations may use this to @@ -984,7 +991,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { &mut self, memory_index: MemoryIndex, base: Option, - offset: usize, + offset: u32, data: &'data [u8], ) -> WasmResult<()>; diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index c617c9943c..bd4dcd1136 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -377,7 +377,7 @@ pub fn parse_element_section<'data>( } => { let mut init_expr_reader = init_expr.get_binary_reader(); let (base, offset) = match init_expr_reader.read_operator()? { - Operator::I32Const { value } => (None, value as u32 as usize), + Operator::I32Const { value } => (None, value as u32), Operator::GlobalGet { global_index } => { (Some(GlobalIndex::from_u32(global_index)), 0) } @@ -388,12 +388,6 @@ pub fn parse_element_section<'data>( )); } }; - // Check for offset + len overflow - if offset.checked_add(segments.len()).is_none() { - return Err(wasm_unsupported!( - "element segment offset and length overflows" - )); - } environ.declare_table_elements( TableIndex::from_u32(table_index), base, @@ -406,7 +400,7 @@ pub fn parse_element_section<'data>( environ.declare_passive_element(index, segments)?; } ElementKind::Declared => { - // Nothing to do here. + environ.declare_elements(segments)?; } } } @@ -429,7 +423,7 @@ pub fn parse_data_section<'data>( } => { let mut init_expr_reader = init_expr.get_binary_reader(); let (base, offset) = match init_expr_reader.read_operator()? { - Operator::I32Const { value } => (None, value as u32 as usize), + Operator::I32Const { value } => (None, value as u32), Operator::GlobalGet { global_index } => { (Some(GlobalIndex::from_u32(global_index)), 0) } @@ -440,12 +434,6 @@ pub fn parse_data_section<'data>( )) } }; - // Check for offset + len overflow - if offset.checked_add(data.len()).is_none() { - return Err(wasm_unsupported!( - "data segment offset and length overflows" - )); - } environ.declare_data_initialization( MemoryIndex::from_u32(memory_index), base, diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h index d6b0349e64..f163f923bb 100644 --- a/crates/c-api/include/wasmtime.h +++ b/crates/c-api/include/wasmtime.h @@ -283,14 +283,6 @@ WASMTIME_CONFIG_PROP(void, static_memory_guard_size, uint64_t) */ WASMTIME_CONFIG_PROP(void, dynamic_memory_guard_size, uint64_t) -/** - * \brief Configures the maximum number of instances that can be created. - * - * For more information see the Rust documentation at - * https://bytecodealliance.github.io/wasmtime/api/wasmtime/struct.Config.html#method.max_instances. - */ -WASMTIME_CONFIG_PROP(void, max_instances, size_t) - /** * \brief Enables Wasmtime's cache and loads configuration from the specified * path. @@ -1000,9 +992,13 @@ WASM_API_EXTERN own wasmtime_error_t* wasmtime_module_serialize( /** * \brief Build a module from serialized data. - * * + * * This function does not take ownership of any of its arguments, but the * returned error and module are owned by the caller. + * + * This function is not safe to receive arbitrary user input. See the Rust + * documentation for more information on what inputs are safe to pass in here + * (e.g. only that of #wasmtime_module_serialize) */ WASM_API_EXTERN own wasmtime_error_t *wasmtime_module_deserialize( wasm_engine_t *engine, diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index f84bc320cd..3e6e313ba9 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -176,8 +176,3 @@ pub extern "C" fn wasmtime_config_static_memory_guard_size_set(c: &mut wasm_conf pub extern "C" fn wasmtime_config_dynamic_memory_guard_size_set(c: &mut wasm_config_t, size: u64) { c.config.dynamic_memory_guard_size(size); } - -#[no_mangle] -pub extern "C" fn wasmtime_config_max_instances_set(c: &mut wasm_config_t, limit: usize) { - c.config.max_instances(limit); -} diff --git a/crates/c-api/src/memory.rs b/crates/c-api/src/memory.rs index 55a001deed..54d3936849 100644 --- a/crates/c-api/src/memory.rs +++ b/crates/c-api/src/memory.rs @@ -31,13 +31,13 @@ impl wasm_memory_t { pub extern "C" fn wasm_memory_new( store: &wasm_store_t, mt: &wasm_memorytype_t, -) -> Box { - let memory = Memory::new(&store.store, mt.ty().ty.clone()); - Box::new(wasm_memory_t { +) -> Option> { + let memory = Memory::new(&store.store, mt.ty().ty.clone()).ok()?; + Some(Box::new(wasm_memory_t { ext: wasm_extern_t { which: memory.into(), }, - }) + })) } #[no_mangle] diff --git a/crates/c-api/src/module.rs b/crates/c-api/src/module.rs index 1757191026..4b60c67e73 100644 --- a/crates/c-api/src/module.rs +++ b/crates/c-api/src/module.rs @@ -185,10 +185,13 @@ pub extern "C" fn wasmtime_module_deserialize( binary: &wasm_byte_vec_t, ret: &mut *mut wasm_module_t, ) -> Option> { - handle_result(Module::new(&engine.engine, binary.as_slice()), |module| { - let module = Box::new(wasm_module_t::new(module)); - *ret = Box::into_raw(module); - }) + handle_result( + unsafe { Module::deserialize(&engine.engine, binary.as_slice()) }, + |module| { + let module = Box::new(wasm_module_t::new(module)); + *ret = Box::into_raw(module); + }, + ) } #[no_mangle] diff --git a/crates/debug/src/lib.rs b/crates/debug/src/lib.rs index 597ab2e25c..b6d6830d51 100644 --- a/crates/debug/src/lib.rs +++ b/crates/debug/src/lib.rs @@ -3,6 +3,7 @@ #![allow(clippy::cast_ptr_alignment)] use anyhow::{bail, ensure, Error}; +use object::endian::{BigEndian, Endian, Endianness, LittleEndian}; use object::{RelocationEncoding, RelocationKind}; use std::collections::HashMap; @@ -18,13 +19,20 @@ pub fn create_gdbjit_image( defined_funcs_offset: usize, funcs: &[*const u8], ) -> Result, Error> { - ensure_supported_elf_format(&mut bytes)?; + let e = ensure_supported_elf_format(&mut bytes)?; // patch relocs relocate_dwarf_sections(&mut bytes, defined_funcs_offset, funcs)?; // elf is still missing details... - convert_object_elf_to_loadable_file(&mut bytes, code_region); + match e { + Endianness::Little => { + convert_object_elf_to_loadable_file::(&mut bytes, code_region) + } + Endianness::Big => { + convert_object_elf_to_loadable_file::(&mut bytes, code_region) + } + } // let mut file = ::std::fs::File::create(::std::path::Path::new("test.o")).expect("file"); // ::std::io::Write::write_all(&mut file, &bytes).expect("write"); @@ -83,20 +91,36 @@ fn relocate_dwarf_sections( Ok(()) } -fn ensure_supported_elf_format(bytes: &mut Vec) -> Result<(), Error> { +fn ensure_supported_elf_format(bytes: &mut Vec) -> Result { use object::elf::*; - use object::endian::LittleEndian; + use object::read::elf::*; + use object::Bytes; use std::mem::size_of; - let e = LittleEndian; - let header: &FileHeader64 = - unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) }; - ensure!( - header.e_ident.class == ELFCLASS64 && header.e_ident.data == ELFDATA2LSB, - "bits and endianess in .ELF", - ); + let kind = match object::FileKind::parse(bytes) { + Ok(file) => file, + Err(err) => { + bail!("Failed to parse file: {}", err); + } + }; + let header = match kind { + object::FileKind::Elf64 => { + match object::elf::FileHeader64::::parse(Bytes(bytes)) { + Ok(header) => header, + Err(err) => { + bail!("Unsupported ELF file: {}", err); + } + } + } + _ => { + bail!("only 64-bit ELF files currently supported") + } + }; + let e = header.endian().unwrap(); + match header.e_machine.get(e) { EM_X86_64 => (), + EM_S390 => (), machine => { bail!("Unsupported ELF target machine: {:x}", machine); } @@ -106,23 +130,25 @@ fn ensure_supported_elf_format(bytes: &mut Vec) -> Result<(), Error> { "program header table is empty" ); let e_shentsize = header.e_shentsize.get(e); - ensure!( - e_shentsize as usize == size_of::>(), - "size of sh" - ); - Ok(()) + let req_shentsize = match e { + Endianness::Little => size_of::>(), + Endianness::Big => size_of::>(), + }; + ensure!(e_shentsize as usize == req_shentsize, "size of sh"); + Ok(e) } -fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const u8, usize)) { +fn convert_object_elf_to_loadable_file( + bytes: &mut Vec, + code_region: (*const u8, usize), +) { use object::elf::*; - use object::endian::LittleEndian; use std::ffi::CStr; use std::mem::size_of; use std::os::raw::c_char; - let e = LittleEndian; - let header: &FileHeader64 = - unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) }; + let e = E::default(); + let header: &FileHeader64 = unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) }; let e_shentsize = header.e_shentsize.get(e); let e_shoff = header.e_shoff.get(e); @@ -130,7 +156,7 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const let mut shstrtab_off = 0; for i in 0..e_shnum { let off = e_shoff as isize + i as isize * e_shentsize as isize; - let section: &SectionHeader64 = + let section: &SectionHeader64 = unsafe { &*(bytes.as_ptr().offset(off) as *const SectionHeader64<_>) }; if section.sh_type.get(e) != SHT_STRTAB { continue; @@ -140,7 +166,7 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const let mut segment: Option<_> = None; for i in 0..e_shnum { let off = e_shoff as isize + i as isize * e_shentsize as isize; - let section: &mut SectionHeader64 = + let section: &mut SectionHeader64 = unsafe { &mut *(bytes.as_mut_ptr().offset(off) as *mut SectionHeader64<_>) }; if section.sh_type.get(e) != SHT_PROGBITS { continue; @@ -171,12 +197,12 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const // LLDB wants segment with virtual address set, placing them at the end of ELF. let ph_off = bytes.len(); - let e_phentsize = size_of::>(); + let e_phentsize = size_of::>(); let e_phnum = 1; bytes.resize(ph_off + e_phentsize * e_phnum, 0); if let Some((sh_offset, sh_size)) = segment { let (v_offset, size) = code_region; - let program: &mut ProgramHeader64 = + let program: &mut ProgramHeader64 = unsafe { &mut *(bytes.as_ptr().add(ph_off) as *mut ProgramHeader64<_>) }; program.p_type.set(e, PT_LOAD); program.p_offset.set(e, sh_offset); @@ -189,7 +215,7 @@ fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const } // It is somewhat loadable ELF file at this moment. - let header: &mut FileHeader64 = + let header: &mut FileHeader64 = unsafe { &mut *(bytes.as_mut_ptr() as *mut FileHeader64<_>) }; header.e_type.set(e, ET_DYN); header.e_phoff.set(e, ph_off as u64); diff --git a/crates/debug/src/transform/expression.rs b/crates/debug/src/transform/expression.rs index 57f3cc7a21..556a460905 100644 --- a/crates/debug/src/transform/expression.rs +++ b/crates/debug/src/transform/expression.rs @@ -512,24 +512,28 @@ where } }; } + // Find all landing pads by scanning bytes, do not care about // false location at this moment. // Looks hacky but it is fast; does not need to be really exact. - for i in 0..buf.len() - 2 { - let op = buf[i]; - if op == gimli::constants::DW_OP_bra.0 || op == gimli::constants::DW_OP_skip.0 { - // TODO fix for big-endian - let offset = i16::from_le_bytes([buf[i + 1], buf[i + 2]]); - let origin = i + 3; - // Discarding out-of-bounds jumps (also some of falsely detected ops) - if (offset >= 0 && offset as usize + origin <= buf.len()) - || (offset < 0 && -offset as usize <= origin) - { - let target = buf.len() as isize - origin as isize - offset as isize; - jump_targets.insert(target as u64, JumpTargetMarker::new()); + if buf.len() > 2 { + for i in 0..buf.len() - 2 { + let op = buf[i]; + if op == gimli::constants::DW_OP_bra.0 || op == gimli::constants::DW_OP_skip.0 { + // TODO fix for big-endian + let offset = i16::from_le_bytes([buf[i + 1], buf[i + 2]]); + let origin = i + 3; + // Discarding out-of-bounds jumps (also some of falsely detected ops) + if (offset >= 0 && offset as usize + origin <= buf.len()) + || (offset < 0 && -offset as usize <= origin) + { + let target = buf.len() as isize - origin as isize - offset as isize; + jump_targets.insert(target as u64, JumpTargetMarker::new()); + } } } } + while !pc.is_empty() { let unread_bytes = pc.len().into_u64(); if let Some(marker) = jump_targets.get(&unread_bytes) { diff --git a/crates/debug/src/transform/unit.rs b/crates/debug/src/transform/unit.rs index 15f165418a..655d7738ee 100644 --- a/crates/debug/src/transform/unit.rs +++ b/crates/debug/src/transform/unit.rs @@ -10,6 +10,7 @@ use anyhow::{Context, Error}; use gimli::write; use gimli::{AttributeValue, DebuggingInformationEntry, Unit}; use std::collections::HashSet; +use wasmtime_environ::ir::Endianness; use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::{CompiledFunctions, ModuleMemoryOffset}; @@ -463,6 +464,19 @@ where isa, )?; + // Data in WebAssembly memory always uses little-endian byte order. + // If the native architecture is big-endian, we need to mark all + // base types used to refer to WebAssembly memory as little-endian + // using the DW_AT_endianity attribute, so that the debugger will + // be able to correctly access them. + if entry.tag() == gimli::DW_TAG_base_type && isa.endianness() == Endianness::Big { + let current_scope = comp_unit.get_mut(die_id); + current_scope.set( + gimli::DW_AT_endianity, + write::AttributeValue::Endianity(gimli::DW_END_little), + ); + } + if entry.tag() == gimli::DW_TAG_subprogram && !current_scope_ranges.is_empty() { append_vmctx_info( comp_unit, diff --git a/crates/debug/src/write_debuginfo.rs b/crates/debug/src/write_debuginfo.rs index 56c7231db0..491267b495 100644 --- a/crates/debug/src/write_debuginfo.rs +++ b/crates/debug/src/write_debuginfo.rs @@ -2,6 +2,7 @@ pub use crate::transform::transform_dwarf; use gimli::write::{Address, Dwarf, EndianVec, FrameTable, Result, Sections, Writer}; use gimli::{RunTimeEndian, SectionId}; use wasmtime_environ::entity::EntityRef; +use wasmtime_environ::ir::Endianness; use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; use wasmtime_environ::{CompiledFunctions, DebugInfoData, ModuleMemoryOffset}; @@ -26,10 +27,19 @@ pub struct DwarfSection { } fn emit_dwarf_sections( + isa: &dyn TargetIsa, mut dwarf: Dwarf, frames: Option, ) -> anyhow::Result> { - let mut sections = Sections::new(WriterRelocate::default()); + let endian = match isa.endianness() { + Endianness::Little => RunTimeEndian::Little, + Endianness::Big => RunTimeEndian::Big, + }; + let writer = WriterRelocate { + relocs: Vec::new(), + writer: EndianVec::new(endian), + }; + let mut sections = Sections::new(writer); dwarf.write(&mut sections)?; if let Some(frames) = frames { frames.write_debug_frame(&mut sections.debug_frame)?; @@ -54,15 +64,6 @@ pub struct WriterRelocate { writer: EndianVec, } -impl Default for WriterRelocate { - fn default() -> Self { - WriterRelocate { - relocs: Vec::new(), - writer: EndianVec::new(RunTimeEndian::Little), - } - } -} - impl Writer for WriterRelocate { type Endian = RunTimeEndian; @@ -156,6 +157,6 @@ pub fn emit_dwarf<'a>( ) -> anyhow::Result> { let dwarf = transform_dwarf(isa, debuginfo_data, funcs, memory_offset)?; let frame_table = create_frame_table(isa, funcs); - let sections = emit_dwarf_sections(dwarf, frame_table)?; + let sections = emit_dwarf_sections(isa, dwarf, frame_table)?; Ok(sections) } diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 751be91cb6..383d030b43 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -12,8 +12,6 @@ readme = "README.md" edition = "2018" [dependencies] -anyhow = "1.0" -region = "2.2.0" cranelift-codegen = { path = "../../cranelift/codegen", version = "0.73.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.73.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.73.0", features = ["enable-serde"] } diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 36eec310ec..12b321d779 100644 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -3,8 +3,8 @@ pub mod ir { pub use cranelift_codegen::binemit::{Reloc, StackMap}; pub use cranelift_codegen::ir::{ - types, AbiParam, ArgumentPurpose, JumpTableOffsets, LabelValueLoc, LibCall, Signature, - SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc, + types, AbiParam, ArgumentPurpose, Endianness, JumpTableOffsets, LabelValueLoc, LibCall, + Signature, SourceLoc, StackSlots, TrapCode, Type, ValueLabel, ValueLoc, }; pub use cranelift_codegen::{ValueLabelsRanges, ValueLocRange}; } diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index dd98484118..38a4293825 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -7,6 +7,7 @@ use cranelift_wasm::*; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; +use std::convert::TryFrom; use std::sync::Arc; /// Implemenation styles for WebAssembly linear memory. @@ -86,7 +87,7 @@ pub struct MemoryInitializer { /// Optionally, a global variable giving a base index. pub base: Option, /// The offset to add to the base. - pub offset: usize, + pub offset: u32, /// The data to write into the linear memory. pub data: Box<[u8]>, } @@ -168,7 +169,15 @@ impl MemoryInitialization { // Perform a bounds check on the segment // As this segment is referencing a defined memory without a global base, the last byte // written to by the segment cannot exceed the memory's initial minimum size - if (initializer.offset + initializer.data.len()) + let offset = usize::try_from(initializer.offset).unwrap(); + let end = match offset.checked_add(initializer.data.len()) { + Some(end) => end, + None => { + out_of_bounds = true; + continue; + } + }; + if end > ((module.memory_plans[initializer.memory_index].memory.minimum as usize) * WASM_PAGE_SIZE) @@ -178,8 +187,8 @@ impl MemoryInitialization { } let pages = &mut map[index]; - let mut page_index = initializer.offset / WASM_PAGE_SIZE; - let mut page_offset = initializer.offset % WASM_PAGE_SIZE; + let mut page_index = offset / WASM_PAGE_SIZE; + let mut page_offset = offset % WASM_PAGE_SIZE; let mut data_offset = 0; let mut data_remaining = initializer.data.len(); @@ -268,7 +277,7 @@ pub struct TableInitializer { /// Optionally, a global variable giving a base index. pub base: Option, /// The offset to add to the base. - pub offset: usize, + pub offset: u32, /// The values to write into the table elements. pub elements: Box<[FuncIndex]>, } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 8106c1a3fb..324c4b9296 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -705,7 +705,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data &mut self, table_index: TableIndex, base: Option, - offset: usize, + offset: u32, elements: Box<[FuncIndex]>, ) -> WasmResult<()> { for element in elements.iter() { @@ -746,6 +746,13 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data Ok(()) } + fn declare_elements(&mut self, segments: Box<[FuncIndex]>) -> WasmResult<()> { + for element in segments.iter() { + self.flag_func_possibly_exported(*element); + } + Ok(()) + } + fn reserve_function_bodies(&mut self, _count: u32, offset: u64) { self.result.debuginfo.wasm_file.code_section_offset = offset; } @@ -794,7 +801,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data &mut self, memory_index: MemoryIndex, base: Option, - offset: usize, + offset: u32, data: &'data [u8], ) -> WasmResult<()> { match &mut self.result.module.memory_initialization { diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index e5d6604610..6fc68f5e28 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -6,7 +6,7 @@ // struct VMContext { // interrupts: *const VMInterrupts, // externref_activations_table: *mut VMExternRefActivationsTable, -// stack_map_registry: *mut StackMapRegistry, +// module_info_lookup: *const dyn ModuleInfoLookup, // signature_ids: [VMSharedSignatureIndex; module.num_signature_ids], // imported_functions: [VMFunctionImport; module.num_imported_functions], // imported_tables: [VMTableImport; module.num_imported_tables], @@ -77,7 +77,7 @@ pub struct VMOffsets { // precalculated offsets of various member fields interrupts: u32, externref_activations_table: u32, - stack_map_registry: u32, + module_info_lookup: u32, signature_ids: u32, imported_functions: u32, imported_tables: u32, @@ -149,7 +149,7 @@ impl From for VMOffsets { num_defined_globals: fields.num_defined_globals, interrupts: 0, externref_activations_table: 0, - stack_map_registry: 0, + module_info_lookup: 0, signature_ids: 0, imported_functions: 0, imported_tables: 0, @@ -168,13 +168,13 @@ impl From for VMOffsets { .interrupts .checked_add(u32::from(fields.pointer_size)) .unwrap(); - ret.stack_map_registry = ret + ret.module_info_lookup = ret .externref_activations_table .checked_add(u32::from(fields.pointer_size)) .unwrap(); ret.signature_ids = ret - .stack_map_registry - .checked_add(u32::from(fields.pointer_size)) + .module_info_lookup + .checked_add(u32::from(fields.pointer_size * 2)) .unwrap(); ret.imported_functions = ret .signature_ids @@ -507,10 +507,10 @@ impl VMOffsets { self.externref_activations_table } - /// The offset of the `*mut StackMapRegistry` member. + /// The offset of the `*const dyn ModuleInfoLookup` member. #[inline] - pub fn vmctx_stack_map_registry(&self) -> u32 { - self.stack_map_registry + pub fn vmctx_module_info_lookup(&self) -> u32 { + self.module_info_lookup } /// The offset of the `signature_ids` array. diff --git a/crates/fiber/src/arch/s390x.S b/crates/fiber/src/arch/s390x.S new file mode 100644 index 0000000000..8d9548bbb0 --- /dev/null +++ b/crates/fiber/src/arch/s390x.S @@ -0,0 +1,112 @@ +// A WORD OF CAUTION +// +// This entire file basically needs to be kept in sync with itself. It's not +// really possible to modify just one bit of this file without understanding +// all the other bits. Documentation tries to reference various bits here and +// there but try to make sure to read over everything before tweaking things! +// +// Also at this time this file is heavily based off the x86_64 file, so you'll +// probably want to read that one as well. + +#include "header.h" + +// fn(top_of_stack(%x0): *mut u8) +HIDDEN(wasmtime_fiber_switch) +GLOBL(wasmtime_fiber_switch) +.p2align 2 +TYPE(wasmtime_fiber_switch) +FUNCTION(wasmtime_fiber_switch): + // Save all callee-saved registers on the stack since we're assuming + // they're clobbered as a result of the stack switch. + stmg %r6, %r15, 48(%r15) + aghi %r15, -64 + std %f8, 0(%r15) + std %f9, 8(%r15) + std %f10, 16(%r15) + std %f11, 24(%r15) + std %f12, 32(%r15) + std %f13, 40(%r15) + std %f14, 48(%r15) + std %f15, 56(%r15) + + // Load our previously saved stack pointer to resume to, and save off our + // current stack pointer on where to come back to eventually. + lg %r1, -16(%r2) + stg %r15, -16(%r2) + + // Switch to the new stack and restore all our callee-saved registers after + // the switch and return to our new stack. + ld %f8, 0(%r1) + ld %f9, 8(%r1) + ld %f10, 16(%r1) + ld %f11, 24(%r1) + ld %f12, 32(%r1) + ld %f13, 40(%r1) + ld %f14, 48(%r1) + ld %f15, 56(%r1) + lmg %r6, %r15, 112(%r1) + br %r14 +SIZE(wasmtime_fiber_switch) + +// fn( +// top_of_stack(%x0): *mut u8, +// entry_point(%x1): extern fn(*mut u8, *mut u8), +// entry_arg0(%x2): *mut u8, +// ) +HIDDEN(wasmtime_fiber_init) +GLOBL(wasmtime_fiber_init) +.p2align 2 +TYPE(wasmtime_fiber_init) +FUNCTION(wasmtime_fiber_init): + larl %r1, FUNCTION(wasmtime_fiber_start) + stg %r1, -48(%r2) // wasmtime_fiber_start - restored into %r14 + stg %r2, -112(%r2) // top_of_stack - restored into %r6 + stg %r3, -104(%r2) // entry_point - restored into %r7 + stg %r4, -96(%r2) // entry_arg0 - restored into %r8 + aghi %r2, -160 // 160 bytes register save area + stg %r2, 120(%r2) // bottom of register save area - restored into %r15 + + // `wasmtime_fiber_switch` has a 64 byte stack. + aghi %r2, -64 + stg %r2, 208(%r2) + br %r14 +SIZE(wasmtime_fiber_init) + +.p2align 2 +TYPE(wasmtime_fiber_start) +FUNCTION(wasmtime_fiber_start): +.cfi_startproc simple + + // See the x86_64 file for more commentary on what these CFI directives are + // doing. Like over there note that the relative offsets to registers here + // match the frame layout in `wasmtime_fiber_switch`. + .cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \ + 7, /* the byte length of this expression */ \ + 0x7f, 0x90, 0x1, /* DW_OP_breg15 0x90 */ \ + 0x06, /* DW_OP_deref */ \ + 0x23, 0xe0, 0x1 /* DW_OP_plus_uconst 0xe0 */ + + .cfi_rel_offset 6, -112 + .cfi_rel_offset 7, -104 + .cfi_rel_offset 8, -96 + .cfi_rel_offset 9, -88 + .cfi_rel_offset 10, -80 + .cfi_rel_offset 11, -72 + .cfi_rel_offset 12, -64 + .cfi_rel_offset 13, -56 + .cfi_rel_offset 14, -48 + .cfi_rel_offset 15, -40 + + // Load our two arguments prepared by `wasmtime_fiber_init`. + lgr %r2, %r8 // entry_arg0 + lgr %r3, %r6 // top_of_stack + + // ... and then we call the function! Note that this is a function call so + // our frame stays on the stack to backtrace through. + basr %r14, %r7 // entry_point + // .. technically we shouldn't get here, so just trap. + .word 0x0000 + .cfi_endproc +SIZE(wasmtime_fiber_start) + +FOOTER diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs index 6e4e991c4c..7ef3382411 100644 --- a/crates/fuzzing/src/lib.rs +++ b/crates/fuzzing/src/lib.rs @@ -39,13 +39,6 @@ pub fn fuzz_default_config(strategy: wasmtime::Strategy) -> anyhow::Result Store { + Store::new_with_limits( + &engine, + StoreLimitsBuilder::new() + // The limits here are chosen based on the default "maximum type size" + // configured in wasm-smith, which is 1000. This means that instances + // are allowed to, for example, export up to 1000 memories. We bump that + // a little bit here to give us some slop. + .instances(1100) + .tables(1100) + .memories(1100) + .build(), + ) +} + /// Methods of timing out execution of a WebAssembly module #[derive(Debug)] pub enum Timeout { @@ -95,7 +110,7 @@ pub fn instantiate_with_config( _ => false, }); let engine = Engine::new(&config).unwrap(); - let store = Store::new(&engine); + let store = create_store(&engine); let mut timeout_state = SignalOnDrop::default(); match timeout { @@ -203,7 +218,7 @@ pub fn differential_execution( config.wasm_module_linking(false); let engine = Engine::new(&config).unwrap(); - let store = Store::new(&engine); + let store = create_store(&engine); let module = Module::new(&engine, &wasm).unwrap(); @@ -348,7 +363,7 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) { ApiCall::StoreNew => { log::trace!("creating store"); assert!(store.is_none()); - store = Some(Store::new(engine.as_ref().unwrap())); + store = Some(create_store(engine.as_ref().unwrap())); } ApiCall::ModuleNew { id, wasm } => { @@ -439,7 +454,7 @@ pub fn spectest(fuzz_config: crate::generators::Config, test: crate::generators: config.wasm_reference_types(false); config.wasm_bulk_memory(false); config.wasm_module_linking(false); - let store = Store::new(&Engine::new(&config).unwrap()); + let store = create_store(&Engine::new(&config).unwrap()); if fuzz_config.consume_fuel { store.add_fuel(u64::max_value()).unwrap(); } @@ -463,7 +478,7 @@ pub fn table_ops( let mut config = fuzz_config.to_wasmtime(); config.wasm_reference_types(true); let engine = Engine::new(&config).unwrap(); - let store = Store::new(&engine); + let store = create_store(&engine); if fuzz_config.consume_fuel { store.add_fuel(u64::max_value()).unwrap(); } @@ -578,7 +593,7 @@ pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Con let mut wasmtime_config = config.to_wasmtime(); wasmtime_config.cranelift_nan_canonicalization(true); let wasmtime_engine = Engine::new(&wasmtime_config).unwrap(); - let wasmtime_store = Store::new(&wasmtime_engine); + let wasmtime_store = create_store(&wasmtime_engine); if config.consume_fuel { wasmtime_store.add_fuel(u64::max_value()).unwrap(); } diff --git a/crates/fuzzing/src/oracles/dummy.rs b/crates/fuzzing/src/oracles/dummy.rs index f03bbfa067..99b5be736b 100644 --- a/crates/fuzzing/src/oracles/dummy.rs +++ b/crates/fuzzing/src/oracles/dummy.rs @@ -87,7 +87,7 @@ pub fn dummy_table(store: &Store, ty: TableType) -> Table { /// Construct a dummy memory for the given memory type. pub fn dummy_memory(store: &Store, ty: MemoryType) -> Memory { - Memory::new(store, ty) + Memory::new(store, ty).unwrap() } /// Construct a dummy instance for the given instance type. diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 2d155e3d7b..3ff67d9c7a 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -176,11 +176,13 @@ struct FinishedFunctions(PrimaryMap); unsafe impl Send for FinishedFunctions {} unsafe impl Sync for FinishedFunctions {} +/// Information about a function, such as trap information, address map, +/// and stack maps. #[derive(Serialize, Deserialize, Clone)] -struct FunctionInfo { - traps: Vec, - address_map: FunctionAddressMap, - stack_maps: Vec, +pub struct FunctionInfo { + pub traps: Vec, + pub address_map: FunctionAddressMap, + pub stack_maps: Vec, } /// This is intended to mirror the type tables in `wasmtime_environ`, except that @@ -362,11 +364,10 @@ impl CompiledModule { } /// Gets the function information for a given function index. - pub fn func_info(&self, index: DefinedFuncIndex) -> (&FunctionAddressMap, &[TrapInformation]) { + pub fn func_info(&self, index: DefinedFuncIndex) -> &FunctionInfo { self.artifacts .funcs .get(index) - .map(|f| (&f.address_map, f.traps.as_ref())) .expect("defined function should be present") } diff --git a/crates/jit/src/link.rs b/crates/jit/src/link.rs index 9c58a138a5..fffd9ecca8 100644 --- a/crates/jit/src/link.rs +++ b/crates/jit/src/link.rs @@ -111,6 +111,19 @@ fn apply_reloc( ); write_unaligned(reloc_address as *mut u32, reloc_delta_u64 as u32); }, + #[cfg(target_pointer_width = "64")] + (RelocationKind::Relative, RelocationEncoding::S390xDbl, 32) => unsafe { + let reloc_address = body.add(offset as usize) as usize; + let reloc_addend = r.addend() as isize; + let reloc_delta_u64 = (target_func_address as u64) + .wrapping_sub(reloc_address as u64) + .wrapping_add(reloc_addend as u64); + assert!( + (reloc_delta_u64 as isize) >> 1 <= i32::max_value() as isize, + "relocation too large to fit in i32" + ); + write_unaligned(reloc_address as *mut u32, (reloc_delta_u64 >> 1) as u32); + }, (RelocationKind::Elf(elf::R_AARCH64_CALL26), RelocationEncoding::Generic, 32) => unsafe { let reloc_address = body.add(offset as usize) as usize; let reloc_addend = r.addend() as isize; diff --git a/crates/lightbeam/Cargo.toml b/crates/lightbeam/Cargo.toml index 7e6e5f6509..6f30be3d49 100644 --- a/crates/lightbeam/Cargo.toml +++ b/crates/lightbeam/Cargo.toml @@ -17,7 +17,7 @@ cranelift-codegen = { path = "../../cranelift/codegen", version = "0.73.0" } derive_more = "0.99" dynasm = "1.0.0" dynasmrt = "1.0.0" -iter-enum = "0.2" +iter-enum = "1" itertools = "0.10.0" memoffset = "0.6.0" more-asserts = "0.2.1" diff --git a/crates/obj/src/builder.rs b/crates/obj/src/builder.rs index 805adbb806..f606d72078 100644 --- a/crates/obj/src/builder.rs +++ b/crates/obj/src/builder.rs @@ -80,6 +80,7 @@ fn to_object_relocations<'a>( RelocationEncoding::Generic, 32, ), + Reloc::S390xPCRel32Dbl => (RelocationKind::Relative, RelocationEncoding::S390xDbl, 32), other => unimplemented!("Unimplemented relocation {:?}", other), }; Some(ObjectRelocation { @@ -102,6 +103,7 @@ fn to_object_architecture( X86_64 => Architecture::X86_64, Arm(_) => Architecture::Arm, Aarch64(_) => Architecture::Aarch64, + S390x => Architecture::S390x, architecture => { anyhow::bail!("target architecture {:?} is unsupported", architecture,); } diff --git a/crates/profiling/src/jitdump_linux.rs b/crates/profiling/src/jitdump_linux.rs index 0f3a4bb00d..b43e8155af 100644 --- a/crates/profiling/src/jitdump_linux.rs +++ b/crates/profiling/src/jitdump_linux.rs @@ -241,6 +241,7 @@ impl State { Architecture::X86_32(_) => elf::EM_386 as u32, Architecture::Arm(_) => elf::EM_ARM as u32, Architecture::Aarch64(_) => elf::EM_AARCH64 as u32, + Architecture::S390x => elf::EM_S390 as u32, _ => unimplemented!("unrecognized architecture"), } } diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index cefce0d507..4d7b8f9c47 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -99,18 +99,16 @@ //! Examination of Deferred Reference Counting and Cycle Detection* by Quinane: //! -use std::alloc::Layout; use std::any::Any; use std::cell::{Cell, RefCell, UnsafeCell}; use std::cmp::Ordering; -use std::collections::BTreeMap; use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::mem; use std::ops::Deref; use std::ptr::{self, NonNull}; -use std::rc::Rc; -use wasmtime_environ::{ir::StackMap, StackMapInformation}; +use std::{alloc::Layout, sync::Arc}; +use wasmtime_environ::ir::StackMap; /// An external reference to some opaque data. /// @@ -596,10 +594,10 @@ impl VMExternRefActivationsTable { pub unsafe fn insert_with_gc( &self, externref: VMExternRef, - stack_maps_registry: &StackMapRegistry, + module_info_lookup: &dyn ModuleInfoLookup, ) { if let Err(externref) = self.try_insert(externref) { - self.gc_and_insert_slow(externref, stack_maps_registry); + self.gc_and_insert_slow(externref, module_info_lookup); } } @@ -607,9 +605,9 @@ impl VMExternRefActivationsTable { unsafe fn gc_and_insert_slow( &self, externref: VMExternRef, - stack_maps_registry: &StackMapRegistry, + module_info_lookup: &dyn ModuleInfoLookup, ) { - gc(stack_maps_registry, self); + gc(module_info_lookup, self); // Might as well insert right into the hash set, rather than the bump // chunk, since we are already on a slow path and we get de-duplication @@ -743,182 +741,28 @@ impl VMExternRefActivationsTable { } } -/// A registry of stack maps for currently active Wasm modules. -#[derive(Default)] -pub struct StackMapRegistry { - inner: RefCell, +/// Used by the runtime to lookup information about a module given a +/// program counter value. +pub trait ModuleInfoLookup: 'static { + /// Lookup the module information from a program counter value. + fn lookup(&self, pc: usize) -> Option>; } -#[derive(Default)] -struct StackMapRegistryInner { - /// A map from the highest pc in a module, to its stack maps. - /// - /// For details, see the comment above `GlobalFrameInfo::ranges`. - ranges: BTreeMap, +/// Used by the runtime to query module information. +pub trait ModuleInfo { + /// Lookup the stack map at a program counter value. + fn lookup_stack_map(&self, pc: usize) -> Option<&StackMap>; } -#[derive(Debug)] -struct ModuleStackMaps { - /// The range of PCs that this module covers. Different modules must always - /// have distinct ranges. - range: std::ops::Range, +pub(crate) struct EmptyModuleInfoLookup; - /// A map from a PC in this module (that is a GC safepoint) to its - /// associated stack map. If `None` then it means that the PC is the start - /// of a range which has no stack map. - pc_to_stack_map: Vec<(usize, Option>)>, -} - -impl StackMapRegistry { - /// Register the stack maps for a given module. - /// - /// The stack maps should be given as an iterator over a function's PC range - /// in memory (that is, where the JIT actually allocated and emitted the - /// function's code at), and the stack maps and code offsets within that - /// range for each of its GC safepoints. - pub fn register_stack_maps<'a>( - &self, - stack_maps: impl IntoIterator, &'a [StackMapInformation])>, - ) { - let mut min = usize::max_value(); - let mut max = 0; - let mut pc_to_stack_map = vec![]; - let mut last_is_none_marker = true; - - for (range, infos) in stack_maps { - let len = range.end - range.start; - - min = std::cmp::min(min, range.start); - max = std::cmp::max(max, range.end); - - // Add a marker between functions indicating that this function's pc - // starts with no stack map so when our binary search later on finds - // a pc between the start of the function and the function's first - // stack map it doesn't think the previous stack map is our stack - // map. - // - // We skip this if the previous entry pushed was also a `None` - // marker, in which case the starting pc already has no stack map. - // This is also skipped if the first `code_offset` is zero since - // what we'll push applies for the first pc anyway. - if !last_is_none_marker && (infos.is_empty() || infos[0].code_offset > 0) { - pc_to_stack_map.push((range.start, None)); - last_is_none_marker = true; - } - - for info in infos { - assert!((info.code_offset as usize) < len); - pc_to_stack_map.push(( - range.start + (info.code_offset as usize), - Some(Rc::new(info.stack_map.clone())), - )); - last_is_none_marker = false; - } - } - - if pc_to_stack_map.is_empty() { - // Nothing to register. - return; - } - - let module_stack_maps = ModuleStackMaps { - range: min..max, - pc_to_stack_map, - }; - - let mut inner = self.inner.borrow_mut(); - - // Assert that this chunk of ranges doesn't collide with any other known - // chunks. - if let Some((_, prev)) = inner.ranges.range(max..).next() { - assert!(prev.range.start > max); - } - if let Some((prev_end, _)) = inner.ranges.range(..=min).next_back() { - assert!(*prev_end < min); - } - - let old = inner.ranges.insert(max, module_stack_maps); - assert!(old.is_none()); - } - - /// Lookup the stack map for the given PC, if any. - pub fn lookup_stack_map(&self, pc: usize) -> Option> { - let inner = self.inner.borrow(); - let stack_maps = inner.module_stack_maps(pc)?; - - // Do a binary search to find the stack map for the given PC. - // - // Because GC safepoints are technically only associated with a single - // PC, we should ideally only care about `Ok(index)` values returned - // from the binary search. However, safepoints are inserted right before - // calls, and there are two things that can disturb the PC/offset - // associated with the safepoint versus the PC we actually use to query - // for the stack map: - // - // 1. The `backtrace` crate gives us the PC in a frame that will be - // *returned to*, and where execution will continue from, rather than - // the PC of the call we are currently at. So we would need to - // disassemble one instruction backwards to query the actual PC for - // the stack map. - // - // TODO: One thing we *could* do to make this a little less error - // prone, would be to assert/check that the nearest GC safepoint - // found is within `max_encoded_size(any kind of call instruction)` - // our queried PC for the target architecture. - // - // 2. Cranelift's stack maps only handle the stack, not - // registers. However, some references that are arguments to a call - // may need to be in registers. In these cases, what Cranelift will - // do is: - // - // a. spill all the live references, - // b. insert a GC safepoint for those references, - // c. reload the references into registers, and finally - // d. make the call. - // - // Step (c) adds drift between the GC safepoint and the location of - // the call, which is where we actually walk the stack frame and - // collect its live references. - // - // Luckily, the spill stack slots for the live references are still - // up to date, so we can still find all the on-stack roots. - // Furthermore, we do not have a moving GC, so we don't need to worry - // whether the following code will reuse the references in registers - // (which would not have been updated to point to the moved objects) - // or reload from the stack slots (which would have been updated to - // point to the moved objects). - let index = match stack_maps - .pc_to_stack_map - .binary_search_by_key(&pc, |(pc, _stack_map)| *pc) - { - // Exact hit. - Ok(i) => i, - - // `Err(0)` means that the associated stack map would have been the - // first element in the array if this pc had an associated stack - // map, but this pc does not have an associated stack map. This can - // only happen inside a Wasm frame if there are no live refs at this - // pc. - Err(0) => return None, - - Err(n) => n - 1, - }; - - let stack_map = stack_maps.pc_to_stack_map[index].1.as_ref()?.clone(); - Some(stack_map) +impl ModuleInfoLookup for EmptyModuleInfoLookup { + fn lookup(&self, _pc: usize) -> Option> { + None } } -impl StackMapRegistryInner { - fn module_stack_maps(&self, pc: usize) -> Option<&ModuleStackMaps> { - let (end, stack_maps) = self.ranges.range(pc..).next()?; - if pc < stack_maps.range.start || *end < pc { - None - } else { - Some(stack_maps) - } - } -} +pub(crate) const EMPTY_MODULE_LOOKUP: EmptyModuleInfoLookup = EmptyModuleInfoLookup; #[derive(Debug, Default)] struct DebugOnly { @@ -965,7 +809,7 @@ impl std::ops::DerefMut for DebugOnly { /// Additionally, you must have registered the stack maps for every Wasm module /// that has frames on the stack with the given `stack_maps_registry`. pub unsafe fn gc( - stack_maps_registry: &StackMapRegistry, + module_info_lookup: &dyn ModuleInfoLookup, externref_activations_table: &VMExternRefActivationsTable, ) { // We borrow the precise stack roots `RefCell` for the whole duration of @@ -1003,8 +847,7 @@ pub unsafe fn gc( if cfg!(debug_assertions) { // Assert that there aren't any Wasm frames on the stack. backtrace::trace(|frame| { - let stack_map = stack_maps_registry.lookup_stack_map(frame.ip() as usize); - assert!(stack_map.is_none()); + assert!(module_info_lookup.lookup(frame.ip() as usize).is_none()); true }); } @@ -1048,28 +891,30 @@ pub unsafe fn gc( let pc = frame.ip() as usize; let sp = frame.sp() as usize; - if let Some(stack_map) = stack_maps_registry.lookup_stack_map(pc) { - debug_assert!(sp != 0, "we should always get a valid SP for Wasm frames"); + if let Some(module_info) = module_info_lookup.lookup(pc) { + if let Some(stack_map) = module_info.lookup_stack_map(pc) { + debug_assert!(sp != 0, "we should always get a valid SP for Wasm frames"); - for i in 0..(stack_map.mapped_words() as usize) { - if stack_map.get_bit(i) { - // Stack maps have one bit per word in the frame, and the - // zero^th bit is the *lowest* addressed word in the frame, - // i.e. the closest to the SP. So to get the `i`^th word in - // this frame, we add `i * sizeof(word)` to the SP. - let ptr_to_ref = sp + i * mem::size_of::(); + for i in 0..(stack_map.mapped_words() as usize) { + if stack_map.get_bit(i) { + // Stack maps have one bit per word in the frame, and the + // zero^th bit is the *lowest* addressed word in the frame, + // i.e. the closest to the SP. So to get the `i`^th word in + // this frame, we add `i * sizeof(word)` to the SP. + let ptr_to_ref = sp + i * mem::size_of::(); - let r = std::ptr::read(ptr_to_ref as *const *mut VMExternData); - debug_assert!( - r.is_null() || activations_table_set.contains(&r), - "every on-stack externref inside a Wasm frame should \ - have an entry in the VMExternRefActivationsTable" - ); - if let Some(r) = NonNull::new(r) { - VMExternRefActivationsTable::insert_precise_stack_root( - &mut precise_stack_roots, - r, + let r = std::ptr::read(ptr_to_ref as *const *mut VMExternData); + debug_assert!( + r.is_null() || activations_table_set.contains(&r), + "every on-stack externref inside a Wasm frame should \ + have an entry in the VMExternRefActivationsTable" ); + if let Some(r) = NonNull::new(r) { + VMExternRefActivationsTable::insert_precise_stack_root( + &mut precise_stack_roots, + r, + ); + } } } } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index bf4234f6b8..f2143681fa 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -3,7 +3,7 @@ //! `InstanceHandle` is a reference-counting handle for an `Instance`. use crate::export::Export; -use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; +use crate::externref::{ModuleInfoLookup, VMExternRefActivationsTable}; use crate::memory::{Memory, RuntimeMemoryCreator}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; @@ -37,6 +37,52 @@ mod allocator; pub use allocator::*; +/// Used by hosts to limit resource consumption of instances. +/// +/// An instance can be created with a resource limiter so that hosts can take into account +/// non-WebAssembly resource usage to determine if a linear memory or table should grow. +pub trait ResourceLimiter { + /// Notifies the resource limiter that an instance's linear memory has been requested to grow. + /// + /// * `current` is the current size of the linear memory in WebAssembly page units. + /// * `desired` is the desired size of the linear memory in WebAssembly page units. + /// * `maximum` is either the linear memory's maximum or a maximum from an instance allocator, + /// also in WebAssembly page units. A value of `None` indicates that the linear memory is + /// unbounded. + /// + /// This function should return `true` to indicate that the growing operation is permitted or + /// `false` if not permitted. Returning `true` when a maximum has been exceeded will have no + /// effect as the linear memory will not grow. + fn memory_growing(&self, current: u32, desired: u32, maximum: Option) -> bool; + + /// Notifies the resource limiter that an instance's table has been requested to grow. + /// + /// * `current` is the current number of elements in the table. + /// * `desired` is the desired number of elements in the table. + /// * `maximum` is either the table's maximum or a maximum from an instance allocator. + /// A value of `None` indicates that the table is unbounded. + /// + /// This function should return `true` to indicate that the growing operation is permitted or + /// `false` if not permitted. Returning `true` when a maximum has been exceeded will have no + /// effect as the table will not grow. + fn table_growing(&self, current: u32, desired: u32, maximum: Option) -> bool; + + /// The maximum number of instances that can be created for a `Store`. + /// + /// Module instantiation will fail if this limit is exceeded. + fn instances(&self) -> usize; + + /// The maximum number of tables that can be created for a `Store`. + /// + /// Module instantiation will fail if this limit is exceeded. + fn tables(&self) -> usize; + + /// The maximum number of tables that can be created for a `Store`. + /// + /// Module instantiation will fail if this limit is exceeded. + fn memories(&self) -> usize; +} + /// Runtime representation of an instance value, which erases all `Instance` /// information since instances are just a collection of values. pub type RuntimeInstance = Rc>; @@ -249,9 +295,9 @@ impl Instance { unsafe { self.vmctx_plus_offset(self.offsets.vmctx_externref_activations_table()) } } - /// Return a pointer to the `StackMapRegistry`. - pub fn stack_map_registry(&self) -> *mut *mut StackMapRegistry { - unsafe { self.vmctx_plus_offset(self.offsets.vmctx_stack_map_registry()) } + /// Return a pointer to the `ModuleInfoLookup`. + pub fn module_info_lookup(&self) -> *mut *const dyn ModuleInfoLookup { + unsafe { self.vmctx_plus_offset(self.offsets.vmctx_module_info_lookup()) } } /// Return a reference to the vmctx used by compiled wasm code. @@ -378,11 +424,12 @@ impl Instance { /// Returns `None` if memory can't be grown by the specified amount /// of pages. pub(crate) fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option { - let result = self + let memory = self .memories .get(memory_index) - .unwrap_or_else(|| panic!("no memory for index {}", memory_index.index())) - .grow(delta); + .unwrap_or_else(|| panic!("no memory for index {}", memory_index.index())); + + let result = unsafe { memory.grow(delta) }; // Keep current the VMContext pointers used by compiled wasm code. self.set_memory(memory_index, self.memories[memory_index].vmmemory()); @@ -460,19 +507,18 @@ impl Instance { delta: u32, init_value: TableElement, ) -> Option { - unsafe { - let orig_size = self - .tables - .get(table_index) - .unwrap_or_else(|| panic!("no table for index {}", table_index.index())) - .grow(delta, init_value)?; + let table = self + .tables + .get(table_index) + .unwrap_or_else(|| panic!("no table for index {}", table_index.index())); - // Keep the `VMContext` pointers used by compiled Wasm code up to - // date. - self.set_table(table_index, self.tables[table_index].vmtable()); + let result = unsafe { table.grow(delta, init_value) }; - Some(orig_size) - } + // Keep the `VMContext` pointers used by compiled Wasm code up to + // date. + self.set_table(table_index, self.tables[table_index].vmtable()); + + result } pub(crate) fn defined_table_fill( @@ -527,11 +573,11 @@ impl Instance { return None; } - Some(unsafe { &*self.anyfunc_ptr(index) }) + unsafe { Some(&*self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index))) } } - unsafe fn anyfunc_ptr(&self, index: FuncIndex) -> *mut VMCallerCheckedAnyfunc { - self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index)) + unsafe fn anyfunc_base(&self) -> *mut VMCallerCheckedAnyfunc { + self.vmctx_plus_offset(self.offsets.vmctx_anyfuncs_begin()) } fn find_passive_segment<'a, I, D, T>( @@ -565,38 +611,56 @@ impl Instance { src: u32, len: u32, ) -> Result<(), Trap> { - // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init - - let table = self.get_table(table_index); - let elements = Self::find_passive_segment( elem_index, &self.module.passive_elements_map, &self.module.passive_elements, &self.dropped_elements, ); + self.table_init_segment(table_index, elements, dst, src, len) + } - if src - .checked_add(len) - .map_or(true, |n| n as usize > elements.len()) - || dst.checked_add(len).map_or(true, |m| m > table.size()) + pub(crate) fn table_init_segment( + &self, + table_index: TableIndex, + elements: &[FuncIndex], + dst: u32, + src: u32, + len: u32, + ) -> Result<(), Trap> { + // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init + + let table = self.get_table(table_index); + + let elements = match elements + .get(usize::try_from(src).unwrap()..) + .and_then(|s| s.get(..usize::try_from(len).unwrap())) { - return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); + Some(elements) => elements, + None => return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)), + }; + + match table.element_type() { + TableElementType::Func => unsafe { + let base = self.anyfunc_base(); + table.init_funcs( + dst, + elements.iter().map(|idx| { + if *idx == FuncIndex::reserved_value() { + ptr::null_mut() + } else { + debug_assert!(idx.as_u32() < self.offsets.num_defined_functions); + base.add(usize::try_from(idx.as_u32()).unwrap()) + } + }), + )?; + }, + + TableElementType::Val(_) => { + debug_assert!(elements.iter().all(|e| *e == FuncIndex::reserved_value())); + table.fill(dst, TableElement::ExternRef(None), len)?; + } } - - // TODO(#983): investigate replacing this get/set loop with a `memcpy`. - for (dst, src) in (dst..dst + len).zip(src..src + len) { - let elem = self - .get_caller_checked_anyfunc(elements[src as usize]) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut _ - }); - - table - .set(dst, TableElement::FuncRef(elem)) - .expect("should never panic because we already did the bounds check above"); - } - Ok(()) } @@ -727,16 +791,26 @@ impl Instance { src: u32, len: u32, ) -> Result<(), Trap> { - // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init - - let memory = self.get_memory(memory_index); - let data = Self::find_passive_segment( data_index, &self.module.passive_data_map, &self.module.passive_data, &self.dropped_data, ); + self.memory_init_segment(memory_index, &data, dst, src, len) + } + + pub(crate) fn memory_init_segment( + &self, + memory_index: MemoryIndex, + data: &[u8], + dst: u32, + src: u32, + len: u32, + ) -> Result<(), Trap> { + // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init + + let memory = self.get_memory(memory_index); if src .checked_add(len) @@ -818,10 +892,6 @@ pub struct InstanceHandle { } impl InstanceHandle { - pub(crate) unsafe fn new(instance: *mut Instance) -> Self { - Self { instance } - } - /// Create a new `InstanceHandle` pointing at the instance /// pointed to by the given `VMContext` pointer. /// diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index d236b4723c..c91f89a74f 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -1,8 +1,8 @@ -use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; +use crate::externref::{ModuleInfoLookup, VMExternRefActivationsTable, EMPTY_MODULE_LOOKUP}; use crate::imports::Imports; -use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator}; +use crate::instance::{Instance, InstanceHandle, ResourceLimiter, RuntimeMemoryCreator}; use crate::memory::{DefaultMemoryCreator, Memory}; -use crate::table::{Table, TableElement}; +use crate::table::Table; use crate::traphandlers::Trap; use crate::vmcontext::{ VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, @@ -15,13 +15,13 @@ use std::any::Any; use std::cell::RefCell; use std::convert::TryFrom; use std::ptr::{self, NonNull}; +use std::rc::Rc; use std::slice; use std::sync::Arc; use thiserror::Error; -use wasmtime_environ::entity::{packed_option::ReservedValue, EntityRef, EntitySet, PrimaryMap}; +use wasmtime_environ::entity::{EntityRef, EntitySet, PrimaryMap}; use wasmtime_environ::wasm::{ - DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalInit, SignatureIndex, - TableElementType, WasmType, + DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, GlobalInit, SignatureIndex, WasmType, }; use wasmtime_environ::{ ir, MemoryInitialization, MemoryInitializer, Module, ModuleType, TableInitializer, VMOffsets, @@ -57,8 +57,11 @@ pub struct InstanceAllocationRequest<'a> { /// The pointer to the reference activations table to use for the instance. pub externref_activations_table: *mut VMExternRefActivationsTable, - /// The pointer to the stack map registry to use for the instance. - pub stack_map_registry: *mut StackMapRegistry, + /// The pointer to the module info lookup to use for the instance. + pub module_info_lookup: Option<*const dyn ModuleInfoLookup>, + + /// The resource limiter to use for the instance. + pub limiter: Option<&'a Rc>, } /// An link error while instantiating a module. @@ -208,7 +211,7 @@ impl<'a> From<&'a PrimaryMap> for Shared fn get_table_init_start( init: &TableInitializer, instance: &Instance, -) -> Result { +) -> Result { match init.base { Some(base) => { let val = unsafe { @@ -219,7 +222,7 @@ fn get_table_init_start( } }; - init.offset.checked_add(val as usize).ok_or_else(|| { + init.offset.checked_add(val).ok_or_else(|| { InstantiationError::Link(LinkError( "element segment global base overflows".to_owned(), )) @@ -233,6 +236,7 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError for init in &instance.module.table_initializers { let table = instance.get_table(init.table_index); let start = get_table_init_start(init, instance)?; + let start = usize::try_from(start).unwrap(); let end = start.checked_add(init.elements.len()); match end { @@ -252,34 +256,15 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { for init in &instance.module.table_initializers { - let table = instance.get_table(init.table_index); - let start = get_table_init_start(init, instance)?; - let end = start.checked_add(init.elements.len()); - - match end { - Some(end) if end <= table.size() as usize => { - for (i, func_idx) in init.elements.iter().enumerate() { - let item = match table.element_type() { - TableElementType::Func => instance - .get_caller_checked_anyfunc(*func_idx) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc - }) - .into(), - TableElementType::Val(_) => { - assert!(*func_idx == FuncIndex::reserved_value()); - TableElement::ExternRef(None) - } - }; - table.set(u32::try_from(start + i).unwrap(), item).unwrap(); - } - } - _ => { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::TableOutOfBounds, - ))) - } - } + instance + .table_init_segment( + init.table_index, + &init.elements, + get_table_init_start(init, instance)?, + 0, + init.elements.len() as u32, + ) + .map_err(InstantiationError::Trap)?; } Ok(()) @@ -288,7 +273,7 @@ fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { fn get_memory_init_start( init: &MemoryInitializer, instance: &Instance, -) -> Result { +) -> Result { match init.base { Some(base) => { let val = unsafe { @@ -299,7 +284,7 @@ fn get_memory_init_start( } }; - init.offset.checked_add(val as usize).ok_or_else(|| { + init.offset.checked_add(val).ok_or_else(|| { InstantiationError::Link(LinkError("data segment global base overflows".to_owned())) }) } @@ -307,24 +292,6 @@ fn get_memory_init_start( } } -unsafe fn get_memory_slice<'instance>( - init: &MemoryInitializer, - instance: &'instance Instance, -) -> &'instance mut [u8] { - let memory = if let Some(defined_memory_index) = - instance.module.defined_memory_index(init.memory_index) - { - instance.memory(defined_memory_index) - } else { - let import = instance.imported_memory(init.memory_index); - let foreign_instance = (&mut *(import).vmctx).instance(); - let foreign_memory = &mut *(import).from; - let foreign_index = foreign_instance.memory_index(foreign_memory); - foreign_instance.memory(foreign_index) - }; - &mut *ptr::slice_from_raw_parts_mut(memory.base, memory.current_length) -} - fn check_memory_init_bounds( instance: &Instance, initializers: &[MemoryInitializer], @@ -332,6 +299,7 @@ fn check_memory_init_bounds( for init in initializers { let memory = instance.get_memory(init.memory_index); let start = get_memory_init_start(init, instance)?; + let start = usize::try_from(start).unwrap(); let end = start.checked_add(init.data.len()); match end { @@ -354,21 +322,15 @@ fn initialize_memories( initializers: &[MemoryInitializer], ) -> Result<(), InstantiationError> { for init in initializers { - let memory = instance.get_memory(init.memory_index); - let start = get_memory_init_start(init, instance)?; - let end = start.checked_add(init.data.len()); - - match end { - Some(end) if end <= memory.current_length => { - let mem_slice = unsafe { get_memory_slice(init, instance) }; - mem_slice[start..end].copy_from_slice(&init.data); - } - _ => { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::HeapOutOfBounds, - ))) - } - } + instance + .memory_init_segment( + init.memory_index, + &init.data, + get_memory_init_start(init, instance)?, + 0, + init.data.len() as u32, + ) + .map_err(InstantiationError::Trap)?; } Ok(()) @@ -447,7 +409,7 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque *instance.interrupts() = req.interrupts; *instance.externref_activations_table() = req.externref_activations_table; - *instance.stack_map_registry() = req.stack_map_registry; + *instance.module_info_lookup() = req.module_info_lookup.unwrap_or(&EMPTY_MODULE_LOOKUP); // Initialize shared signatures let mut ptr = instance.signature_ids_ptr(); @@ -492,6 +454,7 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque ); // Initialize the functions + let mut base = instance.anyfunc_base(); for (index, sig) in instance.module.functions.iter() { let type_index = req.shared_signatures.lookup(*sig); @@ -506,13 +469,14 @@ unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationReque }; ptr::write( - instance.anyfunc_ptr(index), + base, VMCallerCheckedAnyfunc { func_ptr, type_index, vmctx, }, ); + base = base.add(1); } // Initialize the defined tables @@ -590,19 +554,23 @@ impl OnDemandInstanceAllocator { } } - fn create_tables(module: &Module) -> PrimaryMap { + fn create_tables( + module: &Module, + limiter: Option<&Rc>, + ) -> Result, InstantiationError> { let num_imports = module.num_imported_tables; let mut tables: PrimaryMap = PrimaryMap::with_capacity(module.table_plans.len() - num_imports); for table in &module.table_plans.values().as_slice()[num_imports..] { - tables.push(Table::new_dynamic(table)); + tables.push(Table::new_dynamic(table, limiter).map_err(InstantiationError::Resource)?); } - tables + Ok(tables) } fn create_memories( &self, module: &Module, + limiter: Option<&Rc>, ) -> Result, InstantiationError> { let creator = self .mem_creator @@ -612,8 +580,10 @@ impl OnDemandInstanceAllocator { let mut memories: PrimaryMap = PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); for plan in &module.memory_plans.values().as_slice()[num_imports..] { - memories - .push(Memory::new_dynamic(plan, creator).map_err(InstantiationError::Resource)?); + memories.push( + Memory::new_dynamic(plan, creator, limiter) + .map_err(InstantiationError::Resource)?, + ); } Ok(memories) } @@ -633,8 +603,8 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { &self, mut req: InstanceAllocationRequest, ) -> Result { - let memories = self.create_memories(&req.module)?; - let tables = Self::create_tables(&req.module); + let memories = self.create_memories(&req.module, req.limiter)?; + let tables = Self::create_tables(&req.module, req.limiter)?; let host_state = std::mem::replace(&mut req.host_state, Box::new(())); @@ -657,7 +627,9 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { alloc::handle_alloc_error(layout); } ptr::write(instance_ptr, instance); - InstanceHandle::new(instance_ptr) + InstanceHandle { + instance: instance_ptr, + } }; initialize_vmcontext(handle.instance(), req); diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index d9604c348f..7a17b2b143 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -9,7 +9,7 @@ use super::{ initialize_instance, initialize_vmcontext, InstanceAllocationRequest, InstanceAllocator, - InstanceHandle, InstantiationError, + InstanceHandle, InstantiationError, ResourceLimiter, }; use crate::{instance::Instance, Memory, Mmap, Table, VMContext}; use anyhow::{anyhow, bail, Context, Result}; @@ -18,6 +18,7 @@ use std::cell::RefCell; use std::cmp::min; use std::convert::TryFrom; use std::mem; +use std::rc::Rc; use std::sync::{Arc, Mutex}; use wasmtime_environ::{ entity::{EntitySet, PrimaryMap}, @@ -376,10 +377,45 @@ impl InstancePool { } } + unsafe fn setup_instance( + &self, + index: usize, + mut req: InstanceAllocationRequest, + ) -> Result { + let instance = self.instance(index); + + instance.module = req.module.clone(); + instance.offsets = VMOffsets::new( + std::mem::size_of::<*const u8>() as u8, + instance.module.as_ref(), + ); + instance.host_state = std::mem::replace(&mut req.host_state, Box::new(())); + + Self::set_instance_memories( + instance, + self.memories.get(index), + self.memories.max_wasm_pages, + req.limiter, + )?; + + Self::set_instance_tables( + instance, + self.tables.get(index), + self.tables.max_elements, + req.limiter, + )?; + + initialize_vmcontext(instance, req); + + Ok(InstanceHandle { + instance: instance as _, + }) + } + fn allocate( &self, strategy: PoolingAllocationStrategy, - mut req: InstanceAllocationRequest, + req: InstanceAllocationRequest, ) -> Result { let index = { let mut free_list = self.free_list.lock().unwrap(); @@ -390,28 +426,15 @@ impl InstancePool { free_list.swap_remove(free_index) }; - let host_state = std::mem::replace(&mut req.host_state, Box::new(())); - unsafe { - let instance = self.instance(index); - - instance.module = req.module.clone(); - instance.offsets = VMOffsets::new( - std::mem::size_of::<*const u8>() as u8, - instance.module.as_ref(), - ); - instance.host_state = host_state; - - Self::set_instance_memories( - instance, - self.memories.get(index), - self.memories.max_wasm_pages, - )?; - Self::set_instance_tables(instance, self.tables.get(index), self.tables.max_elements)?; - - initialize_vmcontext(instance, req); - - Ok(InstanceHandle::new(instance as _)) + self.setup_instance(index, req).or_else(|e| { + // Deallocate the allocated instance on error + let instance = self.instance(index); + self.deallocate(&InstanceHandle { + instance: instance as _, + }); + Err(e) + }) } } @@ -473,6 +496,7 @@ impl InstancePool { instance: &mut Instance, mut memories: impl Iterator, max_pages: u32, + limiter: Option<&Rc>, ) -> Result<(), InstantiationError> { let module = instance.module.as_ref(); @@ -487,6 +511,7 @@ impl InstancePool { memories.next().unwrap(), max_pages, commit_memory_pages, + limiter, ) .map_err(InstantiationError::Resource)?, ); @@ -503,6 +528,7 @@ impl InstancePool { instance: &mut Instance, mut tables: impl Iterator, max_elements: u32, + limiter: Option<&Rc>, ) -> Result<(), InstantiationError> { let module = instance.module.as_ref(); @@ -514,9 +540,10 @@ impl InstancePool { commit_table_pages(base, max_elements as usize * mem::size_of::<*mut u8>()) .map_err(InstantiationError::Resource)?; - instance - .tables - .push(Table::new_static(plan, base as _, max_elements)); + instance.tables.push( + Table::new_static(plan, base as _, max_elements, limiter) + .map_err(InstantiationError::Resource)?, + ); } let mut dropped_elements = instance.dropped_elements.borrow_mut(); @@ -1370,7 +1397,8 @@ mod test { host_state: Box::new(()), interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_registry: std::ptr::null_mut(), + module_info_lookup: None, + limiter: None, }, ) .expect("allocation should succeed"), @@ -1394,7 +1422,8 @@ mod test { host_state: Box::new(()), interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_registry: std::ptr::null_mut(), + module_info_lookup: None, + limiter: None, }, ) { Err(InstantiationError::Limit(3)) => {} diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs index 5578ca1746..43ba9a654a 100644 --- a/crates/runtime/src/instance/allocator/pooling/uffd.rs +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -523,7 +523,8 @@ mod test { host_state: Box::new(()), interrupts: ptr::null(), externref_activations_table: ptr::null_mut(), - stack_map_registry: ptr::null_mut(), + module_info_lookup: None, + limiter: None, }, ) .expect("instance should allocate"), diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 20f28ffd97..1c3ce53a3d 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -40,7 +40,7 @@ pub use crate::imports::Imports; pub use crate::instance::{ InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstanceLimits, InstantiationError, LinkError, ModuleLimits, OnDemandInstanceAllocator, - PoolingAllocationStrategy, PoolingInstanceAllocator, RuntimeInstance, + PoolingAllocationStrategy, PoolingInstanceAllocator, ResourceLimiter, RuntimeInstance, }; pub use crate::jit_int::GdbJitImageRegistration; pub use crate::memory::{Memory, RuntimeLinearMemory, RuntimeMemoryCreator}; diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index 026738a8a1..9c5800fd8e 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -449,8 +449,8 @@ pub unsafe extern "C" fn wasmtime_activations_table_insert_with_gc( let externref = VMExternRef::clone_from_raw(externref); let instance = (&mut *vmctx).instance(); let activations_table = &**instance.externref_activations_table(); - let registry = &**instance.stack_map_registry(); - activations_table.insert_with_gc(externref, registry); + let module_info_lookup = &**instance.module_info_lookup(); + activations_table.insert_with_gc(externref, module_info_lookup); } /// Perform a Wasm `global.get` for `externref` globals. @@ -466,8 +466,8 @@ pub unsafe extern "C" fn wasmtime_externref_global_get( Some(externref) => { let raw = externref.as_raw(); let activations_table = &**instance.externref_activations_table(); - let registry = &**instance.stack_map_registry(); - activations_table.insert_with_gc(externref, registry); + let module_info_lookup = &**instance.module_info_lookup(); + activations_table.insert_with_gc(externref, module_info_lookup); raw } } diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index 024d901246..973016224d 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -4,12 +4,14 @@ use crate::mmap::Mmap; use crate::vmcontext::VMMemoryDefinition; -use anyhow::Result; +use crate::ResourceLimiter; +use anyhow::{bail, Result}; use more_asserts::{assert_ge, assert_le}; use std::cell::{Cell, RefCell}; use std::cmp::min; use std::convert::TryFrom; use std::ptr; +use std::rc::Rc; use wasmtime_environ::{MemoryPlan, MemoryStyle, WASM_MAX_PAGES, WASM_PAGE_SIZE}; /// A memory allocator @@ -33,6 +35,10 @@ pub trait RuntimeLinearMemory { /// Returns the number of allocated wasm pages. fn size(&self) -> u32; + /// Returns the maximum number of pages the memory can grow to. + /// Returns `None` if the memory is unbounded. + fn maximum(&self) -> Option; + /// Grow memory by the specified amount of wasm pages. /// /// Returns `None` if memory can't be grown by the specified amount @@ -105,6 +111,12 @@ impl RuntimeLinearMemory for MmapMemory { self.mmap.borrow().size } + /// Returns the maximum number of pages the memory can grow to. + /// Returns `None` if the memory is unbounded. + fn maximum(&self) -> Option { + self.maximum + } + /// Grow memory by the specified amount of wasm pages. /// /// Returns `None` if memory can't be grown by the specified amount @@ -189,12 +201,23 @@ enum MemoryStorage { } /// Represents an instantiation of a WebAssembly memory. -pub struct Memory(MemoryStorage); +pub struct Memory { + storage: MemoryStorage, + limiter: Option>, +} impl Memory { /// Create a new dynamic (movable) memory instance for the specified plan. - pub fn new_dynamic(plan: &MemoryPlan, creator: &dyn RuntimeMemoryCreator) -> Result { - Ok(Self(MemoryStorage::Dynamic(creator.new_memory(plan)?))) + pub fn new_dynamic( + plan: &MemoryPlan, + creator: &dyn RuntimeMemoryCreator, + limiter: Option<&Rc>, + ) -> Result { + Self::new( + plan, + MemoryStorage::Dynamic(creator.new_memory(plan)?), + limiter, + ) } /// Create a new static (immovable) memory instance for the specified plan. @@ -203,32 +226,78 @@ impl Memory { base: *mut u8, maximum: u32, make_accessible: fn(*mut u8, usize) -> Result<()>, + limiter: Option<&Rc>, ) -> Result { - if plan.memory.minimum > 0 { - make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize)?; - } - - Ok(Self(MemoryStorage::Static { + let storage = MemoryStorage::Static { base, size: Cell::new(plan.memory.minimum), maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum), make_accessible, #[cfg(all(feature = "uffd", target_os = "linux"))] guard_page_faults: RefCell::new(Vec::new()), - })) + }; + + Self::new(plan, storage, limiter) + } + + fn new( + plan: &MemoryPlan, + storage: MemoryStorage, + limiter: Option<&Rc>, + ) -> Result { + if let Some(limiter) = limiter { + if !limiter.memory_growing(0, plan.memory.minimum, plan.memory.maximum) { + bail!( + "memory minimum size of {} pages exceeds memory limits", + plan.memory.minimum + ); + } + } + + if let MemoryStorage::Static { + base, + make_accessible, + .. + } = &storage + { + if plan.memory.minimum > 0 { + make_accessible( + *base, + plan.memory.minimum as usize * WASM_PAGE_SIZE as usize, + )?; + } + } + + Ok(Self { + storage, + limiter: limiter.cloned(), + }) } /// Returns the number of allocated wasm pages. pub fn size(&self) -> u32 { - match &self.0 { + match &self.storage { MemoryStorage::Static { size, .. } => size.get(), MemoryStorage::Dynamic(mem) => mem.size(), } } + /// Returns the maximum number of pages the memory can grow to at runtime. + /// + /// Returns `None` if the memory is unbounded. + /// + /// The runtime maximum may not be equal to the maximum from the linear memory's + /// Wasm type when it is being constrained by an instance allocator. + pub fn maximum(&self) -> Option { + match &self.storage { + MemoryStorage::Static { maximum, .. } => Some(*maximum), + MemoryStorage::Dynamic(mem) => mem.maximum(), + } + } + /// Returns whether or not the underlying storage of the memory is "static". pub(crate) fn is_static(&self) -> bool { - if let MemoryStorage::Static { .. } = &self.0 { + if let MemoryStorage::Static { .. } = &self.storage { true } else { false @@ -239,8 +308,30 @@ impl Memory { /// /// Returns `None` if memory can't be grown by the specified amount /// of wasm pages. - pub fn grow(&self, delta: u32) -> Option { - match &self.0 { + /// + /// # Safety + /// + /// Resizing the memory can reallocate the memory buffer for dynamic memories. + /// An instance's `VMContext` may have pointers to the memory's base and will + /// need to be fixed up after growing the memory. + /// + /// Generally, prefer using `InstanceHandle::memory_grow`, which encapsulates + /// this unsafety. + pub unsafe fn grow(&self, delta: u32) -> Option { + let old_size = self.size(); + if delta == 0 { + return Some(old_size); + } + + let new_size = old_size.checked_add(delta)?; + + if let Some(limiter) = &self.limiter { + if !limiter.memory_growing(old_size, new_size, self.maximum()) { + return None; + } + } + + match &self.storage { MemoryStorage::Static { base, size, @@ -252,13 +343,6 @@ impl Memory { #[cfg(all(feature = "uffd", target_os = "linux"))] self.reset_guard_pages().ok()?; - let old_size = size.get(); - if delta == 0 { - return Some(old_size); - } - - let new_size = old_size.checked_add(delta)?; - if new_size > *maximum || new_size >= WASM_MAX_PAGES { return None; } @@ -266,7 +350,7 @@ impl Memory { let start = usize::try_from(old_size).unwrap() * WASM_PAGE_SIZE as usize; let len = usize::try_from(delta).unwrap() * WASM_PAGE_SIZE as usize; - make_accessible(unsafe { base.add(start) }, len).ok()?; + make_accessible(base.add(start), len).ok()?; size.set(new_size); @@ -278,7 +362,7 @@ impl Memory { /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. pub fn vmmemory(&self) -> VMMemoryDefinition { - match &self.0 { + match &self.storage { MemoryStorage::Static { base, size, .. } => VMMemoryDefinition { base: *base, current_length: size.get() as usize * WASM_PAGE_SIZE as usize, @@ -299,7 +383,7 @@ impl Memory { size: usize, reset: fn(*mut u8, usize) -> Result<()>, ) { - match &self.0 { + match &self.storage { MemoryStorage::Static { guard_page_faults, .. } => { @@ -320,7 +404,7 @@ impl Memory { /// This function will panic if called on a dynamic memory. #[cfg(all(feature = "uffd", target_os = "linux"))] pub(crate) fn reset_guard_pages(&self) -> Result<()> { - match &self.0 { + match &self.storage { MemoryStorage::Static { guard_page_faults, .. } => { @@ -345,13 +429,16 @@ impl Default for Memory { unreachable!() } - Self(MemoryStorage::Static { - base: ptr::null_mut(), - size: Cell::new(0), - maximum: 0, - make_accessible, - #[cfg(all(feature = "uffd", target_os = "linux"))] - guard_page_faults: RefCell::new(Vec::new()), - }) + Self { + storage: MemoryStorage::Static { + base: ptr::null_mut(), + size: Cell::new(0), + maximum: 0, + make_accessible, + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), + }, + limiter: None, + } } } diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index 8c857add45..9e26721fb4 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -3,19 +3,21 @@ //! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories. use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition}; -use crate::{Trap, VMExternRef}; +use crate::{ResourceLimiter, Trap, VMExternRef}; +use anyhow::{bail, Result}; use std::cell::{Cell, RefCell}; use std::cmp::min; -use std::convert::TryInto; +use std::convert::{TryFrom, TryInto}; use std::ops::Range; use std::ptr; +use std::rc::Rc; use wasmtime_environ::wasm::TableElementType; use wasmtime_environ::{ir, TablePlan}; /// An element going into or coming out of a table. /// /// Table elements are stored as pointers and are default-initialized with `ptr::null_mut`. -#[derive(Clone, Debug)] +#[derive(Clone)] pub enum TableElement { /// A `funcref`. FuncRef(*mut VMCallerCheckedAnyfunc), @@ -69,7 +71,7 @@ impl TableElement { unsafe fn into_raw(self) -> *mut u8 { match self { Self::FuncRef(e) => e as _, - Self::ExternRef(e) => e.map(|e| e.into_raw()).unwrap_or(ptr::null_mut()), + Self::ExternRef(e) => e.map_or(ptr::null_mut(), |e| e.into_raw()), } } } @@ -92,7 +94,6 @@ impl From for TableElement { } } -#[derive(Debug)] enum TableStorage { Static { data: *mut *mut u8, @@ -108,38 +109,74 @@ enum TableStorage { } /// Represents an instance's table. -#[derive(Debug)] -pub struct Table(TableStorage); +pub struct Table { + storage: TableStorage, + limiter: Option>, +} impl Table { /// Create a new dynamic (movable) table instance for the specified table plan. - pub fn new_dynamic(plan: &TablePlan) -> Self { + pub fn new_dynamic( + plan: &TablePlan, + limiter: Option<&Rc>, + ) -> Result { let elements = RefCell::new(vec![ptr::null_mut(); plan.table.minimum as usize]); let ty = plan.table.ty.clone(); let maximum = plan.table.maximum; - Self(TableStorage::Dynamic { + + let storage = TableStorage::Dynamic { elements, ty, maximum, - }) + }; + + Self::new(plan, storage, limiter) } /// Create a new static (immovable) table instance for the specified table plan. - pub fn new_static(plan: &TablePlan, data: *mut *mut u8, maximum: u32) -> Self { + pub fn new_static( + plan: &TablePlan, + data: *mut *mut u8, + maximum: u32, + limiter: Option<&Rc>, + ) -> Result { let size = Cell::new(plan.table.minimum); let ty = plan.table.ty.clone(); let maximum = min(plan.table.maximum.unwrap_or(maximum), maximum); - Self(TableStorage::Static { + + let storage = TableStorage::Static { data, size, ty, maximum, + }; + + Self::new(plan, storage, limiter) + } + + fn new( + plan: &TablePlan, + storage: TableStorage, + limiter: Option<&Rc>, + ) -> Result { + if let Some(limiter) = limiter { + if !limiter.table_growing(0, plan.table.minimum, plan.table.maximum) { + bail!( + "table minimum size of {} elements exceeds table limits", + plan.table.minimum + ); + } + } + + Ok(Self { + storage, + limiter: limiter.cloned(), }) } /// Returns the type of the elements in this table. pub fn element_type(&self) -> TableElementType { - match &self.0 { + match &self.storage { TableStorage::Static { ty, .. } => *ty, TableStorage::Dynamic { ty, .. } => *ty, } @@ -147,7 +184,7 @@ impl Table { /// Returns whether or not the underlying storage of the table is "static". pub(crate) fn is_static(&self) -> bool { - if let TableStorage::Static { .. } = &self.0 { + if let TableStorage::Static { .. } = &self.storage { true } else { false @@ -156,20 +193,51 @@ impl Table { /// Returns the number of allocated elements. pub fn size(&self) -> u32 { - match &self.0 { + match &self.storage { TableStorage::Static { size, .. } => size.get(), TableStorage::Dynamic { elements, .. } => elements.borrow().len().try_into().unwrap(), } } - /// Returns the maximum number of elements. + /// Returns the maximum number of elements at runtime. + /// + /// Returns `None` if the table is unbounded. + /// + /// The runtime maximum may not be equal to the maximum from the table's Wasm type + /// when it is being constrained by an instance allocator. pub fn maximum(&self) -> Option { - match &self.0 { + match &self.storage { TableStorage::Static { maximum, .. } => Some(*maximum), TableStorage::Dynamic { maximum, .. } => maximum.clone(), } } + /// Fill `table[dst..]` with values from `items` + /// + /// Returns a trap error on out-of-bounds accesses. + pub fn init_funcs( + &self, + dst: u32, + items: impl ExactSizeIterator, + ) -> Result<(), Trap> { + assert!(self.element_type() == TableElementType::Func); + + self.with_elements_mut(|elements| { + let elements = match elements + .get_mut(usize::try_from(dst).unwrap()..) + .and_then(|s| s.get_mut(..items.len())) + { + Some(elements) => elements, + None => return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)), + }; + + for (item, slot) in items.zip(elements) { + *slot = item as *mut u8; + } + Ok(()) + }) + } + /// Fill `table[dst..dst + len]` with `val`. /// /// Returns a trap error on out-of-bounds accesses. @@ -218,8 +286,14 @@ impl Table { /// this unsafety. pub unsafe fn grow(&self, delta: u32, init_value: TableElement) -> Option { let old_size = self.size(); - let new_size = old_size.checked_add(delta)?; + + if let Some(limiter) = &self.limiter { + if !limiter.table_growing(old_size, new_size, self.maximum()) { + return None; + } + } + if let Some(max) = self.maximum() { if new_size > max { return None; @@ -229,7 +303,7 @@ impl Table { debug_assert!(self.type_matches(&init_value)); // First resize the storage and then fill with the init value - match &self.0 { + match &self.storage { TableStorage::Static { size, .. } => { size.set(new_size); } @@ -319,7 +393,7 @@ impl Table { /// Return a `VMTableDefinition` for exposing the table to compiled wasm code. pub fn vmtable(&self) -> VMTableDefinition { - match &self.0 { + match &self.storage { TableStorage::Static { data, size, .. } => VMTableDefinition { base: *data as _, current_elements: size.get(), @@ -346,7 +420,7 @@ impl Table { where F: FnOnce(&[*mut u8]) -> R, { - match &self.0 { + match &self.storage { TableStorage::Static { data, size, .. } => unsafe { f(std::slice::from_raw_parts(*data, size.get() as usize)) }, @@ -361,7 +435,7 @@ impl Table { where F: FnOnce(&mut [*mut u8]) -> R, { - match &self.0 { + match &self.storage { TableStorage::Static { data, size, .. } => unsafe { f(std::slice::from_raw_parts_mut(*data, size.get() as usize)) }, @@ -463,11 +537,14 @@ impl Drop for Table { // The default table representation is an empty funcref table that cannot grow. impl Default for Table { fn default() -> Self { - Self(TableStorage::Static { - data: std::ptr::null_mut(), - size: Cell::new(0), - ty: TableElementType::Func, - maximum: 0, - }) + Self { + storage: TableStorage::Static { + data: std::ptr::null_mut(), + size: Cell::new(0), + ty: TableElementType::Func, + maximum: 0, + }, + limiter: None, + } } } diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 36d7ceba86..75f7a7d6d4 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -58,13 +58,12 @@ static mut IS_WASM_PC: fn(usize) -> bool = |_| false; /// program counter is the pc of an actual wasm trap or not. This is then used /// to disambiguate faults that happen due to wasm and faults that happen due to /// bugs in Rust or elsewhere. -pub fn init_traps(is_wasm_pc: fn(usize) -> bool) -> Result<(), Trap> { +pub fn init_traps(is_wasm_pc: fn(usize) -> bool) { static INIT: Once = Once::new(); INIT.call_once(|| unsafe { IS_WASM_PC = is_wasm_pc; sys::platform_init(); }); - sys::lazy_per_thread_init() } /// Raises a user-defined trap immediately. @@ -256,7 +255,7 @@ impl<'a> CallThreadState<'a> { } fn with(self, closure: impl FnOnce(&CallThreadState) -> i32) -> Result<(), Trap> { - let ret = tls::set(&self, || closure(&self)); + let ret = tls::set(&self, || closure(&self))?; if ret != 0 { return Ok(()); } @@ -366,6 +365,7 @@ impl Drop for ResetCell<'_, T> { // the caller to the trap site. mod tls { use super::CallThreadState; + use crate::Trap; use std::mem; use std::ptr; @@ -384,21 +384,38 @@ mod tls { // these TLS values when the runtime may have crossed threads. mod raw { use super::CallThreadState; + use crate::Trap; use std::cell::Cell; use std::ptr; pub type Ptr = *const CallThreadState<'static>; - thread_local!(static PTR: Cell = Cell::new(ptr::null())); + // The first entry here is the `Ptr` which is what's used as part of the + // public interface of this module. The second entry is a boolean which + // allows the runtime to perform per-thread initialization if necessary + // for handling traps (e.g. setting up ports on macOS and sigaltstack on + // Unix). + thread_local!(static PTR: Cell<(Ptr, bool)> = Cell::new((ptr::null(), false))); #[inline(never)] // see module docs for why this is here - pub fn replace(val: Ptr) -> Ptr { - PTR.with(|p| p.replace(val)) + pub fn replace(val: Ptr) -> Result { + PTR.with(|p| { + // When a new value is configured that means that we may be + // entering WebAssembly so check to see if this thread has + // performed per-thread initialization for traps. + let (prev, mut initialized) = p.get(); + if !initialized { + super::super::sys::lazy_per_thread_init()?; + initialized = true; + } + p.set((val, initialized)); + Ok(prev) + }) } #[inline(never)] // see module docs for why this is here pub fn get() -> Ptr { - PTR.with(|p| p.get()) + PTR.with(|p| p.get().0) } } @@ -412,7 +429,7 @@ mod tls { /// /// This is not a safe operation since it's intended to only be used /// with stack switching found with fibers and async wasmtime. - pub unsafe fn take() -> TlsRestore { + pub unsafe fn take() -> Result { // Our tls pointer must be set at this time, and it must not be // null. We need to restore the previous pointer since we're // removing ourselves from the call-stack, and in the process we @@ -421,8 +438,8 @@ mod tls { let raw = raw::get(); assert!(!raw.is_null()); let prev = (*raw).prev.replace(ptr::null()); - raw::replace(prev); - TlsRestore(raw) + raw::replace(prev)?; + Ok(TlsRestore(raw)) } /// Restores a previous tls state back into this thread's TLS. @@ -430,17 +447,12 @@ mod tls { /// This is unsafe because it's intended to only be used within the /// context of stack switching within wasmtime. pub unsafe fn replace(self) -> Result<(), super::Trap> { - // When replacing to the previous value of TLS, we might have - // crossed a thread: make sure the trap-handling lazy initializer - // runs. - super::sys::lazy_per_thread_init()?; - // We need to configure our previous TLS pointer to whatever is in // TLS at this time, and then we set the current state to ourselves. let prev = raw::get(); assert!((*self.0).prev.get().is_null()); (*self.0).prev.set(prev); - raw::replace(self.0); + raw::replace(self.0)?; Ok(()) } } @@ -448,13 +460,14 @@ mod tls { /// Configures thread local state such that for the duration of the /// execution of `closure` any call to `with` will yield `ptr`, unless this /// is recursively called again. - pub fn set(state: &CallThreadState<'_>, closure: impl FnOnce() -> R) -> R { + pub fn set(state: &CallThreadState<'_>, closure: impl FnOnce() -> R) -> Result { struct Reset<'a, 'b>(&'a CallThreadState<'b>); impl Drop for Reset<'_, '_> { #[inline] fn drop(&mut self) { - raw::replace(self.0.prev.replace(ptr::null())); + raw::replace(self.0.prev.replace(ptr::null())) + .expect("tls should be previously initialized"); } } @@ -464,10 +477,10 @@ mod tls { let ptr = unsafe { mem::transmute::<*const CallThreadState<'_>, *const CallThreadState<'static>>(state) }; - let prev = raw::replace(ptr); + let prev = raw::replace(ptr)?; state.prev.set(prev); let _reset = Reset(state); - closure() + Ok(closure()) } /// Returns the last pointer configured with `set` above. Panics if `set` diff --git a/crates/runtime/src/traphandlers/macos.rs b/crates/runtime/src/traphandlers/macos.rs index 0f5e593012..f48ae034a1 100644 --- a/crates/runtime/src/traphandlers/macos.rs +++ b/crates/runtime/src/traphandlers/macos.rs @@ -25,7 +25,7 @@ //! use a thread-local to store information about how to unwind. Additionally //! this requires that the check of whether a pc is a wasm trap or not is a //! global check rather than a per-thread check. This necessitates the existence -//! of `GlobalFrameInfo` in the `wasmtime` crate. +//! of `GlobalModuleRegistry` in the `wasmtime` crate. //! //! Otherwise this file heavily uses the `mach` Rust crate for type and //! function declarations. Many bits and pieces are copied or translated from @@ -42,7 +42,6 @@ use mach::message::*; use mach::port::*; use mach::thread_act::*; use mach::traps::*; -use std::cell::Cell; use std::mem; use std::thread; @@ -425,26 +424,16 @@ impl Drop for ClosePort { /// task-level port which is where we'd expected things like breakpad/crashpad /// exception handlers to get registered. pub fn lazy_per_thread_init() -> Result<(), Trap> { - thread_local! { - static PORTS_SET: Cell = Cell::new(false); + unsafe { + assert!(WASMTIME_PORT != MACH_PORT_NULL); + let kret = thread_set_exception_ports( + MY_PORT.with(|p| p.0), + EXC_MASK_BAD_ACCESS | EXC_MASK_BAD_INSTRUCTION, + WASMTIME_PORT, + EXCEPTION_DEFAULT | MACH_EXCEPTION_CODES, + mach_addons::THREAD_STATE_NONE, + ); + assert_eq!(kret, KERN_SUCCESS, "failed to set thread exception port"); } - - PORTS_SET.with(|ports| { - if ports.replace(true) { - return; - } - - unsafe { - assert!(WASMTIME_PORT != MACH_PORT_NULL); - let kret = thread_set_exception_ports( - MY_PORT.with(|p| p.0), - EXC_MASK_BAD_ACCESS | EXC_MASK_BAD_INSTRUCTION, - WASMTIME_PORT, - EXCEPTION_DEFAULT | MACH_EXCEPTION_CODES, - mach_addons::THREAD_STATE_NONE, - ); - assert_eq!(kret, KERN_SUCCESS, "failed to set thread exception port"); - } - }); Ok(()) } diff --git a/crates/runtime/src/traphandlers/unix.rs b/crates/runtime/src/traphandlers/unix.rs index 579acf59c3..b4046f0944 100644 --- a/crates/runtime/src/traphandlers/unix.rs +++ b/crates/runtime/src/traphandlers/unix.rs @@ -47,8 +47,8 @@ pub unsafe fn platform_init() { // Handle `unreachable` instructions which execute `ud2` right now register(&mut PREV_SIGILL, libc::SIGILL); - // x86 uses SIGFPE to report division by zero - if cfg!(target_arch = "x86") || cfg!(target_arch = "x86_64") { + // x86 and s390x use SIGFPE to report division by zero + if cfg!(target_arch = "x86") || cfg!(target_arch = "x86_64") || cfg!(target_arch = "s390x") { register(&mut PREV_SIGFPE, libc::SIGFPE); } @@ -85,7 +85,7 @@ unsafe extern "C" fn trap_handler( // Otherwise flag ourselves as handling a trap, do the trap // handling, and reset our trap handling flag. Then we figure // out what to do based on the result of the trap handling. - let pc = get_pc(context); + let pc = get_pc(context, signum); let jmp_buf = info.jmp_buf_if_trap(pc, |handler| handler(signum, siginfo, context)); // Figure out what to do based on the result of this handling of @@ -127,7 +127,7 @@ unsafe extern "C" fn trap_handler( } } -unsafe fn get_pc(cx: *mut libc::c_void) -> *const u8 { +unsafe fn get_pc(cx: *mut libc::c_void, _signum: libc::c_int) -> *const u8 { cfg_if::cfg_if! { if #[cfg(all(target_os = "linux", target_arch = "x86_64"))] { let cx = &*(cx as *const libc::ucontext_t); @@ -138,6 +138,23 @@ unsafe fn get_pc(cx: *mut libc::c_void) -> *const u8 { } else if #[cfg(all(any(target_os = "linux", target_os = "android"), target_arch = "aarch64"))] { let cx = &*(cx as *const libc::ucontext_t); cx.uc_mcontext.pc as *const u8 + } else if #[cfg(all(target_os = "linux", target_arch = "s390x"))] { + // On s390x, SIGILL and SIGFPE are delivered with the PSW address + // pointing *after* the faulting instruction, while SIGSEGV and + // SIGBUS are delivered with the PSW address pointing *to* the + // faulting instruction. To handle this, the code generator registers + // any trap that results in one of "late" signals on the last byte + // of the instruction, and any trap that results in one of the "early" + // signals on the first byte of the instruction (as usual). This + // means we simply need to decrement the reported PSW address by + // one in the case of a "late" signal here to ensure we always + // correctly find the associated trap handler. + let trap_offset = match _signum { + libc::SIGILL | libc::SIGFPE => 1, + _ => 0, + }; + let cx = &*(cx as *const libc::ucontext_t); + (cx.uc_mcontext.psw.addr - trap_offset) as *const u8 } else if #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] { let cx = &*(cx as *const libc::ucontext_t); cx.uc_mcontext.mc_rip as *const u8 @@ -154,41 +171,35 @@ unsafe fn get_pc(cx: *mut libc::c_void) -> *const u8 { /// and registering our own alternate stack that is large enough and has a guard /// page. pub fn lazy_per_thread_init() -> Result<(), Trap> { + // This thread local is purely used to register a `Stack` to get deallocated + // when the thread exists. Otherwise this function is only ever called at + // most once per-thread. thread_local! { - /// Thread-local state is lazy-initialized on the first time it's used, - /// and dropped when the thread exits. - static TLS: RefCell = RefCell::new(Tls::None); + static STACK: RefCell> = RefCell::new(None); } /// The size of the sigaltstack (not including the guard, which will be /// added). Make this large enough to run our signal handlers. const MIN_STACK_SIZE: usize = 16 * 4096; - enum Tls { - None, - Allocated { - mmap_ptr: *mut libc::c_void, - mmap_size: usize, - }, - BigEnough, + struct Stack { + mmap_ptr: *mut libc::c_void, + mmap_size: usize, } - return TLS.with(|slot| unsafe { - let mut slot = slot.borrow_mut(); - match *slot { - Tls::None => {} - // already checked - _ => return Ok(()), - } + return STACK.with(|s| { + *s.borrow_mut() = unsafe { allocate_sigaltstack()? }; + Ok(()) + }); + unsafe fn allocate_sigaltstack() -> Result, Trap> { // Check to see if the existing sigaltstack, if it exists, is big // enough. If so we don't need to allocate our own. let mut old_stack = mem::zeroed(); let r = libc::sigaltstack(ptr::null(), &mut old_stack); assert_eq!(r, 0, "learning about sigaltstack failed"); if old_stack.ss_flags & libc::SS_DISABLE == 0 && old_stack.ss_size >= MIN_STACK_SIZE { - *slot = Tls::BigEnough; - return Ok(()); + return Ok(None); } // ... but failing that we need to allocate our own, so do all that @@ -226,25 +237,17 @@ pub fn lazy_per_thread_init() -> Result<(), Trap> { let r = libc::sigaltstack(&new_stack, ptr::null_mut()); assert_eq!(r, 0, "registering new sigaltstack failed"); - *slot = Tls::Allocated { + Ok(Some(Stack { mmap_ptr: ptr, mmap_size: alloc_size, - }; - Ok(()) - }); + })) + } - impl Drop for Tls { + impl Drop for Stack { fn drop(&mut self) { - let (ptr, size) = match self { - Tls::Allocated { - mmap_ptr, - mmap_size, - } => (*mmap_ptr, *mmap_size), - _ => return, - }; unsafe { // Deallocate the stack memory. - let r = libc::munmap(ptr, size); + let r = libc::munmap(self.mmap_ptr, self.mmap_size); debug_assert_eq!(r, 0, "munmap failed during thread shutdown"); } } diff --git a/crates/wasi-common/cap-std-sync/Cargo.toml b/crates/wasi-common/cap-std-sync/Cargo.toml index eab62d0721..3d1ee935dd 100644 --- a/crates/wasi-common/cap-std-sync/Cargo.toml +++ b/crates/wasi-common/cap-std-sync/Cargo.toml @@ -15,10 +15,10 @@ include = ["src/**/*", "LICENSE" ] wasi-common = { path = "../", version = "0.26.0" } async-trait = "0.1" anyhow = "1.0" -cap-std = "0.13.7" -cap-fs-ext = "0.13.7" -cap-time-ext = "0.13.7" -cap-rand = "0.13.2" +cap-std = "0.13.9" +cap-fs-ext = "0.13.9" +cap-time-ext = "0.13.9" +cap-rand = "0.13.9" fs-set-times = "0.3.1" unsafe-io = "0.6.5" system-interface = { version = "0.6.3", features = ["cap_std_impls"] } diff --git a/crates/wasi-crypto/src/wiggle_interfaces/error.rs b/crates/wasi-crypto/src/wiggle_interfaces/error.rs index fd13a87331..e54a104b03 100644 --- a/crates/wasi-crypto/src/wiggle_interfaces/error.rs +++ b/crates/wasi-crypto/src/wiggle_interfaces/error.rs @@ -1,4 +1,4 @@ -use super::{guest_types, WasiCryptoCtx}; +use super::guest_types; use std::num::TryFromIntError; use wasi_crypto::CryptoError; diff --git a/crates/wasi-nn/Cargo.toml b/crates/wasi-nn/Cargo.toml index 1ce73aeed7..a5de00ed3a 100644 --- a/crates/wasi-nn/Cargo.toml +++ b/crates/wasi-nn/Cargo.toml @@ -22,7 +22,7 @@ wasmtime-wasi = { path = "../wasi", version = "0.26.0" } wiggle = { path = "../wiggle", version = "0.26.0" } # These dependencies are necessary for the wasi-nn implementation: -openvino = "0.1.5" +openvino = { version = "0.3.1", features = ["runtime-linking"] } thiserror = "1.0" [build-dependencies] diff --git a/crates/wasi-nn/build.rs b/crates/wasi-nn/build.rs index 189b9513a6..535d0e0f80 100644 --- a/crates/wasi-nn/build.rs +++ b/crates/wasi-nn/build.rs @@ -1,11 +1,9 @@ //! This build script: //! - has the configuration necessary for the wiggle and witx macros. - -use std::path::PathBuf; - fn main() { // This is necessary for Wiggle/Witx macros. - let wasi_root = PathBuf::from("./spec").canonicalize().unwrap(); + let cwd = std::env::current_dir().unwrap(); + let wasi_root = cwd.join("spec"); println!("cargo:rustc-env=WASI_ROOT={}", wasi_root.display()); // Also automatically rebuild if the Witx files change diff --git a/crates/wasi-nn/src/ctx.rs b/crates/wasi-nn/src/ctx.rs index f180f113fb..dbe93c7224 100644 --- a/crates/wasi-nn/src/ctx.rs +++ b/crates/wasi-nn/src/ctx.rs @@ -2,7 +2,7 @@ //! wasi-nn API. use crate::r#impl::UsageError; use crate::witx::types::{Graph, GraphExecutionContext}; -use openvino::InferenceError; +use openvino::{InferenceError, SetupError}; use std::cell::RefCell; use std::collections::HashMap; use std::hash::Hash; @@ -14,8 +14,10 @@ use wiggle::GuestError; pub enum WasiNnError { #[error("guest error")] GuestError(#[from] GuestError), - #[error("openvino error")] - OpenvinoError(#[from] InferenceError), + #[error("openvino inference error")] + OpenvinoInferenceError(#[from] InferenceError), + #[error("openvino setup error")] + OpenvinoSetupError(#[from] SetupError), #[error("usage error")] UsageError(#[from] UsageError), } @@ -74,7 +76,7 @@ impl ExecutionContext { /// Capture the state necessary for calling into `openvino`. pub struct Ctx { - pub(crate) core: openvino::Core, + pub(crate) core: Option, pub(crate) graphs: Table, pub(crate) executions: Table, } @@ -83,7 +85,7 @@ impl Ctx { /// Make a new `WasiNnCtx` with the default settings. pub fn new() -> WasiNnResult { Ok(Self { - core: openvino::Core::new(None)?, + core: Option::default(), graphs: Table::default(), executions: Table::default(), }) diff --git a/crates/wasi-nn/src/impl.rs b/crates/wasi-nn/src/impl.rs index e18caafdc2..47fd323fdb 100644 --- a/crates/wasi-nn/src/impl.rs +++ b/crates/wasi-nn/src/impl.rs @@ -12,6 +12,8 @@ use wiggle::GuestPtr; #[derive(Debug, Error)] pub enum UsageError { + #[error("Invalid context; has the load function been called?")] + InvalidContext, #[error("Only OpenVINO's IR is currently supported, passed encoding: {0:?}")] InvalidEncoding(GraphEncoding), #[error("OpenVINO expects only two buffers (i.e. [ir, weights]), passed: {0}")] @@ -34,9 +36,21 @@ impl<'a> WasiEphemeralNn for WasiNnCtx { if encoding != GraphEncoding::Openvino { return Err(UsageError::InvalidEncoding(encoding).into()); } + if builders.len() != 2 { return Err(UsageError::InvalidNumberOfBuilders(builders.len()).into()); } + + // Construct the context if none is present; this is done lazily (i.e. upon actually loading + // a model) because it may fail to find and load the OpenVINO libraries. The laziness limits + // the extent of the error only to wasi-nn users, not all WASI users. + if self.ctx.borrow().core.is_none() { + self.ctx + .borrow_mut() + .core + .replace(openvino::Core::new(None)?); + } + let builders = builders.as_ptr(); let xml = builders.read()?.as_slice()?; let weights = builders.add(1)?.read()?.as_slice()?; @@ -44,11 +58,15 @@ impl<'a> WasiEphemeralNn for WasiNnCtx { .ctx .borrow_mut() .core + .as_mut() + .ok_or(UsageError::InvalidContext)? .read_network_from_buffer(&xml, &weights)?; let executable_graph = self .ctx .borrow_mut() .core + .as_mut() + .ok_or(UsageError::InvalidContext)? .load_network(&graph, map_execution_target_to_string(target))?; let id = self .ctx @@ -94,7 +112,7 @@ impl<'a> WasiEphemeralNn for WasiNnCtx { .dimensions .as_slice()? .iter() - .map(|d| *d as u64) + .map(|d| *d as usize) .collect::>(); let precision = match tensor.type_ { TensorType::F16 => Precision::FP16, diff --git a/crates/wasi-nn/src/witx.rs b/crates/wasi-nn/src/witx.rs index 32cc0167d8..8244f99644 100644 --- a/crates/wasi-nn/src/witx.rs +++ b/crates/wasi-nn/src/witx.rs @@ -14,7 +14,8 @@ impl<'a> types::UserErrorConversion for WasiNnCtx { fn nn_errno_from_wasi_nn_error(&self, e: WasiNnError) -> Result { eprintln!("Host error: {:?}", e); match e { - WasiNnError::OpenvinoError(_) => unimplemented!(), + WasiNnError::OpenvinoSetupError(_) => unimplemented!(), + WasiNnError::OpenvinoInferenceError(_) => unimplemented!(), WasiNnError::GuestError(_) => unimplemented!(), WasiNnError::UsageError(_) => unimplemented!(), } diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 43bd809c84..f133fa787b 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -319,6 +319,10 @@ impl HostFuncMap { fn async_required(&self) -> bool { self.funcs.values().any(|f| f.1) } + + fn iter(&self) -> impl Iterator { + self.funcs.values().map(|v| &*v.0) + } } macro_rules! generate_wrap_async_host_func { @@ -379,9 +383,6 @@ pub struct Config { pub(crate) max_wasm_stack: usize, pub(crate) features: WasmFeatures, pub(crate) wasm_backtrace_details_env_used: bool, - pub(crate) max_instances: usize, - pub(crate) max_tables: usize, - pub(crate) max_memories: usize, #[cfg(feature = "async")] pub(crate) async_stack_size: usize, host_funcs: HostFuncMap, @@ -418,9 +419,6 @@ impl Config { max_wasm_stack: 1 << 20, wasm_backtrace_details_env_used: false, features: WasmFeatures::default(), - max_instances: 10_000, - max_tables: 10_000, - max_memories: 10_000, #[cfg(feature = "async")] async_stack_size: 2 << 20, host_funcs: HostFuncMap::new(), @@ -1192,39 +1190,6 @@ impl Config { self } - /// Configures the maximum number of instances which can be created within - /// this `Store`. - /// - /// Instantiation will fail with an error if this limit is exceeded. - /// - /// This value defaults to 10,000. - pub fn max_instances(&mut self, instances: usize) -> &mut Self { - self.max_instances = instances; - self - } - - /// Configures the maximum number of tables which can be created within - /// this `Store`. - /// - /// Instantiation will fail with an error if this limit is exceeded. - /// - /// This value defaults to 10,000. - pub fn max_tables(&mut self, tables: usize) -> &mut Self { - self.max_tables = tables; - self - } - - /// Configures the maximum number of memories which can be created within - /// this `Store`. - /// - /// Instantiation will fail with an error if this limit is exceeded. - /// - /// This value defaults to 10,000. - pub fn max_memories(&mut self, memories: usize) -> &mut Self { - self.max_memories = memories; - self - } - /// Defines a host function for the [`Config`] for the given callback. /// /// Use [`Store::get_host_func`](crate::Store::get_host_func) to get a [`Func`](crate::Func) representing the function. @@ -1318,6 +1283,10 @@ impl Config { for_each_function_signature!(generate_wrap_async_host_func); + pub(crate) fn host_funcs(&self) -> impl Iterator { + self.host_funcs.iter() + } + pub(crate) fn get_host_func(&self, module: &str, name: &str) -> Option<&HostFunc> { self.host_funcs.get(module, name) } diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index f7d9c4eaa4..8876eb042d 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -1,10 +1,41 @@ +use crate::signatures::{SignatureCollection, SignatureRegistry}; use crate::Config; use anyhow::Result; +use std::collections::HashMap; use std::sync::Arc; #[cfg(feature = "cache")] use wasmtime_cache::CacheConfig; use wasmtime_jit::Compiler; -use wasmtime_runtime::{debug_builtins, InstanceAllocator}; +use wasmtime_runtime::{debug_builtins, InstanceAllocator, InstanceHandle, VMCallerCheckedAnyfunc}; + +/// This is used as a Send+Sync wrapper around two data structures relating to +/// host functions defined on `Config`: +/// +/// * `anyfuncs` - this stores a mapping between the host function instance and +/// a `VMCallerCheckedAnyfunc` that can be used as the function's value in Wasmtime's ABI. +/// The address of the anyfunc needs to be stable, thus the boxed value. +/// +/// * `signatures` - this stores the collection of shared signatures registered for every +/// usable host functions with this engine. +struct EngineHostFuncs { + anyfuncs: HashMap>, + signatures: SignatureCollection, +} + +impl EngineHostFuncs { + fn new(registry: &SignatureRegistry) -> Self { + Self { + anyfuncs: HashMap::new(), + signatures: SignatureCollection::new(registry), + } + } +} + +// This is safe for send and sync as it is read-only once the +// engine is constructed and the host functions live with the config, +// which the engine keeps a strong reference to. +unsafe impl Send for EngineHostFuncs {} +unsafe impl Sync for EngineHostFuncs {} /// An `Engine` which is a global context for compilation and management of wasm /// modules. @@ -37,6 +68,8 @@ struct EngineInner { config: Config, compiler: Compiler, allocator: Box, + signatures: SignatureRegistry, + host_funcs: EngineHostFuncs, } impl Engine { @@ -46,11 +79,29 @@ impl Engine { debug_builtins::ensure_exported(); config.validate()?; let allocator = config.build_allocator()?; + let registry = SignatureRegistry::new(); + let mut host_funcs = EngineHostFuncs::new(®istry); + + // Register all the host function signatures with the collection + for func in config.host_funcs() { + let sig = host_funcs + .signatures + .register(func.ty.as_wasm_func_type(), func.trampoline); + + // Cloning the instance handle is safe as host functions outlive the engine + host_funcs.anyfuncs.insert( + unsafe { func.instance.clone() }, + Box::new(func.anyfunc(sig)), + ); + } + Ok(Engine { inner: Arc::new(EngineInner { config: config.clone(), compiler: config.build_compiler(allocator.as_ref()), allocator, + signatures: registry, + host_funcs, }), }) } @@ -79,6 +130,25 @@ impl Engine { Arc::ptr_eq(&a.inner, &b.inner) } + pub(crate) fn signatures(&self) -> &SignatureRegistry { + &self.inner.signatures + } + + pub(crate) fn host_func_signatures(&self) -> &SignatureCollection { + &self.inner.host_funcs.signatures + } + + pub(crate) fn host_func_anyfunc( + &self, + instance: &InstanceHandle, + ) -> Option<&VMCallerCheckedAnyfunc> { + self.inner + .host_funcs + .anyfuncs + .get(instance) + .map(AsRef::as_ref) + } + /// Ahead-of-time (AOT) compiles a WebAssembly module. /// /// The `bytes` provided must be in one of two formats: @@ -90,8 +160,9 @@ impl Engine { /// Note that the `wat` feature is enabled by default. /// /// This method may be used to compile a module for use with a different target - /// host. The output of this method may be used with [`Module::new`](crate::Module::new) - /// on hosts compatible with the [`Config`] associated with this [`Engine`]. + /// host. The output of this method may be used with + /// [`Module::deserialize`](crate::Module::deserialize) on hosts compatible + /// with the [`Config`] associated with this [`Engine`]. /// /// The output of this method is safe to send to another host machine for later /// execution. As the output is already a compiled module, translation and code diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 82805e2c11..0319ee0002 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -1,4 +1,4 @@ -use crate::{sig_registry::SignatureRegistry, trampoline::StoreInstanceHandle}; +use crate::trampoline::StoreInstanceHandle; use crate::{Config, Extern, FuncType, Store, Trap, Val, ValType}; use anyhow::{bail, Context as _, Result}; use smallvec::{smallvec, SmallVec}; @@ -22,9 +22,9 @@ use wasmtime_runtime::{ /// This differs from `Func` in that it is not associated with a `Store`. /// Host functions are associated with a `Config`. pub(crate) struct HostFunc { - ty: FuncType, - instance: InstanceHandle, - trampoline: VMTrampoline, + pub ty: FuncType, + pub instance: InstanceHandle, + pub trampoline: VMTrampoline, } impl HostFunc { @@ -73,6 +73,23 @@ impl HostFunc { } } + /// Gets a caller-checked anyfunc for this host function given a shared signature index. + /// + /// The shared signature index must have been registered for the signature of + /// this host function. + pub fn anyfunc(&self, sig: VMSharedSignatureIndex) -> VMCallerCheckedAnyfunc { + let mut anyfunc = match self + .instance + .lookup_by_declaration(&EntityIndex::Function(FuncIndex::from_u32(0))) + { + wasmtime_runtime::Export::Function(f) => unsafe { f.anyfunc.as_ref() }.clone(), + _ => unreachable!(), + }; + + anyfunc.type_index = sig; + anyfunc + } + /// Converts a `HostFunc` to a `Func`. /// /// # Safety @@ -88,11 +105,11 @@ impl HostFunc { }; let export = ExportFunction { - anyfunc: std::ptr::NonNull::new_unchecked(store.get_host_anyfunc( - &self.instance, - &self.ty, - self.trampoline, - )), + anyfunc: store + .engine() + .host_func_anyfunc(&self.instance) + .unwrap() + .into(), }; Func { @@ -408,13 +425,9 @@ impl Func { Func::invoke(&store, &ty_clone, caller_vmctx, values_vec, &func) }); - let (instance, trampoline) = crate::trampoline::create_function( - &ty, - func, - store.engine().config(), - Some(&mut store.signatures().borrow_mut()), - ) - .expect("failed to create function"); + let (instance, trampoline) = + crate::trampoline::create_function(&ty, func, store.engine().config(), Some(store)) + .expect("failed to create function"); let idx = EntityIndex::Function(FuncIndex::from_u32(0)); let (instance, export) = match instance.lookup_by_declaration(&idx) { @@ -734,7 +747,7 @@ impl Func { /// # } /// ``` pub fn wrap(store: &Store, func: impl IntoFunc) -> Func { - let (_, instance, trampoline) = func.into_func(Some(&mut store.signatures().borrow_mut())); + let (_, instance, trampoline) = func.into_func(Some(store)); let (instance, export) = unsafe { let idx = EntityIndex::Function(FuncIndex::from_u32(0)); @@ -759,35 +772,26 @@ impl Func { /// Returns the underlying wasm type that this `Func` has. pub fn ty(&self) -> FuncType { - // Signatures should always be registered in the store's registry of + // Signatures should always be registered in the engine's registry of // shared signatures, so we should be able to unwrap safely here. - let signatures = self.instance.store.signatures().borrow(); - let (wft, _) = signatures - .lookup_shared(self.sig_index()) - .expect("signature should be registered"); - - // This is only called with `Export::Function`, and since it's coming - // from wasmtime_runtime itself we should support all the types coming - // out of it, so assert such here. - FuncType::from_wasm_func_type(&wft) + FuncType::from_wasm_func_type( + self.instance + .store + .engine() + .signatures() + .lookup_type(self.sig_index()) + .expect("signature should be registered"), + ) } /// Returns the number of parameters that this function takes. pub fn param_arity(&self) -> usize { - let signatures = self.instance.store.signatures().borrow(); - let (sig, _) = signatures - .lookup_shared(self.sig_index()) - .expect("signature should be registered"); - sig.params.len() + self.ty().params().len() } /// Returns the number of results this function produces. pub fn result_arity(&self) -> usize { - let signatures = self.instance.store.signatures().borrow(); - let (sig, _) = signatures - .lookup_shared(self.sig_index()) - .expect("signature should be registered"); - sig.returns.len() + self.ty().results().len() } /// Invokes this function with the `params` given, returning the results and @@ -907,21 +911,12 @@ impl Func { } pub(crate) unsafe fn from_wasmtime_function(export: &ExportFunction, store: &Store) -> Self { - // Each function signature in a module should have a trampoline stored - // on that module as well, so unwrap the result here since otherwise - // it's a bug in wasmtime. let anyfunc = export.anyfunc.as_ref(); - let trampoline = store - .signatures() - .borrow() - .lookup_shared(anyfunc.type_index) - .expect("failed to retrieve trampoline from module") - .1; Func { instance: store.existing_vmctx(anyfunc.vmctx), export: export.clone(), - trampoline, + trampoline: store.lookup_trampoline(&*anyfunc), } } @@ -1542,10 +1537,7 @@ for_each_function_signature!(impl_host_abi); /// as an implementation detail of this crate. pub trait IntoFunc { #[doc(hidden)] - fn into_func( - self, - registry: Option<&mut SignatureRegistry>, - ) -> (FuncType, InstanceHandle, VMTrampoline); + fn into_func(self, store: Option<&Store>) -> (FuncType, InstanceHandle, VMTrampoline); } /// A structure representing the *caller's* context when creating a function @@ -1658,12 +1650,12 @@ macro_rules! impl_into_func { $($args: WasmTy,)* R: WasmRet, { - fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { + fn into_func(self, store: Option<&Store>) -> (FuncType, InstanceHandle, VMTrampoline) { let f = move |_: Caller<'_>, $($args:$args),*| { self($($args),*) }; - f.into_func(registry) + f.into_func(store) } } @@ -1674,7 +1666,7 @@ macro_rules! impl_into_func { $($args: WasmTy,)* R: WasmRet, { - fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { + fn into_func(self, store: Option<&Store>) -> (FuncType, InstanceHandle, VMTrampoline) { /// This shim is called by Wasm code, constructs a `Caller`, /// calls the wrapped host function, and returns the translated /// result back to Wasm. @@ -1807,10 +1799,10 @@ macro_rules! impl_into_func { let trampoline = host_trampoline::<$($args,)* R>; - // If not given a registry, use a default signature index that is guaranteed to trap - // if the function is called indirectly without first being associated with a store (a bug condition). - let shared_signature_id = registry - .map(|r| r.register(ty.as_wasm_func_type(), trampoline)) + // If not given a store, use a default signature index that is guaranteed to trap. + // If the function is called indirectly without first being associated with a store (a bug condition). + let shared_signature_id = store + .map(|s| s.signatures().borrow_mut().register(ty.as_wasm_func_type(), trampoline)) .unwrap_or(VMSharedSignatureIndex::default()); let instance = unsafe { diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index c0a1d834da..107810f84b 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -207,7 +207,7 @@ unsafe impl WasmTy for Option { unsafe { store .externref_activations_table() - .insert_with_gc(x.inner, store.stack_map_registry()); + .insert_with_gc(x.inner, store.module_info_lookup()); } abi } else { diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 01765d02bd..7a30f850aa 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -13,9 +13,9 @@ use wasmtime_environ::wasm::{ }; use wasmtime_environ::Initializer; use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstantiationError, RuntimeInstance, StackMapRegistry, - VMContext, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, - VMMemoryImport, VMTableImport, + Imports, InstanceAllocationRequest, InstantiationError, RuntimeInstance, VMContext, + VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, + VMTableImport, }; /// An instantiated WebAssembly module. @@ -362,6 +362,7 @@ impl<'a> Instantiator<'a> { let expected_ty = self.cur.module.compiled_module().module().type_of(*index); matching::MatchCx { + signatures: self.cur.module.signatures(), types: self.cur.module.types(), store: self.store, } @@ -505,29 +506,26 @@ impl<'a> Instantiator<'a> { fn instantiate_raw(&self) -> Result { let compiled_module = self.cur.module.compiled_module(); - // Register the module just before instantiation to ensure we have a - // trampoline registered for every signature and to preserve the module's - // compiled JIT code within the `Store`. - self.store.register_module(&self.cur.module); + // Register the module just before instantiation to ensure we keep the module + // properly referenced while in use by the store. + self.store.modules().borrow_mut().register(&self.cur.module); unsafe { let engine = self.store.engine(); let allocator = engine.allocator(); - let signatures = self.store.signatures().borrow(); - let signatures = signatures.lookup_table(&self.cur.module); let instance = allocator.allocate(InstanceAllocationRequest { module: compiled_module.module().clone(), finished_functions: compiled_module.finished_functions(), imports: self.cur.build(), - shared_signatures: (&signatures).into(), + shared_signatures: self.cur.module.signatures().as_module_map().into(), host_state: Box::new(()), interrupts: self.store.interrupts(), externref_activations_table: self.store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_registry: self.store.stack_map_registry() as *const StackMapRegistry - as *mut _, + module_info_lookup: Some(self.store.module_info_lookup()), + limiter: self.store.limiter().as_ref(), })?; // After we've created the `InstanceHandle` we still need to run diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index d843b9db43..332f63b4d2 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -282,13 +282,13 @@ mod func; mod config; mod engine; mod externals; -mod frame_info; mod instance; +mod limits; mod linker; mod memory; mod module; mod r#ref; -mod sig_registry; +mod signatures; mod store; mod trampoline; mod trap; @@ -298,12 +298,12 @@ mod values; pub use crate::config::*; pub use crate::engine::*; pub use crate::externals::*; -pub use crate::frame_info::{FrameInfo, FrameSymbol}; pub use crate::func::*; pub use crate::instance::Instance; +pub use crate::limits::*; pub use crate::linker::*; pub use crate::memory::*; -pub use crate::module::Module; +pub use crate::module::{FrameInfo, FrameSymbol, Module}; pub use crate::r#ref::ExternRef; pub use crate::store::*; pub use crate::trap::*; diff --git a/crates/wasmtime/src/limits.rs b/crates/wasmtime/src/limits.rs new file mode 100644 index 0000000000..fc65aa2e90 --- /dev/null +++ b/crates/wasmtime/src/limits.rs @@ -0,0 +1,208 @@ +pub(crate) const DEFAULT_INSTANCE_LIMIT: usize = 10000; +pub(crate) const DEFAULT_TABLE_LIMIT: usize = 10000; +pub(crate) const DEFAULT_MEMORY_LIMIT: usize = 10000; + +/// Used by hosts to limit resource consumption of instances at runtime. +/// +/// [`Store::new_with_limits`](crate::Store::new_with_limits) can be used +/// with a resource limiter to take into account non-WebAssembly resource +/// usage to determine if a linear memory or table should be grown. +pub trait ResourceLimiter { + /// Notifies the resource limiter that an instance's linear memory has been requested to grow. + /// + /// * `current` is the current size of the linear memory in WebAssembly page units. + /// * `desired` is the desired size of the linear memory in WebAssembly page units. + /// * `maximum` is either the linear memory's maximum or a maximum from an instance allocator, + /// also in WebAssembly page units. A value of `None` indicates that the linear memory is + /// unbounded. + /// + /// This function should return `true` to indicate that the growing operation is permitted or + /// `false` if not permitted. + /// + /// Note that this function will be called even when the desired count exceeds the given maximum. + /// + /// Returning `true` when a maximum has been exceeded will have no effect as the linear memory + /// will not be grown. + fn memory_growing(&self, current: u32, desired: u32, maximum: Option) -> bool; + + /// Notifies the resource limiter that an instance's table has been requested to grow. + /// + /// * `current` is the current number of elements in the table. + /// * `desired` is the desired number of elements in the table. + /// * `maximum` is either the table's maximum or a maximum from an instance allocator, + /// A value of `None` indicates that the table is unbounded. + /// + /// This function should return `true` to indicate that the growing operation is permitted or + /// `false` if not permitted. + /// + /// Note that this function will be called even when the desired count exceeds the given maximum. + /// + /// Returning `true` when a maximum has been exceeded will have no effect as the table will + /// not be grown. + fn table_growing(&self, current: u32, desired: u32, maximum: Option) -> bool; + + /// The maximum number of instances that can be created for a [`Store`](crate::Store). + /// + /// Module instantiation will fail if this limit is exceeded. + /// + /// This value defaults to 10,000. + fn instances(&self) -> usize { + DEFAULT_INSTANCE_LIMIT + } + + /// The maximum number of tables that can be created for a [`Store`](crate::Store). + /// + /// Module instantiation will fail if this limit is exceeded. + /// + /// This value defaults to 10,000. + fn tables(&self) -> usize { + DEFAULT_TABLE_LIMIT + } + + /// The maximum number of linear memories that can be created for a [`Store`](crate::Store). + /// + /// Instantiation will fail with an error if this limit is exceeded. + /// + /// This value defaults to 10,000. + fn memories(&self) -> usize { + DEFAULT_MEMORY_LIMIT + } +} + +pub(crate) struct ResourceLimiterProxy(pub T); + +impl wasmtime_runtime::ResourceLimiter for ResourceLimiterProxy { + fn memory_growing(&self, current: u32, desired: u32, maximum: Option) -> bool { + self.0.memory_growing(current, desired, maximum) + } + + fn table_growing(&self, current: u32, desired: u32, maximum: Option) -> bool { + self.0.table_growing(current, desired, maximum) + } + + fn instances(&self) -> usize { + self.0.instances() + } + + fn tables(&self) -> usize { + self.0.tables() + } + + fn memories(&self) -> usize { + self.0.memories() + } +} + +/// Used to build [`StoreLimits`]. +pub struct StoreLimitsBuilder(StoreLimits); + +impl StoreLimitsBuilder { + /// Creates a new [`StoreLimitsBuilder`]. + pub fn new() -> Self { + Self(StoreLimits::default()) + } + + /// The maximum number of WebAssembly pages a linear memory can grow to. + /// + /// Growing a linear memory beyond this limit will fail. + /// + /// By default, linear memory pages will not be limited. + pub fn memory_pages(mut self, limit: u32) -> Self { + self.0.memory_pages = Some(limit); + self + } + + /// The maximum number of elements in a table. + /// + /// Growing a table beyond this limit will fail. + /// + /// By default, table elements will not be limited. + pub fn table_elements(mut self, limit: u32) -> Self { + self.0.table_elements = Some(limit); + self + } + + /// The maximum number of instances that can be created for a [`Store`](crate::Store). + /// + /// Module instantiation will fail if this limit is exceeded. + /// + /// This value defaults to 10,000. + pub fn instances(mut self, limit: usize) -> Self { + self.0.instances = limit; + self + } + + /// The maximum number of tables that can be created for a [`Store`](crate::Store). + /// + /// Module instantiation will fail if this limit is exceeded. + /// + /// This value defaults to 10,000. + pub fn tables(mut self, tables: usize) -> Self { + self.0.tables = tables; + self + } + + /// The maximum number of linear memories that can be created for a [`Store`](crate::Store). + /// + /// Instantiation will fail with an error if this limit is exceeded. + /// + /// This value defaults to 10,000. + pub fn memories(mut self, memories: usize) -> Self { + self.0.memories = memories; + self + } + + /// Consumes this builder and returns the [`StoreLimits`]. + pub fn build(self) -> StoreLimits { + self.0 + } +} + +/// Provides limits for a [`Store`](crate::Store). +pub struct StoreLimits { + memory_pages: Option, + table_elements: Option, + instances: usize, + tables: usize, + memories: usize, +} + +impl Default for StoreLimits { + fn default() -> Self { + Self { + memory_pages: None, + table_elements: None, + instances: DEFAULT_INSTANCE_LIMIT, + tables: DEFAULT_TABLE_LIMIT, + memories: DEFAULT_MEMORY_LIMIT, + } + } +} + +impl ResourceLimiter for StoreLimits { + fn memory_growing(&self, _current: u32, desired: u32, _maximum: Option) -> bool { + match self.memory_pages { + Some(limit) if desired > limit => false, + _ => true, + } + } + + fn table_growing(&self, _current: u32, desired: u32, _maximum: Option) -> bool { + match self.table_elements { + Some(limit) if desired > limit => false, + _ => true, + } + } + + fn instances(&self) -> usize { + self.instances + } + + fn tables(&self) -> usize { + self.tables + } + + fn memories(&self) -> usize { + self.memories + } +} diff --git a/crates/wasmtime/src/memory.rs b/crates/wasmtime/src/memory.rs index 39b6349274..57aee031ff 100644 --- a/crates/wasmtime/src/memory.rs +++ b/crates/wasmtime/src/memory.rs @@ -262,7 +262,7 @@ impl Memory { /// let store = Store::new(&engine); /// /// let memory_ty = MemoryType::new(Limits::new(1, None)); - /// let memory = Memory::new(&store, memory_ty); + /// let memory = Memory::new(&store, memory_ty)?; /// /// let module = Module::new(&engine, "(module (memory (import \"\" \"\") 1))")?; /// let instance = Instance::new(&store, &module, &[memory.into()])?; @@ -270,13 +270,12 @@ impl Memory { /// # Ok(()) /// # } /// ``` - pub fn new(store: &Store, ty: MemoryType) -> Memory { - let (instance, wasmtime_export) = - generate_memory_export(store, &ty).expect("generated memory"); - Memory { + pub fn new(store: &Store, ty: MemoryType) -> Result { + let (instance, wasmtime_export) = generate_memory_export(store, &ty)?; + Ok(Memory { instance, wasmtime_export, - } + }) } /// Returns the underlying type of this memory. @@ -454,7 +453,7 @@ impl Memory { .memory_index(unsafe { &*self.wasmtime_export.definition }); self.instance .memory_grow(index, delta) - .ok_or_else(|| anyhow!("failed to grow memory")) + .ok_or_else(|| anyhow!("failed to grow memory by `{}`", delta)) } pub(crate) unsafe fn from_wasmtime_memory( @@ -500,6 +499,10 @@ pub unsafe trait LinearMemory { /// Returns the number of allocated wasm pages. fn size(&self) -> u32; + /// Returns the maximum number of pages the memory can grow to. + /// Returns `None` if the memory is unbounded. + fn maximum(&self) -> Option; + /// Grow memory by the specified amount of wasm pages. /// /// Returns `None` if memory can't be grown by the specified amount @@ -568,7 +571,7 @@ mod tests { .dynamic_memory_guard_size(0); let store = Store::new(&Engine::new(&cfg).unwrap()); let ty = MemoryType::new(Limits::new(1, None)); - let mem = Memory::new(&store, ty); + let mem = Memory::new(&store, ty).unwrap(); assert_eq!(mem.wasmtime_export.memory.offset_guard_size, 0); match mem.wasmtime_export.memory.style { wasmtime_environ::MemoryStyle::Dynamic => {} diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 8b460207cf..4964189d7b 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -1,4 +1,7 @@ -use crate::types::{ExportType, ExternType, ImportType}; +use crate::{ + signatures::SignatureCollection, + types::{ExportType, ExternType, ImportType}, +}; use crate::{Engine, ModuleType}; use anyhow::{bail, Context, Result}; use std::fs; @@ -11,8 +14,10 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::ModuleIndex; use wasmtime_jit::{CompilationArtifacts, CompiledModule, TypeTables}; +mod registry; mod serialization; +pub use registry::{FrameInfo, FrameSymbol, GlobalModuleRegistry, ModuleRegistry}; pub use serialization::SerializedModule; /// A compiled WebAssembly module, ready to be instantiated. @@ -102,6 +107,8 @@ struct ModuleInner { /// Type information of this module and all `artifact_upvars` compiled /// modules. types: Arc, + /// Registered shared signature for the module. + signatures: Arc, } impl Module { @@ -114,9 +121,6 @@ impl Module { /// This is only supported when the `wat` feature of this crate is enabled. /// If this is supplied then the text format will be parsed before validation. /// Note that the `wat` feature is enabled by default. - /// * A module serialized with [`Module::serialize`]. - /// * A module compiled with [`Engine::precompile_module`] or the - /// `wasmtime compile` command. /// /// The data for the wasm module must be loaded in-memory if it's present /// elsewhere, for example on disk. This requires that the entire binary is @@ -175,11 +179,6 @@ impl Module { /// ``` pub fn new(engine: &Engine, bytes: impl AsRef<[u8]>) -> Result { let bytes = bytes.as_ref(); - - if let Some(module) = SerializedModule::from_bytes(bytes)? { - return module.into_module(engine); - } - #[cfg(feature = "wat")] let bytes = wat::parse_bytes(bytes)?; Self::from_binary(engine, &bytes) @@ -251,10 +250,10 @@ impl Module { /// data. /// /// This is similar to [`Module::new`] except that it requires that the - /// `binary` input is a WebAssembly binary or a compiled module, the - /// text format is not supported by this function. It's generally - /// recommended to use [`Module::new`], but if it's required to not - /// support the text format this function can be used instead. + /// `binary` input is a WebAssembly binary, the text format is not supported + /// by this function. It's generally recommended to use [`Module::new`], but + /// if it's required to not support the text format this function can be + /// used instead. /// /// # Examples /// @@ -279,10 +278,6 @@ impl Module { /// # } /// ``` pub fn from_binary(engine: &Engine, binary: &[u8]) -> Result { - if let Some(module) = SerializedModule::from_bytes(binary)? { - return module.into_module(engine); - } - // Check to see that the config's target matches the host let target = engine.config().isa_flags.triple(); if *target != target_lexicon::Triple::host() { @@ -313,25 +308,138 @@ impl Module { } }; - let mut modules = CompiledModule::from_artifacts_list( + let modules = CompiledModule::from_artifacts_list( artifacts, engine.compiler().isa(), &*engine.config().profiler, )?; + + Self::from_parts(engine, modules, main_module, Arc::new(types), &[]) + } + + /// Deserializes an in-memory compiled module previously created with + /// [`Module::serialize`] or [`Engine::precompile_module`]. + /// + /// This function will deserialize the binary blobs emitted by + /// [`Module::serialize`] and [`Engine::precompile_module`] back into an + /// in-memory [`Module`] that's ready to be instantiated. + /// + /// # Unsafety + /// + /// This function is marked as `unsafe` because if fed invalid input or used + /// improperly this could lead to memory safety vulnerabilities. This method + /// should not, for example, be exposed to arbitrary user input. + /// + /// The structure of the binary blob read here is only lightly validated + /// internally in `wasmtime`. This is intended to be an efficient + /// "rehydration" for a [`Module`] which has very few runtime checks beyond + /// deserialization. Arbitrary input could, for example, replace valid + /// compiled code with any other valid compiled code, meaning that this can + /// trivially be used to execute arbitrary code otherwise. + /// + /// For these reasons this function is `unsafe`. This function is only + /// designed to receive the previous input from [`Module::serialize`] and + /// [`Engine::precompile_module`]. If the exact output of those functions + /// (unmodified) is passed to this function then calls to this function can + /// be considered safe. It is the caller's responsibility to provide the + /// guarantee that only previously-serialized bytes are being passed in + /// here. + /// + /// Note that this function is designed to be safe receiving output from + /// *any* compiled version of `wasmtime` itself. This means that it is safe + /// to feed output from older versions of Wasmtime into this function, in + /// addition to newer versions of wasmtime (from the future!). These inputs + /// will deterministically and safely produce an `Err`. This function only + /// successfully accepts inputs from the same version of `wasmtime`, but the + /// safety guarantee only applies to externally-defined blobs of bytes, not + /// those defined by any version of wasmtime. (this means that if you cache + /// blobs across versions of wasmtime you can be safely guaranteed that + /// future versions of wasmtime will reject old cache entries). + pub unsafe fn deserialize(engine: &Engine, bytes: impl AsRef<[u8]>) -> Result { + let module = SerializedModule::from_bytes(bytes.as_ref())?; + module.into_module(engine) + } + + fn from_parts( + engine: &Engine, + mut modules: Vec>, + main_module: usize, + types: Arc, + module_upvars: &[serialization::SerializedModuleUpvar], + ) -> Result { + // Validate the module can be used with the current allocator + engine.allocator().validate(modules[main_module].module())?; + + let signatures = Arc::new(SignatureCollection::new_for_module( + engine.signatures(), + &types.wasm_signatures, + modules.iter().flat_map(|m| m.trampolines().iter().cloned()), + )); + let module = modules.remove(main_module); - // Validate the module can be used with the current allocator - engine.allocator().validate(module.module())?; + let module_upvars = module_upvars + .iter() + .map(|m| { + mk( + engine, + &modules, + &types, + m.index, + &m.artifact_upvars, + &m.module_upvars, + &signatures, + ) + }) + .collect::>>()?; - Ok(Module { + return Ok(Self { inner: Arc::new(ModuleInner { engine: engine.clone(), + types, module, - types: Arc::new(types), artifact_upvars: modules, - module_upvars: Vec::new(), + module_upvars, + signatures, }), - }) + }); + + fn mk( + engine: &Engine, + artifacts: &[Arc], + types: &Arc, + module_index: usize, + artifact_upvars: &[usize], + module_upvars: &[serialization::SerializedModuleUpvar], + signatures: &Arc, + ) -> Result { + Ok(Module { + inner: Arc::new(ModuleInner { + engine: engine.clone(), + types: types.clone(), + module: artifacts[module_index].clone(), + artifact_upvars: artifact_upvars + .iter() + .map(|i| artifacts[*i].clone()) + .collect(), + module_upvars: module_upvars + .into_iter() + .map(|m| { + mk( + engine, + artifacts, + types, + m.index, + &m.artifact_upvars, + &m.module_upvars, + signatures, + ) + }) + .collect::>>()?, + signatures: signatures.clone(), + }), + }) + } } /// Validates `binary` input data as a WebAssembly binary given the @@ -416,8 +524,8 @@ impl Module { ) -> Module { Module { inner: Arc::new(ModuleInner { - types: self.types().clone(), - engine: self.engine().clone(), + types: self.inner.types.clone(), + engine: self.inner.engine.clone(), module: self.inner.artifact_upvars[artifact_index].clone(), artifact_upvars: artifact_upvars .iter() @@ -432,6 +540,7 @@ impl Module { wasmtime_environ::ModuleUpvar::Local(i) => modules[i].clone(), }) .collect(), + signatures: self.inner.signatures.clone(), }), } } @@ -448,6 +557,10 @@ impl Module { &self.inner.types } + pub(crate) fn signatures(&self) -> &Arc { + &self.inner.signatures + } + /// Looks up the module upvar value at the `index` specified. /// /// Note that this panics if `index` is out of bounds since this should diff --git a/crates/wasmtime/src/frame_info.rs b/crates/wasmtime/src/module/registry.rs similarity index 62% rename from crates/wasmtime/src/frame_info.rs rename to crates/wasmtime/src/module/registry.rs index 8bd326ebdb..6687119106 100644 --- a/crates/wasmtime/src/frame_info.rs +++ b/crates/wasmtime/src/module/registry.rs @@ -1,35 +1,38 @@ -use std::collections::BTreeMap; -use std::sync::Arc; -use std::sync::Mutex; -use wasmtime_environ::entity::EntityRef; -use wasmtime_environ::ir; -use wasmtime_environ::wasm::DefinedFuncIndex; -use wasmtime_environ::{FunctionAddressMap, TrapInformation}; -use wasmtime_jit::CompiledModule; +//! Implements a registry of modules for a store. -/// This is a structure that lives within a `Store` and retains information -/// about all modules registered with the `Store` via instantiation. -/// -/// "frame information" here refers to things like determining whether a -/// program counter is a wasm program counter, and additionally mapping program -/// counters to wasm filenames, modules, line numbers, etc. This store of -/// information lives as long as a `Store` lives since modules are never -/// unloaded today. -#[derive(Default)] -pub struct StoreFrameInfo { - /// An internal map that keeps track of backtrace frame information for - /// each module. - /// - /// This map is morally a map of ranges to a map of information for that - /// module. Each module is expected to reside in a disjoint section of - /// contiguous memory. No modules can overlap. - /// - /// The key of this map is the highest address in the module and the value - /// is the module's information, which also contains the start address. - ranges: BTreeMap, +use crate::{signatures::SignatureCollection, Module}; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; +use wasmtime_environ::{ + entity::EntityRef, + ir::{self, StackMap}, + wasm::DefinedFuncIndex, + FunctionAddressMap, TrapInformation, +}; +use wasmtime_jit::CompiledModule; +use wasmtime_runtime::{ModuleInfo, VMCallerCheckedAnyfunc, VMTrampoline}; + +lazy_static::lazy_static! { + static ref GLOBAL_MODULES: Mutex = Default::default(); } -impl StoreFrameInfo { +fn func_by_pc(module: &CompiledModule, pc: usize) -> Option<(DefinedFuncIndex, u32)> { + let (index, start, _) = module.func_by_pc(pc)?; + Some((index, (pc - start) as u32)) +} + +/// Used for registering modules with a store. +/// +/// The map is from the ending (exclusive) address for the module code to +/// the registered module. +/// +/// The `BTreeMap` is used to quickly locate a module based on a program counter value. +#[derive(Default)] +pub struct ModuleRegistry(BTreeMap>); + +impl ModuleRegistry { /// Fetches frame information about a program counter in a backtrace. /// /// Returns an object if this `pc` is known to some previously registered @@ -48,8 +51,14 @@ impl StoreFrameInfo { self.module(pc)?.lookup_trap_info(pc) } - fn module(&self, pc: usize) -> Option<&ModuleFrameInfo> { - let (end, info) = self.ranges.range(pc..).next()?; + /// Fetches information about a registered module given a program counter value. + pub fn lookup_module(&self, pc: usize) -> Option> { + self.module(pc) + .map(|m| -> Arc { m.clone() }) + } + + fn module(&self, pc: usize) -> Option<&Arc> { + let (end, info) = self.0.range(pc..).next()?; if pc < info.start || *end < pc { return None; } @@ -57,12 +66,13 @@ impl StoreFrameInfo { Some(info) } - /// Registers a new compiled module's frame information. - pub fn register(&mut self, module: &Arc) { - let (start, end) = module.code().range(); + /// Registers a new module with the registry. + pub fn register(&mut self, module: &Module) { + let compiled_module = module.compiled_module(); + let (start, end) = compiled_module.code().range(); // Ignore modules with no code or finished functions - if start == end || module.finished_functions().is_empty() { + if start == end || compiled_module.finished_functions().is_empty() { return; } @@ -70,44 +80,58 @@ impl StoreFrameInfo { // may be a valid PC value let end = end - 1; + // Ensure the module isn't already present in the registry + // This is expected when a module is instantiated multiple times in the same store + if let Some(m) = self.0.get(&end) { + assert_eq!(m.start, start); + return; + } + // Assert that this module's code doesn't collide with any other registered modules - if let Some((_, prev)) = self.ranges.range(end..).next() { + if let Some((_, prev)) = self.0.range(end..).next() { assert!(prev.start > end); } - if let Some((prev_end, _)) = self.ranges.range(..=start).next_back() { + + if let Some((prev_end, _)) = self.0.range(..=start).next_back() { assert!(*prev_end < start); } - let prev = self.ranges.insert( + let prev = self.0.insert( end, - ModuleFrameInfo { + Arc::new(RegisteredModule { start, - module: module.clone(), - }, + module: compiled_module.clone(), + signatures: module.signatures().clone(), + }), ); assert!(prev.is_none()); - GLOBAL_INFO.lock().unwrap().register(start, end, module); + GLOBAL_MODULES.lock().unwrap().register(start, end, module); + } + + /// Looks up a trampoline from an anyfunc. + pub fn lookup_trampoline(&self, anyfunc: &VMCallerCheckedAnyfunc) -> Option { + let module = self.module(anyfunc.func_ptr.as_ptr() as usize)?; + module.signatures.trampoline(anyfunc.type_index) } } -impl Drop for StoreFrameInfo { +impl Drop for ModuleRegistry { fn drop(&mut self) { - let mut info = GLOBAL_INFO.lock().unwrap(); - for end in self.ranges.keys() { + let mut info = GLOBAL_MODULES.lock().unwrap(); + for end in self.0.keys() { info.unregister(*end); } } } -/// Represents a module's frame information. -#[derive(Clone)] -pub struct ModuleFrameInfo { +struct RegisteredModule { start: usize, module: Arc, + signatures: Arc, } -impl ModuleFrameInfo { +impl RegisteredModule { /// Determines if the related module has unparsed debug information. pub fn has_unparsed_debuginfo(&self) -> bool { self.module.has_unparsed_debuginfo() @@ -118,9 +142,9 @@ impl ModuleFrameInfo { /// Returns an object if this `pc` is known to this module, or returns `None` /// if no information can be found. pub fn lookup_frame_info(&self, pc: usize) -> Option { - let (index, offset) = self.func(pc)?; - let (addr_map, _) = self.module.func_info(index); - let pos = Self::instr_pos(offset, addr_map); + let (index, offset) = func_by_pc(&self.module, pc)?; + let info = self.module.func_info(index); + let pos = Self::instr_pos(offset, &info.address_map); // In debug mode for now assert that we found a mapping for `pc` within // the function, because otherwise something is buggy along the way and @@ -129,8 +153,8 @@ impl ModuleFrameInfo { debug_assert!(pos.is_some(), "failed to find instruction for {:x}", pc); let instr = match pos { - Some(pos) => addr_map.instructions[pos].srcloc, - None => addr_map.start_srcloc, + Some(pos) => info.address_map.instructions[pos].srcloc, + None => info.address_map.start_srcloc, }; // Use our wasm-relative pc to symbolize this frame. If there's a @@ -173,24 +197,20 @@ impl ModuleFrameInfo { func_index: index.index() as u32, func_name: module.func_names.get(&index).cloned(), instr, - func_start: addr_map.start_srcloc, + func_start: info.address_map.start_srcloc, symbols, }) } /// Fetches trap information about a program counter in a backtrace. pub fn lookup_trap_info(&self, pc: usize) -> Option<&TrapInformation> { - let (index, offset) = self.func(pc)?; - let (_, traps) = self.module.func_info(index); - let idx = traps + let (index, offset) = func_by_pc(&self.module, pc)?; + let info = self.module.func_info(index); + let idx = info + .traps .binary_search_by_key(&offset, |info| info.code_offset) .ok()?; - Some(&traps[idx]) - } - - fn func(&self, pc: usize) -> Option<(DefinedFuncIndex, u32)> { - let (index, start, _) = self.module.func_by_pc(pc)?; - Some((index, (pc - start) as u32)) + Some(&info.traps[idx]) } fn instr_pos(offset: u32, addr_map: &FunctionAddressMap) -> Option { @@ -214,56 +234,117 @@ impl ModuleFrameInfo { } } -/// This is the dual of `StoreFrameInfo` and is stored globally (as the name -/// implies) rather than simply in one `Store`. +impl ModuleInfo for RegisteredModule { + fn lookup_stack_map(&self, pc: usize) -> Option<&StackMap> { + let (index, offset) = func_by_pc(&self.module, pc)?; + let info = self.module.func_info(index); + + // Do a binary search to find the stack map for the given offset. + // + // Because GC safepoints are technically only associated with a single + // PC, we should ideally only care about `Ok(index)` values returned + // from the binary search. However, safepoints are inserted right before + // calls, and there are two things that can disturb the PC/offset + // associated with the safepoint versus the PC we actually use to query + // for the stack map: + // + // 1. The `backtrace` crate gives us the PC in a frame that will be + // *returned to*, and where execution will continue from, rather than + // the PC of the call we are currently at. So we would need to + // disassemble one instruction backwards to query the actual PC for + // the stack map. + // + // TODO: One thing we *could* do to make this a little less error + // prone, would be to assert/check that the nearest GC safepoint + // found is within `max_encoded_size(any kind of call instruction)` + // our queried PC for the target architecture. + // + // 2. Cranelift's stack maps only handle the stack, not + // registers. However, some references that are arguments to a call + // may need to be in registers. In these cases, what Cranelift will + // do is: + // + // a. spill all the live references, + // b. insert a GC safepoint for those references, + // c. reload the references into registers, and finally + // d. make the call. + // + // Step (c) adds drift between the GC safepoint and the location of + // the call, which is where we actually walk the stack frame and + // collect its live references. + // + // Luckily, the spill stack slots for the live references are still + // up to date, so we can still find all the on-stack roots. + // Furthermore, we do not have a moving GC, so we don't need to worry + // whether the following code will reuse the references in registers + // (which would not have been updated to point to the moved objects) + // or reload from the stack slots (which would have been updated to + // point to the moved objects). + + let index = match info + .stack_maps + .binary_search_by_key(&offset, |i| i.code_offset) + { + // Exact hit. + Ok(i) => i, + + // `Err(0)` means that the associated stack map would have been the + // first element in the array if this pc had an associated stack + // map, but this pc does not have an associated stack map. This can + // only happen inside a Wasm frame if there are no live refs at this + // pc. + Err(0) => return None, + + Err(i) => i - 1, + }; + + Some(&info.stack_maps[index].stack_map) + } +} + +// Counterpart to `RegisteredModule`, but stored in the global registry. +struct GlobalRegisteredModule { + start: usize, + module: Arc, + /// Note that modules can be instantiated in many stores, so the purpose of + /// this field is to keep track of how many stores have registered a + /// module. Information is only removed from the global registry when this + /// reference count reaches 0. + references: usize, +} + +/// This is the global module registry that stores information for all modules +/// that are currently in use by any `Store`. /// /// The purpose of this map is to be called from signal handlers to determine /// whether a program counter is a wasm trap or not. Specifically macOS has /// no contextual information about the thread available, hence the necessity /// for global state rather than using thread local state. /// -/// This is similar to `StoreFrameInfo` except that it has less information and -/// supports removal. Any time anything is registered with a `StoreFrameInfo` -/// it is also automatically registered with the singleton global frame -/// information. When a `StoreFrameInfo` is destroyed then all of its entries -/// are removed from the global frame information. +/// This is similar to `ModuleRegistry` except that it has less information and +/// supports removal. Any time anything is registered with a `ModuleRegistry` +/// it is also automatically registered with the singleton global module +/// registry. When a `ModuleRegistry` is destroyed then all of its entries +/// are removed from the global module registry. #[derive(Default)] -pub struct GlobalFrameInfo { - // The map here behaves the same way as `StoreFrameInfo`. - ranges: BTreeMap, -} +pub struct GlobalModuleRegistry(BTreeMap); -/// This is the equivalent of `ModuleFrameInfo` except it keeps a reference count. -struct GlobalModuleFrameInfo { - module: ModuleFrameInfo, - - /// Note that modules can be instantiated in many stores, so the purpose of - /// this field is to keep track of how many stores have registered a - /// module. Information is only removed from the global store when this - /// reference count reaches 0. - references: usize, -} - -lazy_static::lazy_static! { - static ref GLOBAL_INFO: Mutex = Default::default(); -} - -impl GlobalFrameInfo { +impl GlobalModuleRegistry { /// Returns whether the `pc`, according to globally registered information, /// is a wasm trap or not. pub(crate) fn is_wasm_pc(pc: usize) -> bool { - let info = GLOBAL_INFO.lock().unwrap(); + let modules = GLOBAL_MODULES.lock().unwrap(); - match info.ranges.range(pc..).next() { - Some((end, info)) => { - if pc < info.module.start || *end < pc { + match modules.0.range(pc..).next() { + Some((end, entry)) => { + if pc < entry.start || *end < pc { return false; } - match info.module.func(pc) { + match func_by_pc(&entry.module, pc) { Some((index, offset)) => { - let (addr_map, _) = info.module.module.func_info(index); - ModuleFrameInfo::instr_pos(offset, addr_map).is_some() + let info = entry.module.func_info(index); + RegisteredModule::instr_pos(offset, &info.address_map).is_some() } None => false, } @@ -274,32 +355,27 @@ impl GlobalFrameInfo { /// Registers a new region of code, described by `(start, end)` and with /// the given function information, with the global information. - fn register(&mut self, start: usize, end: usize, module: &Arc) { - let info = self - .ranges - .entry(end) - .or_insert_with(|| GlobalModuleFrameInfo { - module: ModuleFrameInfo { - start, - module: module.clone(), - }, - references: 0, - }); + fn register(&mut self, start: usize, end: usize, module: &Module) { + let info = self.0.entry(end).or_insert_with(|| GlobalRegisteredModule { + start, + module: module.compiled_module().clone(), + references: 0, + }); // Note that ideally we'd debug_assert that the information previously // stored, if any, matches the `functions` we were given, but for now we // just do some simple checks to hope it's the same. - assert_eq!(info.module.start, start); + assert_eq!(info.start, start); info.references += 1; } - /// Unregisters a region of code (keyed by the `end` address) from this + /// Unregisters a region of code (keyed by the `end` address) from the /// global information. fn unregister(&mut self, end: usize) { - let info = self.ranges.get_mut(&end).unwrap(); + let info = self.0.get_mut(&end).unwrap(); info.references -= 1; if info.references == 0 { - self.ranges.remove(&end); + self.0.remove(&end); } } } @@ -321,19 +397,6 @@ pub struct FrameInfo { symbols: Vec, } -/// Debug information for a symbol that is attached to a [`FrameInfo`]. -/// -/// When DWARF debug information is present in a wasm file then this structure -/// can be found on a [`FrameInfo`] and can be used to learn about filenames, -/// line numbers, etc, which are the origin of a function in a stack trace. -#[derive(Debug)] -pub struct FrameSymbol { - name: Option, - file: Option, - line: Option, - column: Option, -} - impl FrameInfo { /// Returns the WebAssembly function index for this frame. /// @@ -405,6 +468,19 @@ impl FrameInfo { } } +/// Debug information for a symbol that is attached to a [`FrameInfo`]. +/// +/// When DWARF debug information is present in a wasm file then this structure +/// can be found on a [`FrameInfo`] and can be used to learn about filenames, +/// line numbers, etc, which are the origin of a function in a stack trace. +#[derive(Debug)] +pub struct FrameSymbol { + name: Option, + file: Option, + line: Option, + column: Option, +} + impl FrameSymbol { /// Returns the function name associated with this symbol. /// @@ -463,7 +539,7 @@ fn test_frame_info() -> Result<(), anyhow::Error> { )?; // Create an instance to ensure the frame information is registered. Instance::new(&store, &module, &[])?; - let info = store.frame_info().borrow(); + let modules = store.modules().borrow(); for (i, alloc) in module.compiled_module().finished_functions() { let (start, end) = unsafe { let ptr = (**alloc).as_ptr(); @@ -471,7 +547,7 @@ fn test_frame_info() -> Result<(), anyhow::Error> { (ptr as usize, ptr as usize + len) }; for pc in start..end { - let (frame, _) = info.lookup_frame_info(pc).unwrap(); + let (frame, _) = modules.lookup_frame_info(pc).unwrap(); assert!(frame.func_index() == i.as_u32()); } } diff --git a/crates/wasmtime/src/module/serialization.rs b/crates/wasmtime/src/module/serialization.rs index cc28cdeaf7..e566d01ed5 100644 --- a/crates/wasmtime/src/module/serialization.rs +++ b/crates/wasmtime/src/module/serialization.rs @@ -1,6 +1,5 @@ //! Implements module serialization. -use super::ModuleInner; use crate::{Engine, Module, OptLevel}; use anyhow::{anyhow, bail, Context, Result}; use bincode::Options; @@ -10,8 +9,7 @@ use std::fmt; use std::str::FromStr; use std::sync::Arc; use std::{collections::HashMap, fmt::Display}; -use wasmtime_environ::Tunables; -use wasmtime_environ::{isa::TargetIsa, settings}; +use wasmtime_environ::{isa::TargetIsa, settings, Tunables}; use wasmtime_jit::{ CompilationArtifacts, CompilationStrategy, CompiledModule, Compiler, TypeTables, }; @@ -123,55 +121,44 @@ impl From for OptLevel { } } -/// A small helper struct which defines modules are serialized. +/// A small helper struct for serialized module upvars. #[derive(Serialize, Deserialize)] -struct SerializedModuleData<'a> { - /// All compiled artifacts needed by this module, where the last entry in - /// this list is the artifacts for the module itself. - artifacts: Vec>, +pub struct SerializedModuleUpvar { + /// The module's index into the compilation artifact. + pub index: usize, + /// Indexes into the list of all compilation artifacts for this module. + pub artifact_upvars: Vec, /// Closed-over module values that are also needed for this module. - modules: Vec>, - /// The index into the list of type tables that are used for this module's - /// type tables. - type_tables: usize, + pub module_upvars: Vec, } -impl<'a> SerializedModuleData<'a> { - pub fn new(module: &'a Module) -> (Self, Vec>) { - let mut pushed = HashMap::new(); - let mut tables = Vec::new(); - return (module_data(module, &mut pushed, &mut tables), tables); +impl SerializedModuleUpvar { + pub fn new(module: &Module, artifacts: &[Arc]) -> Self { + // TODO: improve upon the linear searches in the artifact list + let index = artifacts + .iter() + .position(|a| Arc::as_ptr(a) == Arc::as_ptr(&module.inner.module)) + .expect("module should be in artifacts list"); - fn module_data<'a>( - module: &'a Module, - type_tables_pushed: &mut HashMap, - type_tables: &mut Vec>, - ) -> SerializedModuleData<'a> { - // Deduplicate `Arc` using our two parameters to ensure we - // serialize type tables as little as possible. - let ptr = Arc::as_ptr(module.types()); - let type_tables_idx = *type_tables_pushed.entry(ptr as usize).or_insert_with(|| { - type_tables.push(MyCow::Borrowed(module.types())); - type_tables.len() - 1 - }); - SerializedModuleData { - artifacts: module - .inner - .artifact_upvars - .iter() - .map(|i| MyCow::Borrowed(i.compilation_artifacts())) - .chain(Some(MyCow::Borrowed( - module.compiled_module().compilation_artifacts(), - ))) - .collect(), - modules: module - .inner - .module_upvars - .iter() - .map(|i| module_data(i, type_tables_pushed, type_tables)) - .collect(), - type_tables: type_tables_idx, - } + SerializedModuleUpvar { + index, + artifact_upvars: module + .inner + .artifact_upvars + .iter() + .map(|m| { + artifacts + .iter() + .position(|a| Arc::as_ptr(a) == Arc::as_ptr(m)) + .expect("artifact should be in artifacts list") + }) + .collect(), + module_upvars: module + .inner + .module_upvars + .iter() + .map(|m| SerializedModuleUpvar::new(m, artifacts)) + .collect(), } } } @@ -212,14 +199,36 @@ pub struct SerializedModule<'a> { strategy: CompilationStrategy, tunables: Tunables, features: WasmFeatures, - data: SerializedModuleData<'a>, - tables: Vec>, + artifacts: Vec>, + module_upvars: Vec, + types: MyCow<'a, TypeTables>, } impl<'a> SerializedModule<'a> { pub fn new(module: &'a Module) -> Self { - let (data, tables) = SerializedModuleData::new(module); - Self::with_data(module.engine().compiler(), data, tables) + let compiler = module.engine().compiler(); + let artifacts = module + .inner + .artifact_upvars + .iter() + .map(|m| MyCow::Borrowed(m.compilation_artifacts())) + .chain(Some(MyCow::Borrowed( + module.inner.module.compilation_artifacts(), + ))) + .collect::>(); + let module_upvars = module + .inner + .module_upvars + .iter() + .map(|m| SerializedModuleUpvar::new(m, &module.inner.artifact_upvars)) + .collect::>(); + + Self::with_data( + compiler, + artifacts, + module_upvars, + MyCow::Borrowed(module.types()), + ) } pub fn from_artifacts( @@ -229,19 +238,17 @@ impl<'a> SerializedModule<'a> { ) -> Self { Self::with_data( compiler, - SerializedModuleData { - artifacts: artifacts.iter().map(MyCow::Borrowed).collect(), - modules: Vec::new(), - type_tables: 0, - }, - vec![MyCow::Borrowed(types)], + artifacts.iter().map(MyCow::Borrowed).collect(), + Vec::new(), + MyCow::Borrowed(types), ) } fn with_data( compiler: &Compiler, - data: SerializedModuleData<'a>, - tables: Vec>, + artifacts: Vec>, + module_upvars: Vec, + types: MyCow<'a, TypeTables>, ) -> Self { let isa = compiler.isa(); @@ -260,8 +267,9 @@ impl<'a> SerializedModule<'a> { strategy: compiler.strategy(), tunables: compiler.tunables().clone(), features: compiler.features().into(), - data, - tables, + artifacts, + module_upvars, + types, } } @@ -276,47 +284,26 @@ impl<'a> SerializedModule<'a> { self.check_tunables(compiler)?; self.check_features(compiler)?; - let types = self - .tables - .into_iter() - .map(|t| Arc::new(t.unwrap_owned())) - .collect::>(); - let module = mk(engine, &types, self.data)?; + let modules = CompiledModule::from_artifacts_list( + self.artifacts + .into_iter() + .map(|i| i.unwrap_owned()) + .collect(), + engine.compiler().isa(), + &*engine.config().profiler, + )?; - // Validate the module can be used with the current allocator - engine.allocator().validate(module.inner.module.module())?; + assert!(!modules.is_empty()); - return Ok(module); + let main_module = modules.len() - 1; - fn mk( - engine: &Engine, - types: &Vec>, - data: SerializedModuleData<'_>, - ) -> Result { - let mut artifacts = CompiledModule::from_artifacts_list( - data.artifacts - .into_iter() - .map(|i| i.unwrap_owned()) - .collect(), - engine.compiler().isa(), - &*engine.config().profiler, - )?; - let inner = ModuleInner { - engine: engine.clone(), - types: types[data.type_tables].clone(), - module: artifacts.pop().unwrap(), - artifact_upvars: artifacts, - module_upvars: data - .modules - .into_iter() - .map(|m| mk(engine, types, m)) - .collect::>>()?, - }; - - Ok(Module { - inner: Arc::new(inner), - }) - } + Module::from_parts( + engine, + modules, + main_module, + Arc::new(self.types.unwrap_owned()), + &self.module_upvars, + ) } pub fn to_bytes(&self) -> Result> { @@ -342,9 +329,9 @@ impl<'a> SerializedModule<'a> { Ok(bytes) } - pub fn from_bytes(bytes: &[u8]) -> Result> { + pub fn from_bytes(bytes: &[u8]) -> Result { if !bytes.starts_with(HEADER) { - return Ok(None); + bail!("bytes are not a compatible serialized wasmtime module"); } let bytes = &bytes[HEADER.len()..]; @@ -366,11 +353,9 @@ impl<'a> SerializedModule<'a> { ); } - Ok(Some( - bincode_options() - .deserialize::>(&bytes[1 + version_len..]) - .context("deserialize compilation artifacts")?, - )) + Ok(bincode_options() + .deserialize::>(&bytes[1 + version_len..]) + .context("deserialize compilation artifacts")?) } fn check_triple(&self, isa: &dyn TargetIsa) -> Result<()> { diff --git a/crates/wasmtime/src/sig_registry.rs b/crates/wasmtime/src/sig_registry.rs deleted file mode 100644 index e5811a8424..0000000000 --- a/crates/wasmtime/src/sig_registry.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! Implement a registry of function signatures, for fast indirect call -//! signature checking. - -use crate::Module; -use std::collections::{hash_map, HashMap}; -use std::convert::TryFrom; -use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::wasm::{SignatureIndex, WasmFuncType}; -use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline}; - -/// WebAssembly requires that the caller and callee signatures in an indirect -/// call must match. To implement this efficiently, keep a registry of all -/// signatures, shared by all instances, so that call sites can just do an -/// index comparison. -#[derive(Debug, Default)] -pub struct SignatureRegistry { - // Map from a wasm actual function type to the index that it is assigned, - // shared amongst all wasm modules. - wasm2index: HashMap, - - // Map of all known wasm function signatures in this registry. This is - // keyed by `VMSharedSignatureIndex` above. - index_map: Vec, -} - -#[derive(Debug)] -struct Entry { - // The WebAssembly type signature, using wasm types. - wasm: WasmFuncType, - // The native trampoline used to invoke this type signature from `Func`. - // Note that the code memory for this trampoline is not owned by this - // type, but instead it's expected to be owned by the store that this - // registry lives within. - trampoline: Option, -} - -impl SignatureRegistry { - /// Registers all signatures within a module into this registry all at once. - /// - /// This will also internally register trampolines compiled in the module. - pub fn register_module(&mut self, module: &Module) { - // Register a unique index for all types in this module, even if they - // don't have a trampoline. - let signatures = &module.types().wasm_signatures; - for ty in module.compiled_module().module().types.values() { - if let wasmtime_environ::ModuleType::Function(index) = ty { - self.register_one(&signatures[*index], None); - } - } - - // Once we've got a shared index for all types used then also fill in - // any trampolines that the module has compiled as well. - for (index, trampoline) in module.compiled_module().trampolines() { - let shared = self.wasm2index[&signatures[*index]]; - let entry = &mut self.index_map[shared.bits() as usize]; - if entry.trampoline.is_none() { - entry.trampoline = Some(*trampoline); - } - } - } - - /// Register a signature and return its unique index. - pub fn register( - &mut self, - wasm: &WasmFuncType, - trampoline: VMTrampoline, - ) -> VMSharedSignatureIndex { - self.register_one(wasm, Some(trampoline)) - } - - fn register_one( - &mut self, - wasm: &WasmFuncType, - trampoline: Option, - ) -> VMSharedSignatureIndex { - let len = self.wasm2index.len(); - - match self.wasm2index.entry(wasm.clone()) { - hash_map::Entry::Occupied(entry) => { - let ret = *entry.get(); - let entry = &mut self.index_map[ret.bits() as usize]; - // If the entry does not previously have a trampoline, then - // overwrite it with whatever was specified by this function. - if entry.trampoline.is_none() { - entry.trampoline = trampoline; - } - ret - } - hash_map::Entry::Vacant(entry) => { - // Keep `signature_hash` len under 2**32 -- VMSharedSignatureIndex::new(std::u32::MAX) - // is reserved for VMSharedSignatureIndex::default(). - assert!( - len < std::u32::MAX as usize, - "Invariant check: signature_hash.len() < std::u32::MAX" - ); - debug_assert_eq!(len, self.index_map.len()); - let index = VMSharedSignatureIndex::new(u32::try_from(len).unwrap()); - self.index_map.push(Entry { - wasm: wasm.clone(), - trampoline, - }); - entry.insert(index); - index - } - } - } - - /// Looks up a shared index from the wasm signature itself. - pub fn lookup(&self, wasm: &WasmFuncType) -> Option { - self.wasm2index.get(wasm).cloned() - } - - /// Builds a lookup table for a module from the possible module's signature - /// indices to the shared signature index within this registry. - pub fn lookup_table( - &self, - module: &Module, - ) -> PrimaryMap { - // For module-linking using modules this builds up a map that is - // too large. This builds up a map for everything in `TypeTables` but - // that's all the types for all modules in a whole module linking graph, - // which our `module` may not be using. - // - // For all non-module-linking-using modules, though, this is not an - // issue. This is optimizing for the non-module-linking case right now - // and it seems like module linking will likely change to the point that - // this will no longer be an issue in the future. - let signatures = &module.types().wasm_signatures; - let mut map = PrimaryMap::with_capacity(signatures.len()); - for wasm in signatures.values() { - map.push( - self.wasm2index - .get(wasm) - .cloned() - .unwrap_or(VMSharedSignatureIndex::new(u32::MAX)), - ); - } - map - } - - /// Looks up information known about a shared signature index. - /// - /// Note that for this operation to be semantically correct the `idx` must - /// have previously come from a call to `register` of this same object. - pub fn lookup_shared( - &self, - idx: VMSharedSignatureIndex, - ) -> Option<(&WasmFuncType, VMTrampoline)> { - let (wasm, trampoline) = self - .index_map - .get(idx.bits() as usize) - .map(|e| (&e.wasm, e.trampoline))?; - Some((wasm, trampoline?)) - } -} diff --git a/crates/wasmtime/src/signatures.rs b/crates/wasmtime/src/signatures.rs new file mode 100644 index 0000000000..2faf3fa059 --- /dev/null +++ b/crates/wasmtime/src/signatures.rs @@ -0,0 +1,262 @@ +//! Implement a registry of function signatures, for fast indirect call +//! signature checking. + +use std::{ + collections::{hash_map::Entry, HashMap}, + sync::RwLock, +}; +use std::{convert::TryFrom, sync::Arc}; +use wasmtime_environ::entity::PrimaryMap; +use wasmtime_environ::wasm::{SignatureIndex, WasmFuncType}; +use wasmtime_runtime::{VMSharedSignatureIndex, VMTrampoline}; + +/// Represents a collection of shared signatures. +/// +/// This is used to register shared signatures with a shared signature registry. +/// +/// The collection will unregister any contained signatures with the registry +/// when dropped. +#[derive(Debug)] +pub struct SignatureCollection { + registry: Arc>, + signatures: PrimaryMap, + trampolines: HashMap, +} + +impl SignatureCollection { + /// Creates a new, empty signature collection given a signature registry. + pub fn new(registry: &SignatureRegistry) -> Self { + Self { + registry: registry.0.clone(), + signatures: PrimaryMap::new(), + trampolines: HashMap::new(), + } + } + + /// Creates a signature collection for a module given the module's signatures + /// and trampolines. + pub fn new_for_module( + registry: &SignatureRegistry, + signatures: &PrimaryMap, + trampolines: impl Iterator, + ) -> Self { + let (signatures, trampolines) = registry + .0 + .write() + .unwrap() + .register_for_module(signatures, trampolines); + + Self { + registry: registry.0.clone(), + signatures, + trampolines, + } + } + + /// Treats the signature collection as a map from a module signature index to + /// registered shared signature indexes. + /// + /// This is used for looking up module shared signature indexes during module + /// instantiation. + pub fn as_module_map(&self) -> &PrimaryMap { + &self.signatures + } + + /// Gets the shared signature index given a module signature index. + pub fn shared_signature(&self, index: SignatureIndex) -> Option { + self.signatures.get(index).copied() + } + + /// Gets a trampoline for a registered signature. + pub fn trampoline(&self, index: VMSharedSignatureIndex) -> Option { + self.trampolines + .get(&index) + .map(|(_, trampoline)| *trampoline) + } + + /// Registers a single function with the collection. + /// + /// Returns the shared signature index for the function. + pub fn register( + &mut self, + ty: &WasmFuncType, + trampoline: VMTrampoline, + ) -> VMSharedSignatureIndex { + let index = self.registry.write().unwrap().register(ty); + + let entry = match self.trampolines.entry(index) { + Entry::Occupied(e) => e.into_mut(), + Entry::Vacant(e) => e.insert((0, trampoline)), + }; + + // Increment the ref count + entry.0 += 1; + + index + } +} + +impl Drop for SignatureCollection { + fn drop(&mut self) { + if !self.signatures.is_empty() || !self.trampolines.is_empty() { + self.registry.write().unwrap().unregister_signatures(self); + } + } +} + +#[derive(Debug)] +struct RegistryEntry { + references: usize, + ty: WasmFuncType, +} + +#[derive(Debug, Default)] +struct SignatureRegistryInner { + map: HashMap, + entries: Vec>, + free: Vec, +} + +impl SignatureRegistryInner { + fn register_for_module( + &mut self, + signatures: &PrimaryMap, + trampolines: impl Iterator, + ) -> ( + PrimaryMap, + HashMap, + ) { + let mut sigs = PrimaryMap::default(); + let mut map = HashMap::default(); + + for (_, ty) in signatures.iter() { + sigs.push(self.register(ty)); + } + + for (index, trampoline) in trampolines { + map.insert(sigs[index], (1, trampoline)); + } + + (sigs, map) + } + + fn register(&mut self, ty: &WasmFuncType) -> VMSharedSignatureIndex { + let len = self.map.len(); + + let index = match self.map.entry(ty.clone()) { + Entry::Occupied(e) => *e.get(), + Entry::Vacant(e) => { + let (index, entry) = match self.free.pop() { + Some(index) => (index, &mut self.entries[index.bits() as usize]), + None => { + // Keep `index_map` len under 2**32 -- VMSharedSignatureIndex::new(std::u32::MAX) + // is reserved for VMSharedSignatureIndex::default(). + assert!( + len < std::u32::MAX as usize, + "Invariant check: index_map.len() < std::u32::MAX" + ); + debug_assert_eq!(len, self.entries.len()); + + let index = VMSharedSignatureIndex::new(u32::try_from(len).unwrap()); + self.entries.push(None); + + (index, self.entries.last_mut().unwrap()) + } + }; + + // The entry should be missing for one just allocated or + // taken from the free list + assert!(entry.is_none()); + + *entry = Some(RegistryEntry { + references: 0, + ty: ty.clone(), + }); + + *e.insert(index) + } + }; + + self.entries[index.bits() as usize] + .as_mut() + .unwrap() + .references += 1; + + index + } + + fn unregister_signatures(&mut self, collection: &SignatureCollection) { + // If the collection has a populated signatures map, use it to deregister + // This is always 1:1 from entry to registration + if !collection.signatures.is_empty() { + for (_, index) in collection.signatures.iter() { + self.unregister_entry(*index, 1); + } + } else { + // Otherwise, use the trampolines map, which has reference counts related + // to the stored index + for (index, (count, _)) in collection.trampolines.iter() { + self.unregister_entry(*index, *count); + } + } + } + + fn unregister_entry(&mut self, index: VMSharedSignatureIndex, count: usize) { + let removed = { + let entry = self.entries[index.bits() as usize].as_mut().unwrap(); + + debug_assert!(entry.references >= count); + entry.references -= count; + + if entry.references == 0 { + self.map.remove(&entry.ty); + self.free.push(index); + true + } else { + false + } + }; + + if removed { + self.entries[index.bits() as usize] = None; + } + } +} + +// `SignatureRegistryInner` implements `Drop` in debug builds to assert that +// all signatures have been unregistered for the registry. +#[cfg(debug_assertions)] +impl Drop for SignatureRegistryInner { + fn drop(&mut self) { + assert!( + self.map.is_empty() && self.free.len() == self.entries.len(), + "signature registry not empty" + ); + } +} + +/// Implements a shared signature registry. +/// +/// WebAssembly requires that the caller and callee signatures in an indirect +/// call must match. To implement this efficiently, keep a registry of all +/// signatures, shared by all instances, so that call sites can just do an +/// index comparison. +#[derive(Debug)] +pub struct SignatureRegistry(Arc>); + +impl SignatureRegistry { + /// Creates a new shared signature registry. + pub fn new() -> Self { + Self(Arc::new(RwLock::new(SignatureRegistryInner::default()))) + } + + /// Looks up a function type from a shared signature index. + pub fn lookup_type(&self, index: VMSharedSignatureIndex) -> Option { + self.0 + .read() + .unwrap() + .entries + .get(index.bits() as usize) + .and_then(|e| e.as_ref().map(|e| &e.ty).cloned()) + } +} diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 18dbde303c..c3737b3bff 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -1,12 +1,12 @@ -use crate::frame_info; -use crate::frame_info::StoreFrameInfo; -use crate::sig_registry::SignatureRegistry; -use crate::trampoline::StoreInstanceHandle; -use crate::{Engine, Func, FuncType, Module, Trap}; +use crate::{ + module::ModuleRegistry, signatures::SignatureCollection, trampoline::StoreInstanceHandle, + Engine, Func, Module, ResourceLimiter, ResourceLimiterProxy, Trap, DEFAULT_INSTANCE_LIMIT, + DEFAULT_MEMORY_LIMIT, DEFAULT_TABLE_LIMIT, +}; use anyhow::{bail, Result}; use std::any::{Any, TypeId}; use std::cell::{Cell, RefCell}; -use std::collections::{hash_map::Entry, HashMap, HashSet}; +use std::collections::{hash_map::Entry, HashMap}; use std::convert::TryFrom; use std::fmt; use std::future::Future; @@ -16,12 +16,10 @@ use std::ptr; use std::rc::Rc; use std::sync::Arc; use std::task::{Context, Poll}; -use wasmtime_environ::wasm; -use wasmtime_jit::{CompiledModule, ModuleCode}; use wasmtime_runtime::{ - Export, InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, SignalHandler, - StackMapRegistry, TrapInfo, VMCallerCheckedAnyfunc, VMContext, VMExternRef, - VMExternRefActivationsTable, VMInterrupts, VMTrampoline, + InstanceAllocator, InstanceHandle, ModuleInfo, OnDemandInstanceAllocator, SignalHandler, + TrapInfo, VMCallerCheckedAnyfunc, VMContext, VMExternRef, VMExternRefActivationsTable, + VMInterrupts, VMTrampoline, }; /// Used to associate instances with the store. @@ -72,20 +70,13 @@ pub struct Store { pub(crate) struct StoreInner { engine: Engine, - /// The map of all host functions registered with this store's signature registry - host_funcs: RefCell>>, interrupts: Arc, - signatures: RefCell, instances: RefCell>, signal_handler: RefCell>>>, externref_activations_table: VMExternRefActivationsTable, - stack_map_registry: StackMapRegistry, - /// Information about JIT code which allows us to test if a program counter - /// is in JIT code, lookup trap information, etc. - frame_info: RefCell, - /// Set of all compiled modules that we're holding a strong reference to - /// the module's code for. This includes JIT functions, trampolines, etc. - modules: RefCell>, + modules: RefCell, + // The signatures and trampolines for `Func` objects + signatures: RefCell, // Numbers of resources instantiated in this store. instance_count: Cell, memory_count: Cell, @@ -99,6 +90,7 @@ pub(crate) struct StoreInner { current_poll_cx: Cell<*mut Context<'static>>, out_of_gas_behavior: Cell, context_values: RefCell>>, + limiter: Option>, } #[derive(Copy, Clone)] @@ -130,28 +122,57 @@ impl Hash for HostInfoKey { } impl Store { - /// Creates a new store to be associated with the given [`Engine`]. - pub fn new(engine: &Engine) -> Store { - // Ensure that wasmtime_runtime's signal handlers are configured. Note - // that at the `Store` level it means we should perform this - // once-per-thread. Platforms like Unix, however, only require this - // once-per-program. In any case this is safe to call many times and - // each one that's not relevant just won't do anything. - wasmtime_runtime::init_traps(frame_info::GlobalFrameInfo::is_wasm_pc) - .expect("failed to initialize trap handling"); + /// Creates a new [`Store`] to be associated with the given [`Engine`]. + /// + /// The created [`Store`] will place no additional limits on the size of linear + /// memories or tables at runtime. Linear memories and tables will be allowed to + /// grow to any upper limit specified in their definitions. + /// + /// The store will limit the number of instances, linear memories, and tables created to 10,000. + /// + /// Use [`Store::new_with_limits`] with a [`StoreLimitsBuilder`](crate::StoreLimitsBuilder) to + /// specify different limits for the store. + pub fn new(engine: &Engine) -> Self { + Self::new_(engine, None) + } - Store { + /// Creates a new [`Store`] to be associated with the given [`Engine`] and using the supplied + /// resource limiter. + /// + /// A [`ResourceLimiter`] can be implemented by hosts to control the size of WebAssembly + /// linear memories and tables when a request is made to grow them. + /// + /// [`StoreLimitsBuilder`](crate::StoreLimitsBuilder) can be used to create a + /// [`StoreLimits`](crate::StoreLimits) that implements [`ResourceLimiter`] using + /// static limit values. + /// + /// # Example + /// + /// ```rust + /// # use wasmtime::{Engine, Store, StoreLimitsBuilder}; + /// // Place a limit on linear memories so they cannot grow beyond 1 MiB + /// let engine = Engine::default(); + /// let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memory_pages(16).build()); + /// ``` + pub fn new_with_limits(engine: &Engine, limiter: impl ResourceLimiter + 'static) -> Self { + Self::new_(engine, Some(Rc::new(ResourceLimiterProxy(limiter)))) + } + + fn new_(engine: &Engine, limiter: Option>) -> Self { + // Ensure that wasmtime_runtime's signal handlers are configured. This + // is the per-program initialization required for handling traps, such + // as configuring signals, vectored exception handlers, etc. + wasmtime_runtime::init_traps(crate::module::GlobalModuleRegistry::is_wasm_pc); + + Self { inner: Rc::new(StoreInner { engine: engine.clone(), - host_funcs: RefCell::new(HashMap::new()), interrupts: Arc::new(Default::default()), - signatures: RefCell::new(Default::default()), instances: RefCell::new(Vec::new()), signal_handler: RefCell::new(None), externref_activations_table: VMExternRefActivationsTable::new(), - stack_map_registry: StackMapRegistry::default(), - frame_info: Default::default(), - modules: Default::default(), + modules: RefCell::new(ModuleRegistry::default()), + signatures: RefCell::new(SignatureCollection::new(engine.signatures())), instance_count: Default::default(), memory_count: Default::default(), table_count: Default::default(), @@ -162,6 +183,7 @@ impl Store { current_poll_cx: Cell::new(ptr::null_mut()), out_of_gas_behavior: Cell::new(OutOfGas::Trap), context_values: RefCell::new(HashMap::new()), + limiter, }), } } @@ -181,35 +203,6 @@ impl Store { }) } - pub(crate) fn get_host_anyfunc( - &self, - instance: &InstanceHandle, - ty: &FuncType, - trampoline: VMTrampoline, - ) -> *mut VMCallerCheckedAnyfunc { - let mut funcs = self.inner.host_funcs.borrow_mut(); - - let anyfunc = funcs.entry(unsafe { instance.clone() }).or_insert_with(|| { - let mut anyfunc = match instance - .lookup_by_declaration(&wasm::EntityIndex::Function(wasm::FuncIndex::from_u32(0))) - { - Export::Function(f) => unsafe { f.anyfunc.as_ref() }.clone(), - _ => unreachable!(), - }; - - // Register the function with this store's signature registry - anyfunc.type_index = self - .inner - .signatures - .borrow_mut() - .register(ty.as_wasm_func_type(), trampoline); - - Box::new(anyfunc) - }); - - &mut **anyfunc - } - /// Returns the [`Engine`] that this store is associated with. #[inline] pub fn engine(&self) -> &Engine { @@ -244,69 +237,39 @@ impl Store { } } - pub(crate) fn signatures(&self) -> &RefCell { + pub(crate) fn limiter(&self) -> &Option> { + &self.inner.limiter + } + + pub(crate) fn signatures(&self) -> &RefCell { &self.inner.signatures } - pub(crate) fn register_module(&self, module: &Module) { - // With a module being instantiated into this `Store` we need to - // preserve its jit-code. References to this module's code and - // trampolines are not owning-references so it's our responsibility to - // keep it all alive within the `Store`. - // - // If this module is already present in the store then we skip all - // further registration steps. - let first = self + pub(crate) fn lookup_trampoline(&self, anyfunc: &VMCallerCheckedAnyfunc) -> VMTrampoline { + // Look up the trampoline with the store's trampolines (from `Func`). + if let Some(trampoline) = self .inner - .modules - .borrow_mut() - .insert(ArcModuleCode(module.compiled_module().code().clone())); - if !first { - return; + .signatures + .borrow() + .trampoline(anyfunc.type_index) + { + return trampoline; } - // All modules register their JIT code in a store for two reasons - // currently: - // - // * First we only catch signals/traps if the program counter falls - // within the jit code of an instantiated wasm module. This ensures - // we don't catch accidental Rust/host segfaults. - // - // * Second when generating a backtrace we'll use this mapping to - // only generate wasm frames for instruction pointers that fall - // within jit code. + // Look up the trampoline with the registered modules + if let Some(trampoline) = self.inner.modules.borrow().lookup_trampoline(anyfunc) { + return trampoline; + } + + // Lastly, check with the engine (for `HostFunc`) self.inner - .frame_info - .borrow_mut() - .register(module.compiled_module()); - - // We need to know about all the stack maps of all instantiated modules - // so when performing a GC we know about all wasm frames that we find - // on the stack. - self.register_stack_maps(module.compiled_module()); - - // Signatures are loaded into our `SignatureRegistry` here - // once-per-module (and once-per-signature). This allows us to create - // a `Func` wrapper for any function in the module, which requires that - // we know about the signature and trampoline for all instances. - self.signatures().borrow_mut().register_module(module); - } - - fn register_stack_maps(&self, module: &CompiledModule) { - self.stack_map_registry() - .register_stack_maps(module.stack_maps().map(|(func, stack_maps)| unsafe { - let ptr = (*func).as_ptr(); - let len = (*func).len(); - let start = ptr as usize; - let end = ptr as usize + len; - let range = start..end; - (range, stack_maps) - })); + .engine + .host_func_signatures() + .trampoline(anyfunc.type_index) + .expect("trampoline missing") } pub(crate) fn bump_resource_counts(&self, module: &Module) -> Result<()> { - let config = self.engine().config(); - fn bump(slot: &Cell, max: usize, amt: usize, desc: &str) -> Result<()> { let new = slot.get().saturating_add(amt); if new > max { @@ -323,20 +286,11 @@ impl Store { let module = module.env_module(); let memories = module.memory_plans.len() - module.num_imported_memories; let tables = module.table_plans.len() - module.num_imported_tables; + let (max_instances, max_memories, max_tables) = self.limits(); - bump( - &self.inner.instance_count, - config.max_instances, - 1, - "instance", - )?; - bump( - &self.inner.memory_count, - config.max_memories, - memories, - "memory", - )?; - bump(&self.inner.table_count, config.max_tables, tables, "table")?; + bump(&self.inner.instance_count, max_instances, 1, "instance")?; + bump(&self.inner.memory_count, max_memories, memories, "memory")?; + bump(&self.inner.table_count, max_tables, tables, "table")?; Ok(()) } @@ -363,7 +317,7 @@ impl Store { .borrow() .iter() .any(|i| i.handle.vmctx_ptr() == handle.vmctx_ptr()) - || self.inner.host_funcs.borrow().get(&handle).is_some() + || self.inner.engine.host_func_anyfunc(&handle).is_some() ); StoreInstanceHandle { store: self.clone(), @@ -490,24 +444,21 @@ impl Store { } #[inline] - pub(crate) fn stack_map_registry(&self) -> &StackMapRegistry { - &self.inner.stack_map_registry + pub(crate) fn modules(&self) -> &RefCell { + &self.inner.modules } - pub(crate) fn frame_info(&self) -> &RefCell { - &self.inner.frame_info + #[inline] + pub(crate) fn module_info_lookup(&self) -> &dyn wasmtime_runtime::ModuleInfoLookup { + self.inner.as_ref() } /// Perform garbage collection of `ExternRef`s. pub fn gc(&self) { // For this crate's API, we ensure that `set_stack_canary` invariants - // are upheld for all host-->Wasm calls, and we register every module - // used with this store in `self.inner.stack_map_registry`. + // are upheld for all host-->Wasm calls. unsafe { - wasmtime_runtime::gc( - &self.inner.stack_map_registry, - &self.inner.externref_activations_table, - ); + wasmtime_runtime::gc(self.inner.as_ref(), &self.inner.externref_activations_table); } } @@ -700,7 +651,8 @@ impl Store { } unsafe { - let before = wasmtime_runtime::TlsRestore::take(); + let before = wasmtime_runtime::TlsRestore::take() + .map_err(|e| Trap::from_runtime(self, e))?; let res = (*suspend).suspend(()); before.replace().map_err(|e| Trap::from_runtime(self, e))?; res?; @@ -896,6 +848,18 @@ impl Store { Err(trap) => unsafe { wasmtime_runtime::raise_user_trap(trap.into()) }, } } + + fn limits(&self) -> (usize, usize, usize) { + self.inner + .limiter + .as_ref() + .map(|l| (l.instances(), l.memories(), l.tables())) + .unwrap_or(( + DEFAULT_INSTANCE_LIMIT, + DEFAULT_MEMORY_LIMIT, + DEFAULT_TABLE_LIMIT, + )) + } } unsafe impl TrapInfo for Store { @@ -968,6 +932,12 @@ impl Drop for StoreInner { } } +impl wasmtime_runtime::ModuleInfoLookup for StoreInner { + fn lookup(&self, pc: usize) -> Option> { + self.modules.borrow().lookup_module(pc) + } +} + /// A threadsafe handle used to interrupt instances executing within a /// particular `Store`. /// @@ -996,24 +966,6 @@ impl InterruptHandle { } } -// Wrapper struct to implement hash/equality based on the pointer value of the -// `Arc` in question. -struct ArcModuleCode(Arc); - -impl PartialEq for ArcModuleCode { - fn eq(&self, other: &ArcModuleCode) -> bool { - Arc::ptr_eq(&self.0, &other.0) - } -} - -impl Eq for ArcModuleCode {} - -impl Hash for ArcModuleCode { - fn hash(&self, hasher: &mut H) { - Arc::as_ptr(&self.0).hash(hasher) - } -} - struct Reset<'a, T: Copy>(&'a Cell, T); impl Drop for Reset<'_, T> { diff --git a/crates/wasmtime/src/trampoline.rs b/crates/wasmtime/src/trampoline.rs index 7cb588f7d4..5ebc436eba 100644 --- a/crates/wasmtime/src/trampoline.rs +++ b/crates/wasmtime/src/trampoline.rs @@ -19,8 +19,8 @@ use std::sync::Arc; use wasmtime_environ::{entity::PrimaryMap, wasm, Module}; use wasmtime_runtime::{ Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, - OnDemandInstanceAllocator, StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, - VMFunctionImport, VMSharedSignatureIndex, + OnDemandInstanceAllocator, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, + VMSharedSignatureIndex, }; /// A wrapper around `wasmtime_runtime::InstanceHandle` which pairs it with the @@ -77,7 +77,8 @@ fn create_handle( externref_activations_table: store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, + module_info_lookup: Some(store.module_info_lookup()), + limiter: store.limiter().as_ref(), }, )?; diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs deleted file mode 100644 index dde937f81c..0000000000 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Support for a calling of an imported function. - -use crate::trampoline::StoreInstanceHandle; -use crate::Store; -use anyhow::Result; -use std::any::Any; -use std::sync::Arc; -use wasmtime_environ::entity::PrimaryMap; -use wasmtime_environ::wasm::DefinedFuncIndex; -use wasmtime_environ::Module; -use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstanceAllocator, StackMapRegistry, - VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMSharedSignatureIndex, -}; - -pub(crate) fn create_handle( - module: Module, - store: &Store, - finished_functions: PrimaryMap, - host_state: Box, - func_imports: &[VMFunctionImport], - shared_signature_id: Option, -) -> Result { - let mut imports = Imports::default(); - imports.functions = func_imports; - let module = Arc::new(module); - - unsafe { - // Use the default allocator when creating handles associated with host objects - // The configured instance allocator should only be used when creating module instances - // as we don't want host objects to count towards instance limits. - let handle = store - .engine() - .config() - .default_instance_allocator - .allocate(InstanceAllocationRequest { - module: module.clone(), - finished_functions: &finished_functions, - imports, - shared_signatures: shared_signature_id.into(), - host_state, - interrupts: store.interrupts(), - externref_activations_table: store.externref_activations_table() - as *const VMExternRefActivationsTable - as *mut _, - stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, - })?; - - Ok(store.add_instance(handle, true)) - } -} diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 9879880900..b381c561ea 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -1,6 +1,6 @@ //! Support for a calling of an imported function. -use crate::{sig_registry::SignatureRegistry, Config, FuncType, Trap}; +use crate::{Config, FuncType, Store, Trap}; use anyhow::Result; use std::any::Any; use std::cmp; @@ -262,15 +262,19 @@ pub fn create_function( ft: &FuncType, func: Box Result<(), Trap>>, config: &Config, - registry: Option<&mut SignatureRegistry>, + store: Option<&Store>, ) -> Result<(InstanceHandle, VMTrampoline)> { let (module, finished_functions, trampoline, trampoline_state) = create_function_trampoline(config, ft, func)?; - // If there is no signature registry, use the default signature index which is + // If there is no store, use the default signature index which is // guaranteed to trap if there is ever an indirect call on the function (should not happen) - let shared_signature_id = registry - .map(|r| r.register(ft.as_wasm_func_type(), trampoline)) + let shared_signature_id = store + .map(|s| { + s.signatures() + .borrow_mut() + .register(ft.as_wasm_func_type(), trampoline) + }) .unwrap_or(VMSharedSignatureIndex::default()); unsafe { @@ -283,7 +287,8 @@ pub fn create_function( host_state: Box::new(trampoline_state), interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_registry: std::ptr::null_mut(), + module_info_lookup: None, + limiter: None, })?, trampoline, )) @@ -315,7 +320,8 @@ pub unsafe fn create_raw_function( host_state, interrupts: std::ptr::null(), externref_activations_table: std::ptr::null_mut(), - stack_map_registry: std::ptr::null_mut(), + module_info_lookup: None, + limiter: None, })?, ) } diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs index 0876ead225..9f3ea772b0 100644 --- a/crates/wasmtime/src/trampoline/memory.rs +++ b/crates/wasmtime/src/trampoline/memory.rs @@ -37,6 +37,10 @@ impl RuntimeLinearMemory for LinearMemoryProxy { self.mem.size() } + fn maximum(&self) -> Option { + self.mem.maximum() + } + fn grow(&self, delta: u32) -> Option { self.mem.grow(delta) } diff --git a/crates/wasmtime/src/trap.rs b/crates/wasmtime/src/trap.rs index b50f5353ee..1d59aa564c 100644 --- a/crates/wasmtime/src/trap.rs +++ b/crates/wasmtime/src/trap.rs @@ -161,7 +161,7 @@ impl Trap { maybe_interrupted, } => { let mut code = store - .frame_info() + .modules() .borrow() .lookup_trap_info(pc) .map(|info| info.trap_code) @@ -239,7 +239,7 @@ impl Trap { // (the call instruction) so we subtract one as the lookup. let pc_to_lookup = if Some(pc) == trap_pc { pc } else { pc - 1 }; if let Some((info, has_unparsed_debuginfo)) = - store.frame_info().borrow().lookup_frame_info(pc_to_lookup) + store.modules().borrow().lookup_frame_info(pc_to_lookup) { wasm_trace.push(info); diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs index 2a91af473c..0241aee282 100644 --- a/crates/wasmtime/src/types.rs +++ b/crates/wasmtime/src/types.rs @@ -204,8 +204,7 @@ impl ExternType { ) -> ExternType { match ty { EntityType::Function(idx) => { - let sig = &types.wasm_signatures[*idx]; - FuncType::from_wasm_func_type(sig).into() + FuncType::from_wasm_func_type(types.wasm_signatures[*idx].clone()).into() } EntityType::Global(ty) => GlobalType::from_wasmtime_global(ty).into(), EntityType::Memory(ty) => MemoryType::from_wasmtime_memory(ty).into(), @@ -298,8 +297,8 @@ impl FuncType { &self.sig } - pub(crate) fn from_wasm_func_type(sig: &wasm::WasmFuncType) -> FuncType { - FuncType { sig: sig.clone() } + pub(crate) fn from_wasm_func_type(sig: wasm::WasmFuncType) -> FuncType { + Self { sig } } } diff --git a/crates/wasmtime/src/types/matching.rs b/crates/wasmtime/src/types/matching.rs index a767bbb7e1..7405194c12 100644 --- a/crates/wasmtime/src/types/matching.rs +++ b/crates/wasmtime/src/types/matching.rs @@ -1,4 +1,4 @@ -use crate::{Extern, Store}; +use crate::{signatures::SignatureCollection, Extern, Store}; use anyhow::{bail, Context, Result}; use wasmtime_environ::wasm::{ EntityType, Global, InstanceTypeIndex, Memory, ModuleTypeIndex, SignatureIndex, Table, @@ -6,6 +6,7 @@ use wasmtime_environ::wasm::{ use wasmtime_jit::TypeTables; pub struct MatchCx<'a> { + pub signatures: &'a SignatureCollection, pub types: &'a TypeTables, pub store: &'a Store, } @@ -70,12 +71,7 @@ impl MatchCx<'_> { } pub fn func(&self, expected: SignatureIndex, actual: &crate::Func) -> Result<()> { - let matches = match self - .store - .signatures() - .borrow() - .lookup(&self.types.wasm_signatures[expected]) - { + let matches = match self.signatures.shared_signature(expected) { Some(idx) => actual.sig_index() == idx, // If our expected signature isn't registered, then there's no way // that `actual` can match it. @@ -114,15 +110,19 @@ impl MatchCx<'_> { let module = actual.compiled_module().module(); self.imports_match( expected, + actual.signatures(), actual.types(), module.imports().map(|(name, field, ty)| { assert!(field.is_none()); // should be true if module linking is enabled (name, ty) }), )?; - self.exports_match(expected_sig.exports, actual.types(), |name| { - module.exports.get(name).map(|idx| module.type_of(*idx)) - })?; + self.exports_match( + expected_sig.exports, + actual.signatures(), + actual.types(), + |name| module.exports.get(name).map(|idx| module.type_of(*idx)), + )?; Ok(()) } @@ -133,6 +133,7 @@ impl MatchCx<'_> { fn imports_match<'a>( &self, expected: ModuleTypeIndex, + actual_signatures: &SignatureCollection, actual_types: &TypeTables, actual_imports: impl Iterator, ) -> Result<()> { @@ -146,10 +147,11 @@ impl MatchCx<'_> { None => bail!("expected type doesn't import {:?}", name), }; MatchCx { + signatures: actual_signatures, types: actual_types, store: self.store, } - .extern_ty_matches(&actual_ty, expected_ty, self.types) + .extern_ty_matches(&actual_ty, expected_ty, self.signatures, self.types) .with_context(|| format!("module import {:?} incompatible", name))?; } Ok(()) @@ -160,6 +162,7 @@ impl MatchCx<'_> { fn exports_match( &self, expected: InstanceTypeIndex, + actual_signatures: &SignatureCollection, actual_types: &TypeTables, lookup: impl Fn(&str) -> Option, ) -> Result<()> { @@ -169,7 +172,7 @@ impl MatchCx<'_> { for (name, expected) in self.types.instance_signatures[expected].exports.iter() { match lookup(name) { Some(ty) => self - .extern_ty_matches(expected, &ty, actual_types) + .extern_ty_matches(expected, &ty, actual_signatures, actual_types) .with_context(|| format!("export {:?} incompatible", name))?, None => bail!("failed to find export {:?}", name), } @@ -183,6 +186,7 @@ impl MatchCx<'_> { &self, expected: &EntityType, actual_ty: &EntityType, + actual_signatures: &SignatureCollection, actual_types: &TypeTables, ) -> Result<()> { let actual_desc = match actual_ty { @@ -221,7 +225,7 @@ impl MatchCx<'_> { EntityType::Instance(expected) => match actual_ty { EntityType::Instance(actual) => { let sig = &actual_types.instance_signatures[*actual]; - self.exports_match(*expected, actual_types, |name| { + self.exports_match(*expected, actual_signatures, actual_types, |name| { sig.exports.get(name).cloned() })?; Ok(()) @@ -237,15 +241,19 @@ impl MatchCx<'_> { self.imports_match( *expected, + actual_signatures, actual_types, actual_module_sig .imports .iter() .map(|(module, ty)| (module.as_str(), ty.clone())), )?; - self.exports_match(expected_module_sig.exports, actual_types, |name| { - actual_instance_sig.exports.get(name).cloned() - })?; + self.exports_match( + expected_module_sig.exports, + actual_signatures, + actual_types, + |name| actual_instance_sig.exports.get(name).cloned(), + )?; Ok(()) } _ => bail!("expected module, but found {}", actual_desc), diff --git a/crates/wasmtime/src/values.rs b/crates/wasmtime/src/values.rs index 4dc6474513..7d6adbbedc 100644 --- a/crates/wasmtime/src/values.rs +++ b/crates/wasmtime/src/values.rs @@ -98,7 +98,7 @@ impl Val { let externref_ptr = x.inner.as_raw(); store .externref_activations_table() - .insert_with_gc(x.inner, store.stack_map_registry()); + .insert_with_gc(x.inner, store.module_info_lookup()); ptr::write(p as *mut *mut u8, externref_ptr) } Val::FuncRef(f) => ptr::write( diff --git a/crates/wast/src/spectest.rs b/crates/wast/src/spectest.rs index ef744fa42d..cd8b7bc9d1 100644 --- a/crates/wast/src/spectest.rs +++ b/crates/wast/src/spectest.rs @@ -39,7 +39,7 @@ pub fn link_spectest(linker: &mut Linker) -> Result<()> { linker.define("spectest", "table", table)?; let ty = MemoryType::new(Limits::new(1, Some(2))); - let memory = Memory::new(linker.store(), ty); + let memory = Memory::new(linker.store(), ty)?; linker.define("spectest", "memory", memory)?; Ok(()) diff --git a/deny.toml b/deny.toml index d9baefa136..7f909d470e 100644 --- a/deny.toml +++ b/deny.toml @@ -14,6 +14,7 @@ vulnerability = "deny" unmaintained = "deny" yanked = "deny" ignore = [ + "RUSTSEC-2021-0064" ] # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html @@ -23,6 +24,7 @@ allow = [ "Apache-2.0", "BSD-2-Clause", "CC0-1.0", + "ISC", "MIT", "MPL-2.0", "Zlib", diff --git a/examples/fuel.c b/examples/fuel.c new file mode 100644 index 0000000000..af3b51782d --- /dev/null +++ b/examples/fuel.c @@ -0,0 +1,132 @@ +/* +Example of instantiating of the WebAssembly module and invoking its exported +function. + +You can compile and run this example on Linux with: + + cargo build --release -p wasmtime-c-api + cc examples/fuel.c \ + -I crates/c-api/include \ + -I crates/c-api/wasm-c-api/include \ + target/release/libwasmtime.a \ + -lpthread -ldl -lm \ + -o fuel + ./fuel + +Note that on Windows and macOS the command will be similar, but you'll need +to tweak the `-lpthread` and such annotations. +*/ + +#include +#include +#include +#include +#include + +static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap); + +int main() { + wasmtime_error_t *error = NULL; + + wasm_config_t *config = wasm_config_new(); + assert(config != NULL); + wasmtime_config_consume_fuel_set(config, true); + + // Create an *engine*, which is a compilation context, with our configured options. + wasm_engine_t *engine = wasm_engine_new_with_config(config); + assert(engine != NULL); + wasm_store_t *store = wasm_store_new(engine); + assert(store != NULL); + error = wasmtime_store_add_fuel(store, 10000); + if (error != NULL) + exit_with_error("failed to add fuel", error, NULL); + + // Load our input file to parse it next + FILE* file = fopen("examples/fuel.wat", "r"); + if (!file) { + printf("> Error loading file!\n"); + return 1; + } + fseek(file, 0L, SEEK_END); + size_t file_size = ftell(file); + fseek(file, 0L, SEEK_SET); + wasm_byte_vec_t wat; + wasm_byte_vec_new_uninitialized(&wat, file_size); + if (fread(wat.data, file_size, 1, file) != 1) { + printf("> Error loading module!\n"); + return 1; + } + fclose(file); + + // Parse the wat into the binary wasm format + wasm_byte_vec_t wasm; + error = wasmtime_wat2wasm(&wat, &wasm); + if (error != NULL) + exit_with_error("failed to parse wat", error, NULL); + wasm_byte_vec_delete(&wat); + + // Compile and instantiate our module + wasm_module_t *module = NULL; + error = wasmtime_module_new(engine, &wasm, &module); + if (module == NULL) + exit_with_error("failed to compile module", error, NULL); + wasm_byte_vec_delete(&wasm); + wasm_trap_t *trap = NULL; + wasm_instance_t *instance = NULL; + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + error = wasmtime_instance_new(store, module, &imports, &instance, &trap); + if (instance == NULL) + exit_with_error("failed to instantiate", error, trap); + + // Lookup our `fibonacci` export function + wasm_extern_vec_t externs; + wasm_instance_exports(instance, &externs); + assert(externs.size == 1); + wasm_func_t *fibonacci = wasm_extern_as_func(externs.data[0]); + assert(fibonacci != NULL); + + // Call it repeatedly until it fails + for (int n = 1; ; n++) { + uint64_t fuel_before; + wasmtime_store_fuel_consumed(store, &fuel_before); + wasm_val_t params[1] = { WASM_I32_VAL(n) }; + wasm_val_t results[1]; + wasm_val_vec_t params_vec = WASM_ARRAY_VEC(params); + wasm_val_vec_t results_vec = WASM_ARRAY_VEC(results); + error = wasmtime_func_call(fibonacci, ¶ms_vec, &results_vec, &trap); + if (error != NULL || trap != NULL) { + printf("Exhausted fuel computing fib(%d)\n", n); + break; + } + + uint64_t fuel_after; + wasmtime_store_fuel_consumed(store, &fuel_after); + assert(results[0].kind == WASM_I32); + printf("fib(%d) = %d [consumed %lld fuel]\n", n, results[0].of.i32, fuel_after - fuel_before); + + error = wasmtime_store_add_fuel(store, fuel_after - fuel_before); + if (error != NULL) + exit_with_error("failed to add fuel", error, NULL); + } + + // Clean up after ourselves at this point + wasm_extern_vec_delete(&externs); + wasm_instance_delete(instance); + wasm_module_delete(module); + wasm_store_delete(store); + wasm_engine_delete(engine); + return 0; +} + +static void exit_with_error(const char *message, wasmtime_error_t *error, wasm_trap_t *trap) { + fprintf(stderr, "error: %s\n", message); + wasm_byte_vec_t error_message; + if (error != NULL) { + wasmtime_error_message(error, &error_message); + } else { + wasm_trap_message(trap, &error_message); + } + fprintf(stderr, "%.*s\n", (int) error_message.size, error_message.data); + wasm_byte_vec_delete(&error_message); + exit(1); +} diff --git a/examples/fuel.rs b/examples/fuel.rs new file mode 100644 index 0000000000..00181a67e1 --- /dev/null +++ b/examples/fuel.rs @@ -0,0 +1,33 @@ +//! Example of limiting a WebAssembly function's runtime using "fuel consumption". + +// You can execute this example with `cargo run --example fuel` + +use anyhow::Result; +use wasmtime::*; + +fn main() -> Result<()> { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config)?; + let store = Store::new(&engine); + store.add_fuel(10_000)?; + let module = Module::from_file(store.engine(), "examples/fuel.wat")?; + let instance = Instance::new(&store, &module, &[])?; + + // Invoke `fibonacci` export with higher and higher numbers until we exhaust our fuel. + let fibonacci = instance.get_typed_func::("fibonacci")?; + for n in 1.. { + let fuel_before = store.fuel_consumed().unwrap(); + let output = match fibonacci.call(n) { + Ok(v) => v, + Err(_) => { + println!("Exhausted fuel computing fib({})", n); + break; + } + }; + let fuel_consumed = store.fuel_consumed().unwrap() - fuel_before; + println!("fib({}) = {} [consumed {} fuel]", n, output, fuel_consumed); + store.add_fuel(fuel_consumed)?; + } + Ok(()) +} diff --git a/examples/fuel.wat b/examples/fuel.wat new file mode 100644 index 0000000000..48622c2e21 --- /dev/null +++ b/examples/fuel.wat @@ -0,0 +1,13 @@ +(module + (func $fibonacci (param $n i32) (result i32) + (if + (i32.lt_s (local.get $n) (i32.const 2)) + (return (local.get $n)) + ) + (i32.add + (call $fibonacci (i32.sub (local.get $n) (i32.const 1))) + (call $fibonacci (i32.sub (local.get $n) (i32.const 2))) + ) + ) + (export "fibonacci" (func $fibonacci)) +) diff --git a/examples/memory.rs b/examples/memory.rs index 70e1b724a9..e47c249e5f 100644 --- a/examples/memory.rs +++ b/examples/memory.rs @@ -75,7 +75,7 @@ fn main() -> Result<()> { println!("Creating stand-alone memory..."); let memorytype = MemoryType::new(Limits::new(5, Some(5))); - let memory2 = Memory::new(&wasmtime_store, memorytype); + let memory2 = Memory::new(&wasmtime_store, memorytype)?; assert_eq!(memory2.size(), 5); assert!(memory2.grow(1).is_err()); assert!(memory2.grow(0).is_ok()); diff --git a/examples/serialize.rs b/examples/serialize.rs index 3cd709e748..70a875c3bf 100644 --- a/examples/serialize.rs +++ b/examples/serialize.rs @@ -29,9 +29,12 @@ fn deserialize(buffer: &[u8]) -> Result<()> { println!("Initializing..."); let store = Store::default(); - // Compile the wasm binary into an in-memory instance of a `Module`. + // Compile the wasm binary into an in-memory instance of a `Module`. Note + // that this is `unsafe` because it is our responsibility for guaranteeing + // that these bytes are valid precompiled module bytes. We know that from + // the structure of this example program. println!("Deserialize module..."); - let module = Module::new(store.engine(), buffer)?; + let module = unsafe { Module::deserialize(store.engine(), buffer)? }; // Here we handle the imports of the module, which in this case is our // `HelloCallback` type and its associated implementation of `Callback. diff --git a/src/commands/compile.rs b/src/commands/compile.rs index 7b9a7ebcc2..9d601ab7a0 100644 --- a/src/commands/compile.rs +++ b/src/commands/compile.rs @@ -28,7 +28,7 @@ lazy_static::lazy_static! { Compiling for a specific platform (Linux) and CPU preset (Skylake):\n\ \n \ wasmtime compile --target x86_64-unknown-linux --cranelift-enable skylake foo.wasm\n", - crate::WASM_FEATURES.as_str() + crate::FLAG_EXPLANATIONS.as_str() ) }; } @@ -126,7 +126,8 @@ mod test { command.execute()?; let engine = Engine::default(); - let module = Module::from_file(&engine, output_path)?; + let contents = std::fs::read(output_path)?; + let module = unsafe { Module::deserialize(&engine, contents)? }; let store = Store::new(&engine); let instance = Instance::new(&store, &module, &[])?; let f = instance.get_typed_func::("f")?; diff --git a/src/commands/run.rs b/src/commands/run.rs index f1f8da46e5..eeec81b2b7 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -1,6 +1,6 @@ //! The module that implements the `wasmtime run` command. -use crate::CommonOptions; +use crate::{CommonOptions, WasiModules}; use anyhow::{bail, Context as _, Result}; use std::thread; use std::time::Duration; @@ -67,7 +67,7 @@ fn parse_preloads(s: &str) -> Result<(String, PathBuf)> { lazy_static::lazy_static! { static ref AFTER_HELP: String = { - crate::WASM_FEATURES.to_string() + crate::FLAG_EXPLANATIONS.to_string() }; } @@ -143,7 +143,13 @@ impl RunCommand { let argv = self.compute_argv(); let mut linker = Linker::new(&store); - populate_with_wasi(&mut linker, preopen_dirs, &argv, &self.vars)?; + populate_with_wasi( + &mut linker, + preopen_dirs, + &argv, + &self.vars, + &self.common.wasi_modules.unwrap_or(WasiModules::default()), + )?; // Load the preload wasm modules. for (name, path) in self.preloads.iter() { @@ -348,6 +354,7 @@ fn populate_with_wasi( preopen_dirs: Vec<(String, Dir)>, argv: &[String], vars: &[(String, String)], + wasi_modules: &WasiModules, ) -> Result<()> { // Add the current snapshot to the linker. let mut builder = WasiCtxBuilder::new(); @@ -357,25 +364,40 @@ fn populate_with_wasi( builder = builder.preopened_dir(dir, name)?; } - Wasi::new(linker.store(), builder.build()?).add_to_linker(linker)?; - - #[cfg(feature = "wasi-nn")] - { - use std::cell::RefCell; - use std::rc::Rc; - let wasi_nn = WasiNn::new(linker.store(), Rc::new(RefCell::new(WasiNnCtx::new()?))); - wasi_nn.add_to_linker(linker)?; + if wasi_modules.wasi_common { + Wasi::new(linker.store(), builder.build()?).add_to_linker(linker)?; } - #[cfg(feature = "wasi-crypto")] - { - use std::cell::RefCell; - use std::rc::Rc; - let cx_crypto = Rc::new(RefCell::new(WasiCryptoCtx::new())); - WasiCryptoCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; - WasiCryptoAsymmetricCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; - WasiCryptoSignatures::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; - WasiCryptoSymmetric::new(linker.store(), cx_crypto).add_to_linker(linker)?; + if wasi_modules.wasi_nn { + #[cfg(not(feature = "wasi-nn"))] + { + bail!("Cannot enable wasi-nn when the binary is not compiled with this feature."); + } + #[cfg(feature = "wasi-nn")] + { + use std::cell::RefCell; + use std::rc::Rc; + let wasi_nn = WasiNn::new(linker.store(), Rc::new(RefCell::new(WasiNnCtx::new()?))); + wasi_nn.add_to_linker(linker)?; + } + } + + if wasi_modules.wasi_crypto { + #[cfg(not(feature = "wasi-crypto"))] + { + bail!("Cannot enable wasi-crypto when the binary is not compiled with this feature."); + } + #[cfg(feature = "wasi-crypto")] + { + use std::cell::RefCell; + use std::rc::Rc; + let cx_crypto = Rc::new(RefCell::new(WasiCryptoCtx::new())); + WasiCryptoCommon::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; + WasiCryptoAsymmetricCommon::new(linker.store(), cx_crypto.clone()) + .add_to_linker(linker)?; + WasiCryptoSignatures::new(linker.store(), cx_crypto.clone()).add_to_linker(linker)?; + WasiCryptoSymmetric::new(linker.store(), cx_crypto).add_to_linker(linker)?; + } } Ok(()) diff --git a/src/commands/wasm2obj.rs b/src/commands/wasm2obj.rs index ab914d8edc..b6426ed13a 100644 --- a/src/commands/wasm2obj.rs +++ b/src/commands/wasm2obj.rs @@ -18,7 +18,7 @@ lazy_static::lazy_static! { The default is a dummy environment that produces placeholder values.\n\ \n\ {}", - crate::WASM_FEATURES.as_str() + crate::FLAG_EXPLANATIONS.as_str() ) }; } diff --git a/src/commands/wast.rs b/src/commands/wast.rs index cd749e61a6..08ad8e0a2a 100644 --- a/src/commands/wast.rs +++ b/src/commands/wast.rs @@ -9,7 +9,7 @@ use wasmtime_wast::WastContext; lazy_static::lazy_static! { static ref AFTER_HELP: String = { - crate::WASM_FEATURES.to_string() + crate::FLAG_EXPLANATIONS.to_string() }; } diff --git a/src/lib.rs b/src/lib.rs index 3485b62889..facb0383c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,19 +43,47 @@ const SUPPORTED_WASM_FEATURES: &[(&str, &str)] = &[ ("threads", "enables support for WebAssembly threads"), ]; +const SUPPORTED_WASI_MODULES: &[(&str, &str)] = &[ + ( + "default", + "enables all stable WASI modules (no experimental modules)", + ), + ( + "wasi-common", + "enables support for the WASI common APIs, see https://github.com/WebAssembly/WASI", + ), + ( + "experimental-wasi-nn", + "enables support for the WASI neural network API (experimental), see https://github.com/WebAssembly/wasi-nn", + ), + ( + "experimental-wasi-crypto", + "enables support for the WASI cryptography APIs (experimental), see https://github.com/WebAssembly/wasi-crypto", + ), +]; + lazy_static::lazy_static! { - static ref WASM_FEATURES: String = { + static ref FLAG_EXPLANATIONS: String = { use std::fmt::Write; let mut s = String::new(); + + // Explain --wasm-features. writeln!(&mut s, "Supported values for `--wasm-features`:").unwrap(); writeln!(&mut s).unwrap(); - let max = SUPPORTED_WASM_FEATURES.iter().max_by_key(|(name, _)| name.len()).unwrap(); - for (name, desc) in SUPPORTED_WASM_FEATURES.iter() { writeln!(&mut s, "{:width$} {}", name, desc, width = max.0.len() + 2).unwrap(); } + writeln!(&mut s).unwrap(); + + // Explain --wasi-modules. + writeln!(&mut s, "Supported values for `--wasi-modules`:").unwrap(); + writeln!(&mut s).unwrap(); + let max = SUPPORTED_WASI_MODULES.iter().max_by_key(|(name, _)| name.len()).unwrap(); + for (name, desc) in SUPPORTED_WASI_MODULES.iter() { + writeln!(&mut s, "{:width$} {}", name, desc, width = max.0.len() + 2).unwrap(); + } writeln!(&mut s).unwrap(); writeln!(&mut s, "Features prefixed with '-' will be disabled.").unwrap(); @@ -186,6 +214,10 @@ struct CommonOptions { #[structopt(long, value_name = "FEATURE,FEATURE,...", parse(try_from_str = parse_wasm_features))] wasm_features: Option, + /// Enables or disables WASI modules + #[structopt(long, value_name = "MODULE,MODULE,...", parse(try_from_str = parse_wasi_modules))] + wasi_modules: Option, + /// Use Lightbeam for all compilation #[structopt(long, conflicts_with = "cranelift")] lightbeam: bool, @@ -408,6 +440,75 @@ fn parse_wasm_features(features: &str) -> Result { memory64: false, }) } + +fn parse_wasi_modules(modules: &str) -> Result { + let modules = modules.trim(); + match modules { + "default" => Ok(WasiModules::default()), + "-default" => Ok(WasiModules::none()), + _ => { + // Starting from the default set of WASI modules, enable or disable a list of + // comma-separated modules. + let mut wasi_modules = WasiModules::default(); + let mut set = |module: &str, enable: bool| match module { + "" => Ok(()), + "wasi-common" => Ok(wasi_modules.wasi_common = enable), + "experimental-wasi-nn" => Ok(wasi_modules.wasi_nn = enable), + "experimental-wasi-crypto" => Ok(wasi_modules.wasi_crypto = enable), + "default" => bail!("'default' cannot be specified with other WASI modules"), + _ => bail!("unsupported WASI module '{}'", module), + }; + + for module in modules.split(',') { + let module = module.trim(); + let (module, value) = if module.starts_with('-') { + (&module[1..], false) + } else { + (module, true) + }; + set(module, value)?; + } + + Ok(wasi_modules) + } + } +} + +/// Select which WASI modules are available at runtime for use by Wasm programs. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct WasiModules { + /// Enable the wasi-common implementation; eventually this should be split into its separate + /// parts once the implementation allows for it (e.g. wasi-fs, wasi-clocks, etc.). + pub wasi_common: bool, + + /// Enable the experimental wasi-nn implementation. + pub wasi_nn: bool, + + /// Enable the experimental wasi-crypto implementation. + pub wasi_crypto: bool, +} + +impl Default for WasiModules { + fn default() -> Self { + Self { + wasi_common: true, + wasi_nn: false, + wasi_crypto: false, + } + } +} + +impl WasiModules { + /// Enable no modules. + pub fn none() -> Self { + Self { + wasi_common: false, + wasi_nn: false, + wasi_crypto: false, + } + } +} + fn parse_cranelift_flag(name_and_value: &str) -> Result<(String, String)> { let mut split = name_and_value.splitn(2, '='); let name = if let Some(name) = split.next() { @@ -574,4 +675,61 @@ mod test { feature_test!(test_simd_feature, simd, "simd"); feature_test!(test_threads_feature, threads, "threads"); feature_test!(test_multi_memory_feature, multi_memory, "multi-memory"); + + #[test] + fn test_default_modules() { + let options = CommonOptions::from_iter_safe(vec!["foo", "--wasi-modules=default"]).unwrap(); + assert_eq!( + options.wasi_modules.unwrap(), + WasiModules { + wasi_common: true, + wasi_nn: false, + wasi_crypto: false + } + ); + } + + #[test] + fn test_empty_modules() { + let options = CommonOptions::from_iter_safe(vec!["foo", "--wasi-modules="]).unwrap(); + assert_eq!( + options.wasi_modules.unwrap(), + WasiModules { + wasi_common: true, + wasi_nn: false, + wasi_crypto: false + } + ); + } + + #[test] + fn test_some_modules() { + let options = CommonOptions::from_iter_safe(vec![ + "foo", + "--wasi-modules=experimental-wasi-nn,-wasi-common", + ]) + .unwrap(); + assert_eq!( + options.wasi_modules.unwrap(), + WasiModules { + wasi_common: false, + wasi_nn: true, + wasi_crypto: false + } + ); + } + + #[test] + fn test_no_modules() { + let options = + CommonOptions::from_iter_safe(vec!["foo", "--wasi-modules=-default"]).unwrap(); + assert_eq!( + options.wasi_modules.unwrap(), + WasiModules { + wasi_common: false, + wasi_nn: false, + wasi_crypto: false + } + ); + } } diff --git a/tests/all/externals.rs b/tests/all/externals.rs index 9bffa2c08d..0e26166f07 100644 --- a/tests/all/externals.rs +++ b/tests/all/externals.rs @@ -67,7 +67,7 @@ fn cross_store() -> anyhow::Result<()> { let ty = GlobalType::new(ValType::I32, Mutability::Const); let global = Global::new(&store2, ty, Val::I32(0))?; let ty = MemoryType::new(Limits::new(1, None)); - let memory = Memory::new(&store2, ty); + let memory = Memory::new(&store2, ty)?; let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); let table = Table::new(&store2, ty, Val::FuncRef(None))?; @@ -356,7 +356,7 @@ fn read_write_memory_via_api() { let cfg = Config::new(); let store = Store::new(&Engine::new(&cfg).unwrap()); let ty = MemoryType::new(Limits::new(1, None)); - let mem = Memory::new(&store, ty); + let mem = Memory::new(&store, ty).unwrap(); mem.grow(1).unwrap(); let value = b"hello wasm"; diff --git a/tests/all/func.rs b/tests/all/func.rs index e3bee038c2..67e4f9ddc2 100644 --- a/tests/all/func.rs +++ b/tests/all/func.rs @@ -774,3 +774,30 @@ fn wrap_multiple_results() -> anyhow::Result<()> { f64 "f64" F64 f64::from_bits(a), } } + +#[test] +fn trampoline_for_declared_elem() -> anyhow::Result<()> { + let engine = Engine::default(); + + let module = Module::new( + &engine, + r#" + (module + (elem declare func $f) + (func $f) + (func (export "g") (result funcref) + (ref.func $f) + ) + ) + "#, + )?; + + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + + let g = instance.get_typed_func::<(), Option>("g")?; + + let func = g.call(())?; + func.unwrap().call(&[])?; + Ok(()) +} diff --git a/tests/all/limits.rs b/tests/all/limits.rs new file mode 100644 index 0000000000..f4c74be612 --- /dev/null +++ b/tests/all/limits.rs @@ -0,0 +1,381 @@ +use anyhow::Result; +use std::cell::RefCell; +use std::rc::Rc; +use wasmtime::*; + +#[test] +fn test_limits() -> Result<()> { + let engine = Engine::default(); + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (table (export "t") 0 anyfunc))"#, + )?; + + let store = Store::new_with_limits( + &engine, + StoreLimitsBuilder::new() + .memory_pages(10) + .table_elements(5) + .build(), + ); + + let instance = Instance::new(&store, &module, &[])?; + + // Test instance exports and host objects hitting the limit + for memory in std::array::IntoIter::new([ + instance.get_memory("m").unwrap(), + Memory::new(&store, MemoryType::new(Limits::new(0, None)))?, + ]) { + memory.grow(3)?; + memory.grow(5)?; + memory.grow(2)?; + + assert_eq!( + memory.grow(1).map_err(|e| e.to_string()).unwrap_err(), + "failed to grow memory by `1`" + ); + } + + // Test instance exports and host objects hitting the limit + for table in std::array::IntoIter::new([ + instance.get_table("t").unwrap(), + Table::new( + &store, + TableType::new(ValType::FuncRef, Limits::new(0, None)), + Val::FuncRef(None), + )?, + ]) { + table.grow(2, Val::FuncRef(None))?; + table.grow(1, Val::FuncRef(None))?; + table.grow(2, Val::FuncRef(None))?; + + assert_eq!( + table + .grow(1, Val::FuncRef(None)) + .map_err(|e| e.to_string()) + .unwrap_err(), + "failed to grow table by `1`" + ); + } + + Ok(()) +} + +#[test] +fn test_limits_memory_only() -> Result<()> { + let engine = Engine::default(); + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (table (export "t") 0 anyfunc))"#, + )?; + + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memory_pages(10).build()); + + let instance = Instance::new(&store, &module, &[])?; + + // Test instance exports and host objects hitting the limit + for memory in std::array::IntoIter::new([ + instance.get_memory("m").unwrap(), + Memory::new(&store, MemoryType::new(Limits::new(0, None)))?, + ]) { + memory.grow(3)?; + memory.grow(5)?; + memory.grow(2)?; + + assert_eq!( + memory.grow(1).map_err(|e| e.to_string()).unwrap_err(), + "failed to grow memory by `1`" + ); + } + + // Test instance exports and host objects *not* hitting the limit + for table in std::array::IntoIter::new([ + instance.get_table("t").unwrap(), + Table::new( + &store, + TableType::new(ValType::FuncRef, Limits::new(0, None)), + Val::FuncRef(None), + )?, + ]) { + table.grow(2, Val::FuncRef(None))?; + table.grow(1, Val::FuncRef(None))?; + table.grow(2, Val::FuncRef(None))?; + table.grow(1, Val::FuncRef(None))?; + } + + Ok(()) +} + +#[test] +fn test_initial_memory_limits_exceeded() -> Result<()> { + let engine = Engine::default(); + let module = Module::new(&engine, r#"(module (memory (export "m") 11))"#)?; + + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memory_pages(10).build()); + + match Instance::new(&store, &module, &[]) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: memory minimum size of 11 pages exceeds memory limits" + ), + } + + match Memory::new(&store, MemoryType::new(Limits::new(25, None))) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: memory minimum size of 25 pages exceeds memory limits" + ), + } + + Ok(()) +} + +#[test] +fn test_limits_table_only() -> Result<()> { + let engine = Engine::default(); + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (table (export "t") 0 anyfunc))"#, + )?; + + let store = + Store::new_with_limits(&engine, StoreLimitsBuilder::new().table_elements(5).build()); + + let instance = Instance::new(&store, &module, &[])?; + + // Test instance exports and host objects *not* hitting the limit + for memory in std::array::IntoIter::new([ + instance.get_memory("m").unwrap(), + Memory::new(&store, MemoryType::new(Limits::new(0, None)))?, + ]) { + memory.grow(3)?; + memory.grow(5)?; + memory.grow(2)?; + memory.grow(1)?; + } + + // Test instance exports and host objects hitting the limit + for table in std::array::IntoIter::new([ + instance.get_table("t").unwrap(), + Table::new( + &store, + TableType::new(ValType::FuncRef, Limits::new(0, None)), + Val::FuncRef(None), + )?, + ]) { + table.grow(2, Val::FuncRef(None))?; + table.grow(1, Val::FuncRef(None))?; + table.grow(2, Val::FuncRef(None))?; + + assert_eq!( + table + .grow(1, Val::FuncRef(None)) + .map_err(|e| e.to_string()) + .unwrap_err(), + "failed to grow table by `1`" + ); + } + + Ok(()) +} + +#[test] +fn test_initial_table_limits_exceeded() -> Result<()> { + let engine = Engine::default(); + let module = Module::new(&engine, r#"(module (table (export "t") 23 anyfunc))"#)?; + + let store = + Store::new_with_limits(&engine, StoreLimitsBuilder::new().table_elements(4).build()); + + match Instance::new(&store, &module, &[]) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: table minimum size of 23 elements exceeds table limits" + ), + } + + match Table::new( + &store, + TableType::new(ValType::FuncRef, Limits::new(99, None)), + Val::FuncRef(None), + ) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: table minimum size of 99 elements exceeds table limits" + ), + } + + Ok(()) +} + +#[test] +fn test_pooling_allocator_initial_limits_exceeded() -> Result<()> { + let mut config = Config::new(); + config.wasm_multi_memory(true); + config.allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memories: 2, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + ..Default::default() + }, + }); + + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#"(module (memory (export "m1") 2) (memory (export "m2") 5))"#, + )?; + + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memory_pages(3).build()); + + match Instance::new(&store, &module, &[]) { + Ok(_) => unreachable!(), + Err(e) => assert_eq!( + e.to_string(), + "Insufficient resources: memory minimum size of 5 pages exceeds memory limits" + ), + } + + // An instance should still be able to be created after the failure above + let module = Module::new(&engine, r#"(module (memory (export "m") 2))"#)?; + + Instance::new(&store, &module, &[])?; + + Ok(()) +} + +struct MemoryContext { + host_memory_used: usize, + wasm_memory_used: usize, + memory_limit: usize, + limit_exceeded: bool, + limiter_dropped: bool, +} + +struct HostMemoryLimiter(Rc>); + +impl ResourceLimiter for HostMemoryLimiter { + fn memory_growing(&self, current: u32, desired: u32, maximum: Option) -> bool { + let mut ctx = self.0.borrow_mut(); + + // Check if the desired exceeds a maximum (either from Wasm or from the host) + if desired > maximum.unwrap_or(u32::MAX) { + ctx.limit_exceeded = true; + return false; + } + + assert_eq!(current as usize * 0x10000, ctx.wasm_memory_used); + let desired = desired as usize * 0x10000; + + if desired + ctx.host_memory_used > ctx.memory_limit { + ctx.limit_exceeded = true; + return false; + } + + ctx.wasm_memory_used = desired; + true + } + + fn table_growing(&self, _current: u32, _desired: u32, _maximum: Option) -> bool { + true + } +} + +impl Drop for HostMemoryLimiter { + fn drop(&mut self) { + self.0.borrow_mut().limiter_dropped = true; + } +} + +#[test] +fn test_custom_limiter() -> Result<()> { + let mut config = Config::default(); + + // This approximates a function that would "allocate" resources that the host tracks. + // Here this is a simple function that increments the current host memory "used". + config.wrap_host_func("", "alloc", |caller: Caller, size: u32| -> u32 { + if let Some(ctx) = caller.store().get::>>() { + let mut ctx = ctx.borrow_mut(); + let size = size as usize; + + if size + ctx.host_memory_used + ctx.wasm_memory_used <= ctx.memory_limit { + ctx.host_memory_used += size; + return 1; + } + + ctx.limit_exceeded = true; + } + + 0 + }); + + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#"(module (import "" "alloc" (func $alloc (param i32) (result i32))) (memory (export "m") 0) (func (export "f") (param i32) (result i32) local.get 0 call $alloc))"#, + )?; + + let context = Rc::new(RefCell::new(MemoryContext { + host_memory_used: 0, + wasm_memory_used: 0, + memory_limit: 1 << 20, // 16 wasm pages is the limit for both wasm + host memory + limit_exceeded: false, + limiter_dropped: false, + })); + + let store = Store::new_with_limits(&engine, HostMemoryLimiter(context.clone())); + + assert!(store.set(context.clone()).is_ok()); + + let linker = Linker::new(&store); + let instance = linker.instantiate(&module)?; + let memory = instance.get_memory("m").unwrap(); + + // Grow the memory by 640 KiB + memory.grow(3)?; + memory.grow(5)?; + memory.grow(2)?; + + assert!(!context.borrow().limit_exceeded); + + // Grow the host "memory" by 384 KiB + let f = instance.get_typed_func::("f")?; + + assert_eq!(f.call(1 * 0x10000).unwrap(), 1); + assert_eq!(f.call(3 * 0x10000).unwrap(), 1); + assert_eq!(f.call(2 * 0x10000).unwrap(), 1); + + // Memory is at the maximum, but the limit hasn't been exceeded + assert!(!context.borrow().limit_exceeded); + + // Try to grow the memory again + assert_eq!( + memory.grow(1).map_err(|e| e.to_string()).unwrap_err(), + "failed to grow memory by `1`" + ); + + assert!(context.borrow().limit_exceeded); + + // Try to grow the host "memory" again + assert_eq!(f.call(1).unwrap(), 0); + + assert!(context.borrow().limit_exceeded); + + drop(f); + drop(memory); + drop(instance); + drop(linker); + drop(store); + + assert!(context.borrow().limiter_dropped); + + Ok(()) +} diff --git a/tests/all/linker.rs b/tests/all/linker.rs index e3d21e96bb..2f1707dd71 100644 --- a/tests/all/linker.rs +++ b/tests/all/linker.rs @@ -51,11 +51,11 @@ fn link_twice_bad() -> Result<()> { // memories let ty = MemoryType::new(Limits::new(1, None)); - let memory = Memory::new(&store, ty); + let memory = Memory::new(&store, ty)?; linker.define("m", "", memory.clone())?; assert!(linker.define("m", "", memory.clone()).is_err()); let ty = MemoryType::new(Limits::new(2, None)); - let memory = Memory::new(&store, ty); + let memory = Memory::new(&store, ty)?; assert!(linker.define("m", "", memory.clone()).is_err()); // tables diff --git a/tests/all/main.rs b/tests/all/main.rs index 4c921e60a3..89c686119e 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -13,6 +13,7 @@ mod import_calling_export; mod import_indexes; mod instance; mod invoke_func_via_table; +mod limits; mod linker; mod memory_creator; mod module; diff --git a/tests/all/memory_creator.rs b/tests/all/memory_creator.rs index e877b5e22d..5c49414706 100644 --- a/tests/all/memory_creator.rs +++ b/tests/all/memory_creator.rs @@ -17,6 +17,7 @@ mod not_for_windows { struct CustomMemory { mem: *mut c_void, size: usize, + guard_size: usize, used_wasm_pages: RefCell, glob_page_counter: Arc>, } @@ -43,6 +44,7 @@ mod not_for_windows { Self { mem, size, + guard_size, used_wasm_pages: RefCell::new(num_wasm_pages), glob_page_counter: glob_counter, } @@ -63,6 +65,10 @@ mod not_for_windows { *self.used_wasm_pages.borrow() } + fn maximum(&self) -> Option { + Some((self.size as u32 - self.guard_size as u32) / WASM_PAGE_SIZE) + } + fn grow(&self, delta: u32) -> Option { let delta_size = (delta as usize).checked_mul(WASM_PAGE_SIZE as usize)?; @@ -70,11 +76,8 @@ mod not_for_windows { let prev_size = (prev_pages as usize).checked_mul(WASM_PAGE_SIZE as usize)?; let new_pages = prev_pages.checked_add(delta)?; - let new_size = (new_pages as usize).checked_mul(WASM_PAGE_SIZE as usize)?; - let guard_size = unsafe { sysconf(_SC_PAGESIZE) as usize }; - - if new_size > self.size - guard_size { + if new_pages > self.maximum().unwrap() { return None; } unsafe { diff --git a/tests/all/module.rs b/tests/all/module.rs index 279c91e829..f6b3b5b0d0 100644 --- a/tests/all/module.rs +++ b/tests/all/module.rs @@ -27,35 +27,37 @@ fn caches_across_engines() { .serialize() .unwrap(); - let res = Module::new(&Engine::new(&Config::new()).unwrap(), &bytes); - assert!(res.is_ok()); + unsafe { + let res = Module::deserialize(&Engine::new(&Config::new()).unwrap(), &bytes); + assert!(res.is_ok()); - // differ in shared cranelift flags - let res = Module::new( - &Engine::new(Config::new().cranelift_nan_canonicalization(true)).unwrap(), - &bytes, - ); - assert!(res.is_err()); - - // differ in cranelift settings - let res = Module::new( - &Engine::new(Config::new().cranelift_opt_level(OptLevel::None)).unwrap(), - &bytes, - ); - assert!(res.is_err()); - - // Missing required cpu flags - if cfg!(target_arch = "x86_64") { - let res = Module::new( - &Engine::new( - Config::new() - .target(&target_lexicon::Triple::host().to_string()) - .unwrap(), - ) - .unwrap(), + // differ in shared cranelift flags + let res = Module::deserialize( + &Engine::new(Config::new().cranelift_nan_canonicalization(true)).unwrap(), &bytes, ); assert!(res.is_err()); + + // differ in cranelift settings + let res = Module::deserialize( + &Engine::new(Config::new().cranelift_opt_level(OptLevel::None)).unwrap(), + &bytes, + ); + assert!(res.is_err()); + + // Missing required cpu flags + if cfg!(target_arch = "x86_64") { + let res = Module::deserialize( + &Engine::new( + Config::new() + .target(&target_lexicon::Triple::host().to_string()) + .unwrap(), + ) + .unwrap(), + &bytes, + ); + assert!(res.is_err()); + } } } @@ -66,7 +68,7 @@ fn aot_compiles() -> Result<()> { "(module (func (export \"f\") (param i32) (result i32) local.get 0))".as_bytes(), )?; - let module = Module::from_binary(&engine, &bytes)?; + let module = unsafe { Module::deserialize(&engine, &bytes)? }; let store = Store::new(&engine); let instance = Instance::new(&store, &module, &[])?; diff --git a/tests/all/module_linking.rs b/tests/all/module_linking.rs index 101655eff3..28fb802fae 100644 --- a/tests/all/module_linking.rs +++ b/tests/all/module_linking.rs @@ -39,7 +39,9 @@ fn compile() -> Result<()> { assert_eq!(m.imports().len(), 0); assert_eq!(m.exports().len(), 0); let bytes = m.serialize()?; - Module::new(&engine, &bytes)?; + unsafe { + Module::deserialize(&engine, &bytes)?; + } assert_eq!(m.imports().len(), 0); assert_eq!(m.exports().len(), 0); Ok(()) @@ -190,7 +192,6 @@ fn imports_exports() -> Result<()> { fn limit_instances() -> Result<()> { let mut config = Config::new(); config.wasm_module_linking(true); - config.max_instances(10); let engine = Engine::new(&config)?; let module = Module::new( &engine, @@ -216,7 +217,7 @@ fn limit_instances() -> Result<()> { ) "#, )?; - let store = Store::new(&engine); + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().instances(10).build()); let err = Instance::new(&store, &module, &[]).err().unwrap(); assert!( err.to_string().contains("resource limit exceeded"), @@ -231,7 +232,6 @@ fn limit_memories() -> Result<()> { let mut config = Config::new(); config.wasm_module_linking(true); config.wasm_multi_memory(true); - config.max_memories(10); let engine = Engine::new(&config)?; let module = Module::new( &engine, @@ -252,7 +252,7 @@ fn limit_memories() -> Result<()> { ) "#, )?; - let store = Store::new(&engine); + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().memories(10).build()); let err = Instance::new(&store, &module, &[]).err().unwrap(); assert!( err.to_string().contains("resource limit exceeded"), @@ -266,7 +266,6 @@ fn limit_memories() -> Result<()> { fn limit_tables() -> Result<()> { let mut config = Config::new(); config.wasm_module_linking(true); - config.max_tables(10); let engine = Engine::new(&config)?; let module = Module::new( &engine, @@ -287,7 +286,7 @@ fn limit_tables() -> Result<()> { ) "#, )?; - let store = Store::new(&engine); + let store = Store::new_with_limits(&engine, StoreLimitsBuilder::new().tables(10).build()); let err = Instance::new(&store, &module, &[]).err().unwrap(); assert!( err.to_string().contains("resource limit exceeded"), diff --git a/tests/all/module_serialize.rs b/tests/all/module_serialize.rs index 4409659fcd..7f68b48a58 100644 --- a/tests/all/module_serialize.rs +++ b/tests/all/module_serialize.rs @@ -6,8 +6,8 @@ fn serialize(engine: &Engine, wat: &'static str) -> Result> { Ok(module.serialize()?) } -fn deserialize_and_instantiate(store: &Store, buffer: &[u8]) -> Result { - let module = Module::new(store.engine(), buffer)?; +unsafe fn deserialize_and_instantiate(store: &Store, buffer: &[u8]) -> Result { + let module = Module::deserialize(store.engine(), buffer)?; Ok(Instance::new(&store, &module, &[])?) } @@ -17,7 +17,7 @@ fn test_version_mismatch() -> Result<()> { let mut buffer = serialize(&engine, "(module)")?; buffer[13 /* header length */ + 1 /* version length */] = 'x' as u8; - match Module::new(&engine, &buffer) { + match unsafe { Module::deserialize(&engine, &buffer) } { Ok(_) => bail!("expected deserialization to fail"), Err(e) => assert!(e .to_string() @@ -35,7 +35,7 @@ fn test_module_serialize_simple() -> Result<()> { )?; let store = Store::default(); - let instance = deserialize_and_instantiate(&store, &buffer)?; + let instance = unsafe { deserialize_and_instantiate(&store, &buffer)? }; let run = instance.get_typed_func::<(), i32>("run")?; let result = run.call(())?; @@ -53,7 +53,7 @@ fn test_module_serialize_fail() -> Result<()> { let mut config = Config::new(); config.cranelift_opt_level(OptLevel::None); let store = Store::new(&Engine::new(&config)?); - match deserialize_and_instantiate(&store, &buffer) { + match unsafe { deserialize_and_instantiate(&store, &buffer) } { Ok(_) => bail!("expected failure at deserialization"), Err(_) => (), } diff --git a/tests/all/traps.rs b/tests/all/traps.rs index 4fcc6f3882..af702c247b 100644 --- a/tests/all/traps.rs +++ b/tests/all/traps.rs @@ -591,3 +591,39 @@ note: run with `WASMTIME_BACKTRACE_DETAILS=1` environment variable to display mo ); Ok(()) } + +#[test] +fn multithreaded_traps() -> Result<()> { + // Compile and run unreachable on a thread, then moves over the whole store to another thread, + // and make sure traps are still correctly caught after notifying the store of the move. + let instance = { + let store = Store::default(); + let module = Module::new( + store.engine(), + r#"(module (func (export "run") unreachable))"#, + )?; + Instance::new(&store, &module, &[])? + }; + + assert!(instance.get_typed_func::<(), ()>("run")?.call(()).is_err()); + + struct SendInstance { + inner: Instance, + } + unsafe impl Send for SendInstance {} + + let instance = SendInstance { inner: instance }; + + let handle = std::thread::spawn(move || { + let instance = instance.inner; + assert!(instance + .get_typed_func::<(), ()>("run") + .unwrap() + .call(()) + .is_err()); + }); + + handle.join().expect("couldn't join thread"); + + Ok(()) +}