diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02185746bc..47efbda923 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,14 +73,11 @@ jobs: - uses: actions/checkout@v2 with: submodules: true - # Note that we use nightly Rust here to get intra-doc links which are a - # nightly-only feature right now. + # Note that we use nightly Rust for the doc_cfg feature (enabled via `nightlydoc` above) + # This version is an older nightly for the new x64 backend (see below) - uses: ./.github/actions/install-rust with: - # TODO (rust-lang/rust#79661): We are seeing an internal compiler error when - # building with the latest (2020-12-06) nightly; pin on a slightly older - # version for now. - toolchain: nightly-2020-11-29 + toolchain: nightly-2020-12-26 - run: cargo doc --no-deps --all --exclude wasmtime-cli --exclude test-programs --exclude cranelift-codegen-meta - run: cargo doc --package cranelift-codegen-meta --document-private-items - uses: actions/upload-artifact@v1 @@ -122,6 +119,7 @@ jobs: - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features jitdump - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features cache - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features async + - run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features uffd # Check some feature combinations of the `wasmtime-c-api` crate - run: cargo check --manifest-path crates/c-api/Cargo.toml --no-default-features @@ -167,7 +165,7 @@ jobs: # flags to rustc. - uses: ./.github/actions/install-rust with: - toolchain: nightly-2020-11-29 + toolchain: nightly - run: cargo install cargo-fuzz --vers "^0.8" - run: cargo fetch working-directory: ./fuzz @@ -224,7 +222,7 @@ jobs: rust: beta - build: nightly os: ubuntu-latest - rust: nightly-2020-11-29 + rust: nightly - build: macos os: macos-latest rust: stable @@ -303,6 +301,15 @@ jobs: env: RUST_BACKTRACE: 1 + # Test uffd functionality on Linux + - run: | + cargo test --features uffd -p wasmtime-runtime instance::allocator::pooling + cargo test --features uffd -p wasmtime-cli pooling_allocator + cargo test --features uffd -p wasmtime-cli wast::Cranelift + if: matrix.os == 'ubuntu-latest' + env: + RUST_BACKTRACE: 1 + # Build and test lightbeam. Note that # Lightbeam tests fail right now, but we don't want to block on that. - run: cargo build --package lightbeam @@ -312,8 +319,10 @@ jobs: RUST_BACKTRACE: 1 # Perform all tests (debug mode) for `wasmtime` with the experimental x64 - # backend. This runs on the nightly channel of Rust (because of issues with - # unifying Cargo features on stable) on Ubuntu. + # backend. This runs on an older nightly of Rust (because of issues with + # unifying Cargo features on stable) on Ubuntu such that it's new enough + # to build Wasmtime, but old enough where the -Z options being used + # haven't been stabilized yet. test_x64: name: Test x64 new backend runs-on: ubuntu-latest @@ -323,7 +332,7 @@ jobs: submodules: true - uses: ./.github/actions/install-rust with: - toolchain: nightly-2020-11-29 + toolchain: nightly-2020-12-26 - uses: ./.github/actions/define-llvm-env # Install wasm32 targets in order to build various tests throughout the @@ -334,7 +343,7 @@ jobs: # Run the x64 CI script. - run: ./ci/run-experimental-x64-ci.sh env: - CARGO_VERSION: "+nightly-2020-11-29" + CARGO_VERSION: "+nightly-2020-12-26" RUST_BACKTRACE: 1 # Build and test the wasi-nn module. @@ -347,7 +356,7 @@ jobs: submodules: true - uses: ./.github/actions/install-rust with: - toolchain: nightly-2020-11-29 + toolchain: nightly - run: rustup target add wasm32-wasi - uses: ./.github/actions/install-openvino - run: ./ci/run-wasi-nn-example.sh diff --git a/Cargo.lock b/Cargo.lock index 21f104be30..24627ff86e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -213,6 +213,25 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4865004a46a0aafb2a0a5eb19d3c9fc46ee5f063a6cfc605c69ac9ecf5263d" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", +] + [[package]] name = "bit-set" version = "0.5.2" @@ -1567,6 +1586,19 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" +[[package]] +name = "nix" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" +dependencies = [ + "bitflags", + "cc", + "cfg-if 0.1.10", + "libc", + "void", +] + [[package]] name = "nom" version = "5.1.2" @@ -1714,7 +1746,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fb64bef270a1ff665b0b2e28ebfa213e6205a007ce88223d020730225d6008f" dependencies = [ - "bindgen", + "bindgen 0.55.1", "cmake", ] @@ -2929,6 +2961,30 @@ dependencies = [ "winapi", ] +[[package]] +name = "userfaultfd" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d8164d4a8198fa546e7553b529f53e82907214a25fafda4a6f90d978b30a5c" +dependencies = [ + "bitflags", + "libc", + "nix", + "thiserror", + "userfaultfd-sys", +] + +[[package]] +name = "userfaultfd-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ada4f4ae167325015f52cc65f9fb6c251b868d8fb3b6dd0ce2d60e497c4870a" +dependencies = [ + "bindgen 0.57.0", + "cc", + "cfg-if 0.1.10", +] + [[package]] name = "vec_map" version = "0.8.2" @@ -2941,6 +2997,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "wait-timeout" version = "0.2.0" @@ -3273,6 +3335,7 @@ dependencies = [ "indexmap", "log", "more-asserts", + "region", "serde", "thiserror", "wasmparser", @@ -3398,6 +3461,7 @@ dependencies = [ name = "wasmtime-runtime" version = "0.24.0" dependencies = [ + "anyhow", "backtrace", "cc", "cfg-if 1.0.0", @@ -3408,8 +3472,10 @@ dependencies = [ "memoffset", "more-asserts", "psm", + "rand 0.7.3", "region", "thiserror", + "userfaultfd", "wasmtime-environ", "winapi", ] diff --git a/Cargo.toml b/Cargo.toml index ac76f1eaae..2ac7ce4331 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,7 @@ jitdump = ["wasmtime/jitdump"] vtune = ["wasmtime/vtune"] wasi-crypto = ["wasmtime-wasi-crypto"] wasi-nn = ["wasmtime-wasi-nn"] +uffd = ["wasmtime/uffd"] # Try the experimental, work-in-progress new x86_64 backend. This is not stable # as of June 2020. diff --git a/README.md b/README.md index 53c268baa6..2095e08ba1 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

build status zulip chat - min rustc + supported rustc stable Documentation Status

diff --git a/build.rs b/build.rs index 88307adf97..95f1c24be7 100644 --- a/build.rs +++ b/build.rs @@ -111,7 +111,8 @@ fn test_directory( let testsuite = &extract_name(path); for entry in dir_entries.iter() { - write_testsuite_tests(out, entry, testsuite, strategy)?; + write_testsuite_tests(out, entry, testsuite, strategy, false)?; + write_testsuite_tests(out, entry, testsuite, strategy, true)?; } Ok(dir_entries.len()) @@ -148,6 +149,7 @@ fn write_testsuite_tests( path: impl AsRef, testsuite: &str, strategy: &str, + pooling: bool, ) -> anyhow::Result<()> { let path = path.as_ref(); let testname = extract_name(path); @@ -160,14 +162,24 @@ fn write_testsuite_tests( )?; } else if ignore(testsuite, &testname, strategy) { writeln!(out, "#[ignore]")?; + } else if pooling { + // Ignore on aarch64 due to using QEMU for running tests (limited memory) + writeln!(out, r#"#[cfg_attr(target_arch = "aarch64", ignore)]"#)?; } - writeln!(out, "fn r#{}() {{", &testname)?; + + writeln!( + out, + "fn r#{}{}() {{", + &testname, + if pooling { "_pooling" } else { "" } + )?; writeln!(out, " let _ = env_logger::try_init();")?; writeln!( out, - " crate::wast::run_wast(r#\"{}\"#, crate::wast::Strategy::{}).unwrap();", + " crate::wast::run_wast(r#\"{}\"#, crate::wast::Strategy::{}, {}).unwrap();", path.display(), - strategy + strategy, + pooling )?; writeln!(out, "}}")?; writeln!(out)?; diff --git a/cranelift/wasm/src/module_translator.rs b/cranelift/wasm/src/module_translator.rs index ed71d7ce1c..819834f20e 100644 --- a/cranelift/wasm/src/module_translator.rs +++ b/cranelift/wasm/src/module_translator.rs @@ -101,6 +101,8 @@ pub fn translate_module<'data>( Payload::DataCountSection { count, range } => { validator.data_count_section(count, &range)?; + + // NOTE: the count here is the total segment count, not the passive segment count environ.reserve_passive_data(count)?; } diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index 3906c02393..a674658358 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -401,6 +401,12 @@ pub fn parse_element_section<'data>( )); } }; + // Check for offset + len overflow + if offset.checked_add(segments.len()).is_none() { + return Err(wasm_unsupported!( + "element segment offset and length overflows" + )); + } environ.declare_table_elements( TableIndex::from_u32(table_index), base, @@ -447,6 +453,12 @@ pub fn parse_data_section<'data>( )) } }; + // Check for offset + len overflow + if offset.checked_add(data.len()).is_none() { + return Err(wasm_unsupported!( + "data segment offset and length overflows" + )); + } environ.declare_data_initialization( MemoryIndex::from_u32(memory_index), base, diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index 3bb7f0c5c5..f84bc320cd 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -61,8 +61,8 @@ pub extern "C" fn wasmtime_config_consume_fuel_set(c: &mut wasm_config_t, enable } #[no_mangle] -pub extern "C" fn wasmtime_config_max_wasm_stack_set(c: &mut wasm_config_t, size: usize) { - c.config.max_wasm_stack(size); +pub extern "C" fn wasmtime_config_max_wasm_stack_set(c: &mut wasm_config_t, size: usize) -> bool { + c.config.max_wasm_stack(size).is_ok() } #[no_mangle] diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 230295aecb..12c39d6ac6 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -13,6 +13,7 @@ edition = "2018" [dependencies] anyhow = "1.0" +region = "2.2.0" cranelift-codegen = { path = "../../cranelift/codegen", version = "0.71.0", features = ["enable-serde"] } cranelift-entity = { path = "../../cranelift/entity", version = "0.71.0", features = ["enable-serde"] } cranelift-wasm = { path = "../../cranelift/wasm", version = "0.71.0", features = ["enable-serde"] } diff --git a/crates/environ/src/data_structures.rs b/crates/environ/src/data_structures.rs index 2fc9e9ecd0..07f2aedaec 100644 --- a/crates/environ/src/data_structures.rs +++ b/crates/environ/src/data_structures.rs @@ -20,7 +20,7 @@ pub mod isa { } pub mod entity { - pub use cranelift_entity::{packed_option, BoxedSlice, EntityRef, PrimaryMap}; + pub use cranelift_entity::{packed_option, BoxedSlice, EntityRef, EntitySet, PrimaryMap}; } pub mod wasm { diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index b25682acf2..ee4ff050dc 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -6,24 +6,10 @@ use cranelift_codegen::ir; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_wasm::*; use indexmap::IndexMap; -use more_asserts::assert_ge; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; -/// A WebAssembly table initializer. -#[derive(Clone, Debug, Hash, Serialize, Deserialize)] -pub struct TableElements { - /// The index of a table to initialize. - pub table_index: TableIndex, - /// Optionally, a global variable giving a base index. - pub base: Option, - /// The offset to add to the base. - pub offset: usize, - /// The values to write into the table elements. - pub elements: Box<[FuncIndex]>, -} - /// Implemenation styles for WebAssembly linear memory. #[derive(Debug, Clone, Hash, Serialize, Deserialize)] pub enum MemoryStyle { @@ -42,10 +28,20 @@ impl MemoryStyle { // A heap with a maximum that doesn't exceed the static memory bound specified by the // tunables make it static. // - // If the module doesn't declare an explicit maximum treat it as 4GiB. - let maximum = memory.maximum.unwrap_or(WASM_MAX_PAGES); - if maximum <= tunables.static_memory_bound { - assert_ge!(tunables.static_memory_bound, memory.minimum); + // If the module doesn't declare an explicit maximum treat it as 4GiB when not + // requested to use the static memory bound itself as the maximum. + let maximum = std::cmp::min( + memory.maximum.unwrap_or(WASM_MAX_PAGES), + if tunables.static_memory_bound_is_maximum { + std::cmp::min(tunables.static_memory_bound, WASM_MAX_PAGES) + } else { + WASM_MAX_PAGES + }, + ); + + // Ensure the minimum is less than the maximum; the minimum might exceed the maximum + // when the memory is artificially bounded via `static_memory_bound_is_maximum` above + if memory.minimum <= maximum && maximum <= tunables.static_memory_bound { return ( Self::Static { bound: tunables.static_memory_bound, @@ -83,7 +79,157 @@ impl MemoryPlan { } } -/// Implemenation styles for WebAssembly tables. +/// A WebAssembly linear memory initializer. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MemoryInitializer { + /// The index of a linear memory to initialize. + pub memory_index: MemoryIndex, + /// Optionally, a global variable giving a base index. + pub base: Option, + /// The offset to add to the base. + pub offset: usize, + /// The data to write into the linear memory. + pub data: Box<[u8]>, +} + +/// The type of WebAssembly linear memory initialization to use for a module. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum MemoryInitialization { + /// Memory initialization is segmented. + /// + /// Segmented initialization can be used for any module, but it is required if: + /// + /// * A data segment referenced an imported memory. + /// * A data segment uses a global base. + /// + /// Segmented initialization is performed by processing the complete set of data segments + /// when the module is instantiated. + /// + /// This is the default memory initialization type. + Segmented(Vec), + /// Memory initialization is paged. + /// + /// To be paged, the following requirements must be met: + /// + /// * All data segments must reference defined memories. + /// * All data segments must not use a global base. + /// + /// Paged initialization is performed by copying (or mapping) entire WebAssembly pages to each linear memory. + /// + /// The `uffd` feature makes use of this type of memory initialization because it can instruct the kernel + /// to back an entire WebAssembly page from an existing set of in-memory pages. + /// + /// By processing the data segments at module compilation time, the uffd fault handler doesn't have to do + /// any work to point the kernel at the right linear memory page to use. + Paged { + /// The map of defined memory index to a list of initialization pages. + /// The list of page data is sparse, with None representing a zero page. + /// Each page of initialization data is WebAssembly page-sized (64 KiB). + /// The size of the list will be the maximum page written to by a data segment. + map: PrimaryMap>>>, + /// Whether or not an out-of-bounds data segment was observed. + /// This is used to fail module instantiation after the pages are initialized. + out_of_bounds: bool, + }, +} + +impl MemoryInitialization { + /// Attempts to convert segmented memory initialization into paged initialization for the given module. + /// + /// Returns `None` if the initialization cannot be paged or if it is already paged. + pub fn to_paged(&self, module: &Module) -> Option { + const WASM_PAGE_SIZE: usize = crate::WASM_PAGE_SIZE as usize; + + match self { + Self::Paged { .. } => None, + Self::Segmented(initializers) => { + let num_defined_memories = module.memory_plans.len() - module.num_imported_memories; + let mut out_of_bounds = false; + let mut map = PrimaryMap::with_capacity(num_defined_memories); + + for _ in 0..num_defined_memories { + map.push(Vec::new()); + } + + for initializer in initializers { + match ( + module.defined_memory_index(initializer.memory_index), + initializer.base.is_some(), + ) { + (None, _) | (_, true) => { + // If the initializer references an imported memory or uses a global base, + // the complete set of segments will need to be processed at module instantiation + return None; + } + (Some(index), false) => { + if out_of_bounds { + continue; + } + + // Perform a bounds check on the segment + // As this segment is referencing a defined memory without a global base, the last byte + // written to by the segment cannot exceed the memory's initial minimum size + if (initializer.offset + initializer.data.len()) + > ((module.memory_plans[initializer.memory_index].memory.minimum + as usize) + * WASM_PAGE_SIZE) + { + out_of_bounds = true; + continue; + } + + let pages = &mut map[index]; + let mut page_index = initializer.offset / WASM_PAGE_SIZE; + let mut page_offset = initializer.offset % WASM_PAGE_SIZE; + let mut data_offset = 0; + let mut data_remaining = initializer.data.len(); + + if data_remaining == 0 { + continue; + } + + // Copy the initialization data by each WebAssembly-sized page (64 KiB) + loop { + if page_index >= pages.len() { + pages.resize(page_index + 1, None); + } + + let page = pages[page_index].get_or_insert_with(|| { + vec![0; WASM_PAGE_SIZE].into_boxed_slice() + }); + let len = + std::cmp::min(data_remaining, WASM_PAGE_SIZE - page_offset); + + page[page_offset..page_offset + len].copy_from_slice( + &initializer.data[data_offset..(data_offset + len)], + ); + + if len == data_remaining { + break; + } + + page_index += 1; + page_offset = 0; + data_offset += len; + data_remaining -= len; + } + } + }; + } + + Some(Self::Paged { map, out_of_bounds }) + } + } + } +} + +impl Default for MemoryInitialization { + fn default() -> Self { + Self::Segmented(Vec::new()) + } +} + +/// Implementation styles for WebAssembly tables. #[derive(Debug, Clone, Hash, Serialize, Deserialize)] pub enum TableStyle { /// Signatures are stored in the table and checked in the caller. @@ -115,6 +261,19 @@ impl TablePlan { } } +/// A WebAssembly table initializer. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TableInitializer { + /// The index of a table to initialize. + pub table_index: TableIndex, + /// Optionally, a global variable giving a base index. + pub base: Option, + /// The offset to add to the base. + pub offset: usize, + /// The values to write into the table elements. + pub elements: Box<[FuncIndex]>, +} + /// Different types that can appear in a module. /// /// Note that each of these variants are intended to index further into a @@ -155,16 +314,25 @@ pub struct Module { pub start_func: Option, /// WebAssembly table initializers. - pub table_elements: Vec, + pub table_initializers: Vec, + + /// WebAssembly linear memory initializer. + pub memory_initialization: MemoryInitialization, /// WebAssembly passive elements. - pub passive_elements: HashMap>, + pub passive_elements: Vec>, + + /// The map from passive element index (element segment index space) to index in `passive_elements`. + pub passive_elements_map: HashMap, /// WebAssembly passive data segments. #[serde(with = "passive_data_serde")] - pub passive_data: HashMap>, + pub passive_data: Vec>, - /// WebAssembly table initializers. + /// The map from passive data index (data segment index space) to index in `passive_data`. + pub passive_data_map: HashMap, + + /// WebAssembly function names. pub func_names: HashMap, /// Types declared in the wasm module. @@ -229,7 +397,7 @@ pub enum Initializer { export: String, }, - /// A module is being instantiated with previously configured intializers + /// A module is being instantiated with previously configured initializers /// as arguments. Instantiate { /// The module that this instance is instantiating. @@ -241,7 +409,7 @@ pub enum Initializer { /// A module is being created from a set of compiled artifacts. CreateModule { - /// The index of the artifact that's being convereted into a module. + /// The index of the artifact that's being converted into a module. artifact_index: usize, /// The list of artifacts that this module value will be inheriting. artifacts: Vec, @@ -272,7 +440,8 @@ impl Module { /// Get the given passive element, if it exists. pub fn get_passive_element(&self, index: ElemIndex) -> Option<&[FuncIndex]> { - self.passive_elements.get(&index).map(|es| &**es) + let index = *self.passive_elements_map.get(&index)?; + Some(self.passive_elements[index].as_ref()) } /// Convert a `DefinedFuncIndex` into a `FuncIndex`. @@ -419,47 +588,45 @@ pub struct InstanceSignature { } mod passive_data_serde { - use super::{Arc, DataIndex, HashMap}; - use serde::{de::MapAccess, de::Visitor, ser::SerializeMap, Deserializer, Serializer}; + use super::Arc; + use serde::{de::SeqAccess, de::Visitor, ser::SerializeSeq, Deserializer, Serializer}; use std::fmt; - pub(super) fn serialize( - data: &HashMap>, - ser: S, - ) -> Result + pub(super) fn serialize(data: &Vec>, ser: S) -> Result where S: Serializer, { - let mut map = ser.serialize_map(Some(data.len()))?; - for (k, v) in data { - map.serialize_entry(k, v.as_ref())?; + let mut seq = ser.serialize_seq(Some(data.len()))?; + for v in data { + seq.serialize_element(v.as_ref())?; } - map.end() + seq.end() } struct PassiveDataVisitor; impl<'de> Visitor<'de> for PassiveDataVisitor { - type Value = HashMap>; + type Value = Vec>; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a passive_data map") + formatter.write_str("a passive data sequence") } - fn visit_map(self, mut access: M) -> Result + + fn visit_seq(self, mut access: M) -> Result where - M: MapAccess<'de>, + M: SeqAccess<'de>, { - let mut map = HashMap::with_capacity(access.size_hint().unwrap_or(0)); - while let Some((key, value)) = access.next_entry::<_, Vec>()? { - map.insert(key, value.into()); + let mut data = Vec::with_capacity(access.size_hint().unwrap_or(0)); + while let Some(value) = access.next_element::>()? { + data.push(value.into()); } - Ok(map) + Ok(data) } } - pub(super) fn deserialize<'de, D>(de: D) -> Result>, D::Error> + pub(super) fn deserialize<'de, D>(de: D) -> Result>, D::Error> where D: Deserializer<'de>, { - de.deserialize_map(PassiveDataVisitor) + de.deserialize_seq(PassiveDataVisitor) } } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 9d1f89cb1f..2f53bb4af8 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -1,6 +1,6 @@ use crate::module::{ - Initializer, InstanceSignature, MemoryPlan, Module, ModuleSignature, ModuleType, ModuleUpvar, - TableElements, TablePlan, TypeTables, + Initializer, InstanceSignature, MemoryInitialization, MemoryInitializer, MemoryPlan, Module, + ModuleSignature, ModuleType, ModuleUpvar, TableInitializer, TablePlan, TypeTables, }; use crate::tunables::Tunables; use cranelift_codegen::ir; @@ -13,7 +13,6 @@ use cranelift_wasm::{ ModuleIndex, ModuleTypeIndex, SignatureIndex, Table, TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult, }; -use serde::{Deserialize, Serialize}; use std::collections::{hash_map::Entry, HashMap}; use std::convert::TryFrom; use std::mem; @@ -60,9 +59,6 @@ pub struct ModuleTranslation<'data> { /// References to the function bodies. pub function_body_inputs: PrimaryMap>, - /// References to the data initializers. - pub data_initializers: Vec>, - /// DWARF debug information, if enabled, parsed from the module. pub debuginfo: DebugInfoData<'data>, @@ -684,7 +680,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn reserve_table_elements(&mut self, num: u32) -> WasmResult<()> { self.result .module - .table_elements + .table_initializers .reserve_exact(usize::try_from(num).unwrap()); Ok(()) } @@ -696,12 +692,15 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data offset: usize, elements: Box<[FuncIndex]>, ) -> WasmResult<()> { - self.result.module.table_elements.push(TableElements { - table_index, - base, - offset, - elements, - }); + self.result + .module + .table_initializers + .push(TableInitializer { + table_index, + base, + offset, + elements, + }); Ok(()) } @@ -710,11 +709,13 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data elem_index: ElemIndex, segments: Box<[FuncIndex]>, ) -> WasmResult<()> { + let index = self.result.module.passive_elements.len(); + self.result.module.passive_elements.push(segments); let old = self .result .module - .passive_elements - .insert(elem_index, segments); + .passive_elements_map + .insert(elem_index, index); debug_assert!( old.is_none(), "should never get duplicate element indices, that would be a bug in `cranelift_wasm`'s \ @@ -758,9 +759,12 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn reserve_data_initializers(&mut self, num: u32) -> WasmResult<()> { - self.result - .data_initializers - .reserve_exact(usize::try_from(num).unwrap()); + match &mut self.result.module.memory_initialization { + MemoryInitialization::Segmented(initializers) => { + initializers.reserve_exact(usize::try_from(num).unwrap()) + } + _ => unreachable!(), + } Ok(()) } @@ -771,28 +775,35 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data offset: usize, data: &'data [u8], ) -> WasmResult<()> { - self.result.data_initializers.push(DataInitializer { - location: DataInitializerLocation { - memory_index, - base, - offset, - }, - data, - }); + match &mut self.result.module.memory_initialization { + MemoryInitialization::Segmented(initializers) => { + initializers.push(MemoryInitializer { + memory_index, + base, + offset, + data: data.into(), + }); + } + _ => unreachable!(), + } Ok(()) } - fn reserve_passive_data(&mut self, count: u32) -> WasmResult<()> { - self.result.module.passive_data.reserve(count as usize); + fn reserve_passive_data(&mut self, _count: u32) -> WasmResult<()> { + // Note: the count passed in here is the *total* segment count + // There is no way to reserve for just the passive segments as they are discovered when iterating the data section entries + // Given that the total segment count might be much larger than the passive count, do not reserve Ok(()) } fn declare_passive_data(&mut self, data_index: DataIndex, data: &'data [u8]) -> WasmResult<()> { + let index = self.result.module.passive_data.len(); + self.result.module.passive_data.push(Arc::from(data)); let old = self .result .module - .passive_data - .insert(data_index, Arc::from(data)); + .passive_data_map + .insert(data_index, index); debug_assert!( old.is_none(), "a module can't have duplicate indices, this would be a cranelift-wasm bug" @@ -1065,26 +1076,3 @@ pub fn translate_signature(mut sig: ir::Signature, pointer_type: ir::Type) -> ir sig.params.insert(1, AbiParam::new(pointer_type)); sig } - -/// A memory index and offset within that memory where a data initialization -/// should is to be performed. -#[derive(Clone, Serialize, Deserialize)] -pub struct DataInitializerLocation { - /// The index of the memory to initialize. - pub memory_index: MemoryIndex, - - /// Optionally a globalvar base to initialize at. - pub base: Option, - - /// A constant offset to initialize at. - pub offset: usize, -} - -/// A data initializer for linear memory. -pub struct DataInitializer<'data> { - /// The location where the initialization is to be performed. - pub location: DataInitializerLocation, - - /// The initialization data. - pub data: &'data [u8], -} diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs index bd86fef237..4e5aba9145 100644 --- a/crates/environ/src/tunables.rs +++ b/crates/environ/src/tunables.rs @@ -27,6 +27,9 @@ pub struct Tunables { /// Whether or not fuel is enabled for generated code, meaning that fuel /// will be consumed every time a wasm instruction is executed. pub consume_fuel: bool, + + /// Whether or not to treat the static memory bound as the maximum for unbounded heaps. + pub static_memory_bound_is_maximum: bool, } impl Default for Tunables { @@ -62,6 +65,7 @@ impl Default for Tunables { parse_wasm_debuginfo: true, interruptable: false, consume_fuel: false, + static_memory_bound_is_maximum: false, } } } diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 7f74f46754..042f0b9dcf 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -50,6 +50,7 @@ fn align(offset: u32, width: u32) -> u32 { /// This class computes offsets to fields within `VMContext` and other /// related structs that JIT code accesses directly. +#[derive(Debug, Clone, Copy)] pub struct VMOffsets { /// The size in bytes of a pointer on the target. pub pointer_size: u8, diff --git a/crates/fiber/src/lib.rs b/crates/fiber/src/lib.rs index bafae2f01c..9a8d057f35 100644 --- a/crates/fiber/src/lib.rs +++ b/crates/fiber/src/lib.rs @@ -51,6 +51,27 @@ impl<'a, Resume, Yield, Return> Fiber<'a, Resume, Yield, Return> { }) } + /// Creates a new fiber with existing stack space that will execute `func`. + /// + /// This function returns a `Fiber` which, when resumed, will execute `func` + /// to completion. When desired the `func` can suspend itself via + /// `Fiber::suspend`. + /// + /// # Safety + /// + /// The caller must properly allocate the stack space with a guard page and + /// make the pages accessible for correct behavior. + pub unsafe fn new_with_stack( + top_of_stack: *mut u8, + func: impl FnOnce(Resume, &Suspend) -> Return + 'a, + ) -> io::Result> { + Ok(Fiber { + inner: imp::Fiber::new_with_stack(top_of_stack, func)?, + done: Cell::new(false), + _phantom: PhantomData, + }) + } + /// Resumes execution of this fiber. /// /// This function will transfer execution to the fiber and resume from where diff --git a/crates/fiber/src/unix.rs b/crates/fiber/src/unix.rs index 2c1069041e..0cc57ca319 100644 --- a/crates/fiber/src/unix.rs +++ b/crates/fiber/src/unix.rs @@ -35,10 +35,10 @@ use std::io; use std::ptr; pub struct Fiber { - // Description of the mmap region we own. This should be abstracted - // eventually so we aren't personally mmap-ing this region. - mmap: *mut libc::c_void, - mmap_len: usize, + // The top of the stack; for stacks allocated by the fiber implementation itself, + // the base address of the allocation will be `top_of_stack.sub(alloc_len.unwrap())` + top_of_stack: *mut u8, + alloc_len: Option, } pub struct Suspend { @@ -66,21 +66,40 @@ where } impl Fiber { - pub fn new(stack_size: usize, func: F) -> io::Result + pub fn new(stack_size: usize, func: F) -> io::Result + where + F: FnOnce(A, &super::Suspend) -> C, + { + let fiber = Self::alloc_with_stack(stack_size)?; + fiber.init(func); + Ok(fiber) + } + + pub fn new_with_stack(top_of_stack: *mut u8, func: F) -> io::Result + where + F: FnOnce(A, &super::Suspend) -> C, + { + let fiber = Self { + top_of_stack, + alloc_len: None, + }; + + fiber.init(func); + + Ok(fiber) + } + + fn init(&self, func: F) where F: FnOnce(A, &super::Suspend) -> C, { - let fiber = Fiber::alloc_with_stack(stack_size)?; unsafe { - // Initialize the top of the stack to be resumed from - let top_of_stack = fiber.top_of_stack(); let data = Box::into_raw(Box::new(func)).cast(); - wasmtime_fiber_init(top_of_stack, fiber_start::, data); - Ok(fiber) + wasmtime_fiber_init(self.top_of_stack, fiber_start::, data); } } - fn alloc_with_stack(stack_size: usize) -> io::Result { + fn alloc_with_stack(stack_size: usize) -> io::Result { unsafe { // Round up our stack size request to the nearest multiple of the // page size. @@ -104,7 +123,10 @@ impl Fiber { if mmap == libc::MAP_FAILED { return Err(io::Error::last_os_error()); } - let ret = Fiber { mmap, mmap_len }; + let ret = Self { + top_of_stack: mmap.cast::().add(mmap_len), + alloc_len: Some(mmap_len), + }; let res = libc::mprotect( mmap.cast::().add(page_size).cast(), stack_size, @@ -124,27 +146,24 @@ impl Fiber { // stack, otherwise known as our reserved slot for this information. // // In the diagram above this is updating address 0xAff8 - let top_of_stack = self.top_of_stack(); - let addr = top_of_stack.cast::().offset(-1); + let addr = self.top_of_stack.cast::().offset(-1); addr.write(result as *const _ as usize); - wasmtime_fiber_switch(top_of_stack); + wasmtime_fiber_switch(self.top_of_stack); // null this out to help catch use-after-free addr.write(0); } } - - unsafe fn top_of_stack(&self) -> *mut u8 { - self.mmap.cast::().add(self.mmap_len) - } } impl Drop for Fiber { fn drop(&mut self) { unsafe { - let ret = libc::munmap(self.mmap, self.mmap_len); - debug_assert!(ret == 0); + if let Some(alloc_len) = self.alloc_len { + let ret = libc::munmap(self.top_of_stack.sub(alloc_len) as _, alloc_len); + debug_assert!(ret == 0); + } } } } diff --git a/crates/fiber/src/windows.rs b/crates/fiber/src/windows.rs index 69a5f161e6..b2d657eb88 100644 --- a/crates/fiber/src/windows.rs +++ b/crates/fiber/src/windows.rs @@ -3,6 +3,7 @@ use std::cell::Cell; use std::io; use std::ptr; use winapi::shared::minwindef::*; +use winapi::shared::winerror::ERROR_NOT_SUPPORTED; use winapi::um::fibersapi::*; use winapi::um::winbase::*; @@ -40,7 +41,7 @@ where } impl Fiber { - pub fn new(stack_size: usize, func: F) -> io::Result + pub fn new(stack_size: usize, func: F) -> io::Result where F: FnOnce(A, &super::Suspend) -> C, { @@ -61,11 +62,18 @@ impl Fiber { drop(Box::from_raw(state.initial_closure.get().cast::())); Err(io::Error::last_os_error()) } else { - Ok(Fiber { fiber, state }) + Ok(Self { fiber, state }) } } } + pub fn new_with_stack(_top_of_stack: *mut u8, _func: F) -> io::Result + where + F: FnOnce(A, &super::Suspend) -> C, + { + Err(io::Error::from_raw_os_error(ERROR_NOT_SUPPORTED as i32)) + } + pub(crate) fn resume(&self, result: &Cell>) { unsafe { let is_fiber = IsThreadAFiber() != 0; diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 803d88eb43..b4f6faef74 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -25,7 +25,7 @@ wasmtime-debug = { path = "../debug", version = "0.24.0" } wasmtime-profiling = { path = "../profiling", version = "0.24.0" } wasmtime-obj = { path = "../obj", version = "0.24.0" } rayon = { version = "1.0", optional = true } -region = "2.1.0" +region = "2.2.0" thiserror = "1.0.4" target-lexicon = { version = "0.11.0", default-features = false } wasmparser = "0.76" diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index f4ba5c3017..49a2d0ecd4 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -25,7 +25,7 @@ struct CodeMemoryEntry { impl CodeMemoryEntry { fn with_capacity(cap: usize) -> Result { - let mmap = ManuallyDrop::new(Mmap::with_at_least(cap)?); + let mmap = ManuallyDrop::new(Mmap::with_at_least(cap).map_err(|e| e.to_string())?); let registry = ManuallyDrop::new(UnwindRegistry::new(mmap.as_ptr() as usize)); Ok(Self { mmap, diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 655432e00f..df6a17fa24 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -11,7 +11,6 @@ use object::File as ObjectFile; #[cfg(feature = "parallel-compilation")] use rayon::prelude::*; use serde::{Deserialize, Serialize}; -use std::any::Any; use std::ops::Range; use std::sync::Arc; use thiserror::Error; @@ -22,16 +21,11 @@ use wasmtime_environ::wasm::{ DefinedFuncIndex, InstanceTypeIndex, ModuleTypeIndex, SignatureIndex, WasmFuncType, }; use wasmtime_environ::{ - CompileError, DataInitializer, DataInitializerLocation, DebugInfoData, FunctionAddressMap, - InstanceSignature, Module, ModuleEnvironment, ModuleSignature, ModuleTranslation, - StackMapInformation, TrapInformation, + CompileError, DebugInfoData, FunctionAddressMap, InstanceSignature, Module, ModuleEnvironment, + ModuleSignature, ModuleTranslation, StackMapInformation, TrapInformation, }; use wasmtime_profiling::ProfilingAgent; -use wasmtime_runtime::{ - GdbJitImageRegistration, Imports, InstanceHandle, InstantiationError, RuntimeMemoryCreator, - StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, VMInterrupts, - VMSharedSignatureIndex, VMTrampoline, -}; +use wasmtime_runtime::{GdbJitImageRegistration, InstantiationError, VMFunctionBody, VMTrampoline}; /// An error condition while setting up a wasm instance, be it validation, /// compilation, or instantiation. @@ -59,7 +53,8 @@ pub enum SetupError { #[derive(Serialize, Deserialize)] pub struct CompilationArtifacts { /// Module metadata. - module: Module, + #[serde(with = "arc_serde")] + module: Arc, /// ELF image with functions code. obj: Box<[u8]>, @@ -67,9 +62,6 @@ pub struct CompilationArtifacts { /// Unwind information for function code. unwind_info: Box<[ObjectUnwindInfo]>, - /// Data initiailizers. - data_initializers: Box<[OwnedDataInitializer]>, - /// Descriptions of compiled functions funcs: PrimaryMap, @@ -102,9 +94,14 @@ struct DebugInfo { impl CompilationArtifacts { /// Creates a `CompilationArtifacts` for a singular translated wasm module. + /// + /// The `use_paged_init` argument controls whether or not an attempt is made to + /// organize linear memory initialization data as entire pages or to leave + /// the memory initialization data as individual segments. pub fn build( compiler: &Compiler, data: &[u8], + use_paged_mem_init: bool, ) -> Result<(usize, Vec, TypeTables), SetupError> { let (main_module, translations, types) = ModuleEnvironment::new( compiler.frontend_config(), @@ -123,30 +120,28 @@ impl CompilationArtifacts { } = compiler.compile(&mut translation, &types)?; let ModuleTranslation { - module, - data_initializers, + mut module, debuginfo, has_unparsed_debuginfo, .. } = translation; - let data_initializers = data_initializers - .into_iter() - .map(OwnedDataInitializer::new) - .collect::>() - .into_boxed_slice(); + if use_paged_mem_init { + if let Some(init) = module.memory_initialization.to_paged(&module) { + module.memory_initialization = init; + } + } let obj = obj.write().map_err(|_| { - SetupError::Instantiate(InstantiationError::Resource( - "failed to create image memory".to_string(), - )) + SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!( + "failed to create image memory" + ))) })?; Ok(CompilationArtifacts { - module, + module: Arc::new(module), obj: obj.into_boxed_slice(), unwind_info: unwind_info.into_boxed_slice(), - data_initializers, funcs: funcs .into_iter() .map(|(_, func)| FunctionInfo { @@ -208,7 +203,6 @@ pub struct ModuleCode { /// A compiled wasm module, ready to be instantiated. pub struct CompiledModule { artifacts: CompilationArtifacts, - module: Arc, code: Arc, finished_functions: FinishedFunctions, trampolines: PrimaryMap, @@ -242,7 +236,7 @@ impl CompiledModule { &artifacts.unwind_info, ) .map_err(|message| { - SetupError::Instantiate(InstantiationError::Resource(format!( + SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!( "failed to build code memory for functions: {}", message ))) @@ -267,7 +261,6 @@ impl CompiledModule { let finished_functions = FinishedFunctions(finished_functions); Ok(Arc::new(Self { - module: Arc::new(artifacts.module.clone()), artifacts, code: Arc::new(ModuleCode { code_memory, @@ -278,62 +271,19 @@ impl CompiledModule { })) } - /// Crate an `Instance` from this `CompiledModule`. - /// - /// Note that if only one instance of this module is needed, it may be more - /// efficient to call the top-level `instantiate`, since that avoids copying - /// the data initializers. - /// - /// # Unsafety - /// - /// See `InstanceHandle::new` - pub unsafe fn instantiate( - &self, - imports: Imports<'_>, - lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, - mem_creator: Option<&dyn RuntimeMemoryCreator>, - interrupts: *const VMInterrupts, - host_state: Box, - externref_activations_table: *mut VMExternRefActivationsTable, - stack_map_registry: *mut StackMapRegistry, - ) -> Result { - InstanceHandle::new( - self.module.clone(), - &self.finished_functions.0, - imports, - mem_creator, - lookup_shared_signature, - host_state, - interrupts, - externref_activations_table, - stack_map_registry, - ) - } /// Extracts `CompilationArtifacts` from the compiled module. pub fn compilation_artifacts(&self) -> &CompilationArtifacts { &self.artifacts } - /// Returns data initializers to pass to `InstanceHandle::initialize` - pub fn data_initializers(&self) -> Vec> { - self.artifacts - .data_initializers - .iter() - .map(|init| DataInitializer { - location: init.location.clone(), - data: &*init.data, - }) - .collect() - } - /// Return a reference-counting pointer to a module. pub fn module(&self) -> &Arc { - &self.module + &self.artifacts.module } /// Return a reference to a mutable module (if possible). pub fn module_mut(&mut self) -> Option<&mut Module> { - Arc::get_mut(&mut self.module) + Arc::get_mut(&mut self.artifacts.module) } /// Returns the map of all finished JIT functions compiled for this module @@ -470,26 +420,6 @@ impl SymbolizeContext { } } -/// Similar to `DataInitializer`, but owns its own copy of the data rather -/// than holding a slice of the original module. -#[derive(Clone, Serialize, Deserialize)] -pub struct OwnedDataInitializer { - /// The location where the initialization is to be performed. - location: DataInitializerLocation, - - /// The initialization data. - data: Box<[u8]>, -} - -impl OwnedDataInitializer { - fn new(borrowed: DataInitializer<'_>) -> Self { - Self { - location: borrowed.location.clone(), - data: borrowed.data.to_vec().into_boxed_slice(), - } - } -} - fn create_dbg_image( obj: Vec, code_range: (*const u8, usize), @@ -586,3 +516,24 @@ impl From> for DebugInfo { } } } + +mod arc_serde { + use super::Arc; + use serde::{de::Deserialize, ser::Serialize, Deserializer, Serializer}; + + pub(super) fn serialize(arc: &Arc, ser: S) -> Result + where + S: Serializer, + T: Serialize, + { + (**arc).serialize(ser) + } + + pub(super) fn deserialize<'de, D, T>(de: D) -> Result, D::Error> + where + D: Deserializer<'de>, + T: Deserialize<'de>, + { + Ok(Arc::new(T::deserialize(de)?)) + } +} diff --git a/crates/jit/src/trampoline.rs b/crates/jit/src/trampoline.rs index aebc4f1bc8..bb470dbcb3 100644 --- a/crates/jit/src/trampoline.rs +++ b/crates/jit/src/trampoline.rs @@ -38,7 +38,9 @@ pub fn make_trampoline( assert!(compiled_function.relocations.is_empty()); let ptr = code_memory .allocate_for_function(&compiled_function) - .map_err(|message| SetupError::Instantiate(InstantiationError::Resource(message)))? + .map_err(|message| { + SetupError::Instantiate(InstantiationError::Resource(anyhow::anyhow!(message))) + })? .as_ptr(); Ok(unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) }) } diff --git a/crates/obj/src/data_segment.rs b/crates/obj/src/data_segment.rs index 159922f25b..3e4184eb20 100644 --- a/crates/obj/src/data_segment.rs +++ b/crates/obj/src/data_segment.rs @@ -1,12 +1,12 @@ use anyhow::Result; use object::write::{Object, StandardSection, Symbol, SymbolSection}; use object::{SymbolFlags, SymbolKind, SymbolScope}; -use wasmtime_environ::DataInitializer; +use wasmtime_environ::MemoryInitializer; /// Declares data segment symbol pub fn declare_data_segment( obj: &mut Object, - _data_initaliazer: &DataInitializer, + _memory_initializer: &MemoryInitializer, index: usize, ) -> Result<()> { let name = format!("_memory_{}", index); @@ -26,12 +26,12 @@ pub fn declare_data_segment( /// Emit segment data and initialization location pub fn emit_data_segment( obj: &mut Object, - data_initaliazer: &DataInitializer, + memory_initializer: &MemoryInitializer, index: usize, ) -> Result<()> { let name = format!("_memory_{}", index); let symbol_id = obj.symbol_id(name.as_bytes()).unwrap(); let section_id = obj.section_id(StandardSection::Data); - obj.add_symbol_data(symbol_id, section_id, data_initaliazer.data, 1); + obj.add_symbol_data(symbol_id, section_id, &memory_initializer.data, 1); Ok(()) } diff --git a/crates/obj/src/module.rs b/crates/obj/src/module.rs index 2adf1aa393..4150d1c8b8 100644 --- a/crates/obj/src/module.rs +++ b/crates/obj/src/module.rs @@ -7,7 +7,7 @@ use object::write::{Object, Relocation, StandardSection, Symbol, SymbolSection}; use object::{RelocationEncoding, RelocationKind, SymbolFlags, SymbolKind, SymbolScope}; use wasmtime_debug::DwarfSection; use wasmtime_environ::isa::TargetFrontendConfig; -use wasmtime_environ::{CompiledFunctions, DataInitializer, Module}; +use wasmtime_environ::{CompiledFunctions, MemoryInitialization, Module}; fn emit_vmcontext_init( obj: &mut Object, @@ -54,24 +54,32 @@ pub fn emit_module( target_config: &TargetFrontendConfig, compilation: CompiledFunctions, dwarf_sections: Vec, - data_initializers: &[DataInitializer], ) -> Result { let mut builder = ObjectBuilder::new(target, module, &compilation); builder.set_dwarf_sections(dwarf_sections); let mut obj = builder.build()?; // Append data, table and vmcontext_init code to the object file. - - for (i, initializer) in data_initializers.iter().enumerate() { - declare_data_segment(&mut obj, initializer, i)?; + match &module.memory_initialization { + MemoryInitialization::Segmented(initializers) => { + for (i, initializer) in initializers.iter().enumerate() { + declare_data_segment(&mut obj, initializer, i)?; + } + } + _ => unimplemented!(), } for i in 0..module.table_plans.len() { declare_table(&mut obj, i)?; } - for (i, initializer) in data_initializers.iter().enumerate() { - emit_data_segment(&mut obj, initializer, i)?; + match &module.memory_initialization { + MemoryInitialization::Segmented(initializers) => { + for (i, initializer) in initializers.iter().enumerate() { + emit_data_segment(&mut obj, initializer, i)?; + } + } + _ => unimplemented!(), } for i in 0..module.table_plans.len() { diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 7cea17df86..31a020fd4e 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -24,12 +24,23 @@ cfg-if = "1.0" backtrace = "0.3.55" lazy_static = "1.3.0" psm = "0.1.11" +rand = "0.7.3" +anyhow = "1.0.38" [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3.7", features = ["winbase", "memoryapi", "errhandlingapi"] } +[target.'cfg(target_os = "linux")'.dependencies] +userfaultfd = { version = "0.3.0", optional = true } + [build-dependencies] cc = "1.0" [badges] maintenance = { status = "actively-developed" } + +[features] +default = [] + +# Enables support for userfaultfd in the pooling allocator when building on Linux +uffd = ["userfaultfd"] diff --git a/crates/runtime/src/externref.rs b/crates/runtime/src/externref.rs index b5a8ef30bb..7fd964b8e3 100644 --- a/crates/runtime/src/externref.rs +++ b/crates/runtime/src/externref.rs @@ -351,8 +351,35 @@ impl VMExternRef { ptr } + /// Consume this `VMExternRef` into a raw, untyped pointer. + /// + /// # Safety + /// + /// This method forgets self, so it is possible to create a leak of the + /// underlying reference counted data if not used carefully. + /// + /// Use `from_raw` to recreate the `VMExternRef`. + pub unsafe fn into_raw(self) -> *mut u8 { + let ptr = self.0.cast::().as_ptr(); + std::mem::forget(self); + ptr + } + /// Recreate a `VMExternRef` from a pointer returned from a previous call to - /// `VMExternRef::as_raw`. + /// `as_raw`. + /// + /// # Safety + /// + /// Unlike `clone_from_raw`, this does not increment the reference count of the + /// underlying data. It is not safe to continue to use the pointer passed to this + /// function. + pub unsafe fn from_raw(ptr: *mut u8) -> Self { + debug_assert!(!ptr.is_null()); + VMExternRef(NonNull::new_unchecked(ptr).cast()) + } + + /// Recreate a `VMExternRef` from a pointer returned from a previous call to + /// `as_raw`. /// /// # Safety /// diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 1df1cfb640..9273870367 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -4,12 +4,11 @@ use crate::export::Export; use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; -use crate::imports::Imports; -use crate::memory::{DefaultMemoryCreator, RuntimeLinearMemory, RuntimeMemoryCreator}; +use crate::memory::{Memory, RuntimeMemoryCreator}; use crate::table::{Table, TableElement}; use crate::traphandlers::Trap; use crate::vmcontext::{ - VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, + VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, }; @@ -17,23 +16,26 @@ use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable}; use indexmap::IndexMap; use memoffset::offset_of; use more_asserts::assert_lt; -use std::alloc::{self, Layout}; +use std::alloc::Layout; use std::any::Any; use std::cell::RefCell; use std::collections::HashMap; use std::convert::TryFrom; +use std::hash::Hash; use std::ptr::NonNull; use std::rc::Rc; use std::sync::Arc; use std::{mem, ptr, slice}; -use thiserror::Error; -use wasmtime_environ::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap}; +use wasmtime_environ::entity::{packed_option::ReservedValue, EntityRef, EntitySet, PrimaryMap}; use wasmtime_environ::wasm::{ - DataIndex, DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, - ElemIndex, EntityIndex, FuncIndex, GlobalIndex, GlobalInit, MemoryIndex, SignatureIndex, - TableElementType, TableIndex, WasmType, + DataIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, ElemIndex, EntityIndex, + FuncIndex, GlobalIndex, MemoryIndex, TableElementType, TableIndex, }; -use wasmtime_environ::{ir, DataInitializer, Module, ModuleType, TableElements, VMOffsets}; +use wasmtime_environ::{ir, Module, VMOffsets}; + +mod allocator; + +pub use allocator::*; /// Runtime representation of an instance value, which erases all `Instance` /// information since instances are just a collection of values. @@ -51,19 +53,18 @@ pub(crate) struct Instance { offsets: VMOffsets, /// WebAssembly linear memory data. - memories: BoxedSlice>, + memories: PrimaryMap, /// WebAssembly table data. - tables: BoxedSlice, + tables: PrimaryMap, - /// Passive elements in this instantiation. As `elem.drop`s happen, these - /// entries get removed. A missing entry is considered equivalent to an - /// empty slice. - passive_elements: RefCell>>, + /// Stores the dropped passive element segments in this instantiation by index. + /// If the index is present in the set, the segment has been dropped. + dropped_elements: RefCell>, - /// Passive data segments from our module. As `data.drop`s happen, entries - /// get removed. A missing entry is considered equivalent to an empty slice. - passive_data: RefCell>>, + /// Stores the dropped passive data segments in this instantiation by index. + /// If the index is present in the set, the segment has been dropped. + dropped_data: RefCell>, /// Hosts can store arbitrary per-instance information here. host_state: Box, @@ -533,6 +534,22 @@ impl Instance { self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index)) } + fn find_passive_segment<'a, I, D, T>( + index: I, + index_map: &HashMap, + data: &'a Vec, + dropped: &RefCell>, + ) -> &'a [T] + where + D: AsRef<[T]>, + I: EntityRef + Hash, + { + match index_map.get(&index) { + Some(index) if !dropped.borrow().contains(I::new(*index)) => data[*index].as_ref(), + _ => &[], + } + } + /// The `table.init` operation: initializes a portion of a table with a /// passive element. /// @@ -551,15 +568,17 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-init let table = self.get_table(table_index); - let passive_elements = self.passive_elements.borrow(); - let elem = passive_elements - .get(&elem_index) - .map(|e| &**e) - .unwrap_or_else(|| &[]); + + let elements = Self::find_passive_segment( + elem_index, + &self.module.passive_elements_map, + &self.module.passive_elements, + &self.dropped_elements, + ); if src .checked_add(len) - .map_or(true, |n| n as usize > elem.len()) + .map_or(true, |n| n as usize > elements.len()) || dst.checked_add(len).map_or(true, |m| m > table.size()) { return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); @@ -567,8 +586,14 @@ impl Instance { // TODO(#983): investigate replacing this get/set loop with a `memcpy`. for (dst, src) in (dst..dst + len).zip(src..src + len) { + let elem = self + .get_caller_checked_anyfunc(elements[src as usize]) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut _ + }); + table - .set(dst, TableElement::FuncRef(elem[src as usize])) + .set(dst, TableElement::FuncRef(elem)) .expect("should never panic because we already did the bounds check above"); } @@ -579,10 +604,14 @@ impl Instance { pub(crate) fn elem_drop(&self, elem_index: ElemIndex) { // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop - let mut passive_elements = self.passive_elements.borrow_mut(); - passive_elements.remove(&elem_index); - // Note that we don't check that we actually removed an element because - // dropping a non-passive element is a no-op (not a trap). + if let Some(index) = self.module.passive_elements_map.get(&elem_index) { + self.dropped_elements + .borrow_mut() + .insert(ElemIndex::new(*index)); + } + + // Note that we don't check that we actually removed a segment because + // dropping a non-passive segment is a no-op (not a trap). } /// Do a `memory.copy` @@ -701,10 +730,13 @@ impl Instance { // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-memory-init let memory = self.get_memory(memory_index); - let passive_data = self.passive_data.borrow(); - let data = passive_data - .get(&data_index) - .map_or(&[][..], |data| &**data); + + let data = Self::find_passive_segment( + data_index, + &self.module.passive_data_map, + &self.module.passive_data, + &self.dropped_data, + ); if src .checked_add(len) @@ -729,8 +761,14 @@ impl Instance { /// Drop the given data segment, truncating its length to zero. pub(crate) fn data_drop(&self, data_index: DataIndex) { - let mut passive_data = self.passive_data.borrow_mut(); - passive_data.remove(&data_index); + if let Some(index) = self.module.passive_data_map.get(&data_index) { + self.dropped_data + .borrow_mut() + .insert(DataIndex::new(*index)); + } + + // Note that we don't check that we actually removed a segment because + // dropping a non-passive segment is a no-op (not a trap). } /// Get a table by index regardless of whether it is locally-defined or an @@ -780,197 +818,8 @@ pub struct InstanceHandle { } impl InstanceHandle { - /// Create a new `InstanceHandle` pointing at a new `Instance`. - /// - /// # Unsafety - /// - /// This method is not necessarily inherently unsafe to call, but in general - /// the APIs of an `Instance` are quite unsafe and have not been really - /// audited for safety that much. As a result the unsafety here on this - /// method is a low-overhead way of saying "this is an extremely unsafe type - /// to work with". - /// - /// Extreme care must be taken when working with `InstanceHandle` and it's - /// recommended to have relatively intimate knowledge of how it works - /// internally if you'd like to do so. If possible it's recommended to use - /// the `wasmtime` crate API rather than this type since that is vetted for - /// safety. - /// - /// It is your responsibility to ensure that the given raw - /// `externref_activations_table` and `stack_map_registry` outlive this - /// instance. - pub unsafe fn new( - module: Arc, - finished_functions: &PrimaryMap, - imports: Imports, - mem_creator: Option<&dyn RuntimeMemoryCreator>, - lookup_shared_signature: &dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, - host_state: Box, - interrupts: *const VMInterrupts, - externref_activations_table: *mut VMExternRefActivationsTable, - stack_map_registry: *mut StackMapRegistry, - ) -> Result { - debug_assert!(!externref_activations_table.is_null()); - debug_assert!(!stack_map_registry.is_null()); - - let tables = create_tables(&module); - let memories = create_memories(&module, mem_creator.unwrap_or(&DefaultMemoryCreator {}))?; - - let vmctx_tables = tables - .values() - .map(Table::vmtable) - .collect::>() - .into_boxed_slice(); - - let vmctx_memories = memories - .values() - .map(|a| a.vmmemory()) - .collect::>() - .into_boxed_slice(); - - let vmctx_globals = create_globals(&module); - - let offsets = VMOffsets::new(mem::size_of::<*const u8>() as u8, &module); - - let passive_data = RefCell::new(module.passive_data.clone()); - - let handle = { - let instance = Instance { - module, - offsets, - memories, - tables, - passive_elements: Default::default(), - passive_data, - host_state, - vmctx: VMContext {}, - }; - let layout = instance.alloc_layout(); - let instance_ptr = alloc::alloc(layout) as *mut Instance; - if instance_ptr.is_null() { - alloc::handle_alloc_error(layout); - } - ptr::write(instance_ptr, instance); - InstanceHandle { - instance: instance_ptr, - } - }; - let instance = handle.instance(); - - let mut ptr = instance.signature_ids_ptr(); - for sig in handle.module().types.values() { - *ptr = match sig { - ModuleType::Function(sig) => lookup_shared_signature(*sig), - _ => VMSharedSignatureIndex::new(u32::max_value()), - }; - ptr = ptr.add(1); - } - - debug_assert_eq!(imports.functions.len(), handle.module().num_imported_funcs); - ptr::copy( - imports.functions.as_ptr(), - instance.imported_functions_ptr() as *mut VMFunctionImport, - imports.functions.len(), - ); - debug_assert_eq!(imports.tables.len(), handle.module().num_imported_tables); - ptr::copy( - imports.tables.as_ptr(), - instance.imported_tables_ptr() as *mut VMTableImport, - imports.tables.len(), - ); - debug_assert_eq!( - imports.memories.len(), - handle.module().num_imported_memories - ); - ptr::copy( - imports.memories.as_ptr(), - instance.imported_memories_ptr() as *mut VMMemoryImport, - imports.memories.len(), - ); - debug_assert_eq!(imports.globals.len(), handle.module().num_imported_globals); - ptr::copy( - imports.globals.as_ptr(), - instance.imported_globals_ptr() as *mut VMGlobalImport, - imports.globals.len(), - ); - ptr::copy( - vmctx_tables.values().as_slice().as_ptr(), - instance.tables_ptr() as *mut VMTableDefinition, - vmctx_tables.len(), - ); - ptr::copy( - vmctx_memories.values().as_slice().as_ptr(), - instance.memories_ptr() as *mut VMMemoryDefinition, - vmctx_memories.len(), - ); - ptr::copy( - vmctx_globals.values().as_slice().as_ptr(), - instance.globals_ptr() as *mut VMGlobalDefinition, - vmctx_globals.len(), - ); - ptr::write( - instance.builtin_functions_ptr() as *mut VMBuiltinFunctionsArray, - VMBuiltinFunctionsArray::initialized(), - ); - *instance.interrupts() = interrupts; - *instance.externref_activations_table() = externref_activations_table; - *instance.stack_map_registry() = stack_map_registry; - - for (index, sig) in instance.module.functions.iter() { - let type_index = lookup_shared_signature(*sig); - - let (func_ptr, vmctx) = - if let Some(def_index) = instance.module.defined_func_index(index) { - ( - NonNull::new(finished_functions[def_index] as *mut _).unwrap(), - instance.vmctx_ptr(), - ) - } else { - let import = instance.imported_function(index); - (import.body, import.vmctx) - }; - - ptr::write( - instance.anyfunc_ptr(index), - VMCallerCheckedAnyfunc { - func_ptr, - type_index, - vmctx, - }, - ); - } - - // Perform infallible initialization in this constructor, while fallible - // initialization is deferred to the `initialize` method. - initialize_passive_elements(instance); - initialize_globals(instance); - - Ok(handle) - } - - /// Finishes the instantiation process started by `Instance::new`. - /// - /// Only safe to call immediately after instantiation. - pub unsafe fn initialize( - &self, - is_bulk_memory: bool, - data_initializers: &[DataInitializer<'_>], - ) -> Result<(), InstantiationError> { - // Check initializer bounds before initializing anything. Only do this - // when bulk memory is disabled, since the bulk memory proposal changes - // instantiation such that the intermediate results of failed - // initializations are visible. - if !is_bulk_memory { - check_table_init_bounds(self.instance())?; - check_memory_init_bounds(self.instance(), data_initializers)?; - } - - // Apply fallible initializers. Note that this can "leak" state even if - // it fails. - initialize_tables(self.instance())?; - initialize_memories(self.instance(), data_initializers)?; - - Ok(()) + pub(crate) unsafe fn new(instance: *mut Instance) -> Self { + Self { instance } } /// Create a new `InstanceHandle` pointing at the instance @@ -1126,305 +975,4 @@ impl InstanceHandle { instance: self.instance, } } - - /// Deallocates memory associated with this instance. - /// - /// Note that this is unsafe because there might be other handles to this - /// `InstanceHandle` elsewhere, and there's nothing preventing usage of - /// this handle after this function is called. - pub unsafe fn dealloc(&self) { - let instance = self.instance(); - let layout = instance.alloc_layout(); - ptr::drop_in_place(self.instance); - alloc::dealloc(self.instance.cast(), layout); - } -} - -fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module().table_elements { - let start = get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - let size = usize::try_from(table.size()).unwrap(); - if size < start + init.elements.len() { - return Err(InstantiationError::Link(LinkError( - "table out of bounds: elements segment does not fit".to_owned(), - ))); - } - } - - Ok(()) -} - -/// Compute the offset for a memory data initializer. -fn get_memory_init_start(init: &DataInitializer<'_>, instance: &Instance) -> usize { - let mut start = init.location.offset; - - if let Some(base) = init.location.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start -} - -/// Return a byte-slice view of a memory's data. -unsafe fn get_memory_slice<'instance>( - init: &DataInitializer<'_>, - instance: &'instance Instance, -) -> &'instance mut [u8] { - let memory = if let Some(defined_memory_index) = instance - .module - .defined_memory_index(init.location.memory_index) - { - instance.memory(defined_memory_index) - } else { - let import = instance.imported_memory(init.location.memory_index); - let foreign_instance = (&mut *(import).vmctx).instance(); - let foreign_memory = &mut *(import).from; - let foreign_index = foreign_instance.memory_index(foreign_memory); - foreign_instance.memory(foreign_index) - }; - slice::from_raw_parts_mut(memory.base, memory.current_length) -} - -fn check_memory_init_bounds( - instance: &Instance, - data_initializers: &[DataInitializer<'_>], -) -> Result<(), InstantiationError> { - for init in data_initializers { - let start = get_memory_init_start(init, instance); - unsafe { - let mem_slice = get_memory_slice(init, instance); - if mem_slice.get_mut(start..start + init.data.len()).is_none() { - return Err(InstantiationError::Link(LinkError( - "memory out of bounds: data segment does not fit".into(), - ))); - } - } - } - - Ok(()) -} - -/// Allocate memory for just the tables of the current module. -fn create_tables(module: &Module) -> BoxedSlice { - let num_imports = module.num_imported_tables; - let mut tables: PrimaryMap = - PrimaryMap::with_capacity(module.table_plans.len() - num_imports); - for table in &module.table_plans.values().as_slice()[num_imports..] { - tables.push(Table::new(table)); - } - tables.into_boxed_slice() -} - -/// Compute the offset for a table element initializer. -fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize { - let mut start = init.offset; - - if let Some(base) = init.base { - let val = unsafe { - if let Some(def_index) = instance.module.defined_global_index(base) { - *instance.global(def_index).as_u32() - } else { - *(*instance.imported_global(base).from).as_u32() - } - }; - start += usize::try_from(val).unwrap(); - } - - start -} - -/// Initialize the table memory from the provided initializers. -fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { - for init in &instance.module().table_elements { - let start = get_table_init_start(init, instance); - let table = instance.get_table(init.table_index); - - if start - .checked_add(init.elements.len()) - .map_or(true, |end| end > table.size() as usize) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::TableOutOfBounds, - ))); - } - - for (i, func_idx) in init.elements.iter().enumerate() { - let item = match table.element_type() { - TableElementType::Func => instance - .get_caller_checked_anyfunc(*func_idx) - .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc - }) - .into(), - TableElementType::Val(_) => { - assert!(*func_idx == FuncIndex::reserved_value()); - TableElement::ExternRef(None) - } - }; - table.set(u32::try_from(start + i).unwrap(), item).unwrap(); - } - } - - Ok(()) -} - -/// Initialize the `Instance::passive_elements` map by resolving the -/// `Module::passive_elements`'s `FuncIndex`s into `VMCallerCheckedAnyfunc`s for -/// this instance. -fn initialize_passive_elements(instance: &Instance) { - let mut passive_elements = instance.passive_elements.borrow_mut(); - debug_assert!( - passive_elements.is_empty(), - "should only be called once, at initialization time" - ); - - passive_elements.extend( - instance - .module - .passive_elements - .iter() - .filter(|(_, segments)| !segments.is_empty()) - .map(|(idx, segments)| { - ( - *idx, - segments - .iter() - .map(|s| { - instance.get_caller_checked_anyfunc(*s).map_or( - ptr::null_mut(), - |f: &VMCallerCheckedAnyfunc| { - f as *const VMCallerCheckedAnyfunc as *mut _ - }, - ) - }) - .collect(), - ) - }), - ); -} - -/// Allocate memory for just the memories of the current module. -fn create_memories( - module: &Module, - mem_creator: &dyn RuntimeMemoryCreator, -) -> Result>, InstantiationError> { - let num_imports = module.num_imported_memories; - let mut memories: PrimaryMap = - PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); - for plan in &module.memory_plans.values().as_slice()[num_imports..] { - memories.push( - mem_creator - .new_memory(plan) - .map_err(InstantiationError::Resource)?, - ); - } - Ok(memories.into_boxed_slice()) -} - -/// Initialize the table memory from the provided initializers. -fn initialize_memories( - instance: &Instance, - data_initializers: &[DataInitializer<'_>], -) -> Result<(), InstantiationError> { - for init in data_initializers { - let memory = instance.get_memory(init.location.memory_index); - - let start = get_memory_init_start(init, instance); - if start - .checked_add(init.data.len()) - .map_or(true, |end| end > memory.current_length) - { - return Err(InstantiationError::Trap(Trap::wasm( - ir::TrapCode::HeapOutOfBounds, - ))); - } - - unsafe { - let mem_slice = get_memory_slice(init, instance); - let end = start + init.data.len(); - let to_init = &mut mem_slice[start..end]; - to_init.copy_from_slice(init.data); - } - } - - Ok(()) -} - -/// Allocate memory for just the globals of the current module, -/// with initializers applied. -fn create_globals(module: &Module) -> BoxedSlice { - let num_imports = module.num_imported_globals; - let mut vmctx_globals = PrimaryMap::with_capacity(module.globals.len() - num_imports); - - for _ in &module.globals.values().as_slice()[num_imports..] { - vmctx_globals.push(VMGlobalDefinition::new()); - } - - vmctx_globals.into_boxed_slice() -} - -fn initialize_globals(instance: &Instance) { - let module = instance.module(); - let num_imports = module.num_imported_globals; - for (index, global) in module.globals.iter().skip(num_imports) { - let def_index = module.defined_global_index(index).unwrap(); - unsafe { - let to = instance.global_ptr(def_index); - match global.initializer { - GlobalInit::I32Const(x) => *(*to).as_i32_mut() = x, - GlobalInit::I64Const(x) => *(*to).as_i64_mut() = x, - GlobalInit::F32Const(x) => *(*to).as_f32_bits_mut() = x, - GlobalInit::F64Const(x) => *(*to).as_f64_bits_mut() = x, - GlobalInit::V128Const(x) => *(*to).as_u128_bits_mut() = x.0, - GlobalInit::GetGlobal(x) => { - let from = if let Some(def_x) = module.defined_global_index(x) { - instance.global(def_x) - } else { - *instance.imported_global(x).from - }; - *to = from; - } - GlobalInit::RefFunc(f) => { - *(*to).as_anyfunc_mut() = instance.get_caller_checked_anyfunc(f).unwrap() - as *const VMCallerCheckedAnyfunc; - } - GlobalInit::RefNullConst => match global.wasm_ty { - WasmType::FuncRef => *(*to).as_anyfunc_mut() = ptr::null(), - WasmType::ExternRef => *(*to).as_externref_mut() = None, - ty => panic!("unsupported reference type for global: {:?}", ty), - }, - GlobalInit::Import => panic!("locally-defined global initialized as import"), - } - } - } -} - -/// An link error while instantiating a module. -#[derive(Error, Debug)] -#[error("Link error: {0}")] -pub struct LinkError(pub String); - -/// An error while instantiating a module. -#[derive(Error, Debug)] -pub enum InstantiationError { - /// Insufficient resources available for execution. - #[error("Insufficient resources: {0}")] - Resource(String), - - /// A wasm link error occured. - #[error("Failed to link module")] - Link(#[from] LinkError), - - /// A trap ocurred during instantiation, after linking. - #[error("Trap occurred during instantiation")] - Trap(Trap), } diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs new file mode 100644 index 0000000000..c56499994e --- /dev/null +++ b/crates/runtime/src/instance/allocator.rs @@ -0,0 +1,642 @@ +use crate::externref::{StackMapRegistry, VMExternRefActivationsTable}; +use crate::imports::Imports; +use crate::instance::{Instance, InstanceHandle, RuntimeMemoryCreator}; +use crate::memory::{DefaultMemoryCreator, Memory}; +use crate::table::{Table, TableElement}; +use crate::traphandlers::Trap; +use crate::vmcontext::{ + VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, + VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryImport, VMSharedSignatureIndex, + VMTableImport, +}; +use anyhow::Result; +use std::alloc; +use std::any::Any; +use std::cell::RefCell; +use std::convert::TryFrom; +use std::ptr::{self, NonNull}; +use std::slice; +use std::sync::Arc; +use thiserror::Error; +use wasmtime_environ::entity::{packed_option::ReservedValue, EntityRef, EntitySet, PrimaryMap}; +use wasmtime_environ::wasm::{ + DefinedFuncIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, GlobalInit, SignatureIndex, + TableElementType, WasmType, +}; +use wasmtime_environ::{ + ir, MemoryInitialization, MemoryInitializer, Module, ModuleType, TableInitializer, VMOffsets, + WASM_PAGE_SIZE, +}; + +mod pooling; + +pub use self::pooling::{ + InstanceLimits, ModuleLimits, PoolingAllocationStrategy, PoolingInstanceAllocator, +}; + +/// Represents a request for a new runtime instance. +pub struct InstanceAllocationRequest<'a> { + /// The module being instantiated. + pub module: Arc, + + /// The finished (JIT) functions for the module. + pub finished_functions: &'a PrimaryMap, + + /// The imports to use for the instantiation. + pub imports: Imports<'a>, + + /// A callback for looking up shared signature indexes. + pub lookup_shared_signature: &'a dyn Fn(SignatureIndex) -> VMSharedSignatureIndex, + + /// The host state to associate with the instance. + pub host_state: Box, + + /// The pointer to the VM interrupts structure to use for the instance. + pub interrupts: *const VMInterrupts, + + /// The pointer to the reference activations table to use for the instance. + pub externref_activations_table: *mut VMExternRefActivationsTable, + + /// The pointer to the stack map registry to use for the instance. + pub stack_map_registry: *mut StackMapRegistry, +} + +/// An link error while instantiating a module. +#[derive(Error, Debug)] +#[error("Link error: {0}")] +pub struct LinkError(pub String); + +/// An error while instantiating a module. +#[derive(Error, Debug)] +pub enum InstantiationError { + /// Insufficient resources available for execution. + #[error("Insufficient resources: {0}")] + Resource(anyhow::Error), + + /// A wasm link error occured. + #[error("Failed to link module")] + Link(#[from] LinkError), + + /// A trap ocurred during instantiation, after linking. + #[error("Trap occurred during instantiation")] + Trap(Trap), + + /// A limit on how many instances are supported has been reached. + #[error("Limit of {0} concurrent instances has been reached")] + Limit(u32), +} + +/// An error while creating a fiber stack. +#[derive(Error, Debug)] +pub enum FiberStackError { + /// Insufficient resources available for the request. + #[error("Insufficient resources: {0}")] + Resource(anyhow::Error), + /// An error for when the allocator doesn't support custom fiber stacks. + #[error("Custom fiber stacks are not supported by the allocator")] + NotSupported, + /// A limit on how many fibers are supported has been reached. + #[error("Limit of {0} concurrent fibers has been reached")] + Limit(u32), +} + +/// Represents a runtime instance allocator. +/// +/// # Safety +/// +/// This trait is unsafe as it requires knowledge of Wasmtime's runtime internals to implement correctly. +pub unsafe trait InstanceAllocator: Send + Sync { + /// Validates that a module is supported by the allocator. + fn validate(&self, module: &Module) -> Result<()> { + drop(module); + Ok(()) + } + + /// Adjusts the tunables prior to creation of any JIT compiler. + /// + /// This method allows the instance allocator control over tunables passed to a `wasmtime_jit::Compiler`. + fn adjust_tunables(&self, tunables: &mut wasmtime_environ::Tunables) { + drop(tunables); + } + + /// Allocates an instance for the given allocation request. + /// + /// # Safety + /// + /// This method is not inherently unsafe, but care must be made to ensure + /// pointers passed in the allocation request outlive the returned instance. + unsafe fn allocate( + &self, + req: InstanceAllocationRequest, + ) -> Result; + + /// Finishes the instantiation process started by an instance allocator. + /// + /// # Safety + /// + /// This method is only safe to call immediately after an instance has been allocated. + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + ) -> Result<(), InstantiationError>; + + /// Deallocates a previously allocated instance. + /// + /// # Safety + /// + /// This function is unsafe because there are no guarantees that the given handle + /// is the only owner of the underlying instance to deallocate. + /// + /// Use extreme care when deallocating an instance so that there are no dangling instance pointers. + unsafe fn deallocate(&self, handle: &InstanceHandle); + + /// Allocates a fiber stack for calling async functions on. + /// + /// Returns the top of the fiber stack if successfully allocated. + fn allocate_fiber_stack(&self) -> Result<*mut u8, FiberStackError>; + + /// Deallocates a fiber stack that was previously allocated. + /// + /// # Safety + /// + /// This function is unsafe because there are no guarantees that the given stack + /// is no longer in use. + /// + /// Additionally, passing a stack pointer that was not returned from `allocate_fiber_stack` + /// will lead to undefined behavior. + unsafe fn deallocate_fiber_stack(&self, stack: *mut u8); +} + +fn get_table_init_start( + init: &TableInitializer, + instance: &Instance, +) -> Result { + match init.base { + Some(base) => { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + + init.offset.checked_add(val as usize).ok_or_else(|| { + InstantiationError::Link(LinkError( + "element segment global base overflows".to_owned(), + )) + }) + } + None => Ok(init.offset), + } +} + +fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module.table_initializers { + let table = instance.get_table(init.table_index); + let start = get_table_init_start(init, instance)?; + let end = start.checked_add(init.elements.len()); + + match end { + Some(end) if end <= table.size() as usize => { + // Initializer is in bounds + } + _ => { + return Err(InstantiationError::Link(LinkError( + "table out of bounds: elements segment does not fit".to_owned(), + ))) + } + } + } + + Ok(()) +} + +fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { + for init in &instance.module.table_initializers { + let table = instance.get_table(init.table_index); + let start = get_table_init_start(init, instance)?; + let end = start.checked_add(init.elements.len()); + + match end { + Some(end) if end <= table.size() as usize => { + for (i, func_idx) in init.elements.iter().enumerate() { + let item = match table.element_type() { + TableElementType::Func => instance + .get_caller_checked_anyfunc(*func_idx) + .map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| { + f as *const VMCallerCheckedAnyfunc as *mut VMCallerCheckedAnyfunc + }) + .into(), + TableElementType::Val(_) => { + assert!(*func_idx == FuncIndex::reserved_value()); + TableElement::ExternRef(None) + } + }; + table.set(u32::try_from(start + i).unwrap(), item).unwrap(); + } + } + _ => { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::TableOutOfBounds, + ))) + } + } + } + + Ok(()) +} + +fn get_memory_init_start( + init: &MemoryInitializer, + instance: &Instance, +) -> Result { + match init.base { + Some(base) => { + let val = unsafe { + if let Some(def_index) = instance.module.defined_global_index(base) { + *instance.global(def_index).as_u32() + } else { + *(*instance.imported_global(base).from).as_u32() + } + }; + + init.offset.checked_add(val as usize).ok_or_else(|| { + InstantiationError::Link(LinkError("data segment global base overflows".to_owned())) + }) + } + None => Ok(init.offset), + } +} + +unsafe fn get_memory_slice<'instance>( + init: &MemoryInitializer, + instance: &'instance Instance, +) -> &'instance mut [u8] { + let memory = if let Some(defined_memory_index) = + instance.module.defined_memory_index(init.memory_index) + { + instance.memory(defined_memory_index) + } else { + let import = instance.imported_memory(init.memory_index); + let foreign_instance = (&mut *(import).vmctx).instance(); + let foreign_memory = &mut *(import).from; + let foreign_index = foreign_instance.memory_index(foreign_memory); + foreign_instance.memory(foreign_index) + }; + &mut *ptr::slice_from_raw_parts_mut(memory.base, memory.current_length) +} + +fn check_memory_init_bounds( + instance: &Instance, + initializers: &[MemoryInitializer], +) -> Result<(), InstantiationError> { + for init in initializers { + let memory = instance.get_memory(init.memory_index); + let start = get_memory_init_start(init, instance)?; + let end = start.checked_add(init.data.len()); + + match end { + Some(end) if end <= memory.current_length => { + // Initializer is in bounds + } + _ => { + return Err(InstantiationError::Link(LinkError( + "memory out of bounds: data segment does not fit".into(), + ))) + } + } + } + + Ok(()) +} + +fn initialize_memories( + instance: &Instance, + initializers: &[MemoryInitializer], +) -> Result<(), InstantiationError> { + for init in initializers { + let memory = instance.get_memory(init.memory_index); + let start = get_memory_init_start(init, instance)?; + let end = start.checked_add(init.data.len()); + + match end { + Some(end) if end <= memory.current_length => { + let mem_slice = unsafe { get_memory_slice(init, instance) }; + mem_slice[start..end].copy_from_slice(&init.data); + } + _ => { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::HeapOutOfBounds, + ))) + } + } + } + + Ok(()) +} + +fn check_init_bounds(instance: &Instance) -> Result<(), InstantiationError> { + check_table_init_bounds(instance)?; + + match &instance.module.memory_initialization { + MemoryInitialization::Paged { out_of_bounds, .. } => { + if *out_of_bounds { + return Err(InstantiationError::Link(LinkError( + "memory out of bounds: data segment does not fit".into(), + ))); + } + } + MemoryInitialization::Segmented(initializers) => { + check_memory_init_bounds(instance, initializers)?; + } + } + + Ok(()) +} + +fn initialize_instance( + instance: &Instance, + is_bulk_memory: bool, +) -> Result<(), InstantiationError> { + // If bulk memory is not enabled, bounds check the data and element segments before + // making any changes. With bulk memory enabled, initializers are processed + // in-order and side effects are observed up to the point of an out-of-bounds + // initializer, so the early checking is not desired. + if !is_bulk_memory { + check_init_bounds(instance)?; + } + + // Initialize the tables + initialize_tables(instance)?; + + // Initialize the memories + match &instance.module.memory_initialization { + MemoryInitialization::Paged { map, out_of_bounds } => { + for (index, pages) in map { + let memory = instance.memory(index); + let slice = + unsafe { slice::from_raw_parts_mut(memory.base, memory.current_length) }; + + for (page_index, page) in pages.iter().enumerate() { + if let Some(data) = page { + debug_assert_eq!(data.len(), WASM_PAGE_SIZE as usize); + slice[page_index * WASM_PAGE_SIZE as usize..].copy_from_slice(data); + } + } + } + + // Check for out of bound access after initializing the pages to maintain + // the expected behavior of the bulk memory spec. + if *out_of_bounds { + return Err(InstantiationError::Trap(Trap::wasm( + ir::TrapCode::HeapOutOfBounds, + ))); + } + } + MemoryInitialization::Segmented(initializers) => { + initialize_memories(instance, initializers)?; + } + } + + Ok(()) +} + +unsafe fn initialize_vmcontext(instance: &Instance, req: InstanceAllocationRequest) { + let module = &instance.module; + + *instance.interrupts() = req.interrupts; + *instance.externref_activations_table() = req.externref_activations_table; + *instance.stack_map_registry() = req.stack_map_registry; + + // Initialize shared signatures + let mut ptr = instance.signature_ids_ptr(); + for sig in module.types.values() { + *ptr = match sig { + ModuleType::Function(sig) => (req.lookup_shared_signature)(*sig), + _ => VMSharedSignatureIndex::new(u32::max_value()), + }; + ptr = ptr.add(1); + } + + // Initialize the built-in functions + ptr::write( + instance.builtin_functions_ptr() as *mut VMBuiltinFunctionsArray, + VMBuiltinFunctionsArray::initialized(), + ); + + // Initialize the imports + debug_assert_eq!(req.imports.functions.len(), module.num_imported_funcs); + ptr::copy( + req.imports.functions.as_ptr(), + instance.imported_functions_ptr() as *mut VMFunctionImport, + req.imports.functions.len(), + ); + debug_assert_eq!(req.imports.tables.len(), module.num_imported_tables); + ptr::copy( + req.imports.tables.as_ptr(), + instance.imported_tables_ptr() as *mut VMTableImport, + req.imports.tables.len(), + ); + debug_assert_eq!(req.imports.memories.len(), module.num_imported_memories); + ptr::copy( + req.imports.memories.as_ptr(), + instance.imported_memories_ptr() as *mut VMMemoryImport, + req.imports.memories.len(), + ); + debug_assert_eq!(req.imports.globals.len(), module.num_imported_globals); + ptr::copy( + req.imports.globals.as_ptr(), + instance.imported_globals_ptr() as *mut VMGlobalImport, + req.imports.globals.len(), + ); + + // Initialize the functions + for (index, sig) in instance.module.functions.iter() { + let type_index = (req.lookup_shared_signature)(*sig); + + let (func_ptr, vmctx) = if let Some(def_index) = instance.module.defined_func_index(index) { + ( + NonNull::new(req.finished_functions[def_index] as *mut _).unwrap(), + instance.vmctx_ptr(), + ) + } else { + let import = instance.imported_function(index); + (import.body, import.vmctx) + }; + + ptr::write( + instance.anyfunc_ptr(index), + VMCallerCheckedAnyfunc { + func_ptr, + type_index, + vmctx, + }, + ); + } + + // Initialize the defined tables + let mut ptr = instance.tables_ptr(); + for i in 0..module.table_plans.len() - module.num_imported_tables { + ptr::write(ptr, instance.tables[DefinedTableIndex::new(i)].vmtable()); + ptr = ptr.add(1); + } + + // Initialize the defined memories + let mut ptr = instance.memories_ptr(); + for i in 0..module.memory_plans.len() - module.num_imported_memories { + ptr::write( + ptr, + instance.memories[DefinedMemoryIndex::new(i)].vmmemory(), + ); + ptr = ptr.add(1); + } + + // Initialize the defined globals + initialize_vmcontext_globals(instance); +} + +unsafe fn initialize_vmcontext_globals(instance: &Instance) { + let module = &instance.module; + let num_imports = module.num_imported_globals; + for (index, global) in module.globals.iter().skip(num_imports) { + let def_index = module.defined_global_index(index).unwrap(); + let to = instance.global_ptr(def_index); + + // Initialize the global before writing to it + ptr::write(to, VMGlobalDefinition::new()); + + match global.initializer { + GlobalInit::I32Const(x) => *(*to).as_i32_mut() = x, + GlobalInit::I64Const(x) => *(*to).as_i64_mut() = x, + GlobalInit::F32Const(x) => *(*to).as_f32_bits_mut() = x, + GlobalInit::F64Const(x) => *(*to).as_f64_bits_mut() = x, + GlobalInit::V128Const(x) => *(*to).as_u128_bits_mut() = x.0, + GlobalInit::GetGlobal(x) => { + let from = if let Some(def_x) = module.defined_global_index(x) { + instance.global(def_x) + } else { + *instance.imported_global(x).from + }; + *to = from; + } + GlobalInit::RefFunc(f) => { + *(*to).as_anyfunc_mut() = instance.get_caller_checked_anyfunc(f).unwrap() + as *const VMCallerCheckedAnyfunc; + } + GlobalInit::RefNullConst => match global.wasm_ty { + WasmType::FuncRef => *(*to).as_anyfunc_mut() = ptr::null(), + WasmType::ExternRef => *(*to).as_externref_mut() = None, + ty => panic!("unsupported reference type for global: {:?}", ty), + }, + GlobalInit::Import => panic!("locally-defined global initialized as import"), + } + } +} + +/// Represents the on-demand instance allocator. +#[derive(Clone)] +pub struct OnDemandInstanceAllocator { + mem_creator: Option>, +} + +impl OnDemandInstanceAllocator { + /// Creates a new on-demand instance allocator. + pub fn new(mem_creator: Option>) -> Self { + Self { mem_creator } + } + + fn create_tables(module: &Module) -> PrimaryMap { + let num_imports = module.num_imported_tables; + let mut tables: PrimaryMap = + PrimaryMap::with_capacity(module.table_plans.len() - num_imports); + for table in &module.table_plans.values().as_slice()[num_imports..] { + tables.push(Table::new_dynamic(table)); + } + tables + } + + fn create_memories( + &self, + module: &Module, + ) -> Result, InstantiationError> { + let creator = self + .mem_creator + .as_deref() + .unwrap_or_else(|| &DefaultMemoryCreator); + let num_imports = module.num_imported_memories; + let mut memories: PrimaryMap = + PrimaryMap::with_capacity(module.memory_plans.len() - num_imports); + for plan in &module.memory_plans.values().as_slice()[num_imports..] { + memories + .push(Memory::new_dynamic(plan, creator).map_err(InstantiationError::Resource)?); + } + Ok(memories) + } +} + +unsafe impl InstanceAllocator for OnDemandInstanceAllocator { + unsafe fn allocate( + &self, + mut req: InstanceAllocationRequest, + ) -> Result { + debug_assert!(!req.externref_activations_table.is_null()); + debug_assert!(!req.stack_map_registry.is_null()); + + let memories = self.create_memories(&req.module)?; + let tables = Self::create_tables(&req.module); + + let host_state = std::mem::replace(&mut req.host_state, Box::new(())); + + let handle = { + let instance = Instance { + module: req.module.clone(), + offsets: VMOffsets::new(std::mem::size_of::<*const u8>() as u8, &req.module), + memories, + tables, + dropped_elements: RefCell::new(EntitySet::with_capacity( + req.module.passive_elements.len(), + )), + dropped_data: RefCell::new(EntitySet::with_capacity(req.module.passive_data.len())), + host_state, + vmctx: VMContext {}, + }; + let layout = instance.alloc_layout(); + let instance_ptr = alloc::alloc(layout) as *mut Instance; + if instance_ptr.is_null() { + alloc::handle_alloc_error(layout); + } + ptr::write(instance_ptr, instance); + InstanceHandle::new(instance_ptr) + }; + + initialize_vmcontext(handle.instance(), req); + + Ok(handle) + } + + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + ) -> Result<(), InstantiationError> { + initialize_instance(handle.instance(), is_bulk_memory) + } + + unsafe fn deallocate(&self, handle: &InstanceHandle) { + let layout = handle.instance().alloc_layout(); + ptr::drop_in_place(handle.instance); + alloc::dealloc(handle.instance.cast(), layout); + } + + fn allocate_fiber_stack(&self) -> Result<*mut u8, FiberStackError> { + // The on-demand allocator does not support allocating fiber stacks + Err(FiberStackError::NotSupported) + } + + unsafe fn deallocate_fiber_stack(&self, _stack: *mut u8) { + // This should never be called as `allocate_fiber_stack` never returns success + unreachable!() + } +} diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs new file mode 100644 index 0000000000..2aa8c4b7ff --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -0,0 +1,1626 @@ +//! Implements the pooling instance allocator. +//! +//! The pooling instance allocator maps memory in advance +//! and allocates instances, memories, tables, and stacks from +//! a pool of available resources. +//! +//! Using the pooling instance allocator can speed up module instantiation +//! when modules can be constrained based on configurable limits. + +use super::{ + initialize_instance, initialize_vmcontext, FiberStackError, InstanceAllocationRequest, + InstanceAllocator, InstanceHandle, InstantiationError, +}; +use crate::{instance::Instance, Memory, Mmap, Table, VMContext}; +use anyhow::{anyhow, bail, Context, Result}; +use rand::Rng; +use std::cell::RefCell; +use std::cmp::min; +use std::convert::TryFrom; +use std::mem; +use std::sync::{Arc, Mutex}; +use wasmtime_environ::{ + entity::{EntitySet, PrimaryMap}, + MemoryStyle, Module, Tunables, VMOffsets, WASM_PAGE_SIZE, +}; + +cfg_if::cfg_if! { + if #[cfg(windows)] { + mod windows; + use windows as imp; + } else if #[cfg(all(feature = "uffd", target_os = "linux"))] { + mod uffd; + use uffd as imp; + use imp::initialize_memory_pool; + } else if #[cfg(target_os = "linux")] { + mod linux; + use linux as imp; + } else { + mod unix; + use unix as imp; + } +} + +use imp::{ + commit_memory_pages, commit_stack_pages, commit_table_pages, decommit_memory_pages, + decommit_stack_pages, decommit_table_pages, +}; + +fn round_up_to_pow2(n: usize, to: usize) -> usize { + debug_assert!(to > 0); + debug_assert!(to.is_power_of_two()); + (n + to - 1) & !(to - 1) +} + +/// Represents the limits placed on a module for compiling with the pooling instance allocator. +#[derive(Debug, Copy, Clone)] +pub struct ModuleLimits { + /// The maximum number of imported functions for a module. + pub imported_functions: u32, + + /// The maximum number of imported tables for a module. + pub imported_tables: u32, + + /// The maximum number of imported linear memories for a module. + pub imported_memories: u32, + + /// The maximum number of imported globals for a module. + pub imported_globals: u32, + + /// The maximum number of defined types for a module. + pub types: u32, + + /// The maximum number of defined functions for a module. + pub functions: u32, + + /// The maximum number of defined tables for a module. + pub tables: u32, + + /// The maximum number of defined linear memories for a module. + pub memories: u32, + + /// The maximum number of defined globals for a module. + pub globals: u32, + + /// The maximum table elements for any table defined in a module. + pub table_elements: u32, + + /// The maximum number of pages for any linear memory defined in a module. + pub memory_pages: u32, +} + +impl ModuleLimits { + fn validate(&self, module: &Module) -> Result<()> { + if module.num_imported_funcs > self.imported_functions as usize { + bail!( + "imported function count of {} exceeds the limit of {}", + module.num_imported_funcs, + self.imported_functions + ); + } + + if module.num_imported_tables > self.imported_tables as usize { + bail!( + "imported tables count of {} exceeds the limit of {}", + module.num_imported_tables, + self.imported_tables + ); + } + + if module.num_imported_memories > self.imported_memories as usize { + bail!( + "imported memories count of {} exceeds the limit of {}", + module.num_imported_memories, + self.imported_memories + ); + } + + if module.num_imported_globals > self.imported_globals as usize { + bail!( + "imported globals count of {} exceeds the limit of {}", + module.num_imported_globals, + self.imported_globals + ); + } + + if module.types.len() > self.types as usize { + bail!( + "defined types count of {} exceeds the limit of {}", + module.types.len(), + self.types + ); + } + + let functions = module.functions.len() - module.num_imported_funcs; + if functions > self.functions as usize { + bail!( + "defined functions count of {} exceeds the limit of {}", + functions, + self.functions + ); + } + + let tables = module.table_plans.len() - module.num_imported_tables; + if tables > self.tables as usize { + bail!( + "defined tables count of {} exceeds the limit of {}", + tables, + self.tables + ); + } + + let memories = module.memory_plans.len() - module.num_imported_memories; + if memories > self.memories as usize { + bail!( + "defined memories count of {} exceeds the limit of {}", + memories, + self.memories + ); + } + + let globals = module.globals.len() - module.num_imported_globals; + if globals > self.globals as usize { + bail!( + "defined globals count of {} exceeds the limit of {}", + globals, + self.globals + ); + } + + for (i, plan) in module.table_plans.values().as_slice()[module.num_imported_tables..] + .iter() + .enumerate() + { + if plan.table.minimum > self.table_elements { + bail!( + "table index {} has a minimum element size of {} which exceeds the limit of {}", + i, + plan.table.minimum, + self.table_elements + ); + } + } + + for (i, plan) in module.memory_plans.values().as_slice()[module.num_imported_memories..] + .iter() + .enumerate() + { + if plan.memory.minimum > self.memory_pages { + bail!( + "memory index {} has a minimum page size of {} which exceeds the limit of {}", + i, + plan.memory.minimum, + self.memory_pages + ); + } + + if let MemoryStyle::Dynamic = plan.style { + bail!( + "memory index {} has an unsupported dynamic memory plan style", + i, + ); + } + } + + Ok(()) + } +} + +impl Default for ModuleLimits { + fn default() -> Self { + // See doc comments for `wasmtime::ModuleLimits` for these default values + Self { + imported_functions: 1000, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 100, + functions: 10000, + tables: 1, + memories: 1, + globals: 10, + table_elements: 10000, + memory_pages: 160, + } + } +} + +/// Represents the limits placed on instances by the pooling instance allocator. +#[derive(Debug, Copy, Clone)] +pub struct InstanceLimits { + /// The maximum number of concurrent instances supported. + pub count: u32, + + /// The maximum size, in bytes, of host address space to reserve for each linear memory of an instance. + pub memory_reservation_size: u64, +} + +impl Default for InstanceLimits { + fn default() -> Self { + // See doc comments for `wasmtime::InstanceLimits` for these default values + Self { + count: 1000, + #[cfg(target_pointer_width = "32")] + memory_reservation_size: 10 * (1 << 20), // 10 MiB, + #[cfg(target_pointer_width = "64")] + memory_reservation_size: 6 * (1 << 30), // 6 GiB, + } + } +} + +/// The allocation strategy to use for the pooling instance allocator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolingAllocationStrategy { + /// Allocate from the next available instance. + NextAvailable, + /// Allocate from a random available instance. + Random, +} + +impl PoolingAllocationStrategy { + fn next(&self, free_count: usize) -> usize { + debug_assert!(free_count > 0); + + match self { + Self::NextAvailable => free_count - 1, + Self::Random => rand::thread_rng().gen_range(0, free_count), + } + } +} + +impl Default for PoolingAllocationStrategy { + fn default() -> Self { + Self::NextAvailable + } +} + +/// Represents a pool of maximal `Instance` structures. +/// +/// Each index in the pool provides enough space for a maximal `Instance` +/// structure depending on the limits used to create the pool. +/// +/// The pool maintains a free list for fast instance allocation. +/// +/// The userfault handler relies on how instances are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. +#[derive(Debug)] +struct InstancePool { + mapping: Mmap, + offsets: VMOffsets, + instance_size: usize, + max_instances: usize, + free_list: Mutex>, + memories: MemoryPool, + tables: TablePool, +} + +impl InstancePool { + fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { + let page_size = region::page::size(); + + // Calculate the maximum size of an Instance structure given the limits + let offsets = VMOffsets { + pointer_size: std::mem::size_of::<*const u8>() as u8, + num_signature_ids: module_limits.types, + num_imported_functions: module_limits.imported_functions, + num_imported_tables: module_limits.imported_tables, + num_imported_memories: module_limits.imported_memories, + num_imported_globals: module_limits.imported_globals, + num_defined_functions: module_limits.functions, + num_defined_tables: module_limits.tables, + num_defined_memories: module_limits.memories, + num_defined_globals: module_limits.globals, + }; + + let instance_size = round_up_to_pow2( + mem::size_of::() + .checked_add(offsets.size_of_vmctx() as usize) + .ok_or_else(|| anyhow!("instance size exceeds addressable memory"))?, + page_size, + ); + + let max_instances = instance_limits.count as usize; + + let allocation_size = instance_size + .checked_mul(max_instances) + .ok_or_else(|| anyhow!("total size of instance data exceeds addressable memory"))?; + + let mapping = Mmap::accessible_reserved(allocation_size, allocation_size) + .context("failed to create instance pool mapping")?; + + let pool = Self { + mapping, + offsets, + instance_size, + max_instances, + free_list: Mutex::new((0..max_instances).collect()), + memories: MemoryPool::new(module_limits, instance_limits)?, + tables: TablePool::new(module_limits, instance_limits)?, + }; + + // Use a default module to initialize the instances to start + let module = Arc::new(Module::default()); + for i in 0..instance_limits.count as usize { + pool.initialize(i, &module); + } + + Ok(pool) + } + + unsafe fn instance(&self, index: usize) -> &mut Instance { + debug_assert!(index < self.max_instances); + &mut *(self.mapping.as_mut_ptr().add(index * self.instance_size) as *mut Instance) + } + + fn initialize(&self, index: usize, module: &Arc) { + unsafe { + let instance = self.instance(index); + + // Write a default instance with preallocated memory/table map storage to the ptr + std::ptr::write( + instance as _, + Instance { + module: module.clone(), + offsets: self.offsets, + memories: PrimaryMap::with_capacity(self.offsets.num_defined_memories as usize), + tables: PrimaryMap::with_capacity(self.offsets.num_defined_tables as usize), + dropped_elements: RefCell::new(EntitySet::new()), + dropped_data: RefCell::new(EntitySet::new()), + host_state: Box::new(()), + vmctx: VMContext {}, + }, + ); + } + } + + fn allocate( + &self, + strategy: PoolingAllocationStrategy, + mut req: InstanceAllocationRequest, + ) -> Result { + let index = { + let mut free_list = self.free_list.lock().unwrap(); + if free_list.is_empty() { + return Err(InstantiationError::Limit(self.max_instances as u32)); + } + let free_index = strategy.next(free_list.len()); + free_list.swap_remove(free_index) + }; + + let host_state = std::mem::replace(&mut req.host_state, Box::new(())); + + unsafe { + let instance = self.instance(index); + + instance.module = req.module.clone(); + instance.offsets = VMOffsets::new( + std::mem::size_of::<*const u8>() as u8, + instance.module.as_ref(), + ); + instance.host_state = host_state; + + Self::set_instance_memories( + instance, + self.memories.get(index), + self.memories.max_wasm_pages, + )?; + Self::set_instance_tables(instance, self.tables.get(index), self.tables.max_elements)?; + + initialize_vmcontext(instance, req); + + Ok(InstanceHandle::new(instance as _)) + } + } + + fn deallocate(&self, handle: &InstanceHandle) { + let addr = handle.instance as usize; + let base = self.mapping.as_ptr() as usize; + + debug_assert!(addr >= base && addr < base + self.mapping.len()); + debug_assert!((addr - base) % self.instance_size == 0); + + let index = (addr - base) / self.instance_size; + debug_assert!(index < self.max_instances); + + let instance = unsafe { &mut *handle.instance }; + + // Decommit any linear memories that were used + for (memory, base) in instance.memories.values_mut().zip(self.memories.get(index)) { + let memory = mem::take(memory); + debug_assert!(memory.is_static()); + + // Reset any faulted guard pages as the physical memory may be reused for another instance in the future + #[cfg(all(feature = "uffd", target_os = "linux"))] + memory + .reset_guard_pages() + .expect("failed to reset guard pages"); + + let size = (memory.size() * WASM_PAGE_SIZE) as usize; + drop(memory); + decommit_memory_pages(base, size).expect("failed to decommit linear memory pages"); + } + + instance.memories.clear(); + instance.dropped_data.borrow_mut().clear(); + + // Decommit any tables that were used + for (table, base) in instance.tables.values_mut().zip(self.tables.get(index)) { + let table = mem::take(table); + debug_assert!(table.is_static()); + + let size = round_up_to_pow2( + table.size() as usize * mem::size_of::<*mut u8>(), + self.tables.page_size, + ); + + drop(table); + decommit_table_pages(base, size).expect("failed to decommit table pages"); + } + + instance.tables.clear(); + instance.dropped_elements.borrow_mut().clear(); + + // Drop any host state + instance.host_state = Box::new(()); + + self.free_list.lock().unwrap().push(index); + } + + fn set_instance_memories( + instance: &mut Instance, + mut memories: impl Iterator, + max_pages: u32, + ) -> Result<(), InstantiationError> { + let module = instance.module.as_ref(); + + debug_assert!(instance.memories.is_empty()); + + for plan in + (&module.memory_plans.values().as_slice()[module.num_imported_memories..]).iter() + { + instance.memories.push( + Memory::new_static( + plan, + memories.next().unwrap(), + max_pages, + commit_memory_pages, + ) + .map_err(InstantiationError::Resource)?, + ); + } + + let mut dropped_data = instance.dropped_data.borrow_mut(); + debug_assert!(dropped_data.is_empty()); + dropped_data.resize(module.passive_data.len()); + + Ok(()) + } + + fn set_instance_tables( + instance: &mut Instance, + mut tables: impl Iterator, + max_elements: u32, + ) -> Result<(), InstantiationError> { + let module = instance.module.as_ref(); + + debug_assert!(instance.tables.is_empty()); + + for plan in (&module.table_plans.values().as_slice()[module.num_imported_tables..]).iter() { + let base = tables.next().unwrap(); + + commit_table_pages(base, max_elements as usize * mem::size_of::<*mut u8>()) + .map_err(InstantiationError::Resource)?; + + instance + .tables + .push(Table::new_static(plan, base as _, max_elements)); + } + + let mut dropped_elements = instance.dropped_elements.borrow_mut(); + debug_assert!(dropped_elements.is_empty()); + dropped_elements.resize(module.passive_elements.len()); + + Ok(()) + } +} + +impl Drop for InstancePool { + fn drop(&mut self) { + unsafe { + for i in 0..self.max_instances { + let ptr = self.mapping.as_mut_ptr().add(i * self.instance_size) as *mut Instance; + std::ptr::drop_in_place(ptr); + } + } + } +} + +/// Represents a pool of WebAssembly linear memories. +/// +/// A linear memory is divided into accessible pages and guard pages. +/// +/// Each instance index into the pool returns an iterator over the base addresses +/// of the instance's linear memories. +/// +/// +/// The userfault handler relies on how memories are stored in the mapping, +/// so make sure the uffd implementation is kept up-to-date. +#[derive(Debug)] +struct MemoryPool { + mapping: Mmap, + memory_size: usize, + max_memories: usize, + max_instances: usize, + max_wasm_pages: u32, +} + +impl MemoryPool { + fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { + // The maximum module memory page count cannot exceed 65536 pages + if module_limits.memory_pages > 0x10000 { + bail!( + "module memory page limit of {} exceeds the maximum of 65536", + module_limits.memory_pages + ); + } + + // The maximum module memory page count cannot exceed the memory reservation size + if (module_limits.memory_pages * WASM_PAGE_SIZE) as u64 + > instance_limits.memory_reservation_size + { + bail!( + "module memory page limit of {} pages exceeds the memory reservation size limit of {} bytes", + module_limits.memory_pages, + instance_limits.memory_reservation_size + ); + } + + let memory_size = if module_limits.memory_pages > 0 { + usize::try_from(instance_limits.memory_reservation_size) + .map_err(|_| anyhow!("memory reservation size exceeds addressable memory"))? + } else { + 0 + }; + + debug_assert!( + memory_size % region::page::size() == 0, + "memory size {} is not a multiple of system page size", + memory_size + ); + + let max_instances = instance_limits.count as usize; + let max_memories = module_limits.memories as usize; + + let allocation_size = memory_size + .checked_mul(max_memories) + .and_then(|c| c.checked_mul(max_instances)) + .ok_or_else(|| { + anyhow!("total size of memory reservation exceeds addressable memory") + })?; + + // Create a completely inaccessible region to start + let mapping = Mmap::accessible_reserved(0, allocation_size) + .context("failed to create memory pool mapping")?; + + let pool = Self { + mapping, + memory_size, + max_memories, + max_instances, + max_wasm_pages: module_limits.memory_pages, + }; + + // uffd support requires some special setup for the memory pool + #[cfg(all(feature = "uffd", target_os = "linux"))] + initialize_memory_pool(&pool)?; + + Ok(pool) + } + + fn get(&self, instance_index: usize) -> impl Iterator { + debug_assert!(instance_index < self.max_instances); + + let base: *mut u8 = unsafe { + self.mapping + .as_mut_ptr() + .add(instance_index * self.memory_size * self.max_memories) as _ + }; + + let size = self.memory_size; + (0..self.max_memories).map(move |i| unsafe { base.add(i * size) }) + } +} + +/// Represents a pool of WebAssembly tables. +/// +/// Each instance index into the pool returns an iterator over the base addresses +/// of the instance's tables. +#[derive(Debug)] +struct TablePool { + mapping: Mmap, + table_size: usize, + max_tables: usize, + max_instances: usize, + page_size: usize, + max_elements: u32, +} + +impl TablePool { + fn new(module_limits: &ModuleLimits, instance_limits: &InstanceLimits) -> Result { + let page_size = region::page::size(); + + let table_size = if module_limits.table_elements > 0 { + round_up_to_pow2( + mem::size_of::<*mut u8>() + .checked_mul(module_limits.table_elements as usize) + .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?, + page_size, + ) + } else { + 0 + }; + + let max_instances = instance_limits.count as usize; + let max_tables = module_limits.tables as usize; + + let allocation_size = table_size + .checked_mul(max_tables) + .and_then(|c| c.checked_mul(max_instances)) + .ok_or_else(|| anyhow!("total size of instance tables exceeds addressable memory"))?; + + let mapping = Mmap::accessible_reserved(allocation_size, allocation_size) + .context("failed to create table pool mapping")?; + + Ok(Self { + mapping, + table_size, + max_tables, + max_instances, + page_size, + max_elements: module_limits.table_elements, + }) + } + + fn get(&self, instance_index: usize) -> impl Iterator { + debug_assert!(instance_index < self.max_instances); + + let base: *mut u8 = unsafe { + self.mapping + .as_mut_ptr() + .add(instance_index * self.table_size * self.max_tables) as _ + }; + + let size = self.table_size; + (0..self.max_tables).map(move |i| unsafe { base.add(i * size) }) + } +} + +/// Represents a pool of execution stacks (used for the async fiber implementation). +/// +/// Each index into the pool represents a single execution stack. The maximum number of +/// stacks is the same as the maximum number of instances. +/// +/// As stacks grow downwards, each stack starts (lowest address) with a guard page +/// that can be used to detect stack overflow. +/// +/// The top of the stack (starting stack pointer) is returned when a stack is allocated +/// from the pool. +#[derive(Debug)] +struct StackPool { + mapping: Mmap, + stack_size: usize, + max_instances: usize, + page_size: usize, + free_list: Mutex>, +} + +impl StackPool { + fn new(instance_limits: &InstanceLimits, stack_size: usize) -> Result { + let page_size = region::page::size(); + + // On Windows, don't allocate any fiber stacks as native fibers are always used + // Add a page to the stack size for the guard page when using fiber stacks + let stack_size = if cfg!(windows) || stack_size == 0 { + 0 + } else { + round_up_to_pow2(stack_size, page_size) + .checked_add(page_size) + .ok_or_else(|| anyhow!("stack size exceeds addressable memory"))? + }; + + let max_instances = instance_limits.count as usize; + + let allocation_size = stack_size + .checked_mul(max_instances) + .ok_or_else(|| anyhow!("total size of execution stacks exceeds addressable memory"))?; + + let mapping = Mmap::accessible_reserved(allocation_size, allocation_size) + .context("failed to create stack pool mapping")?; + + // Set up the stack guard pages + if allocation_size > 0 { + unsafe { + for i in 0..max_instances { + // Make the stack guard page inaccessible + let bottom_of_stack = mapping.as_mut_ptr().add(i * stack_size); + region::protect(bottom_of_stack, page_size, region::Protection::NONE) + .context("failed to protect stack guard page")?; + } + } + } + + Ok(Self { + mapping, + stack_size, + max_instances, + page_size, + free_list: Mutex::new((0..max_instances).collect()), + }) + } + + fn allocate(&self, strategy: PoolingAllocationStrategy) -> Result<*mut u8, FiberStackError> { + // Stacks are not supported if nothing was allocated + if self.stack_size == 0 { + return Err(FiberStackError::NotSupported); + } + + let index = { + let mut free_list = self.free_list.lock().unwrap(); + if free_list.is_empty() { + return Err(FiberStackError::Limit(self.max_instances as u32)); + } + let free_index = strategy.next(free_list.len()); + free_list.swap_remove(free_index) + }; + + debug_assert!(index < self.max_instances); + + unsafe { + // Remove the guard page from the size + let size_without_guard = self.stack_size - self.page_size; + + let bottom_of_stack = self + .mapping + .as_mut_ptr() + .add((index * self.stack_size) + self.page_size); + + commit_stack_pages(bottom_of_stack, size_without_guard) + .map_err(FiberStackError::Resource)?; + + // The top of the stack should be returned + Ok(bottom_of_stack.add(size_without_guard)) + } + } + + fn deallocate(&self, top_of_stack: *mut u8) { + debug_assert!(!top_of_stack.is_null()); + + unsafe { + // Remove the guard page from the size + let stack_size = self.stack_size - self.page_size; + let bottom_of_stack = top_of_stack.sub(stack_size); + + let base = self.mapping.as_ptr() as usize; + let start_of_stack = (bottom_of_stack as usize) - self.page_size; + + debug_assert!(start_of_stack >= base && start_of_stack < (base + self.mapping.len())); + debug_assert!((start_of_stack - base) % self.stack_size == 0); + + let index = (start_of_stack - base) / self.stack_size; + debug_assert!(index < self.max_instances); + + decommit_stack_pages(bottom_of_stack, stack_size).unwrap(); + + self.free_list.lock().unwrap().push(index); + } + } +} + +/// Implements the pooling instance allocator. +/// +/// This allocator internally maintains pools of instances, memories, tables, and stacks. +/// +/// Note: the resource pools are manually dropped so that the fault handler terminates correctly. +#[derive(Debug)] +pub struct PoolingInstanceAllocator { + strategy: PoolingAllocationStrategy, + module_limits: ModuleLimits, + instance_limits: InstanceLimits, + // This is manually drop so that the pools unmap their memory before the page fault handler drops. + instances: mem::ManuallyDrop, + stacks: StackPool, + #[cfg(all(feature = "uffd", target_os = "linux"))] + _fault_handler: imp::PageFaultHandler, +} + +impl PoolingInstanceAllocator { + /// Creates a new pooling instance allocator with the given strategy and limits. + pub fn new( + strategy: PoolingAllocationStrategy, + module_limits: ModuleLimits, + mut instance_limits: InstanceLimits, + stack_size: usize, + ) -> Result { + if instance_limits.count == 0 { + bail!("the instance count limit cannot be zero"); + } + + // Round the memory reservation size to the nearest Wasm page size + instance_limits.memory_reservation_size = u64::try_from(round_up_to_pow2( + usize::try_from(instance_limits.memory_reservation_size).unwrap(), + WASM_PAGE_SIZE as usize, + )) + .unwrap(); + + // Cap the memory reservation size to 8 GiB (maximum 4 GiB accessible + 4 GiB of guard region) + instance_limits.memory_reservation_size = + min(instance_limits.memory_reservation_size, 0x200000000); + + let instances = InstancePool::new(&module_limits, &instance_limits)?; + let stacks = StackPool::new(&instance_limits, stack_size)?; + + #[cfg(all(feature = "uffd", target_os = "linux"))] + let _fault_handler = imp::PageFaultHandler::new(&instances)?; + + Ok(Self { + strategy, + module_limits, + instance_limits, + instances: mem::ManuallyDrop::new(instances), + stacks, + #[cfg(all(feature = "uffd", target_os = "linux"))] + _fault_handler, + }) + } +} + +impl Drop for PoolingInstanceAllocator { + fn drop(&mut self) { + // Manually drop the pools before the fault handler (if uffd is enabled) + // This ensures that any fault handler thread monitoring the pool memory terminates + unsafe { + mem::ManuallyDrop::drop(&mut self.instances); + } + } +} + +unsafe impl InstanceAllocator for PoolingInstanceAllocator { + fn validate(&self, module: &Module) -> Result<()> { + self.module_limits.validate(module) + } + + fn adjust_tunables(&self, tunables: &mut Tunables) { + let memory_reservation_size = self.instance_limits.memory_reservation_size; + + // For reservation sizes larger than 4 GiB, use a guard region to elide bounds checks + if memory_reservation_size >= 0x100000000 { + tunables.static_memory_bound = 0x10000; // in Wasm pages + tunables.static_memory_offset_guard_size = memory_reservation_size - 0x100000000; + } else { + tunables.static_memory_bound = + u32::try_from(memory_reservation_size).unwrap() / WASM_PAGE_SIZE; + tunables.static_memory_offset_guard_size = 0; + } + + // Treat the static memory bound as the maximum for unbounded Wasm memories + // Because we guarantee a module cannot compile unless it fits in the limits of + // the pool allocator, this ensures all memories are treated as static (i.e. immovable). + tunables.static_memory_bound_is_maximum = true; + } + + unsafe fn allocate( + &self, + req: InstanceAllocationRequest, + ) -> Result { + self.instances.allocate(self.strategy, req) + } + + unsafe fn initialize( + &self, + handle: &InstanceHandle, + is_bulk_memory: bool, + ) -> Result<(), InstantiationError> { + let instance = handle.instance(); + + cfg_if::cfg_if! { + if #[cfg(all(feature = "uffd", target_os = "linux"))] { + match &instance.module.memory_initialization { + wasmtime_environ::MemoryInitialization::Paged{ out_of_bounds, .. } => { + if !is_bulk_memory { + super::check_init_bounds(instance)?; + } + + // Initialize the tables + super::initialize_tables(instance)?; + + // Don't initialize the memory; the fault handler will back the pages when accessed + + // If there was an out of bounds access observed in initialization, return a trap + if *out_of_bounds { + return Err(InstantiationError::Trap(crate::traphandlers::Trap::wasm( + wasmtime_environ::ir::TrapCode::HeapOutOfBounds, + ))); + } + + Ok(()) + }, + _ => initialize_instance(instance, is_bulk_memory) + } + } else { + initialize_instance(instance, is_bulk_memory) + } + } + } + + unsafe fn deallocate(&self, handle: &InstanceHandle) { + self.instances.deallocate(handle); + } + + fn allocate_fiber_stack(&self) -> Result<*mut u8, FiberStackError> { + self.stacks.allocate(self.strategy) + } + + unsafe fn deallocate_fiber_stack(&self, stack: *mut u8) { + self.stacks.deallocate(stack); + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{Imports, VMSharedSignatureIndex}; + use wasmtime_environ::{ + entity::EntityRef, + ir::Type, + wasm::{Global, GlobalInit, Memory, SignatureIndex, Table, TableElementType, WasmType}, + MemoryPlan, ModuleType, TablePlan, TableStyle, + }; + + #[test] + fn test_module_imported_functions_limit() { + let limits = ModuleLimits { + imported_functions: 0, + ..Default::default() + }; + + let mut module = Module::default(); + + module.functions.push(SignatureIndex::new(0)); + assert!(limits.validate(&module).is_ok()); + + module.num_imported_funcs = 1; + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("imported function count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_imported_tables_limit() { + let limits = ModuleLimits { + imported_tables: 0, + ..Default::default() + }; + + let mut module = Module::default(); + + module.table_plans.push(TablePlan { + style: TableStyle::CallerChecksSignature, + table: Table { + wasm_ty: WasmType::FuncRef, + ty: TableElementType::Func, + minimum: 0, + maximum: None, + }, + }); + + assert!(limits.validate(&module).is_ok()); + + module.num_imported_tables = 1; + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("imported tables count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_imported_memories_limit() { + let limits = ModuleLimits { + imported_memories: 0, + ..Default::default() + }; + + let mut module = Module::default(); + + module.memory_plans.push(MemoryPlan { + style: MemoryStyle::Static { bound: 0 }, + memory: Memory { + minimum: 0, + maximum: None, + shared: false, + }, + offset_guard_size: 0, + }); + + assert!(limits.validate(&module).is_ok()); + + module.num_imported_memories = 1; + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("imported memories count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_imported_globals_limit() { + let limits = ModuleLimits { + imported_globals: 0, + ..Default::default() + }; + + let mut module = Module::default(); + + module.globals.push(Global { + wasm_ty: WasmType::I32, + ty: Type::int(32).unwrap(), + mutability: false, + initializer: GlobalInit::I32Const(0), + }); + + assert!(limits.validate(&module).is_ok()); + + module.num_imported_globals = 1; + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("imported globals count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_types_limit() { + let limits = ModuleLimits { + types: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert!(limits.validate(&module).is_ok()); + + module + .types + .push(ModuleType::Function(SignatureIndex::new(0))); + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("defined types count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_functions_limit() { + let limits = ModuleLimits { + functions: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert!(limits.validate(&module).is_ok()); + + module.functions.push(SignatureIndex::new(0)); + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("defined functions count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_tables_limit() { + let limits = ModuleLimits { + tables: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert!(limits.validate(&module).is_ok()); + + module.table_plans.push(TablePlan { + style: TableStyle::CallerChecksSignature, + table: Table { + wasm_ty: WasmType::FuncRef, + ty: TableElementType::Func, + minimum: 0, + maximum: None, + }, + }); + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("defined tables count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_memories_limit() { + let limits = ModuleLimits { + memories: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert!(limits.validate(&module).is_ok()); + + module.memory_plans.push(MemoryPlan { + style: MemoryStyle::Static { bound: 0 }, + memory: Memory { + minimum: 0, + maximum: None, + shared: false, + }, + offset_guard_size: 0, + }); + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("defined memories count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_defined_globals_limit() { + let limits = ModuleLimits { + globals: 0, + ..Default::default() + }; + + let mut module = Module::default(); + assert!(limits.validate(&module).is_ok()); + + module.globals.push(Global { + wasm_ty: WasmType::I32, + ty: Type::int(32).unwrap(), + mutability: false, + initializer: GlobalInit::I32Const(0), + }); + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("defined globals count of 1 exceeds the limit of 0".into()) + ); + } + + #[test] + fn test_module_table_minimum_elements_limit() { + let limits = ModuleLimits { + tables: 1, + table_elements: 10, + ..Default::default() + }; + + let mut module = Module::default(); + module.table_plans.push(TablePlan { + style: TableStyle::CallerChecksSignature, + table: Table { + wasm_ty: WasmType::FuncRef, + ty: TableElementType::Func, + minimum: 11, + maximum: None, + }, + }); + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err( + "table index 0 has a minimum element size of 11 which exceeds the limit of 10" + .into() + ) + ); + } + + #[test] + fn test_module_memory_minimum_size_limit() { + let limits = ModuleLimits { + memories: 1, + memory_pages: 5, + ..Default::default() + }; + + let mut module = Module::default(); + module.memory_plans.push(MemoryPlan { + style: MemoryStyle::Static { bound: 0 }, + memory: Memory { + minimum: 6, + maximum: None, + shared: false, + }, + offset_guard_size: 0, + }); + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("memory index 0 has a minimum page size of 6 which exceeds the limit of 5".into()) + ); + } + + #[test] + fn test_module_with_dynamic_memory_style() { + let limits = ModuleLimits { + memories: 1, + memory_pages: 5, + ..Default::default() + }; + + let mut module = Module::default(); + module.memory_plans.push(MemoryPlan { + style: MemoryStyle::Dynamic, + memory: Memory { + minimum: 1, + maximum: None, + shared: false, + }, + offset_guard_size: 0, + }); + assert_eq!( + limits.validate(&module).map_err(|e| e.to_string()), + Err("memory index 0 has an unsupported dynamic memory plan style".into()) + ); + } + + #[test] + fn test_next_available_allocation_strategy() { + let strat = PoolingAllocationStrategy::NextAvailable; + assert_eq!(strat.next(10), 9); + assert_eq!(strat.next(5), 4); + assert_eq!(strat.next(1), 0); + } + + #[test] + fn test_random_allocation_strategy() { + let strat = PoolingAllocationStrategy::Random; + assert!(strat.next(100) < 100); + assert_eq!(strat.next(1), 0); + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_instance_pool() -> Result<()> { + let module_limits = ModuleLimits { + imported_functions: 0, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 0, + functions: 0, + tables: 1, + memories: 1, + globals: 0, + table_elements: 10, + memory_pages: 1, + }; + let instance_limits = InstanceLimits { + count: 3, + memory_reservation_size: WASM_PAGE_SIZE as u64, + }; + + let instances = InstancePool::new(&module_limits, &instance_limits)?; + + assert_eq!( + instances.offsets.pointer_size, + std::mem::size_of::<*const u8>() as u8 + ); + assert_eq!(instances.offsets.num_signature_ids, 0); + assert_eq!(instances.offsets.num_imported_functions, 0); + assert_eq!(instances.offsets.num_imported_tables, 0); + assert_eq!(instances.offsets.num_imported_memories, 0); + assert_eq!(instances.offsets.num_imported_globals, 0); + assert_eq!(instances.offsets.num_defined_functions, 0); + assert_eq!(instances.offsets.num_defined_tables, 1); + assert_eq!(instances.offsets.num_defined_memories, 1); + assert_eq!(instances.offsets.num_defined_globals, 0); + assert_eq!(instances.instance_size, 4096); + assert_eq!(instances.max_instances, 3); + + assert_eq!(&*instances.free_list.lock().unwrap(), &[0, 1, 2]); + + let mut handles = Vec::new(); + let module = Arc::new(Module::default()); + let finished_functions = &PrimaryMap::new(); + + for _ in (0..3).rev() { + handles.push( + instances + .allocate( + PoolingAllocationStrategy::NextAvailable, + InstanceAllocationRequest { + module: module.clone(), + finished_functions, + imports: Imports { + functions: &[], + tables: &[], + memories: &[], + globals: &[], + }, + lookup_shared_signature: &|_| VMSharedSignatureIndex::default(), + host_state: Box::new(()), + interrupts: std::ptr::null(), + externref_activations_table: std::ptr::null_mut(), + stack_map_registry: std::ptr::null_mut(), + }, + ) + .expect("allocation should succeed"), + ); + } + + assert_eq!(&*instances.free_list.lock().unwrap(), &[]); + + match instances.allocate( + PoolingAllocationStrategy::NextAvailable, + InstanceAllocationRequest { + module: module.clone(), + finished_functions, + imports: Imports { + functions: &[], + tables: &[], + memories: &[], + globals: &[], + }, + lookup_shared_signature: &|_| VMSharedSignatureIndex::default(), + host_state: Box::new(()), + interrupts: std::ptr::null(), + externref_activations_table: std::ptr::null_mut(), + stack_map_registry: std::ptr::null_mut(), + }, + ) { + Err(InstantiationError::Limit(3)) => {} + _ => panic!("unexpected error"), + }; + + for handle in handles.drain(..) { + instances.deallocate(&handle); + } + + assert_eq!(&*instances.free_list.lock().unwrap(), &[2, 1, 0]); + + Ok(()) + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_memory_pool() -> Result<()> { + let pool = MemoryPool::new( + &ModuleLimits { + imported_functions: 0, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 0, + functions: 0, + tables: 0, + memories: 3, + globals: 0, + table_elements: 0, + memory_pages: 1, + }, + &InstanceLimits { + count: 5, + memory_reservation_size: WASM_PAGE_SIZE as u64, + }, + )?; + + assert_eq!(pool.memory_size, WASM_PAGE_SIZE as usize); + assert_eq!(pool.max_memories, 3); + assert_eq!(pool.max_instances, 5); + assert_eq!(pool.max_wasm_pages, 1); + + let base = pool.mapping.as_ptr() as usize; + + for i in 0..5 { + let mut iter = pool.get(i); + + for j in 0..3 { + assert_eq!( + iter.next().unwrap() as usize - base, + ((i * 3) + j) * pool.memory_size + ); + } + + assert_eq!(iter.next(), None); + } + + Ok(()) + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_table_pool() -> Result<()> { + let pool = TablePool::new( + &ModuleLimits { + imported_functions: 0, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 0, + functions: 0, + tables: 4, + memories: 0, + globals: 0, + table_elements: 100, + memory_pages: 0, + }, + &InstanceLimits { + count: 7, + memory_reservation_size: WASM_PAGE_SIZE as u64, + }, + )?; + + assert_eq!(pool.table_size, 4096); + assert_eq!(pool.max_tables, 4); + assert_eq!(pool.max_instances, 7); + assert_eq!(pool.page_size, 4096); + assert_eq!(pool.max_elements, 100); + + let base = pool.mapping.as_ptr() as usize; + + for i in 0..7 { + let mut iter = pool.get(i); + + for j in 0..4 { + assert_eq!( + iter.next().unwrap() as usize - base, + ((i * 4) + j) * pool.table_size + ); + } + + assert_eq!(iter.next(), None); + } + + Ok(()) + } + + #[cfg(all(unix, target_pointer_width = "64"))] + #[test] + fn test_stack_pool() -> Result<()> { + let pool = StackPool::new( + &InstanceLimits { + count: 10, + memory_reservation_size: 0, + }, + 1, + )?; + + assert_eq!(pool.stack_size, 8192); + assert_eq!(pool.max_instances, 10); + assert_eq!(pool.page_size, 4096); + + assert_eq!( + &*pool.free_list.lock().unwrap(), + &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ); + + let base = pool.mapping.as_ptr() as usize; + + let mut stacks = Vec::new(); + for i in (0..10).rev() { + let stack = pool + .allocate(PoolingAllocationStrategy::NextAvailable) + .expect("allocation should succeed"); + assert_eq!(((stack as usize - base) / pool.stack_size) - 1, i); + stacks.push(stack); + } + + assert_eq!(&*pool.free_list.lock().unwrap(), &[]); + + match pool + .allocate(PoolingAllocationStrategy::NextAvailable) + .unwrap_err() + { + FiberStackError::Limit(10) => {} + _ => panic!("unexpected error"), + }; + + for stack in stacks { + pool.deallocate(stack); + } + + assert_eq!( + &*pool.free_list.lock().unwrap(), + &[9, 8, 7, 6, 5, 4, 3, 2, 1, 0], + ); + + Ok(()) + } + + #[test] + fn test_pooling_allocator_with_zero_instance_count() { + assert_eq!( + PoolingInstanceAllocator::new( + PoolingAllocationStrategy::Random, + ModuleLimits::default(), + InstanceLimits { + count: 0, + ..Default::default() + }, + 4096 + ) + .map_err(|e| e.to_string()) + .expect_err("expected a failure constructing instance allocator"), + "the instance count limit cannot be zero" + ); + } + + #[test] + fn test_pooling_allocator_with_memory_pages_exeeded() { + assert_eq!( + PoolingInstanceAllocator::new( + PoolingAllocationStrategy::Random, + ModuleLimits { + memory_pages: 0x10001, + ..Default::default() + }, + InstanceLimits { + count: 1, + memory_reservation_size: 1, + }, + 4096 + ) + .map_err(|e| e.to_string()) + .expect_err("expected a failure constructing instance allocator"), + "module memory page limit of 65537 exceeds the maximum of 65536" + ); + } + + #[test] + fn test_pooling_allocator_with_reservation_size_exeeded() { + assert_eq!( + PoolingInstanceAllocator::new( + PoolingAllocationStrategy::Random, + ModuleLimits { + memory_pages: 2, + ..Default::default() + }, + InstanceLimits { + count: 1, + memory_reservation_size: 1, + }, + 4096, + ) + .map_err(|e| e.to_string()) + .expect_err("expected a failure constructing instance allocator"), + "module memory page limit of 2 pages exceeds the memory reservation size limit of 65536 bytes" + ); + } + + #[cfg_attr(target_arch = "aarch64", ignore)] // https://github.com/bytecodealliance/wasmtime/pull/2518#issuecomment-747280133 + #[cfg(all(unix, target_pointer_width = "64"))] + #[test] + fn test_stack_zeroed() -> Result<()> { + let allocator = PoolingInstanceAllocator::new( + PoolingAllocationStrategy::NextAvailable, + ModuleLimits { + imported_functions: 0, + types: 0, + functions: 0, + tables: 0, + memories: 0, + globals: 0, + table_elements: 0, + memory_pages: 0, + ..Default::default() + }, + InstanceLimits { + count: 1, + memory_reservation_size: 1, + }, + 4096, + )?; + + unsafe { + for _ in 0..10 { + let stack = allocator.allocate_fiber_stack()?; + + // The stack pointer is at the top, so decerement it first + let addr = stack.sub(1); + + assert_eq!(*addr, 0); + *addr = 1; + + allocator.deallocate_fiber_stack(stack); + } + } + + Ok(()) + } +} diff --git a/crates/runtime/src/instance/allocator/pooling/linux.rs b/crates/runtime/src/instance/allocator/pooling/linux.rs new file mode 100644 index 0000000000..324200efe4 --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling/linux.rs @@ -0,0 +1,58 @@ +use anyhow::{bail, Context, Result}; + +fn decommit(addr: *mut u8, len: usize, protect: bool) -> Result<()> { + if len == 0 { + return Ok(()); + } + + unsafe { + if protect { + region::protect(addr, len, region::Protection::NONE) + .context("failed to protect memory pages")?; + } + + // On Linux, this is enough to cause the kernel to initialize the pages to 0 on next access + if libc::madvise(addr as _, len, libc::MADV_DONTNEED) != 0 { + bail!( + "madvise failed to decommit: {}", + std::io::Error::last_os_error() + ); + } + } + + Ok(()) +} + +pub fn commit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + // Just change the protection level to READ|WRITE + unsafe { + region::protect(addr, len, region::Protection::READ_WRITE) + .context("failed to make linear memory pages read/write") + } +} + +pub fn decommit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, true) +} + +pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as table pages remain READ|WRITE + Ok(()) +} + +pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, false) +} + +pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as stack pages remain READ|WRITE + Ok(()) +} + +pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, false) +} diff --git a/crates/runtime/src/instance/allocator/pooling/uffd.rs b/crates/runtime/src/instance/allocator/pooling/uffd.rs new file mode 100644 index 0000000000..ebe5effbc3 --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling/uffd.rs @@ -0,0 +1,596 @@ +//! This module implements user space page fault handling with the `userfaultfd` ("uffd") system call on Linux. +//! +//! Handling page faults for memory accesses in regions relating to WebAssembly instances +//! enables the runtime to protect guard pages in user space rather than kernel space (i.e. without `mprotect`). +//! +//! Additionally, linear memories can be lazy-initialized upon first access. +//! +//! Handling faults in user space is slower than handling faults in the kernel. However, +//! in use cases where there is a high number of concurrently executing instances, handling the faults +//! in user space requires rarely changing memory protection levels. This can improve concurrency +//! by not taking kernel memory manager locks and may decrease TLB shootdowns as fewer page table entries need +//! to continually change. +//! +//! Here's how the `uffd` feature works: +//! +//! 1. A user fault file descriptor is created to monitor specific areas of the address space. +//! 2. A thread is spawned to continually read events from the user fault file descriptor. +//! 3. When a page fault event is received, the handler thread calculates where the fault occurred: +//! a) If the fault occurs on a linear memory page, it is handled by either copying the page from +//! initialization data or zeroing it. +//! b) If the fault occurs on a guard page, the protection level of the guard page is changed to +//! force the kernel to signal SIGBUS on the next retry. The faulting page is recorded so the +//! protection level can be reset in the future. +//! 4. Faults to address space relating to an instance may occur from both Wasmtime (e.g. instance +//! initialization) or from WebAssembly code (e.g. reading from or writing to linear memory), +//! therefore the user fault handling must do as little work as possible to handle the fault. +//! 5. When the pooling allocator is dropped, it will drop the memory mappings relating to the pool; this +//! generates unmap events for the fault handling thread, which responds by decrementing the mapping +//! count. When the count reaches zero, the user fault handling thread will gracefully terminate. +//! +//! This feature requires a Linux kernel 4.11 or newer to use. + +use super::{InstancePool, MemoryPool}; +use crate::instance::Instance; +use anyhow::{bail, Context, Result}; +use std::thread; +use userfaultfd::{Event, FeatureFlags, IoctlFlags, Uffd, UffdBuilder}; +use wasmtime_environ::{entity::EntityRef, wasm::DefinedMemoryIndex, MemoryInitialization}; + +const WASM_PAGE_SIZE: usize = wasmtime_environ::WASM_PAGE_SIZE as usize; + +fn decommit(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + unsafe { + // On Linux, this tells the kernel to discard the backing of the pages in the range. + // If the discarded pages are part of a uffd region, then the next access will fault + // and the user fault handler will receive the event. + // If the pages are not monitored by uffd, the kernel will zero the page on next access, + // as if it were mmap'd for the first time. + if libc::madvise(addr as _, len, libc::MADV_DONTNEED) != 0 { + bail!( + "madvise failed to decommit: {}", + std::io::Error::last_os_error() + ); + } + } + + Ok(()) +} + +pub fn commit_memory_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as memory pages remain READ|WRITE with uffd + Ok(()) +} + +pub fn decommit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) +} + +pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as table pages remain READ|WRITE + Ok(()) +} + +pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) +} + +pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as stack pages remain READ|WRITE + Ok(()) +} + +pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) +} + +/// This is used to initialize the memory pool when uffd is enabled. +/// +/// Without uffd, all of the memory pool's pages are initially protected with `NONE` to treat the entire +/// range as guard pages. When an instance is created, the initial pages of the memory are +/// changed to `READ_WRITE`. +/// +/// With uffd, however, the potentially accessible pages of the each linear memory are made `READ_WRITE` and +/// the page fault handler will detect an out of bounds access and treat the page, temporarily, +/// as a guard page. +pub(super) fn initialize_memory_pool(pool: &MemoryPool) -> Result<()> { + if pool.memory_size == 0 || pool.max_wasm_pages == 0 { + return Ok(()); + } + + for i in 0..pool.max_instances { + for base in pool.get(i) { + unsafe { + region::protect( + base as _, + pool.max_wasm_pages as usize * WASM_PAGE_SIZE, + region::Protection::READ_WRITE, + ) + .context("failed to initialize memory pool for uffd")?; + } + } + } + + Ok(()) +} + +/// This is used to reset a linear memory's guard page back to read-write as the page might be accessible +/// again in the future depending on how the linear memory grows. +fn reset_guard_page(addr: *mut u8, len: usize) -> Result<()> { + unsafe { + region::protect(addr, len, region::Protection::READ_WRITE) + .context("failed to reset guard page") + } +} + +/// Represents a location of a page fault within monitored regions of memory. +enum FaultLocation<'a> { + /// The address location is in a WebAssembly linear memory page. + /// The fault handler will copy the pages from initialization data if necessary. + MemoryPage { + /// The address of the page being accessed. + page_addr: *mut u8, + /// The length of the page being accessed. + len: usize, + /// The instance related to the memory page that was accessed. + instance: &'a Instance, + /// The index of the memory that was accessed. + memory_index: DefinedMemoryIndex, + /// The Wasm page index to initialize if the access was not a guard page. + page_index: Option, + }, +} + +/// Used to resolve fault addresses to a location. +/// +/// This implementation relies heavily on how the linear memory pool organizes its memory. +/// +/// `usize` is used here instead of pointers to keep this `Send` as it gets sent to the handler thread. +struct FaultLocator { + instances_start: usize, + instance_size: usize, + max_instances: usize, + memories_start: usize, + memories_end: usize, + memory_size: usize, + max_memories: usize, +} + +impl FaultLocator { + fn new(instances: &InstancePool) -> Self { + let instances_start = instances.mapping.as_ptr() as usize; + let memories_start = instances.memories.mapping.as_ptr() as usize; + let memories_end = memories_start + instances.memories.mapping.len(); + + // Should always have instances + debug_assert!(instances_start != 0); + + Self { + instances_start, + instance_size: instances.instance_size, + max_instances: instances.max_instances, + memories_start, + memories_end, + memory_size: instances.memories.memory_size, + max_memories: instances.memories.max_memories, + } + } + + /// This is super-duper unsafe as it is used from the handler thread + /// to access instance data without any locking primitives. + /// + /// It is assumed that the thread that owns the instance being accessed is + /// currently suspended waiting on a fault to be handled. + /// + /// Of course a stray faulting memory access from a thread that does not own + /// the instance might introduce a race, but this implementation considers + /// such to be a serious soundness bug not originating in this code. + /// + /// If the assumption holds true, accessing the instance data from the handler thread + /// should, in theory, be safe. + unsafe fn get_instance(&self, index: usize) -> &Instance { + debug_assert!(index < self.max_instances); + &*((self.instances_start + (index * self.instance_size)) as *const Instance) + } + + unsafe fn locate(&self, addr: usize) -> Option { + // Check for a linear memory location + if addr >= self.memories_start && addr < self.memories_end { + let index = (addr - self.memories_start) / self.memory_size; + let memory_index = DefinedMemoryIndex::new(index % self.max_memories); + let memory_start = self.memories_start + (index * self.memory_size); + let page_index = (addr - memory_start) / WASM_PAGE_SIZE; + let instance = self.get_instance(index / self.max_memories); + + let init_page_index = instance.memories.get(memory_index).and_then(|m| { + if page_index < m.size() as usize { + Some(page_index) + } else { + None + } + }); + + return Some(FaultLocation::MemoryPage { + page_addr: (memory_start + page_index * WASM_PAGE_SIZE) as _, + len: WASM_PAGE_SIZE, + instance, + memory_index, + page_index: init_page_index, + }); + } + + None + } +} + +/// This is called following a fault on a guard page. +/// +/// Because the region being monitored is protected read-write, this needs to set the +/// protection level to `NONE` before waking the page. +/// +/// This will cause the kernel to raise a SIGBUS when retrying the fault. +unsafe fn wake_guard_page_access(uffd: &Uffd, page_addr: *const u8, len: usize) -> Result<()> { + // Set the page to NONE to induce a SIGBUS for the access on the next retry + region::protect(page_addr, len, region::Protection::NONE) + .context("failed to change guard page protection")?; + + uffd.wake(page_addr as _, len) + .context("failed to wake guard page access")?; + + Ok(()) +} + +/// This is called to initialize a linear memory page (64 KiB). +/// +/// If paged initialization is used for the module, then we can instruct the kernel to back the page with +/// what is already stored in the initialization data; if the page isn't in the initialization data, +/// it will be zeroed instead. +/// +/// If paged initialization isn't being used, we zero the page. Initialization happens +/// at module instantiation in this case and the segment data will be then copied to the zeroed page. +unsafe fn initialize_wasm_page( + uffd: &Uffd, + instance: &Instance, + page_addr: *const u8, + memory_index: DefinedMemoryIndex, + page_index: usize, +) -> Result<()> { + // Check for paged initialization and copy the page if present in the initialization data + if let MemoryInitialization::Paged { map, .. } = &instance.module.memory_initialization { + let pages = &map[memory_index]; + + if let Some(Some(data)) = pages.get(page_index) { + debug_assert_eq!(data.len(), WASM_PAGE_SIZE); + + log::trace!( + "copying linear memory page from {:p} to {:p}", + data.as_ptr(), + page_addr + ); + + uffd.copy(data.as_ptr() as _, page_addr as _, WASM_PAGE_SIZE, true) + .context("failed to copy linear memory page")?; + + return Ok(()); + } + } + + log::trace!("zeroing linear memory page at {:p}", page_addr); + + uffd.zeropage(page_addr as _, WASM_PAGE_SIZE, true) + .context("failed to zero linear memory page")?; + + Ok(()) +} + +unsafe fn handle_page_fault( + uffd: &Uffd, + locator: &FaultLocator, + addr: *mut std::ffi::c_void, +) -> Result<()> { + match locator.locate(addr as usize) { + Some(FaultLocation::MemoryPage { + page_addr, + len, + instance, + memory_index, + page_index, + }) => { + log::trace!( + "handling fault in linear memory at address {:p} on page {:p}", + addr, + page_addr + ); + + match page_index { + Some(page_index) => { + initialize_wasm_page(&uffd, instance, page_addr, memory_index, page_index)?; + } + None => { + log::trace!("out of bounds memory access at {:p}", addr); + + // Record the guard page fault so the page protection level can be reset later + instance.memories[memory_index].record_guard_page_fault( + page_addr, + len, + reset_guard_page, + ); + wake_guard_page_access(&uffd, page_addr, len)?; + } + } + } + None => { + bail!( + "failed to locate fault address {:p} in registered memory regions", + addr + ); + } + } + + Ok(()) +} + +fn fault_handler_thread(uffd: Uffd, locator: FaultLocator) -> Result<()> { + loop { + match uffd.read_event().expect("failed to read event") { + Some(Event::Unmap { start, end }) => { + log::trace!("memory region unmapped: {:p}-{:p}", start, end); + + let (start, end) = (start as usize, end as usize); + + if start == locator.memories_start && end == locator.memories_end { + break; + } else { + panic!("unexpected memory region unmapped"); + } + } + Some(Event::Pagefault { addr, .. }) => unsafe { + handle_page_fault(&uffd, &locator, addr as _)? + }, + Some(_) => continue, + None => bail!("no event was read from the user fault descriptor"), + } + } + + log::trace!("fault handler thread has successfully terminated"); + + Ok(()) +} + +#[derive(Debug)] +pub struct PageFaultHandler { + thread: Option>>, +} + +impl PageFaultHandler { + pub(super) fn new(instances: &InstancePool) -> Result { + let uffd = UffdBuilder::new() + .close_on_exec(true) + .require_features(FeatureFlags::EVENT_UNMAP) + .create() + .context("failed to create user fault descriptor")?; + + // Register the linear memory pool with the userfault fd + let start = instances.memories.mapping.as_ptr(); + let len = instances.memories.mapping.len(); + + let thread = if !start.is_null() && len > 0 { + let ioctls = uffd + .register(start as _, len) + .context("failed to register user fault range")?; + + if !ioctls.contains(IoctlFlags::WAKE | IoctlFlags::COPY | IoctlFlags::ZEROPAGE) { + bail!( + "required user fault ioctls not supported by the kernel; found: {:?}", + ioctls, + ); + } + + log::trace!( + "user fault handling enabled on linear memory pool at {:p} with size {}", + start, + len + ); + + let locator = FaultLocator::new(&instances); + + Some( + thread::Builder::new() + .name("page fault handler".into()) + .spawn(move || fault_handler_thread(uffd, locator)) + .context("failed to spawn page fault handler thread")?, + ) + } else { + log::trace!("user fault handling disabled as there is no linear memory pool"); + None + }; + + Ok(Self { thread }) + } +} + +impl Drop for PageFaultHandler { + fn drop(&mut self) { + // The handler thread should terminate once all monitored regions of memory are unmapped. + // The pooling instance allocator ensures that the regions are unmapped prior to dropping + // the page fault handler. + if let Some(thread) = self.thread.take() { + thread + .join() + .expect("failed to join page fault handler thread") + .expect("fault handler thread failed"); + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{ + Imports, InstanceAllocationRequest, InstanceLimits, ModuleLimits, + PoolingAllocationStrategy, VMSharedSignatureIndex, + }; + use std::ptr; + use std::sync::Arc; + use wasmtime_environ::{entity::PrimaryMap, wasm::Memory, MemoryPlan, MemoryStyle, Module}; + + #[cfg(target_pointer_width = "64")] + #[test] + fn test_address_locator() { + let module_limits = ModuleLimits { + imported_functions: 0, + imported_tables: 0, + imported_memories: 0, + imported_globals: 0, + types: 0, + functions: 0, + tables: 0, + memories: 2, + globals: 0, + table_elements: 0, + memory_pages: 2, + }; + let instance_limits = InstanceLimits { + count: 3, + memory_reservation_size: (WASM_PAGE_SIZE * 10) as u64, + }; + + let instances = + InstancePool::new(&module_limits, &instance_limits).expect("should allocate"); + + let locator = FaultLocator::new(&instances); + + assert_eq!(locator.instances_start, instances.mapping.as_ptr() as usize); + assert_eq!(locator.instance_size, 4096); + assert_eq!(locator.max_instances, 3); + assert_eq!( + locator.memories_start, + instances.memories.mapping.as_ptr() as usize + ); + assert_eq!( + locator.memories_end, + locator.memories_start + instances.memories.mapping.len() + ); + assert_eq!(locator.memory_size, WASM_PAGE_SIZE * 10); + assert_eq!(locator.max_memories, 2); + + unsafe { + assert!(locator.locate(0).is_none()); + assert!(locator.locate(locator.memories_end).is_none()); + + let mut module = Module::new(); + + for _ in 0..module_limits.memories { + module.memory_plans.push(MemoryPlan { + memory: Memory { + minimum: 2, + maximum: Some(2), + shared: false, + }, + style: MemoryStyle::Static { bound: 1 }, + offset_guard_size: 0, + }); + } + + module_limits.validate(&module).expect("should validate"); + + let mut handles = Vec::new(); + let module = Arc::new(module); + let finished_functions = &PrimaryMap::new(); + + // Allocate the maximum number of instances with the maximum number of memories + for _ in 0..instances.max_instances { + handles.push( + instances + .allocate( + PoolingAllocationStrategy::Random, + InstanceAllocationRequest { + module: module.clone(), + finished_functions, + imports: Imports { + functions: &[], + tables: &[], + memories: &[], + globals: &[], + }, + lookup_shared_signature: &|_| VMSharedSignatureIndex::default(), + host_state: Box::new(()), + interrupts: ptr::null(), + externref_activations_table: ptr::null_mut(), + stack_map_registry: ptr::null_mut(), + }, + ) + .expect("instance should allocate"), + ); + } + + // Validate memory locations + for instance_index in 0..instances.max_instances { + for memory_index in 0..instances.memories.max_memories { + let memory_start = locator.memories_start + + (instance_index * locator.memory_size * locator.max_memories) + + (memory_index * locator.memory_size); + + // Test for access to first page + match locator.locate(memory_start + 10000) { + Some(FaultLocation::MemoryPage { + page_addr, + len, + instance: _, + memory_index: mem_index, + page_index, + }) => { + assert_eq!(page_addr, memory_start as _); + assert_eq!(len, WASM_PAGE_SIZE); + assert_eq!(mem_index, DefinedMemoryIndex::new(memory_index)); + assert_eq!(page_index, Some(0)); + } + _ => panic!("expected a memory page location"), + } + + // Test for access to second page + match locator.locate(memory_start + 1024 + WASM_PAGE_SIZE) { + Some(FaultLocation::MemoryPage { + page_addr, + len, + instance: _, + memory_index: mem_index, + page_index, + }) => { + assert_eq!(page_addr, (memory_start + WASM_PAGE_SIZE) as _); + assert_eq!(len, WASM_PAGE_SIZE); + assert_eq!(mem_index, DefinedMemoryIndex::new(memory_index)); + assert_eq!(page_index, Some(1)); + } + _ => panic!("expected a memory page location"), + } + + // Test for guard page + match locator.locate(memory_start + 10 + 9 * WASM_PAGE_SIZE) { + Some(FaultLocation::MemoryPage { + page_addr, + len, + instance: _, + memory_index: mem_index, + page_index, + }) => { + assert_eq!(page_addr, (memory_start + (9 * WASM_PAGE_SIZE)) as _); + assert_eq!(len, WASM_PAGE_SIZE); + assert_eq!(mem_index, DefinedMemoryIndex::new(memory_index)); + assert_eq!(page_index, None); + } + _ => panic!("expected a memory page location"), + } + } + } + + for handle in handles.drain(..) { + instances.deallocate(&handle); + } + } + } +} diff --git a/crates/runtime/src/instance/allocator/pooling/unix.rs b/crates/runtime/src/instance/allocator/pooling/unix.rs new file mode 100644 index 0000000000..d172f411eb --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling/unix.rs @@ -0,0 +1,68 @@ +use anyhow::{bail, Context, Result}; + +fn decommit(addr: *mut u8, len: usize, protect: bool) -> Result<()> { + if len == 0 { + return Ok(()); + } + + // By creating a new mapping at the same location, this will discard the + // mapping for the pages in the given range. + // The new mapping will be to the CoW zero page, so this effectively + // zeroes the pages. + if unsafe { + libc::mmap( + addr as _, + len, + if protect { + libc::PROT_NONE + } else { + libc::PROT_READ | libc::PROT_WRITE + }, + libc::MAP_PRIVATE | libc::MAP_ANON | libc::MAP_FIXED, + -1, + 0, + ) as *mut u8 + } != addr + { + bail!( + "mmap failed to remap pages: {}", + std::io::Error::last_os_error() + ); + } + + Ok(()) +} + +pub fn commit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + // Just change the protection level to READ|WRITE + unsafe { + region::protect(addr, len, region::Protection::READ_WRITE) + .context("failed to make linear memory pages read/write") + } +} + +pub fn decommit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, true) +} + +pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as table pages remain READ|WRITE + Ok(()) +} + +pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, false) +} + +pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> { + // A no-op as stack pages remain READ|WRITE + Ok(()) +} + +pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len, false) +} diff --git a/crates/runtime/src/instance/allocator/pooling/windows.rs b/crates/runtime/src/instance/allocator/pooling/windows.rs new file mode 100644 index 0000000000..286cd459fe --- /dev/null +++ b/crates/runtime/src/instance/allocator/pooling/windows.rs @@ -0,0 +1,55 @@ +use anyhow::{bail, Result}; +use winapi::um::memoryapi::{VirtualAlloc, VirtualFree}; +use winapi::um::winnt::{MEM_COMMIT, MEM_DECOMMIT, PAGE_READWRITE}; + +pub fn commit(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + // Memory needs to be committed, so don't use the `region` crate + if unsafe { VirtualAlloc(addr as _, len, MEM_COMMIT, PAGE_READWRITE).is_null() } { + bail!("failed to commit memory as read/write"); + } + + Ok(()) +} + +pub fn decommit(addr: *mut u8, len: usize) -> Result<()> { + if len == 0 { + return Ok(()); + } + + if unsafe { VirtualFree(addr as _, len, MEM_DECOMMIT) } == 0 { + bail!( + "failed to decommit memory pages: {}", + std::io::Error::last_os_error() + ); + } + + Ok(()) +} + +pub fn commit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + commit(addr, len) +} + +pub fn decommit_memory_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) +} + +pub fn commit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + commit(addr, len) +} + +pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) +} + +pub fn commit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + commit(addr, len) +} + +pub fn decommit_stack_pages(addr: *mut u8, len: usize) -> Result<()> { + decommit(addr, len) +} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 00539c36b4..b53e73c162 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -37,9 +37,13 @@ pub mod libcalls; pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; -pub use crate::instance::{InstanceHandle, InstantiationError, LinkError, RuntimeInstance}; +pub use crate::instance::{ + FiberStackError, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstanceLimits, + InstantiationError, LinkError, ModuleLimits, OnDemandInstanceAllocator, + PoolingAllocationStrategy, PoolingInstanceAllocator, RuntimeInstance, +}; pub use crate::jit_int::GdbJitImageRegistration; -pub use crate::memory::{RuntimeLinearMemory, RuntimeMemoryCreator}; +pub use crate::memory::{Memory, RuntimeLinearMemory, RuntimeMemoryCreator}; pub use crate::mmap::Mmap; pub use crate::table::{Table, TableElement}; pub use crate::traphandlers::{ diff --git a/crates/runtime/src/memory.rs b/crates/runtime/src/memory.rs index 340feb3130..024d901246 100644 --- a/crates/runtime/src/memory.rs +++ b/crates/runtime/src/memory.rs @@ -4,15 +4,18 @@ use crate::mmap::Mmap; use crate::vmcontext::VMMemoryDefinition; +use anyhow::Result; use more_asserts::{assert_ge, assert_le}; -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; +use std::cmp::min; use std::convert::TryFrom; +use std::ptr; use wasmtime_environ::{MemoryPlan, MemoryStyle, WASM_MAX_PAGES, WASM_PAGE_SIZE}; /// A memory allocator pub trait RuntimeMemoryCreator: Send + Sync { /// Create new RuntimeLinearMemory - fn new_memory(&self, plan: &MemoryPlan) -> Result, String>; + fn new_memory(&self, plan: &MemoryPlan) -> Result>; } /// A default memory allocator used by Wasmtime @@ -20,8 +23,8 @@ pub struct DefaultMemoryCreator; impl RuntimeMemoryCreator for DefaultMemoryCreator { /// Create new MmapMemory - fn new_memory(&self, plan: &MemoryPlan) -> Result, String> { - Ok(Box::new(MmapMemory::new(plan)?) as Box) + fn new_memory(&self, plan: &MemoryPlan) -> Result> { + Ok(Box::new(MmapMemory::new(plan)?) as _) } } @@ -52,10 +55,6 @@ pub struct MmapMemory { // Size in bytes of extra guard pages after the end to optimize loads and stores with // constant offsets. offset_guard_size: usize, - - // Records whether we're using a bounds-checking strategy which requires - // handlers to catch trapping accesses. - pub(crate) needs_signal_handlers: bool, } #[derive(Debug)] @@ -68,22 +67,13 @@ struct WasmMmap { impl MmapMemory { /// Create a new linear memory instance with specified minimum and maximum number of wasm pages. - pub fn new(plan: &MemoryPlan) -> Result { + pub fn new(plan: &MemoryPlan) -> Result { // `maximum` cannot be set to more than `65536` pages. assert_le!(plan.memory.minimum, WASM_MAX_PAGES); assert!(plan.memory.maximum.is_none() || plan.memory.maximum.unwrap() <= WASM_MAX_PAGES); let offset_guard_bytes = plan.offset_guard_size as usize; - // If we have an offset guard, or if we're doing the static memory - // allocation strategy, we need signal handlers to catch out of bounds - // acceses. - let needs_signal_handlers = offset_guard_bytes > 0 - || match plan.style { - MemoryStyle::Dynamic => false, - MemoryStyle::Static { .. } => true, - }; - let minimum_pages = match plan.style { MemoryStyle::Dynamic => plan.memory.minimum, MemoryStyle::Static { bound } => { @@ -105,7 +95,6 @@ impl MmapMemory { mmap: mmap.into(), maximum: plan.memory.maximum, offset_guard_size: offset_guard_bytes, - needs_signal_handlers, }) } } @@ -177,10 +166,192 @@ impl RuntimeLinearMemory for MmapMemory { /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. fn vmmemory(&self) -> VMMemoryDefinition { - let mut mmap = self.mmap.borrow_mut(); + let mmap = self.mmap.borrow(); VMMemoryDefinition { base: mmap.alloc.as_mut_ptr(), current_length: mmap.size as usize * WASM_PAGE_SIZE as usize, } } } + +enum MemoryStorage { + Static { + base: *mut u8, + size: Cell, + maximum: u32, + make_accessible: fn(*mut u8, usize) -> Result<()>, + /// Stores the pages in the linear memory that have faulted as guard pages when using the `uffd` feature. + /// These pages need their protection level reset before the memory can grow. + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell Result<()>)>>, + }, + Dynamic(Box), +} + +/// Represents an instantiation of a WebAssembly memory. +pub struct Memory(MemoryStorage); + +impl Memory { + /// Create a new dynamic (movable) memory instance for the specified plan. + pub fn new_dynamic(plan: &MemoryPlan, creator: &dyn RuntimeMemoryCreator) -> Result { + Ok(Self(MemoryStorage::Dynamic(creator.new_memory(plan)?))) + } + + /// Create a new static (immovable) memory instance for the specified plan. + pub fn new_static( + plan: &MemoryPlan, + base: *mut u8, + maximum: u32, + make_accessible: fn(*mut u8, usize) -> Result<()>, + ) -> Result { + if plan.memory.minimum > 0 { + make_accessible(base, plan.memory.minimum as usize * WASM_PAGE_SIZE as usize)?; + } + + Ok(Self(MemoryStorage::Static { + base, + size: Cell::new(plan.memory.minimum), + maximum: min(plan.memory.maximum.unwrap_or(maximum), maximum), + make_accessible, + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), + })) + } + + /// Returns the number of allocated wasm pages. + pub fn size(&self) -> u32 { + match &self.0 { + MemoryStorage::Static { size, .. } => size.get(), + MemoryStorage::Dynamic(mem) => mem.size(), + } + } + + /// Returns whether or not the underlying storage of the memory is "static". + pub(crate) fn is_static(&self) -> bool { + if let MemoryStorage::Static { .. } = &self.0 { + true + } else { + false + } + } + + /// Grow memory by the specified amount of wasm pages. + /// + /// Returns `None` if memory can't be grown by the specified amount + /// of wasm pages. + pub fn grow(&self, delta: u32) -> Option { + match &self.0 { + MemoryStorage::Static { + base, + size, + maximum, + make_accessible, + .. + } => { + // Reset any faulted guard pages before growing the memory. + #[cfg(all(feature = "uffd", target_os = "linux"))] + self.reset_guard_pages().ok()?; + + let old_size = size.get(); + if delta == 0 { + return Some(old_size); + } + + let new_size = old_size.checked_add(delta)?; + + if new_size > *maximum || new_size >= WASM_MAX_PAGES { + return None; + } + + let start = usize::try_from(old_size).unwrap() * WASM_PAGE_SIZE as usize; + let len = usize::try_from(delta).unwrap() * WASM_PAGE_SIZE as usize; + + make_accessible(unsafe { base.add(start) }, len).ok()?; + + size.set(new_size); + + Some(old_size) + } + MemoryStorage::Dynamic(mem) => mem.grow(delta), + } + } + + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. + pub fn vmmemory(&self) -> VMMemoryDefinition { + match &self.0 { + MemoryStorage::Static { base, size, .. } => VMMemoryDefinition { + base: *base, + current_length: size.get() as usize * WASM_PAGE_SIZE as usize, + }, + MemoryStorage::Dynamic(mem) => mem.vmmemory(), + } + } + + /// Records a faulted guard page in a static memory. + /// + /// This is used to track faulted guard pages that need to be reset for the uffd feature. + /// + /// This function will panic if called on a dynamic memory. + #[cfg(all(feature = "uffd", target_os = "linux"))] + pub(crate) fn record_guard_page_fault( + &self, + page_addr: *mut u8, + size: usize, + reset: fn(*mut u8, usize) -> Result<()>, + ) { + match &self.0 { + MemoryStorage::Static { + guard_page_faults, .. + } => { + guard_page_faults + .borrow_mut() + .push((page_addr, size, reset)); + } + MemoryStorage::Dynamic(_) => { + unreachable!("dynamic memories should not have guard page faults") + } + } + } + + /// Resets the previously faulted guard pages of a static memory. + /// + /// This is used to reset the protection of any guard pages that were previously faulted. + /// + /// This function will panic if called on a dynamic memory. + #[cfg(all(feature = "uffd", target_os = "linux"))] + pub(crate) fn reset_guard_pages(&self) -> Result<()> { + match &self.0 { + MemoryStorage::Static { + guard_page_faults, .. + } => { + let mut faults = guard_page_faults.borrow_mut(); + for (addr, len, reset) in faults.drain(..) { + reset(addr, len)?; + } + } + MemoryStorage::Dynamic(_) => { + unreachable!("dynamic memories should not have guard page faults") + } + } + + Ok(()) + } +} + +// The default memory representation is an empty memory that cannot grow. +impl Default for Memory { + fn default() -> Self { + fn make_accessible(_ptr: *mut u8, _len: usize) -> Result<()> { + unreachable!() + } + + Self(MemoryStorage::Static { + base: ptr::null_mut(), + size: Cell::new(0), + maximum: 0, + make_accessible, + #[cfg(all(feature = "uffd", target_os = "linux"))] + guard_page_faults: RefCell::new(Vec::new()), + }) + } +} diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs index 483ec5be09..b0612f91c9 100644 --- a/crates/runtime/src/mmap.rs +++ b/crates/runtime/src/mmap.rs @@ -1,6 +1,7 @@ //! Low-level abstraction for allocating and managing zero-filled pages //! of memory. +use anyhow::{bail, Result}; use more_asserts::assert_le; use more_asserts::assert_lt; use std::io; @@ -38,7 +39,7 @@ impl Mmap { } /// Create a new `Mmap` pointing to at least `size` bytes of page-aligned accessible memory. - pub fn with_at_least(size: usize) -> Result { + pub fn with_at_least(size: usize) -> Result { let page_size = region::page::size(); let rounded_size = round_up_to_page_size(size, page_size); Self::accessible_reserved(rounded_size, rounded_size) @@ -48,10 +49,7 @@ impl Mmap { /// within a reserved mapping of `mapping_size` bytes. `accessible_size` and `mapping_size` /// must be native page-size multiples. #[cfg(not(target_os = "windows"))] - pub fn accessible_reserved( - accessible_size: usize, - mapping_size: usize, - ) -> Result { + pub fn accessible_reserved(accessible_size: usize, mapping_size: usize) -> Result { let page_size = region::page::size(); assert_le!(accessible_size, mapping_size); assert_eq!(mapping_size & (page_size - 1), 0); @@ -76,7 +74,7 @@ impl Mmap { ) }; if ptr as isize == -1_isize { - return Err(io::Error::last_os_error().to_string()); + bail!("mmap failed: {}", io::Error::last_os_error()); } Self { @@ -96,7 +94,7 @@ impl Mmap { ) }; if ptr as isize == -1_isize { - return Err(io::Error::last_os_error().to_string()); + bail!("mmap failed: {}", io::Error::last_os_error()); } let mut result = Self { @@ -117,13 +115,14 @@ impl Mmap { /// within a reserved mapping of `mapping_size` bytes. `accessible_size` and `mapping_size` /// must be native page-size multiples. #[cfg(target_os = "windows")] - pub fn accessible_reserved( - accessible_size: usize, - mapping_size: usize, - ) -> Result { + pub fn accessible_reserved(accessible_size: usize, mapping_size: usize) -> Result { use winapi::um::memoryapi::VirtualAlloc; use winapi::um::winnt::{MEM_COMMIT, MEM_RESERVE, PAGE_NOACCESS, PAGE_READWRITE}; + if mapping_size == 0 { + return Ok(Self::new()); + } + let page_size = region::page::size(); assert_le!(accessible_size, mapping_size); assert_eq!(mapping_size & (page_size - 1), 0); @@ -140,7 +139,7 @@ impl Mmap { ) }; if ptr.is_null() { - return Err(io::Error::last_os_error().to_string()); + bail!("VirtualAlloc failed: {}", io::Error::last_os_error()); } Self { @@ -152,7 +151,7 @@ impl Mmap { let ptr = unsafe { VirtualAlloc(ptr::null_mut(), mapping_size, MEM_RESERVE, PAGE_NOACCESS) }; if ptr.is_null() { - return Err(io::Error::last_os_error().to_string()); + bail!("VirtualAlloc failed: {}", io::Error::last_os_error()); } let mut result = Self { @@ -173,7 +172,7 @@ impl Mmap { /// `start` and `len` must be native page-size multiples and describe a range within /// `self`'s reserved memory. #[cfg(not(target_os = "windows"))] - pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<(), String> { + pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<()> { let page_size = region::page::size(); assert_eq!(start & (page_size - 1), 0); assert_eq!(len & (page_size - 1), 0); @@ -182,15 +181,18 @@ impl Mmap { // Commit the accessible size. let ptr = self.ptr as *const u8; - unsafe { region::protect(ptr.add(start), len, region::Protection::READ_WRITE) } - .map_err(|e| e.to_string()) + unsafe { + region::protect(ptr.add(start), len, region::Protection::READ_WRITE)?; + } + + Ok(()) } /// Make the memory starting at `start` and extending for `len` bytes accessible. /// `start` and `len` must be native page-size multiples and describe a range within /// `self`'s reserved memory. #[cfg(target_os = "windows")] - pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<(), String> { + pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<()> { use winapi::ctypes::c_void; use winapi::um::memoryapi::VirtualAlloc; use winapi::um::winnt::{MEM_COMMIT, PAGE_READWRITE}; @@ -212,7 +214,7 @@ impl Mmap { } .is_null() { - return Err(io::Error::last_os_error().to_string()); + bail!("VirtualAlloc failed: {}", io::Error::last_os_error()); } Ok(()) @@ -234,7 +236,7 @@ impl Mmap { } /// Return the allocated memory as a mutable pointer to u8. - pub fn as_mut_ptr(&mut self) -> *mut u8 { + pub fn as_mut_ptr(&self) -> *mut u8 { self.ptr as *mut u8 } @@ -247,6 +249,11 @@ impl Mmap { pub fn is_empty(&self) -> bool { self.len() == 0 } + + #[allow(dead_code)] + pub(crate) unsafe fn from_raw(ptr: usize, len: usize) -> Self { + Self { ptr, len } + } } impl Drop for Mmap { diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index 462480bc76..8c857add45 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -4,20 +4,17 @@ use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition}; use crate::{Trap, VMExternRef}; -use std::cell::RefCell; -use std::convert::{TryFrom, TryInto}; +use std::cell::{Cell, RefCell}; +use std::cmp::min; +use std::convert::TryInto; +use std::ops::Range; use std::ptr; use wasmtime_environ::wasm::TableElementType; -use wasmtime_environ::{ir, TablePlan, TableStyle}; - -/// A table instance. -#[derive(Debug)] -pub struct Table { - elements: RefCell, - maximum: Option, -} +use wasmtime_environ::{ir, TablePlan}; /// An element going into or coming out of a table. +/// +/// Table elements are stored as pointers and are default-initialized with `ptr::null_mut`. #[derive(Clone, Debug)] pub enum TableElement { /// A `funcref`. @@ -26,44 +23,150 @@ pub enum TableElement { ExternRef(Option), } -#[derive(Debug)] -enum TableElements { - FuncRefs(Vec<*mut VMCallerCheckedAnyfunc>), - ExternRefs(Vec>), +impl TableElement { + /// Consumes the given raw pointer into a table element. + /// + /// # Safety + /// + /// This is unsafe as it will *not* clone any externref, leaving the reference count unchanged. + /// + /// This should only be used if the raw pointer is no longer in use. + unsafe fn from_raw(ty: TableElementType, ptr: *mut u8) -> Self { + match ty { + TableElementType::Func => Self::FuncRef(ptr as _), + TableElementType::Val(_) => Self::ExternRef(if ptr.is_null() { + None + } else { + Some(VMExternRef::from_raw(ptr)) + }), + } + } + + /// Clones a table element from the underlying raw pointer. + /// + /// # Safety + /// + /// This is unsafe as it will clone any externref, incrementing the reference count. + unsafe fn clone_from_raw(ty: TableElementType, ptr: *mut u8) -> Self { + match ty { + TableElementType::Func => Self::FuncRef(ptr as _), + TableElementType::Val(_) => Self::ExternRef(if ptr.is_null() { + None + } else { + Some(VMExternRef::clone_from_raw(ptr)) + }), + } + } + + /// Consumes a table element into a raw pointer. + /// + /// # Safety + /// + /// This is unsafe as it will consume any underlying externref into a raw pointer without modifying + /// the reference count. + /// + /// Use `from_raw` to properly drop any table elements stored as raw pointers. + unsafe fn into_raw(self) -> *mut u8 { + match self { + Self::FuncRef(e) => e as _, + Self::ExternRef(e) => e.map(|e| e.into_raw()).unwrap_or(ptr::null_mut()), + } + } } +impl From<*mut VMCallerCheckedAnyfunc> for TableElement { + fn from(f: *mut VMCallerCheckedAnyfunc) -> TableElement { + TableElement::FuncRef(f) + } +} + +impl From> for TableElement { + fn from(x: Option) -> TableElement { + TableElement::ExternRef(x) + } +} + +impl From for TableElement { + fn from(x: VMExternRef) -> TableElement { + TableElement::ExternRef(Some(x)) + } +} + +#[derive(Debug)] +enum TableStorage { + Static { + data: *mut *mut u8, + size: Cell, + ty: TableElementType, + maximum: u32, + }, + Dynamic { + elements: RefCell>, + ty: TableElementType, + maximum: Option, + }, +} + +/// Represents an instance's table. +#[derive(Debug)] +pub struct Table(TableStorage); + impl Table { - /// Create a new table instance with specified minimum and maximum number of elements. - pub fn new(plan: &TablePlan) -> Self { - let min = usize::try_from(plan.table.minimum).unwrap(); - let elements = RefCell::new(match plan.table.ty { - TableElementType::Func => TableElements::FuncRefs(vec![ptr::null_mut(); min]), - TableElementType::Val(ty) => { - debug_assert_eq!(ty, crate::ref_type()); - TableElements::ExternRefs(vec![None; min]) - } - }); - match plan.style { - TableStyle::CallerChecksSignature => Self { - elements, - maximum: plan.table.maximum, - }, - } + /// Create a new dynamic (movable) table instance for the specified table plan. + pub fn new_dynamic(plan: &TablePlan) -> Self { + let elements = RefCell::new(vec![ptr::null_mut(); plan.table.minimum as usize]); + let ty = plan.table.ty.clone(); + let maximum = plan.table.maximum; + Self(TableStorage::Dynamic { + elements, + ty, + maximum, + }) + } + + /// Create a new static (immovable) table instance for the specified table plan. + pub fn new_static(plan: &TablePlan, data: *mut *mut u8, maximum: u32) -> Self { + let size = Cell::new(plan.table.minimum); + let ty = plan.table.ty.clone(); + let maximum = min(plan.table.maximum.unwrap_or(maximum), maximum); + Self(TableStorage::Static { + data, + size, + ty, + maximum, + }) } /// Returns the type of the elements in this table. pub fn element_type(&self) -> TableElementType { - match &*self.elements.borrow() { - TableElements::FuncRefs(_) => TableElementType::Func, - TableElements::ExternRefs(_) => TableElementType::Val(crate::ref_type()), + match &self.0 { + TableStorage::Static { ty, .. } => *ty, + TableStorage::Dynamic { ty, .. } => *ty, + } + } + + /// Returns whether or not the underlying storage of the table is "static". + pub(crate) fn is_static(&self) -> bool { + if let TableStorage::Static { .. } = &self.0 { + true + } else { + false } } /// Returns the number of allocated elements. pub fn size(&self) -> u32 { - match &*self.elements.borrow() { - TableElements::FuncRefs(x) => x.len().try_into().unwrap(), - TableElements::ExternRefs(x) => x.len().try_into().unwrap(), + match &self.0 { + TableStorage::Static { size, .. } => size.get(), + TableStorage::Dynamic { elements, .. } => elements.borrow().len().try_into().unwrap(), + } + } + + /// Returns the maximum number of elements. + pub fn maximum(&self) -> Option { + match &self.0 { + TableStorage::Static { maximum, .. } => Some(*maximum), + TableStorage::Dynamic { maximum, .. } => maximum.clone(), } } @@ -71,20 +174,30 @@ impl Table { /// /// Returns a trap error on out-of-bounds accesses. pub fn fill(&self, dst: u32, val: TableElement, len: u32) -> Result<(), Trap> { - let start = dst; + let start = dst as usize; let end = start - .checked_add(len) + .checked_add(len as usize) .ok_or_else(|| Trap::wasm(ir::TrapCode::TableOutOfBounds))?; - if end > self.size() { + if end > self.size() as usize { return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); } - for i in start..end { - self.set(i, val.clone()).unwrap(); - } + debug_assert!(self.type_matches(&val)); - Ok(()) + self.with_elements_mut(|elements| { + if let Some((last, elements)) = elements[start..end].split_last_mut() { + let ty = self.element_type(); + + for e in elements { + Self::set_raw(ty, e, val.clone()); + } + + Self::set_raw(self.element_type(), last, val); + } + + Ok(()) + }) } /// Grow table by the specified amount of elements. @@ -104,40 +217,43 @@ impl Table { /// Generally, prefer using `InstanceHandle::table_grow`, which encapsulates /// this unsafety. pub unsafe fn grow(&self, delta: u32, init_value: TableElement) -> Option { - let size = self.size(); + let old_size = self.size(); - let new_len = size.checked_add(delta)?; - if let Some(max) = self.maximum { - if new_len > max { + let new_size = old_size.checked_add(delta)?; + if let Some(max) = self.maximum() { + if new_size > max { return None; } } - let new_len = usize::try_from(new_len).unwrap(); - match &mut *self.elements.borrow_mut() { - TableElements::FuncRefs(x) => { - let init_value = init_value.try_into().ok()?; - x.resize(new_len, init_value) + debug_assert!(self.type_matches(&init_value)); + + // First resize the storage and then fill with the init value + match &self.0 { + TableStorage::Static { size, .. } => { + size.set(new_size); } - TableElements::ExternRefs(x) => { - let init_value = init_value.try_into().ok()?; - x.resize(new_len, init_value) + TableStorage::Dynamic { elements, .. } => { + let mut elements = elements.borrow_mut(); + elements.resize(new_size as usize, ptr::null_mut()); } } - Some(size) + self.fill(old_size, init_value, delta) + .expect("table should not be out of bounds"); + + Some(old_size) } /// Get reference to the specified element. /// /// Returns `None` if the index is out of bounds. pub fn get(&self, index: u32) -> Option { - match &*self.elements.borrow() { - TableElements::FuncRefs(x) => x.get(index as usize).cloned().map(TableElement::FuncRef), - TableElements::ExternRefs(x) => { - x.get(index as usize).cloned().map(TableElement::ExternRef) - } - } + self.with_elements(|elements| { + elements + .get(index as usize) + .map(|p| unsafe { TableElement::clone_from_raw(self.element_type(), *p) }) + }) } /// Set reference to the specified element. @@ -147,18 +263,15 @@ impl Table { /// Returns an error if `index` is out of bounds or if this table type does /// not match the element type. pub fn set(&self, index: u32, elem: TableElement) -> Result<(), ()> { - let mut elems = self.elements.borrow_mut(); - match &mut *elems { - TableElements::FuncRefs(x) => { - let slot = x.get_mut(index as usize).ok_or(())?; - *slot = elem.try_into().or(Err(()))?; - } - TableElements::ExternRefs(x) => { - let slot = x.get_mut(index as usize).ok_or(())?; - *slot = elem.try_into().or(Err(()))?; - } + if !self.type_matches(&elem) { + return Err(()); } - Ok(()) + + self.with_elements_mut(|elements| { + let e = elements.get_mut(index as usize).ok_or(())?; + Self::set_raw(self.element_type(), e, elem); + Ok(()) + }) } /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`. @@ -186,21 +299,19 @@ impl Table { return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds)); } - let srcs = src_index..src_index + len; - let dsts = dst_index..dst_index + len; + debug_assert!( + dst_table.element_type() == src_table.element_type(), + "table element type mismatch" + ); - // Note on the unwraps: the bounds check above means that these will - // never panic. - // - // TODO(#983): investigate replacing this get/set loop with a `memcpy`. - if dst_index <= src_index { - for (s, d) in (srcs).zip(dsts) { - dst_table.set(d, src_table.get(s).unwrap()).unwrap(); - } + let src_range = src_index as usize..src_index as usize + len as usize; + let dst_range = dst_index as usize..dst_index as usize + len as usize; + + // Check if the tables are the same as we cannot mutably borrow and also borrow the same `RefCell` + if ptr::eq(dst_table, src_table) { + Self::copy_elements_within(dst_table, dst_range, src_range); } else { - for (s, d) in srcs.rev().zip(dsts.rev()) { - dst_table.set(d, src_table.get(s).unwrap()).unwrap(); - } + Self::copy_elements(dst_table, src_table, dst_range, src_range); } Ok(()) @@ -208,55 +319,155 @@ impl Table { /// Return a `VMTableDefinition` for exposing the table to compiled wasm code. pub fn vmtable(&self) -> VMTableDefinition { - match &*self.elements.borrow() { - TableElements::FuncRefs(x) => VMTableDefinition { - base: x.as_ptr() as *const u8 as *mut u8, - current_elements: x.len().try_into().unwrap(), + match &self.0 { + TableStorage::Static { data, size, .. } => VMTableDefinition { + base: *data as _, + current_elements: size.get(), }, - TableElements::ExternRefs(x) => VMTableDefinition { - base: x.as_ptr() as *const u8 as *mut u8, - current_elements: x.len().try_into().unwrap(), + TableStorage::Dynamic { elements, .. } => { + let elements = elements.borrow(); + VMTableDefinition { + base: elements.as_ptr() as _, + current_elements: elements.len().try_into().unwrap(), + } + } + } + } + + fn type_matches(&self, val: &TableElement) -> bool { + match (&val, self.element_type()) { + (TableElement::FuncRef(_), TableElementType::Func) => true, + (TableElement::ExternRef(_), TableElementType::Val(_)) => true, + _ => false, + } + } + + fn with_elements(&self, f: F) -> R + where + F: FnOnce(&[*mut u8]) -> R, + { + match &self.0 { + TableStorage::Static { data, size, .. } => unsafe { + f(std::slice::from_raw_parts(*data, size.get() as usize)) }, + TableStorage::Dynamic { elements, .. } => { + let elements = elements.borrow(); + f(elements.as_slice()) + } + } + } + + fn with_elements_mut(&self, f: F) -> R + where + F: FnOnce(&mut [*mut u8]) -> R, + { + match &self.0 { + TableStorage::Static { data, size, .. } => unsafe { + f(std::slice::from_raw_parts_mut(*data, size.get() as usize)) + }, + TableStorage::Dynamic { elements, .. } => { + let mut elements = elements.borrow_mut(); + f(elements.as_mut_slice()) + } + } + } + + fn set_raw(ty: TableElementType, elem: &mut *mut u8, val: TableElement) { + unsafe { + let old = *elem; + *elem = val.into_raw(); + + // Drop the old element + let _ = TableElement::from_raw(ty, old); + } + } + + fn copy_elements( + dst_table: &Self, + src_table: &Self, + dst_range: Range, + src_range: Range, + ) { + // This can only be used when copying between different tables + debug_assert!(!ptr::eq(dst_table, src_table)); + + let ty = dst_table.element_type(); + + match ty { + TableElementType::Func => { + // `funcref` are `Copy`, so just do a mempcy + dst_table.with_elements_mut(|dst| { + src_table.with_elements(|src| dst[dst_range].copy_from_slice(&src[src_range])) + }); + } + TableElementType::Val(_) => { + // We need to clone each `externref` + dst_table.with_elements_mut(|dst| { + src_table.with_elements(|src| { + for (s, d) in src_range.zip(dst_range) { + let elem = unsafe { TableElement::clone_from_raw(ty, src[s]) }; + Self::set_raw(ty, &mut dst[d], elem); + } + }) + }); + } + } + } + + fn copy_elements_within(table: &Self, dst_range: Range, src_range: Range) { + let ty = table.element_type(); + + match ty { + TableElementType::Func => { + // `funcref` are `Copy`, so just do a memmove + table.with_elements_mut(|dst| dst.copy_within(src_range, dst_range.start)); + } + TableElementType::Val(_) => { + // We need to clone each `externref` while handling overlapping ranges + table.with_elements_mut(|dst| { + if dst_range.start <= src_range.start { + for (s, d) in src_range.zip(dst_range) { + let elem = unsafe { TableElement::clone_from_raw(ty, dst[s]) }; + Self::set_raw(ty, &mut dst[d], elem); + } + } else { + for (s, d) in src_range.rev().zip(dst_range.rev()) { + let elem = unsafe { TableElement::clone_from_raw(ty, dst[s]) }; + Self::set_raw(ty, &mut dst[d], elem); + } + } + }); + } } } } -impl TryFrom for *mut VMCallerCheckedAnyfunc { - type Error = TableElement; +impl Drop for Table { + fn drop(&mut self) { + let ty = self.element_type(); - fn try_from(e: TableElement) -> Result { - match e { - TableElement::FuncRef(f) => Ok(f), - _ => Err(e), + // funcref tables can skip this + if let TableElementType::Func = ty { + return; } + + // Properly drop any table elements stored in the table + self.with_elements(|elements| { + for element in elements.iter() { + let _ = unsafe { TableElement::from_raw(ty, *element) }; + } + }); } } -impl TryFrom for Option { - type Error = TableElement; - - fn try_from(e: TableElement) -> Result { - match e { - TableElement::ExternRef(x) => Ok(x), - _ => Err(e), - } - } -} - -impl From<*mut VMCallerCheckedAnyfunc> for TableElement { - fn from(f: *mut VMCallerCheckedAnyfunc) -> TableElement { - TableElement::FuncRef(f) - } -} - -impl From> for TableElement { - fn from(x: Option) -> TableElement { - TableElement::ExternRef(x) - } -} - -impl From for TableElement { - fn from(x: VMExternRef) -> TableElement { - TableElement::ExternRef(Some(x)) +// The default table representation is an empty funcref table that cannot grow. +impl Default for Table { + fn default() -> Self { + Self(TableStorage::Static { + data: std::ptr::null_mut(), + size: Cell::new(0), + ty: TableElementType::Func, + maximum: 0, + }) } } diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index c20a42b5ed..f4dffeee2c 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -750,7 +750,7 @@ impl VMContext { } } -/// +/// Trampoline function pointer type. pub type VMTrampoline = unsafe extern "C" fn( *mut VMContext, // callee vmctx *mut VMContext, // caller vmctx diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 752b3c7ff8..466e3cb61d 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -73,3 +73,6 @@ experimental_x64 = ["wasmtime-jit/experimental_x64"] # Enables support for "async stores" as well as defining host functions as # `async fn` and calling functions asynchronously. async = ["wasmtime-fiber"] + +# Enables userfaultfd support in the runtime's pooling allocator when building on Linux +uffd = ["wasmtime-runtime/uffd"] diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 6ddaa022c4..f3135dd6c8 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -14,6 +14,257 @@ use wasmtime_environ::settings::{self, Configurable, SetError}; use wasmtime_environ::{isa, isa::TargetIsa, Tunables}; use wasmtime_jit::{native, CompilationStrategy, Compiler}; use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent, VTuneAgent}; +use wasmtime_runtime::{InstanceAllocator, OnDemandInstanceAllocator, PoolingInstanceAllocator}; + +/// Represents the limits placed on a module for compiling with the pooling instance allocation strategy. +#[derive(Debug, Copy, Clone)] +pub struct ModuleLimits { + /// The maximum number of imported functions for a module (default is 1000). + pub imported_functions: u32, + + /// The maximum number of imported tables for a module (default is 0). + pub imported_tables: u32, + + /// The maximum number of imported linear memories for a module (default is 0). + pub imported_memories: u32, + + /// The maximum number of imported globals for a module (default is 0). + pub imported_globals: u32, + + /// The maximum number of defined types for a module (default is 100). + pub types: u32, + + /// The maximum number of defined functions for a module (default is 10000). + pub functions: u32, + + /// The maximum number of defined tables for a module (default is 1). + pub tables: u32, + + /// The maximum number of defined linear memories for a module (default is 1). + pub memories: u32, + + /// The maximum number of defined globals for a module (default is 10). + pub globals: u32, + + /// The maximum table elements for any table defined in a module (default is 10000). + /// + /// If a table's minimum element limit is greater than this value, the module will + /// fail to compile. + /// + /// If a table's maximum element limit is unbounded or greater than this value, + /// the maximum will be `table_elements` for the purpose of any `table.grow` instruction. + pub table_elements: u32, + + /// The maximum number of pages for any linear memory defined in a module (default is 160). + /// + /// The default of 160 means at most 10 MiB of host memory may be committed for each instance. + /// + /// If a memory's minimum page limit is greater than this value, the module will + /// fail to compile. + /// + /// If a memory's maximum page limit is unbounded or greater than this value, + /// the maximum will be `memory_pages` for the purpose of any `memory.grow` instruction. + /// + /// This value cannot exceed any memory reservation size limits placed on instances. + pub memory_pages: u32, +} + +impl Default for ModuleLimits { + fn default() -> Self { + // Use the defaults from the runtime + let wasmtime_runtime::ModuleLimits { + imported_functions, + imported_tables, + imported_memories, + imported_globals, + types, + functions, + tables, + memories, + globals, + table_elements, + memory_pages, + } = wasmtime_runtime::ModuleLimits::default(); + + Self { + imported_functions, + imported_tables, + imported_memories, + imported_globals, + types, + functions, + tables, + memories, + globals, + table_elements, + memory_pages, + } + } +} + +// This exists so we can convert between the public Wasmtime API and the runtime representation +// without having to export runtime types from the Wasmtime API. +#[doc(hidden)] +impl Into for ModuleLimits { + fn into(self) -> wasmtime_runtime::ModuleLimits { + let Self { + imported_functions, + imported_tables, + imported_memories, + imported_globals, + types, + functions, + tables, + memories, + globals, + table_elements, + memory_pages, + } = self; + + wasmtime_runtime::ModuleLimits { + imported_functions, + imported_tables, + imported_memories, + imported_globals, + types, + functions, + tables, + memories, + globals, + table_elements, + memory_pages, + } + } +} + +/// Represents the limits placed on instances by the pooling instance allocation strategy. +#[derive(Debug, Copy, Clone)] +pub struct InstanceLimits { + /// The maximum number of concurrent instances supported (default is 1000). + pub count: u32, + + /// The maximum size, in bytes, of host address space to reserve for each linear memory of an instance. + /// + /// Note: this value has important performance ramifications. + /// + /// On 64-bit platforms, the default for this value will be 6 GiB. A value of less than 4 GiB will + /// force runtime bounds checking for memory accesses and thus will negatively impact performance. + /// Any value above 4 GiB will start eliding bounds checks provided the `offset` of the memory access is + /// less than (`memory_reservation_size` - 4 GiB). A value of 8 GiB will completely elide *all* bounds + /// checks; consequently, 8 GiB will be the maximum supported value. The default of 6 GiB reserves + /// less host address space for each instance, but a memory access with an offset above 2 GiB will incur + /// runtime bounds checks. + /// + /// On 32-bit platforms, the default for this value will be 10 MiB. A 32-bit host has very limited address + /// space to reserve for a lot of concurrent instances. As a result, runtime bounds checking will be used + /// for all memory accesses. For better runtime performance, a 64-bit host is recommended. + /// + /// This value will be rounded up by the WebAssembly page size (64 KiB). + pub memory_reservation_size: u64, +} + +impl Default for InstanceLimits { + fn default() -> Self { + let wasmtime_runtime::InstanceLimits { + count, + memory_reservation_size, + } = wasmtime_runtime::InstanceLimits::default(); + + Self { + count, + memory_reservation_size, + } + } +} + +// This exists so we can convert between the public Wasmtime API and the runtime representation +// without having to export runtime types from the Wasmtime API. +#[doc(hidden)] +impl Into for InstanceLimits { + fn into(self) -> wasmtime_runtime::InstanceLimits { + let Self { + count, + memory_reservation_size, + } = self; + + wasmtime_runtime::InstanceLimits { + count, + memory_reservation_size, + } + } +} + +/// The allocation strategy to use for the pooling instance allocation strategy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolingAllocationStrategy { + /// Allocate from the next available instance. + NextAvailable, + /// Allocate from a random available instance. + Random, +} + +impl Default for PoolingAllocationStrategy { + fn default() -> Self { + match wasmtime_runtime::PoolingAllocationStrategy::default() { + wasmtime_runtime::PoolingAllocationStrategy::NextAvailable => Self::NextAvailable, + wasmtime_runtime::PoolingAllocationStrategy::Random => Self::Random, + } + } +} + +// This exists so we can convert between the public Wasmtime API and the runtime representation +// without having to export runtime types from the Wasmtime API. +#[doc(hidden)] +impl Into for PoolingAllocationStrategy { + fn into(self) -> wasmtime_runtime::PoolingAllocationStrategy { + match self { + Self::NextAvailable => wasmtime_runtime::PoolingAllocationStrategy::NextAvailable, + Self::Random => wasmtime_runtime::PoolingAllocationStrategy::Random, + } + } +} + +/// Represents the module instance allocation strategy to use. +#[derive(Clone)] +pub enum InstanceAllocationStrategy { + /// The on-demand instance allocation strategy. + /// + /// Resources related to a module instance are allocated at instantiation time and + /// immediately deallocated when the `Store` referencing the instance is dropped. + /// + /// This is the default allocation strategy for Wasmtime. + OnDemand, + /// The pooling instance allocation strategy. + /// + /// A pool of resources is created in advance and module instantiation reuses resources + /// from the pool. Resources are returned to the pool when the `Store` referencing the instance + /// is dropped. + Pooling { + /// The allocation strategy to use. + strategy: PoolingAllocationStrategy, + /// The module limits to use. + module_limits: ModuleLimits, + /// The instance limits to use. + instance_limits: InstanceLimits, + }, +} + +impl InstanceAllocationStrategy { + /// The default pooling instance allocation strategy. + pub fn pooling() -> Self { + Self::Pooling { + strategy: PoolingAllocationStrategy::default(), + module_limits: ModuleLimits::default(), + instance_limits: InstanceLimits::default(), + } + } +} + +impl Default for InstanceAllocationStrategy { + fn default() -> Self { + Self::OnDemand + } +} /// Global configuration options used to create an [`Engine`](crate::Engine) /// and customize its behavior. @@ -29,13 +280,18 @@ pub struct Config { #[cfg(feature = "cache")] pub(crate) cache_config: CacheConfig, pub(crate) profiler: Arc, - pub(crate) memory_creator: Option, + pub(crate) instance_allocator: Option>, + // The default instance allocator is used for instantiating host objects + // and for module instantiation when `instance_allocator` is None + pub(crate) default_instance_allocator: OnDemandInstanceAllocator, pub(crate) max_wasm_stack: usize, pub(crate) features: WasmFeatures, pub(crate) wasm_backtrace_details_env_used: bool, pub(crate) max_instances: usize, pub(crate) max_tables: usize, pub(crate) max_memories: usize, + #[cfg(feature = "async")] + pub(crate) async_stack_size: usize, } impl Config { @@ -73,7 +329,8 @@ impl Config { #[cfg(feature = "cache")] cache_config: CacheConfig::new_cache_disabled(), profiler: Arc::new(NullProfilerAgent), - memory_creator: None, + instance_allocator: None, + default_instance_allocator: OnDemandInstanceAllocator::new(None), max_wasm_stack: 1 << 20, wasm_backtrace_details_env_used: false, features: WasmFeatures { @@ -85,6 +342,8 @@ impl Config { max_instances: 10_000, max_tables: 10_000, max_memories: 10_000, + #[cfg(feature = "async")] + async_stack_size: 2 << 20, }; ret.wasm_backtrace_details(WasmBacktraceDetails::Environment); return ret; @@ -159,23 +418,75 @@ impl Config { self } - /// Configures the maximum amount of native stack space available to + /// Configures the maximum amount of stack space available for /// executing WebAssembly code. /// - /// WebAssembly code currently executes on the native call stack for its own - /// call frames. WebAssembly, however, also has well-defined semantics on - /// stack overflow. This is intended to be a knob which can help configure - /// how much native stack space a wasm module is allowed to consume. Note - /// that the number here is not super-precise, but rather wasm will take at - /// most "pretty close to this much" stack space. + /// WebAssembly has well-defined semantics on stack overflow. This is + /// intended to be a knob which can help configure how much stack space + /// wasm execution is allowed to consume. Note that the number here is not + /// super-precise, but rather wasm will take at most "pretty close to this + /// much" stack space. /// /// If a wasm call (or series of nested wasm calls) take more stack space /// than the `size` specified then a stack overflow trap will be raised. /// - /// By default this option is 1 MB. - pub fn max_wasm_stack(&mut self, size: usize) -> &mut Self { + /// When the `async` feature is enabled, this value cannot exceed the + /// `async_stack_size` option. Be careful not to set this value too close + /// to `async_stack_size` as doing so may limit how much stack space + /// is available for host functions. Unlike wasm functions that trap + /// on stack overflow, a host function that overflows the stack will + /// abort the process. + /// + /// `max_wasm_stack` must be set prior to setting an instance allocation + /// strategy. + /// + /// By default this option is 1 MiB. + pub fn max_wasm_stack(&mut self, size: usize) -> Result<&mut Self> { + #[cfg(feature = "async")] + if size > self.async_stack_size { + bail!("wasm stack size cannot exceed the async stack size"); + } + + if size == 0 { + bail!("wasm stack size cannot be zero"); + } + + if self.instance_allocator.is_some() { + bail!( + "wasm stack size cannot be modified after setting an instance allocation strategy" + ); + } + self.max_wasm_stack = size; - self + Ok(self) + } + + /// Configures the size of the stacks used for asynchronous execution. + /// + /// This setting configures the size of the stacks that are allocated for + /// asynchronous execution. The value cannot be less than `max_wasm_stack`. + /// + /// The amount of stack space guaranteed for host functions is + /// `async_stack_size - max_wasm_stack`, so take care not to set these two values + /// close to one another; doing so may cause host functions to overflow the + /// stack and abort the process. + /// + /// `async_stack_size` must be set prior to setting an instance allocation + /// strategy. + /// + /// By default this option is 2 MiB. + #[cfg(feature = "async")] + pub fn async_stack_size(&mut self, size: usize) -> Result<&mut Self> { + if size < self.max_wasm_stack { + bail!("async stack size cannot be less than the maximum wasm stack size"); + } + if self.instance_allocator.is_some() { + bail!( + "async stack size cannot be modified after setting an instance allocation strategy" + ); + } + self.async_stack_size = size; + Ok(self) } /// Configures whether the WebAssembly threads proposal will be enabled for @@ -504,12 +815,51 @@ impl Config { Ok(self) } - /// Sets a custom memory creator + /// Sets a custom memory creator. + /// + /// Custom memory creators are used when creating host `Memory` objects or when + /// creating instance linear memories for the on-demand instance allocation strategy. pub fn with_host_memory(&mut self, mem_creator: Arc) -> &mut Self { - self.memory_creator = Some(MemoryCreatorProxy { mem_creator }); + self.default_instance_allocator = + OnDemandInstanceAllocator::new(Some(Arc::new(MemoryCreatorProxy(mem_creator)))); self } + /// Sets the instance allocation strategy to use. + /// + /// When using the pooling instance allocation strategy, all linear memories will be created as "static". + /// + /// This means the [`Config::static_memory_maximum_size`] and [`Config::static_memory_guard_size`] options + /// will be ignored in favor of [`InstanceLimits::memory_reservation_size`] when the pooling instance + /// allocation strategy is used. + pub fn with_allocation_strategy( + &mut self, + strategy: InstanceAllocationStrategy, + ) -> Result<&mut Self> { + self.instance_allocator = match strategy { + InstanceAllocationStrategy::OnDemand => None, + InstanceAllocationStrategy::Pooling { + strategy, + module_limits, + instance_limits, + } => { + #[cfg(feature = "async")] + let stack_size = self.async_stack_size; + + #[cfg(not(feature = "async"))] + let stack_size = 0; + + Some(Arc::new(PoolingInstanceAllocator::new( + strategy.into(), + module_limits.into(), + instance_limits.into(), + stack_size, + )?)) + } + }; + Ok(self) + } + /// Configures the maximum size, in bytes, where a linear memory is /// considered static, above which it'll be considered dynamic. /// @@ -726,7 +1076,15 @@ impl Config { pub(crate) fn build_compiler(&self) -> Compiler { let isa = self.target_isa(); - Compiler::new(isa, self.strategy, self.tunables.clone(), self.features) + let mut tunables = self.tunables.clone(); + self.instance_allocator().adjust_tunables(&mut tunables); + Compiler::new(isa, self.strategy, tunables, self.features) + } + + pub(crate) fn instance_allocator(&self) -> &dyn InstanceAllocator { + self.instance_allocator + .as_deref() + .unwrap_or(&self.default_instance_allocator) } } diff --git a/crates/wasmtime/src/externals.rs b/crates/wasmtime/src/externals.rs index e71fb7d75c..665acf9f02 100644 --- a/crates/wasmtime/src/externals.rs +++ b/crates/wasmtime/src/externals.rs @@ -547,10 +547,13 @@ impl Table { bail!("cross-`Store` table copies are not supported"); } + if dst_table.ty() != src_table.ty() { + bail!("tables do not have the same element type"); + } + // NB: We must use the `dst_table`'s `wasmtime_handle` for the // `dst_table_index` and vice versa for `src_table` since each table can // come from different modules. - let dst_table_index = dst_table.wasmtime_table_index(); let dst_table_index = dst_table.instance.get_defined_table(dst_table_index); @@ -579,6 +582,11 @@ impl Table { bail!("cross-`Store` table fills are not supported"); } + // Ensure the fill value is the correct type + if self.ty().element() != &val.ty() { + bail!("mismatched element fill type"); + } + let table_index = self.wasmtime_table_index(); self.instance .handle diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index 2ddaee2a68..9e2992c9b8 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -12,9 +12,9 @@ use wasmtime_environ::wasm::{ }; use wasmtime_environ::Initializer; use wasmtime_runtime::{ - Imports, InstantiationError, RuntimeInstance, StackMapRegistry, VMContext, - VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, VMMemoryImport, - VMTableImport, + Imports, InstanceAllocationRequest, InstantiationError, RuntimeInstance, StackMapRegistry, + VMContext, VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMGlobalImport, + VMMemoryImport, VMTableImport, }; /// An instantiated WebAssembly module. @@ -492,18 +492,26 @@ impl<'a> Instantiator<'a> { // compiled JIT code within the `Store`. self.store.register_module(&self.cur.module); - let config = self.store.engine().config(); unsafe { - let instance = compiled_module.instantiate( - self.cur.build(), - &self.store.lookup_shared_signature(self.cur.module.types()), - config.memory_creator.as_ref().map(|a| a as _), - self.store.interrupts(), - Box::new(()), - self.store.externref_activations_table() as *const VMExternRefActivationsTable + let config = self.store.engine().config(); + + let allocator = config.instance_allocator(); + + let instance = allocator.allocate(InstanceAllocationRequest { + module: compiled_module.module().clone(), + finished_functions: compiled_module.finished_functions(), + imports: self.cur.build(), + lookup_shared_signature: &self + .store + .lookup_shared_signature(self.cur.module.types()), + host_state: Box::new(()), + interrupts: self.store.interrupts(), + externref_activations_table: self.store.externref_activations_table() + as *const VMExternRefActivationsTable as *mut _, - self.store.stack_map_registry() as *const StackMapRegistry as *mut _, - )?; + stack_map_registry: self.store.stack_map_registry() as *const StackMapRegistry + as *mut _, + })?; // After we've created the `InstanceHandle` we still need to run // initialization to set up data/elements/etc. We do this after adding @@ -512,12 +520,9 @@ impl<'a> Instantiator<'a> { // initializers may have run which placed elements into other instance's // tables. This means that from this point on, regardless of whether // initialization is successful, we need to keep the instance alive. - let instance = self.store.add_instance(instance); - instance - .initialize( - config.features.bulk_memory, - &compiled_module.data_initializers(), - ) + let instance = self.store.add_instance(instance, false); + allocator + .initialize(&instance.handle, config.features.bulk_memory) .map_err(|e| -> Error { match e { InstantiationError::Trap(trap) => { diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index 12dd72b456..f0116362c9 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -172,6 +172,12 @@ //! * `vtune` - Not enabled by default, this feature compiles in support for //! supporting VTune profiling of JIT code. //! +//! * `uffd` - Not enabled by default. This feature enables `userfaultfd` support +//! when using the pooling instance allocator. As handling page faults in user space +//! comes with a performance penalty, this feature should only be enabled when kernel +//! lock contention is hampering multithreading throughput. This feature is only +//! supported on Linux and requires a Linux kernel version 4.11 or higher. +//! //! ## Examples //! //! In addition to the examples below be sure to check out the [online embedding diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 0bcf8f5686..e3c9213adc 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -307,15 +307,22 @@ impl Module { /// # } /// ``` pub fn from_binary(engine: &Engine, binary: &[u8]) -> Result { - #[cfg(feature = "cache")] - let (main_module, artifacts, types) = - ModuleCacheEntry::new("wasmtime", engine.cache_config()) + const USE_PAGED_MEM_INIT: bool = cfg!(all(feature = "uffd", target_os = "linux")); + + cfg_if::cfg_if! { + if #[cfg(feature = "cache")] { + let (main_module, artifacts, types) = ModuleCacheEntry::new( + "wasmtime", + engine.cache_config(), + ) .get_data((engine.compiler(), binary), |(compiler, binary)| { - CompilationArtifacts::build(compiler, binary) + CompilationArtifacts::build(compiler, binary, USE_PAGED_MEM_INIT) })?; - #[cfg(not(feature = "cache"))] - let (main_module, artifacts, types) = - CompilationArtifacts::build(engine.compiler(), binary)?; + } else { + let (main_module, artifacts, types) = + CompilationArtifacts::build(engine.compiler(), binary, USE_PAGED_MEM_INIT)?; + } + }; let mut modules = CompiledModule::from_artifacts_list( artifacts, @@ -324,6 +331,12 @@ impl Module { )?; let module = modules.remove(main_module); + // Validate the module can be used with the current allocator + engine + .config() + .instance_allocator() + .validate(module.module())?; + Ok(Module { inner: Arc::new(ModuleInner { engine: engine.clone(), diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 1b771417be..53a7bf7ec7 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -18,10 +18,19 @@ use std::task::{Context, Poll}; use wasmtime_environ::wasm; use wasmtime_jit::{CompiledModule, ModuleCode, TypeTables}; use wasmtime_runtime::{ - InstanceHandle, RuntimeMemoryCreator, SignalHandler, StackMapRegistry, TrapInfo, VMContext, + InstanceAllocator, InstanceHandle, SignalHandler, StackMapRegistry, TrapInfo, VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, }; +/// Used to associate instances with the store. +/// +/// This is needed to track if the instance was allocated expliclty with the default +/// instance allocator. +struct StoreInstance { + handle: InstanceHandle, + use_default_allocator: bool, +} + /// A `Store` is a collection of WebAssembly instances and host-defined items. /// /// All WebAssembly instances and items will be attached to and refer to a @@ -63,7 +72,7 @@ pub(crate) struct StoreInner { engine: Engine, interrupts: Arc, signatures: RefCell, - instances: RefCell>, + instances: RefCell>, signal_handler: RefCell>>>, externref_activations_table: VMExternRefActivationsTable, stack_map_registry: StackMapRegistry, @@ -254,15 +263,6 @@ impl Store { &self.inner.engine } - /// Returns an optional reference to a ['RuntimeMemoryCreator'] - pub(crate) fn memory_creator(&self) -> Option<&dyn RuntimeMemoryCreator> { - self.engine() - .config() - .memory_creator - .as_ref() - .map(|x| x as _) - } - pub(crate) fn signatures(&self) -> &RefCell { &self.inner.signatures } @@ -383,8 +383,15 @@ impl Store { Ok(()) } - pub(crate) unsafe fn add_instance(&self, handle: InstanceHandle) -> StoreInstanceHandle { - self.inner.instances.borrow_mut().push(handle.clone()); + pub(crate) unsafe fn add_instance( + &self, + handle: InstanceHandle, + use_default_allocator: bool, + ) -> StoreInstanceHandle { + self.inner.instances.borrow_mut().push(StoreInstance { + handle: handle.clone(), + use_default_allocator, + }); StoreInstanceHandle { store: self.clone(), handle, @@ -397,7 +404,7 @@ impl Store { .instances .borrow() .iter() - .any(|i| i.vmctx_ptr() == handle.vmctx_ptr())); + .any(|i| i.handle.vmctx_ptr() == handle.vmctx_ptr())); StoreInstanceHandle { store: self.clone(), handle, @@ -752,12 +759,14 @@ impl Store { /// that the various comments are illuminating as to what's going on here. #[cfg(feature = "async")] pub(crate) async fn on_fiber(&self, func: impl FnOnce() -> R) -> Result { - debug_assert!(self.is_async()); + let config = self.inner.engine.config(); - // TODO: allocation of a fiber should be much more abstract where we - // shouldn't be allocating huge stacks on every async wasm function call. + debug_assert!(self.is_async()); + debug_assert!(config.async_stack_size > 0); + + type SuspendType = wasmtime_fiber::Suspend, (), Result<(), Trap>>; let mut slot = None; - let fiber = wasmtime_fiber::Fiber::new(10 * 1024 * 1024, |keep_going, suspend| { + let func = |keep_going, suspend: &SuspendType| { // First check and see if we were interrupted/dropped, and only // continue if we haven't been. keep_going?; @@ -775,18 +784,46 @@ impl Store { slot = Some(func()); Ok(()) - }) - .map_err(|e| Trap::from(anyhow::Error::from(e)))?; + }; + + let (fiber, stack) = match config.instance_allocator().allocate_fiber_stack() { + Ok(stack) => { + // Use the returned stack and deallocate it when finished + ( + unsafe { + wasmtime_fiber::Fiber::new_with_stack(stack, func) + .map_err(|e| Trap::from(anyhow::Error::from(e)))? + }, + stack, + ) + } + Err(wasmtime_runtime::FiberStackError::NotSupported) => { + // The allocator doesn't support custom fiber stacks for the current platform + // Request that the fiber itself allocate the stack + ( + wasmtime_fiber::Fiber::new(config.async_stack_size, func) + .map_err(|e| Trap::from(anyhow::Error::from(e)))?, + std::ptr::null_mut(), + ) + } + Err(e) => return Err(Trap::from(anyhow::Error::from(e))), + }; // Once we have the fiber representing our synchronous computation, we // wrap that in a custom future implementation which does the // translation from the future protocol to our fiber API. - FiberFuture { fiber, store: self }.await?; + FiberFuture { + fiber, + store: self, + stack, + } + .await?; return Ok(slot.unwrap()); struct FiberFuture<'a> { fiber: wasmtime_fiber::Fiber<'a, Result<(), Trap>, (), Result<(), Trap>>, store: &'a Store, + stack: *mut u8, } impl Future for FiberFuture<'_> { @@ -843,15 +880,23 @@ impl Store { // completion. impl Drop for FiberFuture<'_> { fn drop(&mut self) { - if self.fiber.done() { - return; + if !self.fiber.done() { + let result = self.fiber.resume(Err(Trap::new("future dropped"))); + // This resumption with an error should always complete the + // fiber. While it's technically possible for host code to catch + // the trap and re-resume, we'd ideally like to signal that to + // callers that they shouldn't be doing that. + debug_assert!(result.is_ok()); + } + if !self.stack.is_null() { + unsafe { + self.store + .engine() + .config() + .instance_allocator() + .deallocate_fiber_stack(self.stack) + }; } - let result = self.fiber.resume(Err(Trap::new("future dropped"))); - // This resumption with an error should always complete the - // fiber. While it's technically possible for host code to catch - // the trap and re-resume, we'd ideally like to signal that to - // callers that they shouldn't be doing that. - debug_assert!(result.is_ok()); } } } @@ -974,9 +1019,17 @@ impl fmt::Debug for Store { impl Drop for StoreInner { fn drop(&mut self) { - for instance in self.instances.get_mut().iter() { + let allocator = self.engine.config().instance_allocator(); + for instance in self.instances.borrow().iter() { unsafe { - instance.dealloc(); + if instance.use_default_allocator { + self.engine + .config() + .default_instance_allocator + .deallocate(&instance.handle); + } else { + allocator.deallocate(&instance.handle); + } } } } diff --git a/crates/wasmtime/src/trampoline/create_handle.rs b/crates/wasmtime/src/trampoline/create_handle.rs index f597987e1b..4ecf261b1e 100644 --- a/crates/wasmtime/src/trampoline/create_handle.rs +++ b/crates/wasmtime/src/trampoline/create_handle.rs @@ -9,15 +9,15 @@ use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::DefinedFuncIndex; use wasmtime_environ::Module; use wasmtime_runtime::{ - Imports, InstanceHandle, StackMapRegistry, VMExternRefActivationsTable, VMFunctionBody, - VMFunctionImport, VMSharedSignatureIndex, + Imports, InstanceAllocationRequest, InstanceAllocator, StackMapRegistry, + VMExternRefActivationsTable, VMFunctionBody, VMFunctionImport, VMSharedSignatureIndex, }; pub(crate) fn create_handle( module: Module, store: &Store, finished_functions: PrimaryMap, - state: Box, + host_state: Box, func_imports: &[VMFunctionImport], shared_signature_id: Option, ) -> Result { @@ -26,17 +26,26 @@ pub(crate) fn create_handle( let module = Arc::new(module); unsafe { - let handle = InstanceHandle::new( - module, - &finished_functions, - imports, - store.memory_creator(), - &|_| shared_signature_id.unwrap(), - state, - store.interrupts(), - store.externref_activations_table() as *const VMExternRefActivationsTable as *mut _, - store.stack_map_registry() as *const StackMapRegistry as *mut _, - )?; - Ok(store.add_instance(handle)) + // Use the default allocator when creating handles associated with host objects + // The configured instance allocator should only be used when creating module instances + // as we don't want host objects to count towards instance limits. + let handle = store + .engine() + .config() + .default_instance_allocator + .allocate(InstanceAllocationRequest { + module: module.clone(), + finished_functions: &finished_functions, + imports, + lookup_shared_signature: &|_| shared_signature_id.unwrap(), + host_state, + interrupts: store.interrupts(), + externref_activations_table: store.externref_activations_table() + as *const VMExternRefActivationsTable + as *mut _, + stack_map_registry: store.stack_map_registry() as *const StackMapRegistry as *mut _, + })?; + + Ok(store.add_instance(handle, true)) } } diff --git a/crates/wasmtime/src/trampoline/memory.rs b/crates/wasmtime/src/trampoline/memory.rs index 0bfb2bfff9..37c14c51f6 100644 --- a/crates/wasmtime/src/trampoline/memory.rs +++ b/crates/wasmtime/src/trampoline/memory.rs @@ -3,7 +3,7 @@ use crate::memory::{LinearMemory, MemoryCreator}; use crate::trampoline::StoreInstanceHandle; use crate::Store; use crate::{Limits, MemoryType}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::{wasm, MemoryPlan, MemoryStyle, Module, WASM_PAGE_SIZE}; use wasmtime_runtime::{RuntimeLinearMemory, RuntimeMemoryCreator, VMMemoryDefinition}; @@ -54,19 +54,18 @@ impl RuntimeLinearMemory for LinearMemoryProxy { } #[derive(Clone)] -pub(crate) struct MemoryCreatorProxy { - pub(crate) mem_creator: Arc, -} +pub(crate) struct MemoryCreatorProxy(pub Arc); impl RuntimeMemoryCreator for MemoryCreatorProxy { - fn new_memory(&self, plan: &MemoryPlan) -> Result, String> { + fn new_memory(&self, plan: &MemoryPlan) -> Result> { let ty = MemoryType::new(Limits::new(plan.memory.minimum, plan.memory.maximum)); let reserved_size_in_bytes = match plan.style { MemoryStyle::Static { bound } => Some(bound as u64 * WASM_PAGE_SIZE as u64), MemoryStyle::Dynamic => None, }; - self.mem_creator + self.0 .new_memory(ty, reserved_size_in_bytes, plan.offset_guard_size) .map(|mem| Box::new(LinearMemoryProxy { mem }) as Box) + .map_err(|e| anyhow!(e)) } } diff --git a/tests/all/async_functions.rs b/tests/all/async_functions.rs index 2f83b62bf0..6f90657bf4 100644 --- a/tests/all/async_functions.rs +++ b/tests/all/async_functions.rs @@ -364,3 +364,37 @@ fn fuel_eventually_finishes() { let instance = Instance::new_async(&store, &module, &[]); run(instance).unwrap(); } + +#[test] +fn async_with_pooling_stacks() { + let mut config = Config::new(); + config + .with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 0, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + memory_reservation_size: 1, + }, + }) + .expect("pooling allocator created"); + + let engine = Engine::new(&config); + let store = Store::new_async(&engine); + let func = Func::new_async( + &store, + FuncType::new(None, None), + (), + move |_caller, _state, _params, _results| Box::new(async { Ok(()) }), + ); + run(func.call_async(&[])).unwrap(); + run(func.call_async(&[])).unwrap(); + let future1 = func.call_async(&[]); + let future2 = func.call_async(&[]); + run(future2).unwrap(); + run(future1).unwrap(); +} diff --git a/tests/all/main.rs b/tests/all/main.rs index 526c5eb4c6..d965bbd1ac 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -18,6 +18,7 @@ mod module; mod module_linking; mod module_serialize; mod name; +mod pooling_allocator; mod stack_overflow; mod table; mod traps; diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs new file mode 100644 index 0000000000..bdad287aa9 --- /dev/null +++ b/tests/all/pooling_allocator.rs @@ -0,0 +1,436 @@ +use anyhow::Result; +use wasmtime::*; + +#[test] +fn successful_instantiation() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 10, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + memory_reservation_size: 1, + }, + })?; + + let engine = Engine::new(&config); + let module = Module::new(&engine, r#"(module (memory 1) (table 10 funcref))"#)?; + + // Module should instantiate + let store = Store::new(&engine); + Instance::new(&store, &module, &[])?; + + Ok(()) +} + +#[test] +fn memory_limit() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 3, + table_elements: 10, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + memory_reservation_size: 196608, + }, + })?; + + let engine = Engine::new(&config); + + // Module should fail to validate because the minimum is greater than the configured limit + match Module::new(&engine, r#"(module (memory 4))"#) { + Ok(_) => panic!("module compilation should fail"), + Err(e) => assert_eq!( + e.to_string(), + "memory index 0 has a minimum page size of 4 which exceeds the limit of 3" + ), + } + + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (func (export "f") (result i32) (memory.grow (i32.const 1))))"#, + )?; + + // Instantiate the module and grow the memory via the `f` function + { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let f = instance.get_func("f").unwrap().get0::().unwrap(); + + assert_eq!(f().expect("function should not trap"), 0); + assert_eq!(f().expect("function should not trap"), 1); + assert_eq!(f().expect("function should not trap"), 2); + assert_eq!(f().expect("function should not trap"), -1); + assert_eq!(f().expect("function should not trap"), -1); + } + + // Instantiate the module and grow the memory via the Wasmtime API + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + + let memory = instance.get_memory("m").unwrap(); + assert_eq!(memory.size(), 0); + assert_eq!(memory.grow(1).expect("memory should grow"), 0); + assert_eq!(memory.size(), 1); + assert_eq!(memory.grow(1).expect("memory should grow"), 1); + assert_eq!(memory.size(), 2); + assert_eq!(memory.grow(1).expect("memory should grow"), 2); + assert_eq!(memory.size(), 3); + assert!(memory.grow(1).is_err()); + + Ok(()) +} + +#[test] +fn memory_init() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 2, + table_elements: 0, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + ..Default::default() + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new( + &engine, + r#"(module (memory (export "m") 2) (data (i32.const 65530) "this data spans multiple pages") (data (i32.const 10) "hello world"))"#, + )?; + + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let memory = instance.get_memory("m").unwrap(); + + unsafe { + assert_eq!( + &memory.data_unchecked()[65530..65560], + b"this data spans multiple pages" + ); + assert_eq!(&memory.data_unchecked()[10..21], b"hello world"); + } + + Ok(()) +} + +#[test] +fn memory_guard_page_trap() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 2, + table_elements: 0, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + ..Default::default() + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new( + &engine, + r#"(module (memory (export "m") 0) (func (export "f") (param i32) local.get 0 i32.load drop))"#, + )?; + + // Instantiate the module and check for out of bounds trap + for _ in 0..10 { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let m = instance.get_memory("m").unwrap(); + let f = instance.get_func("f").unwrap().get1::().unwrap(); + + let trap = f(0).expect_err("function should trap"); + assert!(trap.to_string().contains("out of bounds")); + + let trap = f(1).expect_err("function should trap"); + assert!(trap.to_string().contains("out of bounds")); + + m.grow(1).expect("memory should grow"); + f(0).expect("function should not trap"); + + let trap = f(65536).expect_err("function should trap"); + assert!(trap.to_string().contains("out of bounds")); + + let trap = f(65537).expect_err("function should trap"); + assert!(trap.to_string().contains("out of bounds")); + + m.grow(1).expect("memory should grow"); + f(65536).expect("function should not trap"); + + m.grow(1).expect_err("memory should be at the limit"); + } + + Ok(()) +} + +#[test] +#[cfg_attr(target_arch = "aarch64", ignore)] // https://github.com/bytecodealliance/wasmtime/pull/2518#issuecomment-747280133 +fn memory_zeroed() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 0, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + memory_reservation_size: 1, + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new(&engine, r#"(module (memory (export "m") 1))"#)?; + + // Instantiate the module repeatedly after writing data to the entire memory + for _ in 0..10 { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let memory = instance.get_memory("m").unwrap(); + + assert_eq!(memory.size(), 1); + assert_eq!(memory.data_size(), 65536); + + let ptr = memory.data_ptr(); + + unsafe { + for i in 0..8192 { + assert_eq!(*ptr.cast::().offset(i), 0); + } + std::ptr::write_bytes(ptr, 0xFE, memory.data_size()); + } + } + + Ok(()) +} + +#[test] +fn table_limit() -> Result<()> { + const TABLE_ELEMENTS: u32 = 10; + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: TABLE_ELEMENTS, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + memory_reservation_size: 1, + }, + })?; + + let engine = Engine::new(&config); + + // Module should fail to validate because the minimum is greater than the configured limit + match Module::new(&engine, r#"(module (table 31 funcref))"#) { + Ok(_) => panic!("module compilation should fail"), + Err(e) => assert_eq!( + e.to_string(), + "table index 0 has a minimum element size of 31 which exceeds the limit of 10" + ), + } + + let module = Module::new( + &engine, + r#"(module (table (export "t") 0 funcref) (func (export "f") (result i32) (table.grow (ref.null func) (i32.const 1))))"#, + )?; + + // Instantiate the module and grow the table via the `f` function + { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let f = instance.get_func("f").unwrap().get0::().unwrap(); + + for i in 0..TABLE_ELEMENTS { + assert_eq!(f().expect("function should not trap"), i as i32); + } + + assert_eq!(f().expect("function should not trap"), -1); + assert_eq!(f().expect("function should not trap"), -1); + } + + // Instantiate the module and grow the table via the Wasmtime API + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + + let table = instance.get_table("t").unwrap(); + + for i in 0..TABLE_ELEMENTS { + assert_eq!(table.size(), i); + assert_eq!( + table + .grow(1, Val::FuncRef(None)) + .expect("table should grow"), + i + ); + } + + assert_eq!(table.size(), TABLE_ELEMENTS); + assert!(table.grow(1, Val::FuncRef(None)).is_err()); + + Ok(()) +} + +#[test] +fn table_init() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 0, + table_elements: 6, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + ..Default::default() + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new( + &engine, + r#"(module (table (export "t") 6 funcref) (elem (i32.const 1) 1 2 3 4) (elem (i32.const 0) 0) (func) (func (param i32)) (func (param i32 i32)) (func (param i32 i32 i32)) (func (param i32 i32 i32 i32)))"#, + )?; + + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let table = instance.get_table("t").unwrap(); + + for i in 0..5 { + let v = table.get(i).expect("table should have entry"); + let f = v + .funcref() + .expect("expected funcref") + .expect("expected non-null value"); + assert_eq!(f.ty().params().len(), i as usize); + } + + assert!( + table + .get(5) + .expect("table should have entry") + .funcref() + .expect("expected funcref") + .is_none(), + "funcref should be null" + ); + + Ok(()) +} + +#[test] +#[cfg_attr(target_arch = "aarch64", ignore)] // https://github.com/bytecodealliance/wasmtime/pull/2518#issuecomment-747280133 +fn table_zeroed() -> Result<()> { + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 10, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 1, + memory_reservation_size: 1, + }, + })?; + + let engine = Engine::new(&config); + + let module = Module::new(&engine, r#"(module (table (export "t") 10 funcref))"#)?; + + // Instantiate the module repeatedly after filling table elements + for _ in 0..10 { + let store = Store::new(&engine); + let instance = Instance::new(&store, &module, &[])?; + let table = instance.get_table("t").unwrap(); + let f = Func::wrap(&store, || {}); + + assert_eq!(table.size(), 10); + + for i in 0..10 { + match table.get(i).unwrap() { + Val::FuncRef(r) => assert!(r.is_none()), + _ => panic!("expected a funcref"), + } + table.set(i, Val::FuncRef(Some(f.clone()))).unwrap(); + } + } + + Ok(()) +} + +#[test] +fn instantiation_limit() -> Result<()> { + const INSTANCE_LIMIT: u32 = 10; + let mut config = Config::new(); + config.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + memory_pages: 1, + table_elements: 10, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: INSTANCE_LIMIT, + memory_reservation_size: 1, + }, + })?; + + let engine = Engine::new(&config); + let module = Module::new(&engine, r#"(module)"#)?; + + // Instantiate to the limit + { + let store = Store::new(&engine); + + for _ in 0..INSTANCE_LIMIT { + Instance::new(&store, &module, &[])?; + } + + match Instance::new(&store, &module, &[]) { + Ok(_) => panic!("instantiation should fail"), + Err(e) => assert_eq!( + e.to_string(), + format!( + "Limit of {} concurrent instances has been reached", + INSTANCE_LIMIT + ) + ), + } + } + + // With the above store dropped, ensure instantiations can be made + + let store = Store::new(&engine); + + for _ in 0..INSTANCE_LIMIT { + Instance::new(&store, &module, &[])?; + } + + Ok(()) +} diff --git a/tests/all/table.rs b/tests/all/table.rs index 26830c4ea9..abdddccc4a 100644 --- a/tests/all/table.rs +++ b/tests/all/table.rs @@ -11,3 +11,42 @@ fn get_none() { } assert!(table.get(1).is_none()); } + +#[test] +fn fill_wrong() { + let store = Store::default(); + let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); + let table = Table::new(&store, ty, Val::FuncRef(None)).unwrap(); + assert_eq!( + table + .fill(0, Val::ExternRef(None), 1) + .map_err(|e| e.to_string()) + .unwrap_err(), + "mismatched element fill type" + ); + + let ty = TableType::new(ValType::ExternRef, Limits::new(1, None)); + let table = Table::new(&store, ty, Val::ExternRef(None)).unwrap(); + assert_eq!( + table + .fill(0, Val::FuncRef(None), 1) + .map_err(|e| e.to_string()) + .unwrap_err(), + "mismatched element fill type" + ); +} + +#[test] +fn copy_wrong() { + let store = Store::default(); + let ty = TableType::new(ValType::FuncRef, Limits::new(1, None)); + let table1 = Table::new(&store, ty, Val::FuncRef(None)).unwrap(); + let ty = TableType::new(ValType::ExternRef, Limits::new(1, None)); + let table2 = Table::new(&store, ty, Val::ExternRef(None)).unwrap(); + assert_eq!( + Table::copy(&table1, 0, &table2, 0, 1) + .map_err(|e| e.to_string()) + .unwrap_err(), + "tables do not have the same element type" + ); +} diff --git a/tests/all/wast.rs b/tests/all/wast.rs index 362dca274b..fd8a4f3a5b 100644 --- a/tests/all/wast.rs +++ b/tests/all/wast.rs @@ -1,5 +1,8 @@ use std::path::Path; -use wasmtime::{Config, Engine, Store, Strategy}; +use wasmtime::{ + Config, Engine, InstanceAllocationStrategy, InstanceLimits, ModuleLimits, + PoolingAllocationStrategy, Store, Strategy, +}; use wasmtime_wast::WastContext; include!(concat!(env!("OUT_DIR"), "/wast_testsuite_tests.rs")); @@ -7,7 +10,7 @@ include!(concat!(env!("OUT_DIR"), "/wast_testsuite_tests.rs")); // Each of the tests included from `wast_testsuite_tests` will call this // function which actually executes the `wast` test suite given the `strategy` // to compile it. -fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> { +fn run_wast(wast: &str, strategy: Strategy, pooling: bool) -> anyhow::Result<()> { let wast = Path::new(wast); let simd = wast.iter().any(|s| s == "simd"); @@ -44,6 +47,30 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> { cfg.static_memory_maximum_size(0); } + if pooling { + // The limits here are crafted such that the wast tests should pass. + // However, these limits may become insufficient in the future as the wast tests change. + // If a wast test fails because of a limit being "exceeded" or if memory/table + // fails to grow, the values here will need to be adjusted. + cfg.with_allocation_strategy(InstanceAllocationStrategy::Pooling { + strategy: PoolingAllocationStrategy::NextAvailable, + module_limits: ModuleLimits { + imported_memories: 2, + imported_tables: 2, + imported_globals: 11, + memories: 2, + tables: 4, + globals: 11, + memory_pages: 805, + ..Default::default() + }, + instance_limits: InstanceLimits { + count: 450, + ..Default::default() + }, + })?; + } + let store = Store::new(&Engine::new(&cfg)); let mut wast_context = WastContext::new(store); wast_context.register_spectest()?;