diff --git a/Cargo.lock b/Cargo.lock index 6ee438fc75..0f897137f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3457,7 +3457,6 @@ dependencies = [ "wasmtime-cache", "wasmtime-cranelift", "wasmtime-environ", - "wasmtime-fuzzing", "wasmtime-runtime", "wasmtime-wasi", "wasmtime-wasi-crypto", @@ -3528,7 +3527,6 @@ dependencies = [ "cranelift-wasm", "libfuzzer-sys", "target-lexicon", - "wasm-smith", "wasmtime", "wasmtime-fuzzing", ] diff --git a/Cargo.toml b/Cargo.toml index 6e8d7661ab..ba9cd8dd2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,12 +44,13 @@ lazy_static = "1.4.0" rustix = "0.31.0" [dev-dependencies] +# depend again on wasmtime to activate its default features for tests +wasmtime = { path = "crates/wasmtime", version = "0.33.0" } env_logger = "0.8.1" filecheck = "0.5.0" more-asserts = "0.2.1" tempfile = "3.1.0" test-programs = { path = "crates/test-programs" } -wasmtime-fuzzing = { path = "crates/fuzzing" } wasmtime-runtime = { path = "crates/runtime" } tokio = { version = "1.8.0", features = ["rt", "time", "macros", "rt-multi-thread"] } tracing-subscriber = "0.3.1" diff --git a/crates/fuzzing/src/generators.rs b/crates/fuzzing/src/generators.rs index 9c52c30f6c..6d025f665f 100644 --- a/crates/fuzzing/src/generators.rs +++ b/crates/fuzzing/src/generators.rs @@ -9,13 +9,15 @@ //! `Arbitrary` trait for the wrapped external tool. pub mod api; - pub mod table_ops; +use crate::oracles::{StoreLimits, Timeout}; use anyhow::Result; use arbitrary::{Arbitrary, Unstructured}; use std::sync::Arc; -use wasmtime::{LinearMemory, MemoryCreator, MemoryType}; +use std::time::Duration; +use wasm_smith::SwarmConfig; +use wasmtime::{Engine, LinearMemory, MemoryCreator, MemoryType, Store}; #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, Hash)] enum OptLevel { @@ -34,20 +36,34 @@ impl OptLevel { } } -/// Implementation of generating a `wasmtime::Config` arbitrarily -#[derive(Arbitrary, Debug, Eq, Hash, PartialEq)] +/// Configuration for `wasmtime::Config` and generated modules for a session of +/// fuzzing. +/// +/// This configuration guides what modules are generated, how wasmtime +/// configuration is generated, and is typically itself generated through a call +/// to `Arbitrary` which allows for a form of "swarm testing". +#[derive(Arbitrary, Debug)] pub struct Config { + /// Configuration related to the `wasmtime::Config`. + pub wasmtime: WasmtimeConfig, + /// Configuration related to generated modules. + pub module_config: ModuleConfig, +} + +/// Configuration related to `wasmtime::Config` and the various settings which +/// can be tweaked from within. +#[derive(Arbitrary, Clone, Debug, Eq, Hash, PartialEq)] +pub struct WasmtimeConfig { opt_level: OptLevel, debug_info: bool, canonicalize_nans: bool, interruptable: bool, - #[allow(missing_docs)] - pub consume_fuel: bool, + consume_fuel: bool, memory_config: MemoryConfig, force_jump_veneers: bool, } -#[derive(Arbitrary, Debug, Eq, Hash, PartialEq)] +#[derive(Arbitrary, Clone, Debug, Eq, Hash, PartialEq)] enum MemoryConfig { /// Configuration for linear memories which correspond to normal /// configuration settings in `wasmtime` itself. This will tweak various @@ -71,21 +87,42 @@ enum MemoryConfig { impl Config { /// Converts this to a `wasmtime::Config` object pub fn to_wasmtime(&self) -> wasmtime::Config { - let mut cfg = crate::fuzz_default_config(wasmtime::Strategy::Auto).unwrap(); - cfg.debug_info(self.debug_info) - .cranelift_nan_canonicalization(self.canonicalize_nans) - .cranelift_opt_level(self.opt_level.to_wasmtime()) - .interruptable(self.interruptable) - .consume_fuel(self.consume_fuel); + crate::init_fuzzing(); - if self.force_jump_veneers { + let mut cfg = wasmtime::Config::new(); + cfg.wasm_bulk_memory(true) + .wasm_reference_types(true) + .wasm_module_linking(self.module_config.config.module_linking_enabled) + .wasm_multi_memory(self.module_config.config.max_memories > 1) + .wasm_simd(self.module_config.config.simd_enabled) + .wasm_memory64(self.module_config.config.memory64_enabled) + .cranelift_nan_canonicalization(self.wasmtime.canonicalize_nans) + .cranelift_opt_level(self.wasmtime.opt_level.to_wasmtime()) + .interruptable(self.wasmtime.interruptable) + .consume_fuel(self.wasmtime.consume_fuel); + + // If the wasm-smith-generated module use nan canonicalization then we + // don't need to enable it, but if it doesn't enable it already then we + // enable this codegen option. + cfg.cranelift_nan_canonicalization(!self.module_config.config.canonicalize_nans); + + // Enabling the verifier will at-least-double compilation time, which + // with a 20-30x slowdown in fuzzing can cause issues related to + // timeouts. If generated modules can have more than a small handful of + // functions then disable the verifier when fuzzing to try to lessen the + // impact of timeouts. + if self.module_config.config.max_funcs > 10 { + cfg.cranelift_debug_verifier(false); + } + + if self.wasmtime.force_jump_veneers { unsafe { cfg.cranelift_flag_set("wasmtime_linkopt_force_jump_veneer", "true") .unwrap(); } } - match &self.memory_config { + match &self.wasmtime.memory_config { MemoryConfig::Normal { static_memory_maximum_size, static_memory_guard_size, @@ -107,6 +144,30 @@ impl Config { } return cfg; } + + /// Convenience function for generating a `Store` using this + /// configuration. + pub fn to_store(&self) -> Store { + let engine = Engine::new(&self.to_wasmtime()).unwrap(); + let mut store = Store::new(&engine, StoreLimits::new()); + store.limiter(|s| s as &mut dyn wasmtime::ResourceLimiter); + if self.wasmtime.consume_fuel { + store.add_fuel(u64::max_value()).unwrap(); + } + return store; + } + + /// Generates an arbitrary method of timing out an instance, ensuring that + /// this configuration supports the returned timeout. + pub fn generate_timeout(&mut self, u: &mut Unstructured<'_>) -> arbitrary::Result { + if u.arbitrary()? { + self.wasmtime.interruptable = true; + Ok(Timeout::Time(Duration::from_secs(20))) + } else { + self.wasmtime.consume_fuel = true; + Ok(Timeout::Fuel(100_000)) + } + } } struct UnalignedMemoryCreator; @@ -194,40 +255,89 @@ impl<'a> Arbitrary<'a> for SpecTest { } } -/// Type alias for wasm-smith generated modules using wasmtime's default -/// configuration. -pub type GeneratedModule = wasm_smith::ConfiguredModule; +/// Default module-level configuration for fuzzing Wasmtime. +/// +/// Internally this uses `wasm-smith`'s own `SwarmConfig` but we further refine +/// the defaults here as well. +#[derive(Debug, Clone)] +pub struct ModuleConfig { + #[allow(missing_docs)] + pub config: SwarmConfig, +} -/// Wasmtime-specific default configuration for wasm-smith-generated modules. -#[derive(Arbitrary, Clone, Debug)] -pub struct WasmtimeDefaultConfig; - -impl wasm_smith::Config for WasmtimeDefaultConfig { - // Allow multi-memory to get exercised - fn max_memories(&self) -> usize { - 2 +impl ModuleConfig { + /// Uses this configuration and the supplied source of data to generate + /// a wasm module. + pub fn generate(&self, input: &mut Unstructured<'_>) -> arbitrary::Result { + wasm_smith::Module::new(self.config.clone(), input) } - // Allow multi-table (reference types) to get exercised - fn max_tables(&self) -> usize { - 4 + /// Indicates that this configuration should be spec-test-compliant, + /// disabling various features the spec tests assert are disabled. + pub fn set_spectest_compliant(&mut self) { + self.config.memory64_enabled = false; + self.config.simd_enabled = false; + self.config.bulk_memory_enabled = true; + self.config.reference_types_enabled = true; + self.config.max_memories = 1; } - // Turn some wasm features default-on for those that have a finished - // implementation in Wasmtime. - fn simd_enabled(&self) -> bool { - true - } + /// Indicates that this configuration is being used for differential + /// execution so only a single function should be generated since that's all + /// that's going to be exercised. + pub fn set_differential_config(&mut self) { + self.config.allow_start_export = false; + // Make sure there's a type available for the function. + self.config.min_types = 1; + self.config.max_types = 1; - fn reference_types_enabled(&self) -> bool { - true - } + // Generate one and only one function + self.config.min_funcs = 1; + self.config.max_funcs = 1; - fn bulk_memory_enabled(&self) -> bool { - true - } + // Give the function a memory, but keep it small + self.config.min_memories = 1; + self.config.max_memories = 1; + self.config.max_memory_pages = 1; + self.config.memory_max_size_required = true; - fn memory64_enabled(&self) -> bool { - true + // Don't allow any imports + self.config.max_imports = 0; + + // Try to get the function and the memory exported + self.config.min_exports = 2; + self.config.max_exports = 4; + + // NaN is canonicalized at the wasm level for differential fuzzing so we + // can paper over NaN differences between engines. + self.config.canonicalize_nans = true; + + // When diffing against a non-wasmtime engine then disable wasm + // features to get selectively re-enabled against each differential + // engine. + self.config.bulk_memory_enabled = false; + self.config.reference_types_enabled = false; + self.config.simd_enabled = false; + self.config.memory64_enabled = false; + } +} + +impl<'a> Arbitrary<'a> for ModuleConfig { + fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { + let mut config = SwarmConfig::arbitrary(u)?; + + // Allow multi-memory by default. + config.max_memories = config.max_memories.max(2); + + // Allow multi-table by default. + config.max_tables = config.max_tables.max(4); + + // Allow enabling some various wasm proposals by default. + config.bulk_memory_enabled = u.arbitrary()?; + config.reference_types_enabled = u.arbitrary()?; + config.simd_enabled = u.arbitrary()?; + config.memory64_enabled = u.arbitrary()?; + + Ok(ModuleConfig { config }) } } diff --git a/crates/fuzzing/src/generators/api.rs b/crates/fuzzing/src/generators/api.rs index 731a6fe017..b27313b459 100644 --- a/crates/fuzzing/src/generators/api.rs +++ b/crates/fuzzing/src/generators/api.rs @@ -14,13 +14,12 @@ //! //! [swarm testing]: https://www.cs.utah.edu/~regehr/papers/swarm12.pdf +use crate::generators::{Config, ModuleConfig}; use arbitrary::{Arbitrary, Unstructured}; use std::collections::BTreeSet; #[derive(Arbitrary, Debug)] struct Swarm { - config_debug_info: bool, - config_interruptable: bool, module_new: bool, module_drop: bool, instance_new: bool, @@ -32,37 +31,20 @@ struct Swarm { #[derive(Arbitrary, Debug)] #[allow(missing_docs)] pub enum ApiCall { - ConfigNew, - ConfigDebugInfo(bool), - ConfigInterruptable(bool), - EngineNew, - StoreNew, - ModuleNew { - id: usize, - wasm: super::GeneratedModule, - }, - ModuleDrop { - id: usize, - }, - InstanceNew { - id: usize, - module: usize, - }, - InstanceDrop { - id: usize, - }, - CallExportedFunc { - instance: usize, - nth: usize, - }, + StoreNew(Config), + ModuleNew { id: usize, wasm: Vec }, + ModuleDrop { id: usize }, + InstanceNew { id: usize, module: usize }, + InstanceDrop { id: usize }, + CallExportedFunc { instance: usize, nth: usize }, } use ApiCall::*; -#[derive(Default)] struct Scope { id_counter: usize, modules: BTreeSet, instances: BTreeSet, + module_config: ModuleConfig, } impl Scope { @@ -87,11 +69,16 @@ impl<'a> Arbitrary<'a> for ApiCalls { let swarm = Swarm::arbitrary(input)?; let mut calls = vec![]; - arbitrary_config(input, &swarm, &mut calls)?; - calls.push(EngineNew); - calls.push(StoreNew); + let config = Config::arbitrary(input)?; + let module_config = config.module_config.clone(); + calls.push(StoreNew(config)); - let mut scope = Scope::default(); + let mut scope = Scope { + id_counter: 0, + modules: BTreeSet::default(), + instances: BTreeSet::default(), + module_config, + }; // Total limit on number of API calls we'll generate. This exists to // avoid libFuzzer timeouts. @@ -107,10 +94,13 @@ impl<'a> Arbitrary<'a> for ApiCalls { if swarm.module_new { choices.push(|input, scope| { let id = scope.next_id(); - let mut wasm = super::GeneratedModule::arbitrary(input)?; - wasm.module.ensure_termination(1000); + let mut wasm = scope.module_config.generate(input)?; + wasm.ensure_termination(1000); scope.modules.insert(id); - Ok(ModuleNew { id, wasm }) + Ok(ModuleNew { + id, + wasm: wasm.to_bytes(), + }) }); } if swarm.module_drop && !scope.modules.is_empty() { @@ -156,44 +146,4 @@ impl<'a> Arbitrary<'a> for ApiCalls { Ok(ApiCalls { calls }) } - - fn size_hint(depth: usize) -> (usize, Option) { - arbitrary::size_hint::recursion_guard(depth, |depth| { - arbitrary::size_hint::or( - // This is the stuff we unconditionally need, which affects the - // minimum size. - arbitrary::size_hint::and( - ::size_hint(depth), - // `arbitrary_config` uses four bools: - // 2 when `swarm.config_debug_info` is true - // 2 when `swarm.config_interruptable` is true - <(bool, bool, bool, bool) as Arbitrary>::size_hint(depth), - ), - // We can generate arbitrary `WasmOptTtf` instances, which have - // no upper bound on the number of bytes they consume. This sets - // the upper bound to `None`. - ::size_hint(depth), - ) - }) - } -} - -fn arbitrary_config( - input: &mut Unstructured, - swarm: &Swarm, - calls: &mut Vec, -) -> arbitrary::Result<()> { - calls.push(ConfigNew); - - if swarm.config_debug_info && bool::arbitrary(input)? { - calls.push(ConfigDebugInfo(bool::arbitrary(input)?)); - } - - if swarm.config_interruptable && bool::arbitrary(input)? { - calls.push(ConfigInterruptable(bool::arbitrary(input)?)); - } - - // TODO: flags, features, and compilation strategy. - - Ok(()) } diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs index 16b9441caf..e7bbdc35fb 100644 --- a/crates/fuzzing/src/lib.rs +++ b/crates/fuzzing/src/lib.rs @@ -1,7 +1,8 @@ //! Fuzzing infrastructure for Wasmtime. -#![deny(missing_docs, missing_debug_implementations)] +#![deny(missing_docs)] +pub use wasm_smith; pub mod generators; pub mod oracles; @@ -29,19 +30,3 @@ pub(crate) fn init_fuzzing() { .build_global(); }) } - -/// Create default fuzzing config with given strategy -pub fn fuzz_default_config(strategy: wasmtime::Strategy) -> anyhow::Result { - init_fuzzing(); - let mut config = wasmtime::Config::new(); - config - .cranelift_nan_canonicalization(true) - .wasm_bulk_memory(true) - .wasm_reference_types(true) - .wasm_module_linking(true) - .wasm_multi_memory(true) - .wasm_simd(true) - .wasm_memory64(true) - .strategy(strategy)?; - Ok(config) -} diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index d3b440e559..110b2feaa7 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -12,8 +12,8 @@ pub mod dummy; +use crate::generators; use anyhow::Context; -use arbitrary::Arbitrary; use log::{debug, warn}; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::sync::{Arc, Condvar, Mutex}; @@ -28,7 +28,11 @@ mod v8; static CNT: AtomicUsize = AtomicUsize::new(0); -fn log_wasm(wasm: &[u8]) { +/// Logs a wasm file to the filesystem to make it easy to figure out what wasm +/// was used when debugging. +pub fn log_wasm(wasm: &[u8]) { + super::init_fuzzing(); + if !log::log_enabled!(log::Level::Debug) { return; } @@ -47,21 +51,9 @@ fn log_wasm(wasm: &[u8]) { } } -fn create_store(engine: &Engine) -> Store { - let mut store = Store::new( - &engine, - StoreLimits { - // Limits tables/memories within a store to at most 1gb for now to - // exercise some larger address but not overflow various limits. - remaining_memory: 1 << 30, - oom: false, - }, - ); - store.limiter(|s| s as &mut dyn ResourceLimiter); - return store; -} - -struct StoreLimits { +/// The `T` in `Store` for fuzzing stores, used to limit resource +/// consumption during fuzzing. +pub struct StoreLimits { /// Remaining memory, in bytes, left to allocate remaining_memory: usize, /// Whether or not an allocation request has been denied @@ -69,6 +61,16 @@ struct StoreLimits { } impl StoreLimits { + /// Creates the default set of limits for all fuzzing stores. + pub fn new() -> StoreLimits { + StoreLimits { + // Limits tables/memories within a store to at most 1gb for now to + // exercise some larger address but not overflow various limits. + remaining_memory: 1 << 30, + oom: false, + } + } + fn alloc(&mut self, amt: usize) -> bool { match self.remaining_memory.checked_sub(amt) { Some(mem) => { @@ -108,48 +110,22 @@ pub enum Timeout { Fuel(u64), } -/// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected -/// panic or segfault or anything else that can be detected "passively". -/// -/// Performs initial validation, and returns early if the Wasm is invalid. -/// -/// You can control which compiler is used via passing a `Strategy`. -pub fn instantiate(wasm: &[u8], known_valid: bool, strategy: Strategy) { - // Explicitly disable module linking for now since it's a breaking change to - // pre-module-linking modules due to imports - let mut cfg = crate::fuzz_default_config(strategy).unwrap(); - cfg.wasm_module_linking(false); - instantiate_with_config(wasm, known_valid, cfg, Timeout::None); -} - /// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected /// panic or segfault or anything else that can be detected "passively". /// /// The engine will be configured using provided config. -/// -/// See also `instantiate` functions. -pub fn instantiate_with_config( - wasm: &[u8], - known_valid: bool, - mut config: Config, - timeout: Timeout, -) { - crate::init_fuzzing(); - - config.interruptable(match &timeout { - Timeout::Time(_) => true, - _ => false, - }); - config.consume_fuel(match &timeout { - Timeout::Fuel(_) => true, - _ => false, - }); - let engine = Engine::new(&config).unwrap(); - let mut store = create_store(&engine); +pub fn instantiate(wasm: &[u8], known_valid: bool, config: &generators::Config, timeout: Timeout) { + let mut store = config.to_store(); let mut timeout_state = SignalOnDrop::default(); match timeout { - Timeout::Fuel(fuel) => store.add_fuel(fuel).unwrap(), + Timeout::Fuel(fuel) => { + // consume the default fuel in the store ... + let remaining = store.consume_fuel(0).unwrap(); + store.consume_fuel(remaining - 1).unwrap(); + // ... then add back in how much fuel we're allowing here + store.add_fuel(fuel).unwrap(); + } // If a timeout is requested then we spawn a helper thread to wait for // the requested time and then send us a signal to get interrupted. We // also arrange for the thread's sleep to get interrupted if we return @@ -167,7 +143,7 @@ pub fn instantiate_with_config( } log_wasm(wasm); - let module = match Module::new(&engine, wasm) { + let module = match Module::new(store.engine(), wasm) { Ok(module) => module, Err(_) if !known_valid => return, Err(e) => panic!("failed to compile module: {:?}", e), @@ -216,34 +192,17 @@ fn instantiate_with_dummy(store: &mut Store, module: &Module) -> Op panic!("failed to instantiate {:?}", e); } -/// Compile the Wasm buffer, and implicitly fail if we have an unexpected -/// panic or segfault or anything else that can be detected "passively". -/// -/// Performs initial validation, and returns early if the Wasm is invalid. -/// -/// You can control which compiler is used via passing a `Strategy`. -pub fn compile(wasm: &[u8], strategy: Strategy) { - crate::init_fuzzing(); - - let mut config = crate::fuzz_default_config(strategy).unwrap(); - config.wasm_module_linking(false); - let engine = Engine::new(&config).unwrap(); - log_wasm(wasm); - let _ = Module::new(&engine, wasm); -} - /// Instantiate the given Wasm module with each `Config` and call all of its /// exports. Modulo OOM, non-canonical NaNs, and usage of Wasm features that are /// or aren't enabled for different configs, we should get the same results when /// we call the exported functions for all of our different configs. pub fn differential_execution( - module: &crate::generators::GeneratedModule, - configs: &[crate::generators::Config], + wasm: &[u8], + module_config: &generators::ModuleConfig, + configs: &[generators::WasmtimeConfig], ) { use std::collections::{HashMap, HashSet}; - crate::init_fuzzing(); - // We need at least two configs. if configs.len() < 2 // And all the configs should be unique. @@ -252,32 +211,18 @@ pub fn differential_execution( return; } - let configs: Vec<_> = configs.iter().map(|c| (c.to_wasmtime(), c)).collect(); let mut export_func_results: HashMap, Trap>> = Default::default(); - let wasm = module.module.to_bytes(); log_wasm(&wasm); - for (mut config, fuzz_config) in configs { + for fuzz_config in configs { + let fuzz_config = generators::Config { + module_config: module_config.clone(), + wasmtime: fuzz_config.clone(), + }; log::debug!("fuzz config: {:?}", fuzz_config); - // Disable module linking since it isn't enabled by default for - // `GeneratedModule` but is enabled by default for our fuzz config. - // Since module linking is currently a breaking change this is required - // to accept modules that would otherwise be broken by module linking. - config.wasm_module_linking(false); - // We don't want different configurations with different values for nan - // canonicalization since that can affect results. All configs should - // have the same value configured for this option, so `true` is - // arbitrarily chosen here. - config.cranelift_nan_canonicalization(true); - - let engine = Engine::new(&config).unwrap(); - let mut store = create_store(&engine); - if fuzz_config.consume_fuel { - store.add_fuel(u64::max_value()).unwrap(); - } - - let module = Module::new(&engine, &wasm).unwrap(); + let mut store = fuzz_config.to_store(); + let module = Module::new(store.engine(), &wasm).unwrap(); // TODO: we should implement tracing versions of these dummy imports // that record a trace of the order that imported functions were called @@ -367,53 +312,26 @@ fn f64_equal(a: u64, b: u64) -> bool { } /// Invoke the given API calls. -pub fn make_api_calls(api: crate::generators::api::ApiCalls) { +pub fn make_api_calls(api: generators::api::ApiCalls) { use crate::generators::api::ApiCall; use std::collections::HashMap; - crate::init_fuzzing(); - - let mut config: Option = None; - let mut engine: Option = None; let mut store: Option> = None; let mut modules: HashMap = Default::default(); let mut instances: HashMap = Default::default(); for call in api.calls { match call { - ApiCall::ConfigNew => { - log::trace!("creating config"); - assert!(config.is_none()); - config = Some(crate::fuzz_default_config(wasmtime::Strategy::Cranelift).unwrap()); - } - - ApiCall::ConfigDebugInfo(b) => { - log::trace!("enabling debuginfo"); - config.as_mut().unwrap().debug_info(b); - } - - ApiCall::ConfigInterruptable(b) => { - log::trace!("enabling interruption"); - config.as_mut().unwrap().interruptable(b); - } - - ApiCall::EngineNew => { - log::trace!("creating engine"); - assert!(engine.is_none()); - engine = Some(Engine::new(config.as_ref().unwrap()).unwrap()); - } - - ApiCall::StoreNew => { + ApiCall::StoreNew(config) => { log::trace!("creating store"); assert!(store.is_none()); - store = Some(create_store(engine.as_ref().unwrap())); + store = Some(config.to_store()); } ApiCall::ModuleNew { id, wasm } => { log::debug!("creating module: {}", id); - let wasm = wasm.module.to_bytes(); log_wasm(&wasm); - let module = match Module::new(engine.as_ref().unwrap(), &wasm) { + let module = match Module::new(store.as_ref().unwrap().engine(), &wasm) { Ok(m) => m, Err(_) => continue, }; @@ -486,18 +404,10 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) { /// Executes the wast `test` spectest with the `config` specified. /// /// Ensures that spec tests pass regardless of the `Config`. -pub fn spectest(fuzz_config: crate::generators::Config, test: crate::generators::SpecTest) { - crate::init_fuzzing(); +pub fn spectest(mut fuzz_config: generators::Config, test: generators::SpecTest) { + fuzz_config.module_config.set_spectest_compliant(); log::debug!("running {:?} with {:?}", test.file, fuzz_config); - let mut config = fuzz_config.to_wasmtime(); - config.wasm_memory64(false); - config.wasm_module_linking(false); - config.wasm_multi_memory(false); - let mut store = create_store(&Engine::new(&config).unwrap()); - if fuzz_config.consume_fuel { - store.add_fuel(u64::max_value()).unwrap(); - } - let mut wast_context = WastContext::new(store); + let mut wast_context = WastContext::new(fuzz_config.to_store()); wast_context.register_spectest().unwrap(); wast_context .run_buffer(test.file, test.contents.as_bytes()) @@ -505,32 +415,23 @@ pub fn spectest(fuzz_config: crate::generators::Config, test: crate::generators: } /// Execute a series of `table.get` and `table.set` operations. -pub fn table_ops( - fuzz_config: crate::generators::Config, - ops: crate::generators::table_ops::TableOps, -) { +pub fn table_ops(fuzz_config: generators::Config, ops: generators::table_ops::TableOps) { let _ = env_logger::try_init(); let expected_drops = Arc::new(AtomicUsize::new(ops.num_params() as usize)); let num_dropped = Arc::new(AtomicUsize::new(0)); { - let mut config = fuzz_config.to_wasmtime(); - config.wasm_reference_types(true); - config.consume_fuel(true); - - let engine = Engine::new(&config).unwrap(); - let mut store = create_store(&engine); - store.add_fuel(100).unwrap(); + let mut store = fuzz_config.to_store(); let wasm = ops.to_wasm_binary(); log_wasm(&wasm); - let module = match Module::new(&engine, &wasm) { + let module = match Module::new(store.engine(), &wasm) { Ok(m) => m, Err(_) => return, }; - let mut linker = Linker::new(&engine); + let mut linker = Linker::new(store.engine()); // To avoid timeouts, limit the number of explicit GCs we perform per // test case. @@ -648,70 +549,6 @@ pub fn table_ops( } } -/// Configuration options for wasm-smith such that generated modules always -/// conform to certain specifications: one exported function, one exported -/// memory. -#[derive(Default, Debug, Arbitrary, Clone)] -pub struct SingleFunctionModuleConfig; - -impl wasm_smith::Config - for SingleFunctionModuleConfig -{ - fn allow_start_export(&self) -> bool { - false - } - - fn min_types(&self) -> usize { - 1 - } - - fn min_funcs(&self) -> usize { - 1 - } - - fn max_funcs(&self) -> usize { - 1 - } - - fn min_memories(&self) -> u32 { - 1 - } - - fn max_memories(&self) -> usize { - 1 - } - - fn max_imports(&self) -> usize { - 0 - } - - fn min_exports(&self) -> usize { - 2 - } - - fn max_memory_pages(&self, _is_64: bool) -> u64 { - 1 - } - - fn memory_max_size_required(&self) -> bool { - true - } - - // NaN is canonicalized at the wasm level for differential fuzzing so we - // can paper over NaN differences between engines. - fn canonicalize_nans(&self) -> bool { - true - } - - fn simd_enabled(&self) -> bool { - SIMD - } - - fn bulk_memory_enabled(&self) -> bool { - BULK - } -} - /// Perform differential execution between Cranelift and wasmi, diffing the /// resulting memory image when execution terminates. This relies on the /// module-under-test to be instrumented to bound the execution time. Invoke @@ -720,7 +557,7 @@ impl wasm_smith::Config /// /// May return `None` if we early-out due to a rejected fuzz config; these /// should be rare if modules are generated appropriately. -pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Config) -> Option<()> { +pub fn differential_wasmi_execution(wasm: &[u8], config: &generators::Config) -> Option<()> { crate::init_fuzzing(); log_wasm(wasm); @@ -810,7 +647,7 @@ pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Con /// specification interpreter. /// /// May return `None` if we early-out due to a rejected fuzz config. -pub fn differential_spec_execution(wasm: &[u8], config: &crate::generators::Config) -> Option<()> { +pub fn differential_spec_execution(wasm: &[u8], config: &generators::Config) -> Option<()> { crate::init_fuzzing(); debug!("config: {:#?}", config); log_wasm(wasm); @@ -887,20 +724,10 @@ pub fn differential_spec_execution(wasm: &[u8], config: &crate::generators::Conf fn differential_store( wasm: &[u8], - fuzz_config: &crate::generators::Config, + fuzz_config: &generators::Config, ) -> (Module, Store) { - let mut config = fuzz_config.to_wasmtime(); - // forcibly disable NaN canonicalization because wasm-smith has already - // been configured to canonicalize everything at the wasm level. - config.cranelift_nan_canonicalization(false); - let engine = Engine::new(&config).unwrap(); - let mut store = create_store(&engine); - if fuzz_config.consume_fuel { - store.add_fuel(u64::max_value()).unwrap(); - } - - let module = Module::new(&engine, &wasm).expect("Wasmtime can compile module"); - + let store = fuzz_config.to_store(); + let module = Module::new(store.engine(), &wasm).expect("Wasmtime can compile module"); (module, store) } @@ -908,7 +735,7 @@ fn differential_store( /// its `Val` results. fn run_in_wasmtime( wasm: &[u8], - config: &crate::generators::Config, + config: &generators::Config, params: &[Val], ) -> anyhow::Result> { // Instantiate wasmtime module and instance. diff --git a/crates/fuzzing/src/oracles/v8.rs b/crates/fuzzing/src/oracles/v8.rs index be2549f3de..5da046b051 100644 --- a/crates/fuzzing/src/oracles/v8.rs +++ b/crates/fuzzing/src/oracles/v8.rs @@ -15,7 +15,6 @@ use wasmtime::*; /// from happening. pub fn differential_v8_execution(wasm: &[u8], config: &crate::generators::Config) -> Option<()> { // Wasmtime setup - crate::init_fuzzing(); log_wasm(wasm); let (wasmtime_module, mut wasmtime_store) = super::differential_store(wasm, config); log::trace!("compiled module with wasmtime"); @@ -79,11 +78,15 @@ pub fn differential_v8_execution(wasm: &[u8], config: &crate::generators::Config ValType::I64 => Val::I64(0), ValType::F32 => Val::F32(0), ValType::F64 => Val::F64(0), + ValType::FuncRef => Val::FuncRef(None), + ValType::ExternRef => Val::ExternRef(None), _ => unimplemented!(), }); v8_params.push(match param { ValType::I32 | ValType::F32 | ValType::F64 => v8::Number::new(&mut scope, 0.0).into(), ValType::I64 => v8::BigInt::new_from_i64(&mut scope, 0).into(), + ValType::FuncRef => v8::null(&mut scope).into(), + ValType::ExternRef => v8::null(&mut scope).into(), _ => unimplemented!(), }); } @@ -231,6 +234,19 @@ fn assert_val_match(a: &Val, b: &v8::Local<'_, v8::Value>, scope: &mut v8::Handl b.to_number(scope).unwrap().value(), ); } + + // Externref values can only come from us, the embedder, and we only + // give wasm null, so these values should always be null. + Val::ExternRef(ref wasmtime) => { + assert!(wasmtime.is_none()); + assert!(b.is_null()); + } + + // In general we can't equate function references since wasm modules can + // create references to internal functions via `func.ref`, so we don't + // equate values here. + Val::FuncRef(_) => {} + _ => panic!("unsupported match {:?}", a), } diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 9c2b3d1ee9..39f2adf11e 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -19,7 +19,6 @@ libfuzzer-sys = "0.4.0" target-lexicon = "0.12" wasmtime = { path = "../crates/wasmtime" } wasmtime-fuzzing = { path = "../crates/fuzzing" } -wasm-smith = "0.7.0" [features] # Leave a stub feature with no side-effects in place for now: the OSS-Fuzz @@ -76,20 +75,8 @@ test = false doc = false [[bin]] -name = "instantiate-wasm-smith" -path = "fuzz_targets/instantiate-wasm-smith.rs" -test = false -doc = false - -[[bin]] -name = "instantiate-swarm" -path = "fuzz_targets/instantiate-swarm.rs" -test = false -doc = false - -[[bin]] -name = "instantiate-maybe-invalid" -path = "fuzz_targets/instantiate-maybe-invalid.rs" +name = "compile-maybe-invalid" +path = "fuzz_targets/compile-maybe-invalid.rs" test = false doc = false diff --git a/fuzz/fuzz_targets/compile-maybe-invalid.rs b/fuzz/fuzz_targets/compile-maybe-invalid.rs new file mode 100644 index 0000000000..2cdb282f85 --- /dev/null +++ b/fuzz/fuzz_targets/compile-maybe-invalid.rs @@ -0,0 +1,12 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use wasmtime::{Engine, Module}; +use wasmtime_fuzzing::wasm_smith::MaybeInvalidModule; + +fuzz_target!(|module: MaybeInvalidModule| { + let engine = Engine::default(); + let wasm = module.to_bytes(); + wasmtime_fuzzing::oracles::log_wasm(&wasm); + drop(Module::new(&engine, &wasm)); +}); diff --git a/fuzz/fuzz_targets/compile.rs b/fuzz/fuzz_targets/compile.rs index 54284da256..684d75637a 100644 --- a/fuzz/fuzz_targets/compile.rs +++ b/fuzz/fuzz_targets/compile.rs @@ -1,9 +1,10 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use wasmtime::Strategy; -use wasmtime_fuzzing::oracles; +use wasmtime::{Engine, Module}; fuzz_target!(|data: &[u8]| { - oracles::compile(data, Strategy::Cranelift); + let engine = Engine::default(); + wasmtime_fuzzing::oracles::log_wasm(data); + drop(Module::new(&engine, data)); }); diff --git a/fuzz/fuzz_targets/differential.rs b/fuzz/fuzz_targets/differential.rs index ea6092af8f..cdd9835249 100644 --- a/fuzz/fuzz_targets/differential.rs +++ b/fuzz/fuzz_targets/differential.rs @@ -1,14 +1,24 @@ #![no_main] +use libfuzzer_sys::arbitrary::{Result, Unstructured}; use libfuzzer_sys::fuzz_target; use wasmtime_fuzzing::{generators, oracles}; -fuzz_target!(|data: ( - generators::Config, - generators::Config, - generators::GeneratedModule, -)| { - let (lhs, rhs, mut wasm) = data; - wasm.module.ensure_termination(1000); - oracles::differential_execution(&wasm, &[lhs, rhs]); +fuzz_target!(|data: &[u8]| { + // errors in `run` have to do with not enough input in `data`, which we + // ignore here since it doesn't affect how we'd like to fuzz. + drop(run(data)); }); + +fn run(data: &[u8]) -> Result<()> { + let mut u = Unstructured::new(data); + + let lhs: generators::WasmtimeConfig = u.arbitrary()?; + let rhs: generators::WasmtimeConfig = u.arbitrary()?; + let config: generators::ModuleConfig = u.arbitrary()?; + let mut module = config.generate(&mut u)?; + module.ensure_termination(1000); + + oracles::differential_execution(&module.to_bytes(), &config, &[lhs, rhs]); + Ok(()) +} diff --git a/fuzz/fuzz_targets/differential_v8.rs b/fuzz/fuzz_targets/differential_v8.rs index 3870e09759..9304fdec5c 100644 --- a/fuzz/fuzz_targets/differential_v8.rs +++ b/fuzz/fuzz_targets/differential_v8.rs @@ -1,13 +1,27 @@ #![no_main] +use libfuzzer_sys::arbitrary::{Result, Unstructured}; use libfuzzer_sys::fuzz_target; use wasmtime_fuzzing::{generators, oracles}; -fuzz_target!(|data: ( - generators::Config, - wasm_smith::ConfiguredModule> -)| { - let (config, mut wasm) = data; - wasm.module.ensure_termination(1000); - oracles::differential_v8_execution(&wasm.module.to_bytes(), &config); +fuzz_target!(|data: &[u8]| { + // errors in `run` have to do with not enough input in `data`, which we + // ignore here since it doesn't affect how we'd like to fuzz. + drop(run(data)); }); + +fn run(data: &[u8]) -> Result<()> { + let mut u = Unstructured::new(data); + let mut config: generators::Config = u.arbitrary()?; + config.module_config.set_differential_config(); + + // Enable features that v8 has implemented + config.module_config.config.simd_enabled = true; + config.module_config.config.bulk_memory_enabled = true; + config.module_config.config.reference_types_enabled = true; + + let mut module = config.module_config.generate(&mut u)?; + module.ensure_termination(1000); + oracles::differential_v8_execution(&module.to_bytes(), &config); + Ok(()) +} diff --git a/fuzz/fuzz_targets/differential_wasmi.rs b/fuzz/fuzz_targets/differential_wasmi.rs index 29674ee449..fa6931b6fb 100644 --- a/fuzz/fuzz_targets/differential_wasmi.rs +++ b/fuzz/fuzz_targets/differential_wasmi.rs @@ -1,13 +1,21 @@ #![no_main] +use libfuzzer_sys::arbitrary::{Result, Unstructured}; use libfuzzer_sys::fuzz_target; use wasmtime_fuzzing::{generators, oracles}; -fuzz_target!(|data: ( - generators::Config, - wasm_smith::ConfiguredModule> -)| { - let (config, mut wasm) = data; - wasm.module.ensure_termination(1000); - oracles::differential_wasmi_execution(&wasm.module.to_bytes(), &config); +fuzz_target!(|data: &[u8]| { + // errors in `run` have to do with not enough input in `data`, which we + // ignore here since it doesn't affect how we'd like to fuzz. + drop(run(data)); }); + +fn run(data: &[u8]) -> Result<()> { + let mut u = Unstructured::new(data); + let mut config: generators::Config = u.arbitrary()?; + config.module_config.set_differential_config(); + let mut module = config.module_config.generate(&mut u)?; + module.ensure_termination(1000); + oracles::differential_wasmi_execution(&module.to_bytes(), &config); + Ok(()) +} diff --git a/fuzz/fuzz_targets/instantiate-maybe-invalid.rs b/fuzz/fuzz_targets/instantiate-maybe-invalid.rs deleted file mode 100644 index 46a749f5e8..0000000000 --- a/fuzz/fuzz_targets/instantiate-maybe-invalid.rs +++ /dev/null @@ -1,21 +0,0 @@ -#![no_main] - -use libfuzzer_sys::fuzz_target; -use std::time::Duration; -use wasm_smith::MaybeInvalidModule; -use wasmtime::Strategy; -use wasmtime_fuzzing::oracles::{self, Timeout}; - -fuzz_target!(|pair: (bool, MaybeInvalidModule)| { - let (timeout_with_time, module) = pair; - oracles::instantiate_with_config( - &module.to_bytes(), - false, - wasmtime_fuzzing::fuzz_default_config(Strategy::Auto).unwrap(), - if timeout_with_time { - Timeout::Time(Duration::from_secs(20)) - } else { - Timeout::Fuel(100_000) - }, - ); -}); diff --git a/fuzz/fuzz_targets/instantiate-swarm.rs b/fuzz/fuzz_targets/instantiate-swarm.rs deleted file mode 100644 index 0ebc708580..0000000000 --- a/fuzz/fuzz_targets/instantiate-swarm.rs +++ /dev/null @@ -1,41 +0,0 @@ -#![no_main] - -use libfuzzer_sys::arbitrary::{Result, Unstructured}; -use libfuzzer_sys::fuzz_target; -use std::time::Duration; -use wasm_smith::{Module, SwarmConfig}; -use wasmtime::Strategy; -use wasmtime_fuzzing::oracles::{self, Timeout}; - -fuzz_target!(|data: &[u8]| { - // errors in `run` have to do with not enough input in `data`, which we - // ignore here since it doesn't affect how we'd like to fuzz. - drop(run(data)); -}); - -fn run(data: &[u8]) -> Result<()> { - let mut u = Unstructured::new(data); - let timeout = if u.arbitrary()? { - Timeout::Time(Duration::from_secs(20)) - } else { - Timeout::Fuel(100_000) - }; - - // Further configure `SwarmConfig` after we generate one to enable features - // that aren't otherwise enabled by default. We want to test all of these in - // Wasmtime. - let mut config: SwarmConfig = u.arbitrary()?; - config.module_linking_enabled = u.arbitrary()?; - config.memory64_enabled = u.arbitrary()?; - // Don't generate modules that allocate more than 6GB - config.max_memory_pages = 6 << 30; - let module = Module::new(config.clone(), &mut u)?; - - let mut cfg = wasmtime_fuzzing::fuzz_default_config(Strategy::Auto).unwrap(); - cfg.wasm_multi_memory(config.max_memories > 1); - cfg.wasm_module_linking(config.module_linking_enabled); - cfg.wasm_memory64(config.memory64_enabled); - - oracles::instantiate_with_config(&module.to_bytes(), true, cfg, timeout); - Ok(()) -} diff --git a/fuzz/fuzz_targets/instantiate-wasm-smith.rs b/fuzz/fuzz_targets/instantiate-wasm-smith.rs deleted file mode 100644 index 883a459555..0000000000 --- a/fuzz/fuzz_targets/instantiate-wasm-smith.rs +++ /dev/null @@ -1,12 +0,0 @@ -#![no_main] - -use libfuzzer_sys::fuzz_target; -use wasmtime::Strategy; -use wasmtime_fuzzing::{generators::GeneratedModule, oracles}; - -fuzz_target!(|module: GeneratedModule| { - let mut module = module; - module.module.ensure_termination(1000); - let wasm_bytes = module.module.to_bytes(); - oracles::instantiate(&wasm_bytes, true, Strategy::Auto); -}); diff --git a/fuzz/fuzz_targets/instantiate.rs b/fuzz/fuzz_targets/instantiate.rs index ec41c59105..0247d9f21d 100644 --- a/fuzz/fuzz_targets/instantiate.rs +++ b/fuzz/fuzz_targets/instantiate.rs @@ -1,9 +1,36 @@ #![no_main] +use libfuzzer_sys::arbitrary::{Result, Unstructured}; use libfuzzer_sys::fuzz_target; -use wasmtime::Strategy; -use wasmtime_fuzzing::oracles; +use wasmtime_fuzzing::oracles::Timeout; +use wasmtime_fuzzing::{generators, oracles}; fuzz_target!(|data: &[u8]| { - oracles::instantiate(data, false, Strategy::Auto); + // errors in `run` have to do with not enough input in `data`, which we + // ignore here since it doesn't affect how we'd like to fuzz. + drop(run(data)); }); + +fn run(data: &[u8]) -> Result<()> { + let mut u = Unstructured::new(data); + let mut config: generators::Config = u.arbitrary()?; + + // Pick either fuel, duration-based, or module-based timeout. Note that the + // module-based timeout is implemented with wasm-smith's + // `ensure_termination` option. + let timeout = if u.arbitrary()? { + config.generate_timeout(&mut u)? + } else { + Timeout::None + }; + + // Enable module linking for this fuzz target specifically + config.module_config.config.module_linking_enabled = u.arbitrary()?; + + let mut module = config.module_config.generate(&mut u)?; + if let Timeout::None = timeout { + module.ensure_termination(1000); + } + oracles::instantiate(&module.to_bytes(), true, &config, timeout); + Ok(()) +} diff --git a/tests/all/fuzzing.rs b/tests/all/fuzzing.rs deleted file mode 100644 index dcc0673032..0000000000 --- a/tests/all/fuzzing.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Regression tests for bugs found via fuzzing. -//! -//! The `#[test]` goes in here, the Wasm binary goes in -//! `./fuzzing/some-descriptive-name.wasm`, and then the `#[test]` should -//! use the Wasm binary by including it via -//! `include_bytes!("./fuzzing/some-descriptive-name.wasm")`. - -use wasmtime::{Config, Strategy}; -use wasmtime_fuzzing::oracles::{self, Timeout}; - -#[test] -fn instantiate_empty_module() { - let data = wat::parse_str(include_str!("./fuzzing/empty.wat")).unwrap(); - oracles::instantiate(&data, true, Strategy::Auto); -} - -#[test] -fn instantiate_empty_module_with_memory() { - let data = wat::parse_str(include_str!("./fuzzing/empty_with_memory.wat")).unwrap(); - oracles::instantiate(&data, true, Strategy::Auto); -} - -#[test] -fn instantiate_module_that_compiled_to_x64_has_register_32() { - let mut config = Config::new(); - config.debug_info(true); - let data = wat::parse_str(include_str!("./fuzzing/issue694.wat")).unwrap(); - oracles::instantiate_with_config(&data, true, config, Timeout::None); -} diff --git a/tests/all/fuzzing/README.md b/tests/all/fuzzing/README.md deleted file mode 100644 index ff53c97785..0000000000 --- a/tests/all/fuzzing/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This directory contains `.wasm` binaries generated during fuzzing that uncovered -a bug, and which we now use as regression tests in `../fuzzing.rs`. diff --git a/tests/all/fuzzing/empty.wat b/tests/all/fuzzing/empty.wat deleted file mode 100644 index 3af8f25454..0000000000 --- a/tests/all/fuzzing/empty.wat +++ /dev/null @@ -1 +0,0 @@ -(module) diff --git a/tests/all/fuzzing/empty_with_memory.wat b/tests/all/fuzzing/empty_with_memory.wat deleted file mode 100644 index 4503196d52..0000000000 --- a/tests/all/fuzzing/empty_with_memory.wat +++ /dev/null @@ -1 +0,0 @@ -(module (memory 1)) diff --git a/tests/all/main.rs b/tests/all/main.rs index 4e8c9766ef..b1ec333981 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -7,7 +7,6 @@ mod externals; mod fuel; mod func; mod funcref; -mod fuzzing; mod gc; mod globals; mod host_funcs; diff --git a/tests/misc_testsuite/empty.wast b/tests/misc_testsuite/empty.wast index 08be4a08cd..db1950a4c3 100644 --- a/tests/misc_testsuite/empty.wast +++ b/tests/misc_testsuite/empty.wast @@ -44,3 +44,9 @@ "\ff\ff\ff\ff\0f" ;; index == u32::MAX (local) "\00" ;; empty string name ) + +;; empty module +(module) + +;; empty module with memory +(module (memory 1)) diff --git a/tests/all/fuzzing/issue1809.wat b/tests/misc_testsuite/issue1809.wast similarity index 100% rename from tests/all/fuzzing/issue1809.wat rename to tests/misc_testsuite/issue1809.wast diff --git a/tests/all/fuzzing/issue694.wat b/tests/misc_testsuite/issue694.wast similarity index 100% rename from tests/all/fuzzing/issue694.wat rename to tests/misc_testsuite/issue694.wast