diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index cc3bbf67fd..678841a7b9 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -380,136 +380,6 @@ pub fn differential( Ok(()) } -/// Instantiate the given Wasm module with each `Config` and call all of its -/// exports. Modulo OOM, non-canonical NaNs, and usage of Wasm features that are -/// or aren't enabled for different configs, we should get the same results when -/// we call the exported functions for all of our different configs. -/// -/// Returns `None` if a fuzz configuration was rejected (should happen rarely). -pub fn differential_execution( - wasm: &[u8], - module_config: &generators::ModuleConfig, - configs: &[generators::WasmtimeConfig], -) -> Option<()> { - use std::collections::{HashMap, HashSet}; - - // We need at least two configs. - if configs.len() < 2 - // And all the configs should be unique. - || configs.iter().collect::>().len() != configs.len() - { - return None; - } - - let mut export_func_results: HashMap, Trap>> = Default::default(); - log_wasm(&wasm); - - for fuzz_config in configs { - let fuzz_config = generators::Config { - module_config: module_config.clone(), - wasmtime: fuzz_config.clone(), - }; - log::debug!("fuzz config: {:?}", fuzz_config); - - let mut store = fuzz_config.to_store(); - let module = compile_module(store.engine(), &wasm, true, &fuzz_config)?; - - // TODO: we should implement tracing versions of these dummy imports - // that record a trace of the order that imported functions were called - // in and with what values. Like the results of exported functions, - // calls to imports should also yield the same values for each - // configuration, and we should assert that. - let instance = match instantiate_with_dummy(&mut store, &module) { - Some(instance) => instance, - None => continue, - }; - - let exports = instance - .exports(&mut store) - .filter_map(|e| { - let name = e.name().to_string(); - e.into_func().map(|f| (name, f)) - }) - .collect::>(); - for (name, f) in exports { - log::debug!("invoke export {:?}", name); - let ty = f.ty(&store); - let params = dummy::dummy_values(ty.params()); - let mut results = vec![Val::I32(0); ty.results().len()]; - let this_result = f - .call(&mut store, ¶ms, &mut results) - .map(|()| results.into()) - .map_err(|e| e.downcast::().unwrap()); - - let existing_result = export_func_results - .entry(name.to_string()) - .or_insert_with(|| this_result.clone()); - assert_same_export_func_result(&existing_result, &this_result, &name); - } - } - - return Some(()); - - fn assert_same_export_func_result( - lhs: &Result, Trap>, - rhs: &Result, Trap>, - func_name: &str, - ) { - let fail = || { - panic!( - "differential fuzzing failed: exported func {} returned two \ - different results: {:?} != {:?}", - func_name, lhs, rhs - ) - }; - - match (lhs, rhs) { - // Different compilation settings can lead to different amounts - // of stack space being consumed, so if either the lhs or the rhs - // hit a stack overflow then we discard the result of the other side - // since if it ran successfully or trapped that's ok in both - // situations. - (Err(e), _) | (_, Err(e)) if e.trap_code() == Some(TrapCode::StackOverflow) => {} - - (Err(a), Err(b)) => { - if a.trap_code() != b.trap_code() { - fail(); - } - } - (Ok(lhs), Ok(rhs)) => { - if lhs.len() != rhs.len() { - fail(); - } - for (lhs, rhs) in lhs.iter().zip(rhs.iter()) { - match (lhs, rhs) { - (Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue, - (Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue, - (Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue, - (Val::F32(lhs), Val::F32(rhs)) if f32_equal(*lhs, *rhs) => continue, - (Val::F64(lhs), Val::F64(rhs)) if f64_equal(*lhs, *rhs) => continue, - (Val::ExternRef(_), Val::ExternRef(_)) - | (Val::FuncRef(_), Val::FuncRef(_)) => continue, - _ => fail(), - } - } - } - _ => fail(), - } - } -} - -fn f32_equal(a: u32, b: u32) -> bool { - let a = f32::from_bits(a); - let b = f32::from_bits(b); - a == b || (a.is_nan() && b.is_nan()) -} - -fn f64_equal(a: u64, b: u64) -> bool { - let a = f64::from_bits(a); - let b = f64::from_bits(b); - a == b || (a.is_nan() && b.is_nan()) -} - /// Invoke the given API calls. pub fn make_api_calls(api: generators::api::ApiCalls) { use crate::generators::api::ApiCall; diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index e98d19a734..52ab67fb78 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -60,12 +60,6 @@ path = "fuzz_targets/differential.rs" test = false doc = false -[[bin]] -name = "differential_meta" -path = "fuzz_targets/differential_meta.rs" -test = false -doc = false - [[bin]] name = "differential_v8" path = "fuzz_targets/differential_v8.rs" diff --git a/fuzz/README.md b/fuzz/README.md index 40021f386c..ec2b77acab 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -34,9 +34,7 @@ At the time of writing, we have the following fuzz targets: to its source, yielding a function A', and checks that A compiled + incremental compilation generates the same machine code as if A' was compiled from scratch. -* `differential`: Generate a Wasm module and check that Wasmtime returns - the same results when run with two different configurations. -* `differential_meta`: Generate a Wasm module, evaluate each exported function +* `differential`: Generate a Wasm module, evaluate each exported function with random inputs, and check that Wasmtime returns the same results as a choice of another engine: the Wasm spec interpreter (see the `wasm-spec-interpreter` crate), the `wasmi` interpreter, or Wasmtime itself diff --git a/fuzz/fuzz_targets/differential.rs b/fuzz/fuzz_targets/differential.rs index e3e868ea00..9119981123 100644 --- a/fuzz/fuzz_targets/differential.rs +++ b/fuzz/fuzz_targets/differential.rs @@ -2,40 +2,108 @@ use libfuzzer_sys::arbitrary::{Result, Unstructured}; use libfuzzer_sys::fuzz_target; -use wasmtime_fuzzing::generators::InstanceAllocationStrategy; -use wasmtime_fuzzing::{generators, oracles}; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::SeqCst; +use std::sync::Once; +use wasmtime_fuzzing::generators::{Config, DiffValue, SingleInstModule}; +use wasmtime_fuzzing::oracles::diff_spec; +use wasmtime_fuzzing::oracles::diff_wasmtime::WasmtimeInstance; +use wasmtime_fuzzing::oracles::{differential, engine, log_wasm}; + +// Upper limit on the number of invocations for each WebAssembly function +// executed by this fuzz target. +const NUM_INVOCATIONS: usize = 5; + +// Keep track of how many WebAssembly modules we actually executed (i.e. ran to +// completion) versus how many were tried. +static TOTAL_INVOCATIONS: AtomicUsize = AtomicUsize::new(0); +static TOTAL_SUCCESSES: AtomicUsize = AtomicUsize::new(0); +static TOTAL_ATTEMPTED: AtomicUsize = AtomicUsize::new(0); + +// The spec interpreter requires special one-time setup. +static SETUP: Once = Once::new(); fuzz_target!(|data: &[u8]| { - // errors in `run` have to do with not enough input in `data`, which we + // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on + // `setup_ocaml_runtime`. + SETUP.call_once(|| diff_spec::setup_ocaml_runtime()); + + // Errors in `run` have to do with not enough input in `data`, which we // ignore here since it doesn't affect how we'd like to fuzz. - drop(run(data)); + drop(run(&data)); }); fn run(data: &[u8]) -> Result<()> { - let mut u = Unstructured::new(data); - - let mut config: generators::Config = u.arbitrary()?; - let module = config.generate(&mut u, Some(1000))?; - - let lhs = config.wasmtime; - let mut rhs: generators::WasmtimeConfig = u.arbitrary()?; - - // Use the same allocation strategy between the two configs. - // - // Ideally this wouldn't be necessary, but if the lhs is using ondemand - // and the rhs is using the pooling allocator (or vice versa), then - // the module may have been generated in such a way that is incompatible - // with the other allocation strategy. - // - // We can remove this in the future when it's possible to access the - // fields of `wasm_smith::Module` to constrain the pooling allocator - // based on what was actually generated. - rhs.strategy = lhs.strategy.clone(); - if let InstanceAllocationStrategy::Pooling { .. } = &rhs.strategy { - // Also use the same memory configuration when using the pooling allocator - rhs.memory_config = lhs.memory_config.clone(); + let successes = TOTAL_SUCCESSES.load(SeqCst); + let attempts = TOTAL_ATTEMPTED.fetch_add(1, SeqCst); + if attempts > 1 && attempts % 1_000 == 0 { + println!("=== Execution rate ({} successes / {} attempted modules): {}% (total invocations: {}) ===", + successes, + attempts, + successes as f64 / attempts as f64 * 100f64, + TOTAL_INVOCATIONS.load(SeqCst) + ); } - oracles::differential_execution(&module.to_bytes(), &config.module_config, &[lhs, rhs]); + let mut u = Unstructured::new(data); + let mut config: Config = u.arbitrary()?; + config.set_differential_config(); + + // Generate the Wasm module. + let wasm = if u.arbitrary()? { + // TODO figure out if this always eats up the rest of the unstructured; + // can we limit the number of instructions/functions. + let module = config.generate(&mut u, Some(1000))?; + module.to_bytes() + } else { + let module = SingleInstModule::new(&mut u, &mut config.module_config)?; + module.to_bytes() + }; + log_wasm(&wasm); + + // Choose a left-hand side Wasm engine. + let lhs = engine::choose(&mut u, &config)?; + let lhs_instance = lhs.instantiate(&wasm); + + // Choose a right-hand side Wasm engine--this will always be Wasmtime. + let rhs_store = config.to_store(); + let rhs_module = wasmtime::Module::new(rhs_store.engine(), &wasm).unwrap(); + let rhs_instance = WasmtimeInstance::new(rhs_store, rhs_module); + + // If we fail to instantiate, check that both sides do. + let (mut lhs_instance, mut rhs_instance) = match (lhs_instance, rhs_instance) { + (Ok(l), Ok(r)) => (l, r), + (Err(_), Err(_)) => return Ok(()), // TODO match the error messages. + (l, r) => panic!( + "failed to instantiate only one side: {:?} != {:?}", + l.err(), + r.err() + ), + }; + + // Call each exported function with different sets of arguments. + for (name, signature) in rhs_instance.exported_functions() { + let mut invocations = 0; + loop { + let arguments = signature + .params() + .map(|t| DiffValue::arbitrary_of_type(&mut u, t.try_into().unwrap())) + .collect::>>()?; + differential(lhs_instance.as_mut(), &mut rhs_instance, &name, &arguments) + .expect("failed to run differential evaluation"); + + // We evaluate the same function with different arguments until we + // hit a predetermined limit or we run out of unstructured data--it + // does not make sense to re-evaluate the same arguments over and + // over. + invocations += 1; + TOTAL_INVOCATIONS.fetch_add(1, SeqCst); + if invocations > NUM_INVOCATIONS || u.is_empty() { + break; + } + } + } + + TOTAL_SUCCESSES.fetch_add(1, SeqCst); Ok(()) } diff --git a/fuzz/fuzz_targets/differential_meta.rs b/fuzz/fuzz_targets/differential_meta.rs deleted file mode 100644 index 9119981123..0000000000 --- a/fuzz/fuzz_targets/differential_meta.rs +++ /dev/null @@ -1,109 +0,0 @@ -#![no_main] - -use libfuzzer_sys::arbitrary::{Result, Unstructured}; -use libfuzzer_sys::fuzz_target; -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering::SeqCst; -use std::sync::Once; -use wasmtime_fuzzing::generators::{Config, DiffValue, SingleInstModule}; -use wasmtime_fuzzing::oracles::diff_spec; -use wasmtime_fuzzing::oracles::diff_wasmtime::WasmtimeInstance; -use wasmtime_fuzzing::oracles::{differential, engine, log_wasm}; - -// Upper limit on the number of invocations for each WebAssembly function -// executed by this fuzz target. -const NUM_INVOCATIONS: usize = 5; - -// Keep track of how many WebAssembly modules we actually executed (i.e. ran to -// completion) versus how many were tried. -static TOTAL_INVOCATIONS: AtomicUsize = AtomicUsize::new(0); -static TOTAL_SUCCESSES: AtomicUsize = AtomicUsize::new(0); -static TOTAL_ATTEMPTED: AtomicUsize = AtomicUsize::new(0); - -// The spec interpreter requires special one-time setup. -static SETUP: Once = Once::new(); - -fuzz_target!(|data: &[u8]| { - // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on - // `setup_ocaml_runtime`. - SETUP.call_once(|| diff_spec::setup_ocaml_runtime()); - - // Errors in `run` have to do with not enough input in `data`, which we - // ignore here since it doesn't affect how we'd like to fuzz. - drop(run(&data)); -}); - -fn run(data: &[u8]) -> Result<()> { - let successes = TOTAL_SUCCESSES.load(SeqCst); - let attempts = TOTAL_ATTEMPTED.fetch_add(1, SeqCst); - if attempts > 1 && attempts % 1_000 == 0 { - println!("=== Execution rate ({} successes / {} attempted modules): {}% (total invocations: {}) ===", - successes, - attempts, - successes as f64 / attempts as f64 * 100f64, - TOTAL_INVOCATIONS.load(SeqCst) - ); - } - - let mut u = Unstructured::new(data); - let mut config: Config = u.arbitrary()?; - config.set_differential_config(); - - // Generate the Wasm module. - let wasm = if u.arbitrary()? { - // TODO figure out if this always eats up the rest of the unstructured; - // can we limit the number of instructions/functions. - let module = config.generate(&mut u, Some(1000))?; - module.to_bytes() - } else { - let module = SingleInstModule::new(&mut u, &mut config.module_config)?; - module.to_bytes() - }; - log_wasm(&wasm); - - // Choose a left-hand side Wasm engine. - let lhs = engine::choose(&mut u, &config)?; - let lhs_instance = lhs.instantiate(&wasm); - - // Choose a right-hand side Wasm engine--this will always be Wasmtime. - let rhs_store = config.to_store(); - let rhs_module = wasmtime::Module::new(rhs_store.engine(), &wasm).unwrap(); - let rhs_instance = WasmtimeInstance::new(rhs_store, rhs_module); - - // If we fail to instantiate, check that both sides do. - let (mut lhs_instance, mut rhs_instance) = match (lhs_instance, rhs_instance) { - (Ok(l), Ok(r)) => (l, r), - (Err(_), Err(_)) => return Ok(()), // TODO match the error messages. - (l, r) => panic!( - "failed to instantiate only one side: {:?} != {:?}", - l.err(), - r.err() - ), - }; - - // Call each exported function with different sets of arguments. - for (name, signature) in rhs_instance.exported_functions() { - let mut invocations = 0; - loop { - let arguments = signature - .params() - .map(|t| DiffValue::arbitrary_of_type(&mut u, t.try_into().unwrap())) - .collect::>>()?; - differential(lhs_instance.as_mut(), &mut rhs_instance, &name, &arguments) - .expect("failed to run differential evaluation"); - - // We evaluate the same function with different arguments until we - // hit a predetermined limit or we run out of unstructured data--it - // does not make sense to re-evaluate the same arguments over and - // over. - invocations += 1; - TOTAL_INVOCATIONS.fetch_add(1, SeqCst); - if invocations > NUM_INVOCATIONS || u.is_empty() { - break; - } - } - } - - TOTAL_SUCCESSES.fetch_add(1, SeqCst); - Ok(()) -}