diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e827508b63..acaa482bd1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -196,7 +196,8 @@ jobs: with: toolchain: nightly-2022-04-27 - run: cargo install cargo-fuzz --vers "^0.11" - # Install OCaml packages necessary for 'differential_spec' fuzz target. + # Install the OCaml packages necessary for fuzz targets that use the + # `wasm-spec-interpreter`. - run: sudo apt install -y ocaml-nox ocamlbuild ocaml-findlib libzarith-ocaml-dev - run: cargo fetch working-directory: ./fuzz diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 287f1b332d..cc3bbf67fd 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -22,7 +22,6 @@ use self::diff_wasmtime::WasmtimeInstance; use self::engine::DiffInstance; use crate::generators::{self, DiffValue}; use arbitrary::Arbitrary; -use log::debug; pub use stacks::check_stacks; use std::cell::Cell; use std::collections::hash_map::DefaultHasher; @@ -832,254 +831,6 @@ fn table_ops_eventually_gcs() { panic!("after {n} runs nothing ever gc'd, something is probably wrong"); } -/// Perform differential execution between Cranelift and wasmi, diffing the -/// resulting memory image when execution terminates. This relies on the -/// module-under-test to be instrumented to bound the execution time. Invoke -/// with a module generated by `wasm-smith` using the -/// `SingleFunctionModuleConfig` configuration type for best results. -/// -/// May return `None` if we early-out due to a rejected fuzz config; these -/// should be rare if modules are generated appropriately. -pub fn differential_wasmi_execution(wasm: &[u8], config: &generators::Config) -> Option<()> { - crate::init_fuzzing(); - log_wasm(wasm); - - // Instantiate wasmi module and instance. - let wasmi_module = wasmi::Module::from_buffer(&wasm[..]).ok()?; - let wasmi_instance = - wasmi::ModuleInstance::new(&wasmi_module, &wasmi::ImportsBuilder::default()).ok()?; - let wasmi_instance = wasmi_instance.assert_no_start(); - - // If wasmi succeeded then we assert that wasmtime will also succeed. - let (wasmtime_module, mut wasmtime_store) = differential_store(wasm, config); - let wasmtime_module = wasmtime_module?; - let wasmtime_instance = Instance::new(&mut wasmtime_store, &wasmtime_module, &[]) - .expect("Wasmtime can instantiate module"); - - // Introspect wasmtime module to find name of an exported function and of an - // exported memory. - let (func_name, ty) = first_exported_function(&wasmtime_module)?; - - let wasmi_main_export = wasmi_instance.export_by_name(func_name).unwrap(); - let wasmi_main = wasmi_main_export.as_func().unwrap(); - let wasmi_val = wasmi::FuncInstance::invoke(&wasmi_main, &[], &mut wasmi::NopExternals); - - let wasmtime_main = wasmtime_instance - .get_func(&mut wasmtime_store, func_name) - .expect("function export is present"); - let mut wasmtime_results = vec![Val::I32(0); ty.results().len()]; - let wasmtime_val = wasmtime_main - .call(&mut wasmtime_store, &[], &mut wasmtime_results) - .map(|()| wasmtime_results.get(0).cloned()); - - debug!( - "Successful execution: wasmi returned {:?}, wasmtime returned {:?}", - wasmi_val, wasmtime_val - ); - - match (&wasmi_val, &wasmtime_val) { - (&Ok(Some(wasmi::RuntimeValue::I32(a))), &Ok(Some(Val::I32(b)))) if a == b => {} - (&Ok(Some(wasmi::RuntimeValue::F32(a))), &Ok(Some(Val::F32(b)))) - if f32_equal(a.to_bits(), b) => {} - (&Ok(Some(wasmi::RuntimeValue::I64(a))), &Ok(Some(Val::I64(b)))) if a == b => {} - (&Ok(Some(wasmi::RuntimeValue::F64(a))), &Ok(Some(Val::F64(b)))) - if f64_equal(a.to_bits(), b) => {} - (&Ok(None), &Ok(None)) => {} - (&Err(_), &Err(_)) => {} - _ => { - panic!( - "Values do not match: wasmi returned {:?}; wasmtime returned {:?}", - wasmi_val, wasmtime_val - ); - } - } - - // Compare linear memories if there's an exported linear memory - let memory_name = match first_exported_memory(&wasmtime_module) { - Some(name) => name, - None => return Some(()), - }; - let wasmi_mem_export = wasmi_instance.export_by_name(memory_name).unwrap(); - let wasmi_mem = wasmi_mem_export.as_memory().unwrap(); - let wasmtime_mem = wasmtime_instance - .get_memory(&mut wasmtime_store, memory_name) - .expect("memory export is present"); - - if wasmi_mem.current_size().0 != wasmtime_mem.size(&wasmtime_store) as usize { - panic!("resulting memories are not the same size"); - } - - // Wasmi memory may be stored non-contiguously; copy it out to a contiguous chunk. - let mut wasmi_buf: Vec = vec![0; wasmtime_mem.data_size(&wasmtime_store)]; - wasmi_mem - .get_into(0, &mut wasmi_buf[..]) - .expect("can access wasmi memory"); - - let wasmtime_slice = wasmtime_mem.data(&wasmtime_store); - - if wasmi_buf.len() >= 64 { - debug!("-> First 64 bytes of wasmi heap: {:?}", &wasmi_buf[0..64]); - debug!( - "-> First 64 bytes of Wasmtime heap: {:?}", - &wasmtime_slice[0..64] - ); - } - - if &wasmi_buf[..] != &wasmtime_slice[..] { - panic!("memory contents are not equal"); - } - - Some(()) -} - -/// Perform differential execution between Wasmtime and the official WebAssembly -/// specification interpreter. -/// -/// May return `None` if we early-out due to a rejected fuzz config. -#[cfg(feature = "fuzz-spec-interpreter")] -pub fn differential_spec_execution(wasm: &[u8], config: &generators::Config) -> Option<()> { - use anyhow::Context; - - crate::init_fuzzing(); - debug!("config: {:#?}", config); - log_wasm(wasm); - - // Run the spec interpreter first, then Wasmtime. The order is important - // because both sides (OCaml runtime and Wasmtime) register signal handlers; - // Wasmtime uses these signal handlers for catching various WebAssembly - // failures. On certain OSes (e.g. Linux x86_64), the signal handlers - // interfere, observable as an uncaught `SIGSEGV`--not even caught by - // libFuzzer. By running Wasmtime second, its signal handlers are registered - // most recently and they catch failures appropriately. - // - // For now, execute with dummy (zeroed) function arguments. - let spec_vals = wasm_spec_interpreter::interpret(wasm, None); - debug!("spec interpreter returned: {:?}", &spec_vals); - - let (wasmtime_module, mut wasmtime_store) = differential_store(wasm, config); - let wasmtime_module = match wasmtime_module { - Some(m) => m, - None => return None, - }; - - let wasmtime_vals = - Instance::new(&mut wasmtime_store, &wasmtime_module, &[]).and_then(|wasmtime_instance| { - // Find the first exported function. - let (func_name, ty) = first_exported_function(&wasmtime_module) - .context("Cannot find exported function")?; - let wasmtime_main = wasmtime_instance - .get_func(&mut wasmtime_store, &func_name[..]) - .expect("function export is present"); - - let dummy_params = dummy::dummy_values(ty.params()); - - // Execute the function and return the values. - let mut results = vec![Val::I32(0); ty.results().len()]; - wasmtime_main - .call(&mut wasmtime_store, &dummy_params, &mut results) - .map(|()| Some(results)) - }); - - // Match a spec interpreter value against a Wasmtime value. Eventually this - // should support references and `v128` (TODO). - fn matches(spec_val: &wasm_spec_interpreter::Value, wasmtime_val: &wasmtime::Val) -> bool { - match (spec_val, wasmtime_val) { - (wasm_spec_interpreter::Value::I32(a), wasmtime::Val::I32(b)) => a == b, - (wasm_spec_interpreter::Value::I64(a), wasmtime::Val::I64(b)) => a == b, - (wasm_spec_interpreter::Value::F32(a), wasmtime::Val::F32(b)) => { - f32_equal(*a as u32, *b) - } - (wasm_spec_interpreter::Value::F64(a), wasmtime::Val::F64(b)) => { - f64_equal(*a as u64, *b) - } - (wasm_spec_interpreter::Value::V128(a), wasmtime::Val::V128(b)) => { - assert_eq!(a.len(), 16); - let a_num = u128::from_le_bytes(a.as_slice().try_into().unwrap()); - a_num == *b - } - (_, _) => { - unreachable!("TODO: only fuzzing of scalar and vector value types is supported") - } - } - } - - match (&spec_vals, &wasmtime_vals) { - // Compare the returned values, failing if they do not match. - (Ok(spec_vals), Ok(Some(wasmtime_vals))) => { - let all_match = spec_vals - .iter() - .zip(wasmtime_vals) - .all(|(s, w)| matches(s, w)); - if !all_match { - panic!( - "Values do not match: spec returned {:?}; wasmtime returned {:?}", - spec_vals, wasmtime_vals - ); - } - } - (_, Ok(None)) => { - // `run_in_wasmtime` rejected the config - return None; - } - // If both sides fail, skip this fuzz execution. - (Err(spec_error), Err(wasmtime_error)) => { - // The `None` value returned here indicates that both sides - // failed--if we see too many of these we might be failing too often - // to check instruction semantics. At some point it would be - // beneficial to compare the error messages from both sides (TODO). - // It would also be good to keep track of statistics about the - // ratios of the kinds of errors the fuzzer sees (TODO). - log::warn!( - "Both sides failed: spec returned '{}'; wasmtime returned {:?}", - spec_error, - wasmtime_error - ); - return None; - } - // If only one side fails, fail the fuzz the test. - _ => { - panic!( - "Only one side failed: spec returned {:?}; wasmtime returned {:?}", - &spec_vals, &wasmtime_vals - ); - } - } - - // TODO Compare memory contents. - - Some(()) -} - -fn differential_store( - wasm: &[u8], - fuzz_config: &generators::Config, -) -> (Option, Store) { - let store = fuzz_config.to_store(); - let module = compile_module(store.engine(), wasm, true, fuzz_config); - (module, store) -} - -// Introspect wasmtime module to find the name of the first exported function. -fn first_exported_function(module: &wasmtime::Module) -> Option<(&str, FuncType)> { - for e in module.exports() { - match e.ty() { - wasmtime::ExternType::Func(ty) => return Some((e.name(), ty)), - _ => {} - } - } - None -} - -fn first_exported_memory(module: &Module) -> Option<&str> { - for e in module.exports() { - match e.ty() { - wasmtime::ExternType::Memory(..) => return Some(e.name()), - _ => {} - } - } - None -} - #[derive(Default)] struct SignalOnDrop { state: Arc<(Mutex, Condvar)>, diff --git a/crates/fuzzing/src/oracles/v8.rs b/crates/fuzzing/src/oracles/v8.rs index 984b706937..6621511ec4 100644 --- a/crates/fuzzing/src/oracles/v8.rs +++ b/crates/fuzzing/src/oracles/v8.rs @@ -1,4 +1,5 @@ -use super::{first_exported_function, first_exported_memory, log_wasm}; +use super::{compile_module, log_wasm}; +use crate::generators; use std::convert::TryFrom; use std::sync::Once; use wasmtime::*; @@ -16,7 +17,7 @@ use wasmtime::*; pub fn differential_v8_execution(wasm: &[u8], config: &crate::generators::Config) -> Option<()> { // Wasmtime setup log_wasm(wasm); - let (wasmtime_module, mut wasmtime_store) = super::differential_store(wasm, config); + let (wasmtime_module, mut wasmtime_store) = differential_store(wasm, config); let wasmtime_module = wasmtime_module?; log::trace!("compiled module with wasmtime"); @@ -339,3 +340,33 @@ fn assert_error_matches(wasmtime: &anyhow::Error, v8: &str) { } verify_wasmtime("not possibly present in an error, just panic please"); } + +fn differential_store( + wasm: &[u8], + fuzz_config: &generators::Config, +) -> (Option, Store) { + let store = fuzz_config.to_store(); + let module = compile_module(store.engine(), wasm, true, fuzz_config); + (module, store) +} + +// Introspect wasmtime module to find the name of the first exported function. +fn first_exported_function(module: &wasmtime::Module) -> Option<(&str, FuncType)> { + for e in module.exports() { + match e.ty() { + wasmtime::ExternType::Func(ty) => return Some((e.name(), ty)), + _ => {} + } + } + None +} + +fn first_exported_memory(module: &Module) -> Option<&str> { + for e in module.exports() { + match e.ty() { + wasmtime::ExternType::Memory(..) => return Some(e.name()), + _ => {} + } + } + None +} diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 1c4ece1cd3..e98d19a734 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -66,20 +66,6 @@ path = "fuzz_targets/differential_meta.rs" test = false doc = false - -[[bin]] -name = "differential_spec" -path = "fuzz_targets/differential_spec.rs" -test = false -doc = false -required-features = ['fuzz-spec-interpreter'] - -[[bin]] -name = "differential_wasmi" -path = "fuzz_targets/differential_wasmi.rs" -test = false -doc = false - [[bin]] name = "differential_v8" path = "fuzz_targets/differential_v8.rs" diff --git a/fuzz/README.md b/fuzz/README.md index 5d8e53b02f..40021f386c 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -36,13 +36,13 @@ At the time of writing, we have the following fuzz targets: from scratch. * `differential`: Generate a Wasm module and check that Wasmtime returns the same results when run with two different configurations. -* `differential_spec`: Generate a Wasm module and check that Wasmtime returns - the same results as the Wasm spec interpreter (see the `wasm-spec-interpreter` - crate). +* `differential_meta`: Generate a Wasm module, evaluate each exported function + with random inputs, and check that Wasmtime returns the same results as a + choice of another engine: the Wasm spec interpreter (see the + `wasm-spec-interpreter` crate), the `wasmi` interpreter, or Wasmtime itself + run with a different configuration. * `differential_v8`: Generate a Wasm module and check that Wasmtime returns the same results as V8. -* `differential_wasmi`: Generate a Wasm module and check that Wasmtime returns - the same results as the `wasmi` interpreter. * `instantiate`: Generate a Wasm module and Wasmtime configuration and attempt to compile and instantiate with them. * `instantiate-many`: Generate many Wasm modules and attempt to compile and diff --git a/fuzz/fuzz_targets/differential_spec.rs b/fuzz/fuzz_targets/differential_spec.rs deleted file mode 100644 index 96f6ad84f8..0000000000 --- a/fuzz/fuzz_targets/differential_spec.rs +++ /dev/null @@ -1,47 +0,0 @@ -#![no_main] - -use libfuzzer_sys::arbitrary::{Result, Unstructured}; -use libfuzzer_sys::fuzz_target; -use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; -use wasmtime_fuzzing::{generators, oracles}; - -// Keep track of how many WebAssembly modules we actually executed (i.e. ran to -// completion) versus how many were tried. -static TRIED: AtomicUsize = AtomicUsize::new(0); -static EXECUTED: AtomicUsize = AtomicUsize::new(0); - -fuzz_target!(|data: &[u8]| { - // errors in `run` have to do with not enough input in `data`, which we - // ignore here since it doesn't affect how we'd like to fuzz. - drop(run(data)); -}); - -fn run(data: &[u8]) -> Result<()> { - let mut u = Unstructured::new(data); - let mut config: generators::Config = u.arbitrary()?; - config.set_differential_config(); - - // Enable features that the spec interpreter has implemented - config.module_config.config.simd_enabled = true; - - // TODO: this is a best-effort attempt to avoid errors caused by the - // generated module exporting no functions. - config.module_config.config.min_exports = 5; - config.module_config.config.max_exports = 5; - - let module = config.generate(&mut u, Some(1000))?; - let tried = TRIED.fetch_add(1, SeqCst); - let executed = match oracles::differential_spec_execution(&module.to_bytes(), &config) { - Some(_) => EXECUTED.fetch_add(1, SeqCst), - None => EXECUTED.load(SeqCst), - }; - if tried > 0 && tried % 1000 == 0 { - println!( - "=== Execution rate ({} executed modules / {} tried modules): {}% ===", - executed, - tried, - executed as f64 / tried as f64 * 100f64 - ) - } - Ok(()) -} diff --git a/fuzz/fuzz_targets/differential_wasmi.rs b/fuzz/fuzz_targets/differential_wasmi.rs deleted file mode 100644 index fe02ac2509..0000000000 --- a/fuzz/fuzz_targets/differential_wasmi.rs +++ /dev/null @@ -1,20 +0,0 @@ -#![no_main] - -use libfuzzer_sys::arbitrary::{Result, Unstructured}; -use libfuzzer_sys::fuzz_target; -use wasmtime_fuzzing::{generators, oracles}; - -fuzz_target!(|data: &[u8]| { - // errors in `run` have to do with not enough input in `data`, which we - // ignore here since it doesn't affect how we'd like to fuzz. - drop(run(data)); -}); - -fn run(data: &[u8]) -> Result<()> { - let mut u = Unstructured::new(data); - let mut config: generators::Config = u.arbitrary()?; - config.set_differential_config(); - let module = config.generate(&mut u, Some(1000))?; - oracles::differential_wasmi_execution(&module.to_bytes(), &config); - Ok(()) -}