From bbdea06e2d22b85cdacf5406171bccf65166031b Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Wed, 2 Dec 2020 14:52:44 -0800 Subject: [PATCH] Add differential fuzzing against wasmi (a Wasm interpreter). This PR adds a new fuzz target, `differential_wasmi`, that runs a Cranelift-based Wasm backend alongside a simple third-party Wasm interpeter crate (`wasmi`). The fuzzing runs the first function in a given module to completion on each side, and then diffs the return value and linear memory contents. This strategy should provide end-to-end coverage including both the Wasm translation to CLIF (which has seen some subtle and scary bugs at times), the lowering from CLIF to VCode, the register allocation, and the final code emission. This PR also adds a feature `experimental_x64` to the fuzzing crate (and the chain of dependencies down to `cranelift-codegen`) so that we can fuzz the new x86-64 backend as well as the current one. --- Cargo.lock | 82 +++++++++- crates/fuzzing/Cargo.toml | 6 +- crates/fuzzing/src/oracles.rs | 202 +++++++++++++++++++++--- crates/wasmtime/Cargo.toml | 3 + fuzz/Cargo.toml | 11 +- fuzz/fuzz_targets/differential_wasmi.rs | 13 ++ 6 files changed, 294 insertions(+), 23 deletions(-) create mode 100644 fuzz/fuzz_targets/differential_wasmi.rs diff --git a/Cargo.lock b/Cargo.lock index 40c4034ef9..2044cd3a6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -766,6 +766,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + [[package]] name = "dynasm" version = "1.0.0" @@ -1238,6 +1244,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memory_units" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" + [[package]] name = "miniz_oxide" version = "0.4.3" @@ -1264,6 +1276,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1274,6 +1297,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -1352,6 +1387,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "parity-wasm" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -2313,13 +2354,47 @@ dependencies = [ ] [[package]] -name = "wasm-smith" -version = "0.1.10" +name = "wasm-encoder" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd5e4720bb44dc5e46a917139dc9dfa4bb6fee023bf9f77d2c55ec12eeaf9930" +checksum = "49891b7c581cf9e0090b25cd274e6498ad478f0a7819319ea96da2f253caaacb" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-smith" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fdf8c9ba2fdc0d8ffe3f7e5b23b5aac377eeca817a9885c058f27c8de5c500" dependencies = [ "arbitrary", "leb128", + "wasm-encoder", +] + +[[package]] +name = "wasmi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6825d9b2147105789adb4c2d84b9b568719713f3ac39618b637b4dafc86c4" +dependencies = [ + "downcast-rs", + "libc", + "memory_units", + "num-rational", + "num-traits", + "parity-wasm", + "wasmi-validation", +] + +[[package]] +name = "wasmi-validation" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" +dependencies = [ + "parity-wasm", ] [[package]] @@ -2522,6 +2597,7 @@ dependencies = [ "log", "rayon", "wasm-smith", + "wasmi", "wasmparser 0.68.0", "wasmprinter", "wasmtime", diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index c5625d293b..4526982823 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -16,7 +16,11 @@ wasmparser = "0.68.0" wasmprinter = "0.2.15" wasmtime = { path = "../wasmtime" } wasmtime-wast = { path = "../wast" } -wasm-smith = "0.1.10" +wasm-smith = "0.1.12" +wasmi = "0.7.0" [dev-dependencies] wat = "1.0.28" + +[features] +experimental_x64 = ["wasmtime/experimental_x64"] diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index cbd0d97107..2fb6c58682 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -12,7 +12,9 @@ pub mod dummy; +use arbitrary::Arbitrary; use dummy::dummy_imports; +use log::debug; use std::cell::Cell; use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; @@ -249,24 +251,8 @@ pub fn differential_execution( (Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue, (Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue, (Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue, - (Val::F32(lhs), Val::F32(rhs)) => { - let lhs = f32::from_bits(*lhs); - let rhs = f32::from_bits(*rhs); - if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { - continue; - } else { - fail() - } - } - (Val::F64(lhs), Val::F64(rhs)) => { - let lhs = f64::from_bits(*lhs); - let rhs = f64::from_bits(*rhs); - if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { - continue; - } else { - fail() - } - } + (Val::F32(lhs), Val::F32(rhs)) if f32_equal(*lhs, *rhs) => continue, + (Val::F64(lhs), Val::F64(rhs)) if f64_equal(*lhs, *rhs) => continue, (Val::ExternRef(_), Val::ExternRef(_)) | (Val::FuncRef(_), Val::FuncRef(_)) => continue, _ => fail(), @@ -278,6 +264,18 @@ pub fn differential_execution( } } +fn f32_equal(a: u32, b: u32) -> bool { + let a = f32::from_bits(a); + let b = f32::from_bits(b); + a == b || (a.is_nan() && b.is_nan()) +} + +fn f64_equal(a: u64, b: u64) -> bool { + let a = f64::from_bits(a); + let b = f64::from_bits(b); + a == b || (a.is_nan() && b.is_nan()) +} + /// Invoke the given API calls. pub fn make_api_calls(api: crate::generators::api::ApiCalls) { use crate::generators::api::ApiCall; @@ -479,3 +477,171 @@ pub fn table_ops(config: crate::generators::Config, ops: crate::generators::tabl } } } + +/// Configuration options for wasm-smith such that generated modules always +/// conform to certain specifications. +#[derive(Default, Debug, Arbitrary)] +pub struct DifferentialWasmiModuleConfig; + +impl wasm_smith::Config for DifferentialWasmiModuleConfig { + fn allow_start_export(&self) -> bool { + false + } + + fn min_funcs(&self) -> usize { + 1 + } + + fn max_funcs(&self) -> usize { + 1 + } + + fn min_memories(&self) -> u32 { + 1 + } + + fn max_memories(&self) -> u32 { + 1 + } + + fn max_imports(&self) -> usize { + 0 + } + + fn min_exports(&self) -> usize { + 2 + } + + fn max_memory_pages(&self) -> u32 { + 1 + } + + fn memory_max_size_required(&self) -> bool { + true + } +} + +/// Perform differential execution between Cranelift and wasmi, diffing the +/// resulting memory image when execution terminates. This relies on the +/// module-under-test to be instrumented to bound the execution time. Invoke +/// with a module generated by `wasm-smith` using the +/// `DiferentialWasmiModuleConfig` configuration type for best results. +/// +/// May return `None` if we early-out due to a rejected fuzz config; these +/// should be rare if modules are generated appropriately. +pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Config) -> Option<()> { + crate::init_fuzzing(); + + // Instantiate wasmi module and instance. + let wasmi_module = wasmi::Module::from_buffer(&wasm[..]).ok()?; + let wasmi_instance = + wasmi::ModuleInstance::new(&wasmi_module, &wasmi::ImportsBuilder::default()).ok()?; + let wasmi_instance = wasmi_instance.assert_no_start(); + + // TODO(paritytech/wasmi#19): wasmi does not currently canonicalize NaNs. To avoid spurious + // fuzz failures, for now let's fuzz only integer Wasm programs. + if wasmi_module.deny_floating_point().is_err() { + return None; + } + + // Instantiate wasmtime module and instance. + let mut wasmtime_config = config.to_wasmtime(); + wasmtime_config.cranelift_nan_canonicalization(true); + let wasmtime_engine = Engine::new(&wasmtime_config); + let wasmtime_store = Store::new(&wasmtime_engine); + let wasmtime_module = + Module::new(&wasmtime_engine, &wasm).expect("Wasmtime can compile module"); + let wasmtime_instance = Instance::new(&wasmtime_store, &wasmtime_module, &[]) + .expect("Wasmtime can instantiate module"); + + // Introspect wasmtime module to find name of an exported function and of an + // exported memory. Stop when we have one of each. (According to the config + // above, there should be at most one of each.) + let (func_name, memory_name) = { + let mut func_name = None; + let mut memory_name = None; + for e in wasmtime_module.exports() { + match e.ty() { + wasmtime::ExternType::Func(..) => func_name = Some(e.name().to_string()), + wasmtime::ExternType::Memory(..) => memory_name = Some(e.name().to_string()), + _ => {} + } + if func_name.is_some() && memory_name.is_some() { + break; + } + } + (func_name?, memory_name?) + }; + + let wasmi_mem_export = wasmi_instance.export_by_name(&memory_name[..]).unwrap(); + let wasmi_mem = wasmi_mem_export.as_memory().unwrap(); + let wasmi_main_export = wasmi_instance.export_by_name(&func_name[..]).unwrap(); + let wasmi_main = wasmi_main_export.as_func().unwrap(); + let wasmi_val = wasmi::FuncInstance::invoke(&wasmi_main, &[], &mut wasmi::NopExternals); + + let wasmtime_mem = wasmtime_instance + .get_memory(&memory_name[..]) + .expect("memory export is present"); + let wasmtime_main = wasmtime_instance + .get_func(&func_name[..]) + .expect("function export is present"); + let wasmtime_vals = wasmtime_main.call(&[]); + let wasmtime_val = wasmtime_vals.map(|v| v.iter().next().cloned()); + + debug!( + "Successful execution: wasmi returned {:?}, wasmtime returned {:?}", + wasmi_val, wasmtime_val + ); + + let show_wat = || { + if let Ok(s) = wasmprinter::print_bytes(&wasm[..]) { + eprintln!("wat:\n{}\n", s); + } + }; + + match (&wasmi_val, &wasmtime_val) { + (&Ok(Some(wasmi::RuntimeValue::I32(a))), &Ok(Some(Val::I32(b)))) if a == b => {} + (&Ok(Some(wasmi::RuntimeValue::F32(a))), &Ok(Some(Val::F32(b)))) + if f32_equal(a.to_bits(), b) => {} + (&Ok(Some(wasmi::RuntimeValue::I64(a))), &Ok(Some(Val::I64(b)))) if a == b => {} + (&Ok(Some(wasmi::RuntimeValue::F64(a))), &Ok(Some(Val::F64(b)))) + if f64_equal(a.to_bits(), b) => {} + (&Ok(None), &Ok(None)) => {} + (&Err(_), &Err(_)) => {} + _ => { + show_wat(); + panic!( + "Values do not match: wasmi returned {:?}; wasmtime returned {:?}", + wasmi_val, wasmtime_val + ); + } + } + + if wasmi_mem.current_size().0 != wasmtime_mem.size() as usize { + show_wat(); + panic!("resulting memories are not the same size"); + } + + // Wasmi memory may be stored non-contiguously; copy it out to a contiguous chunk. + let mut wasmi_buf: Vec = vec![0; wasmtime_mem.data_size()]; + wasmi_mem + .get_into(0, &mut wasmi_buf[..]) + .expect("can access wasmi memory"); + + let wasmtime_slice = unsafe { wasmtime_mem.data_unchecked() }; + + if wasmi_buf.len() >= 64 { + debug!("-> First 64 bytes of wasmi heap: {:?}", &wasmi_buf[0..64]); + debug!( + "-> First 64 bytes of Wasmtime heap: {:?}", + &wasmtime_slice[0..64] + ); + } + + if &wasmi_buf[..] != &wasmtime_slice[..] { + show_wat(); + panic!("memory contents are not equal"); + } + + Some(()) +} diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 28de26001e..57cdec42a5 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -59,3 +59,6 @@ parallel-compilation = ["wasmtime-jit/parallel-compilation"] # Enables support for automatic cache configuration to be enabled in `Config`. cache = ["wasmtime-cache"] + +# Enables support for new x64 backend. +experimental_x64 = ["wasmtime-jit/experimental_x64"] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index f295655755..32726a7940 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -17,7 +17,10 @@ target-lexicon = "0.11" peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing", optional = true } wasmtime = { path = "../crates/wasmtime" } wasmtime-fuzzing = { path = "../crates/fuzzing" } -wasm-smith = "0.1.5" +wasm-smith = "0.1.12" + +[features] +experimental_x64 = ["wasmtime-fuzzing/experimental_x64"] [[bin]] name = "compile" @@ -43,6 +46,12 @@ path = "fuzz_targets/differential.rs" test = false doc = false +[[bin]] +name = "differential_wasmi" +path = "fuzz_targets/differential_wasmi.rs" +test = false +doc = false + [[bin]] name = "spectests" path = "fuzz_targets/spectests.rs" diff --git a/fuzz/fuzz_targets/differential_wasmi.rs b/fuzz/fuzz_targets/differential_wasmi.rs new file mode 100644 index 0000000000..e4e8fffead --- /dev/null +++ b/fuzz/fuzz_targets/differential_wasmi.rs @@ -0,0 +1,13 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use wasmtime_fuzzing::{generators, oracles}; + +fuzz_target!(|data: ( + generators::Config, + wasm_smith::ConfiguredModule +)| { + let (config, mut wasm) = data; + wasm.ensure_termination(1000); + oracles::differential_wasmi_execution(&wasm.to_bytes()[..], &config); +});