From b0cf8c021f65d87bf36e44b65287eaf2ab403b8f Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 17 Mar 2020 09:29:48 -0700 Subject: [PATCH 1/5] Turn off binaryen in fuzzing by default ... but turn it back on in CI by default. The `binaryen-sys` crate builds binaryen from source, which is a drag on CI for a few reasons: * This is quite large and takes a good deal of time to build * The debug build directory for binaryen is 4GB large In an effort to both save time and disk space on the builders this commit adds a `binaryen` feature to the `wasmtime-fuzz` crate. This feature is enabled specifically when running the fuzzers on CI, but it is disabled during the typical `cargo test --all` command. This means that the test builders should save an extra 4G of space and be a bit speedier now that they don't build a giant wad of C++. We'll need to update the OSS-fuzz integration to enable the `binaryen` feature when executing `cargo fuzz build`, and I'll do that once this gets closer to landing. --- .github/workflows/main.yml | 2 +- crates/fuzzing/Cargo.toml | 2 +- crates/fuzzing/src/generators.rs | 15 +++++++++------ crates/fuzzing/src/oracles.rs | 7 ++++--- fuzz/Cargo.toml | 6 ++++++ fuzz/fuzz_targets/differential.rs | 2 +- 6 files changed, 22 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b9131355b..31d53cd738 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,7 +79,7 @@ jobs: - run: cargo install cargo-fuzz --vers "^0.7" - run: cargo fetch working-directory: ./fuzz - - run: cargo fuzz build --release --debug-assertions + - run: cargo fuzz build --release --debug-assertions --features binaryen # Our corpora are too large to run in full on every pull request, they just # take too long. Instead, we sample some of them and make sure that running # our fuzzers over the sampled inputs still works OK. diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index 807ac2bae9..872c623462 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -9,7 +9,7 @@ version = "0.12.0" [dependencies] anyhow = "1.0.22" arbitrary = { version = "0.4.0", features = ["derive"] } -binaryen = "0.10.0" +binaryen = { version = "0.10.0", optional = true } env_logger = "0.7.1" log = "0.4.8" rayon = "1.2.1" diff --git a/crates/fuzzing/src/generators.rs b/crates/fuzzing/src/generators.rs index b2504fc3e1..15715b3a44 100644 --- a/crates/fuzzing/src/generators.rs +++ b/crates/fuzzing/src/generators.rs @@ -8,20 +8,22 @@ //! wrapper over an external tool, such that the wrapper implements the //! `Arbitrary` trait for the wrapped external tool. +#[cfg(feature = "binaryen")] pub mod api; -use arbitrary::{Arbitrary, Unstructured}; -use std::fmt; +use arbitrary::Arbitrary; /// A Wasm test case generator that is powered by Binaryen's `wasm-opt -ttf`. #[derive(Clone)] +#[cfg(feature = "binaryen")] pub struct WasmOptTtf { /// The raw, encoded Wasm bytes. pub wasm: Vec, } -impl fmt::Debug for WasmOptTtf { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { +#[cfg(feature = "binaryen")] +impl std::fmt::Debug for WasmOptTtf { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( f, "WasmOptTtf {{ wasm: wat::parse_str(r###\"\n{}\n\"###).unwrap() }}", @@ -30,8 +32,9 @@ impl fmt::Debug for WasmOptTtf { } } +#[cfg(feature = "binaryen")] impl Arbitrary for WasmOptTtf { - fn arbitrary(input: &mut Unstructured) -> arbitrary::Result { + fn arbitrary(input: &mut arbitrary::Unstructured) -> arbitrary::Result { crate::init_fuzzing(); let seed: Vec = Arbitrary::arbitrary(input)?; let module = binaryen::tools::translate_to_fuzz_mvp(&seed); @@ -39,7 +42,7 @@ impl Arbitrary for WasmOptTtf { Ok(WasmOptTtf { wasm }) } - fn arbitrary_take_rest(input: Unstructured) -> arbitrary::Result { + fn arbitrary_take_rest(input: arbitrary::Unstructured) -> arbitrary::Result { crate::init_fuzzing(); let seed: Vec = Arbitrary::arbitrary_take_rest(input)?; let module = binaryen::tools::translate_to_fuzz_mvp(&seed); diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 3d7eb6eef2..361e5b2b70 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -110,7 +110,7 @@ pub fn compile(wasm: &[u8], strategy: Strategy) { /// or aren't enabled for different configs, we should get the same results when /// we call the exported functions for all of our different configs. pub fn differential_execution( - ttf: &crate::generators::WasmOptTtf, + wasm: &[u8], configs: &[crate::generators::DifferentialConfig], ) { crate::init_fuzzing(); @@ -131,13 +131,13 @@ pub fn differential_execution( }; let mut export_func_results: HashMap, Trap>> = Default::default(); - log_wasm(&ttf.wasm); + log_wasm(wasm); for config in &configs { let engine = Engine::new(config); let store = Store::new(&engine); - let module = match Module::new(&store, &ttf.wasm) { + let module = match Module::new(&store, wasm) { Ok(module) => module, // The module might rely on some feature that our config didn't // enable or something like that. @@ -283,6 +283,7 @@ fn assert_same_export_func_result( } /// Invoke the given API calls. +#[cfg(feature = "binaryen")] pub fn make_api_calls(api: crate::generators::api::ApiCalls) { use crate::generators::api::ApiCall; diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index b8f1a527d4..f32af6f4c0 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -34,15 +34,21 @@ name = "instantiate_translated" path = "fuzz_targets/instantiate_translated.rs" test = false doc = false +required-features = ['binaryen'] [[bin]] name = "api_calls" path = "fuzz_targets/api_calls.rs" test = false doc = false +required-features = ['binaryen'] [[bin]] name = "differential" path = "fuzz_targets/differential.rs" test = false doc = false +required-features = ['binaryen'] + +[features] +binaryen = ['wasmtime-fuzzing/binaryen'] diff --git a/fuzz/fuzz_targets/differential.rs b/fuzz/fuzz_targets/differential.rs index 5cf14f0523..edccf2daf8 100755 --- a/fuzz/fuzz_targets/differential.rs +++ b/fuzz/fuzz_targets/differential.rs @@ -9,5 +9,5 @@ fuzz_target!(|data: ( generators::WasmOptTtf )| { let (lhs, rhs, wasm) = data; - oracles::differential_execution(&wasm, &[lhs, rhs]); + oracles::differential_execution(&wasm.wasm, &[lhs, rhs]); }); From 23bc79f66d398dde570d52d69c061adfb63e8ba3 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 17 Mar 2020 09:36:40 -0700 Subject: [PATCH 2/5] rustfmt --- crates/fuzzing/src/oracles.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 361e5b2b70..c79bdba7c6 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -109,10 +109,7 @@ pub fn compile(wasm: &[u8], strategy: Strategy) { /// exports. Modulo OOM, non-canonical NaNs, and usage of Wasm features that are /// or aren't enabled for different configs, we should get the same results when /// we call the exported functions for all of our different configs. -pub fn differential_execution( - wasm: &[u8], - configs: &[crate::generators::DifferentialConfig], -) { +pub fn differential_execution(wasm: &[u8], configs: &[crate::generators::DifferentialConfig]) { crate::init_fuzzing(); // We need at least two configs. From 5f47068eb131f1fa4f3886c30b2247fdf2b72967 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 17 Mar 2020 09:51:51 -0700 Subject: [PATCH 3/5] take ttf in differential --- crates/fuzzing/src/oracles.rs | 10 +++++++--- fuzz/fuzz_targets/differential.rs | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index c79bdba7c6..4a9955e10f 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -109,7 +109,11 @@ pub fn compile(wasm: &[u8], strategy: Strategy) { /// exports. Modulo OOM, non-canonical NaNs, and usage of Wasm features that are /// or aren't enabled for different configs, we should get the same results when /// we call the exported functions for all of our different configs. -pub fn differential_execution(wasm: &[u8], configs: &[crate::generators::DifferentialConfig]) { +#[cfg(feature = "binaryen")] +pub fn differential_execution( + ttf: &crate::generators::WasmOptTtf, + configs: &[crate::generators::DifferentialConfig], +) { crate::init_fuzzing(); // We need at least two configs. @@ -128,13 +132,13 @@ pub fn differential_execution(wasm: &[u8], configs: &[crate::generators::Differe }; let mut export_func_results: HashMap, Trap>> = Default::default(); - log_wasm(wasm); + log_wasm(&ttf.wasm); for config in &configs { let engine = Engine::new(config); let store = Store::new(&engine); - let module = match Module::new(&store, wasm) { + let module = match Module::new(&store, &ttf.wasm) { Ok(module) => module, // The module might rely on some feature that our config didn't // enable or something like that. diff --git a/fuzz/fuzz_targets/differential.rs b/fuzz/fuzz_targets/differential.rs index edccf2daf8..5cf14f0523 100755 --- a/fuzz/fuzz_targets/differential.rs +++ b/fuzz/fuzz_targets/differential.rs @@ -9,5 +9,5 @@ fuzz_target!(|data: ( generators::WasmOptTtf )| { let (lhs, rhs, wasm) = data; - oracles::differential_execution(&wasm.wasm, &[lhs, rhs]); + oracles::differential_execution(&wasm, &[lhs, rhs]); }); From 532422a5d9ac750bdd726649ef856e96576ef17f Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 17 Mar 2020 11:58:48 -0700 Subject: [PATCH 4/5] Fix CI fuzz runners --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 31d53cd738..190516ad21 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,35 +87,35 @@ jobs: find fuzz/corpus/compile -type f \ | shuf \ | head -n 3000 \ - | xargs cargo fuzz run compile --release --debug-assertions + | xargs cargo fuzz run compile --release --debug-assertions --features binaryen env: RUST_BACKTRACE: 1 - run: | find fuzz/corpus/instantiate -type f \ | shuf \ | head -n 2000 \ - | xargs cargo fuzz run instantiate --release --debug-assertions + | xargs cargo fuzz run instantiate --release --debug-assertions --features binaryen env: RUST_BACKTRACE: 1 - run: | find fuzz/corpus/instantiate_translated -type f \ | shuf \ | head -n 1000 \ - | xargs cargo fuzz run instantiate_translated --release --debug-assertions + | xargs cargo fuzz run instantiate_translated --release --debug-assertions --features binaryen env: RUST_BACKTRACE: 1 - run: | find fuzz/corpus/api_calls -type f \ | shuf \ | head -n 100 \ - | xargs cargo fuzz run api_calls --release --debug-assertions + | xargs cargo fuzz run api_calls --release --debug-assertions --features binaryen env: RUST_BACKTRACE: 1 - run: | find fuzz/corpus/differential -type f \ | shuf \ | head -n 100 \ - | xargs cargo fuzz run differential --release --debug-assertions + | xargs cargo fuzz run differential --release --debug-assertions --features binaryen env: RUST_BACKTRACE: 1 From 210bfddfa9070643f83b8dcc747d3d2e2710b79b Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 17 Mar 2020 12:01:11 -0700 Subject: [PATCH 5/5] Fix unused imports in oracles --- crates/fuzzing/src/oracles.rs | 120 +++++++++++++++++----------------- 1 file changed, 61 insertions(+), 59 deletions(-) diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 4a9955e10f..ebee9cc436 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -12,8 +12,7 @@ pub mod dummy; -use dummy::{dummy_imports, dummy_values}; -use std::collections::{HashMap, HashSet}; +use dummy::dummy_imports; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use wasmtime::*; @@ -114,6 +113,8 @@ pub fn differential_execution( ttf: &crate::generators::WasmOptTtf, configs: &[crate::generators::DifferentialConfig], ) { + use std::collections::{HashMap, HashSet}; + crate::init_fuzzing(); // We need at least two configs. @@ -205,7 +206,7 @@ pub fn differential_execution( }; let ty = f.ty(); - let params = match dummy_values(ty.params()) { + let params = match dummy::dummy_values(ty.params()) { Ok(p) => p, Err(_) => continue, }; @@ -217,69 +218,69 @@ pub fn differential_execution( assert_same_export_func_result(&existing_result, &this_result, name); } } -} -fn init_hang_limit(instance: &Instance) { - match instance.get_export("hangLimitInitializer") { - None => return, - Some(Extern::Func(f)) => { - f.call(&[]) - .expect("initializing the hang limit should not fail"); - } - Some(_) => panic!("unexpected hangLimitInitializer export"), - } -} - -fn assert_same_export_func_result( - lhs: &Result, Trap>, - rhs: &Result, Trap>, - func_name: &str, -) { - let fail = || { - panic!( - "differential fuzzing failed: exported func {} returned two \ - different results: {:?} != {:?}", - func_name, lhs, rhs - ) - }; - - match (lhs, rhs) { - (Err(_), Err(_)) => {} - (Ok(lhs), Ok(rhs)) => { - if lhs.len() != rhs.len() { - fail(); + fn init_hang_limit(instance: &Instance) { + match instance.get_export("hangLimitInitializer") { + None => return, + Some(Extern::Func(f)) => { + f.call(&[]) + .expect("initializing the hang limit should not fail"); } - for (lhs, rhs) in lhs.iter().zip(rhs.iter()) { - match (lhs, rhs) { - (Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue, - (Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue, - (Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue, - (Val::F32(lhs), Val::F32(rhs)) => { - let lhs = f32::from_bits(*lhs); - let rhs = f32::from_bits(*rhs); - if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { - continue; - } else { - fail() + Some(_) => panic!("unexpected hangLimitInitializer export"), + } + } + + fn assert_same_export_func_result( + lhs: &Result, Trap>, + rhs: &Result, Trap>, + func_name: &str, + ) { + let fail = || { + panic!( + "differential fuzzing failed: exported func {} returned two \ + different results: {:?} != {:?}", + func_name, lhs, rhs + ) + }; + + match (lhs, rhs) { + (Err(_), Err(_)) => {} + (Ok(lhs), Ok(rhs)) => { + if lhs.len() != rhs.len() { + fail(); + } + for (lhs, rhs) in lhs.iter().zip(rhs.iter()) { + match (lhs, rhs) { + (Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue, + (Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue, + (Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue, + (Val::F32(lhs), Val::F32(rhs)) => { + let lhs = f32::from_bits(*lhs); + let rhs = f32::from_bits(*rhs); + if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { + continue; + } else { + fail() + } } - } - (Val::F64(lhs), Val::F64(rhs)) => { - let lhs = f64::from_bits(*lhs); - let rhs = f64::from_bits(*rhs); - if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { - continue; - } else { - fail() + (Val::F64(lhs), Val::F64(rhs)) => { + let lhs = f64::from_bits(*lhs); + let rhs = f64::from_bits(*rhs); + if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) { + continue; + } else { + fail() + } } + (Val::AnyRef(_), Val::AnyRef(_)) | (Val::FuncRef(_), Val::FuncRef(_)) => { + continue + } + _ => fail(), } - (Val::AnyRef(_), Val::AnyRef(_)) | (Val::FuncRef(_), Val::FuncRef(_)) => { - continue - } - _ => fail(), } } + _ => fail(), } - _ => fail(), } } @@ -287,6 +288,7 @@ fn assert_same_export_func_result( #[cfg(feature = "binaryen")] pub fn make_api_calls(api: crate::generators::api::ApiCalls) { use crate::generators::api::ApiCall; + use std::collections::HashMap; crate::init_fuzzing(); @@ -401,7 +403,7 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) { let nth = nth % funcs.len(); let f = &funcs[nth]; let ty = f.ty(); - let params = match dummy_values(ty.params()) { + let params = match dummy::dummy_values(ty.params()) { Ok(p) => p, Err(_) => continue, };