T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[fuzz] Remove more fuzz targets (#4737)

Browse Source 
* [fuzz] Remove the `differential` fuzz target

This functionality is already covered by the `differential_meta` target.

* [fuzz] Rename `differential_meta` to `differential`

Now that the `differential_meta` fuzz target does everything that the
existing `differential` target did and more, it can take over the
original name.
This commit is contained in:
Andrew Brown
2022-08-19 10:39:56 -07:00
committed by GitHub
parent 8b7fb19b1d
commit 9758f5420e
5 changed files with 96 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
crates/fuzzing/src/oracles.rs
View File

@@ -380,136 +380,6 @@ pub fn differential(
Ok(()) Ok(())
} }
/// Instantiate the given Wasm module with each `Config` and call all of its
/// exports. Modulo OOM, non-canonical NaNs, and usage of Wasm features that are
/// or aren't enabled for different configs, we should get the same results when
/// we call the exported functions for all of our different configs.
///
/// Returns `None` if a fuzz configuration was rejected (should happen rarely).
pub fn differential_execution(
wasm: &[u8],
module_config: &generators::ModuleConfig,
configs: &[generators::WasmtimeConfig],
) -> Option<()> {
use std::collections::{HashMap, HashSet};
// We need at least two configs.
if configs.len() < 2
// And all the configs should be unique.
|| configs.iter().collect::<HashSet<_>>().len() != configs.len()
{
return None;
}
let mut export_func_results: HashMap<String, Result<Box<[Val]>, Trap>> = Default::default();
log_wasm(&wasm);
for fuzz_config in configs {
let fuzz_config = generators::Config {
module_config: module_config.clone(),
wasmtime: fuzz_config.clone(),
};
log::debug!("fuzz config: {:?}", fuzz_config);
let mut store = fuzz_config.to_store();
let module = compile_module(store.engine(), &wasm, true, &fuzz_config)?;
// TODO: we should implement tracing versions of these dummy imports
// that record a trace of the order that imported functions were called
// in and with what values. Like the results of exported functions,
// calls to imports should also yield the same values for each
// configuration, and we should assert that.
let instance = match instantiate_with_dummy(&mut store, &module) {
Some(instance) => instance,
None => continue,
};
let exports = instance
.exports(&mut store)
.filter_map(|e| {
let name = e.name().to_string();
e.into_func().map(|f| (name, f))
})
.collect::<Vec<_>>();
for (name, f) in exports {
log::debug!("invoke export {:?}", name);
let ty = f.ty(&store);
let params = dummy::dummy_values(ty.params());
let mut results = vec![Val::I32(0); ty.results().len()];
let this_result = f
.call(&mut store, &params, &mut results)
.map(|()| results.into())
.map_err(|e| e.downcast::<Trap>().unwrap());
let existing_result = export_func_results
.entry(name.to_string())
.or_insert_with(|| this_result.clone());
assert_same_export_func_result(&existing_result, &this_result, &name);
}
}
return Some(());
fn assert_same_export_func_result(
lhs: &Result<Box<[Val]>, Trap>,
rhs: &Result<Box<[Val]>, Trap>,
func_name: &str,
) {
let fail = || {
panic!(
"differential fuzzing failed: exported func {} returned two \
different results: {:?} != {:?}",
func_name, lhs, rhs
)
};
match (lhs, rhs) {
// Different compilation settings can lead to different amounts
// of stack space being consumed, so if either the lhs or the rhs
// hit a stack overflow then we discard the result of the other side
// since if it ran successfully or trapped that's ok in both
// situations.
(Err(e), _) | (_, Err(e)) if e.trap_code() == Some(TrapCode::StackOverflow) => {}
(Err(a), Err(b)) => {
if a.trap_code() != b.trap_code() {
fail();
}
}
(Ok(lhs), Ok(rhs)) => {
if lhs.len() != rhs.len() {
fail();
}
for (lhs, rhs) in lhs.iter().zip(rhs.iter()) {
match (lhs, rhs) {
(Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue,
(Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue,
(Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue,
(Val::F32(lhs), Val::F32(rhs)) if f32_equal(*lhs, *rhs) => continue,
(Val::F64(lhs), Val::F64(rhs)) if f64_equal(*lhs, *rhs) => continue,
(Val::ExternRef(_), Val::ExternRef(_))
| (Val::FuncRef(_), Val::FuncRef(_)) => continue,
_ => fail(),
}
}
}
_ => fail(),
}
}
}
fn f32_equal(a: u32, b: u32) -> bool {
let a = f32::from_bits(a);
let b = f32::from_bits(b);
a == b || (a.is_nan() && b.is_nan())
}
fn f64_equal(a: u64, b: u64) -> bool {
let a = f64::from_bits(a);
let b = f64::from_bits(b);
a == b || (a.is_nan() && b.is_nan())
}
/// Invoke the given API calls. /// Invoke the given API calls.
pub fn make_api_calls(api: generators::api::ApiCalls) { pub fn make_api_calls(api: generators::api::ApiCalls) {
use crate::generators::api::ApiCall; use crate::generators::api::ApiCall;

6
fuzz/Cargo.toml
View File

@@ -60,12 +60,6 @@ path = "fuzz_targets/differential.rs"
test = false test = false
doc = false doc = false
[[bin]]
name = "differential_meta"
path = "fuzz_targets/differential_meta.rs"
test = false
doc = false
[[bin]] [[bin]]
name = "differential_v8" name = "differential_v8"
path = "fuzz_targets/differential_v8.rs" path = "fuzz_targets/differential_v8.rs"

4
fuzz/README.md
View File

@@ -34,9 +34,7 @@ At the time of writing, we have the following fuzz targets:
to its source, yielding a function A', and checks that A compiled + to its source, yielding a function A', and checks that A compiled +
incremental compilation generates the same machine code as if A' was compiled incremental compilation generates the same machine code as if A' was compiled
from scratch. from scratch.
* `differential`: Generate a Wasm module and check that Wasmtime returns * `differential`: Generate a Wasm module, evaluate each exported function
the same results when run with two different configurations.
* `differential_meta`: Generate a Wasm module, evaluate each exported function
with random inputs, and check that Wasmtime returns the same results as a with random inputs, and check that Wasmtime returns the same results as a
choice of another engine: the Wasm spec interpreter (see the choice of another engine: the Wasm spec interpreter (see the
`wasm-spec-interpreter` crate), the `wasmi` interpreter, or Wasmtime itself `wasm-spec-interpreter` crate), the `wasmi` interpreter, or Wasmtime itself

122
fuzz/fuzz_targets/differential.rs
View File

@@ -2,40 +2,108 @@
use libfuzzer_sys::arbitrary::{Result, Unstructured}; use libfuzzer_sys::arbitrary::{Result, Unstructured};
use libfuzzer_sys::fuzz_target; use libfuzzer_sys::fuzz_target;
use wasmtime_fuzzing::generators::InstanceAllocationStrategy; use std::sync::atomic::AtomicUsize;
use wasmtime_fuzzing::{generators, oracles}; use std::sync::atomic::Ordering::SeqCst;
use std::sync::Once;
use wasmtime_fuzzing::generators::{Config, DiffValue, SingleInstModule};
use wasmtime_fuzzing::oracles::diff_spec;
use wasmtime_fuzzing::oracles::diff_wasmtime::WasmtimeInstance;
use wasmtime_fuzzing::oracles::{differential, engine, log_wasm};
// Upper limit on the number of invocations for each WebAssembly function
// executed by this fuzz target.
const NUM_INVOCATIONS: usize = 5;
// Keep track of how many WebAssembly modules we actually executed (i.e. ran to
// completion) versus how many were tried.
static TOTAL_INVOCATIONS: AtomicUsize = AtomicUsize::new(0);
static TOTAL_SUCCESSES: AtomicUsize = AtomicUsize::new(0);
static TOTAL_ATTEMPTED: AtomicUsize = AtomicUsize::new(0);
// The spec interpreter requires special one-time setup.
static SETUP: Once = Once::new();
fuzz_target!(|data: &[u8]| { fuzz_target!(|data: &[u8]| {
// errors in `run` have to do with not enough input in `data`, which we // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on
// `setup_ocaml_runtime`.
SETUP.call_once(|| diff_spec::setup_ocaml_runtime());
// Errors in `run` have to do with not enough input in `data`, which we
// ignore here since it doesn't affect how we'd like to fuzz. // ignore here since it doesn't affect how we'd like to fuzz.
drop(run(data)); drop(run(&data));
}); });
fn run(data: &[u8]) -> Result<()> { fn run(data: &[u8]) -> Result<()> {
let mut u = Unstructured::new(data); let successes = TOTAL_SUCCESSES.load(SeqCst);
let attempts = TOTAL_ATTEMPTED.fetch_add(1, SeqCst);
let mut config: generators::Config = u.arbitrary()?; if attempts > 1 && attempts % 1_000 == 0 {
let module = config.generate(&mut u, Some(1000))?; println!("=== Execution rate ({} successes / {} attempted modules): {}% (total invocations: {}) ===",
successes,
let lhs = config.wasmtime; attempts,
let mut rhs: generators::WasmtimeConfig = u.arbitrary()?; successes as f64 / attempts as f64 * 100f64,
TOTAL_INVOCATIONS.load(SeqCst)
// Use the same allocation strategy between the two configs. );
//
// Ideally this wouldn't be necessary, but if the lhs is using ondemand
// and the rhs is using the pooling allocator (or vice versa), then
// the module may have been generated in such a way that is incompatible
// with the other allocation strategy.
//
// We can remove this in the future when it's possible to access the
// fields of `wasm_smith::Module` to constrain the pooling allocator
// based on what was actually generated.
rhs.strategy = lhs.strategy.clone();
if let InstanceAllocationStrategy::Pooling { .. } = &rhs.strategy {
// Also use the same memory configuration when using the pooling allocator
rhs.memory_config = lhs.memory_config.clone();
} }
oracles::differential_execution(&module.to_bytes(), &config.module_config, &[lhs, rhs]); let mut u = Unstructured::new(data);
let mut config: Config = u.arbitrary()?;
config.set_differential_config();
// Generate the Wasm module.
let wasm = if u.arbitrary()? {
// TODO figure out if this always eats up the rest of the unstructured;
// can we limit the number of instructions/functions.
let module = config.generate(&mut u, Some(1000))?;
module.to_bytes()
} else {
let module = SingleInstModule::new(&mut u, &mut config.module_config)?;
module.to_bytes()
};
log_wasm(&wasm);
// Choose a left-hand side Wasm engine.
let lhs = engine::choose(&mut u, &config)?;
let lhs_instance = lhs.instantiate(&wasm);
// Choose a right-hand side Wasm engine--this will always be Wasmtime.
let rhs_store = config.to_store();
let rhs_module = wasmtime::Module::new(rhs_store.engine(), &wasm).unwrap();
let rhs_instance = WasmtimeInstance::new(rhs_store, rhs_module);
// If we fail to instantiate, check that both sides do.
let (mut lhs_instance, mut rhs_instance) = match (lhs_instance, rhs_instance) {
(Ok(l), Ok(r)) => (l, r),
(Err(_), Err(_)) => return Ok(()), // TODO match the error messages.
(l, r) => panic!(
"failed to instantiate only one side: {:?} != {:?}",
l.err(),
r.err()
),
};
// Call each exported function with different sets of arguments.
for (name, signature) in rhs_instance.exported_functions() {
let mut invocations = 0;
loop {
let arguments = signature
.params()
.map(|t| DiffValue::arbitrary_of_type(&mut u, t.try_into().unwrap()))
.collect::<Result<Vec<_>>>()?;
differential(lhs_instance.as_mut(), &mut rhs_instance, &name, &arguments)
.expect("failed to run differential evaluation");
// We evaluate the same function with different arguments until we
// hit a predetermined limit or we run out of unstructured data--it
// does not make sense to re-evaluate the same arguments over and
// over.
invocations += 1;
TOTAL_INVOCATIONS.fetch_add(1, SeqCst);
if invocations > NUM_INVOCATIONS || u.is_empty() {
break;
}
}
}
TOTAL_SUCCESSES.fetch_add(1, SeqCst);
Ok(()) Ok(())
} }

109
fuzz/fuzz_targets/differential_meta.rs
View File

@@ -1,109 +0,0 @@
#![no_main]
use libfuzzer_sys::arbitrary::{Result, Unstructured};
use libfuzzer_sys::fuzz_target;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering::SeqCst;
use std::sync::Once;
use wasmtime_fuzzing::generators::{Config, DiffValue, SingleInstModule};
use wasmtime_fuzzing::oracles::diff_spec;
use wasmtime_fuzzing::oracles::diff_wasmtime::WasmtimeInstance;
use wasmtime_fuzzing::oracles::{differential, engine, log_wasm};
// Upper limit on the number of invocations for each WebAssembly function
// executed by this fuzz target.
const NUM_INVOCATIONS: usize = 5;
// Keep track of how many WebAssembly modules we actually executed (i.e. ran to
// completion) versus how many were tried.
static TOTAL_INVOCATIONS: AtomicUsize = AtomicUsize::new(0);
static TOTAL_SUCCESSES: AtomicUsize = AtomicUsize::new(0);
static TOTAL_ATTEMPTED: AtomicUsize = AtomicUsize::new(0);
// The spec interpreter requires special one-time setup.
static SETUP: Once = Once::new();
fuzz_target!(|data: &[u8]| {
// To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on
// `setup_ocaml_runtime`.
SETUP.call_once(|| diff_spec::setup_ocaml_runtime());
// Errors in `run` have to do with not enough input in `data`, which we
// ignore here since it doesn't affect how we'd like to fuzz.
drop(run(&data));
});
fn run(data: &[u8]) -> Result<()> {
let successes = TOTAL_SUCCESSES.load(SeqCst);
let attempts = TOTAL_ATTEMPTED.fetch_add(1, SeqCst);
if attempts > 1 && attempts % 1_000 == 0 {
println!("=== Execution rate ({} successes / {} attempted modules): {}% (total invocations: {}) ===",
successes,
attempts,
successes as f64 / attempts as f64 * 100f64,
TOTAL_INVOCATIONS.load(SeqCst)
);
}
let mut u = Unstructured::new(data);
let mut config: Config = u.arbitrary()?;
config.set_differential_config();
// Generate the Wasm module.
let wasm = if u.arbitrary()? {
// TODO figure out if this always eats up the rest of the unstructured;
// can we limit the number of instructions/functions.
let module = config.generate(&mut u, Some(1000))?;
module.to_bytes()
} else {
let module = SingleInstModule::new(&mut u, &mut config.module_config)?;
module.to_bytes()
};
log_wasm(&wasm);
// Choose a left-hand side Wasm engine.
let lhs = engine::choose(&mut u, &config)?;
let lhs_instance = lhs.instantiate(&wasm);
// Choose a right-hand side Wasm engine--this will always be Wasmtime.
let rhs_store = config.to_store();
let rhs_module = wasmtime::Module::new(rhs_store.engine(), &wasm).unwrap();
let rhs_instance = WasmtimeInstance::new(rhs_store, rhs_module);
// If we fail to instantiate, check that both sides do.
let (mut lhs_instance, mut rhs_instance) = match (lhs_instance, rhs_instance) {
(Ok(l), Ok(r)) => (l, r),
(Err(_), Err(_)) => return Ok(()), // TODO match the error messages.
(l, r) => panic!(
"failed to instantiate only one side: {:?} != {:?}",
l.err(),
r.err()
),
};
// Call each exported function with different sets of arguments.
for (name, signature) in rhs_instance.exported_functions() {
let mut invocations = 0;
loop {
let arguments = signature
.params()
.map(|t| DiffValue::arbitrary_of_type(&mut u, t.try_into().unwrap()))
.collect::<Result<Vec<_>>>()?;
differential(lhs_instance.as_mut(), &mut rhs_instance, &name, &arguments)
.expect("failed to run differential evaluation");
// We evaluate the same function with different arguments until we
// hit a predetermined limit or we run out of unstructured data--it
// does not make sense to re-evaluate the same arguments over and
// over.
invocations += 1;
TOTAL_INVOCATIONS.fetch_add(1, SeqCst);
if invocations > NUM_INVOCATIONS || u.is_empty() {
break;
}
}
}
TOTAL_SUCCESSES.fetch_add(1, SeqCst);
Ok(())
}