Throw out fewer fuzz inputs with differential fuzzer (#4859)

* Throw out fewer fuzz inputs with differential fuzzer

Prior to this commit the differential fuzzer would generate a module and
then select an engine to execute the module against Wasmtime. This
meant, however, that the candidate list of engines were filtered against
the configuration used to generate the module to ensure that the
selected engine could run the generated module.

This commit inverts this logic and instead selects an engine first,
allowing the engine to then tweak the module configuration to ensure
that the generated module is compatible with the engine selected. This
means that fewer fuzz inputs are discarded because every fuzz input will
result in an engine being executed.

Internally the engine constructors have all been updated to update the
configuration to work instead of filtering the configuration. Some other
fixes were applied for the spec interpreter as well to work around #4852

* Fix tests

crates/fuzzing/src/generators/config.rs

@@ -38,11 +38,11 @@ impl Config {
 
         // Make it more likely that there are types available to generate a
         // function with.
-        config.min_types = 1;
+        config.min_types = config.min_types.max(1);
         config.max_types = config.max_types.max(1);
 
         // Generate at least one function
-        config.min_funcs = 1;
+        config.min_funcs = config.min_funcs.max(1);
         config.max_funcs = config.max_funcs.max(1);
 
         // Allow a memory to be generated, but don't let it get too large.
@@ -102,7 +102,7 @@ impl Config {
     /// to ensure termination; as doing so will add an additional global to the module,
     /// the pooling allocator, if configured, will also have its globals limit updated.
     pub fn generate(
-        &mut self,
+        &self,
         input: &mut Unstructured<'_>,
         default_fuel: Option<u32>,
     ) -> arbitrary::Result<wasm_smith::Module> {

crates/fuzzing/src/generators/single_inst_module.rs

@@ -24,17 +24,7 @@ pub struct SingleInstModule<'a> {
 
 impl<'a> SingleInstModule<'a> {
     /// Choose a single-instruction module that matches `config`.
-    pub fn new(u: &mut Unstructured<'a>, config: &mut ModuleConfig) -> arbitrary::Result<&'a Self> {
+    pub fn new(u: &mut Unstructured<'a>, config: &ModuleConfig) -> arbitrary::Result<&'a Self> {
-        // To avoid skipping modules unnecessarily during fuzzing, fix up the
-        // `ModuleConfig` to match the inherent limits of a single-instruction
-        // module.
-        config.config.min_funcs = 1;
-        config.config.max_funcs = 1;
-        config.config.min_tables = 0;
-        config.config.max_tables = 0;
-        config.config.min_memories = 0;
-        config.config.max_memories = 0;
-
         // Only select instructions that match the `ModuleConfig`.
         let instructions = &INSTRUCTIONS
             .iter()

crates/fuzzing/src/oracles/diff_spec.rs

@@ -1,9 +1,9 @@
 //! Evaluate an exported Wasm function using the WebAssembly specification
 //! reference interpreter.
 
-use crate::generators::{DiffValue, DiffValueType, ModuleConfig};
+use crate::generators::{Config, DiffValue, DiffValueType};
 use crate::oracles::engine::{DiffEngine, DiffInstance};
-use anyhow::{anyhow, bail, Error, Result};
+use anyhow::{anyhow, Error, Result};
 use wasm_spec_interpreter::Value;
 use wasmtime::Trap;
 
@@ -11,20 +11,15 @@ use wasmtime::Trap;
 pub struct SpecInterpreter;
 
 impl SpecInterpreter {
-    /// Build a new [`SpecInterpreter`] but only if the configuration does not
+    pub(crate) fn new(config: &mut Config) -> Self {
-    /// rely on features that the current bindings (i.e.,
+        let config = &mut config.module_config.config;
-    /// `wasm-spec-interpreter`) do not support.
+
-    pub fn new(config: &ModuleConfig) -> Result<Self> {
-        if config.config.reference_types_enabled {
-            bail!("the spec interpreter bindings do not support reference types")
-        }
         // TODO: right now the interpreter bindings only execute the first
         // function in the module so if there's possibly more than one function
         // it's not possible to run the other function. This should be fixed
         // with improvements to the ocaml bindings to the interpreter.
-        if config.config.max_funcs > 1 {
+        config.min_funcs = 1;
-            bail!("the spec interpreter bindings can only support one function for now")
+        config.max_funcs = 1;
-        }
 
         // TODO: right now the instantiation step for the interpreter does
         // nothing and the evaluation step performs an instantiation followed by
@@ -32,18 +27,18 @@ impl SpecInterpreter {
         // engines will "succeed" in the interpreter because the error is
         // delayed to the execution. This should be fixed by making
         // instantiation a first-class primitive in our interpreter bindings.
-        if config.config.max_tables > 0 {
+        config.min_tables = 0;
-            bail!("the spec interpreter bindings do not fail as they should with out-of-bounds table accesses")
+        config.max_tables = 0;
-        }
 
-        if config.config.memory64_enabled {
+        config.min_memories = config.min_memories.min(1);
-            bail!("memory64 not implemented in spec interpreter");
+        config.max_memories = config.max_memories.min(1);
-        }
 
-        if config.config.threads_enabled {
+        config.memory64_enabled = false;
-            bail!("spec interpreter does not support the threading proposal");
+        config.threads_enabled = false;
-        }
+        config.bulk_memory_enabled = false;
-        Ok(Self)
+        config.reference_types_enabled = false;
+
+        Self
     }
 }
 
@@ -160,5 +155,5 @@ fn smoke() {
     if !wasm_spec_interpreter::support_compiled_in() {
         return;
     }
-    crate::oracles::engine::smoke_test_engine(|config| SpecInterpreter::new(&config.module_config))
+    crate::oracles::engine::smoke_test_engine(|_, config| Ok(SpecInterpreter::new(config)))
 }

crates/fuzzing/src/oracles/diff_v8.rs

@@ -1,4 +1,4 @@
-use crate::generators::{DiffValue, DiffValueType, ModuleConfig};
+use crate::generators::{Config, DiffValue, DiffValueType};
 use crate::oracles::engine::{DiffEngine, DiffInstance};
 use anyhow::{bail, Error, Result};
 use std::cell::RefCell;
@@ -12,7 +12,7 @@ pub struct V8Engine {
 }
 
 impl V8Engine {
-    pub fn new(config: &ModuleConfig) -> Result<V8Engine> {
+    pub fn new(config: &mut Config) -> V8Engine {
         static INIT: Once = Once::new();
 
         INIT.call_once(|| {
@@ -21,27 +21,22 @@ impl V8Engine {
             v8::V8::initialize();
         });
 
+        let config = &mut config.module_config.config;
         // FIXME: reference types are disabled for now as we seemingly keep finding
         // a segfault in v8. This is found relatively quickly locally and keeps
         // getting found by oss-fuzz and currently we don't think that there's
         // really much we can do about it. For the time being disable reference
         // types entirely. An example bug is
         // https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=45662
-        if config.config.reference_types_enabled {
+        config.reference_types_enabled = false;
-            bail!("reference types are buggy in v8");
-        }
 
-        if config.config.memory64_enabled {
+        config.min_memories = config.min_memories.min(1);
-            bail!("memory64 not enabled by default in v8");
+        config.max_memories = config.max_memories.min(1);
-        }
+        config.memory64_enabled = false;
 
-        if config.config.max_memories > 1 {
+        Self {
-            bail!("multi-memory not enabled by default in v8");
-        }
-
-        Ok(Self {
             isolate: Rc::new(RefCell::new(v8::Isolate::new(Default::default()))),
-        })
+        }
     }
 }
 
@@ -325,5 +320,5 @@ fn get_diff_value(
 
 #[test]
 fn smoke() {
-    crate::oracles::engine::smoke_test_engine(|config| V8Engine::new(&config.module_config))
+    crate::oracles::engine::smoke_test_engine(|_, config| Ok(V8Engine::new(config)))
 }

crates/fuzzing/src/oracles/diff_wasmi.rs

@@ -1,45 +1,30 @@
 //! Evaluate an exported Wasm function using the wasmi interpreter.
 
-use crate::generators::{DiffValue, DiffValueType, ModuleConfig};
+use crate::generators::{Config, DiffValue, DiffValueType};
 use crate::oracles::engine::{DiffEngine, DiffInstance};
-use anyhow::{bail, Context, Error, Result};
+use anyhow::{Context, Error, Result};
 use wasmtime::{Trap, TrapCode};
 
 /// A wrapper for `wasmi` as a [`DiffEngine`].
 pub struct WasmiEngine;
 
 impl WasmiEngine {
-    /// Build a new [`WasmiEngine`] but only if the configuration does not rely
+    pub(crate) fn new(config: &mut Config) -> Self {
-    /// on features that `wasmi` does not support.
+        let config = &mut config.module_config.config;
-    pub fn new(config: &ModuleConfig) -> Result<Self> {
+        config.reference_types_enabled = false;
-        if config.config.reference_types_enabled {
+        config.simd_enabled = false;
-            bail!("wasmi does not support reference types")
+        config.multi_value_enabled = false;
-        }
+        config.saturating_float_to_int_enabled = false;
-        if config.config.simd_enabled {
+        config.sign_extension_enabled = false;
-            bail!("wasmi does not support SIMD")
+        config.memory64_enabled = false;
-        }
+        config.bulk_memory_enabled = false;
-        if config.config.multi_value_enabled {
+        config.threads_enabled = false;
-            bail!("wasmi does not support multi-value")
+        config.max_memories = config.max_memories.min(1);
-        }
+        config.min_memories = config.min_memories.min(1);
-        if config.config.saturating_float_to_int_enabled {
+        config.max_tables = config.max_tables.min(1);
-            bail!("wasmi does not support saturating float-to-int conversions")
+        config.min_tables = config.min_tables.min(1);
-        }
+
-        if config.config.sign_extension_enabled {
+        Self
-            bail!("wasmi does not support sign-extension")
-        }
-        if config.config.memory64_enabled {
-            bail!("wasmi does not support memory64");
-        }
-        if config.config.bulk_memory_enabled {
-            bail!("wasmi does not support bulk memory");
-        }
-        if config.config.threads_enabled {
-            bail!("wasmi does not support threads");
-        }
-        if config.config.max_memories > 1 {
-            bail!("wasmi does not support multi-memory");
-        }
-        Ok(Self)
     }
 }
 
@@ -223,5 +208,5 @@ impl Into<DiffValue> for wasmi::RuntimeValue {
 
 #[test]
 fn smoke() {
-    crate::oracles::engine::smoke_test_engine(|config| WasmiEngine::new(&config.module_config))
+    crate::oracles::engine::smoke_test_engine(|_, config| Ok(WasmiEngine::new(config)))
 }

crates/fuzzing/src/oracles/diff_wasmtime.rs

@@ -1,15 +1,16 @@
 //! Evaluate an exported Wasm function using Wasmtime.
 
-use crate::generators::{self, DiffValue, DiffValueType};
+use crate::generators::{self, DiffValue, DiffValueType, WasmtimeConfig};
 use crate::oracles::dummy;
 use crate::oracles::engine::DiffInstance;
 use crate::oracles::{compile_module, engine::DiffEngine, StoreLimits};
 use anyhow::{Context, Error, Result};
+use arbitrary::Unstructured;
 use wasmtime::{Extern, FuncType, Instance, Module, Store, Trap, TrapCode, Val};
 
 /// A wrapper for using Wasmtime as a [`DiffEngine`].
 pub struct WasmtimeEngine {
-    pub(crate) config: generators::Config,
+    config: generators::Config,
 }
 
 impl WasmtimeEngine {
@@ -17,7 +18,13 @@ impl WasmtimeEngine {
     /// later. Ideally the store and engine could be built here but
     /// `compile_module` takes a [`generators::Config`]; TODO re-factor this if
     /// that ever changes.
-    pub fn new(config: generators::Config) -> Result<Self> {
+    pub fn new(u: &mut Unstructured<'_>, config: &generators::Config) -> arbitrary::Result<Self> {
+        let mut new_config = u.arbitrary::<WasmtimeConfig>()?;
+        new_config.make_compatible_with(&config.wasmtime);
+        let config = generators::Config {
+            wasmtime: new_config,
+            module_config: config.module_config.clone(),
+        };
         Ok(Self { config })
     }
 }
@@ -220,5 +227,5 @@ impl Into<DiffValue> for Val {
 
 #[test]
 fn smoke() {
-    crate::oracles::engine::smoke_test_engine(|config| WasmtimeEngine::new(config))
+    crate::oracles::engine::smoke_test_engine(|u, config| WasmtimeEngine::new(u, config))
 }

crates/fuzzing/src/oracles/engine.rs

@@ -1,68 +1,37 @@
 //! Define the interface for differential evaluation of Wasm functions.
 
-use crate::generators::{Config, DiffValue, DiffValueType, WasmtimeConfig};
+use crate::generators::{Config, DiffValue, DiffValueType};
 use crate::oracles::{diff_wasmi::WasmiEngine, diff_wasmtime::WasmtimeEngine};
 use anyhow::Error;
 use arbitrary::Unstructured;
 use wasmtime::Trap;
 
-/// Pick one of the engines implemented in this module that is:
+/// Returns a function which can be used to build the engine name specified.
-/// - in the list of `allowed` engines
+///
-/// - can evaluate Wasm modules compatible with `existing_config`.
+/// `None` is returned if the named engine does not have support compiled into
-pub fn choose(
+/// this crate.
+pub fn build(
     u: &mut Unstructured<'_>,
-    existing_config: &Config,
+    name: &str,
-    allowed: &[&str],
+    config: &mut Config,
 ) -> arbitrary::Result<Option<Box<dyn DiffEngine>>> {
-    // Filter out any engines that cannot match the `existing_config` or are not
+    let engine: Box<dyn DiffEngine> = match name {
-    // `allowed`.
+        "wasmtime" => Box::new(WasmtimeEngine::new(u, config)?),
-    let mut engines: Vec<Box<dyn DiffEngine>> = vec![];
+        "wasmi" => Box::new(WasmiEngine::new(config)),
-
-    if allowed.contains(&"wasmtime") {
-        let mut new_wasmtime_config: WasmtimeConfig = u.arbitrary()?;
-        new_wasmtime_config.make_compatible_with(&existing_config.wasmtime);
-        let new_config = Config {
-            wasmtime: new_wasmtime_config,
-            module_config: existing_config.module_config.clone(),
-        };
-        if let Result::Ok(e) = WasmtimeEngine::new(new_config) {
-            engines.push(Box::new(e))
-        }
-    }
-
-    if allowed.contains(&"wasmi") {
-        if let Result::Ok(e) = WasmiEngine::new(&existing_config.module_config) {
-            engines.push(Box::new(e))
-        }
-    }
 
         #[cfg(feature = "fuzz-spec-interpreter")]
-    if allowed.contains(&"spec") {
+        "spec" => Box::new(crate::oracles::diff_spec::SpecInterpreter::new(config)),
-        if let Result::Ok(e) =
+        #[cfg(not(feature = "fuzz-spec-interpreter"))]
-            crate::oracles::diff_spec::SpecInterpreter::new(&existing_config.module_config)
+        "spec" => return Ok(None),
-        {
-            engines.push(Box::new(e))
-        }
-    }
 
         #[cfg(not(any(windows, target_arch = "s390x")))]
-    if allowed.contains(&"v8") {
+        "v8" => Box::new(crate::oracles::diff_v8::V8Engine::new(config)),
-        if let Result::Ok(e) =
+        #[cfg(any(windows, target_arch = "s390x"))]
-            crate::oracles::diff_v8::V8Engine::new(&existing_config.module_config)
+        "v8" => return Ok(None),
-        {
-            engines.push(Box::new(e))
-        }
-    }
 
-    if engines.is_empty() {
+        _ => panic!("unknown engine {name}"),
-        return Ok(None);
+    };
-    }
 
-    // Use the input of the fuzzer to pick an engine that we'll be fuzzing
-    // Wasmtime against.
-    let index: usize = u.int_in_range(0..=engines.len() - 1)?;
-    let engine = engines.swap_remove(index);
-    log::debug!("selected engine: {}", engine.name());
     Ok(Some(engine))
 }
 
@@ -187,11 +156,11 @@ pub fn parse_env_list(env_variable: &str) -> Option<Vec<String>> {
 }
 
 #[cfg(test)]
-pub fn smoke_test_engine<T>(mk_engine: impl Fn(Config) -> anyhow::Result<T>)
+pub fn smoke_test_engine<T>(
-where
+    mk_engine: impl Fn(&mut arbitrary::Unstructured<'_>, &mut Config) -> arbitrary::Result<T>,
+) where
     T: DiffEngine,
 {
-    use arbitrary::Arbitrary;
     use rand::prelude::*;
 
     let mut rng = SmallRng::seed_from_u64(0);
@@ -199,8 +168,8 @@ where
     let n = 100;
     for _ in 0..n {
         rng.fill_bytes(&mut buf);
-        let u = Unstructured::new(&buf);
+        let mut u = Unstructured::new(&buf);
-        let mut config = match Config::arbitrary_take_rest(u) {
+        let mut config = match u.arbitrary::<Config>() {
             Ok(config) => config,
             Err(_) => continue,
         };
@@ -208,19 +177,7 @@ where
         // settings, can guaranteed instantiate a module.
         config.set_differential_config();
 
-        // Configure settings to ensure that any filters in engine constructors
+        let mut engine = match mk_engine(&mut u, &mut config) {
-        // try not to filter out this `Config`.
-        config.module_config.config.reference_types_enabled = false;
-        config.module_config.config.bulk_memory_enabled = false;
-        config.module_config.config.memory64_enabled = false;
-        config.module_config.config.threads_enabled = false;
-        config.module_config.config.simd_enabled = false;
-        config.module_config.config.min_funcs = 1;
-        config.module_config.config.max_funcs = 1;
-        config.module_config.config.min_tables = 0;
-        config.module_config.config.max_tables = 0;
-
-        let mut engine = match mk_engine(config) {
             Ok(engine) => engine,
             Err(e) => {
                 println!("skip {:?}", e);

fuzz/fuzz_targets/differential.rs

@@ -60,60 +60,84 @@ fn run(data: &[u8]) -> Result<()> {
     STATS.bump_attempts();
 
     let mut u = Unstructured::new(data);
+
+    // Generate a Wasmtime and module configuration and update its settings
+    // initially to be suitable for differential execution where the generated
+    // wasm will behave the same in two different engines. This will get further
+    // refined below.
     let mut config: Config = u.arbitrary()?;
     config.set_differential_config();
 
-    // Generate the Wasm module; this is specified by either the ALLOWED_MODULES
+    // Choose an engine that Wasmtime will be differentially executed against.
-    // environment variable or a random selection between wasm-smith and
+    // The chosen engine is then created, which might update `config`, and
-    // single-inst.
+    // returned as a trait object.
-    let build_wasm_smith_module = |u: &mut Unstructured, config: &mut Config| -> Result<_> {
+    let lhs = u.choose(unsafe { &ALLOWED_ENGINES })?;
+    let mut lhs = match engine::build(&mut u, lhs, &mut config)? {
+        Some(engine) => engine,
+        // The chosen engine does not have support compiled into the fuzzer,
+        // discard this test case.
+        None => return Ok(()),
+    };
+
+    // Using the now-legalized module configuration generate the Wasm module;
+    // this is specified by either the ALLOWED_MODULES environment variable or a
+    // random selection between wasm-smith and single-inst.
+    let build_wasm_smith_module = |u: &mut Unstructured, config: &Config| -> Result<_> {
         STATS.wasm_smith_modules.fetch_add(1, SeqCst);
         let module = config.generate(u, Some(1000))?;
         Ok(module.to_bytes())
     };
-    let build_single_inst_module = |u: &mut Unstructured, config: &mut Config| -> Result<_> {
+    let build_single_inst_module = |u: &mut Unstructured, config: &Config| -> Result<_> {
         STATS.single_instruction_modules.fetch_add(1, SeqCst);
-        let module = SingleInstModule::new(u, &mut config.module_config)?;
+        let module = SingleInstModule::new(u, &config.module_config)?;
         Ok(module.to_bytes())
     };
     if unsafe { ALLOWED_MODULES.is_empty() } {
         panic!("unable to generate a module to fuzz against; check `ALLOWED_MODULES`")
     }
     let wasm = match *u.choose(unsafe { ALLOWED_MODULES.as_slice() })? {
-        "wasm-smith" => build_wasm_smith_module(&mut u, &mut config)?,
+        "wasm-smith" => build_wasm_smith_module(&mut u, &config)?,
-        "single-inst" => build_single_inst_module(&mut u, &mut config)?,
+        "single-inst" => build_single_inst_module(&mut u, &config)?,
         _ => unreachable!(),
     };
     log_wasm(&wasm);
 
-    // Choose a left-hand side Wasm engine. If no engine could be chosen then
+    // Instantiate the generated wasm file in the chosen differential engine.
-    // that means the configuration selected above doesn't match any allowed
-    // engine (configured via an env var) so the test case is thrown out.
-    let mut lhs = match engine::choose(&mut u, &config, unsafe { &ALLOWED_ENGINES })? {
-        Some(engine) => engine,
-        None => return Ok(()),
-    };
     let lhs_instance = lhs.instantiate(&wasm);
     STATS.bump_engine(lhs.name());
 
-    // Choose a right-hand side Wasm engine--this will always be Wasmtime.
+    // Always use Wasmtime as the second engine to instantiate within.
     let rhs_store = config.to_store();
     let rhs_module = wasmtime::Module::new(rhs_store.engine(), &wasm).unwrap();
     let rhs_instance = WasmtimeInstance::new(rhs_store, rhs_module);
 
-    // If we fail to instantiate, check that both sides do.
     let (mut lhs_instance, mut rhs_instance) = match (lhs_instance, rhs_instance) {
+        // Both sides successful, continue below to invoking exports.
         (Ok(l), Ok(r)) => (l, r),
+
+        // Both sides failed, make sure they failed for the same reason but then
+        // we're done with this fuzz test case.
         (Err(l), Err(r)) => {
             let err = r.downcast::<Trap>().expect("not a trap");
             lhs.assert_error_match(&err, &l);
             return Ok(());
         }
-        (l, r) => panic!(
+
+        // One side succeeded and one side failed, that means a bug happened!
+        (l, r) => {
+            // FIXME(#4852): the spec interpreter doesn't instantiate as part of
+            // the instantiate step so if wasmtime failed and the spec succeeded
+            // that's ok. This clause should be removed once that issue is
+            // fixed.
+            if l.is_ok() && lhs.name() == "spec" {
+                return Ok(());
+            }
+            panic!(
                 "failed to instantiate only one side: {:?} != {:?}",
                 l.err(),
                 r.err()
-        ),
+            )
+        }
     };
 
     // Call each exported function with different sets of arguments.
@@ -148,6 +172,13 @@ fn run(data: &[u8]) -> Result<()> {
                 break 'outer;
             }
 
+            // FIXME(#4852): the spec interpreter only supports one execution
+            // right now because each execution re-instantiates the module in
+            // its bindings. This should be removed once that issue is fixed.
+            if lhs.name() == "spec" {
+                break 'outer;
+            }
+
             // We evaluate the same function with different arguments until we
             // hit a predetermined limit or we run out of unstructured data--it
             // does not make sense to re-evaluate the same arguments over and