Merge pull request #833 from fitzgen/initial-differential-fuzzing

Add initial differential fuzzing

.github/workflows/main.yml

@@ -98,6 +98,11 @@ jobs:
             | shuf \
             | head -n 100 \
             | xargs cargo fuzz run api_calls --release --debug-assertions
+    - run: |
+        find fuzz/corpus/differential -type f \
+            | shuf \
+            | head -n 100 \
+            | xargs cargo fuzz run differential --release --debug-assertions
 
   # Install wasm32-unknown-emscripten target, and ensure `crates/wasi-common`
   # compiles to Emscripten.

crates/fuzzing/src/generators.rs

@@ -45,3 +45,41 @@ impl Arbitrary for WasmOptTtf {
         Ok(WasmOptTtf { wasm })
     }
 }
+
+/// A description of configuration options that we should do differential
+/// testing between.
+#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct DifferentialConfig {
+    strategy: DifferentialStrategy,
+    opt_level: DifferentialOptLevel,
+}
+
+impl DifferentialConfig {
+    /// Convert this differential fuzzing config into a `wasmtime::Config`.
+    pub fn to_wasmtime_config(&self) -> anyhow::Result<wasmtime::Config> {
+        let mut config = wasmtime::Config::new();
+        config.strategy(match self.strategy {
+            DifferentialStrategy::Cranelift => wasmtime::Strategy::Cranelift,
+            DifferentialStrategy::Lightbeam => wasmtime::Strategy::Lightbeam,
+        })?;
+        config.cranelift_opt_level(match self.opt_level {
+            DifferentialOptLevel::None => wasmtime::OptLevel::None,
+            DifferentialOptLevel::Speed => wasmtime::OptLevel::Speed,
+            DifferentialOptLevel::SpeedAndSize => wasmtime::OptLevel::SpeedAndSize,
+        });
+        Ok(config)
+    }
+}
+
+#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, Hash)]
+enum DifferentialStrategy {
+    Cranelift,
+    Lightbeam,
+}
+
+#[derive(Arbitrary, Clone, Debug, PartialEq, Eq, Hash)]
+enum DifferentialOptLevel {
+    None,
+    Speed,
+    SpeedAndSize,
+}

crates/fuzzing/src/oracles.rs

@@ -12,8 +12,8 @@
 
 pub mod dummy;
 
-use dummy::{dummy_imports, dummy_value};
+use dummy::{dummy_imports, dummy_values};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use wasmtime::*;
 
 /// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected
@@ -67,6 +67,180 @@ pub fn compile(wasm: &[u8], strategy: Strategy) {
     let _ = Module::new(&store, wasm);
 }
 
+/// Instantiate the given Wasm module with each `Config` and call all of its
+/// exports. Modulo OOM, non-canonical NaNs, and usage of Wasm features that are
+/// or aren't enabled for different configs, we should get the same results when
+/// we call the exported functions for all of our different configs.
+pub fn differential_execution(
+    ttf: &crate::generators::WasmOptTtf,
+    configs: &[crate::generators::DifferentialConfig],
+) {
+    // We need at least two configs.
+    if configs.len() < 2
+        // And all the configs should be unique.
+        || configs.iter().collect::<HashSet<_>>().len() != configs.len()
+    {
+        return;
+    }
+
+    let configs: Vec<_> = match configs.iter().map(|c| c.to_wasmtime_config()).collect() {
+        Ok(cs) => cs,
+        // If the config is trying to use something that was turned off at
+        // compile time, eg lightbeam, just continue to the next fuzz input.
+        Err(_) => return,
+    };
+
+    let mut export_func_results: HashMap<String, Result<Box<[Val]>, Trap>> = Default::default();
+
+    for config in &configs {
+        let engine = Engine::new(config);
+        let store = Store::new(&engine);
+
+        let module = match Module::new(&store, &ttf.wasm) {
+            Ok(module) => module,
+            // The module might rely on some feature that our config didn't
+            // enable or something like that.
+            Err(e) => {
+                eprintln!("Warning: failed to compile `wasm-opt -ttf` module: {}", e);
+                continue;
+            }
+        };
+
+        // TODO: we should implement tracing versions of these dummy imports
+        // that record a trace of the order that imported functions were called
+        // in and with what values. Like the results of exported functions,
+        // calls to imports should also yield the same values for each
+        // configuration, and we should assert that.
+        let imports = match dummy_imports(&store, module.imports()) {
+            Ok(imps) => imps,
+            Err(e) => {
+                // There are some value types that we can't synthesize a
+                // dummy value for (e.g. anyrefs) and for modules that
+                // import things of these types we skip instantiation.
+                eprintln!("Warning: failed to synthesize dummy imports: {}", e);
+                continue;
+            }
+        };
+
+        // Don't unwrap this: there can be instantiation-/link-time errors that
+        // aren't caught during validation or compilation. For example, an imported
+        // table might not have room for an element segment that we want to
+        // initialize into it.
+        let instance = match Instance::new(&module, &imports) {
+            Ok(instance) => instance,
+            Err(e) => {
+                eprintln!(
+                    "Warning: failed to instantiate `wasm-opt -ttf` module: {}",
+                    e
+                );
+                continue;
+            }
+        };
+
+        let funcs = module
+            .exports()
+            .iter()
+            .filter_map(|e| {
+                if let ExternType::Func(_) = e.ty() {
+                    Some(e.name())
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+
+        for name in funcs {
+            // Always call the hang limit initializer first, so that we don't
+            // infinite loop when calling another export.
+            init_hang_limit(&instance);
+
+            let f = match instance
+                .get_export(&name)
+                .expect("instance should have export from module")
+            {
+                Extern::Func(f) => f.clone(),
+                _ => panic!("export should be a function"),
+            };
+
+            let ty = f.ty();
+            let params = match dummy_values(ty.params()) {
+                Ok(p) => p,
+                Err(_) => continue,
+            };
+            let this_result = f.call(&params);
+
+            let existing_result = export_func_results
+                .entry(name.to_string())
+                .or_insert_with(|| this_result.clone());
+            assert_same_export_func_result(&existing_result, &this_result, name);
+        }
+    }
+}
+
+fn init_hang_limit(instance: &Instance) {
+    match instance.get_export("hangLimitInitializer") {
+        None => return,
+        Some(Extern::Func(f)) => {
+            f.call(&[])
+                .expect("initializing the hang limit should not fail");
+        }
+        Some(_) => panic!("unexpected hangLimitInitializer export"),
+    }
+}
+
+fn assert_same_export_func_result(
+    lhs: &Result<Box<[Val]>, Trap>,
+    rhs: &Result<Box<[Val]>, Trap>,
+    func_name: &str,
+) {
+    let fail = || {
+        panic!(
+            "differential fuzzing failed: exported func {} returned two \
+             different results: {:?} != {:?}",
+            func_name, lhs, rhs
+        )
+    };
+
+    match (lhs, rhs) {
+        (Err(_), Err(_)) => {}
+        (Ok(lhs), Ok(rhs)) => {
+            if lhs.len() != rhs.len() {
+                fail();
+            }
+            for (lhs, rhs) in lhs.iter().zip(rhs.iter()) {
+                match (lhs, rhs) {
+                    (Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue,
+                    (Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue,
+                    (Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue,
+                    (Val::F32(lhs), Val::F32(rhs)) => {
+                        let lhs = f32::from_bits(*lhs);
+                        let rhs = f32::from_bits(*rhs);
+                        if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) {
+                            continue;
+                        } else {
+                            fail()
+                        }
+                    }
+                    (Val::F64(lhs), Val::F64(rhs)) => {
+                        let lhs = f64::from_bits(*lhs);
+                        let rhs = f64::from_bits(*rhs);
+                        if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) {
+                            continue;
+                        } else {
+                            fail()
+                        }
+                    }
+                    (Val::AnyRef(_), Val::AnyRef(_)) | (Val::FuncRef(_), Val::FuncRef(_)) => {
+                        continue
+                    }
+                    _ => fail(),
+                }
+            }
+        }
+        _ => fail(),
+    }
+}
+
 /// Invoke the given API calls.
 pub fn make_api_calls(api: crate::generators::api::ApiCalls) {
     use crate::generators::api::ApiCall;
@@ -170,12 +344,7 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) {
                 let nth = nth % funcs.len();
                 let f = &funcs[nth];
                 let ty = f.ty();
-                let params = match ty
+                let params = match dummy_values(ty.params()) {
-                    .params()
-                    .iter()
-                    .map(|valty| dummy_value(valty))
-                    .collect::<Result<Vec<_>, _>>()
-                {
                     Ok(p) => p,
                     Err(_) => continue,
                 };

crates/fuzzing/src/oracles/dummy.rs

@@ -70,6 +70,11 @@ pub fn dummy_value(val_ty: &ValType) -> Result<Val, Trap> {
     })
 }
 
+/// Construct a sequence of dummy values for the given types.
+pub fn dummy_values(val_tys: &[ValType]) -> Result<Vec<Val>, Trap> {
+    val_tys.iter().map(dummy_value).collect()
+}
+
 /// Construct a dummy global for the given global type.
 pub fn dummy_global(store: &Store, ty: GlobalType) -> Result<Global, Trap> {
     let val = dummy_value(ty.content())?;

fuzz/Cargo.toml

@@ -39,3 +39,9 @@ name = "api_calls"
 path = "fuzz_targets/api_calls.rs"
 test = false
 doc = false
+
+[[bin]]
+name = "differential"
+path = "fuzz_targets/differential.rs"
+test = false
+doc = false

fuzz/fuzz_targets/differential.rs

@@ -0,0 +1,13 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use wasmtime_fuzzing::{generators, oracles};
+
+fuzz_target!(|data: (
+    generators::DifferentialConfig,
+    generators::DifferentialConfig,
+    generators::WasmOptTtf
+)| {
+    let (lhs, rhs, wasm) = data;
+    oracles::differential_execution(&wasm, &[lhs, rhs]);
+});