Merge pull request #2453 from cfallin/differential-fuzz-interp

Add differential fuzzing against wasmi (a Wasm interpreter).

Cargo.lock

@@ -766,6 +766,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "downcast-rs"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
+
 [[package]]
 name = "dynasm"
 version = "1.0.0"
@@ -1238,6 +1244,12 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "memory_units"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.4.3"
@@ -1264,6 +1276,17 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.44"
@@ -1274,6 +1297,18 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "num-rational"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
+dependencies = [
+ "autocfg",
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.14"
@@ -1352,6 +1387,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "parity-wasm"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865"
+
 [[package]]
 name = "peeking_take_while"
 version = "0.1.2"
@@ -2313,13 +2354,47 @@ dependencies = [
 ]
 
 [[package]]
-name = "wasm-smith"
-version = "0.1.10"
+name = "wasm-encoder"
+version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd5e4720bb44dc5e46a917139dc9dfa4bb6fee023bf9f77d2c55ec12eeaf9930"
+checksum = "49891b7c581cf9e0090b25cd274e6498ad478f0a7819319ea96da2f253caaacb"
+dependencies = [
+ "leb128",
+]
+
+[[package]]
+name = "wasm-smith"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7fdf8c9ba2fdc0d8ffe3f7e5b23b5aac377eeca817a9885c058f27c8de5c500"
 dependencies = [
  "arbitrary",
  "leb128",
+ "wasm-encoder",
+]
+
+[[package]]
+name = "wasmi"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bb6825d9b2147105789adb4c2d84b9b568719713f3ac39618b637b4dafc86c4"
+dependencies = [
+ "downcast-rs",
+ "libc",
+ "memory_units",
+ "num-rational",
+ "num-traits",
+ "parity-wasm",
+ "wasmi-validation",
+]
+
+[[package]]
+name = "wasmi-validation"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93"
+dependencies = [
+ "parity-wasm",
 ]
 
 [[package]]
@@ -2522,6 +2597,7 @@ dependencies = [
  "log",
  "rayon",
  "wasm-smith",
+ "wasmi",
  "wasmparser 0.68.0",
  "wasmprinter",
  "wasmtime",

crates/fuzzing/Cargo.toml

@@ -16,7 +16,11 @@ wasmparser = "0.68.0"
 wasmprinter = "0.2.15"
 wasmtime = { path = "../wasmtime" }
 wasmtime-wast = { path = "../wast" }
-wasm-smith = "0.1.10"
+wasm-smith = "0.1.12"
+wasmi = "0.7.0"
 
 [dev-dependencies]
 wat = "1.0.28"
+
+[features]
+experimental_x64 = ["wasmtime/experimental_x64"]

crates/fuzzing/src/oracles.rs

@@ -12,7 +12,9 @@
 
 pub mod dummy;
 
+use arbitrary::Arbitrary;
 use dummy::dummy_imports;
+use log::debug;
 use std::cell::Cell;
 use std::rc::Rc;
 use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
@@ -249,24 +251,8 @@ pub fn differential_execution(
                         (Val::I32(lhs), Val::I32(rhs)) if lhs == rhs => continue,
                         (Val::I64(lhs), Val::I64(rhs)) if lhs == rhs => continue,
                         (Val::V128(lhs), Val::V128(rhs)) if lhs == rhs => continue,
-                        (Val::F32(lhs), Val::F32(rhs)) => {
-                            let lhs = f32::from_bits(*lhs);
-                            let rhs = f32::from_bits(*rhs);
-                            if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) {
-                                continue;
-                            } else {
-                                fail()
-                            }
-                        }
-                        (Val::F64(lhs), Val::F64(rhs)) => {
-                            let lhs = f64::from_bits(*lhs);
-                            let rhs = f64::from_bits(*rhs);
-                            if lhs == rhs || (lhs.is_nan() && rhs.is_nan()) {
-                                continue;
-                            } else {
-                                fail()
-                            }
-                        }
+                        (Val::F32(lhs), Val::F32(rhs)) if f32_equal(*lhs, *rhs) => continue,
+                        (Val::F64(lhs), Val::F64(rhs)) if f64_equal(*lhs, *rhs) => continue,
                         (Val::ExternRef(_), Val::ExternRef(_))
                         | (Val::FuncRef(_), Val::FuncRef(_)) => continue,
                         _ => fail(),
@@ -278,6 +264,18 @@ pub fn differential_execution(
     }
 }
 
+fn f32_equal(a: u32, b: u32) -> bool {
+    let a = f32::from_bits(a);
+    let b = f32::from_bits(b);
+    a == b || (a.is_nan() && b.is_nan())
+}
+
+fn f64_equal(a: u64, b: u64) -> bool {
+    let a = f64::from_bits(a);
+    let b = f64::from_bits(b);
+    a == b || (a.is_nan() && b.is_nan())
+}
+
 /// Invoke the given API calls.
 pub fn make_api_calls(api: crate::generators::api::ApiCalls) {
     use crate::generators::api::ApiCall;
@@ -479,3 +477,171 @@ pub fn table_ops(config: crate::generators::Config, ops: crate::generators::tabl
         }
     }
 }
+
+/// Configuration options for wasm-smith such that generated modules always
+/// conform to certain specifications.
+#[derive(Default, Debug, Arbitrary)]
+pub struct DifferentialWasmiModuleConfig;
+
+impl wasm_smith::Config for DifferentialWasmiModuleConfig {
+    fn allow_start_export(&self) -> bool {
+        false
+    }
+
+    fn min_funcs(&self) -> usize {
+        1
+    }
+
+    fn max_funcs(&self) -> usize {
+        1
+    }
+
+    fn min_memories(&self) -> u32 {
+        1
+    }
+
+    fn max_memories(&self) -> u32 {
+        1
+    }
+
+    fn max_imports(&self) -> usize {
+        0
+    }
+
+    fn min_exports(&self) -> usize {
+        2
+    }
+
+    fn max_memory_pages(&self) -> u32 {
+        1
+    }
+
+    fn memory_max_size_required(&self) -> bool {
+        true
+    }
+}
+
+/// Perform differential execution between Cranelift and wasmi, diffing the
+/// resulting memory image when execution terminates. This relies on the
+/// module-under-test to be instrumented to bound the execution time. Invoke
+/// with a module generated by `wasm-smith` using the
+/// `DiferentialWasmiModuleConfig` configuration type for best results.
+///
+/// May return `None` if we early-out due to a rejected fuzz config; these
+/// should be rare if modules are generated appropriately.
+pub fn differential_wasmi_execution(wasm: &[u8], config: &crate::generators::Config) -> Option<()> {
+    crate::init_fuzzing();
+
+    // Instantiate wasmi module and instance.
+    let wasmi_module = wasmi::Module::from_buffer(&wasm[..]).ok()?;
+    let wasmi_instance =
+        wasmi::ModuleInstance::new(&wasmi_module, &wasmi::ImportsBuilder::default()).ok()?;
+    let wasmi_instance = wasmi_instance.assert_no_start();
+
+    // TODO(paritytech/wasmi#19): wasmi does not currently canonicalize NaNs. To avoid spurious
+    // fuzz failures, for now let's fuzz only integer Wasm programs.
+    if wasmi_module.deny_floating_point().is_err() {
+        return None;
+    }
+
+    // Instantiate wasmtime module and instance.
+    let mut wasmtime_config = config.to_wasmtime();
+    wasmtime_config.cranelift_nan_canonicalization(true);
+    let wasmtime_engine = Engine::new(&wasmtime_config);
+    let wasmtime_store = Store::new(&wasmtime_engine);
+    let wasmtime_module =
+        Module::new(&wasmtime_engine, &wasm).expect("Wasmtime can compile module");
+    let wasmtime_instance = Instance::new(&wasmtime_store, &wasmtime_module, &[])
+        .expect("Wasmtime can instantiate module");
+
+    // Introspect wasmtime module to find name of an exported function and of an
+    // exported memory. Stop when we have one of each. (According to the config
+    // above, there should be at most one of each.)
+    let (func_name, memory_name) = {
+        let mut func_name = None;
+        let mut memory_name = None;
+        for e in wasmtime_module.exports() {
+            match e.ty() {
+                wasmtime::ExternType::Func(..) => func_name = Some(e.name().to_string()),
+                wasmtime::ExternType::Memory(..) => memory_name = Some(e.name().to_string()),
+                _ => {}
+            }
+            if func_name.is_some() && memory_name.is_some() {
+                break;
+            }
+        }
+        (func_name?, memory_name?)
+    };
+
+    let wasmi_mem_export = wasmi_instance.export_by_name(&memory_name[..]).unwrap();
+    let wasmi_mem = wasmi_mem_export.as_memory().unwrap();
+    let wasmi_main_export = wasmi_instance.export_by_name(&func_name[..]).unwrap();
+    let wasmi_main = wasmi_main_export.as_func().unwrap();
+    let wasmi_val = wasmi::FuncInstance::invoke(&wasmi_main, &[], &mut wasmi::NopExternals);
+
+    let wasmtime_mem = wasmtime_instance
+        .get_memory(&memory_name[..])
+        .expect("memory export is present");
+    let wasmtime_main = wasmtime_instance
+        .get_func(&func_name[..])
+        .expect("function export is present");
+    let wasmtime_vals = wasmtime_main.call(&[]);
+    let wasmtime_val = wasmtime_vals.map(|v| v.iter().next().cloned());
+
+    debug!(
+        "Successful execution: wasmi returned {:?}, wasmtime returned {:?}",
+        wasmi_val, wasmtime_val
+    );
+
+    let show_wat = || {
+        if let Ok(s) = wasmprinter::print_bytes(&wasm[..]) {
+            eprintln!("wat:\n{}\n", s);
+        }
+    };
+
+    match (&wasmi_val, &wasmtime_val) {
+        (&Ok(Some(wasmi::RuntimeValue::I32(a))), &Ok(Some(Val::I32(b)))) if a == b => {}
+        (&Ok(Some(wasmi::RuntimeValue::F32(a))), &Ok(Some(Val::F32(b))))
+            if f32_equal(a.to_bits(), b) => {}
+        (&Ok(Some(wasmi::RuntimeValue::I64(a))), &Ok(Some(Val::I64(b)))) if a == b => {}
+        (&Ok(Some(wasmi::RuntimeValue::F64(a))), &Ok(Some(Val::F64(b))))
+            if f64_equal(a.to_bits(), b) => {}
+        (&Ok(None), &Ok(None)) => {}
+        (&Err(_), &Err(_)) => {}
+        _ => {
+            show_wat();
+            panic!(
+                "Values do not match: wasmi returned {:?}; wasmtime returned {:?}",
+                wasmi_val, wasmtime_val
+            );
+        }
+    }
+
+    if wasmi_mem.current_size().0 != wasmtime_mem.size() as usize {
+        show_wat();
+        panic!("resulting memories are not the same size");
+    }
+
+    // Wasmi memory may be stored non-contiguously; copy it out to a contiguous chunk.
+    let mut wasmi_buf: Vec<u8> = vec![0; wasmtime_mem.data_size()];
+    wasmi_mem
+        .get_into(0, &mut wasmi_buf[..])
+        .expect("can access wasmi memory");
+
+    let wasmtime_slice = unsafe { wasmtime_mem.data_unchecked() };
+
+    if wasmi_buf.len() >= 64 {
+        debug!("-> First 64 bytes of wasmi heap: {:?}", &wasmi_buf[0..64]);
+        debug!(
+            "-> First 64 bytes of Wasmtime heap: {:?}",
+            &wasmtime_slice[0..64]
+        );
+    }
+
+    if &wasmi_buf[..] != &wasmtime_slice[..] {
+        show_wat();
+        panic!("memory contents are not equal");
+    }
+
+    Some(())
+}

crates/wasmtime/Cargo.toml

@@ -59,3 +59,6 @@ parallel-compilation = ["wasmtime-jit/parallel-compilation"]
 
 # Enables support for automatic cache configuration to be enabled in `Config`.
 cache = ["wasmtime-cache"]
+
+# Enables support for new x64 backend.
+experimental_x64 = ["wasmtime-jit/experimental_x64"]

fuzz/Cargo.toml

@@ -17,7 +17,10 @@ target-lexicon = "0.11"
 peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing", optional = true }
 wasmtime = { path = "../crates/wasmtime" }
 wasmtime-fuzzing = { path = "../crates/fuzzing" }
-wasm-smith = "0.1.5"
+wasm-smith = "0.1.12"
+
+[features]
+experimental_x64 = ["wasmtime-fuzzing/experimental_x64"]
 
 [[bin]]
 name = "compile"
@@ -43,6 +46,12 @@ path = "fuzz_targets/differential.rs"
 test = false
 doc = false
 
+[[bin]]
+name = "differential_wasmi"
+path = "fuzz_targets/differential_wasmi.rs"
+test = false
+doc = false
+
 [[bin]]
 name = "spectests"
 path = "fuzz_targets/spectests.rs"

fuzz/fuzz_targets/differential_wasmi.rs

@@ -0,0 +1,13 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use wasmtime_fuzzing::{generators, oracles};
+
+fuzz_target!(|data: (
+    generators::Config,
+    wasm_smith::ConfiguredModule<oracles::DifferentialWasmiModuleConfig>
+)| {
+    let (config, mut wasm) = data;
+    wasm.ensure_termination(1000);
+    oracles::differential_wasmi_execution(&wasm.to_bytes()[..], &config);
+});