Add a spec test fuzzer for Config (#1509)

* Add a spec test fuzzer for Config

This commit adds a new fuzzer which is intended to run on oss-fuzz. This
fuzzer creates and arbitrary `Config` which *should* pass spec tests and
then asserts that it does so. The goal here is to weed out any
accidental bugs in global configuration which could cause
non-spec-compliant behavior.

* Move implementation to `fuzzing` crate

Cargo.lock

@@ -2251,6 +2251,7 @@ dependencies = [
  "wasmparser",
  "wasmprinter",
  "wasmtime",
+ "wasmtime-wast",
  "wat",
 ]
 

crates/fuzzing/Cargo.toml

@@ -15,7 +15,8 @@ log = "0.4.8"
 rayon = "1.2.1"
 wasmparser = "0.51.2"
 wasmprinter = "0.2.1"
-wasmtime = { path = "../api", version = "0.15.0" }
+wasmtime = { path = "../api" }
+wasmtime-wast = { path = "../wast" }
 
 [dev-dependencies]
 wat = "1.0.10"

crates/fuzzing/build.rs

@@ -0,0 +1,26 @@
+// A small build script to include the contents of the spec test suite into the
+// final fuzzing binary so the fuzzing binary can be run elsewhere and doesn't
+// rely on the original source tree.
+
+use std::env;
+use std::path::PathBuf;
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+
+    let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
+    let dir = env::current_dir()
+        .unwrap()
+        .join("../../tests/spec_testsuite");
+    let mut code = format!("static FILES: &[(&str, &str)] = &[\n");
+    for entry in dir.read_dir().unwrap() {
+        let entry = entry.unwrap();
+        let path = entry.path().display().to_string();
+        if !path.ends_with(".wast") {
+            continue;
+        }
+        code.push_str(&format!("({:?}, include_str!({0:?})),\n", path));
+    }
+    code.push_str("];\n");
+    std::fs::write(out_dir.join("spectests.rs"), code).unwrap();
+}

crates/fuzzing/src/generators.rs

@@ -11,7 +11,7 @@
 #[cfg(feature = "binaryen")]
 pub mod api;
 
-use arbitrary::Arbitrary;
+use arbitrary::{Arbitrary, Unstructured};
 
 /// A Wasm test case generator that is powered by Binaryen's `wasm-opt -ttf`.
 #[derive(Clone)]
@@ -60,7 +60,7 @@ impl Arbitrary for WasmOptTtf {
 #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, Hash)]
 pub struct DifferentialConfig {
     strategy: DifferentialStrategy,
-    opt_level: DifferentialOptLevel,
+    opt_level: OptLevel,
 }
 
 impl DifferentialConfig {
@@ -70,11 +70,7 @@ impl DifferentialConfig {
             DifferentialStrategy::Cranelift => wasmtime::Strategy::Cranelift,
             DifferentialStrategy::Lightbeam => wasmtime::Strategy::Lightbeam,
         })?;
-        config.cranelift_opt_level(match self.opt_level {
-            DifferentialOptLevel::None => wasmtime::OptLevel::None,
-            DifferentialOptLevel::Speed => wasmtime::OptLevel::Speed,
-            DifferentialOptLevel::SpeedAndSize => wasmtime::OptLevel::SpeedAndSize,
-        });
+        config.cranelift_opt_level(self.opt_level.to_wasmtime());
         Ok(config)
     }
 }
@@ -86,8 +82,65 @@ enum DifferentialStrategy {
 }
 
 #[derive(Arbitrary, Clone, Debug, PartialEq, Eq, Hash)]
-enum DifferentialOptLevel {
+enum OptLevel {
     None,
     Speed,
     SpeedAndSize,
 }
+
+impl OptLevel {
+    fn to_wasmtime(&self) -> wasmtime::OptLevel {
+        match self {
+            OptLevel::None => wasmtime::OptLevel::None,
+            OptLevel::Speed => wasmtime::OptLevel::Speed,
+            OptLevel::SpeedAndSize => wasmtime::OptLevel::SpeedAndSize,
+        }
+    }
+}
+
+/// Implementation of generating a `wasmtime::Config` arbitrarily
+#[derive(Arbitrary, Debug)]
+pub struct Config {
+    opt_level: OptLevel,
+    debug_verifier: bool,
+    debug_info: bool,
+    canonicalize_nans: bool,
+    spectest: usize,
+}
+
+impl Config {
+    /// Converts this to a `wasmtime::Config` object
+    pub fn to_wasmtime(&self) -> wasmtime::Config {
+        let mut cfg = wasmtime::Config::new();
+        cfg.debug_info(self.debug_info)
+            .cranelift_nan_canonicalization(self.canonicalize_nans)
+            .cranelift_debug_verifier(self.debug_verifier)
+            .cranelift_opt_level(self.opt_level.to_wasmtime());
+        return cfg;
+    }
+}
+
+include!(concat!(env!("OUT_DIR"), "/spectests.rs"));
+
+/// A spec test from the upstream wast testsuite, arbitrarily chosen from the
+/// list of known spec tests.
+#[derive(Debug)]
+pub struct SpecTest {
+    /// The filename of the spec test
+    pub file: &'static str,
+    /// The `*.wast` contents of the spec test
+    pub contents: &'static str,
+}
+
+impl Arbitrary for SpecTest {
+    fn arbitrary(u: &mut Unstructured) -> arbitrary::Result<Self> {
+        // NB: this does get a uniform value in the provided range.
+        let i = u.int_in_range(0..=FILES.len() - 1)?;
+        let (file, contents) = FILES[i];
+        Ok(SpecTest { file, contents })
+    }
+
+    fn size_hint(_depth: usize) -> (usize, Option<usize>) {
+        (1, Some(std::mem::size_of::<usize>()))
+    }
+}

crates/fuzzing/src/oracles.rs

@@ -15,6 +15,7 @@ pub mod dummy;
 use dummy::dummy_imports;
 use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
 use wasmtime::*;
+use wasmtime_wast::WastContext;
 
 fn log_wasm(wasm: &[u8]) {
     static CNT: AtomicUsize = AtomicUsize::new(0);
@@ -400,3 +401,15 @@ pub fn make_api_calls(api: crate::generators::api::ApiCalls) {
         }
     }
 }
+
+/// Executes the wast `test` spectest with the `config` specified.
+///
+/// Ensures that spec tests pass regardless of the `Config`.
+pub fn spectest(config: crate::generators::Config, test: crate::generators::SpecTest) {
+    let store = Store::new(&Engine::new(&config.to_wasmtime()));
+    let mut wast_context = WastContext::new(store);
+    wast_context.register_spectest().unwrap();
+    wast_context
+        .run_buffer(test.file, test.contents.as_bytes())
+        .unwrap();
+}

fuzz/Cargo.toml

@@ -50,5 +50,11 @@ test = false
 doc = false
 required-features = ['binaryen']
 
+[[bin]]
+name = "spectests"
+path = "fuzz_targets/spectests.rs"
+test = false
+doc = false
+
 [features]
 binaryen = ['wasmtime-fuzzing/binaryen']

fuzz/fuzz_targets/spectests.rs

@@ -0,0 +1,9 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use wasmtime_fuzzing::generators::{Config, SpecTest};
+
+fuzz_target!(|pair: (Config, SpecTest)| {
+    let (config, test) = pair;
+    wasmtime_fuzzing::oracles::spectest(config, test);
+});