Fuzzing: Add test case logging and regression test template

When the test case that causes the failure can successfully be disassembled to
WAT, we get logs like this:

```
[2019-11-26T18:48:46Z INFO  wasmtime_fuzzing] Wrote WAT disassembly to: /home/fitzgen/wasmtime/crates/fuzzing/target/scratch/8437-0.wat
[2019-11-26T18:48:46Z INFO  wasmtime_fuzzing] If this fuzz test fails, copy `/home/fitzgen/wasmtime/crates/fuzzing/target/scratch/8437-0.wat` to `wasmtime/crates/fuzzing/tests/regressions/my-regression.wat` and add the following test to `wasmtime/crates/fuzzing/tests/regressions.rs`:

    ```
    #[test]
    fn my_fuzzing_regression_test() {
        let data = wat::parse_str(
            include_str!("./regressions/my-regression.wat")
        ).unwrap();
        oracles::instantiate(data, CompilationStrategy::Auto)
    }
    ```
```

If the test case cannot be disassembled to WAT, then we get logs like this:

```
[2019-11-26T18:48:46Z INFO  wasmtime_fuzzing] Wrote Wasm test case to: /home/fitzgen/wasmtime/crates/fuzzing/target/scratch/8437-0.wasm
[2019-11-26T18:48:46Z INFO  wasmtime_fuzzing] Failed to disassemble Wasm into WAT:
    Bad magic number (at offset 0)

    Stack backtrace:
        Run with RUST_LIB_BACKTRACE=1 env variable to display a backtrace

[2019-11-26T18:48:46Z INFO  wasmtime_fuzzing] If this fuzz test fails, copy `/home/fitzgen/wasmtime/crates/fuzzing/target/scratch/8437-0.wasm` to `wasmtime/crates/fuzzing/tests/regressions/my-regression.wasm` and add the following test to `wasmtime/crates/fuzzing/tests/regressions.rs`:

    ```
    #[test]
    fn my_fuzzing_regression_test() {
        let data = include_bytes!("./regressions/my-regression.wasm");
        oracles::instantiate(data, CompilationStrategy::Auto)
    }
    ```
```

crates/fuzzing/Cargo.toml

@@ -9,9 +9,13 @@ version = "0.1.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = "1.0.22"
 arbitrary = "0.2.0"
 binaryen = "0.8.2"
 cranelift-codegen = "0.50.0"
 cranelift-native = "0.50.0"
+env_logger = { version = "0.7.1", optional = true }
+log = "0.4.8"
 wasmparser = "0.42.1"
+wasmprinter = "0.2.0"
 wasmtime-jit = { path = "../jit" }

crates/fuzzing/src/generators.rs

@@ -11,6 +11,7 @@
 use arbitrary::{Arbitrary, Unstructured};
 
 /// A Wasm test case generator that is powered by Binaryen's `wasm-opt -ttf`.
+#[derive(Debug)]
 pub struct WasmOptTtf {
     /// The raw, encoded Wasm bytes.
     pub wasm: Vec<u8>,

crates/fuzzing/src/lib.rs

@@ -1,2 +1,165 @@
+//! Fuzzing infrastructure for Wasmtime.
+
+#![deny(missing_docs, missing_debug_implementations)]
+
 pub mod generators;
 pub mod oracles;
+
+use anyhow::Context;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process;
+use std::sync::{atomic, Once};
+
+/// Run a fuzz test on Wasm test case with automatic logging.
+///
+/// This is intended for defining the body of a `libfuzzer_sys::fuzz_target!`
+/// invocation.
+///
+/// Automatically prints out how to create a regression test that runs the exact
+/// same set of oracles.
+///
+/// It also binds the expression getting the wasm bytes to the variable, for
+/// example below the `wasm` variable is assigned the value
+/// `&my_test_case.as_wasm_bytes()`. This variable can be used within the body.
+///
+/// ```ignore
+/// use wasmtime_fuzzing::{oracles, with_log_wasm_test_case};
+///
+/// with_log_wasm_test_case!(&my_test_case.as_wasm_bytes(), |wasm| {
+///     oracles::compile(wasm);
+///     oracles::instantiate(wasm);
+/// });
+/// ```
+#[macro_export]
+macro_rules! with_log_wasm_test_case {
+    ( $wasm:expr , |$wasm_var:ident| $oracle:expr ) => {{
+        let $wasm_var = $wasm;
+        wasmtime_fuzzing::log_wasm_test_case(
+            &$wasm_var,
+            stringify!($wasm_var),
+            stringify!($oracle),
+        );
+        $oracle;
+    }};
+}
+
+/// Given that we are going to do a fuzz test of the given Wasm buffer, log the
+/// Wasm and its WAT disassembly, and preserve them to the filesystem so that if
+/// we panic or crash, we can easily inspect the test case.
+///
+/// This is intended to be used via the `with_log_wasm_test_case` macro.
+pub fn log_wasm_test_case(wasm: &[u8], wasm_var: &'static str, oracle_expr: &'static str) {
+    init_logging();
+
+    let wasm_path = wasm_test_case_path();
+    fs::write(&wasm_path, wasm)
+        .with_context(|| format!("Failed to write wasm to {}", wasm_path.display()))
+        .unwrap();
+    log::info!("Wrote Wasm test case to: {}", wasm_path.display());
+
+    match wasmprinter::print_bytes(wasm) {
+        Ok(wat) => {
+            log::info!("WAT disassembly:\n{}", wat);
+
+            let wat_path = wat_disassembly_path();
+            fs::write(&wat_path, &wat)
+                .with_context(|| {
+                    format!("Failed to write WAT disassembly to {}", wat_path.display())
+                })
+                .unwrap();
+            log::info!("Wrote WAT disassembly to: {}", wat_path.display());
+
+            log::info!(
+                "If this fuzz test fails, copy `{wat_path}` to `wasmtime/crates/fuzzing/tests/regressions/my-regression.wat` and add the following test to `wasmtime/crates/fuzzing/tests/regressions.rs`:
+
+```
+#[test]
+fn my_fuzzing_regression_test() {{
+    let {wasm_var} = wat::parse_str(
+        include_str!(\"./regressions/my-regression.wat\")
+    ).unwrap();
+    {oracle_expr}
+}}
+```",
+                wat_path = wat_path.display(),
+                wasm_var = wasm_var,
+                oracle_expr = oracle_expr
+            );
+        }
+        Err(e) => {
+            log::info!("Failed to disassemble Wasm into WAT:\n{:?}", e);
+            log::info!(
+                "If this fuzz test fails, copy `{wasm_path}` to `wasmtime/crates/fuzzing/tests/regressions/my-regression.wasm` and add the following test to `wasmtime/crates/fuzzing/tests/regressions.rs`:
+
+```
+#[test]
+fn my_fuzzing_regression_test() {{
+    let {wasm_var} = include_bytes!(\"./regressions/my-regression.wasm\");
+    {oracle_expr}
+}}
+```",
+                wasm_path = wasm_path.display(),
+                wasm_var = wasm_var,
+                oracle_expr = oracle_expr
+            );
+        }
+    }
+}
+
+fn scratch_dir() -> PathBuf {
+    let dir = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("target")
+        .join("scratch");
+
+    static CREATE: Once = Once::new();
+    CREATE.call_once(|| {
+        fs::create_dir_all(&dir)
+            .with_context(|| format!("Failed to create {}", dir.display()))
+            .unwrap();
+    });
+
+    dir
+}
+
+fn wasm_test_case_path() -> PathBuf {
+    static WASM_TEST_CASE_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(0);
+
+    thread_local! {
+        static WASM_TEST_CASE_PATH: PathBuf = {
+            let dir = scratch_dir();
+            dir.join(format!("{}-{}.wasm",
+                             process::id(),
+                             WASM_TEST_CASE_COUNTER.fetch_add(1, atomic::Ordering::SeqCst)
+            ))
+        };
+    }
+
+    WASM_TEST_CASE_PATH.with(|p| p.clone())
+}
+
+fn wat_disassembly_path() -> PathBuf {
+    static WAT_DISASSEMBLY_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(0);
+
+    thread_local! {
+        static WAT_DISASSEMBLY_PATH: PathBuf = {
+            let dir = scratch_dir();
+            dir.join(format!(
+                "{}-{}.wat",
+                process::id(),
+                WAT_DISASSEMBLY_COUNTER.fetch_add(1, atomic::Ordering::SeqCst)
+            ))
+        };
+    }
+
+    WAT_DISASSEMBLY_PATH.with(|p| p.clone())
+}
+
+#[cfg(feature = "env_logger")]
+fn init_logging() {
+    static INIT_LOGGING: Once = Once::new();
+    INIT_LOGGING.call_once(|| env_logger::init());
+}
+
+#[cfg(not(feature = "env_logger"))]
+fn init_logging() {}

crates/fuzzing/tests/regressions.rs

@@ -0,0 +1,9 @@
+//! Regression tests for bugs found via fuzzing.
+//!
+//! The `#[test]` goes in here, the Wasm binary goes in
+//! `./regressions/some-descriptive-name.wasm`, and then the `#[test]` should
+//! use the Wasm binary by including it via
+//! `include_bytes!("./regressions/some-descriptive-name.wasm")`.
+
+#[allow(unused_imports)] // Until we actually have some regression tests...
+use wasmtime_fuzzing::*;

crates/fuzzing/tests/regressions/README.md

@@ -0,0 +1,2 @@
+This directory contains `.wasm` binaries generated during fuzzing that uncovered
+a bug, and which we now use as regression tests in `../regressions.rs`.

fuzz/Cargo.toml

@@ -10,7 +10,7 @@ cargo-fuzz = true
 
 [dependencies]
 arbitrary = "0.2.0"
-wasmtime-fuzzing = { path = "../crates/fuzzing" }
+wasmtime-fuzzing = { path = "../crates/fuzzing", features = ["env_logger"] }
 wasmtime-jit = { path = "../crates/jit" }
 libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer-sys.git" }
 

fuzz/fuzz_targets/compile.rs

@@ -1,14 +1,20 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use wasmtime_fuzzing::oracles;
+use wasmtime_fuzzing::{oracles, with_log_wasm_test_case};
 use wasmtime_jit::CompilationStrategy;
 
 fuzz_target!(|data: &[u8]| {
-    oracles::compile(data, CompilationStrategy::Cranelift);
+    with_log_wasm_test_case!(data, |data| oracles::compile(
+        data,
+        CompilationStrategy::Cranelift
+    ));
 });
 
 #[cfg(feature = "lightbeam")]
 fuzz_target!(|data: &[u8]| {
-    oracles::compile(data, CompilationStrategy::Lightbeam);
+    with_log_wasm_test_case!(data, |data| oracles::compile(
+        data,
+        CompilationStrategy::Lightbeam
+    ));
 });

fuzz/fuzz_targets/instantiate.rs

@@ -1,9 +1,12 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use wasmtime_fuzzing::oracles;
+use wasmtime_fuzzing::{oracles, with_log_wasm_test_case};
-use wasmtime_jit::{CompilationStrategy};
+use wasmtime_jit::CompilationStrategy;
 
 fuzz_target!(|data: &[u8]| {
-    oracles::instantiate(data, CompilationStrategy::Auto);
+    with_log_wasm_test_case!(data, |data| oracles::instantiate(
+        data,
+        CompilationStrategy::Auto
+    ));
 });

fuzz/fuzz_targets/instantiate_translated.rs

@@ -1,9 +1,12 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
-use wasmtime_fuzzing::{generators, oracles};
+use wasmtime_fuzzing::{generators, oracles, with_log_wasm_test_case};
 use wasmtime_jit::CompilationStrategy;
 
 fuzz_target!(|data: generators::WasmOptTtf| {
-    oracles::instantiate(&data.wasm, CompilationStrategy::Auto);
+    with_log_wasm_test_case!(&data.wasm, |wasm| oracles::instantiate(
+        wasm,
+        CompilationStrategy::Auto
+    ));
 });