Merge pull request #619 from fitzgen/introduce-wasmtime-fuzzing-crate

Introduce the `wasmtime-fuzzing` crate

.github/workflows/main.yml

@@ -95,6 +95,21 @@ jobs:
       with:
         toolchain: ${{ matrix.rust }}
 
+    - name: Install libclang
+      # Note: libclang is pre-installed on the macOS and linux images.
+      if: matrix.os == 'windows-latest'
+      run: |
+        Invoke-WebRequest http://releases.llvm.org/9.0.0/LLVM-9.0.0-win64.exe -OutFile llvm-installer.exe
+        7z x llvm-installer.exe -oC:\llvm-binary
+        Write-Host ::set-env name=LIBCLANG_PATH::C:\llvm-binary\bin\libclang.dll
+        Write-Host ::add-path::C:\llvm-binary\bin
+
+    - name: Query Clang Version
+      if: matrix.os == 'windows-latest'
+      run: |
+        Get-Command clang.exe
+        clang.exe --version
+
     # Install wasm32-wasi target in order to build wasi-common's integration
     # tests
     - run: rustup target add wasm32-wasi
@@ -219,7 +234,7 @@ jobs:
     - run: $CENTOS cargo build --release --manifest-path crates/api/Cargo.toml
       shell: bash
     # Test what we just built
-    - run: $CENTOS cargo test --features test_programs --release --all --exclude lightbeam --exclude wasmtime-wasi-c --exclude wasmtime-py --exclude wasmtime
+    - run: $CENTOS cargo test --features test_programs --release --all --exclude lightbeam --exclude wasmtime-wasi-c --exclude wasmtime-py --exclude wasmtime --exclude wasmtime-fuzzing
       shell: bash
       env:
         RUST_BACKTRACE: 1

Cargo.toml

@@ -54,6 +54,7 @@ anyhow = "1.0.19"
 
 [workspace]
 members = [
+  "crates/fuzzing",
   "crates/misc/rust",
   "crates/misc/py",
 ]

crates/fuzzing/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+authors = ["The Wasmtime Project Developers"]
+description = "Fuzzing infrastructure for Wasmtime"
+edition = "2018"
+name = "wasmtime-fuzzing"
+publish = false
+version = "0.1.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+arbitrary = "0.2.0"
+binaryen = "0.8.2"
+cranelift-codegen = "0.50.0"
+cranelift-native = "0.50.0"
+wasmparser = "0.42.1"
+wasmtime-jit = { path = "../jit" }

crates/fuzzing/README.md

@@ -0,0 +1,14 @@
+# Fuzzing Infrastructure for Wasmtime
+
+This crate provides test case generators and oracles for use with fuzzing.
+
+These generators and oracles are generally independent of the fuzzing engine
+that might be using them and driving the whole fuzzing process (e.g. libFuzzer
+or AFL). As such, this crate does *not* contain any actual fuzz targets
+itself. Those are generally just a couple lines of glue code that plug raw input
+from (for example) `libFuzzer` into a generator, and then run one or more
+oracles on the generated test case.
+
+If you're looking for the actual fuzz target definitions we currently have, they
+live in `wasmtime/fuzz/fuzz_targets/*` and are driven by `cargo fuzz` and
+`libFuzzer`.

crates/fuzzing/src/generators.rs

@@ -0,0 +1,29 @@
+//! Test case generators.
+//!
+//! Test case generators take raw, unstructured input from a fuzzer
+//! (e.g. libFuzzer) and translate that into a structured test case (e.g. a
+//! valid Wasm binary).
+//!
+//! These are generally implementations of the `Arbitrary` trait, or some
+//! wrapper over an external tool, such that the wrapper implements the
+//! `Arbitrary` trait for the wrapped external tool.
+
+use arbitrary::{Arbitrary, Unstructured};
+
+/// A Wasm test case generator that is powered by Binaryen's `wasm-opt -ttf`.
+pub struct WasmOptTtf {
+    /// The raw, encoded Wasm bytes.
+    pub wasm: Vec<u8>,
+}
+
+impl Arbitrary for WasmOptTtf {
+    fn arbitrary<U>(input: &mut U) -> Result<Self, U::Error>
+    where
+        U: Unstructured + ?Sized,
+    {
+        let seed: Vec<u8> = Arbitrary::arbitrary(input)?;
+        let module = binaryen::tools::translate_to_fuzz_mvp(&seed);
+        let wasm = module.write();
+        Ok(WasmOptTtf { wasm })
+    }
+}

crates/fuzzing/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod generators;
+pub mod oracles;

crates/fuzzing/src/oracles.rs

@@ -0,0 +1,66 @@
+//! Oracles.
+//!
+//! Oracles take a test case and determine whether we have a bug. For example,
+//! one of the simplest oracles is to take a Wasm binary as our input test case,
+//! validate and instantiate it, and (implicitly) check that no assertions
+//! failed or segfaults happened. A more complicated oracle might compare the
+//! result of executing a Wasm file with and without optimizations enabled, and
+//! make sure that the two executions are observably identical.
+//!
+//! When an oracle finds a bug, it should report it to the fuzzing engine by
+//! panicking.
+
+use cranelift_codegen::settings;
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::rc::Rc;
+use wasmtime_jit::{CompilationStrategy, CompiledModule, Compiler, NullResolver};
+
+fn host_isa() -> Box<dyn cranelift_codegen::isa::TargetIsa> {
+    let flag_builder = settings::builder();
+    let isa_builder = cranelift_native::builder().expect("host machine is not a supported target");
+    isa_builder.finish(settings::Flags::new(flag_builder))
+}
+
+/// Instantiate the Wasm buffer, and implicitly fail if we have an unexpected
+/// panic or segfault or anything else that can be detected "passively".
+///
+/// Performs initial validation, and returns early if the Wasm is invalid.
+///
+/// You can control which compiler is used via passing a `CompilationStrategy`.
+pub fn instantiate(wasm: &[u8], compilation_strategy: CompilationStrategy) {
+    if wasmparser::validate(wasm, None).is_err() {
+        return;
+    }
+
+    let isa = host_isa();
+    let mut compiler = Compiler::new(isa, compilation_strategy);
+    let mut imports_resolver = NullResolver {};
+
+    wasmtime_jit::instantiate(
+        &mut compiler,
+        wasm,
+        &mut imports_resolver,
+        Default::default(),
+        true,
+    )
+    .expect("failed to instantiate valid Wasm!");
+}
+
+/// Compile the Wasm buffer, and implicitly fail if we have an unexpected
+/// panic or segfault or anything else that can be detected "passively".
+///
+/// Performs initial validation, and returns early if the Wasm is invalid.
+///
+/// You can control which compiler is used via passing a `CompilationStrategy`.
+pub fn compile(wasm: &[u8], compilation_strategy: CompilationStrategy) {
+    if wasmparser::validate(wasm, None).is_err() {
+        return;
+    }
+
+    let isa = host_isa();
+    let mut compiler = Compiler::new(isa, compilation_strategy);
+    let mut resolver = NullResolver {};
+    let global_exports = Rc::new(RefCell::new(HashMap::new()));
+    let _ = CompiledModule::new(&mut compiler, wasm, &mut resolver, global_exports, false);
+}

fuzz/Cargo.toml

@@ -9,14 +9,10 @@ publish = false
 cargo-fuzz = true
 
 [dependencies]
-wasmtime-environ = { path = "../crates/environ" }
+arbitrary = "0.2.0"
+wasmtime-fuzzing = { path = "../crates/fuzzing" }
 wasmtime-jit = { path = "../crates/jit" }
-cranelift-codegen = "0.50"
-cranelift-wasm = "0.50"
-cranelift-native = "0.50"
 libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer-sys.git" }
-wasmparser = { version = "0.39.2", default-features = false, features = ["core"] }
-binaryen = "0.8.1"
 
 # Prevent this from interfering with workspaces
 [workspace]

fuzz/fuzz_targets/compile.rs

@@ -1,50 +1,14 @@
 #![no_main]
 
-extern crate libfuzzer_sys;
-
-use cranelift_codegen::settings;
 use libfuzzer_sys::fuzz_target;
-use std::cell::RefCell;
+use wasmtime_fuzzing::oracles;
-use std::collections::HashMap;
+use wasmtime_jit::CompilationStrategy;
-use std::rc::Rc;
-use wasmparser::validate;
-use wasmtime_jit::{CompilationStrategy, CompiledModule, Compiler, NullResolver};
 
 fuzz_target!(|data: &[u8]| {
-    if validate(data, None).is_err() {
+    oracles::compile(data, CompilationStrategy::Cranelift);
-        return;
-    }
-    let flag_builder = settings::builder();
-    let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
-        panic!("host machine is not a supported target");
-    });
-    let isa = isa_builder.finish(settings::Flags::new(flag_builder));
-    let mut compiler = Compiler::new(isa, CompilationStrategy::Cranelift);
-    let mut resolver = NullResolver {};
-    let global_exports = Rc::new(RefCell::new(HashMap::new()));
-    let _compiled =
-        match CompiledModule::new(&mut compiler, data, &mut resolver, global_exports, false) {
-            Ok(x) => x,
-            Err(_) => return,
-        };
 });
 
 #[cfg(feature = "lightbeam")]
 fuzz_target!(|data: &[u8]| {
-    if validate(data, None).is_err() {
+    oracles::compile(data, CompilationStrategy::Lightbeam);
-        return;
-    }
-    let flag_builder = settings::builder();
-    let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
-        panic!("host machine is not a supported target");
-    });
-    let isa = isa_builder.finish(settings::Flags::new(flag_builder));
-    let mut compiler = Compiler::new(isa, CompilationStrategy::Lightbeam);
-    let mut resolver = NullResolver {};
-    let global_exports = Rc::new(RefCell::new(HashMap::new()));
-    let _compiled =
-        match CompiledModule::new(&mut compiler, data, &mut resolver, global_exports, false) {
-            Ok(x) => x,
-            Err(_) => return,
-        };
 });

fuzz/fuzz_targets/instantiate.rs

@@ -1,29 +1,9 @@
 #![no_main]
 
-extern crate libfuzzer_sys;
-
-use cranelift_codegen::settings;
 use libfuzzer_sys::fuzz_target;
-use wasmparser::validate;
+use wasmtime_fuzzing::oracles;
-use wasmtime_jit::{instantiate, CompilationStrategy, Compiler, NullResolver};
+use wasmtime_jit::{CompilationStrategy};
 
 fuzz_target!(|data: &[u8]| {
-    if validate(data, None).is_err() {
+    oracles::instantiate(data, CompilationStrategy::Auto);
-        return;
-    }
-    let flag_builder = settings::builder();
-    let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
-        panic!("host machine is not a supported target");
-    });
-    let isa = isa_builder.finish(settings::Flags::new(flag_builder));
-    let mut compiler = Compiler::new(isa, CompilationStrategy::Auto);
-    let mut imports_resolver = NullResolver {};
-    let _instance = instantiate(
-        &mut compiler,
-        data,
-        &mut imports_resolver,
-        Default::default(),
-        true,
-    )
-    .unwrap();
 });

fuzz/fuzz_targets/instantiate_translated.rs

@@ -1,27 +1,9 @@
 #![no_main]
 
-extern crate libfuzzer_sys;
-
-use cranelift_codegen::settings;
 use libfuzzer_sys::fuzz_target;
-use wasmtime_jit::{instantiate, CompilationStrategy, Compiler, NullResolver};
+use wasmtime_fuzzing::{generators, oracles};
+use wasmtime_jit::CompilationStrategy;
 
-fuzz_target!(|data: &[u8]| {
+fuzz_target!(|data: generators::WasmOptTtf| {
-    let binaryen_module = binaryen::tools::translate_to_fuzz_mvp(data);
+    oracles::instantiate(&data.wasm, CompilationStrategy::Auto);
-    let wasm = binaryen_module.write();
-    let flag_builder = settings::builder();
-    let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
-        panic!("host machine is not a supported target");
-    });
-    let isa = isa_builder.finish(settings::Flags::new(flag_builder));
-    let mut compiler = Compiler::new(isa, CompilationStrategy::Auto);
-    let mut imports_resolver = NullResolver {};
-    let _instance = instantiate(
-        &mut compiler,
-        &wasm,
-        &mut imports_resolver,
-        Default::default(),
-        true,
-    )
-    .unwrap();
 });