Move most wasmtime tests into one test suite (#1544)

* Move most wasmtime tests into one test suite

This commit moves most wasmtime tests into a single test suite which
gets compiled into one executable instead of having lots of test
executables. The goal here is to reduce disk space on CI, and this
should be achieved by having fewer executables which means fewer copies
of `libwasmtime.rlib` linked across binaries on the system. More
importantly though this means that DWARF debug information should only
be in one executable rather than duplicated across many.

* Share more build caches

Globally set `RUSTFLAGS` to `-Dwarnings` instead of individually so all
build steps share the same value.

* Allow some dead code in cranelift-codegen

Prevents having to fix all warnings for all possible feature
combinations, only the main ones which come up.

* Update some debug file paths

.github/actions/install-rust/main.js

@@ -12,6 +12,9 @@ child_process.execFileSync('rustup', ['set', 'profile', 'minimal']);
 child_process.execFileSync('rustup', ['update', toolchain, '--no-self-update']);
 child_process.execFileSync('rustup', ['default', toolchain]);
 
+// Deny warnings on CI to keep our code warning-free as it lands in-tree
+console.log(`::set-env name=RUSTFLAGS::-D warnings`);
+
 // Save disk space by avoiding incremental compilation, and also we don't use
 // any caching so incremental wouldn't help anyway.
 console.log(`::set-env name=CARGO_INCREMENTAL::0`);

.github/workflows/main.yml

@@ -228,14 +228,12 @@ jobs:
     - run: cargo test --features test-programs/test_programs --all --exclude lightbeam -- --nocapture
       env:
         RUST_BACKTRACE: 1
-        RUSTFLAGS: "-D warnings"
 
     # Test debug (DWARF) related functionality.
     - run: cargo test test_debug_dwarf_ -- --ignored --nocapture --test-threads 1
       if: matrix.os == 'ubuntu-latest'
       env:
         RUST_BACKTRACE: 1
-        RUSTFLAGS: "-D warnings"
 
     # Build and test lightbeam if we're using the nightly toolchain. Note that
     # Lightbeam tests fail right now, but we don't want to block on that.

Cargo.lock

@@ -2117,16 +2117,12 @@ dependencies = [
  "anyhow",
  "backtrace",
  "cfg-if",
- "file-per-thread-logger",
  "lazy_static",
  "libc",
- "pretty_env_logger",
- "rayon",
  "region",
  "rustc-demangle",
  "target-lexicon",
  "tempfile",
- "wasi-common",
  "wasmparser",
  "wasmtime-environ",
  "wasmtime-jit",
@@ -2177,6 +2173,7 @@ dependencies = [
  "wasmtime",
  "wasmtime-debug",
  "wasmtime-environ",
+ "wasmtime-fuzzing",
  "wasmtime-jit",
  "wasmtime-obj",
  "wasmtime-runtime",

Cargo.toml

@@ -41,11 +41,12 @@ libc = "0.2.60"
 rayon = "1.2.1"
 
 [dev-dependencies]
-wasmtime-runtime = { path = "crates/runtime", version = "0.15.0" }
-more-asserts = "0.2.1"
-test-programs = { path = "crates/test-programs" }
-tempfile = "3.1.0"
 filecheck = "0.5.0"
+more-asserts = "0.2.1"
+tempfile = "3.1.0"
+test-programs = { path = "crates/test-programs" }
+wasmtime-fuzzing = { path = "crates/fuzzing" }
+wasmtime-runtime = { path = "crates/runtime" }
 
 [build-dependencies]
 anyhow = "1.0.19"
@@ -79,3 +80,7 @@ vtune = ["wasmtime/vtune"]
 
 [badges]
 maintenance = { status = "actively-developed" }
+
+[[test]]
+name = "host_segfault"
+harness = false

build.rs

@@ -157,7 +157,7 @@ fn write_testsuite_tests(
     writeln!(out, "fn r#{}() -> anyhow::Result<()> {{", &testname)?;
     writeln!(
         out,
-        "crate::run_wast(r#\"{}\"#, crate::Strategy::{})",
+        "crate::wast::run_wast(r#\"{}\"#, crate::wast::Strategy::{})",
         path.display(),
         strategy
     )?;

cranelift/codegen/src/lib.rs

@@ -39,6 +39,13 @@
     )
 )]
 #![no_std]
+// Various bits and pieces of this crate might only be used for one platform or
+// another, but it's not really too useful to learn about that all the time. On
+// CI we build at least one version of this crate with `--features all-arch`
+// which means we'll always detect truly dead code, otherwise if this is only
+// built for one platform we don't have to worry too much about trimming
+// everything down.
+#![cfg_attr(not(feature = "all-arch"), allow(dead_code))]
 
 #[allow(unused_imports)] // #[macro_use] is required for no_std
 #[macro_use]

crates/api/Cargo.toml

@@ -29,13 +29,7 @@ wat = { version = "1.0.10", optional = true }
 winapi = "0.3.7"
 
 [dev-dependencies]
-# for wasmtime.rs
-wasi-common = { path = "../wasi-common", version = "0.15.0" }
-pretty_env_logger = "0.4.0"
-rayon = "1.2.1"
-file-per-thread-logger = "0.1.1"
-wat = "1.0.10"
-tempfile = "3.1"
+tempfile = "3.0"
 
 [badges]
 maintenance = { status = "actively-developed" }
@@ -53,7 +47,3 @@ jitdump = ["wasmtime-jit/jitdump"]
 
 # Enables support for the `VTune` profiler
 vtune = ["wasmtime-jit/vtune"]
-
-[[test]]
-name = "host-segfault"
-harness = false

crates/fuzzing/src/lib.rs

@@ -24,10 +24,9 @@ pub(crate) fn init_fuzzing() {
     INIT.call_once(|| {
         let _ = env_logger::try_init();
 
-        rayon::ThreadPoolBuilder::new()
+        let _ = rayon::ThreadPoolBuilder::new()
             .num_threads(1)
-            .build_global()
-            .expect("should only initialize the rayon thread pool once!");
+            .build_global();
     })
 }
 

tests/all/debug/lldb.rs

@@ -63,7 +63,7 @@ pub fn test_debug_dwarf_lldb() -> Result<()> {
     let output = lldb_with_script(
         &[
             "-g",
-            "tests/debug/testsuite/fib-wasm.wasm",
+            "tests/all/debug/testsuite/fib-wasm.wasm",
             "--invoke",
             "fib",
             "3",
@@ -105,7 +105,7 @@ pub fn test_debug_dwarf_ptr() -> Result<()> {
             "-g",
             "--opt-level",
             "0",
-            "tests/debug/testsuite/reverse-str.wasm",
+            "tests/all/debug/testsuite/reverse-str.wasm",
         ],
         r#"b reverse-str.c:9
 r

tests/all/debug/translate.rs

@@ -32,7 +32,7 @@ fn check_wasm(wasm_path: &str, directives: &str) -> Result<()> {
 ))]
 fn test_debug_dwarf_translate() -> Result<()> {
     check_wasm(
-        "tests/debug/testsuite/fib-wasm.wasm",
+        "tests/all/debug/testsuite/fib-wasm.wasm",
         r##"
 check: DW_TAG_compile_unit
 # We have "fib" function

tests/all/fuzzing.rs

@@ -1,22 +1,22 @@
 //! Regression tests for bugs found via fuzzing.
 //!
 //! The `#[test]` goes in here, the Wasm binary goes in
-//! `./regressions/some-descriptive-name.wasm`, and then the `#[test]` should
+//! `./fuzzing/some-descriptive-name.wasm`, and then the `#[test]` should
 //! use the Wasm binary by including it via
-//! `include_bytes!("./regressions/some-descriptive-name.wasm")`.
+//! `include_bytes!("./fuzzing/some-descriptive-name.wasm")`.
 
 use wasmtime::{Config, Strategy};
 use wasmtime_fuzzing::oracles;
 
 #[test]
 fn instantiate_empty_module() {
-    let data = wat::parse_str(include_str!("./regressions/empty.wat")).unwrap();
+    let data = wat::parse_str(include_str!("./fuzzing/empty.wat")).unwrap();
     oracles::instantiate(&data, Strategy::Auto);
 }
 
 #[test]
 fn instantiate_empty_module_with_memory() {
-    let data = wat::parse_str(include_str!("./regressions/empty_with_memory.wat")).unwrap();
+    let data = wat::parse_str(include_str!("./fuzzing/empty_with_memory.wat")).unwrap();
     oracles::instantiate(&data, Strategy::Auto);
 }
 
@@ -24,6 +24,6 @@ fn instantiate_empty_module_with_memory() {
 fn instantiate_module_that_compiled_to_x64_has_register_32() {
     let mut config = Config::new();
     config.debug_info(true);
-    let data = wat::parse_str(include_str!("./regressions/issue694.wat")).unwrap();
+    let data = wat::parse_str(include_str!("./fuzzing/issue694.wat")).unwrap();
     oracles::instantiate_with_config(&data, config);
 }

tests/all/fuzzing/README.md

@@ -1,2 +1,2 @@
 This directory contains `.wasm` binaries generated during fuzzing that uncovered
-a bug, and which we now use as regression tests in `../regressions.rs`.
+a bug, and which we now use as regression tests in `../fuzzing.rs`.

tests/all/main.rs

@@ -0,0 +1,16 @@
+mod cli_tests;
+mod custom_signal_handler;
+mod debug;
+mod externals;
+mod func;
+mod fuzzing;
+mod globals;
+mod import_calling_export;
+mod import_indexes;
+mod instance;
+mod invoke_func_via_table;
+mod linker;
+mod memory_creator;
+mod name;
+mod traps;
+mod wast;