diff --git a/.github/actions/install-rust/main.js b/.github/actions/install-rust/main.js
index f94596aa8e..4adf88d4bc 100644
--- a/.github/actions/install-rust/main.js
+++ b/.github/actions/install-rust/main.js
@@ -12,6 +12,9 @@ child_process.execFileSync('rustup', ['set', 'profile', 'minimal']);
 child_process.execFileSync('rustup', ['update', toolchain, '--no-self-update']);
 child_process.execFileSync('rustup', ['default', toolchain]);
 
+// Deny warnings on CI to keep our code warning-free as it lands in-tree
+console.log(`::set-env name=RUSTFLAGS::-D warnings`);
+
 // Save disk space by avoiding incremental compilation, and also we don't use
 // any caching so incremental wouldn't help anyway.
 console.log(`::set-env name=CARGO_INCREMENTAL::0`);
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index be91e9b7ea..4ebec41ccc 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -228,14 +228,12 @@ jobs:
     - run: cargo test --features test-programs/test_programs --all --exclude lightbeam -- --nocapture
       env:
         RUST_BACKTRACE: 1
-        RUSTFLAGS: "-D warnings"
 
     # Test debug (DWARF) related functionality.
     - run: cargo test test_debug_dwarf_ -- --ignored --nocapture --test-threads 1
       if: matrix.os == 'ubuntu-latest'
       env:
         RUST_BACKTRACE: 1
-        RUSTFLAGS: "-D warnings"
 
     # Build and test lightbeam if we're using the nightly toolchain. Note that
     # Lightbeam tests fail right now, but we don't want to block on that.
diff --git a/Cargo.lock b/Cargo.lock
index 1b516a2bd1..c9d04b8cf8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2117,16 +2117,12 @@ dependencies = [
  "anyhow",
  "backtrace",
  "cfg-if",
- "file-per-thread-logger",
  "lazy_static",
  "libc",
- "pretty_env_logger",
- "rayon",
  "region",
  "rustc-demangle",
  "target-lexicon",
  "tempfile",
- "wasi-common",
  "wasmparser",
  "wasmtime-environ",
  "wasmtime-jit",
@@ -2177,6 +2173,7 @@ dependencies = [
  "wasmtime",
  "wasmtime-debug",
  "wasmtime-environ",
+ "wasmtime-fuzzing",
  "wasmtime-jit",
  "wasmtime-obj",
  "wasmtime-runtime",
diff --git a/Cargo.toml b/Cargo.toml
index 4cb6252962..3161417cbc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -41,11 +41,12 @@ libc = "0.2.60"
 rayon = "1.2.1"
 
 [dev-dependencies]
-wasmtime-runtime = { path = "crates/runtime", version = "0.15.0" }
-more-asserts = "0.2.1"
-test-programs = { path = "crates/test-programs" }
-tempfile = "3.1.0"
 filecheck = "0.5.0"
+more-asserts = "0.2.1"
+tempfile = "3.1.0"
+test-programs = { path = "crates/test-programs" }
+wasmtime-fuzzing = { path = "crates/fuzzing" }
+wasmtime-runtime = { path = "crates/runtime" }
 
 [build-dependencies]
 anyhow = "1.0.19"
@@ -79,3 +80,7 @@ vtune = ["wasmtime/vtune"]
 
 [badges]
 maintenance = { status = "actively-developed" }
+
+[[test]]
+name = "host_segfault"
+harness = false
diff --git a/build.rs b/build.rs
index 3eeb27c179..4764540fe1 100644
--- a/build.rs
+++ b/build.rs
@@ -157,7 +157,7 @@ fn write_testsuite_tests(
     writeln!(out, "fn r#{}() -> anyhow::Result<()> {{", &testname)?;
     writeln!(
         out,
-        "crate::run_wast(r#\"{}\"#, crate::Strategy::{})",
+        "crate::wast::run_wast(r#\"{}\"#, crate::wast::Strategy::{})",
         path.display(),
         strategy
     )?;
diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs
index d87bbf26b8..d178187914 100644
--- a/cranelift/codegen/src/lib.rs
+++ b/cranelift/codegen/src/lib.rs
@@ -39,6 +39,13 @@
     )
 )]
 #![no_std]
+// Various bits and pieces of this crate might only be used for one platform or
+// another, but it's not really too useful to learn about that all the time. On
+// CI we build at least one version of this crate with `--features all-arch`
+// which means we'll always detect truly dead code, otherwise if this is only
+// built for one platform we don't have to worry too much about trimming
+// everything down.
+#![cfg_attr(not(feature = "all-arch"), allow(dead_code))]
 
 #[allow(unused_imports)] // #[macro_use] is required for no_std
 #[macro_use]
diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml
index ea4580f24b..14e6783ff1 100644
--- a/crates/api/Cargo.toml
+++ b/crates/api/Cargo.toml
@@ -29,13 +29,7 @@ wat = { version = "1.0.10", optional = true }
 winapi = "0.3.7"
 
 [dev-dependencies]
-# for wasmtime.rs
-wasi-common = { path = "../wasi-common", version = "0.15.0" }
-pretty_env_logger = "0.4.0"
-rayon = "1.2.1"
-file-per-thread-logger = "0.1.1"
-wat = "1.0.10"
-tempfile = "3.1"
+tempfile = "3.0"
 
 [badges]
 maintenance = { status = "actively-developed" }
@@ -53,7 +47,3 @@ jitdump = ["wasmtime-jit/jitdump"]
 
 # Enables support for the `VTune` profiler
 vtune = ["wasmtime-jit/vtune"]
-
-[[test]]
-name = "host-segfault"
-harness = false
diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs
index d9f884e616..613912683d 100644
--- a/crates/fuzzing/src/lib.rs
+++ b/crates/fuzzing/src/lib.rs
@@ -24,10 +24,9 @@ pub(crate) fn init_fuzzing() {
     INIT.call_once(|| {
         let _ = env_logger::try_init();
 
-        rayon::ThreadPoolBuilder::new()
+        let _ = rayon::ThreadPoolBuilder::new()
             .num_threads(1)
-            .build_global()
-            .expect("should only initialize the rayon thread pool once!");
+            .build_global();
     })
 }
 
diff --git a/tests/cli_tests.rs b/tests/all/cli_tests.rs
similarity index 100%
rename from tests/cli_tests.rs
rename to tests/all/cli_tests.rs
diff --git a/tests/custom_signal_handler.rs b/tests/all/custom_signal_handler.rs
similarity index 100%
rename from tests/custom_signal_handler.rs
rename to tests/all/custom_signal_handler.rs
diff --git a/tests/debug/dump.rs b/tests/all/debug/dump.rs
similarity index 100%
rename from tests/debug/dump.rs
rename to tests/all/debug/dump.rs
diff --git a/tests/debug/lldb.rs b/tests/all/debug/lldb.rs
similarity index 96%
rename from tests/debug/lldb.rs
rename to tests/all/debug/lldb.rs
index abf14f353e..eb444a1454 100644
--- a/tests/debug/lldb.rs
+++ b/tests/all/debug/lldb.rs
@@ -63,7 +63,7 @@ pub fn test_debug_dwarf_lldb() -> Result<()> {
     let output = lldb_with_script(
         &[
             "-g",
-            "tests/debug/testsuite/fib-wasm.wasm",
+            "tests/all/debug/testsuite/fib-wasm.wasm",
             "--invoke",
             "fib",
             "3",
@@ -105,7 +105,7 @@ pub fn test_debug_dwarf_ptr() -> Result<()> {
             "-g",
             "--opt-level",
             "0",
-            "tests/debug/testsuite/reverse-str.wasm",
+            "tests/all/debug/testsuite/reverse-str.wasm",
         ],
         r#"b reverse-str.c:9
 r
diff --git a/tests/debug/main.rs b/tests/all/debug/mod.rs
similarity index 100%
rename from tests/debug/main.rs
rename to tests/all/debug/mod.rs
diff --git a/tests/debug/obj.rs b/tests/all/debug/obj.rs
similarity index 100%
rename from tests/debug/obj.rs
rename to tests/all/debug/obj.rs
diff --git a/tests/debug/simulate.rs b/tests/all/debug/simulate.rs
similarity index 100%
rename from tests/debug/simulate.rs
rename to tests/all/debug/simulate.rs
diff --git a/tests/debug/testsuite/fib-wasm.c b/tests/all/debug/testsuite/fib-wasm.c
similarity index 100%
rename from tests/debug/testsuite/fib-wasm.c
rename to tests/all/debug/testsuite/fib-wasm.c
diff --git a/tests/debug/testsuite/fib-wasm.wasm b/tests/all/debug/testsuite/fib-wasm.wasm
similarity index 100%
rename from tests/debug/testsuite/fib-wasm.wasm
rename to tests/all/debug/testsuite/fib-wasm.wasm
diff --git a/tests/debug/testsuite/reverse-str.c b/tests/all/debug/testsuite/reverse-str.c
similarity index 100%
rename from tests/debug/testsuite/reverse-str.c
rename to tests/all/debug/testsuite/reverse-str.c
diff --git a/tests/debug/testsuite/reverse-str.wasm b/tests/all/debug/testsuite/reverse-str.wasm
similarity index 100%
rename from tests/debug/testsuite/reverse-str.wasm
rename to tests/all/debug/testsuite/reverse-str.wasm
diff --git a/tests/debug/translate.rs b/tests/all/debug/translate.rs
similarity index 97%
rename from tests/debug/translate.rs
rename to tests/all/debug/translate.rs
index e2fb78fa05..fd15b0f555 100644
--- a/tests/debug/translate.rs
+++ b/tests/all/debug/translate.rs
@@ -32,7 +32,7 @@ fn check_wasm(wasm_path: &str, directives: &str) -> Result<()> {
 ))]
 fn test_debug_dwarf_translate() -> Result<()> {
     check_wasm(
-        "tests/debug/testsuite/fib-wasm.wasm",
+        "tests/all/debug/testsuite/fib-wasm.wasm",
         r##"
 check: DW_TAG_compile_unit
 # We have "fib" function
diff --git a/crates/api/tests/externals.rs b/tests/all/externals.rs
similarity index 100%
rename from crates/api/tests/externals.rs
rename to tests/all/externals.rs
diff --git a/crates/api/tests/func.rs b/tests/all/func.rs
similarity index 100%
rename from crates/api/tests/func.rs
rename to tests/all/func.rs
diff --git a/crates/fuzzing/tests/regressions.rs b/tests/all/fuzzing.rs
similarity index 60%
rename from crates/fuzzing/tests/regressions.rs
rename to tests/all/fuzzing.rs
index de29c718a0..9bd841846b 100644
--- a/crates/fuzzing/tests/regressions.rs
+++ b/tests/all/fuzzing.rs
@@ -1,22 +1,22 @@
 //! Regression tests for bugs found via fuzzing.
 //!
 //! The `#[test]` goes in here, the Wasm binary goes in
-//! `./regressions/some-descriptive-name.wasm`, and then the `#[test]` should
+//! `./fuzzing/some-descriptive-name.wasm`, and then the `#[test]` should
 //! use the Wasm binary by including it via
-//! `include_bytes!("./regressions/some-descriptive-name.wasm")`.
+//! `include_bytes!("./fuzzing/some-descriptive-name.wasm")`.
 
 use wasmtime::{Config, Strategy};
 use wasmtime_fuzzing::oracles;
 
 #[test]
 fn instantiate_empty_module() {
-    let data = wat::parse_str(include_str!("./regressions/empty.wat")).unwrap();
+    let data = wat::parse_str(include_str!("./fuzzing/empty.wat")).unwrap();
     oracles::instantiate(&data, Strategy::Auto);
 }
 
 #[test]
 fn instantiate_empty_module_with_memory() {
-    let data = wat::parse_str(include_str!("./regressions/empty_with_memory.wat")).unwrap();
+    let data = wat::parse_str(include_str!("./fuzzing/empty_with_memory.wat")).unwrap();
     oracles::instantiate(&data, Strategy::Auto);
 }
 
@@ -24,6 +24,6 @@ fn instantiate_empty_module_with_memory() {
 fn instantiate_module_that_compiled_to_x64_has_register_32() {
     let mut config = Config::new();
     config.debug_info(true);
-    let data = wat::parse_str(include_str!("./regressions/issue694.wat")).unwrap();
+    let data = wat::parse_str(include_str!("./fuzzing/issue694.wat")).unwrap();
     oracles::instantiate_with_config(&data, config);
 }
diff --git a/crates/fuzzing/tests/regressions/README.md b/tests/all/fuzzing/README.md
similarity index 52%
rename from crates/fuzzing/tests/regressions/README.md
rename to tests/all/fuzzing/README.md
index 3a1630279f..ff53c97785 100644
--- a/crates/fuzzing/tests/regressions/README.md
+++ b/tests/all/fuzzing/README.md
@@ -1,2 +1,2 @@
 This directory contains `.wasm` binaries generated during fuzzing that uncovered
-a bug, and which we now use as regression tests in `../regressions.rs`.
+a bug, and which we now use as regression tests in `../fuzzing.rs`.
diff --git a/crates/fuzzing/tests/regressions/empty.wat b/tests/all/fuzzing/empty.wat
similarity index 100%
rename from crates/fuzzing/tests/regressions/empty.wat
rename to tests/all/fuzzing/empty.wat
diff --git a/crates/fuzzing/tests/regressions/empty_with_memory.wat b/tests/all/fuzzing/empty_with_memory.wat
similarity index 100%
rename from crates/fuzzing/tests/regressions/empty_with_memory.wat
rename to tests/all/fuzzing/empty_with_memory.wat
diff --git a/crates/fuzzing/tests/regressions/issue694.wat b/tests/all/fuzzing/issue694.wat
similarity index 100%
rename from crates/fuzzing/tests/regressions/issue694.wat
rename to tests/all/fuzzing/issue694.wat
diff --git a/crates/api/tests/globals.rs b/tests/all/globals.rs
similarity index 100%
rename from crates/api/tests/globals.rs
rename to tests/all/globals.rs
diff --git a/crates/api/tests/import_calling_export.rs b/tests/all/import_calling_export.rs
similarity index 100%
rename from crates/api/tests/import_calling_export.rs
rename to tests/all/import_calling_export.rs
diff --git a/crates/api/tests/import-indexes.rs b/tests/all/import_indexes.rs
similarity index 100%
rename from crates/api/tests/import-indexes.rs
rename to tests/all/import_indexes.rs
diff --git a/crates/api/tests/instance.rs b/tests/all/instance.rs
similarity index 100%
rename from crates/api/tests/instance.rs
rename to tests/all/instance.rs
diff --git a/crates/api/tests/invoke_func_via_table.rs b/tests/all/invoke_func_via_table.rs
similarity index 100%
rename from crates/api/tests/invoke_func_via_table.rs
rename to tests/all/invoke_func_via_table.rs
diff --git a/crates/api/tests/linker.rs b/tests/all/linker.rs
similarity index 100%
rename from crates/api/tests/linker.rs
rename to tests/all/linker.rs
diff --git a/tests/all/main.rs b/tests/all/main.rs
new file mode 100644
index 0000000000..fddeb9897a
--- /dev/null
+++ b/tests/all/main.rs
@@ -0,0 +1,16 @@
+mod cli_tests;
+mod custom_signal_handler;
+mod debug;
+mod externals;
+mod func;
+mod fuzzing;
+mod globals;
+mod import_calling_export;
+mod import_indexes;
+mod instance;
+mod invoke_func_via_table;
+mod linker;
+mod memory_creator;
+mod name;
+mod traps;
+mod wast;
diff --git a/crates/api/tests/memory_creator.rs b/tests/all/memory_creator.rs
similarity index 100%
rename from crates/api/tests/memory_creator.rs
rename to tests/all/memory_creator.rs
diff --git a/crates/api/tests/name.rs b/tests/all/name.rs
similarity index 100%
rename from crates/api/tests/name.rs
rename to tests/all/name.rs
diff --git a/crates/api/tests/traps.rs b/tests/all/traps.rs
similarity index 100%
rename from crates/api/tests/traps.rs
rename to tests/all/traps.rs
diff --git a/tests/wast_testsuites.rs b/tests/all/wast.rs
similarity index 100%
rename from tests/wast_testsuites.rs
rename to tests/all/wast.rs
diff --git a/crates/api/tests/host-segfault.rs b/tests/host_segfault.rs
similarity index 100%
rename from crates/api/tests/host-segfault.rs
rename to tests/host_segfault.rs