winch: Adding support for integration tests (#5588)

* Adding in the foundations for Winch `filetests`

This commit adds two new crates into the Winch workspace:
`filetests` and `test-macros`. The intent is to mimic the
structure of Cranelift `filetests`, but in a simpler way.

* Updates to documentation

This commits adds a high level document to outline how to test Winch
through the `winch-tools` utility. It also updates some inline
documentation which gets propagated to the CLI.

* Updating test-macro to use a glob instead of only a flat directory

Cargo.lock

@@ -766,9 +766,12 @@ dependencies = [
  "log",
  "pretty_env_logger",
  "rayon",
+ "serde",
+ "similar",
  "target-lexicon",
  "termcolor",
  "thiserror",
+ "toml",
  "walkdir",
  "wat",
 ]
@@ -3949,6 +3952,33 @@ dependencies = [
  "wasmparser",
 ]
 
+[[package]]
+name = "winch-filetests"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "capstone",
+ "cranelift-codegen",
+ "serde",
+ "similar",
+ "target-lexicon",
+ "toml",
+ "wasmtime-environ",
+ "wat",
+ "winch-codegen",
+ "winch-test-macros",
+]
+
+[[package]]
+name = "winch-test-macros"
+version = "0.0.0"
+dependencies = [
+ "glob",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "winch-tools"
 version = "0.0.0"
@@ -3957,11 +3987,17 @@ dependencies = [
  "capstone",
  "clap 3.2.8",
  "cranelift-codegen",
+ "glob",
+ "serde",
+ "similar",
  "target-lexicon",
+ "toml",
  "wasmparser",
  "wasmtime-environ",
  "wat",
  "winch-codegen",
+ "winch-filetests",
+ "winch-test-macros",
 ]
 
 [[package]]

Cargo.toml

@@ -156,6 +156,8 @@ cranelift-bforest = { path = "cranelift/bforest", version = "0.93.0" }
 cranelift = { path = "cranelift/umbrella", version = "0.93.0" }
 
 winch-codegen = { path = "winch/codegen", version = "=0.4.0" }
+winch-filetests = { path = "winch/filetests" }
+winch-test-macros = { path = "winch/test-macros" }
 
 target-lexicon = { version = "0.12.3", default-features = false, features = ["std"] }
 anyhow = "1.0.22"
@@ -186,6 +188,10 @@ bitflags = "1.2"
 thiserror = "1.0.15"
 async-trait = "0.1.42"
 heck = "0.4"
+similar = "2.1.0"
+toml = "0.5.9"
+serde = "1.0.94"
+glob = "0.3.0"
 
 [features]
 default = [

cranelift/Cargo.toml

@@ -46,6 +46,9 @@ thiserror = { workspace = true }
 walkdir = "2.2"
 anyhow = { workspace = true }
 clap = { workspace = true }
+similar = { workspace = true }
+toml = { workspace = true }
+serde = { workspace = true }
 
 [features]
 default = ["disas", "wasm", "cranelift-codegen/all-arch", "cranelift-codegen/trace-log", "souper-harvest"]

cranelift/filetests/Cargo.toml

@@ -26,10 +26,10 @@ num_cpus = "1.8.0"
 target-lexicon = { workspace = true }
 thiserror = { workspace = true }
 anyhow = { workspace = true }
-similar = "2.1.0"
+similar ={ workspace = true }
 wat.workspace = true
-toml = "0.5.9"
-serde = "1.0.94"
+toml = { workspace = true }
+serde = { workspace = true }
 cranelift-wasm.workspace = true
 wasmparser.workspace = true
 cranelift.workspace = true

winch/Cargo.toml

@@ -11,9 +11,10 @@ edition.workspace = true
 name = "winch-tools"
 path = "src/main.rs"
 
-
 [dependencies]
 winch-codegen = { workspace = true }
+winch-filetests = { workspace = true }
+winch-test-macros = { workspace = true }
 wasmtime-environ = { workspace = true }
 target-lexicon = { workspace = true }
 anyhow = { workspace = true }
@@ -22,6 +23,10 @@ clap = { workspace = true }
 wat = { workspace = true }
 cranelift-codegen = { workspace = true }
 capstone = { workspace = true }
+similar = { workspace = true }
+toml = { workspace = true }
+serde = { workspace = true }
+glob = { workspace = true }
 
 [features]
 default = ["all-arch"]

winch/docs/testing.md

@@ -0,0 +1,63 @@
+# Testing Winch
+
+Winch is tested through integration testing using the `winch-filetests` crate
+and manual exploratory testing. A CLI is available to run these tests
+conveniently. To add the `winch-tools` binary to your `PATH`, run `cargo install
+--path winch` from the root of `wasmtime`. The CLI provides two commands: `test`
+and `compile`. To see the help text for each command, run `winch-tools test
+--help` or `winch-tools compile --help`.
+
+## Integration Testing (`winch-tools test`)
+
+The `test` command will run a suite of tests that validates Winch output for a
+WebAssembly module is consistent with our expectations.
+
+### Running `test`
+
+Running `winch-tools test` will run all integration tests in the
+`winch-filetests` crate. All arguments following two dashes (`--`) will be
+passed directly to `cargo test -p winch-filetests`. This will allow you to
+configure the tests to run based on your requirements. All tests in the
+`winch-filetests` crate get named in the following convention:
+`winch_filetests_${filepath}`. This makes it possible to filter if you don't
+want to run the entire suite.
+
+If the output of Winch changes for a test in a run due to code updates, the test
+will fail and the difference between the two outputs will be shown. If the new
+output is expected, the tests can be re-run with an `WINCH_TEST_BLESS`
+environment variable set to `1`.
+
+### Adding a test
+
+To add new tests, create a `.wat` file in the `winch/filetests/filetests` folder
+in the following format:
+
+```wat
+;;! target = "x86_64"
+(module
+  (func (result i32)
+    (i32.const 42)
+  )
+)
+```
+
+It is encouraged to use folders to organize tests. For example, tests targeting
+the x86_64 architecture can be placed in the `winch/filetests/filetests/x64`.
+
+The first block of comments are a TOML compatible configuration passed to Winch
+during compilation with a `!` at the start of each line. The body of the file
+will be the subject of the test. A final block of comments is reserved for the
+output of the compilation, and it will be used to compare the output of the
+current run with the output of previous runs.
+
+## Manual Exploratory Tests (`winch-tools compile`)
+
+The `compile` command will run Winch for particular architecture against
+provided input file, and print the disassembled output to the console. Only
+`.wat` files are supported.
+
+### Running `compile`
+
+```bash
+winch-tools compile $wat_file --target $target_triple
+```

winch/filetests/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+authors = ["The Winch Project Developers"]
+name = "winch-filetests"
+description = "Tests for the Winch compiler based on a set of known valid files"
+license = "Apache-2.0 WITH LLVM-exception"
+repository = "https://github.com/bytecodealliance/wasmtime"
+version = "0.0.0"
+publish = false
+edition.workspace = true
+
+[dependencies]
+winch-test-macros = {workspace = true}
+target-lexicon = { workspace = true }
+winch-codegen = { workspace = true, features = ['all-arch'] }
+wasmtime-environ = { workspace = true }
+anyhow = { workspace = true }
+wat = { workspace = true }
+similar = { workspace = true }
+toml = { workspace = true }
+serde = { workspace = true }
+cranelift-codegen = { workspace = true }
+capstone = { workspace = true }

winch/filetests/filetests/x64/simple.wat

@@ -0,0 +1,12 @@
+;;! target = "x86_64"
+
+(module
+  (func (result i32)
+    (i32.const 42)
+  )
+)
+;;    0:	 55                   	push	rbp
+;;    1:	 4889e5               	mov	rbp, rsp
+;;    4:	 48c7c02a000000       	mov	rax, 0x2a
+;;    b:	 5d                   	pop	rbp
+;;    c:	 c3                   	ret

winch/filetests/src/disasm.rs

@@ -0,0 +1,59 @@
+//! Disassembly utilities.
+
+use anyhow::{bail, Result};
+use capstone::prelude::*;
+use std::fmt::Write;
+use target_lexicon::Architecture;
+use winch_codegen::TargetIsa;
+
+/// Disassemble and print a machine code buffer.
+pub fn disasm(bytes: &[u8], isa: &dyn TargetIsa) -> Result<Vec<String>> {
+    let dis = disassembler_for(isa)?;
+    let insts = dis.disasm_all(bytes, 0x0).unwrap();
+
+    let disassembled_lines = insts
+        .iter()
+        .map(|i| {
+            let mut line = String::new();
+
+            write!(&mut line, "{:4x}:\t ", i.address()).unwrap();
+
+            let mut bytes_str = String::new();
+            let mut len = 0;
+            for b in i.bytes() {
+                write!(&mut bytes_str, "{:02x}", b).unwrap();
+                len += 1;
+            }
+            write!(&mut line, "{:21}\t", bytes_str).unwrap();
+            if len > 8 {
+                write!(&mut line, "\n\t\t\t\t").unwrap();
+            }
+
+            if let Some(s) = i.mnemonic() {
+                write!(&mut line, "{}\t", s).unwrap();
+            }
+
+            if let Some(s) = i.op_str() {
+                write!(&mut line, "{}", s).unwrap();
+            }
+
+            line
+        })
+        .collect();
+
+    Ok(disassembled_lines)
+}
+
+fn disassembler_for(isa: &dyn TargetIsa) -> Result<Capstone> {
+    let disasm = match isa.triple().architecture {
+        Architecture::X86_64 => Capstone::new()
+            .x86()
+            .mode(arch::x86::ArchMode::Mode64)
+            .build()
+            .map_err(|e| anyhow::format_err!("{}", e))?,
+
+        _ => bail!("Unsupported ISA"),
+    };
+
+    Ok(disasm)
+}

winch/filetests/src/lib.rs

@@ -0,0 +1,165 @@
+pub mod disasm;
+
+#[cfg(test)]
+mod test {
+    use super::disasm::disasm;
+    use anyhow::Context;
+    use cranelift_codegen::settings;
+    use serde::{Deserialize, Serialize};
+    use similar::TextDiff;
+    use std::str::FromStr;
+    use target_lexicon::Triple;
+    use wasmtime_environ::{
+        wasmparser::{types::Types, Parser as WasmParser, Validator},
+        DefinedFuncIndex, FunctionBodyData, Module, ModuleEnvironment, Tunables,
+    };
+    use winch_codegen::isa::TargetIsa;
+    use winch_codegen::lookup;
+    use winch_test_macros::generate_file_tests;
+
+    #[derive(Clone, Debug, Serialize, Deserialize)]
+    struct TestConfig {
+        target: String,
+    }
+
+    /// A helper function to parse the test configuration from the top of the file.
+    fn parse_config(wat: &str) -> TestConfig {
+        let config_lines: Vec<_> = wat
+            .lines()
+            .take_while(|l| l.starts_with(";;!"))
+            .map(|l| &l[3..])
+            .collect();
+        let config_text = config_lines.join("\n");
+
+        toml::from_str(&config_text)
+            .context("failed to parse the test configuration")
+            .unwrap()
+    }
+
+    /// A helper function to parse the expected result from the bottom of the file.
+    fn parse_expected_result(wat: &str) -> String {
+        let mut expected_lines: Vec<_> = wat
+            .lines()
+            .rev()
+            .take_while(|l| l.starts_with(";;"))
+            .map(|l| {
+                if l.starts_with(";; ") {
+                    &l[3..]
+                } else {
+                    &l[2..]
+                }
+            })
+            .collect();
+        expected_lines.reverse();
+        expected_lines.join("\n")
+    }
+
+    /// A helper function to rewrite the expected result in the file.
+    fn rewrite_expected(wat: &str, actual: &str) -> String {
+        let old_expectation_line_count = wat
+            .lines()
+            .rev()
+            .take_while(|l| l.starts_with(";;"))
+            .count();
+        let old_wat_line_count = wat.lines().count();
+        let new_wat_lines: Vec<_> = wat
+            .lines()
+            .take(old_wat_line_count - old_expectation_line_count)
+            .map(|l| l.to_string())
+            .chain(actual.lines().map(|l| {
+                if l.is_empty() {
+                    ";;".to_string()
+                } else {
+                    format!(";; {l}")
+                }
+            }))
+            .collect();
+        let mut new_wat = new_wat_lines.join("\n");
+        new_wat.push('\n');
+
+        new_wat
+    }
+
+    #[generate_file_tests]
+    fn run_test(test_path: &str) {
+        let binding = std::fs::read_to_string(test_path).unwrap();
+        let wat = binding.as_str();
+
+        let config = parse_config(wat);
+        let wasm = wat::parse_str(&wat).unwrap();
+        let triple = Triple::from_str(&config.target).unwrap();
+
+        let binding = parse_expected_result(wat);
+        let expected = binding.as_str();
+
+        let shared_flags = settings::Flags::new(settings::builder());
+        let isa_builder = lookup(triple).unwrap();
+        let isa = isa_builder.build(shared_flags).unwrap();
+
+        let mut validator = Validator::new();
+        let parser = WasmParser::new(0);
+        let mut types = Default::default();
+        let tunables = Tunables::default();
+        let mut translation = ModuleEnvironment::new(&tunables, &mut validator, &mut types)
+            .translate(parser, &wasm)
+            .context("Failed to translate WebAssembly module")
+            .unwrap();
+        let _ = types.finish();
+
+        let body_inputs = std::mem::take(&mut translation.function_body_inputs);
+        let module = &translation.module;
+        let types = translation.get_types();
+
+        let binding = body_inputs
+            .into_iter()
+            .flat_map(|func| compile(&*isa, module, types, func))
+            .collect::<Vec<String>>()
+            .join("\n");
+        let actual = binding.as_str();
+
+        if std::env::var("WINCH_TEST_BLESS").unwrap_or_default() == "1" {
+            let new_wat = rewrite_expected(wat, actual);
+
+            std::fs::write(test_path, new_wat)
+                .with_context(|| format!("failed to write file: {}", test_path))
+                .unwrap();
+
+            return;
+        }
+
+        if expected.trim() != actual.trim() {
+            eprintln!(
+                "\n{}",
+                TextDiff::from_lines(expected, actual)
+                    .unified_diff()
+                    .header("expected", "actual")
+            );
+
+            eprintln!(
+                "note: You can re-run with the `WINCH_TEST_BLESS=1` environment variable set to update test expectations.\n"
+            );
+
+            panic!("Did not get the expected translation");
+        }
+    }
+
+    fn compile(
+        isa: &dyn TargetIsa,
+        module: &Module,
+        types: &Types,
+        f: (DefinedFuncIndex, FunctionBodyData<'_>),
+    ) -> Vec<String> {
+        let index = module.func_index(f.0);
+        let sig = types
+            .func_type_at(index.as_u32())
+            .expect(&format!("function type at index {:?}", index.as_u32()));
+        let FunctionBodyData { body, validator } = f.1;
+        let validator = validator.into_validator(Default::default());
+
+        let buffer = isa
+            .compile_function(&sig, &body, validator)
+            .expect("Couldn't compile function");
+
+        disasm(buffer.data(), isa).unwrap()
+    }
+}

winch/src/compile.rs

@@ -0,0 +1,72 @@
+use anyhow::{Context, Result};
+use clap::Parser;
+use cranelift_codegen::settings;
+use std::{fs, path::PathBuf, str::FromStr};
+use target_lexicon::Triple;
+use wasmtime_environ::{
+    wasmparser::{types::Types, Parser as WasmParser, Validator},
+    DefinedFuncIndex, FunctionBodyData, Module, ModuleEnvironment, Tunables,
+};
+use winch_codegen::{lookup, TargetIsa};
+use winch_filetests::disasm::disasm;
+
+#[derive(Parser, Debug)]
+pub struct Options {
+    /// The input file.
+    input: PathBuf,
+
+    /// The target architecture.
+    #[clap(long = "target")]
+    target: String,
+}
+
+pub fn run(opt: &Options) -> Result<()> {
+    let bytes = fs::read(&opt.input)
+        .with_context(|| format!("Failed to read input file {}", opt.input.display()))?;
+    let bytes = wat::parse_bytes(&bytes)?;
+    let triple = Triple::from_str(&opt.target)?;
+    let shared_flags = settings::Flags::new(settings::builder());
+    let isa_builder = lookup(triple)?;
+    let isa = isa_builder.build(shared_flags)?;
+    let mut validator = Validator::new();
+    let parser = WasmParser::new(0);
+    let mut types = Default::default();
+    let tunables = Tunables::default();
+    let mut translation = ModuleEnvironment::new(&tunables, &mut validator, &mut types)
+        .translate(parser, &bytes)
+        .context("Failed to translate WebAssembly module")?;
+    let _ = types.finish();
+
+    let body_inputs = std::mem::take(&mut translation.function_body_inputs);
+    let module = &translation.module;
+    let types = translation.get_types();
+
+    body_inputs
+        .into_iter()
+        .try_for_each(|func| compile(&*isa, module, types, func))?;
+
+    Ok(())
+}
+
+fn compile(
+    isa: &dyn TargetIsa,
+    module: &Module,
+    types: &Types,
+    f: (DefinedFuncIndex, FunctionBodyData<'_>),
+) -> Result<()> {
+    let index = module.func_index(f.0);
+    let sig = types
+        .func_type_at(index.as_u32())
+        .expect(&format!("function type at index {:?}", index.as_u32()));
+    let FunctionBodyData { body, validator } = f.1;
+    let validator = validator.into_validator(Default::default());
+    let buffer = isa
+        .compile_function(&sig, &body, validator)
+        .expect("Couldn't compile function");
+
+    disasm(buffer.data(), isa)?
+        .iter()
+        .for_each(|s| println!("{}", s));
+
+    Ok(())
+}

winch/src/disasm.rs

@@ -1,61 +0,0 @@
-//! Disassembly utilities.
-
-use anyhow::{bail, Result};
-use capstone::prelude::*;
-use std::fmt::Write;
-use target_lexicon::Architecture;
-use winch_codegen::TargetIsa;
-
-/// Disassemble and print a machine code buffer.
-pub fn print(bytes: &[u8], isa: &dyn TargetIsa) -> Result<()> {
-    let dis = disassembler_for(isa)?;
-    let insts = dis.disasm_all(bytes, 0x0).unwrap();
-
-    for i in insts.iter() {
-        let mut line = String::new();
-
-        write!(&mut line, "{:4x}:\t", i.address()).unwrap();
-
-        let mut bytes_str = String::new();
-        let mut len = 0;
-        let mut first = true;
-        for b in i.bytes() {
-            if !first {
-                write!(&mut bytes_str, " ").unwrap();
-            }
-            write!(&mut bytes_str, "{:02x}", b).unwrap();
-            len += 1;
-            first = false;
-        }
-        write!(&mut line, "{:21}\t", bytes_str).unwrap();
-        if len > 8 {
-            write!(&mut line, "\n\t\t\t\t").unwrap();
-        }
-
-        if let Some(s) = i.mnemonic() {
-            write!(&mut line, "{}\t", s).unwrap();
-        }
-
-        if let Some(s) = i.op_str() {
-            write!(&mut line, "{}", s).unwrap();
-        }
-
-        println!("{}", line);
-    }
-
-    Ok(())
-}
-
-fn disassembler_for(isa: &dyn TargetIsa) -> Result<Capstone> {
-    let disasm = match isa.triple().architecture {
-        Architecture::X86_64 => Capstone::new()
-            .x86()
-            .mode(arch::x86::ArchMode::Mode64)
-            .build()
-            .map_err(|e| anyhow::format_err!("{}", e))?,
-
-        _ => bail!("Unsupported ISA"),
-    };
-
-    Ok(disasm)
-}

winch/src/filetests.rs

@@ -0,0 +1,25 @@
+use std::process::Command;
+
+use anyhow::Result;
+use clap::Parser;
+
+#[derive(Parser, Debug)]
+pub struct Options {
+    /// Passes extra arguments to `cargo test --package winch-filetests`. For example, to run a single
+    /// test, use `-- --test-threads 1 --test single_test_name`.
+    #[clap(last = true, value_parser)]
+    cargo_test_args: Vec<String>,
+}
+
+pub fn run(opts: &Options) -> Result<()> {
+    Command::new("cargo")
+        .arg("test")
+        .arg("--package")
+        .arg("winch-filetests")
+        .arg("--")
+        .args(&opts.cargo_test_args)
+        .spawn()?
+        .wait()
+        .map(|_| ())
+        .map_err(|e| anyhow::anyhow!("Failed to run cargo test: {}", e))
+}

winch/src/main.rs

@@ -1,77 +1,21 @@
-//! Winch CLI tool, meant mostly for testing purposes.
-//!
-//! Reads Wasm in binary/text format and compiles them
-//! to any of the supported architectures using Winch.
+mod compile;
+mod filetests;
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 use clap::Parser;
-use cranelift_codegen::settings;
-use std::{fs, path::PathBuf, str::FromStr};
-use target_lexicon::Triple;
-use wasmtime_environ::{
-    wasmparser::{types::Types, Parser as WasmParser, Validator},
-    DefinedFuncIndex, FunctionBodyData, Module, ModuleEnvironment, Tunables,
-};
-use winch_codegen::{lookup, TargetIsa};
 
-mod disasm;
-
-#[derive(Parser, Debug)]
-struct Options {
-    /// The input file.
-    input: PathBuf,
-
-    /// The target architecture.
-    #[clap(long = "target")]
-    target: String,
+/// Winch compilation and testing tool.
+#[derive(Parser)]
+enum Commands {
+    /// Compile a Wasm module to the specified target architecture.
+    Compile(compile::Options),
+    /// Run the filetests.
+    Test(filetests::Options),
 }
 
 fn main() -> Result<()> {
-    let opt = Options::from_args();
-    let bytes = fs::read(&opt.input)
-        .with_context(|| format!("Failed to read input file {}", opt.input.display()))?;
-    let bytes = wat::parse_bytes(&bytes)?;
-    let triple = Triple::from_str(&opt.target)?;
-    let shared_flags = settings::Flags::new(settings::builder());
-    let isa_builder = lookup(triple)?;
-    let isa = isa_builder.build(shared_flags)?;
-    let mut validator = Validator::new();
-    let parser = WasmParser::new(0);
-    let mut types = Default::default();
-    let tunables = Tunables::default();
-    let mut translation = ModuleEnvironment::new(&tunables, &mut validator, &mut types)
-        .translate(parser, &bytes)
-        .context("Failed to translate WebAssembly module")?;
-    let _ = types.finish();
-
-    let body_inputs = std::mem::take(&mut translation.function_body_inputs);
-    let module = &translation.module;
-    let types = translation.get_types();
-
-    body_inputs
-        .into_iter()
-        .try_for_each(|func| compile(&*isa, module, types, func))?;
-
-    Ok(())
-}
-
-fn compile(
-    isa: &dyn TargetIsa,
-    module: &Module,
-    types: &Types,
-    f: (DefinedFuncIndex, FunctionBodyData<'_>),
-) -> Result<()> {
-    let index = module.func_index(f.0);
-    let sig = types
-        .func_type_at(index.as_u32())
-        .expect(&format!("function type at index {:?}", index.as_u32()));
-    let FunctionBodyData { body, validator } = f.1;
-    let validator = validator.into_validator(Default::default());
-    let buffer = isa
-        .compile_function(&sig, &body, validator)
-        .expect("Couldn't compile function");
-
-    disasm::print(buffer.data(), isa)?;
-
-    Ok(())
+    match Commands::parse() {
+        Commands::Compile(c) => compile::run(&c),
+        Commands::Test(t) => filetests::run(&t),
+    }
 }

winch/test-macros/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+authors = ["The Winch Project Developers"]
+name = "winch-test-macros"
+description = "Winch test macros"
+license = "Apache-2.0 WITH LLVM-exception"
+repository = "https://github.com/bytecodealliance/wasmtime"
+version = "0.0.0"
+publish = false
+edition.workspace = true
+
+[lib]
+proc-macro = true
+
+[dependencies]
+quote = "1.0"
+syn = { version = "1.0", features = ["full"]}
+proc-macro2 = "1.0"
+glob = { workspace = true }

winch/test-macros/src/lib.rs

@@ -0,0 +1,82 @@
+extern crate proc_macro;
+
+use std::path::Path;
+
+use glob::glob;
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::ItemFn;
+
+fn get_test_name_for_root(root: &Path, path: &Path) -> String {
+    let test_name = path
+        .strip_prefix(root)
+        .unwrap()
+        .to_str()
+        .unwrap()
+        .replace("/", "_")
+        .replace("\\", "_")
+        .replace(".wat", "");
+
+    format!("winch_filetests_{}", test_name)
+}
+
+/// Generate a test case for every .wat file in the filetests directory.
+/// This should only be used from the filetests crate.
+#[proc_macro_attribute]
+pub fn generate_file_tests(_attr: TokenStream, input: TokenStream) -> TokenStream {
+    // Parse the input as a function.
+    let input = proc_macro2::TokenStream::from(input);
+
+    let fn_ast: ItemFn =
+        syn::parse(input.clone().into()).expect("Failed to parse tokens as function");
+
+    // Get the function's name and body.
+    let name = &fn_ast.sig.ident;
+
+    let filetests_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../filetests/filetests");
+
+    let test_file_entries = glob(format!("{}/**/*.wat", filetests_dir.to_str().unwrap()).as_str())
+        .expect("Failed to read glob pattern");
+
+    // Create a list of test cases by opening every .wat file in the directory.
+    let test_cases = test_file_entries.map(|entry| {
+        let path = entry.expect("Failed to read glob entry");
+
+        let full = path.to_str().expect("Path for file was empty");
+
+        let test_name = proc_macro2::Ident::new(
+            &get_test_name_for_root(&filetests_dir, &path),
+            proc_macro2::Span::call_site(),
+        );
+        quote! {
+            #[test]
+            fn #test_name() {
+                #name(#full);
+            }
+        }
+    });
+
+    // Assemble the output by combining the function and test cases.
+    let output = quote! {
+        #input
+
+        #(#test_cases)*
+    };
+
+    output.into()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_test_name_for_root_unix() {
+        let root = Path::new("/home/user/Documents/winch/filetests/filetests");
+        let path = Path::new("/home/user/Documents/winch/filetests/filetests/simd/simple.wat");
+
+        let test_name = get_test_name_for_root(root, path);
+
+        assert_eq!(test_name, "winch_filetests_simd_simple");
+    }
+}