wasi-threads: run test suite (#5907)

* wasi-threads: run test suite

This change enables the running of the wasi-threads [test suite]. It
relies on a Wasmtime CLI binary being available and runs all `*.wasm`
and `*.wat` files present in the test suite directory. The results of
each execution are compared against a JSON spec file with the same base
name as the WebAssembly module. The spec file defines the expected exit
code, e.g.

This commit does not yet build any `*.c` or `*.s` files from the test
suite. That could be done later, perhaps upstream; in the meantime, this
work is still valuable as it lays the foundation for running other WASI
tests from the in-progress [wasi-testsuite] which share the same JSON
spec infrastructure.

[test suite]: https://github.com/WebAssembly/wasi-threads/tree/main/test/testsuite
[wasi-testsuite]: https://github.com/WebAssembly/wasi-testsuite

* review: move testsuite to top-level tests

* fix: remove now-unnecessary wasi-threads test

* fix: update testsuite submodule name

* fix: ignore tests on Windows

prtest:full

* fix: `cfg_attr` syntax

prtest:full

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "crates/wasi-crypto/spec"]
 	path = crates/wasi-crypto/spec
 	url = https://github.com/WebAssembly/wasi-crypto.git
+[submodule "tests/wasi_testsuite/wasi-threads"]
+	path = tests/wasi_testsuite/wasi-threads
+	url = https://github.com/WebAssembly/wasi-threads

Cargo.toml

@@ -70,6 +70,8 @@ component-macro-test = { path = "crates/misc/component-macro-test" }
 component-test-util = { workspace = true }
 bstr = "0.2.17"
 libc = "0.2.60"
+serde = "1.0"
+serde_json = "1.0"
 
 [target.'cfg(windows)'.dev-dependencies]
 windows-sys = { workspace = true, features = ["Win32_System_Memory"] }

tests/all/cli_tests.rs

@@ -7,7 +7,7 @@ use tempfile::{NamedTempFile, TempDir};
 
 // Run the wasmtime CLI with the provided args and return the `Output`.
 // If the `stdin` is `Some`, opens the file and redirects to the child's stdin.
-fn run_wasmtime_for_output(args: &[&str], stdin: Option<&Path>) -> Result<Output> {
+pub fn run_wasmtime_for_output(args: &[&str], stdin: Option<&Path>) -> Result<Output> {
     let runner = std::env::vars()
         .filter(|(k, _v)| k.starts_with("CARGO_TARGET") && k.ends_with("RUNNER"))
         .next();

tests/all/main.rs

@@ -32,6 +32,7 @@ mod table;
 mod threads;
 mod traps;
 mod wait_notify;
+mod wasi_testsuite;
 mod wast;
 
 /// A helper to compile a module in a new store with reference types enabled.

tests/all/wasi_testsuite.rs

@@ -0,0 +1,79 @@
+//! Run the tests in `wasi_testsuite` using Wasmtime's CLI binary and checking
+//! the results with a [wasi-testsuite] spec.
+//!
+//! [wasi-testsuite]: https://github.com/WebAssembly/wasi-testsuite
+
+use crate::cli_tests::run_wasmtime_for_output;
+use anyhow::Result;
+use serde::Deserialize;
+use std::ffi::OsStr;
+use std::fs::{self, DirEntry};
+use std::path::{Path, PathBuf};
+use std::process::Output;
+
+#[test]
+#[cfg_attr(target_os = "windows", ignore)] // TODO: https://github.com/WebAssembly/WASI/issues/524
+fn wasi_threads_testsuite() -> Result<()> {
+    for module in list_modules("tests/wasi_testsuite/wasi-threads/test/testsuite")? {
+        println!("Testing {}", module.display());
+        let result = run(&module)?;
+        let spec = parse_spec(&module.with_extension("json"))?;
+        assert_eq!(spec, result);
+    }
+    Ok(())
+}
+
+fn list_modules(testsuite_dir: &str) -> Result<impl Iterator<Item = PathBuf>> {
+    Ok(fs::read_dir(testsuite_dir)?
+        .filter_map(Result::ok)
+        .filter(is_wasm)
+        .map(|e| e.path()))
+}
+
+fn is_wasm(entry: &DirEntry) -> bool {
+    let path = entry.path();
+    let ext = path.extension().map(OsStr::to_str).flatten();
+    path.is_file() && (ext == Some("wat") || ext == Some("wasm"))
+}
+
+fn run<P: AsRef<Path>>(module: P) -> Result<Output> {
+    run_wasmtime_for_output(
+        &[
+            "run",
+            "--wasi-modules",
+            "experimental-wasi-threads",
+            "--wasm-features",
+            "threads",
+            "--disable-cache",
+            module.as_ref().to_str().unwrap(),
+        ],
+        None,
+    )
+}
+
+fn parse_spec<P: AsRef<Path>>(spec_file: P) -> Result<Spec> {
+    let contents = fs::read_to_string(spec_file)?;
+    let spec = serde_json::from_str(&contents)?;
+    Ok(spec)
+}
+
+#[derive(Debug, Deserialize)]
+struct Spec {
+    exit_code: i32,
+    stdout: Option<String>,
+    stderr: Option<String>,
+}
+
+impl PartialEq<Output> for Spec {
+    fn eq(&self, other: &Output) -> bool {
+        self.exit_code == other.status.code().unwrap()
+            && matches_or_missing(&self.stdout, &other.stdout)
+            && matches_or_missing(&self.stderr, &other.stderr)
+    }
+}
+
+fn matches_or_missing(a: &Option<String>, b: &[u8]) -> bool {
+    a.as_ref()
+        .map(|s| s == &String::from_utf8_lossy(b))
+        .unwrap_or(true)
+}