Use build.rs to generate test cases automatically

Now, test binaries are bundled with the repo, and
just like in CraneStation/wasmtime, the test cases
are generated automatically using build.rs. So all
it takes is to drop a new test binary in the
testsuite dir to get the test case for it generated
(with some caveats to do with handling preopens).

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "tests/misc-tests"]
-	path = tests/misc-tests
-	url = https://github.com/kubkon/misc-tests.git

Cargo.toml

@@ -37,6 +37,9 @@ target-lexicon = "0.3.0"
 [patch."https://github.com/CraneStation/wasi-common"]
 wasi-common = { path = "." }
 
+[build-dependencies]
+cfg-if = "0.1.9"
+
 [lib]
 name = "wasi_common"
 crate-type = ["rlib", "staticlib", "cdylib"]

appveyor.yml

@@ -9,7 +9,5 @@ install:
   - rustc -vV
   - cargo -vV
   - rustup component add rustfmt
-  - cd %APPVEYOR_BUILD_FOLDER%
-  - git submodule update --init --recursive
 build: false
 test_script: test-all.bat

build.rs

@@ -0,0 +1,155 @@
+//! Build program to generate a program which runs all the testsuites.
+//!
+//! By generating a separate `#[test]` test for each file, we allow cargo test
+//! to automatically run the files in parallel.
+//!
+//! Idea adapted from: https://github.com/CraneStation/wasmtime/blob/master/build.rs
+//! Thanks @sunfishcode
+
+use std::env;
+use std::fs::{read_dir, DirEntry, File};
+use std::io::{self, Write};
+use std::path::{Path, PathBuf};
+
+fn main() {
+    let out_dir =
+        PathBuf::from(env::var("OUT_DIR").expect("The OUT_DIR environment variable must be set"));
+    let mut out = File::create(out_dir.join("misc_testsuite_tests.rs"))
+        .expect("error generating test source file");
+
+    test_directory(&mut out, "misc_testsuite").expect("generating tests");
+}
+
+fn test_directory(out: &mut File, testsuite: &str) -> io::Result<()> {
+    let mut dir_entries: Vec<_> = read_dir(testsuite)
+        .expect("reading testsuite directory")
+        .map(|r| r.expect("reading testsuite directory entry"))
+        .filter(|dir_entry| {
+            let p = dir_entry.path();
+            if let Some(ext) = p.extension() {
+                // Only look at wast files.
+                if ext == "wasm" {
+                    // Ignore files starting with `.`, which could be editor temporary files
+                    if let Some(stem) = p.file_stem() {
+                        if let Some(stemstr) = stem.to_str() {
+                            if !stemstr.starts_with('.') {
+                                return true;
+                            }
+                        }
+                    }
+                }
+            }
+            false
+        })
+        .collect();
+
+    dir_entries.sort_by_key(|dir| dir.path());
+
+    writeln!(
+        out,
+        "mod {} {{",
+        Path::new(testsuite)
+            .file_stem()
+            .expect("testsuite filename should have a stem")
+            .to_str()
+            .expect("testsuite filename should be representable as a string")
+            .replace("-", "_")
+    )?;
+    writeln!(out, "    use super::{{runtime, utils}};")?;
+    for dir_entry in dir_entries {
+        write_testsuite_tests(out, dir_entry, testsuite)?;
+    }
+    writeln!(out, "}}")?;
+    Ok(())
+}
+
+fn write_testsuite_tests(out: &mut File, dir_entry: DirEntry, testsuite: &str) -> io::Result<()> {
+    let path = dir_entry.path();
+    let stemstr = path
+        .file_stem()
+        .expect("file_stem")
+        .to_str()
+        .expect("to_str");
+
+    writeln!(out, "    #[test]")?;
+    if ignore(testsuite, stemstr) {
+        writeln!(out, "    #[ignore]")?;
+    }
+    writeln!(
+        out,
+        "    fn {}() -> Result<(), String> {{",
+        avoid_keywords(&stemstr.replace("-", "_"))
+    )?;
+    write!(out, "        let path = std::path::Path::new(\"")?;
+    // Write out the string with escape_debug to prevent special characters such
+    // as backslash from being reinterpreted.
+    for c in path.display().to_string().chars() {
+        write!(out, "{}", c.escape_debug())?;
+    }
+    writeln!(out, "\");")?;
+    writeln!(out, "        let data = utils::read_wasm(path)?;")?;
+    writeln!(
+        out,
+        "        let bin_name = utils::extract_exec_name_from_path(path)?;"
+    )?;
+    let workspace = if no_preopens(testsuite, stemstr) {
+        "None"
+    } else {
+        "Some(&utils::prepare_workspace(&bin_name)?)"
+    };
+    writeln!(
+        out,
+        "        runtime::instantiate(&data, &bin_name, {})",
+        workspace
+    )?;
+    writeln!(out, "    }}")?;
+    writeln!(out)?;
+    Ok(())
+}
+
+/// Rename tests which have the same name as Rust keywords.
+fn avoid_keywords(name: &str) -> &str {
+    match name {
+        "if" => "if_",
+        "loop" => "loop_",
+        "type" => "type_",
+        "const" => "const_",
+        "return" => "return_",
+        other => other,
+    }
+}
+
+cfg_if::cfg_if! {
+    if #[cfg(not(windows))] {
+        /// Ignore tests that aren't supported yet.
+        fn ignore(_testsuite: &str, _name: &str) -> bool {
+            false
+        }
+    } else {
+        /// Ignore tests that aren't supported yet.
+        fn ignore(testsuite: &str, name: &str) -> bool {
+            if testsuite == "misc_testsuite" {
+                match name {
+                    "big_random_buf" => false,
+                    _ => true,
+                }
+            } else {
+                unreachable!()
+            }
+        }
+    }
+}
+
+/// Mark tests which do not require preopens
+fn no_preopens(testsuite: &str, name: &str) -> bool {
+    if testsuite == "misc_testsuite" {
+        match name {
+            "big_random_buf" => true,
+            "clock_time_get" => true,
+            "sched_yield" => true,
+            _ => false,
+        }
+    } else {
+        unreachable!()
+    }
+}

tests/misc_tests.rs

@@ -1,102 +1,4 @@
 mod runtime;
 mod utils;
 
-use std::path::Path;
+include!(concat!(env!("OUT_DIR"), "/misc_testsuite_tests.rs"));
-
-fn run_test_with_workspace<P: AsRef<Path>>(path: P) -> Result<(), String> {
-    // Load in the wasm testcase
-    let data = utils::read_wasm(path.as_ref())?;
-    let bin_name = utils::extract_exec_name_from_path(path.as_ref())?;
-
-    // Prepare workspace
-    let workspace = utils::prepare_workspace(&bin_name)?;
-
-    // Run!
-    runtime::instantiate(&data, bin_name, Some(workspace))
-}
-
-fn run_test_without_workspace<P: AsRef<Path>>(path: P) -> Result<(), String> {
-    // Load in the wasm testcase
-    let data = utils::read_wasm(path.as_ref())?;
-    let bin_name = utils::extract_exec_name_from_path(path.as_ref())?;
-
-    // Run!
-    runtime::instantiate(&data, bin_name, None)
-}
-
-#[cfg(all(unix))]
-#[test]
-fn sched_yield() -> Result<(), String> {
-    run_test_without_workspace("tests/misc-tests/bin/sched_yield.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn truncation_rights() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/truncation_rights.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn unlink_directory() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/unlink_directory.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn remove_nonempty_directory() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/remove_nonempty_directory.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn interesting_paths() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/interesting_paths.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn nofollow_errors() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/nofollow_errors.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn symlink_loop() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/symlink_loop.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn close_preopen() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/close_preopen.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn clock_time_get() -> Result<(), String> {
-    run_test_without_workspace("tests/misc-tests/bin/clock_time_get.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn readlink_no_buffer() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/readlink_no_buffer.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn isatty() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/isatty.wasm")
-}
-
-#[cfg(all(unix))]
-#[test]
-fn directory_seek() -> Result<(), String> {
-    run_test_with_workspace("tests/misc-tests/bin/directory_seek.wasm")
-}
-
-#[test]
-fn big_random_buf() -> Result<(), String> {
-    run_test_without_workspace("tests/misc-tests/bin/big_random_buf.wasm")
-}