Misc testsuite feature gated (#113)

* Put misc_testsuite behind a feature gate

This PR puts building and generating of misc_testsuite behind
a feature gate "misc_testsuite". This is mainly to allow projects
which pull `wasi-common` as a dependency not to have to have
`wasm32-wasi` target installed in order to build it as it currently
is.

* Update the CI

* Rename feature to wasm_tests

* Explain integration testing in the README

.azure-pipelines.yml

@@ -58,7 +58,7 @@ jobs:
       displayName: Cargo build release
     - script: cargo build
       displayName: Cargo build
-    - bash: RUST_BACKTRACE=1 cargo test --all
+    - bash: RUST_BACKTRACE=1 cargo test --features wasm_tests --all
       displayName: Cargo test
     - script: cargo doc
       condition: eq(variables['Agent.OS'], 'Darwin')

Cargo.toml

@@ -10,6 +10,11 @@ edition = "2018"
 license = "Apache-2.0 WITH LLVM-exception"
 description = "WASI implementation in Rust"
 
+[features]
+# this feature requires wasm32-wasi target installed, and it enables wasm32
+# integration tests when run with `cargo test --features wasm_tests`
+wasm_tests = []
+
 [dependencies]
 wasi-common-cbindgen = { path = "wasi-common-cbindgen" }
 failure = "0.1"

README.md

@@ -39,6 +39,28 @@ In our Windows implementation, we currently support the minimal subset of [WASI
 which allows for running the very basic "Hello world!" style WASM apps. More coming shortly,
 so stay tuned!
 
+## Development hints
+When testing the crate, you may want to enable and run full wasm32 integration testsuite. This
+requires `wasm32-wasi` target installed which can be done as follows using [rustup]
+
+```
+rustup target add wasm32-wasi
+```
+
+[rustup]: https://rustup.rs
+
+Next initiate submodules containing the integration testsuite
+
+```
+git submodule update --init
+```
+
+Now, you should be able to run the integration testsuite by enabling the `wasm_tests` feature
+
+```
+cargo test --features wasm_tests
+```
+
 ## Third-Party Code
 Significant parts of our hostcall implementations are derived from the C implementations in
 `cloudabi-utils`. See [LICENSE.cloudabi-utils](LICENSE.cloudabi-utils) for license information.

build.rs

@@ -6,203 +6,214 @@
 //! Idea adapted from: https://github.com/CraneStation/wasmtime/blob/master/build.rs
 //! Thanks @sunfishcode
 
-use std::env;
-use std::fs::{read_dir, DirEntry, File};
-use std::io::{self, Write};
-use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
-
 fn main() {
-    let out_dir =
+    #[cfg(feature = "wasm_tests")]
-        PathBuf::from(env::var("OUT_DIR").expect("The OUT_DIR environment variable must be set"));
+    wasm_tests::build_and_generate_tests();
-    println!("OUT_DIR is {:?}", out_dir);
-    let mut out = File::create(out_dir.join("misc_testsuite_tests.rs"))
-        .expect("error generating test source file");
-
-    build_tests("misc_testsuite", &out_dir).expect("building tests");
-    test_directory(&mut out, "misc_testsuite", &out_dir).expect("generating tests");
 }
 
-fn build_tests(testsuite: &str, out_dir: &Path) -> io::Result<()> {
+#[cfg(feature = "wasm_tests")]
-    // if the submodule has not been checked out, the build will stall
+mod wasm_tests {
-    if !Path::new(&format!("{}/Cargo.toml", testsuite)).exists() {
+    use std::env;
-        panic!("Testsuite {} not checked out", testsuite);
+    use std::fs::{read_dir, DirEntry, File};
-    }
+    use std::io::{self, Write};
+    use std::path::{Path, PathBuf};
+    use std::process::{Command, Stdio};
 
-    let mut cmd = Command::new("cargo");
+    pub(crate) fn build_and_generate_tests() {
-    cmd.args(&[
+        let out_dir = PathBuf::from(
-        "build",
+            env::var("OUT_DIR").expect("The OUT_DIR environment variable must be set"),
-        "--release",
-        "--target=wasm32-wasi",
-        "--target-dir",
-        out_dir.to_str().unwrap(),
-    ])
-    .stdout(Stdio::inherit())
-    .stderr(Stdio::inherit())
-    .current_dir(testsuite);
-    let output = cmd.output()?;
-
-    let status = output.status;
-    if !status.success() {
-        panic!(
-            "Building tests failed: exit code: {}",
-            status.code().unwrap()
         );
+        let mut out = File::create(out_dir.join("misc_testsuite_tests.rs"))
+            .expect("error generating test source file");
+        build_tests("misc_testsuite", &out_dir).expect("building tests");
+        test_directory(&mut out, "misc_testsuite", &out_dir).expect("generating tests");
     }
 
-    Ok(())
+    fn build_tests(testsuite: &str, out_dir: &Path) -> io::Result<()> {
-}
+        // if the submodule has not been checked out, the build will stall
+        if !Path::new(&format!("{}/Cargo.toml", testsuite)).exists() {
+            panic!("Testsuite {} not checked out", testsuite);
+        }
 
-fn test_directory(out: &mut File, testsuite: &str, out_dir: &Path) -> io::Result<()> {
+        let mut cmd = Command::new("cargo");
-    let mut dir_entries: Vec<_> = read_dir(out_dir.join("wasm32-wasi/release"))
+        cmd.args(&[
-        .expect("reading testsuite directory")
+            "build",
-        .map(|r| r.expect("reading testsuite directory entry"))
+            "--release",
-        .filter(|dir_entry| {
+            "--target=wasm32-wasi",
-            let p = dir_entry.path();
+            "--target-dir",
-            if let Some(ext) = p.extension() {
+            out_dir.to_str().unwrap(),
-                // Only look at wast files.
+        ])
-                if ext == "wasm" {
+        .stdout(Stdio::inherit())
-                    // Ignore files starting with `.`, which could be editor temporary files
+        .stderr(Stdio::inherit())
-                    if let Some(stem) = p.file_stem() {
+        .current_dir(testsuite);
-                        if let Some(stemstr) = stem.to_str() {
+        let output = cmd.output()?;
-                            if !stemstr.starts_with('.') {
+
-                                return true;
+        let status = output.status;
+        if !status.success() {
+            panic!(
+                "Building tests failed: exit code: {}",
+                status.code().unwrap()
+            );
+        }
+
+        Ok(())
+    }
+
+    fn test_directory(out: &mut File, testsuite: &str, out_dir: &Path) -> io::Result<()> {
+        let mut dir_entries: Vec<_> = read_dir(out_dir.join("wasm32-wasi/release"))
+            .expect("reading testsuite directory")
+            .map(|r| r.expect("reading testsuite directory entry"))
+            .filter(|dir_entry| {
+                let p = dir_entry.path();
+                if let Some(ext) = p.extension() {
+                    // Only look at wast files.
+                    if ext == "wasm" {
+                        // Ignore files starting with `.`, which could be editor temporary files
+                        if let Some(stem) = p.file_stem() {
+                            if let Some(stemstr) = stem.to_str() {
+                                if !stemstr.starts_with('.') {
+                                    return true;
+                                }
                             }
                         }
                     }
                 }
-            }
+                false
-            false
+            })
-        })
+            .collect();
-        .collect();
 
-    dir_entries.sort_by_key(|dir| dir.path());
+        dir_entries.sort_by_key(|dir| dir.path());
 
-    writeln!(
-        out,
-        "mod {} {{",
-        Path::new(testsuite)
-            .file_stem()
-            .expect("testsuite filename should have a stem")
-            .to_str()
-            .expect("testsuite filename should be representable as a string")
-            .replace("-", "_")
-    )?;
-    writeln!(out, "    use super::{{runtime, utils, setup_log}};")?;
-    for dir_entry in dir_entries {
-        write_testsuite_tests(out, dir_entry, testsuite)?;
-    }
-    writeln!(out, "}}")?;
-    Ok(())
-}
-
-fn write_testsuite_tests(out: &mut File, dir_entry: DirEntry, testsuite: &str) -> io::Result<()> {
-    let path = dir_entry.path();
-    let stemstr = path
-        .file_stem()
-        .expect("file_stem")
-        .to_str()
-        .expect("to_str");
-
-    writeln!(out, "    #[test]")?;
-    if ignore(testsuite, stemstr) {
-        writeln!(out, "    #[ignore]")?;
-    }
-    writeln!(
-        out,
-        "    fn {}() -> Result<(), String> {{",
-        avoid_keywords(&stemstr.replace("-", "_"))
-    )?;
-    writeln!(out, "        setup_log();")?;
-    write!(out, "        let path = std::path::Path::new(\"")?;
-    // Write out the string with escape_debug to prevent special characters such
-    // as backslash from being reinterpreted.
-    for c in path.display().to_string().chars() {
-        write!(out, "{}", c.escape_debug())?;
-    }
-    writeln!(out, "\");")?;
-    writeln!(out, "        let data = utils::read_wasm(path)?;")?;
-    writeln!(
-        out,
-        "        let bin_name = utils::extract_exec_name_from_path(path)?;"
-    )?;
-    let workspace = if no_preopens(testsuite, stemstr) {
-        "None"
-    } else {
         writeln!(
             out,
-            "        let workspace = utils::prepare_workspace(&bin_name)?;"
+            "mod {} {{",
+            Path::new(testsuite)
+                .file_stem()
+                .expect("testsuite filename should have a stem")
+                .to_str()
+                .expect("testsuite filename should be representable as a string")
+                .replace("-", "_")
         )?;
-        "Some(workspace.path())"
+        writeln!(out, "    use super::{{runtime, utils, setup_log}};")?;
-    };
+        for dir_entry in dir_entries {
-    writeln!(
+            write_testsuite_tests(out, dir_entry, testsuite)?;
-        out,
-        "        runtime::instantiate(&data, &bin_name, {})",
-        workspace
-    )?;
-    writeln!(out, "    }}")?;
-    writeln!(out)?;
-    Ok(())
-}
-
-/// Rename tests which have the same name as Rust keywords.
-fn avoid_keywords(name: &str) -> &str {
-    match name {
-        "if" => "if_",
-        "loop" => "loop_",
-        "type" => "type_",
-        "const" => "const_",
-        "return" => "return_",
-        other => other,
-    }
-}
-
-cfg_if::cfg_if! {
-    if #[cfg(not(windows))] {
-        /// Ignore tests that aren't supported yet.
-        fn ignore(testsuite: &str, name: &str) -> bool {
-            if testsuite == "misc_testsuite" {
-                match name {
-                    "path_symlink_trailing_slashes" => true,
-                    _ => false,
-                }
-            } else {
-                unreachable!()
-            }
-        }
-    } else {
-        /// Ignore tests that aren't supported yet.
-        fn ignore(testsuite: &str, name: &str) -> bool {
-            if testsuite == "misc_testsuite" {
-                match name {
-                    "readlink_no_buffer" => true,
-                    "dangling_symlink" => true,
-                    "symlink_loop" => true,
-                    "clock_time_get" => true,
-                    "truncation_rights" => true,
-                    "fd_readdir" => true,
-                    "path_rename_trailing_slashes" => true,
-                    "path_symlink_trailing_slashes" => true,
-                    "remove_directory_trailing_slashes" => true,
-                    _ => false,
-                }
-            } else {
-                unreachable!()
-            }
         }
+        writeln!(out, "}}")?;
+        Ok(())
     }
-}
 
-/// Mark tests which do not require preopens
+    fn write_testsuite_tests(
-fn no_preopens(testsuite: &str, name: &str) -> bool {
+        out: &mut File,
-    if testsuite == "misc_testsuite" {
+        dir_entry: DirEntry,
+        testsuite: &str,
+    ) -> io::Result<()> {
+        let path = dir_entry.path();
+        let stemstr = path
+            .file_stem()
+            .expect("file_stem")
+            .to_str()
+            .expect("to_str");
+
+        writeln!(out, "    #[test]")?;
+        if ignore(testsuite, stemstr) {
+            writeln!(out, "    #[ignore]")?;
+        }
+        writeln!(
+            out,
+            "    fn {}() -> Result<(), String> {{",
+            avoid_keywords(&stemstr.replace("-", "_"))
+        )?;
+        writeln!(out, "        setup_log();")?;
+        write!(out, "        let path = std::path::Path::new(\"")?;
+        // Write out the string with escape_debug to prevent special characters such
+        // as backslash from being reinterpreted.
+        for c in path.display().to_string().chars() {
+            write!(out, "{}", c.escape_debug())?;
+        }
+        writeln!(out, "\");")?;
+        writeln!(out, "        let data = utils::read_wasm(path)?;")?;
+        writeln!(
+            out,
+            "        let bin_name = utils::extract_exec_name_from_path(path)?;"
+        )?;
+        let workspace = if no_preopens(testsuite, stemstr) {
+            "None"
+        } else {
+            writeln!(
+                out,
+                "        let workspace = utils::prepare_workspace(&bin_name)?;"
+            )?;
+            "Some(workspace.path())"
+        };
+        writeln!(
+            out,
+            "        runtime::instantiate(&data, &bin_name, {})",
+            workspace
+        )?;
+        writeln!(out, "    }}")?;
+        writeln!(out)?;
+        Ok(())
+    }
+
+    /// Rename tests which have the same name as Rust keywords.
+    fn avoid_keywords(name: &str) -> &str {
         match name {
-            "big_random_buf" => true,
+            "if" => "if_",
-            "clock_time_get" => true,
+            "loop" => "loop_",
-            "sched_yield" => true,
+            "type" => "type_",
-            _ => false,
+            "const" => "const_",
+            "return" => "return_",
+            other => other,
+        }
+    }
+
+    cfg_if::cfg_if! {
+        if #[cfg(not(windows))] {
+            /// Ignore tests that aren't supported yet.
+            fn ignore(testsuite: &str, name: &str) -> bool {
+                if testsuite == "misc_testsuite" {
+                    match name {
+                        "path_symlink_trailing_slashes" => true,
+                        _ => false,
+                    }
+                } else {
+                    unreachable!()
+                }
+            }
+        } else {
+            /// Ignore tests that aren't supported yet.
+            fn ignore(testsuite: &str, name: &str) -> bool {
+                if testsuite == "misc_testsuite" {
+                    match name {
+                        "readlink_no_buffer" => true,
+                        "dangling_symlink" => true,
+                        "symlink_loop" => true,
+                        "clock_time_get" => true,
+                        "truncation_rights" => true,
+                        "fd_readdir" => true,
+                        "path_rename_trailing_slashes" => true,
+                        "path_symlink_trailing_slashes" => true,
+                        "remove_directory_trailing_slashes" => true,
+                        _ => false,
+                    }
+                } else {
+                    unreachable!()
+                }
+            }
+        }
+    }
+
+    /// Mark tests which do not require preopens
+    fn no_preopens(testsuite: &str, name: &str) -> bool {
+        if testsuite == "misc_testsuite" {
+            match name {
+                "big_random_buf" => true,
+                "clock_time_get" => true,
+                "sched_yield" => true,
+                _ => false,
+            }
+        } else {
+            unreachable!()
         }
-    } else {
-        unreachable!()
     }
 }

tests/wasm_tests.rs

@@ -1,3 +1,5 @@
+#![cfg(feature = "wasm_tests")]
+
 mod runtime;
 mod utils;