Add more SIMD spec tests (#388)

* Add more SIMD spec tests

Also provides a helper, extract_name, for building the names needed for the generated code

* Use `cargo test ... -- --nocapture` to see test errors in CI

* Use OS-independent paths for WAST files

* Ignore 'skip-stack-guard-page'

* Temporarily disable SIMD tests

* Re-enable SIMD spec tests and only disable on Windows temporarily

.azure-pipelines.yml

@@ -69,7 +69,7 @@ jobs:
       displayName: Fetch cargo dependencies
 
     # Build and test all features except for lightbeam
-    - bash: cargo test --all --exclude lightbeam --exclude wasmtime-wasi-c --exclude wasmtime-py
+    - bash: cargo test --all --exclude lightbeam --exclude wasmtime-wasi-c --exclude wasmtime-py -- --nocapture
       displayName: Cargo test
       env:
         RUST_BACKTRACE: 1
@@ -78,7 +78,7 @@ jobs:
     - bash: cargo build --package lightbeam
       displayName: Cargo build lightbeam
       condition: and(succeeded(), eq(variables['toolchain'], 'nightly'))
-    - bash: cargo test --package lightbeam
+    - bash: cargo test --package lightbeam -- --nocapture
       displayName: Cargo test lightbeam
       # Lightbeam tests fail right now, but we don't want to block on that.
       continueOnError: true

build.rs

@@ -33,15 +33,24 @@ fn main() {
         {
             test_file(
                 &mut out,
-                "spec_testsuite/proposals/simd/simd_const.wast",
+                &to_os_path(&["spec_testsuite", "proposals", "simd", "simd_address.wast"]),
+                strategy,
+            )
+            .expect("generating tests");
+            test_file(
+                &mut out,
+                &to_os_path(&["spec_testsuite", "proposals", "simd", "simd_align.wast"]),
+                strategy,
+            )
+            .expect("generating tests");
+            test_file(
+                &mut out,
+                &to_os_path(&["spec_testsuite", "proposals", "simd", "simd_const.wast"]),
                 strategy,
             )
             .expect("generating tests");
 
-            let multi_value_suite = Path::new("spec_testsuite")
+            let multi_value_suite = &to_os_path(&["spec_testsuite", "proposals", "multi-value"]);
-                .join("proposals")
-                .join("multi-value");
-            let multi_value_suite = multi_value_suite.display().to_string();
             test_directory(&mut out, &multi_value_suite, strategy).expect("generating tests");
         } else {
             println!("cargo:warning=The spec testsuite is disabled. To enable, run `git submodule update --remote`.");
@@ -51,8 +60,14 @@ fn main() {
     }
 }
 
-fn test_directory(out: &mut File, testsuite: &str, strategy: &str) -> io::Result<()> {
+/// Helper for creating OS-independent paths.
-    let mut dir_entries: Vec<_> = read_dir(testsuite)
+fn to_os_path(components: &[&str]) -> String {
+    let path: PathBuf = components.iter().collect();
+    path.display().to_string()
+}
+
+fn test_directory(out: &mut File, path: &str, strategy: &str) -> io::Result<()> {
+    let mut dir_entries: Vec<_> = read_dir(path)
         .expect("reading testsuite directory")
         .map(|r| r.expect("reading testsuite directory entry"))
         .filter(|dir_entry| {
@@ -76,6 +91,7 @@ fn test_directory(out: &mut File, testsuite: &str, strategy: &str) -> io::Result
 
     dir_entries.sort_by_key(|dir| dir.path());
 
+    let testsuite = &extract_name(path);
     start_test_module(out, testsuite)?;
     for dir_entry in dir_entries {
         write_testsuite_tests(out, &dir_entry.path(), testsuite, strategy)?;
@@ -84,25 +100,26 @@ fn test_directory(out: &mut File, testsuite: &str, strategy: &str) -> io::Result
 }
 
 fn test_file(out: &mut File, testfile: &str, strategy: &str) -> io::Result<()> {
-    let testsuite = "single_file_spec_test";
     let path = Path::new(testfile);
-    start_test_module(out, testsuite)?;
+    let testsuite = format!("single_test_{}", extract_name(path));
-    write_testsuite_tests(out, path, testsuite, strategy)?;
+    start_test_module(out, &testsuite)?;
+    write_testsuite_tests(out, path, &testsuite, strategy)?;
     finish_test_module(out)
 }
 
-fn start_test_module(out: &mut File, testsuite: &str) -> io::Result<()> {
+/// Extract a valid Rust identifier from the stem of a path.
-    writeln!(
+fn extract_name(path: impl AsRef<Path>) -> String {
-        out,
+    path.as_ref()
-        "    mod {} {{",
-        Path::new(testsuite)
         .file_stem()
-            .expect("testsuite filename should have a stem")
+        .expect("filename should have a stem")
         .to_str()
-            .expect("testsuite filename should be representable as a string")
+        .expect("filename should be representable as a string")
         .replace("-", "_")
         .replace("/", "_")
-    )?;
+}
+
+fn start_test_module(out: &mut File, testsuite: &str) -> io::Result<()> {
+    writeln!(out, "    mod {} {{", testsuite)?;
     writeln!(
         out,
         "        use super::super::{{native_isa, Path, WastContext, Compiler, Features, CompilationStrategy}};"
@@ -119,17 +136,13 @@ fn write_testsuite_tests(
     testsuite: &str,
     strategy: &str,
 ) -> io::Result<()> {
-    let stemstr = path
+    let testname = extract_name(path);
-        .file_stem()
-        .expect("file_stem")
-        .to_str()
-        .expect("to_str");
 
     writeln!(out, "        #[test]")?;
-    if ignore(testsuite, stemstr, strategy) {
+    if ignore(testsuite, &testname, strategy) {
         writeln!(out, "        #[ignore]")?;
     }
-    writeln!(out, "        fn r#{}() {{", &stemstr.replace("-", "_"))?;
+    writeln!(out, "        fn r#{}() {{", &testname)?;
     writeln!(out, "            let isa = native_isa();")?;
     writeln!(
         out,
@@ -138,8 +151,9 @@ fn write_testsuite_tests(
     )?;
     writeln!(
         out,
-        "            let features = Features {{ simd: true, multi_value: {}, ..Default::default() }};",
+        "            let features = Features {{ simd: {}, multi_value: {}, ..Default::default() }};",
-        testsuite.contains("multi-value")
+        testsuite.contains("simd"),
+        testsuite.contains("multi_value")
     )?;
     writeln!(
         out,
@@ -166,16 +180,16 @@ fn write_testsuite_tests(
 }
 
 /// Ignore tests that aren't supported yet.
-fn ignore(testsuite: &str, name: &str, strategy: &str) -> bool {
+fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
-    let is_multi_value = testsuite.ends_with("multi-value");
+    let is_multi_value = testsuite.ends_with("multi_value");
     match strategy {
         #[cfg(feature = "lightbeam")]
-        "Lightbeam" => match (testsuite, name) {
+        "Lightbeam" => match (testsuite, testname) {
-            ("single_file_spec_test", "simd_const") => return true,
+            (_, _) if testname.starts_with("simd") => return true,
             (_, _) if is_multi_value => return true,
             _ => (),
         },
-        "Cranelift" => match (testsuite, name) {
+        "Cranelift" => match (testsuite, testname) {
             // We don't currently support more return values than available
             // registers, and this contains a function with many, many more
             // return values than that.
@@ -186,13 +200,16 @@ fn ignore(testsuite: &str, name: &str, strategy: &str) -> bool {
     }
 
     if cfg!(windows) {
-        return match (testsuite, name) {
+        return match (testsuite, testname) {
             // Currently, our multi-value support only works with however many
             // extra return registers we have available, and windows' fastcall
             // ABI only has a single return register, so we need to wait on full
             // multi-value support in Cranelift.
             (_, _) if is_multi_value => true,
 
+            // Until Windows unwind information is added we must disable SIMD spec tests that trap.
+            (_, _) if testname.starts_with("simd") => return true,
+
             ("spec_testsuite", "address") => true,
             ("spec_testsuite", "align") => true,
             ("spec_testsuite", "call") => true,
@@ -214,7 +231,7 @@ fn ignore(testsuite: &str, name: &str, strategy: &str) -> bool {
             ("spec_testsuite", "memory_trap") => true,
             ("spec_testsuite", "resizing") => true,
             ("spec_testsuite", "select") => true,
-            ("spec_testsuite", "skip-stack-guard-page") => true,
+            ("spec_testsuite", "skip_stack_guard_page") => true,
             ("spec_testsuite", "start") => true,
             ("spec_testsuite", "traps") => true,
             ("spec_testsuite", "unreachable") => true,
@@ -242,7 +259,7 @@ fn ignore(testsuite: &str, name: &str, strategy: &str) -> bool {
         .to_bits()
             != 0x26800001
         {
-            return match (testsuite, name) {
+            return match (testsuite, testname) {
                 ("spec_testsuite", "const") => true,
                 ("single_file_spec_test", "simd_const") => true,
                 (_, _) => false,