Add a 'test verifier' sub-test.

This test runs the verifier on each function and matches the resulting
verifier error against the "error:" annotation.

Move the existing verifier test into filetests/verifier/ and use the new
syntex.

cranelift/filetests/cfg/loop.cton

@@ -1,5 +1,6 @@
 ; For testing cfg generation. This code is nonsense.
 test print-cfg
+test verifier
 
 function nonsense(i32, i32) -> f32 {
 ; check: digraph nonsense {

cranelift/filetests/cfg/traps_early.cton

@@ -1,12 +1,13 @@
 ; For testing cfg generation. This code explores the implications of encountering
 ; a terminating instruction before any connections have been made.
 test print-cfg
+test verifier
 
 function nonsense(i32) {
 ; check: digraph nonsense {
 
 ebb0(v1: i32):
-    trap
+    trap            ; error: terminator instruction was encountered before the end
     brnz v1, ebb2   ; unordered: ebb0:inst1 -> ebb2
     jump ebb1       ; unordered: ebb0:inst2 -> ebb1
 

cranelift/filetests/verifier/bad_layout.cton

@@ -1,6 +1,8 @@
-function test(i32) {    ; Err(terminator)
+test verifier
+
+function test(i32) {
     ebb0(v0: i32):
-        jump ebb1
+        jump ebb1       ; error: terminator
         return
     ebb1:
         jump ebb2

cranelift/src/tools/filetest/mod.rs

@@ -10,6 +10,7 @@ use filetest::runner::TestRunner;
 pub mod subtest;
 mod runner;
 mod domtree;
+mod verifier;
 
 /// Main entry point for `cton-util test`.
 ///

cranelift/src/tools/filetest/subtest.rs

@@ -13,10 +13,12 @@ pub fn new(parsed: &TestCommand) -> Result<Box<SubTest>> {
     use cat;
     use print_cfg;
     use filetest::domtree;
+    use filetest::verifier;
     match parsed.command {
         "cat" => cat::subtest(parsed),
         "print-cfg" => print_cfg::subtest(parsed),
         "domtree" => domtree::subtest(parsed),
+        "verifier" => verifier::subtest(parsed),
         _ => Err(format!("unknown test command '{}'", parsed.command)),
     }
 }

cranelift/src/tools/filetest/verifier.rs

@@ -0,0 +1,78 @@
+//! Test command for checking the IL verifier.
+//!
+//! The `test verifier` test command looks for annotations on instructions like this:
+//!
+//!     jump ebb3 ; error: jump to non-existent EBB
+//!
+//! This annotation means that the verifier is expected to given an error for the jump instruction
+//! containing the substring "jump to non-existent EBB".
+
+use std::borrow::{Borrow, Cow};
+use cretonne::verify_function;
+use cretonne::ir::Function;
+use cton_reader::TestCommand;
+use filetest::subtest::{SubTest, Context, Result};
+use utils::match_directive;
+
+struct TestVerifier;
+
+pub fn subtest(parsed: &TestCommand) -> Result<Box<SubTest>> {
+    assert_eq!(parsed.command, "verifier");
+    if !parsed.options.is_empty() {
+        Err(format!("No options allowed on {}", parsed))
+    } else {
+        Ok(Box::new(TestVerifier))
+    }
+}
+
+impl SubTest for TestVerifier {
+    fn name(&self) -> Cow<str> {
+        Cow::from("verifier")
+    }
+
+    fn needs_verifier(&self) -> bool {
+        // Running the verifier before this test would defeat its purpose.
+        false
+    }
+
+    fn run(&self, func: Cow<Function>, context: &Context) -> Result<()> {
+        let func = func.borrow();
+
+        // Scan source annotations for "error:" directives.
+        let mut expected = None;
+        for comment in &context.details.comments {
+            if let Some(tail) = match_directive(comment.text, "error:") {
+                // Currently, the verifier can only report one problem at a time.
+                // Reject more than one `error:` directives.
+                if expected.is_some() {
+                    return Err("cannot handle multiple error: directives".to_string());
+                }
+                expected = Some((comment.entity, tail));
+            }
+        }
+
+        match verify_function(func) {
+            Ok(_) => {
+                match expected {
+                    None => Ok(()),
+                    Some((_, msg)) => Err(format!("passed, expected error: {}", msg)),
+                }
+            }
+            Err(got) => {
+                match expected {
+                    None => Err(format!("verifier pass, got {}", got)),
+                    Some((want_loc, want_msg)) if got.message.contains(want_msg) => {
+                        if want_loc == got.location {
+                            Ok(())
+                        } else {
+                            Err(format!("correct error reported on {}, but wanted {}",
+                                        got.location,
+                                        want_loc))
+                        }
+                    }
+                    Some(_) => Err(format!("mismatching error: {}", got)),
+                }
+            }
+        }
+    }
+}

cranelift/src/tools/tests/verifier.rs

@@ -1,66 +0,0 @@
-extern crate cretonne;
-extern crate cton_reader;
-extern crate glob;
-extern crate regex;
-
-use std::env;
-use glob::glob;
-use regex::Regex;
-use std::fs::File;
-use std::io::Read;
-use self::cton_reader::parse_functions;
-use self::cretonne::verifier::Verifier;
-
-/// Compile a function and run verifier tests based on specially formatted
-/// comments in the [function's] source.
-fn verifier_tests_from_source(function_source: &str) {
-    let func_re = Regex::new("^[ \t]*function.*").unwrap();
-    let err_re = Regex::new(";[ \t]*Err\\((.*)+\\)").unwrap();
-
-    // Each entry corresponds to an optional regular expression, where
-    // the index corresponds to the function offset in our source code.
-    // If no error is expected for a given function its entry will be
-    // set to None.
-    let mut verifier_results = Vec::new();
-    for line in function_source.lines() {
-        if func_re.is_match(line) {
-            match err_re.captures(line) {
-                Some(caps) => {
-                    verifier_results.push(Some(Regex::new(caps.at(1).unwrap()).unwrap()));
-                },
-                None => {
-                    verifier_results.push(None);
-                },
-            };
-        }
-    }
-
-    // Run the verifier against each function and compare the output
-    // with the expected result (as determined above).
-    for (i, func) in parse_functions(function_source).unwrap().into_iter().enumerate() {
-        let result = Verifier::new(&func).run();
-        match verifier_results[i] {
-            Some(ref re) => {
-                assert_eq!(re.is_match(&result.err().unwrap().message), true);
-            },
-            None => {
-                assert_eq!(result, Ok(()));
-            }
-        }
-    }
-}
-
-#[test]
-fn test_all() {
-    let testdir = format!("{}/tests/verifier_testdata/*.cton",
-                          env::current_dir().unwrap().display());
-
-    for entry in glob(&testdir).unwrap() {
-        let path = entry.unwrap();
-        println!("Testing {:?}", path);
-        let mut file = File::open(&path).unwrap();
-        let mut buffer = String::new();
-        file.read_to_string(&mut buffer).unwrap();
-        verifier_tests_from_source(&buffer);
-    }
-}