From a98d6e5256ed231acfffa7a32bd913aa1694c78c Mon Sep 17 00:00:00 2001 From: Jakob Stoklund Olesen Date: Fri, 16 Sep 2016 16:19:47 -0700 Subject: [PATCH] Add a 'test verifier' sub-test. This test runs the verifier on each function and matches the resulting verifier error against the "error:" annotation. Move the existing verifier test into filetests/verifier/ and use the new syntex. --- cranelift/filetests/cfg/loop.cton | 1 + cranelift/filetests/cfg/traps_early.cton | 3 +- .../verifier}/bad_layout.cton | 6 +- cranelift/src/tools/filetest/mod.rs | 1 + cranelift/src/tools/filetest/subtest.rs | 2 + cranelift/src/tools/filetest/verifier.rs | 78 +++++++++++++++++++ cranelift/src/tools/tests/verifier.rs | 66 ---------------- 7 files changed, 88 insertions(+), 69 deletions(-) rename cranelift/{src/tools/tests/verifier_testdata => filetests/verifier}/bad_layout.cton (71%) create mode 100644 cranelift/src/tools/filetest/verifier.rs delete mode 100644 cranelift/src/tools/tests/verifier.rs diff --git a/cranelift/filetests/cfg/loop.cton b/cranelift/filetests/cfg/loop.cton index 5732ac593a..9a1ca6d7e3 100644 --- a/cranelift/filetests/cfg/loop.cton +++ b/cranelift/filetests/cfg/loop.cton @@ -1,5 +1,6 @@ ; For testing cfg generation. This code is nonsense. test print-cfg +test verifier function nonsense(i32, i32) -> f32 { ; check: digraph nonsense { diff --git a/cranelift/filetests/cfg/traps_early.cton b/cranelift/filetests/cfg/traps_early.cton index a52cac1ba4..9648190bc5 100644 --- a/cranelift/filetests/cfg/traps_early.cton +++ b/cranelift/filetests/cfg/traps_early.cton @@ -1,12 +1,13 @@ ; For testing cfg generation. This code explores the implications of encountering ; a terminating instruction before any connections have been made. test print-cfg +test verifier function nonsense(i32) { ; check: digraph nonsense { ebb0(v1: i32): - trap + trap ; error: terminator instruction was encountered before the end brnz v1, ebb2 ; unordered: ebb0:inst1 -> ebb2 jump ebb1 ; unordered: ebb0:inst2 -> ebb1 diff --git a/cranelift/src/tools/tests/verifier_testdata/bad_layout.cton b/cranelift/filetests/verifier/bad_layout.cton similarity index 71% rename from cranelift/src/tools/tests/verifier_testdata/bad_layout.cton rename to cranelift/filetests/verifier/bad_layout.cton index a99b9b5fe4..ac29452958 100644 --- a/cranelift/src/tools/tests/verifier_testdata/bad_layout.cton +++ b/cranelift/filetests/verifier/bad_layout.cton @@ -1,6 +1,8 @@ -function test(i32) { ; Err(terminator) +test verifier + +function test(i32) { ebb0(v0: i32): - jump ebb1 + jump ebb1 ; error: terminator return ebb1: jump ebb2 diff --git a/cranelift/src/tools/filetest/mod.rs b/cranelift/src/tools/filetest/mod.rs index 0a7bdf0bab..b368959125 100644 --- a/cranelift/src/tools/filetest/mod.rs +++ b/cranelift/src/tools/filetest/mod.rs @@ -10,6 +10,7 @@ use filetest::runner::TestRunner; pub mod subtest; mod runner; mod domtree; +mod verifier; /// Main entry point for `cton-util test`. /// diff --git a/cranelift/src/tools/filetest/subtest.rs b/cranelift/src/tools/filetest/subtest.rs index 0359265d3b..cfc5cec5bf 100644 --- a/cranelift/src/tools/filetest/subtest.rs +++ b/cranelift/src/tools/filetest/subtest.rs @@ -13,10 +13,12 @@ pub fn new(parsed: &TestCommand) -> Result> { use cat; use print_cfg; use filetest::domtree; + use filetest::verifier; match parsed.command { "cat" => cat::subtest(parsed), "print-cfg" => print_cfg::subtest(parsed), "domtree" => domtree::subtest(parsed), + "verifier" => verifier::subtest(parsed), _ => Err(format!("unknown test command '{}'", parsed.command)), } } diff --git a/cranelift/src/tools/filetest/verifier.rs b/cranelift/src/tools/filetest/verifier.rs new file mode 100644 index 0000000000..5812dc9db0 --- /dev/null +++ b/cranelift/src/tools/filetest/verifier.rs @@ -0,0 +1,78 @@ +//! Test command for checking the IL verifier. +//! +//! The `test verifier` test command looks for annotations on instructions like this: +//! +//! jump ebb3 ; error: jump to non-existent EBB +//! +//! This annotation means that the verifier is expected to given an error for the jump instruction +//! containing the substring "jump to non-existent EBB". + +use std::borrow::{Borrow, Cow}; +use cretonne::verify_function; +use cretonne::ir::Function; +use cton_reader::TestCommand; +use filetest::subtest::{SubTest, Context, Result}; +use utils::match_directive; + +struct TestVerifier; + +pub fn subtest(parsed: &TestCommand) -> Result> { + assert_eq!(parsed.command, "verifier"); + if !parsed.options.is_empty() { + Err(format!("No options allowed on {}", parsed)) + } else { + Ok(Box::new(TestVerifier)) + } +} + +impl SubTest for TestVerifier { + fn name(&self) -> Cow { + Cow::from("verifier") + } + + fn needs_verifier(&self) -> bool { + // Running the verifier before this test would defeat its purpose. + false + } + + fn run(&self, func: Cow, context: &Context) -> Result<()> { + let func = func.borrow(); + + // Scan source annotations for "error:" directives. + let mut expected = None; + for comment in &context.details.comments { + if let Some(tail) = match_directive(comment.text, "error:") { + // Currently, the verifier can only report one problem at a time. + // Reject more than one `error:` directives. + if expected.is_some() { + return Err("cannot handle multiple error: directives".to_string()); + } + expected = Some((comment.entity, tail)); + } + } + + match verify_function(func) { + Ok(_) => { + match expected { + None => Ok(()), + Some((_, msg)) => Err(format!("passed, expected error: {}", msg)), + } + } + Err(got) => { + match expected { + None => Err(format!("verifier pass, got {}", got)), + Some((want_loc, want_msg)) if got.message.contains(want_msg) => { + if want_loc == got.location { + Ok(()) + } else { + Err(format!("correct error reported on {}, but wanted {}", + got.location, + want_loc)) + } + } + Some(_) => Err(format!("mismatching error: {}", got)), + } + } + } + } +} diff --git a/cranelift/src/tools/tests/verifier.rs b/cranelift/src/tools/tests/verifier.rs deleted file mode 100644 index 15fcf05ec8..0000000000 --- a/cranelift/src/tools/tests/verifier.rs +++ /dev/null @@ -1,66 +0,0 @@ -extern crate cretonne; -extern crate cton_reader; -extern crate glob; -extern crate regex; - -use std::env; -use glob::glob; -use regex::Regex; -use std::fs::File; -use std::io::Read; -use self::cton_reader::parse_functions; -use self::cretonne::verifier::Verifier; - -/// Compile a function and run verifier tests based on specially formatted -/// comments in the [function's] source. -fn verifier_tests_from_source(function_source: &str) { - let func_re = Regex::new("^[ \t]*function.*").unwrap(); - let err_re = Regex::new(";[ \t]*Err\\((.*)+\\)").unwrap(); - - // Each entry corresponds to an optional regular expression, where - // the index corresponds to the function offset in our source code. - // If no error is expected for a given function its entry will be - // set to None. - let mut verifier_results = Vec::new(); - for line in function_source.lines() { - if func_re.is_match(line) { - match err_re.captures(line) { - Some(caps) => { - verifier_results.push(Some(Regex::new(caps.at(1).unwrap()).unwrap())); - }, - None => { - verifier_results.push(None); - }, - }; - } - } - - // Run the verifier against each function and compare the output - // with the expected result (as determined above). - for (i, func) in parse_functions(function_source).unwrap().into_iter().enumerate() { - let result = Verifier::new(&func).run(); - match verifier_results[i] { - Some(ref re) => { - assert_eq!(re.is_match(&result.err().unwrap().message), true); - }, - None => { - assert_eq!(result, Ok(())); - } - } - } -} - -#[test] -fn test_all() { - let testdir = format!("{}/tests/verifier_testdata/*.cton", - env::current_dir().unwrap().display()); - - for entry in glob(&testdir).unwrap() { - let path = entry.unwrap(); - println!("Testing {:?}", path); - let mut file = File::open(&path).unwrap(); - let mut buffer = String::new(); - file.read_to_string(&mut buffer).unwrap(); - verifier_tests_from_source(&buffer); - } -}