Move the filetest harness into its own crate.

This allows us to run the tests via a library call rather than just
as a command execution. And, it's a step toward a broader goal, which
is to keep the code in the top-level src directory minimal, with
important functionality exposed as crates.

cranelift/Cargo.toml

@@ -18,11 +18,11 @@ cretonne-reader = { path = "lib/reader", version = "0.3.4" }
 cretonne-frontend = { path = "lib/frontend", version = "0.3.4" }
 cretonne-wasm = { path = "lib/wasm", version = "0.3.4" }
 cretonne-native = { path = "lib/native", version = "0.3.4" }
+cretonne-filetests = { path = "lib/filetests", version = "0.3.4" }
 filecheck = { path = "lib/filecheck" }
 docopt = "0.8.0"
 serde = "1.0.8"
 serde_derive = "1.0.8"
-num_cpus = "1.5.1"
 tempdir = "0.3.5"
 term = "0.5"
 

cranelift/docs/testing.rst

@@ -136,6 +136,9 @@ This example will run the legalizer test twice. Both runs will have
 ``opt_level=best``, but they will have different ``is_64bit`` settings. The 32-bit
 run will also have the RISC-V specific flag ``supports_m`` disabled.
 
+The filetests are run automatically as part of `cargo test`, and they can
+also be run manually with the `cton-util test` command.
+
 Filecheck
 ---------
 

cranelift/src/compile.rs

@@ -7,8 +7,9 @@ use std::path::PathBuf;
 use cretonne::Context;
 use cretonne::settings::FlagsOrIsa;
 use cretonne::{binemit, ir};
+use cretonne::print_errors::pretty_error;
 use std::path::Path;
-use utils::{pretty_error, read_to_string, parse_sets_and_isa};
+use utils::{read_to_string, parse_sets_and_isa};
 
 struct PrintRelocs {
     flag_print: bool,

cranelift/src/cton-util.rs

@@ -1,12 +1,11 @@
-#[macro_use(dbg)]
 extern crate cretonne;
 extern crate cton_reader;
 extern crate cton_wasm;
+extern crate cton_filetests;
 extern crate docopt;
 #[macro_use]
 extern crate serde_derive;
 extern crate filecheck;
-extern crate num_cpus;
 extern crate tempdir;
 extern crate term;
 
@@ -16,7 +15,6 @@ use std::io::{self, Write};
 use std::process;
 
 mod utils;
-mod filetest;
 mod cat;
 mod print_cfg;
 mod rsfilecheck;
@@ -88,7 +86,7 @@ fn cton_util() -> CommandResult {
 
     // Find the sub-command to execute.
     let result = if args.cmd_test {
-        filetest::run(args.flag_verbose, args.arg_file)
+        cton_filetests::run(args.flag_verbose, args.arg_file).map(|_time| ())
     } else if args.cmd_cat {
         cat::run(args.arg_file)
     } else if args.cmd_filecheck {

cranelift/src/rsfilecheck.rs

@@ -1,3 +1,7 @@
+//! The `filecheck` sub-command.
+//!
+//! This file is named to avoid a name collision with the filecheck crate.
+
 use CommandResult;
 use utils::read_to_string;
 use filecheck::{CheckerBuilder, Checker, NO_VARIABLES};

cranelift/src/utils.rs

@@ -1,13 +1,9 @@
 //! Utility functions.
 
-use cretonne::ir::entities::AnyEntity;
-use cretonne::{ir, verifier};
-use cretonne::result::CtonError;
 use cretonne::isa::TargetIsa;
 use cretonne::settings::{self, FlagsOrIsa};
 use cretonne::isa;
 use cton_reader::{parse_options, Location};
-use std::fmt::Write;
 use std::fs::File;
 use std::io::{self, Read};
 use std::path::Path;
@@ -28,51 +24,6 @@ pub fn read_to_end<P: AsRef<Path>>(path: P) -> io::Result<Vec<u8>> {
     Ok(buffer)
 }
 
-/// Look for a directive in a comment string.
-/// The directive is of the form "foo:" and should follow the leading `;` in the comment:
-///
-/// ; dominates: ebb3 ebb4
-///
-/// Return the comment text following the directive.
-pub fn match_directive<'a>(comment: &'a str, directive: &str) -> Option<&'a str> {
-    assert!(
-        directive.ends_with(':'),
-        "Directive must include trailing colon"
-    );
-    let text = comment.trim_left_matches(';').trim_left();
-    if text.starts_with(directive) {
-        Some(text[directive.len()..].trim())
-    } else {
-        None
-    }
-}
-
-/// Pretty-print a verifier error.
-pub fn pretty_verifier_error(
-    func: &ir::Function,
-    isa: Option<&TargetIsa>,
-    err: verifier::Error,
-) -> String {
-    let mut msg = err.to_string();
-    match err.location {
-        AnyEntity::Inst(inst) => {
-            write!(msg, "\n{}: {}\n\n", inst, func.dfg.display_inst(inst, isa)).unwrap()
-        }
-        _ => msg.push('\n'),
-    }
-    write!(msg, "{}", func.display(isa)).unwrap();
-    msg
-}
-
-/// Pretty-print a Cretonne error.
-pub fn pretty_error(func: &ir::Function, isa: Option<&TargetIsa>, err: CtonError) -> String {
-    if let CtonError::Verifier(e) = err {
-        pretty_verifier_error(func, isa, e)
-    } else {
-        err.to_string()
-    }
-}
-
 /// Like `FlagsOrIsa`, but holds ownership.
 pub enum OwnedFlagsOrIsa {
     Flags(settings::Flags),
@@ -119,12 +70,3 @@ pub fn parse_sets_and_isa(
         Ok(OwnedFlagsOrIsa::Flags(settings::Flags::new(&flag_builder)))
     }
 }
-
-#[test]
-fn test_match_directive() {
-    assert_eq!(match_directive("; foo: bar  ", "foo:"), Some("bar"));
-    assert_eq!(match_directive(" foo:bar", "foo:"), Some("bar"));
-    assert_eq!(match_directive("foo:bar", "foo:"), Some("bar"));
-    assert_eq!(match_directive(";x foo: bar", "foo:"), None);
-    assert_eq!(match_directive(";;; foo: bar", "foo:"), Some("bar"));
-}

cranelift/src/wasm.rs

@@ -6,6 +6,7 @@ use cton_wasm::{translate_module, DummyEnvironment, ModuleEnvironment};
 use std::path::PathBuf;
 use cretonne::Context;
 use cretonne::settings::FlagsOrIsa;
+use cretonne::print_errors::{pretty_error, pretty_verifier_error};
 use std::fs::File;
 use std::error::Error;
 use std::io;
@@ -13,7 +14,7 @@ use std::path::Path;
 use std::process::Command;
 use tempdir::TempDir;
 use term;
-use utils::{pretty_verifier_error, pretty_error, parse_sets_and_isa, read_to_end};
+use utils::{parse_sets_and_isa, read_to_end};
 
 macro_rules! vprintln {
     ($x: expr, $($tts:tt)*) => {

cranelift/test-all.sh

@@ -3,11 +3,10 @@ set -euo pipefail
 
 # This is the top-level test script:
 #
-# - Build documentation for Rust code in 'src/tools/target/doc'.
-# - Run unit tests for all Rust crates.
-# - Make a debug build of all crates.
-# - Make a release build of cton-util.
-# - Run file-level tests with the release build of cton-util.
+# - Make a debug build.
+# - Make a release build.
+# - Run unit tests for all Rust crates (including the filetests)
+# - Build API documentation.
 #
 # All tests run by this script should be passing at all times.
 

cranelift/tests/cton-util-test.rs

@@ -1,34 +0,0 @@
-//! Run `cton-util test` on all available testcases.
-
-use std::process::{Command, Output};
-use std::env;
-use std::path::PathBuf;
-use std::io::{self, Write};
-
-/// Returns the target directory, where we can find build artifacts
-/// and such for the current configuration.
-fn get_target_dir() -> PathBuf {
-    let mut path = env::current_exe().unwrap();
-    path.pop(); // chop off exe name
-    path.pop(); // chop off deps name
-    path
-}
-
-#[test]
-fn cton_util_test() {
-    let mut cmd = Command::new(&get_target_dir().join("cton-util"));
-    cmd.arg("test");
-
-    // We have testcases in the following directories:
-    cmd.arg("filetests");
-    cmd.arg("docs");
-
-    let Output {
-        status,
-        stdout,
-        stderr,
-    } = cmd.output().unwrap();
-    io::stdout().write(&stdout).unwrap();
-    io::stderr().write(&stderr).unwrap();
-    assert!(status.success(), "failed with exit status {}", status);
-}

cranelift/tests/filetests.rs

@@ -0,0 +1,7 @@
+extern crate cton_filetests;
+
+#[test]
+fn filetests() {
+    // Run all the filetests in the following directories.
+    cton_filetests::run(false, vec!["filetests".into(), "docs".into()]).expect("test harness");
+}

lib/cretonne/src/lib.rs

@@ -27,6 +27,7 @@ pub mod ir;
 pub mod isa;
 pub mod loop_analysis;
 pub mod packed_option;
+pub mod print_errors;
 pub mod result;
 pub mod settings;
 pub mod timing;

lib/cretonne/src/print_errors.rs

@@ -0,0 +1,33 @@
+//! Utility routines for pretty-printing error messages.
+
+use ir;
+use verifier;
+use result::CtonError;
+use isa::TargetIsa;
+use std::fmt::Write;
+
+/// Pretty-print a verifier error.
+pub fn pretty_verifier_error(
+    func: &ir::Function,
+    isa: Option<&TargetIsa>,
+    err: verifier::Error,
+) -> String {
+    let mut msg = err.to_string();
+    match err.location {
+        ir::entities::AnyEntity::Inst(inst) => {
+            write!(msg, "\n{}: {}\n\n", inst, func.dfg.display_inst(inst, isa)).unwrap()
+        }
+        _ => msg.push('\n'),
+    }
+    write!(msg, "{}", func.display(isa)).unwrap();
+    msg
+}
+
+/// Pretty-print a Cretonne error.
+pub fn pretty_error(func: &ir::Function, isa: Option<&TargetIsa>, err: CtonError) -> String {
+    if let CtonError::Verifier(e) = err {
+        pretty_verifier_error(func, isa, e)
+    } else {
+        err.to_string()
+    }
+}

lib/filetests/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "cretonne-filetests"
+authors = ["The Cretonne Project Developers"]
+version = "0.3.4"
+description = "Test driver and implementations of the filetest commands"
+license = "Apache-2.0"
+documentation = "http://cretonne.readthedocs.io/en/latest/testing.html#file-tests"
+repository = "https://github.com/Cretonne/cretonne"
+publish = false
+
+[lib]
+name = "cton_filetests"
+
+[dependencies]
+cretonne = { path = "../cretonne", version = "0.3.4" }
+cretonne-reader = { path = "../reader", version = "0.3.4" }
+filecheck = { path = "../filecheck", version = "0.1.0" }
+num_cpus = "1.5.1"

lib/filetests/src/concurrent.rs

@@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex};
 use std::thread;
 use std::time::Duration;
 use num_cpus;
-use filetest::{TestResult, runone};
+use {TestResult, runone};
 
 // Request sent to worker threads contains jobid and path.
 struct Request(usize, PathBuf);

lib/filetests/src/lib.rs

@@ -1,18 +1,24 @@
 //! File tests.
 //!
-//! This module contains the main driver for `cton-util test` as well as implementations of the
-//! available test commands.
+//! This crate contains the main test driver as well as implementations of the
+//! available filetest commands.
+
+#[macro_use(dbg)]
+extern crate cretonne;
+extern crate cton_reader;
+extern crate filecheck;
+extern crate num_cpus;
 
 use std::path::Path;
 use std::time;
 use cton_reader::TestCommand;
-use CommandResult;
-use filetest::runner::TestRunner;
+use runner::TestRunner;
 
 mod concurrent;
 mod runner;
 mod runone;
 mod subtest;
+mod match_directive;
 
 mod test_binemit;
 mod test_cat;
@@ -38,7 +44,7 @@ type TestResult = Result<time::Duration, String>;
 /// Directories are scanned recursively for test cases ending in `.cton`. These test cases are
 /// executed on background threads.
 ///
-pub fn run(verbose: bool, files: Vec<String>) -> CommandResult {
+pub fn run(verbose: bool, files: Vec<String>) -> TestResult {
     let mut runner = TestRunner::new(verbose);
 
     for path in files.iter().map(Path::new) {

lib/filetests/src/match_directive.rs

@@ -0,0 +1,27 @@
+/// Look for a directive in a comment string.
+/// The directive is of the form "foo:" and should follow the leading `;` in the comment:
+///
+/// ; dominates: ebb3 ebb4
+///
+/// Return the comment text following the directive.
+pub fn match_directive<'a>(comment: &'a str, directive: &str) -> Option<&'a str> {
+    assert!(
+        directive.ends_with(':'),
+        "Directive must include trailing colon"
+    );
+    let text = comment.trim_left_matches(';').trim_left();
+    if text.starts_with(directive) {
+        Some(text[directive.len()..].trim())
+    } else {
+        None
+    }
+}
+
+#[test]
+fn test_match_directive() {
+    assert_eq!(match_directive("; foo: bar  ", "foo:"), Some("bar"));
+    assert_eq!(match_directive(" foo:bar", "foo:"), Some("bar"));
+    assert_eq!(match_directive("foo:bar", "foo:"), Some("bar"));
+    assert_eq!(match_directive(";x foo: bar", "foo:"), None);
+    assert_eq!(match_directive(";;; foo: bar", "foo:"), Some("bar"));
+}

lib/filetests/src/runner.rs

@@ -7,9 +7,9 @@ use std::error::Error;
 use std::fmt::{self, Display};
 use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
-use filetest::{TestResult, runone};
-use filetest::concurrent::{ConcurrentRunner, Reply};
-use CommandResult;
+use std::time;
+use {TestResult, runone};
+use concurrent::{ConcurrentRunner, Reply};
 
 // Timeout in seconds when we're not making progress.
 const TIMEOUT_PANIC: usize = 10;
@@ -323,14 +323,15 @@ impl TestRunner {
     }
 
     /// Scan pushed directories for tests and run them.
-    pub fn run(&mut self) -> CommandResult {
+    pub fn run(&mut self) -> TestResult {
+        let started = time::Instant::now();
         self.scan_dirs();
         self.schedule_jobs();
         self.drain_threads();
         self.report_slow_tests();
         println!("{} tests", self.tests.len());
         match self.errors {
-            0 => Ok(()),
+            0 => Ok(started.elapsed()),
             1 => Err("1 failure".to_string()),
             n => Err(format!("{} failures", n)),
         }

lib/filetests/src/runone.rs

@@ -3,16 +3,26 @@
 use std::borrow::Cow;
 use std::path::Path;
 use std::time;
+use std::io::{self, Read};
+use std::fs;
 use cretonne::ir::Function;
 use cretonne::isa::TargetIsa;
 use cretonne::settings::Flags;
 use cretonne::timing;
 use cretonne::verify_function;
+use cretonne::print_errors::pretty_verifier_error;
 use cton_reader::parse_test;
 use cton_reader::IsaSpec;
-use utils::{read_to_string, pretty_verifier_error};
-use filetest::{TestResult, new_subtest};
-use filetest::subtest::{SubTest, Context, Result};
+use {TestResult, new_subtest};
+use subtest::{SubTest, Context, Result};
+
+/// Read an entire file into a string.
+fn read_to_string<P: AsRef<Path>>(path: P) -> io::Result<String> {
+    let mut file = fs::File::open(path)?;
+    let mut buffer = String::new();
+    file.read_to_string(&mut buffer)?;
+    Ok(buffer)
+}
 
 /// Load `path` and run the test in it.
 ///

lib/filetests/src/test_binemit.rs

@@ -11,9 +11,10 @@ use cretonne::dbg::DisplayList;
 use cretonne::ir;
 use cretonne::ir::entities::AnyEntity;
 use cretonne::binemit::RegDiversions;
+use cretonne::print_errors::pretty_error;
 use cton_reader::TestCommand;
-use filetest::subtest::{SubTest, Context, Result};
-use utils::{match_directive, pretty_error};
+use subtest::{SubTest, Context, Result};
+use match_directive::match_directive;
 
 struct TestBinEmit;
 

lib/filetests/src/test_cat.rs

@@ -3,7 +3,7 @@
 use std::borrow::Cow;
 use cretonne::ir::Function;
 use cton_reader::TestCommand;
-use filetest::subtest::{self, SubTest, Context, Result as STResult};
+use subtest::{self, SubTest, Context, Result as STResult};
 
 /// Object implementing the `test cat` sub-test.
 ///

lib/filetests/src/test_compile.rs

@@ -5,11 +5,11 @@
 use cretonne::binemit;
 use cretonne::ir;
 use cretonne;
+use cretonne::print_errors::pretty_error;
 use cton_reader::TestCommand;
-use filetest::subtest::{SubTest, Context, Result, run_filecheck};
+use subtest::{SubTest, Context, Result, run_filecheck};
 use std::borrow::Cow;
 use std::fmt::Write;
-use utils::pretty_error;
 
 struct TestCompile;
 

lib/filetests/src/test_domtree.rs

@@ -2,7 +2,9 @@
 //!
 //! The `test domtree` test command looks for annotations on instructions like this:
 //!
+//! ```cton
 //!     jump ebb3 ; dominates: ebb3
+//! ```
 //!
 //! This annotation means that the jump instruction is expected to be the immediate dominator of
 //! `ebb3`.
@@ -15,12 +17,12 @@ use cretonne::flowgraph::ControlFlowGraph;
 use cretonne::ir::Function;
 use cretonne::ir::entities::AnyEntity;
 use cton_reader::TestCommand;
-use filetest::subtest::{SubTest, Context, Result, run_filecheck};
+use subtest::{SubTest, Context, Result, run_filecheck};
 use std::borrow::{Borrow, Cow};
 use std::collections::HashMap;
 use std::fmt::{self, Write};
 use std::result;
-use utils::match_directive;
+use match_directive::match_directive;
 
 struct TestDomtree;
 

lib/filetests/src/test_legalizer.rs

@@ -6,10 +6,10 @@
 use std::borrow::Cow;
 use cretonne;
 use cretonne::ir::Function;
+use cretonne::print_errors::pretty_error;
 use cton_reader::TestCommand;
-use filetest::subtest::{SubTest, Context, Result, run_filecheck};
+use subtest::{SubTest, Context, Result, run_filecheck};
 use std::fmt::Write;
-use utils::pretty_error;
 
 struct TestLegalizer;
 

lib/filetests/src/test_licm.rs

@@ -7,11 +7,11 @@
 
 use cretonne::ir::Function;
 use cretonne;
+use cretonne::print_errors::pretty_error;
 use cton_reader::TestCommand;
-use filetest::subtest::{SubTest, Context, Result, run_filecheck};
+use subtest::{SubTest, Context, Result, run_filecheck};
 use std::borrow::Cow;
 use std::fmt::Write;
-use utils::pretty_error;
 
 struct TestLICM;
 

lib/filetests/src/test_preopt.rs

@@ -4,11 +4,11 @@
 
 use cretonne::ir::Function;
 use cretonne;
+use cretonne::print_errors::pretty_error;
 use cton_reader::TestCommand;
-use filetest::subtest::{SubTest, Context, Result, run_filecheck};
+use subtest::{SubTest, Context, Result, run_filecheck};
 use std::borrow::Cow;
 use std::fmt::Write;
-use utils::pretty_error;
 
 struct TestPreopt;
 

lib/filetests/src/test_print_cfg.rs

@@ -8,7 +8,7 @@ use std::borrow::Cow;
 use cretonne::ir::Function;
 use cretonne::cfg_printer::CFGPrinter;
 use cton_reader::TestCommand;
-use filetest::subtest::{self, SubTest, Context, Result as STResult};
+use subtest::{self, SubTest, Context, Result as STResult};
 
 /// Object implementing the `test print-cfg` sub-test.
 struct TestPrintCfg;

lib/filetests/src/test_regalloc.rs

@@ -7,11 +7,11 @@
 
 use cretonne::ir::Function;
 use cretonne;
+use cretonne::print_errors::pretty_error;
 use cton_reader::TestCommand;
-use filetest::subtest::{SubTest, Context, Result, run_filecheck};
+use subtest::{SubTest, Context, Result, run_filecheck};
 use std::borrow::Cow;
 use std::fmt::Write;
-use utils::pretty_error;
 
 struct TestRegalloc;
 

lib/filetests/src/test_simple_gvn.rs

@@ -7,11 +7,11 @@
 
 use cretonne::ir::Function;
 use cretonne;
+use cretonne::print_errors::pretty_error;
 use cton_reader::TestCommand;
-use filetest::subtest::{SubTest, Context, Result, run_filecheck};
+use subtest::{SubTest, Context, Result, run_filecheck};
 use std::borrow::Cow;
 use std::fmt::Write;
-use utils::pretty_error;
 
 struct TestSimpleGVN;
 

lib/filetests/src/test_verifier.rs

@@ -2,7 +2,9 @@
 //!
 //! The `test verifier` test command looks for annotations on instructions like this:
 //!
+//! ```cton
 //!     jump ebb3 ; error: jump to non-existent EBB
+//! ```
 //!
 //! This annotation means that the verifier is expected to given an error for the jump instruction
 //! containing the substring "jump to non-existent EBB".
@@ -11,8 +13,8 @@ use std::borrow::{Borrow, Cow};
 use cretonne::verify_function;
 use cretonne::ir::Function;
 use cton_reader::TestCommand;
-use filetest::subtest::{SubTest, Context, Result};
-use utils::match_directive;
+use subtest::{SubTest, Context, Result};
+use match_directive::match_directive;
 
 struct TestVerifier;
 

lib/wasm/tests/wasm_testsuite.rs

@@ -11,11 +11,9 @@ use std::str;
 use std::io::prelude::*;
 use std::process::Command;
 use std::fs;
-use cretonne::ir;
-use cretonne::ir::entities::AnyEntity;
-use cretonne::isa::TargetIsa;
 use cretonne::settings::{self, Configurable, Flags};
 use cretonne::verifier;
+use cretonne::print_errors::pretty_verifier_error;
 use tempdir::TempDir;
 
 #[test]
@@ -109,25 +107,3 @@ fn handle_module(path: PathBuf, flags: &Flags) {
             .unwrap();
     }
 }
-
-
-/// Pretty-print a verifier error.
-pub fn pretty_verifier_error(
-    func: &ir::Function,
-    isa: Option<&TargetIsa>,
-    err: verifier::Error,
-) -> String {
-    let msg = err.to_string();
-    let str1 = match err.location {
-        AnyEntity::Inst(inst) => {
-            format!(
-                "{}\n{}: {}\n\n",
-                msg,
-                inst,
-                func.dfg.display_inst(inst, isa)
-            )
-        }
-        _ => String::from(format!("{}\n", msg)),
-    };
-    format!("{}{}", str1, func.display(isa))
-}