cranelift: Add ability to auto-update test expectations (#3612)

* cranelift: Add ability to auto-update test expectations

One of the problems of the current `*.clif` testing is that the files
are difficult to update when widespread changes are made (such as
removing modification of the frame pointer). Additionally when changing
register allocation or similar it can cause a large number of changes in
tests but the tests themselves didn't actually break. For this reason
this commit adds the ability to automatically update test expectations.

The idea behind this commit is that tests of the form `test compile` can
also optionally be flagged with the `precise-output` flag:

    test compile precise-output

and when doing so the compiled form of each function is asserted to 100%
match the following comments and their test expectations. If a match is
not found then a `BLESS=1` environment variable can be used to
automatically rewrite the test file itself with the correct assertion.
If the environment variable isn't present and the expectation doesn't
match then the test fails.

It's hoped that, if approved, a follow-up commit can add
`precise-output` to all current `test compile` tests (or make it the
default) and all tests can be mass-updated. When developing locally test
expectations need not be written and instead tests can be run with
`BLESS=1` and the output can be manually verified. The environment
variable will not be present on CI which means that changes to the
output which don't also change the test expectation will cause CI to
fail. Furthermore this should still make updates to the test output
easily readable in review on CI because the test expectations are
intended to look the same as before.

Closes #1539

* Use raw vcode output in tests

* Fix a merge conflict

* Review comments

cranelift/filetests/filetests/isa/aarch64/simd.clif

@@ -1,4 +1,4 @@
-test compile
+test compile precise-output
 set unwind_info=false
 target aarch64
 
@@ -9,10 +9,16 @@ block0:
   return v1
 }
 
-; check:  movz x0, #1
+; VCode_ShowWithRRU {{
-; nextln:  movk x0, #1, LSL #48
+;   Entry block: 0
-; nextln:  dup v0.2d, x0
+; Block 0:
-; nextln:  ret
+;   (original IR block: block0)
+;   (instruction range: 0 .. 4)
+;   Inst 0:   movz x0, #1
+;   Inst 1:   movk x0, #1, LSL #48
+;   Inst 2:   dup v0.2d, x0
+;   Inst 3:   ret
+; }}
 
 function %f2() -> i16x8 {
 block0:
@@ -22,9 +28,15 @@ block0:
   return v2
 }
 
-; check:  movz x0, #42679
+; VCode_ShowWithRRU {{
-; nextln:  dup v0.8h, w0
+;   Entry block: 0
-; nextln:  ret
+; Block 0:
+;   (original IR block: block0)
+;   (instruction range: 0 .. 3)
+;   Inst 0:   movz x0, #42679
+;   Inst 1:   dup v0.8h, w0
+;   Inst 2:   ret
+; }}
 
 function %f3() -> b8x16 {
 block0:
@@ -34,8 +46,14 @@ block0:
   return v2
 }
 
-; check:  movi v0.16b, #255
+; VCode_ShowWithRRU {{
-; nextln:  ret
+;   Entry block: 0
+; Block 0:
+;   (original IR block: block0)
+;   (instruction range: 0 .. 2)
+;   Inst 0:   movi v0.16b, #255
+;   Inst 1:   ret
+; }}
 
 function %f4(i32, i8x16, i8x16) -> i8x16 {
 block0(v0: i32, v1: i8x16, v2: i8x16):
@@ -43,9 +61,15 @@ block0(v0: i32, v1: i8x16, v2: i8x16):
    return v3
 }
 
-; check:  subs wzr, w0, wzr
+; VCode_ShowWithRRU {{
-; nextln:  vcsel v0.16b, v0.16b, v1.16b, ne (if-then-else diamond)
+;   Entry block: 0
-; nextln:  ret
+; Block 0:
+;   (original IR block: block0)
+;   (instruction range: 0 .. 3)
+;   Inst 0:   subs wzr, w0, wzr
+;   Inst 1:   vcsel v0.16b, v0.16b, v1.16b, ne (if-then-else diamond)
+;   Inst 2:   ret
+; }}
 
 function %f5(i64) -> i8x16 {
 block0(v0: i64):
@@ -54,8 +78,14 @@ block0(v0: i64):
   return v2
 }
 
-; check:  ld1r { v0.16b }, [x0]
+; VCode_ShowWithRRU {{
-; nextln:  ret
+;   Entry block: 0
+; Block 0:
+;   (original IR block: block0)
+;   (instruction range: 0 .. 2)
+;   Inst 0:   ld1r { v0.16b }, [x0]
+;   Inst 1:   ret
+; }}
 
 function %f6(i64, i64) -> i8x16, i8x16 {
 block0(v0: i64, v1: i64):
@@ -66,9 +96,15 @@ block0(v0: i64, v1: i64):
   return v4, v5
 }
 
-; check:  ld1r { v0.16b }, [x0]
+; VCode_ShowWithRRU {{
-; nextln:  ld1r { v1.16b }, [x1]
+;   Entry block: 0
-; nextln:  ret
+; Block 0:
+;   (original IR block: block0)
+;   (instruction range: 0 .. 3)
+;   Inst 0:   ld1r { v0.16b }, [x0]
+;   Inst 1:   ld1r { v1.16b }, [x1]
+;   Inst 2:   ret
+; }}
 
 function %f7(i64, i64) -> i8x16, i8x16 {
 block0(v0: i64, v1: i64):
@@ -79,10 +115,16 @@ block0(v0: i64, v1: i64):
   return v4, v5
 }
 
-; check:  ldrb w0, [x0]
+; VCode_ShowWithRRU {{
-; nextln:  ld1r { v0.16b }, [x1]
+;   Entry block: 0
-; nextln:  dup v1.16b, w0
+; Block 0:
-; nextln:  ret
+;   (original IR block: block0)
+;   (instruction range: 0 .. 4)
+;   Inst 0:   ldrb w0, [x0]
+;   Inst 1:   ld1r { v0.16b }, [x1]
+;   Inst 2:   dup v1.16b, w0
+;   Inst 3:   ret
+; }}
 
 function %f8(i64, i64) -> i8x16, i8x16 {
 block0(v0: i64, v1: i64):
@@ -92,10 +134,16 @@ block0(v0: i64, v1: i64):
   return v3, v4
 }
 
-; check:  ldrb w0, [x0]
+; VCode_ShowWithRRU {{
-; nextln:  dup v0.16b, w0
+;   Entry block: 0
-; nextln:  dup v1.16b, w0
+; Block 0:
-; nextln:  ret
+;   (original IR block: block0)
+;   (instruction range: 0 .. 4)
+;   Inst 0:   ldrb w0, [x0]
+;   Inst 1:   dup v0.16b, w0
+;   Inst 2:   dup v1.16b, w0
+;   Inst 3:   ret
+; }}
 
 function %f9() -> i32x2 {
 block0:
@@ -104,9 +152,15 @@ block0:
   return v1
 }
 
-; check:  movi v0.2d, #18374687579166474495
+; VCode_ShowWithRRU {{
-; nextln:  fmov d0, d0
+;   Entry block: 0
-; nextln:  ret
+; Block 0:
+;   (original IR block: block0)
+;   (instruction range: 0 .. 3)
+;   Inst 0:   movi v0.2d, #18374687579166474495
+;   Inst 1:   fmov d0, d0
+;   Inst 2:   ret
+; }}
 
 function %f10() -> i32x4 {
 block0:
@@ -115,8 +169,14 @@ block0:
   return v1
 }
 
-; check:  mvni v0.4s, #15, MSL #16
+; VCode_ShowWithRRU {{
-; nextln:  ret
+;   Entry block: 0
+; Block 0:
+;   (original IR block: block0)
+;   (instruction range: 0 .. 2)
+;   Inst 0:   mvni v0.4s, #15, MSL #16
+;   Inst 1:   ret
+; }}
 
 function %f11() -> f32x4 {
 block0:
@@ -125,5 +185,12 @@ block0:
   return v1
 }
 
-; check:  fmov v0.4s, #1.3125
+; VCode_ShowWithRRU {{
-; nextln:  ret
+;   Entry block: 0
+; Block 0:
+;   (original IR block: block0)
+;   (instruction range: 0 .. 2)
+;   Inst 0:   fmov v0.4s, #1.3125
+;   Inst 1:   ret
+; }}
+

cranelift/filetests/src/runone.rs

@@ -2,18 +2,20 @@
 
 use crate::new_subtest;
 use crate::subtest::{Context, SubTest};
-use anyhow::Context as _;
+use anyhow::{bail, Context as _, Result};
 use cranelift_codegen::ir::Function;
 use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::print_errors::pretty_verifier_error;
 use cranelift_codegen::settings::Flags;
 use cranelift_codegen::timing;
 use cranelift_codegen::verify_function;
-use cranelift_reader::{parse_test, Feature, IsaSpec, ParseOptions, TestFile};
+use cranelift_reader::{parse_test, Feature, IsaSpec, Location, ParseOptions, TestFile};
 use log::info;
 use std::borrow::Cow;
+use std::cell::Cell;
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::str::Lines;
 use std::time;
 
 /// Skip the tests which define features and for which there's a feature mismatch.
@@ -113,6 +115,7 @@ pub fn run(
         Some(t) => t,
     };
 
+    let mut file_update = FileUpdate::new(&path);
     let file_path = path.to_string_lossy();
     for (func, details) in testfile.functions {
         let mut context = Context {
@@ -122,6 +125,7 @@ pub fn run(
             flags,
             isa: None,
             file_path: file_path.as_ref(),
+            file_update: &mut file_update,
         };
 
         for tuple in &tuples {
@@ -187,3 +191,88 @@ fn run_one_test<'a>(
     test.run(func, context).context(test.name())?;
     Ok(())
 }
+
+/// A helper struct to update a file in-place as test expectations are
+/// automatically updated.
+///
+/// This structure automatically handles multiple edits to one file. Our edits
+/// are line-based but if editing a previous portion of the file adds lines then
+/// all future edits need to know to skip over those previous lines. Note that
+/// this assumes that edits are done front-to-back.
+pub struct FileUpdate {
+    path: PathBuf,
+    line_diff: Cell<isize>,
+    last_update: Cell<usize>,
+}
+
+impl FileUpdate {
+    fn new(path: &Path) -> FileUpdate {
+        FileUpdate {
+            path: path.to_path_buf(),
+            line_diff: Cell::new(0),
+            last_update: Cell::new(0),
+        }
+    }
+
+    /// Updates the file that this structure references at the `location`
+    /// specified.
+    ///
+    /// The closure `f` is given first a buffer to push the new test into along
+    /// with a lines iterator for the old test.
+    pub fn update_at(
+        &self,
+        location: &Location,
+        f: impl FnOnce(&mut String, &mut Lines<'_>),
+    ) -> Result<()> {
+        // This is required for correctness of this update.
+        assert!(location.line_number > self.last_update.get());
+        self.last_update.set(location.line_number);
+
+        // Read the old test file and calculate thte new line number we're
+        // preserving up to based on how many lines prior to this have been
+        // removed or added.
+        let old_test = std::fs::read_to_string(&self.path)?;
+        let mut new_test = String::new();
+        let mut lines = old_test.lines();
+        let lines_to_preserve =
+            (((location.line_number - 1) as isize) + self.line_diff.get()) as usize;
+
+        // Push everything leading up to the start of the function
+        for _ in 0..lines_to_preserve {
+            new_test.push_str(lines.next().unwrap());
+            new_test.push_str("\n");
+        }
+
+        // Push the whole function, leading up to the trailing `}`
+        let mut first = true;
+        while let Some(line) = lines.next() {
+            if first && !line.starts_with("function") {
+                bail!(
+                    "line {} in test file {:?} did not start with `function`, \
+                     cannot automatically update test",
+                    location.line_number,
+                    self.path,
+                );
+            }
+            first = false;
+            new_test.push_str(line);
+            new_test.push_str("\n");
+            if line.starts_with("}") {
+                break;
+            }
+        }
+
+        // Use our custom update function to further update the test.
+        f(&mut new_test, &mut lines);
+
+        // Record the difference in line count so future updates can be adjusted
+        // accordingly, and then write the file back out to the filesystem.
+        let old_line_count = old_test.lines().count();
+        let new_line_count = new_test.lines().count();
+        self.line_diff
+            .set(self.line_diff.get() + (new_line_count as isize - old_line_count as isize));
+
+        std::fs::write(&self.path, new_test)?;
+        Ok(())
+    }
+}

cranelift/filetests/src/subtest.rs

@@ -1,5 +1,6 @@
 //! `SubTest` trait.
 
+use crate::runone::FileUpdate;
 use anyhow::Context as _;
 use cranelift_codegen::ir::Function;
 use cranelift_codegen::isa::TargetIsa;
@@ -28,6 +29,10 @@ pub struct Context<'a> {
 
     /// Full path to the file containing the test.
     pub file_path: &'a str,
+
+    /// Context used to update the original `file_path` in-place with its test
+    /// expectations if so configured in the environment.
+    pub file_update: &'a FileUpdate,
 }
 
 impl<'a> Context<'a> {

cranelift/filetests/src/test_compile.rs

@@ -3,21 +3,34 @@
 //! The `compile` test command runs each function through the full code generator pipeline
 
 use crate::subtest::{run_filecheck, Context, SubTest};
-use cranelift_codegen;
+use anyhow::{bail, Result};
 use cranelift_codegen::binemit::{self, CodeInfo};
 use cranelift_codegen::ir;
-use cranelift_reader::TestCommand;
+use cranelift_reader::{TestCommand, TestOption};
 use log::info;
 use std::borrow::Cow;
+use std::env;
 
-struct TestCompile;
+struct TestCompile {
+    /// Flag indicating that the text expectation, comments after the function,
+    /// must be a precise 100% match on the compiled output of the function.
+    /// This test assertion is also automatically-update-able to allow tweaking
+    /// the code generator and easily updating all affected tests.
+    precise_output: bool,
+}
 
-pub fn subtest(parsed: &TestCommand) -> anyhow::Result<Box<dyn SubTest>> {
+pub fn subtest(parsed: &TestCommand) -> Result<Box<dyn SubTest>> {
     assert_eq!(parsed.command, "compile");
-    if !parsed.options.is_empty() {
+    let mut test = TestCompile {
-        anyhow::bail!("No options allowed on {}", parsed);
+        precise_output: false,
+    };
+    for option in parsed.options.iter() {
+        match option {
+            TestOption::Flag("precise-output") => test.precise_output = true,
+            _ => anyhow::bail!("unknown option on {}", parsed),
+        }
     }
-    Ok(Box::new(TestCompile))
+    Ok(Box::new(test))
 }
 
 impl SubTest for TestCompile {
@@ -33,7 +46,7 @@ impl SubTest for TestCompile {
         true
     }
 
-    fn run(&self, func: Cow<ir::Function>, context: &Context) -> anyhow::Result<()> {
+    fn run(&self, func: Cow<ir::Function>, context: &Context) -> Result<()> {
         let isa = context.isa.expect("compile needs an ISA");
         let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned());
 
@@ -54,7 +67,11 @@ impl SubTest for TestCompile {
 
         info!("Generated {} bytes of code:\n{}", total_size, disasm);
 
-        run_filecheck(&disasm, context)
+        if self.precise_output {
+            check_precise_output(&disasm, context)
+        } else {
+            run_filecheck(&disasm, context)
+        }
     }
 }
 
@@ -95,3 +112,74 @@ impl binemit::CodeSink for SizeSink {
     fn trap(&mut self, _code: ir::TrapCode, _srcloc: ir::SourceLoc) {}
     fn end_codegen(&mut self) {}
 }
+
+fn check_precise_output(text: &str, context: &Context) -> Result<()> {
+    let actual = text.lines().collect::<Vec<_>>();
+
+    // Use the comments after the function to build the test expectation.
+    let expected = context
+        .details
+        .comments
+        .iter()
+        .map(|c| c.text.strip_prefix("; ").unwrap_or(c.text))
+        .collect::<Vec<_>>();
+
+    // If the expectation matches what we got, then there's nothing to do.
+    if actual == expected {
+        return Ok(());
+    }
+
+    // If we're supposed to automatically update the test, then do so here.
+    if env::var("CRANELIFT_TEST_BLESS").unwrap_or(String::new()) == "1" {
+        return update_test(&actual, context);
+    }
+
+    // Otherwise this test has failed, and we can print out as such.
+    bail!(
+        "compilation of function on line {} does not match\n\
+         the text expectation\n\
+         \n\
+         expected:\n\
+         {:#?}\n\
+         actual:\n\
+         {:#?}\n\
+         \n\
+         This test assertion can be automatically updated by setting the\n\
+         CRANELIFT_TEST_BLESS=1 environment variable when running this test.
+         ",
+        context.details.location.line_number,
+        expected,
+        actual,
+    )
+}
+
+fn update_test(output: &[&str], context: &Context) -> Result<()> {
+    context
+        .file_update
+        .update_at(&context.details.location, |new_test, old_test| {
+            // blank newline after the function
+            new_test.push_str("\n");
+
+            // Splice in the test output
+            for output in output {
+                new_test.push_str("; ");
+                new_test.push_str(output);
+                new_test.push_str("\n");
+            }
+
+            // blank newline after test assertion
+            new_test.push_str("\n");
+
+            // Drop all remaining commented lines (presumably the old test expectation),
+            // but after we hit a real line then we push all remaining lines.
+            let mut in_next_function = false;
+            for line in old_test {
+                if !in_next_function && (line.trim().is_empty() || line.starts_with(";")) {
+                    continue;
+                }
+                in_next_function = true;
+                new_test.push_str(line);
+                new_test.push_str("\n");
+            }
+        })
+}