From 7ec54a5a019ae1f02e3ac14d4b5c43aeb6bc643c Mon Sep 17 00:00:00 2001 From: Jakob Stoklund Olesen Date: Fri, 23 Sep 2016 09:38:17 -0700 Subject: [PATCH] Add a Cretonne testing guide. Describe the basics of Rust-level tests, and go into more detail about the file-level tests. --- docs/cton_lexer.py | 19 ++- docs/index.rst | 3 +- docs/langref.rst | 10 +- docs/testing.rst | 295 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 317 insertions(+), 10 deletions(-) create mode 100644 docs/testing.rst diff --git a/docs/cton_lexer.py b/docs/cton_lexer.py index eaa856c444..868bb23241 100644 --- a/docs/cton_lexer.py +++ b/docs/cton_lexer.py @@ -19,6 +19,17 @@ class CretonneLexer(RegexLexer): tokens = { 'root': [ + # Test header lines. + (r'^(test|isa|set)(?:( +)([-\w]+)' + + r'(?:(=)(?:(\d+)|(yes|no|true|false|on|off)|(\w+)))?)*' + + r'( *)$', + bygroups(Keyword.Namespace, Whitespace, Name.Attribute, + Operator, Number.Integer, Keyword.Constant, + Name.Constant, Whitespace)), + # Comments with filecheck or other test directive. + (r'(; *)([a-z]+:)(.*?)$', + bygroups(Comment.Single, Comment.Special, Comment.Single)), + # Plain comments. (r';.*?$', Comment.Single), # Strings are in double quotes, support \xx escapes only. (r'"([^"\\]+|\\[0-9a-fA-F]{2})*"', String), @@ -30,16 +41,16 @@ class CretonneLexer(RegexLexer): (r'[-+]?0[xX][0-9a-fA-F]*\.[0-9a-fA-F]*([pP]\d+)?', Number.Hex), (r'[-+]?(\d+\.\d+([eE]\d+)?|[sq]NaN|Inf)', Number.Float), (r'[-+]?\d+', Number.Integer), - # Reserved words. - (keywords('function'), Keyword), # Known attributes. (keywords('align', 'aligntrap', 'uext', 'sext', 'inreg'), Name.Attribute), # Well known value types. (r'\b(b\d+|i\d+|f32|f64)(x\d+)?\b', Keyword.Type), # v = value + # vx = value # ss = stack slot - (r'(v|ss)\d+', Name.Variable), + # jt = jump table + (r'(vx?|ss|jt)\d+', Name.Variable), # ebb = extended basic block (r'(ebb)\d+', Name.Label), # Match instruction names in context. @@ -52,7 +63,7 @@ class CretonneLexer(RegexLexer): (r'->|=|:', Operator), (r'[{}(),.]', Punctuation), (r'[ \t]+', Text), - ] + ], } diff --git a/docs/index.rst b/docs/index.rst index 88d9257213..b5a23ab28b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,10 +4,11 @@ Cretonne Code Generator Contents: .. toctree:: - :maxdepth: 2 + :maxdepth: 1 langref metaref + testing Indices and tables ================== diff --git a/docs/langref.rst b/docs/langref.rst index 848d140db8..10385cb7f4 100644 --- a/docs/langref.rst +++ b/docs/langref.rst @@ -48,11 +48,11 @@ fall through to the next EBB without an explicit branch. A ``.cton`` file consists of a sequence of independent function definitions: .. productionlist:: - function-list : { function } - function : function-spec "{" preamble function-body "}" - function-spec : "function" function-name signature - preamble : { preamble-decl } - function-body : { extended-basic-block } + function_list : { function } + function : function_spec "{" preamble function_body "}" + function_spec : "function" function_name signature + preamble : { preamble_decl } + function_body : { extended_basic_block } Static single assignment form ----------------------------- diff --git a/docs/testing.rst b/docs/testing.rst new file mode 100644 index 0000000000..c902dc70f6 --- /dev/null +++ b/docs/testing.rst @@ -0,0 +1,295 @@ +**************** +Testing Cretonne +**************** + +Cretonne is tested at multiple levels of abstraction and integration. When +possible, Rust unit tests are used to verify single functions and types. When +testing the interaction between compiler passes, file-level tests are +appropriate. + +The top-level shell script :file:`test-all.sh` runs all of the tests in the +Cretonne repository. + +Rust tests +========== + +.. highlight:: rust + +Rust and Cargo have good support for testing. Cretonne uses unit tests, doc +tests, and integration tests where appropriate. + +Unit tests +---------- + +Unit test live in a ``tests`` sub-module of the code they are testing:: + + pub fn add(x: u32, y: u32) { + x + y + } + + #[cfg(test)] + mod tests { + use super::add; + + #[test] + check_add() { + assert_eq!(add(2, 2), 4); + } + } + +Since sub-modules have access to non-public items in a Rust module, unit tests +can be used to test module-internal functions and types too. + +Doc tests +--------- + +Documentation comments can contain code snippets which are also compiled and +tested:: + + //! The `Flags` struct is immutable once it has been created. A `Builder` instance is used to + //! create it. + //! + //! # Example + //! ``` + //! use cretonne::settings::{self, Configurable}; + //! + //! let mut b = settings::builder(); + //! b.set("opt_level", "fastest"); + //! + //! let f = settings::Flags::new(&b); + //! assert_eq!(f.opt_level(), settings::OptLevel::Fastest); + //! ``` + +These tests are useful for demonstrating how to use an API, and running them +regularly makes sure that they stay up to date. Documentation tests are not +appropriate for lots of assertions, use unit tests for that. + +Integration tests +----------------- + +Integration tests are Rust source files thar are compiled and linked +individually. They are used to exercise the external API of the crates under +test. + +These tests are usually found in the :file:`src/tools/tests` sub-directory where they +have access to all the crates in the Cretonne repository. The +:file:`libcretonne` and :file:`libreader` crates have no external dependencies +which can make testing tedious. + +File tests +========== + +.. highlight:: cton + +Compilers work with large data structures representing programs, and it quickly +gets unwieldy to generate test data programmatically. File-level tests make it +easier to provide substantial input functions for the compiler tests. + +File tests are :file:`*.cton` files in the :file:`filetests/` directory +hierarchy. Each file has a header describing what to test followed by a number +of input functions in the :doc:`Cretonne textual intermediate language +`: + +.. productionlist:: + test_file : test_header `function_list` + test_header : test_commands (`isa_specs` | `settings`) + test_commands : test_command { test_command } + test_command : "test" test_name { option } "\n" + +The available test commands are described below. + +Many test comands only make sense in the context of a target instruction set +architecture. These tests require one or more ISA specifications in the test +header: + +.. productionlist:: + isa_specs : { [`settings`] isa_spec } + isa_spec : "isa" isa_name { `option` } "\n" + +The options given on the ``isa`` line modify the ISA-specific settings defined in +:file:`meta/isa/*/setings.py`. + +All types of tests allow shared Cretonne settings to be modified: + +.. productionlist:: + settings : { setting } + setting : "set" { option } "\n" + option : flag | setting "=" value + +The shared settings available for all target ISAs are defined in +:file:`meta/cretonne/settings.py`. + +The ``set`` lines apply settings cumulatively:: + + test legalizer + set opt_level=best + set is_64bit=1 + isa riscv + set is_64bit=0 + isa riscv supports_m=false + + function foo() {} + +This example will run the legalizer test twice. Both runs will have +``opt_level=best``, but they will have different ``is_64bit`` settings. The 32-bit +run will also have the RISC-V specific flag ``supports_m`` disabled. + +Filecheck +--------- + +Many of the test commands bescribed below use *filecheck* to verify their +output. Filecheck is a Rust implementation of the LLVM tool of the same name. +See the :file:`libfilecheck` documentation for details of its syntax. + +Comments in :file:`.cton` files are associated with the entity they follow. +This typically means and instruction or the whole function. Those tests that +use filecheck will extract comments associated with each function (or its +entities) and scan them for filecheck directives. The test output for each +function is then matched againts the filecheck directives for that function. + +Note that LLVM's file tests don't separate filecheck directives by their +associated function. It verifies the concatenated output against all filecheck +directives in the test file. LLVM's :command:`FileCheck` command has a +``CHECK-LABEL:`` directive to help separate the output from different functions. +Cretonne's tests don't need this. + +Filecheck variables +~~~~~~~~~~~~~~~~~~~ + +Cretonne's IL parser causes entities like values and EBBs to be renumbered. It +maintains a source mapping to resolve references in the text, but when a +function is written out as text as part of a test, all of the entities have the +new numbers. This can complicate the filecheck directives since they need to +refer to the new entity numbers, not the ones in the adjacent source text. + +To help with this, the parser's source-to-entity mapping is made available as +predefined filecheck variables. A value by the source name ``v10`` can be +referenced as the filecheck variable ``$v10``. The variable expands to the +renumbered entity name. + +`test cat` +---------- + +This is one of the simplest file tests, used for testing the conversion to and +from textual IL. The ``test cat`` command simply parses each function and +converts it back to text again. The text of each function is then matched +against the associated filecheck directives. + +Example:: + + function r1() -> i32, f32 { + ebb1: + v10 = iconst.i32 3 + v20 = f32const 0.0 + return v10, v20 + } + ; sameln: function r1() -> i32, f32 { + ; nextln: ebb0: + ; nextln: v0 = iconst.i32 3 + ; nextln: v1 = f32const 0.0 + ; nextln: return v0, v1 + ; nextln: } + +Notice that the values ``v10`` and ``v20`` in the source were renumbered to +``v0`` and ``v1`` respectively during parsing. The equivalent test using +filecheck variables would be:: + + function r1() -> i32, f32 { + ebb1: + v10 = iconst.i32 3 + v20 = f32const 0.0 + return v10, v20 + } + ; sameln: function r1() -> i32, f32 { + ; nextln: ebb0: + ; nextln: $v10 = iconst.i32 3 + ; nextln: $v20 = f32const 0.0 + ; nextln: return $v10, $v20 + ; nextln: } + +`test verifier` +--------------- + +Run each function through the IL verifier and check that it produces the +expected error messages. + +Expected error messages are indicated with an ``error:`` directive *on the +instruction that produces the verifier error*. Both the error message and +reported location of the error is verified:: + + test verifier + + function test(i32) { + ebb0(v0: i32): + jump ebb1 ; error: terminator + return + } + +This example test passed if the verifier fails with an error message containing +the sub-string ``"terminator"`` *and* the error is reported for the ``jump`` +instruction. + +If a function contains no ``error:`` annotations, the test passes if the +function verifies correctly. + +`test print-cfg` +---------------- + +Print the control flow graph of each function as a Graphviz graph, and run +filecheck over the result. See also the :command:`cton-util print-cfg` +command:: + + ; For testing cfg generation. This code is nonsense. + test print-cfg + test verifier + + function nonsense(i32, i32) -> f32 { + ; check: digraph nonsense { + ; regex: I=\binst\d+\b + ; check: label="{ebb0 | <$(BRZ=$I)>brz ebb2 | <$(JUMP=$I)>jump ebb1}"] + + ebb0(v1: i32, v2: i32): + brz v2, ebb2 ; unordered: ebb0:$BRZ -> ebb2 + v4 = iconst.i32 0 + jump ebb1(v4) ; unordered: ebb0:$JUMP -> ebb1 + + ebb1(v5: i32): + return v1 + + ebb2: + v100 = f32const 0.0 + return v100 + } + +`test domtree` +-------------- + +Compute the dominator tree of each function and validate it against the +``dominates:`` annotations:: + + test domtree + + function test(i32) { + ebb0(v0: i32): + jump ebb1 ; dominates: ebb1 + ebb1: + brz v0, ebb3 ; dominates: ebb3 + jump ebb2 ; dominates: ebb2 + ebb2: + jump ebb3 + ebb3: + return + } + +Every reachable extended basic block except for the entry block has an +*immediate dominator* which is a jump or branch instruction. This test passes +if the ``dominates:`` annotations on the immediate dominator instructions are +both correct and complete. + +`test legalizer` +---------------- + +Legalize each function for the specified target ISA and run the resulting +function through filecheck. This test command can be used to validate the +encodings selected for legal instructions as well as the instruction +transformations performed by the legalizer.