Try to reduce CI times with a Rust *.wat parser (#1332)

This commit moves the cranelift tests and tools from the `wabt` crate on
crates.io (which compiles the wabt C++ codebase) to the `wat` crate on
crates.io which is a Rust parser for the `*.wat` format. This was
motivated by me noticing that release builds on Windows are ~5 minutes
longer than Linux builds, and local timing graphs showed that `wabt-sys`
was by far the longest build step in the build process.

This commit changes the `clif-util` binary where the `--enable-simd`
flag is no longer respected with the text format as input, since the
`wat` crate has no feature gating. This was already sort of not
respected, though, since `--enable-simd` wasn't consulted for binary
inputs which `clif-util` supports as well. If this isn't ok though then
it should be ok to close this PR!

cranelift/Cargo.toml

@@ -38,7 +38,7 @@ clap = "2.32.0"
 serde = "1.0.8"
 term = "0.6.1"
 capstone = { version = "0.6.0", optional = true }
-wabt = { version = "0.9.1", optional = true }
+wat = { version = "1.0.7", optional = true }
 target-lexicon = "0.10"
 pretty_env_logger = "0.3.0"
 file-per-thread-logger = "0.1.2"
@@ -48,7 +48,7 @@ walkdir = "2.2"
 [features]
 default = ["disas", "wasm", "cranelift-codegen/all-arch", "basic-blocks"]
 disas = ["capstone"]
-wasm = ["wabt", "cranelift-wasm"]
+wasm = ["wat", "cranelift-wasm"]
 basic-blocks = ["cranelift-codegen/basic-blocks", "cranelift-frontend/basic-blocks",
 "cranelift-wasm/basic-blocks", "cranelift-filetests/basic-blocks"]
 

cranelift/src/clif-util.rs

@@ -107,24 +107,6 @@ fn add_debug_flag<'a>() -> clap::Arg<'a, 'a> {
         .help("Enable debug output on stderr/stdout")
 }
 
-fn add_enable_simd_flag<'a>() -> clap::Arg<'a, 'a> {
-    Arg::with_name("enable-simd")
-        .long("enable-simd")
-        .help("Enable WASM's SIMD operations")
-}
-
-fn add_enable_multi_value<'a>() -> clap::Arg<'a, 'a> {
-    Arg::with_name("enable-multi-value")
-        .long("enable-multi-value")
-        .help("Enable WASM's multi-value support")
-}
-
-fn add_enable_reference_types_flag<'a>() -> clap::Arg<'a, 'a> {
-    Arg::with_name("enable-reference-types")
-        .long("enable-reference-types")
-        .help("Enable WASM's reference types operations")
-}
-
 fn add_just_decode_flag<'a>() -> clap::Arg<'a, 'a> {
     Arg::with_name("just-decode")
         .short("t")
@@ -167,9 +149,6 @@ fn add_wasm_or_compile<'a>(cmd: &str) -> clap::App<'a, 'a> {
         .arg(add_target_flag())
         .arg(add_input_file_arg())
         .arg(add_debug_flag())
-        .arg(add_enable_simd_flag())
-        .arg(add_enable_multi_value())
-        .arg(add_enable_reference_types_flag())
         .arg(add_just_decode_flag())
         .arg(add_check_translation_flag())
 }
@@ -321,9 +300,6 @@ fn main() {
                     rest_cmd.is_present("print-size"),
                     rest_cmd.is_present("time-passes"),
                     rest_cmd.is_present("value-ranges"),
-                    rest_cmd.is_present("enable-simd"),
-                    rest_cmd.is_present("enable-multi-value"),
-                    rest_cmd.is_present("enable-reference-types"),
                 )
             };
 

cranelift/src/utils.rs

@@ -24,21 +24,6 @@ pub fn read_to_string<P: AsRef<Path>>(path: P) -> io::Result<String> {
     Ok(buffer)
 }
 
-/// Read an entire file into a vector of bytes.
-#[cfg(feature = "wasm")]
-pub fn read_to_end<P: AsRef<Path>>(path: P) -> io::Result<Vec<u8>> {
-    let mut buffer = Vec::new();
-    if path.as_ref() == Path::new("-") {
-        let stdin = io::stdin();
-        let mut stdin = stdin.lock();
-        stdin.read_to_end(&mut buffer)?;
-    } else {
-        let mut file = File::open(path)?;
-        file.read_to_end(&mut buffer)?;
-    }
-    Ok(buffer)
-}
-
 /// Like `FlagsOrIsa`, but holds ownership.
 pub enum OwnedFlagsOrIsa {
     Flags(settings::Flags),

cranelift/src/wasm.rs

@@ -8,7 +8,7 @@
 )]
 
 use crate::disasm::{print_all, PrintRelocs, PrintStackmaps, PrintTraps};
-use crate::utils::{parse_sets_and_triple, read_to_end};
+use crate::utils::parse_sets_and_triple;
 use cranelift_codegen::ir::DisplayFunctionAnnotations;
 use cranelift_codegen::print_errors::{pretty_error, pretty_verifier_error};
 use cranelift_codegen::settings::FlagsOrIsa;
@@ -16,10 +16,10 @@ use cranelift_codegen::timing;
 use cranelift_codegen::Context;
 use cranelift_entity::EntityRef;
 use cranelift_wasm::{translate_module, DummyEnvironment, FuncIndex, ReturnMode};
+use std::io::Read;
 use std::path::Path;
 use std::path::PathBuf;
 use term;
-use wabt::{wat2wasm_with_features, Features};
 
 macro_rules! vprintln {
     ($x: expr, $($tts:tt)*) => {
@@ -49,9 +49,6 @@ pub fn run(
     flag_print_size: bool,
     flag_report_times: bool,
     flag_calc_value_ranges: bool,
-    flag_enable_simd: bool,
-    flag_enable_multi_value: bool,
-    flag_enable_reference_types: bool,
 ) -> Result<(), String> {
     let parsed = parse_sets_and_triple(flag_set, flag_triple)?;
 
@@ -67,9 +64,6 @@ pub fn run(
             flag_print_disasm,
             flag_report_times,
             flag_calc_value_ranges,
-            flag_enable_simd,
-            flag_enable_multi_value,
-            flag_enable_reference_types,
             &path.to_path_buf(),
             &name,
             parsed.as_fisa(),
@@ -87,9 +81,6 @@ fn handle_module(
     flag_print_disasm: bool,
     flag_report_times: bool,
     flag_calc_value_ranges: bool,
-    flag_enable_simd: bool,
-    flag_enable_multi_value: bool,
-    flag_enable_reference_types: bool,
     path: &PathBuf,
     name: &str,
     fisa: FlagsOrIsa,
@@ -103,25 +94,19 @@ fn handle_module(
     vprint!(flag_verbose, "Translating... ");
     let _ = terminal.reset();
 
-    let mut module_binary = read_to_end(path.clone()).map_err(|err| err.to_string())?;
+    let module_binary = if path.to_str() == Some("-") {
-
+        let stdin = std::io::stdin();
-    if !module_binary.starts_with(&[b'\0', b'a', b's', b'm']) {
+        let mut buf = Vec::new();
-        let mut features = Features::new();
+        stdin
-        if flag_enable_simd {
+            .lock()
-            features.enable_simd();
+            .read_to_end(&mut buf)
-        }
+            .map_err(|e| e.to_string())?;
-        if flag_enable_multi_value {
+        wat::parse_bytes(&buf)
-            features.enable_multi_value();
+            .map_err(|err| format!("{:?}", err))?
-        }
+            .into()
-        if flag_enable_reference_types {
+    } else {
-            features.enable_reference_types();
+        wat::parse_file(path).map_err(|err| format!("{:?}", err))?
-        }
+    };
-
-        module_binary = match wat2wasm_with_features(&module_binary, features) {
-            Ok(data) => data,
-            Err(e) => return Err(e.to_string()),
-        };
-    }
 
     let isa = match fisa.isa {
         Some(isa) => isa,

cranelift/wasm/Cargo.toml

@@ -21,7 +21,7 @@ serde = { version = "1.0.94", features = ["derive"], optional = true }
 thiserror = "1.0.4"
 
 [dev-dependencies]
-wabt = "0.9.1"
+wat = "1.0.7"
 target-lexicon = "0.10"
 
 [features]

cranelift/wasm/tests/wasm_testsuite.rs

@@ -10,7 +10,6 @@ use std::io::prelude::*;
 use std::path::Path;
 use std::str::FromStr;
 use target_lexicon::triple;
-use wabt::{wat2wasm_with_features, Features, Wat2Wasm};
 
 #[test]
 fn testsuite() {
@@ -47,15 +46,14 @@ fn use_fallthrough_return() {
 
 #[test]
 fn use_name_section() {
-    let wat = r#"
+    let data = wat::parse_str(
+        r#"
         (module $module_name
             (func $func_name (local $loc_name i32)
             )
-        )"#;
+        )"#,
-    let data = Wat2Wasm::new()
+    )
-        .write_debug_names(true)
+    .unwrap();
-        .convert(wat)
-        .unwrap_or_else(|e| panic!("error converting wat to wasm: {:?}", e));
 
     let flags = Flags::new(settings::builder());
     let triple = triple!("riscv64");
@@ -79,23 +77,13 @@ fn read_file(path: &Path) -> io::Result<Vec<u8>> {
 }
 
 fn read_module(path: &Path) -> Vec<u8> {
-    let mut features = Features::new();
-    features.enable_all();
     match path.extension() {
         None => {
             panic!("the file extension is not wasm or wat");
         }
         Some(ext) => match ext.to_str() {
             Some("wasm") => read_file(path).expect("error reading wasm file"),
-            Some("wat") => {
+            Some("wat") => wat::parse_file(path).expect("failed to parse wat"),
-                let wat = read_file(path).expect("error reading wat file");
-                match wat2wasm_with_features(&wat, features) {
-                    Ok(wasm) => wasm,
-                    Err(e) => {
-                        panic!("error converting wat to wasm: {:?}", e);
-                    }
-                }
-            }
             None | Some(&_) => panic!("the file extension for {:?} is not wasm or wat", path),
         },
     }