Support parsing the text format in wasmtime crate (#813)

* Support parsing the text format in `wasmtime` crate

This commit adds support to the `wasmtime::Module` type to parse the
text format. This is often quite convenient to support in testing or
tinkering with the runtime. Additionally the `wat` parser is pretty
lightweight and easy to add to builds, so it's relatively easy for us to
support as well!

The exact manner that this is now supported comes with a few updates to
the existing API:

* A new optional feature of the `wasmtime` crate, `wat`, has been added.
  This is enabled by default.
* The `Module::new` API now takes `impl AsRef<[u8]>` instead of just
  `&[u8]`, and when the `wat` feature is enabled it will attempt to
  interpret it either as a wasm binary or as the text format. Note that
  this check is quite cheap since you just check the first byte.
* A `Module::from_file` API was added as a convenience to parse a file
  from disk, allowing error messages for `*.wat` files on disk to be a
  bit nicer.
* APIs like `Module::new_unchecked` and `Module::validate` remain
  unchanged, they require the binary format to be called.

The intention here is to make this as convenient as possible for new
developers of the `wasmtime` crate. By changing the default behavior
though this has ramifications such as, for example, supporting the text
format implicitly through the C API now.

* Handle review comments

* Update more tests to avoid usage of `wat` crate

* Go back to unchecked for now in wasm_module_new

Looks like C# tests rely on this?

.github/workflows/main.yml

@@ -172,6 +172,12 @@ jobs:
     - run: cargo fetch --locked
     - run: cargo fetch --locked --manifest-path crates/test-programs/wasi-tests/Cargo.toml
 
+    # Build some various feature combinations
+    - run: cargo build --manifest-path crates/api/Cargo.toml --no-default-features
+    - run: cargo build --manifest-path crates/api/Cargo.toml --features wat
+    - run: cargo build --manifest-path crates/api/Cargo.toml --features lightbeam
+      if: matrix.rust == 'nightly'
+
     # Build and test all features except for lightbeam
     - run: cargo test --features test_programs --all --exclude lightbeam -- --nocapture
       env:

crates/api/Cargo.toml

@@ -21,6 +21,7 @@ cfg-if = "0.1.9"
 backtrace = "0.3.42"
 rustc-demangle = "0.1.16"
 lazy_static = "1.4"
+wat = { version = "1.0.7", optional = true }
 
 [target.'cfg(target_os = "windows")'.dependencies]
 winapi = "0.3.7"
@@ -37,6 +38,8 @@ wat = "1.0"
 maintenance = { status = "actively-developed" }
 
 [features]
+default = ['wat']
+
 # Enables experimental support for the lightbeam codegen backend, an alternative
 # to cranelift. Requires Nightly Rust currently, and this is not enabled by
 # default.

crates/api/examples/gcd.rs

@@ -36,9 +36,8 @@ const WAT: &str = r#"
 fn main() -> anyhow::Result<()> {
     // Load our WebAssembly (parsed WAT in our case), and then load it into a
     // `Module` which is attached to a `Store` cache.
-    let wasm = wat::parse_str(WAT)?;
     let store = Store::default();
-    let module = Module::new(&store, &wasm)?;
+    let module = Module::new(&store, WAT)?;
 
     // Find index of the `gcd` export.
     let gcd_index = module

crates/api/examples/hello.rs

@@ -21,21 +21,15 @@ fn main() -> Result<()> {
     println!("Initializing...");
     let store = Store::default();
 
-    // Next upload the `*.wasm` binary file, which in this case we're going to
+    // Compile the wasm binary into an in-memory instance of a `Module`.
-    // be parsing an inline text format into a binary.
-    println!("Loading binary...");
-    let binary = wat::parse_str(
-        r#"
-            (module
-              (func $hello (import "" "hello"))
-              (func (export "run") (call $hello))
-            )
-        "#,
-    )?;
-
-    // Compiler the `*.wasm` binary into an in-memory instance of a `Module`.
     println!("Compiling module...");
-    let module = Module::new(&store, &binary).context("> Error compiling module!")?;
+    let wat = r#"
+        (module
+          (func $hello (import "" "hello"))
+          (func (export "run") (call $hello))
+        )
+    "#;
+    let module = Module::new(&store, wat).context("> Error compiling module!")?;
 
     // Here we handle the imports of the module, which in this case is our
     // `HelloCallback` type and its associated implementation of `Callback.

crates/api/examples/memory.rs

@@ -66,27 +66,25 @@ fn main() -> Result<(), Error> {
 
     // Load binary.
     println!("Loading binary...");
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module
-            (module
+          (memory (export "memory") 2 3)
-              (memory (export "memory") 2 3)
 
-              (func (export "size") (result i32) (memory.size))
+          (func (export "size") (result i32) (memory.size))
-              (func (export "load") (param i32) (result i32)
+          (func (export "load") (param i32) (result i32)
-                (i32.load8_s (local.get 0))
+            (i32.load8_s (local.get 0))
-              )
+          )
-              (func (export "store") (param i32 i32)
+          (func (export "store") (param i32 i32)
-                (i32.store8 (local.get 0) (local.get 1))
+            (i32.store8 (local.get 0) (local.get 1))
-              )
+          )
 
-              (data (i32.const 0x1000) "\01\02\03\04")
+          (data (i32.const 0x1000) "\01\02\03\04")
-            )
+        )
-        "#,
+    "#;
-    )?;
 
     // Compile.
     println!("Compiling module...");
-    let module = Module::new(&store, &binary).context("> Error compiling module!")?;
+    let module = Module::new(&store, &wat).context("> Error compiling module!")?;
 
     // Instantiate.
     println!("Instantiating module...");

crates/api/examples/multi.rs

@@ -48,13 +48,9 @@ fn main() -> Result<()> {
     let engine = Engine::new(Config::new().wasm_multi_value(true));
     let store = Store::new(&engine);
 
-    // Load binary.
-    println!("Loading binary...");
-    let binary = wat::parse_str(WAT)?;
-
     // Compile.
     println!("Compiling module...");
-    let module = Module::new(&store, &binary).context("Error compiling module!")?;
+    let module = Module::new(&store, WAT).context("Error compiling module!")?;
 
     // Create external print functions.
     println!("Creating callback...");

crates/api/src/callable.rs

@@ -30,7 +30,7 @@ use wasmtime_runtime::Export;
 /// # fn main () -> Result<(), Box<dyn std::error::Error>> {
 /// // Simple module that imports our host function ("times_two") and re-exports
 /// // it as "run".
-/// let binary = wat::parse_str(r#"
+/// let wat = r#"
 ///    (module
 ///      (func $times_two (import "" "times_two") (param i32) (result i32))
 ///      (func
@@ -40,11 +40,11 @@ use wasmtime_runtime::Export;
 ///        (local.get 0)
 ///        (call $times_two))
 ///    )
-/// "#)?;
+/// "#;
 ///
 /// // Initialise environment and our module.
 /// let store = wasmtime::Store::default();
-/// let module = wasmtime::Module::new(&store, &binary)?;
+/// let module = wasmtime::Module::new(&store, wat)?;
 ///
 /// // Define the type of the function we're going to call.
 /// let times_two_type = wasmtime::FuncType::new(

crates/api/src/module.rs

@@ -7,6 +7,7 @@ use anyhow::{Error, Result};
 use lazy_static::lazy_static;
 use std::cell::Cell;
 use std::collections::HashMap;
+use std::path::Path;
 use std::rc::Rc;
 use std::sync::{Arc, RwLock};
 use wasmparser::{
@@ -107,15 +108,22 @@ lazy_static! {
 }
 
 impl Module {
-    /// Creates a new WebAssembly `Module` from the given in-memory `binary`
+    /// Creates a new WebAssembly `Module` from the given in-memory `bytes`.
-    /// data.
     ///
-    /// The `binary` data provided must be a [binary-encoded][binary]
+    /// The `bytes` provided must be in one of two formats:
-    /// WebAssembly module. This means that the data for the wasm module must be
+    ///
-    /// loaded in-memory if it's present elsewhere, for example on disk.
+    /// * It can be a [binary-encoded][binary] WebAssembly module. This
-    /// Additionally this requires that the entire binary is loaded into memory
+    ///   is always supported.
-    /// all at once, this API does not support streaming compilation of a
+    /// * It may also be a [text-encoded][text] instance of the WebAssembly
-    /// module.
+    ///   text format. This is only supported when the `wat` feature of this
+    ///   crate is enabled. If this is supplied then the text format will be
+    ///   parsed before validation. Note that the `wat` feature is enabled by
+    ///   default.
+    ///
+    /// The data for the wasm module must be loaded in-memory if it's present
+    /// elsewhere, for example on disk. This requires that the entire binary is
+    /// loaded into memory all at once, this API does not support streaming
+    /// compilation of a module.
     ///
     /// The WebAssembly binary will be decoded and validated. It will also be
     /// compiled according to the configuration of the provided `store` and
@@ -137,34 +145,69 @@ impl Module {
     ///   example too many locals)
     /// * The wasm binary may use features that are not enabled in the
     ///   configuration of `store`
+    /// * If the `wat` feature is enabled and the input is text, then it may be
+    ///   rejected if it fails to parse.
     ///
     /// The error returned should contain full information about why module
     /// creation failed if one is returned.
     ///
     /// [binary]: https://webassembly.github.io/spec/core/binary/index.html
-    pub fn new(store: &Store, binary: &[u8]) -> Result<Module> {
+    /// [text]: https://webassembly.github.io/spec/core/text/index.html
-        Module::validate(store, binary)?;
+    pub fn new(store: &Store, bytes: impl AsRef<[u8]>) -> Result<Module> {
-        unsafe { Module::new_unchecked(store, binary) }
+        #[cfg(feature = "wat")]
+        let bytes = wat::parse_bytes(bytes.as_ref())?;
+        Module::from_binary(store, bytes.as_ref())
     }
 
     /// Creates a new WebAssembly `Module` from the given in-memory `binary`
     /// data. The provided `name` will be used in traps/backtrace details.
     ///
     /// See [`Module::new`] for other details.
-    pub fn new_with_name(store: &Store, binary: &[u8], name: &str) -> Result<Module> {
+    pub fn new_with_name(store: &Store, bytes: impl AsRef<[u8]>, name: &str) -> Result<Module> {
-        let mut module = Module::new(store, binary)?;
+        let mut module = Module::new(store, bytes.as_ref())?;
         let inner = Rc::get_mut(&mut module.inner).unwrap();
         Arc::get_mut(&mut inner.names).unwrap().module_name = Some(name.to_string());
         Ok(module)
     }
 
+    /// Creates a new WebAssembly `Module` from the contents of the given
+    /// `file` on disk.
+    ///
+    /// This is a convenience function that will read the `file` provided and
+    /// pass the bytes to the [`Module::new`] function. For more information
+    /// see [`Module::new`]
+    pub fn from_file(store: &Store, file: impl AsRef<Path>) -> Result<Module> {
+        #[cfg(feature = "wat")]
+        let wasm = wat::parse_file(file)?;
+        #[cfg(not(feature = "wat"))]
+        let wasm = std::fs::read(file)?;
+        Module::new(store, &wasm)
+    }
+
+    /// Creates a new WebAssembly `Module` from the given in-memory `binary`
+    /// data.
+    ///
+    /// This is similar to [`Module::new`] except that it requires that the
+    /// `binary` input is a WebAssembly binary, the text format is not supported
+    /// by this function. It's generally recommended to use [`Module::new`],
+    /// but if it's required to not support the text format this function can be
+    /// used instead.
+    pub fn from_binary(store: &Store, binary: &[u8]) -> Result<Module> {
+        Module::validate(store, binary)?;
+        // Note that the call to `validate` here should be ok because we
+        // previously validated the binary, meaning we're guaranteed to pass a
+        // valid binary for `store`.
+        unsafe { Module::from_binary_unchecked(store, binary) }
+    }
+
     /// Creates a new WebAssembly `Module` from the given in-memory `binary`
     /// data, skipping validation and asserting that `binary` is a valid
     /// WebAssembly module.
     ///
     /// This function is the same as [`Module::new`] except that it skips the
-    /// call to [`Module::validate`]. This means that the WebAssembly binary is
+    /// call to [`Module::validate`] and it does not support the text format of
-    /// not validated for correctness and it is simply assumed as valid.
+    /// WebAssembly. The WebAssembly binary is not validated for
+    /// correctness and it is simply assumed as valid.
     ///
     /// For more information about creation of a module and the `store` argument
     /// see the documentation of [`Module::new`].
@@ -184,7 +227,7 @@ impl Module {
     /// While this assumes that the binary is valid it still needs to actually
     /// be somewhat valid for decoding purposes, and the basics of decoding can
     /// still fail.
-    pub unsafe fn new_unchecked(store: &Store, binary: &[u8]) -> Result<Module> {
+    pub unsafe fn from_binary_unchecked(store: &Store, binary: &[u8]) -> Result<Module> {
         let mut ret = Module::compile(store, binary)?;
         ret.read_imports_and_exports(binary)?;
         Ok(ret)
@@ -194,10 +237,11 @@ impl Module {
     /// configuration in `store`.
     ///
     /// This function will perform a speedy validation of the `binary` input
-    /// WebAssembly module (which is in [binary form][binary]) and return either
+    /// WebAssembly module (which is in [binary form][binary], the text format
-    /// `Ok` or `Err` depending on the results of validation. The `store`
+    /// is not accepted by this function) and return either `Ok` or `Err`
-    /// argument indicates configuration for WebAssembly features, for example,
+    /// depending on the results of validation. The `store` argument indicates
-    /// which are used to indicate what should be valid and what shouldn't be.
+    /// configuration for WebAssembly features, for example, which are used to
+    /// indicate what should be valid and what shouldn't be.
     ///
     /// Validation automatically happens as part of [`Module::new`], but is a
     /// requirement for [`Module::new_unchecked`] to be safe.

crates/api/tests/import-indexes.rs

@@ -38,8 +38,7 @@ fn same_import_names_still_distinct() -> anyhow::Result<()> {
     }
 
     let store = Store::default();
-    let wasm = wat::parse_str(WAT)?;
+    let module = Module::new(&store, WAT)?;
-    let module = Module::new(&store, &wasm)?;
 
     let imports = [
         Func::new(

crates/api/tests/import_calling_export.rs

@@ -32,8 +32,7 @@ fn test_import_calling_export() {
     }
 
     let store = Store::default();
-    let wasm = wat::parse_str(WAT).unwrap();
+    let module = Module::new(&store, WAT).expect("failed to create module");
-    let module = Module::new(&store, &wasm).expect("failed to create module");
 
     let callback = Rc::new(Callback {
         other: RefCell::new(None),

crates/api/tests/invoke_func_via_table.rs

@@ -5,17 +5,15 @@ use wasmtime::*;
 fn test_invoke_func_via_table() -> Result<()> {
     let store = Store::default();
 
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+      (module
-          (module
+        (func $f (result i64) (i64.const 42))
-            (func $f (result i64) (i64.const 42))
 
-            (table (export "table") 1 1 anyfunc)
+        (table (export "table") 1 1 anyfunc)
-            (elem (i32.const 0) $f)
+        (elem (i32.const 0) $f)
-          )
+      )
-        "#,
+    "#;
-    )?;
+    let module = Module::new(&store, wat).context("> Error compiling module!")?;
-    let module = Module::new(&store, &binary).context("> Error compiling module!")?;
     let instance = Instance::new(&module, &[]).context("> Error instantiating module!")?;
 
     let f = instance

crates/api/tests/name.rs

@@ -3,15 +3,13 @@ use wasmtime::*;
 #[test]
 fn test_module_no_name() -> anyhow::Result<()> {
     let store = Store::default();
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module
-            (module
+        (func (export "run") (nop))
-            (func (export "run") (nop))
+        )
-            )
+    "#;
-        "#,
-    )?;
 
-    let module = Module::new(&store, &binary)?;
+    let module = Module::new(&store, wat)?;
     assert_eq!(module.name(), None);
 
     Ok(())
@@ -20,18 +18,16 @@ fn test_module_no_name() -> anyhow::Result<()> {
 #[test]
 fn test_module_name() -> anyhow::Result<()> {
     let store = Store::default();
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module $from_name_section
-            (module $from_name_section
+        (func (export "run") (nop))
-            (func (export "run") (nop))
+        )
-            )
+    "#;
-        "#,
-    )?;
 
-    let module = Module::new(&store, &binary)?;
+    let module = Module::new(&store, wat)?;
     assert_eq!(module.name(), Some("from_name_section"));
 
-    let module = Module::new_with_name(&store, &binary, "override")?;
+    let module = Module::new_with_name(&store, wat, "override")?;
     assert_eq!(module.name(), Some("override"));
 
     Ok(())

crates/api/tests/traps.rs

@@ -13,16 +13,14 @@ fn test_trap_return() -> Result<()> {
     }
 
     let store = Store::default();
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module
-            (module
+        (func $hello (import "" "hello"))
-            (func $hello (import "" "hello"))
+        (func (export "run") (call $hello))
-            (func (export "run") (call $hello))
+        )
-            )
+    "#;
-        "#,
-    )?;
 
-    let module = Module::new(&store, &binary)?;
+    let module = Module::new(&store, wat)?;
     let hello_type = FuncType::new(Box::new([]), Box::new([]));
     let hello_func = Func::new(&store, hello_type, Rc::new(HelloCallback));
 
@@ -41,16 +39,14 @@ fn test_trap_return() -> Result<()> {
 #[test]
 fn test_trap_trace() -> Result<()> {
     let store = Store::default();
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module $hello_mod
-            (module $hello_mod
+            (func (export "run") (call $hello))
-                (func (export "run") (call $hello))
+            (func $hello (unreachable))
-                (func $hello (unreachable))
+        )
-            )
+    "#;
-        "#,
-    )?;
 
-    let module = Module::new(&store, &binary)?;
+    let module = Module::new(&store, wat)?;
     let instance = Instance::new(&module, &[])?;
     let run_func = instance.exports()[0]
         .func()
@@ -82,20 +78,18 @@ fn test_trap_trace_cb() -> Result<()> {
     }
 
     let store = Store::default();
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module $hello_mod
-            (module $hello_mod
+            (import "" "throw" (func $throw))
-                (import "" "throw" (func $throw))
+            (func (export "run") (call $hello))
-                (func (export "run") (call $hello))
+            (func $hello (call $throw))
-                (func $hello (call $throw))
+        )
-            )
+    "#;
-        "#,
-    )?;
 
     let fn_type = FuncType::new(Box::new([]), Box::new([]));
     let fn_func = Func::new(&store, fn_type, Rc::new(ThrowCallback));
 
-    let module = Module::new(&store, &binary)?;
+    let module = Module::new(&store, wat)?;
     let instance = Instance::new(&module, &[fn_func.into()])?;
     let run_func = instance.exports()[0]
         .func()
@@ -117,15 +111,13 @@ fn test_trap_trace_cb() -> Result<()> {
 #[test]
 fn test_trap_stack_overflow() -> Result<()> {
     let store = Store::default();
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module $rec_mod
-            (module $rec_mod
+            (func $run (export "run") (call $run))
-                (func $run (export "run") (call $run))
+        )
-            )
+    "#;
-        "#,
-    )?;
 
-    let module = Module::new(&store, &binary)?;
+    let module = Module::new(&store, wat)?;
     let instance = Instance::new(&module, &[])?;
     let run_func = instance.exports()[0]
         .func()
@@ -148,18 +140,16 @@ fn test_trap_stack_overflow() -> Result<()> {
 #[test]
 fn trap_display_pretty() -> Result<()> {
     let store = Store::default();
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module $m
-            (module $m
+            (func $die unreachable)
-                (func $die unreachable)
+            (func call $die)
-                (func call $die)
+            (func $foo call 1)
-                (func $foo call 1)
+            (func (export "bar") call $foo)
-                (func (export "bar") call $foo)
+        )
-            )
+    "#;
-        "#,
-    )?;
 
-    let module = Module::new(&store, &binary)?;
+    let module = Module::new(&store, wat)?;
     let instance = Instance::new(&module, &[])?;
     let run_func = instance.exports()[0]
         .func()
@@ -183,31 +173,27 @@ wasm backtrace:
 #[test]
 fn trap_display_multi_module() -> Result<()> {
     let store = Store::default();
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module $a
-            (module $a
+            (func $die unreachable)
-                (func $die unreachable)
+            (func call $die)
-                (func call $die)
+            (func $foo call 1)
-                (func $foo call 1)
+            (func (export "bar") call $foo)
-                (func (export "bar") call $foo)
+        )
-            )
+    "#;
-        "#,
-    )?;
 
-    let module = Module::new(&store, &binary)?;
+    let module = Module::new(&store, wat)?;
     let instance = Instance::new(&module, &[])?;
     let bar = instance.exports()[0].clone();
 
-    let binary = wat::parse_str(
+    let wat = r#"
-        r#"
+        (module $b
-            (module $b
+            (import "" "" (func $bar))
-                (import "" "" (func $bar))
+            (func $middle call $bar)
-                (func $middle call $bar)
+            (func (export "bar2") call $middle)
-                (func (export "bar2") call $middle)
+        )
-            )
+    "#;
-        "#,
+    let module = Module::new(&store, wat)?;
-    )?;
-    let module = Module::new(&store, &binary)?;
     let instance = Instance::new(&module, &[bar])?;
     let bar2 = instance.exports()[0]
         .func()

crates/c-api/src/lib.rs

@@ -787,7 +787,10 @@ pub unsafe extern "C" fn wasm_module_new(
 ) -> *mut wasm_module_t {
     let binary = (*binary).as_slice();
     let store = &(*store).store.borrow();
-    let module = Module::new_unchecked(store, binary).expect("module");
+    let module = match Module::from_binary_unchecked(store, binary) {
+        Ok(module) => module,
+        Err(_) => return ptr::null_mut(),
+    };
     let imports = module
         .imports()
         .iter()

tests/custom_signal_handler.rs

@@ -107,8 +107,7 @@ mod tests {
     fn test_custom_signal_handler_single_instance() -> Result<()> {
         let engine = Engine::new(&Config::default());
         let store = Store::new(&engine);
-        let data = wat::parse_str(WAT1)?;
+        let module = Module::new(&store, WAT1)?;
-        let module = Module::new(&store, &data)?;
         let instance = Instance::new(&module, &[])?;
 
         let (base, length) = set_up_memory(&instance);
@@ -166,8 +165,7 @@ mod tests {
     fn test_custom_signal_handler_multiple_instances() -> Result<()> {
         let engine = Engine::new(&Config::default());
         let store = Store::new(&engine);
-        let data = wat::parse_str(WAT1)?;
+        let module = Module::new(&store, WAT1)?;
-        let module = Module::new(&store, &data)?;
 
         // Set up multiple instances
 
@@ -261,8 +259,7 @@ mod tests {
         let store = Store::new(&engine);
 
         // instance1 which defines 'read'
-        let data1 = wat::parse_str(WAT1)?;
+        let module1 = Module::new(&store, WAT1)?;
-        let module1 = Module::new(&store, &data1)?;
         let instance1 = Instance::new(&module1, &[])?;
         let (base1, length1) = set_up_memory(&instance1);
         unsafe {
@@ -277,8 +274,7 @@ mod tests {
         let instance1_read = instance1_exports[0].clone();
 
         // instance2 wich calls 'instance1.read'
-        let data2 = wat::parse_str(WAT2)?;
+        let module2 = Module::new(&store, WAT2)?;
-        let module2 = Module::new(&store, &data2)?;
         let instance2 = Instance::new(&module2, &[instance1_read])?;
         // since 'instance2.run' calls 'instance1.read' we need to set up the signal handler to handle
         // SIGSEGV originating from within the memory of instance1