Merge pull request #1910 from bytecodealliance/pch/separate_wasmtime_wiggle_crate

Factor Wiggle's wasmtime integration into a standalone crate

crates/wiggle/wasmtime/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "wasmtime-wiggle"
+version = "0.18.0"
+authors = ["Pat Hickey <phickey@fastly.com>", "Jakub Konka <kubkonk@jakubkonka.com>", "Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+license = "Apache-2.0 WITH LLVM-exception"
+description = "Integrate Wiggle code generator with Wasmtime"
+categories = ["wasm"]
+keywords = ["webassembly", "wasm"]
+repository = "https://github.com/bytecodealliance/wasmtime"
+include = ["src/**/*", "LICENSE"]
+
+[dependencies]
+wasmtime = { path = "../../wasmtime", version = "0.18.0" }
+wasmtime-wiggle-macro = { path = "./macro", version = "0.18.0" }
+witx = { path = "../../wasi-common/WASI/tools/witx", version = "0.8.5", optional = true }
+wiggle = { path = "..", version = "0.18.0" }
+
+[badges]
+maintenance = { status = "actively-developed" }
+
+[features]
+# The wiggle proc-macro emits some code (inside `pub mod metadata`) guarded
+# by the `wiggle_metadata` feature flag. We use this feature flag so that
+# users of wiggle are not forced to take a direct dependency on the `witx`
+# crate unless they want it.
+wiggle_metadata = ['witx', "wiggle/wiggle_metadata"]
+
+# The `tracing` crate can use the `log` ecosystem of backends with this
+# non-default feature. We don't need to provide this by default, but its
+# useful for users that don't want to use `tracing-subscriber` to get
+# the logs out of wiggle-generated libraries.
+tracing_log = [ "wiggle/tracing_log" ]
+
+default = ["wiggle_metadata" ]

crates/wiggle/wasmtime/macro/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "wasmtime-wiggle-macro"
+version = "0.18.0"
+authors = ["Pat Hickey <phickey@fastly.com>", "Jakub Konka <kubkonk@jakubkonka.com>", "Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+license = "Apache-2.0 WITH LLVM-exception"
+description = "Macro for integrating Wiggle code generator with Wasmtime"
+categories = ["wasm"]
+keywords = ["webassembly", "wasm"]
+repository = "https://github.com/bytecodealliance/wasmtime"
+include = ["src/**/*", "LICENSE"]
+
+[lib]
+proc-macro = true
+test = false
+
+[dependencies]
+witx = { path = "../../../wasi-common/WASI/tools/witx", version = "0.8.5" }
+wiggle-generate = { path = "../../generate", version = "0.18.0" }
+quote = "1.0"
+syn = { version = "1.0", features = ["full"] }
+proc-macro2 = "1.0"
+
+[badges]
+maintenance = { status = "actively-developed" }
+

crates/wiggle/wasmtime/macro/src/config.rs

@@ -0,0 +1,319 @@
+use {
+    proc_macro2::{Span, TokenStream},
+    std::collections::HashMap,
+    syn::{
+        braced,
+        parse::{Parse, ParseStream},
+        punctuated::Punctuated,
+        Error, Ident, Path, Result, Token,
+    },
+    wiggle_generate::config::{CtxConf, WitxConf},
+};
+
+#[derive(Debug, Clone)]
+pub struct Config {
+    pub target: TargetConf,
+    pub witx: WitxConf,
+    pub ctx: CtxConf,
+    pub modules: ModulesConf,
+    pub missing_memory: MissingMemoryConf,
+}
+
+#[derive(Debug, Clone)]
+pub enum ConfigField {
+    Target(TargetConf),
+    Witx(WitxConf),
+    Ctx(CtxConf),
+    Modules(ModulesConf),
+    MissingMemory(MissingMemoryConf),
+}
+
+mod kw {
+    syn::custom_keyword!(target);
+    syn::custom_keyword!(witx);
+    syn::custom_keyword!(witx_literal);
+    syn::custom_keyword!(ctx);
+    syn::custom_keyword!(modules);
+    syn::custom_keyword!(name);
+    syn::custom_keyword!(docs);
+    syn::custom_keyword!(missing_memory);
+    syn::custom_keyword!(function_override);
+}
+
+impl Parse for ConfigField {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let lookahead = input.lookahead1();
+        if lookahead.peek(kw::target) {
+            input.parse::<kw::target>()?;
+            input.parse::<Token![:]>()?;
+            Ok(ConfigField::Target(input.parse()?))
+        } else if lookahead.peek(kw::witx) {
+            input.parse::<kw::witx>()?;
+            input.parse::<Token![:]>()?;
+            Ok(ConfigField::Witx(WitxConf::Paths(input.parse()?)))
+        } else if lookahead.peek(kw::witx_literal) {
+            input.parse::<kw::witx_literal>()?;
+            input.parse::<Token![:]>()?;
+            Ok(ConfigField::Witx(WitxConf::Literal(input.parse()?)))
+        } else if lookahead.peek(kw::ctx) {
+            input.parse::<kw::ctx>()?;
+            input.parse::<Token![:]>()?;
+            Ok(ConfigField::Ctx(input.parse()?))
+        } else if lookahead.peek(kw::modules) {
+            input.parse::<kw::modules>()?;
+            input.parse::<Token![:]>()?;
+            Ok(ConfigField::Modules(input.parse()?))
+        } else if lookahead.peek(kw::missing_memory) {
+            input.parse::<kw::missing_memory>()?;
+            input.parse::<Token![:]>()?;
+            Ok(ConfigField::MissingMemory(input.parse()?))
+        } else {
+            Err(lookahead.error())
+        }
+    }
+}
+
+impl Config {
+    pub fn build(fields: impl Iterator<Item = ConfigField>, err_loc: Span) -> Result<Self> {
+        let mut target = None;
+        let mut witx = None;
+        let mut ctx = None;
+        let mut modules = None;
+        let mut missing_memory = None;
+        for f in fields {
+            match f {
+                ConfigField::Target(c) => {
+                    if target.is_some() {
+                        return Err(Error::new(err_loc, "duplicate `target` field"));
+                    }
+                    target = Some(c);
+                }
+                ConfigField::Witx(c) => {
+                    if witx.is_some() {
+                        return Err(Error::new(err_loc, "duplicate `witx` field"));
+                    }
+                    witx = Some(c);
+                }
+                ConfigField::Ctx(c) => {
+                    if ctx.is_some() {
+                        return Err(Error::new(err_loc, "duplicate `ctx` field"));
+                    }
+                    ctx = Some(c);
+                }
+                ConfigField::Modules(c) => {
+                    if modules.is_some() {
+                        return Err(Error::new(err_loc, "duplicate `modules` field"));
+                    }
+                    modules = Some(c);
+                }
+                ConfigField::MissingMemory(c) => {
+                    if missing_memory.is_some() {
+                        return Err(Error::new(err_loc, "duplicate `missing_memory` field"));
+                    }
+                    missing_memory = Some(c);
+                }
+            }
+        }
+        Ok(Config {
+            target: target.ok_or_else(|| Error::new(err_loc, "`target` field required"))?,
+            witx: witx.ok_or_else(|| Error::new(err_loc, "`witx` field required"))?,
+            ctx: ctx.ok_or_else(|| Error::new(err_loc, "`ctx` field required"))?,
+            modules: modules.ok_or_else(|| Error::new(err_loc, "`modules` field required"))?,
+            missing_memory: missing_memory
+                .ok_or_else(|| Error::new(err_loc, "`missing_memory` field required"))?,
+        })
+    }
+
+    /// Load the `witx` document for the configuration.
+    ///
+    /// # Panics
+    ///
+    /// This method will panic if the paths given in the `witx` field were not valid documents.
+    pub fn load_document(&self) -> witx::Document {
+        self.witx.load_document()
+    }
+}
+
+impl Parse for Config {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let contents;
+        let _lbrace = braced!(contents in input);
+        let fields: Punctuated<ConfigField, Token![,]> =
+            contents.parse_terminated(ConfigField::parse)?;
+        Ok(Config::build(fields.into_iter(), input.span())?)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TargetConf {
+    pub path: Path,
+}
+
+impl Parse for TargetConf {
+    fn parse(input: ParseStream) -> Result<Self> {
+        Ok(TargetConf {
+            path: input.parse()?,
+        })
+    }
+}
+
+enum ModuleConfField {
+    Name(Ident),
+    Docs(String),
+    FunctionOverride(FunctionOverrideConf),
+}
+
+impl Parse for ModuleConfField {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let lookahead = input.lookahead1();
+        if lookahead.peek(kw::name) {
+            input.parse::<kw::name>()?;
+            input.parse::<Token![:]>()?;
+            Ok(ModuleConfField::Name(input.parse()?))
+        } else if lookahead.peek(kw::docs) {
+            input.parse::<kw::docs>()?;
+            input.parse::<Token![:]>()?;
+            let docs: syn::LitStr = input.parse()?;
+            Ok(ModuleConfField::Docs(docs.value()))
+        } else if lookahead.peek(kw::function_override) {
+            input.parse::<kw::function_override>()?;
+            input.parse::<Token![:]>()?;
+            Ok(ModuleConfField::FunctionOverride(input.parse()?))
+        } else {
+            Err(lookahead.error())
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ModuleConf {
+    pub name: Ident,
+    pub docs: Option<String>,
+    pub function_override: FunctionOverrideConf,
+}
+
+impl ModuleConf {
+    fn build(fields: impl Iterator<Item = ModuleConfField>, err_loc: Span) -> Result<Self> {
+        let mut name = None;
+        let mut docs = None;
+        let mut function_override = None;
+        for f in fields {
+            match f {
+                ModuleConfField::Name(c) => {
+                    if name.is_some() {
+                        return Err(Error::new(err_loc, "duplicate `name` field"));
+                    }
+                    name = Some(c);
+                }
+                ModuleConfField::Docs(c) => {
+                    if docs.is_some() {
+                        return Err(Error::new(err_loc, "duplicate `docs` field"));
+                    }
+                    docs = Some(c);
+                }
+                ModuleConfField::FunctionOverride(c) => {
+                    if function_override.is_some() {
+                        return Err(Error::new(err_loc, "duplicate `function_override` field"));
+                    }
+                    function_override = Some(c);
+                }
+            }
+        }
+        Ok(ModuleConf {
+            name: name.ok_or_else(|| Error::new(err_loc, "`name` field required"))?,
+            docs,
+            function_override: function_override.unwrap_or_default(),
+        })
+    }
+}
+
+impl Parse for ModuleConf {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let contents;
+        let _lbrace = braced!(contents in input);
+        let fields: Punctuated<ModuleConfField, Token![,]> =
+            contents.parse_terminated(ModuleConfField::parse)?;
+        Ok(ModuleConf::build(fields.into_iter(), input.span())?)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ModulesConf {
+    pub mods: HashMap<String, ModuleConf>,
+}
+
+impl ModulesConf {
+    pub fn iter(&self) -> impl Iterator<Item = (&String, &ModuleConf)> {
+        self.mods.iter()
+    }
+}
+
+impl Parse for ModulesConf {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let contents;
+        let _lbrace = braced!(contents in input);
+        let fields: Punctuated<(String, ModuleConf), Token![,]> =
+            contents.parse_terminated(|i| {
+                let name = i.parse::<Ident>()?.to_string();
+                i.parse::<Token![=>]>()?;
+                let val = i.parse()?;
+                Ok((name, val))
+            })?;
+        Ok(ModulesConf {
+            mods: fields.into_iter().collect(),
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct MissingMemoryConf {
+    pub err: TokenStream,
+}
+impl Parse for MissingMemoryConf {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let contents;
+        let _lbrace = braced!(contents in input);
+        Ok(MissingMemoryConf {
+            err: contents.parse()?,
+        })
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct FunctionOverrideConf {
+    pub funcs: Vec<FunctionOverrideField>,
+}
+impl FunctionOverrideConf {
+    pub fn find(&self, name: &str) -> Option<&Ident> {
+        self.funcs
+            .iter()
+            .find(|f| f.name == name)
+            .map(|f| &f.replacement)
+    }
+}
+
+impl Parse for FunctionOverrideConf {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let contents;
+        let _lbrace = braced!(contents in input);
+        let fields: Punctuated<FunctionOverrideField, Token![,]> =
+            contents.parse_terminated(FunctionOverrideField::parse)?;
+        Ok(FunctionOverrideConf {
+            funcs: fields.into_iter().collect(),
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct FunctionOverrideField {
+    pub name: String,
+    pub replacement: Ident,
+}
+impl Parse for FunctionOverrideField {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let name = input.parse::<Ident>()?.to_string();
+        input.parse::<Token![=>]>()?;
+        let replacement = input.parse::<Ident>()?;
+        Ok(FunctionOverrideField { name, replacement })
+    }
+}

crates/wiggle/wasmtime/macro/src/lib.rs

@@ -0,0 +1,231 @@
+use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
+use quote::quote;
+use syn::parse_macro_input;
+use wiggle_generate::Names;
+
+mod config;
+
+use config::{MissingMemoryConf, ModuleConf, TargetConf};
+
+/// Define the structs required to integrate a Wiggle implementation with Wasmtime.
+///
+/// ## Arguments
+///
+/// Arguments are provided using struct syntax e.g. `{ arg_name: value }`.
+///
+/// * `target`: The path of the module where the Wiggle implementation is defined.
+/// * `witx` or `witx_literal`: the .witx document where the interface is defined.
+///   `witx` takes a list of filesystem paths, e.g. `["/path/to/file1.witx",
+///   "./path/to_file2.witx"]`. Relative paths are relative to the root of the crate
+///   where the macro is invoked. `witx_literal` takes a string of the witx document, e.g.
+///   `"(typename $foo u8)"`.
+/// * `ctx`: The context struct used for the Wiggle implementation. This must be the same
+///   type as the [`wasmtime_wiggle::from_witx`] macro at `target` was invoked with. However, it
+///   must be imported to the current scope so that it is a bare identifier e.g. `CtxType`, not
+///   `path::to::CtxType`.
+/// * `modules`: Describes how any modules in the witx document will be implemented as Wasmtime
+///    instances. `modules` takes a map from the witx module name to a configuration struct, e.g.
+///    `foo => { name: Foo }, bar => { name: Bar }` will generate integrations for the modules
+///    named `foo` and `bar` in the witx document, as `pub struct Foo` and `pub struct Bar`
+///    respectively.
+///    The module configuration uses struct syntax with the following fields:
+///      * `name`: required, gives the name of the struct which encapsulates the instance for
+///         Wasmtime.
+///      * `docs`: optional, a doc string that will be used for the definition of the struct.
+///      * `function_override`: A map of witx function names to Rust function symbols for
+///         functions that should not call the Wiggle-generated functions, but instead use
+///         a separate implementation. This is typically used for functions that need to interact
+///         with Wasmtime in a manner that Wiggle does not permit, e.g. wasi's `proc_exit` function
+///         needs to return a Trap directly to the runtime.
+///    Example:
+///    `modules: { some_module => { name: SomeTypeName, docs: "Doc string for definition of
+///     SomeTypeName here", function_override: { foo => my_own_foo } }`.
+/// * `missing_memory`: Describes the error value to return in case the calling module does not
+///   export a Memory as `"memory"`. This value is given in braces, e.g. `missing_memory: {
+///   wasi_common::wasi::Errno::Inval }`.
+///
+#[proc_macro]
+pub fn wasmtime_integration(args: TokenStream) -> TokenStream {
+    let mut config = parse_macro_input!(args as config::Config);
+    config.witx.make_paths_relative_to(
+        std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR env var"),
+    );
+    let doc = config.load_document();
+    let names = Names::new(&config.ctx.name, quote!(wasmtime_wiggle));
+
+    let modules = config.modules.iter().map(|(name, module_conf)| {
+        let module = doc
+            .module(&witx::Id::new(name))
+            .unwrap_or_else(|| panic!("witx document did not contain module named '{}'", name));
+        generate_module(
+            &module,
+            &module_conf,
+            &names,
+            &config.target,
+            &config.missing_memory,
+        )
+    });
+    quote!( #(#modules)* ).into()
+}
+
+fn generate_module(
+    module: &witx::Module,
+    module_conf: &ModuleConf,
+    names: &Names,
+    target_conf: &TargetConf,
+    missing_mem_conf: &MissingMemoryConf,
+) -> TokenStream2 {
+    let fields = module.funcs().map(|f| {
+        let name_ident = names.func(&f.name);
+        quote! { pub #name_ident: wasmtime::Func }
+    });
+    let get_exports = module.funcs().map(|f| {
+        let func_name = f.name.as_str();
+        let name_ident = names.func(&f.name);
+        quote! { #func_name => Some(&self.#name_ident) }
+    });
+    let ctor_fields = module.funcs().map(|f| names.func(&f.name));
+
+    let module_name = module.name.as_str();
+
+    let linker_add = module.funcs().map(|f| {
+        let func_name = f.name.as_str();
+        let name_ident = names.func(&f.name);
+        quote! {
+            linker.define(#module_name, #func_name, self.#name_ident.clone())?;
+        }
+    });
+
+    let target_path = &target_conf.path;
+    let module_id = names.module(&module.name);
+    let target_module = quote! { #target_path::#module_id };
+
+    let ctor_externs = module.funcs().map(|f| {
+        if let Some(func_override) = module_conf.function_override.find(&f.name.as_str()) {
+            let name_ident = names.func(&f.name);
+            quote! { let #name_ident = wasmtime::Func::wrap(store, #func_override); }
+        } else {
+            generate_func(&f, names, missing_mem_conf, &target_module)
+        }
+    });
+
+    let type_name = module_conf.name.clone();
+    let type_docs = module_conf
+        .docs
+        .as_ref()
+        .map(|docs| quote!( #[doc = #docs] ))
+        .unwrap_or_default();
+    let constructor_docs = format!(
+        "Creates a new [`{}`] instance.
+
+External values are allocated into the `store` provided and
+configuration of the wasi instance itself should be all
+contained in the `cx` parameter.",
+        module_conf.name.to_string()
+    );
+
+    let ctx_type = names.ctx_type();
+
+    quote! {
+        #type_docs
+        pub struct #type_name {
+            #(#fields,)*
+        }
+
+        impl #type_name {
+            #[doc = #constructor_docs]
+            pub fn new(store: &wasmtime::Store, cx: #ctx_type) -> Self {
+                let cx = std::rc::Rc::new(std::cell::RefCell::new(cx));
+                #(#ctor_externs)*
+
+                Self {
+                    #(#ctor_fields,)*
+                }
+            }
+
+            /// Looks up a field called `name` in this structure, returning it
+            /// if found.
+            ///
+            /// This is often useful when instantiating a `wasmtime` instance
+            /// where name resolution often happens with strings.
+            pub fn get_export(&self, name: &str) -> Option<&wasmtime::Func> {
+                match name {
+                    #(#get_exports,)*
+                    _ => None,
+                }
+            }
+
+            /// Adds all instance items to the specified `Linker`.
+            pub fn add_to_linker(&self, linker: &mut wasmtime::Linker) -> anyhow::Result<()> {
+                #(#linker_add)*
+                Ok(())
+            }
+        }
+    }
+}
+
+fn generate_func(
+    func: &witx::InterfaceFunc,
+    names: &Names,
+    missing_mem_conf: &MissingMemoryConf,
+    target_module: &TokenStream2,
+) -> TokenStream2 {
+    let missing_mem_err = &missing_mem_conf.err;
+    let name_ident = names.func(&func.name);
+
+    let coretype = func.core_type();
+
+    let arg_decls = coretype.args.iter().map(|arg| {
+        let name = names.func_core_arg(arg);
+        let atom = names.atom_type(arg.repr());
+        quote! { #name: #atom }
+    });
+    let arg_names = coretype.args.iter().map(|arg| names.func_core_arg(arg));
+
+    let (ret_ty, handle_early_error) = if let Some(ret) = &coretype.ret {
+        let ret_ty = match ret.signifies {
+            witx::CoreParamSignifies::Value(atom) => names.atom_type(atom),
+            _ => unreachable!("coretype ret should always be passed by value"),
+        };
+        (quote! { #ret_ty }, quote! { return e.into(); })
+    } else {
+        (
+            quote! {()},
+            quote! { panic!("unrecoverable error in {}: {}", stringify!(#name_ident), e) },
+        )
+    };
+
+    let runtime = names.runtime_mod();
+
+    quote! {
+        let my_cx = cx.clone();
+        let #name_ident = wasmtime::Func::wrap(
+            store,
+            move |caller: wasmtime::Caller<'_> #(,#arg_decls)*| -> #ret_ty {
+                unsafe {
+                    let mem = match caller.get_export("memory") {
+                        Some(wasmtime::Extern::Memory(m)) => m,
+                        _ => {
+                            log::warn!("callee does not export a memory as \"memory\"");
+                            let e = { #missing_mem_err };
+                            #handle_early_error
+                        }
+                    };
+                    // Wiggle does not expose any methods for functions to re-enter the WebAssembly
+                    // instance, or expose the memory via non-wiggle mechanisms. However, the
+                    // user-defined code may end up re-entering the instance, in which case this
+                    // is an incorrect implementation - we require exactly one BorrowChecker exist
+                    // per instance.
+                    let bc = #runtime::BorrowChecker::new();
+                    let mem = #runtime::WasmtimeGuestMemory::new( mem, bc );
+                    #target_module::#name_ident(
+                        &mut my_cx.borrow_mut(),
+                        &mem,
+                        #(#arg_names),*
+                    )
+                }
+            }
+        );
+    }
+}

crates/wiggle/wasmtime/src/lib.rs

@@ -0,0 +1,24 @@
+pub use wasmtime_wiggle_macro::*;
+pub use wiggle::*;
+
+/// Lightweight `wasmtime::Memory` wrapper so we can implement the
+/// `wiggle::GuestMemory` trait on it.
+pub struct WasmtimeGuestMemory {
+    mem: wasmtime::Memory,
+    bc: BorrowChecker,
+}
+
+impl WasmtimeGuestMemory {
+    pub fn new(mem: wasmtime::Memory, bc: BorrowChecker) -> Self {
+        Self { mem, bc }
+    }
+}
+
+unsafe impl GuestMemory for WasmtimeGuestMemory {
+    fn base(&self) -> (*mut u8, u32) {
+        (self.mem.data_ptr(), self.mem.data_size() as _)
+    }
+    fn borrow_checker(&self) -> &BorrowChecker {
+        &self.bc
+    }
+}