Update WIT tooling used by Wasmtime (#5565)

* Update WIT tooling used by Wasmtime

This commit updates the WIT tooling, namely the wasm-tools family of
crates, with recent updates. Notably:

* bytecodealliance/wasm-tools#867
* bytecodealliance/wasm-tools#871

This updates index spaces in components and additionally bumps the
minimum required version of the component binary format to be consumed
by Wasmtime (because of the index space changes). Additionally WIT
tooling now fully supports `use`.

Note that WIT tooling doesn't, at this time, fully support packages and
depending on remotely defined WIT packages. Currently WIT still needs to
be vendored in the project. It's hoped that future work with `cargo
component` and possible integration here could make the story about
depending on remotely-defined WIT more ergonomic and streamlined.

* Fix `bindgen!` codegen tests

* Add a test for `use` paths an implement support

* Update to crates.io versions of wasm-tools

* Uncomment codegen tests

crates/component-macro/Cargo.toml

@@ -16,6 +16,7 @@ test = false
 doctest = false
 
 [dependencies]
+anyhow = "1.0"
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = { version = "1.0", features = ["extra-traits"] }

crates/component-macro/src/bindgen.rs

@@ -3,14 +3,14 @@ use std::path::{Path, PathBuf};
 use syn::parse::{Error, Parse, ParseStream, Result};
 use syn::punctuated::Punctuated;
 use syn::{braced, token, Ident, Token};
-use wasmtime_wit_bindgen::Opts;
-use wit_parser::{Document, World};
+use wasmtime_wit_bindgen::{Opts, TrappableError};
+use wit_parser::{PackageId, Resolve, UnresolvedPackage, WorldId};
 
-#[derive(Default)]
 pub struct Config {
-    opts: Opts, // ...
-    world: World,
-    files: Vec<String>,
+    opts: Opts,
+    resolve: Resolve,
+    world: WorldId,
+    files: Vec<PathBuf>,
 }
 
 pub fn expand(input: &Config) -> Result<TokenStream> {
@@ -21,93 +21,162 @@ pub fn expand(input: &Config) -> Result<TokenStream> {
         ));
     }
 
-    let src = input.opts.generate(&input.world);
+    let src = input.opts.generate(&input.resolve, input.world);
     let mut contents = src.parse::<TokenStream>().unwrap();
 
     // Include a dummy `include_str!` for any files we read so rustc knows that
     // we depend on the contents of those files.
-    let cwd = std::env::var("CARGO_MANIFEST_DIR").unwrap();
     for file in input.files.iter() {
         contents.extend(
-            format!(
-                "const _: &str = include_str!(r#\"{}\"#);\n",
-                Path::new(&cwd).join(file).display()
-            )
-            .parse::<TokenStream>()
-            .unwrap(),
+            format!("const _: &str = include_str!(r#\"{}\"#);\n", file.display())
+                .parse::<TokenStream>()
+                .unwrap(),
         );
     }
 
     Ok(contents)
 }
 
+enum Source {
+    Path(String),
+    Inline(String),
+}
+
 impl Parse for Config {
     fn parse(input: ParseStream<'_>) -> Result<Self> {
         let call_site = Span::call_site();
+        let mut opts = Opts::default();
         let mut world = None;
-        let mut ret = Config::default();
+        let mut source = None;
 
-        if input.peek(token::Brace) {
+        let document = if input.peek(token::Brace) {
             let content;
             syn::braced!(content in input);
             let fields = Punctuated::<Opt, Token![,]>::parse_terminated(&content)?;
+            let mut document = None;
             for field in fields.into_pairs() {
                 match field.into_value() {
-                    Opt::Path(path) => {
-                        if world.is_some() {
-                            return Err(Error::new(path.span(), "cannot specify second world"));
+                    Opt::Path(s) => {
+                        if source.is_some() {
+                            return Err(Error::new(s.span(), "cannot specify second source"));
                         }
-                        world = Some(ret.parse(path)?);
+                        source = Some(Source::Path(s.value()));
                     }
-                    Opt::Inline(span, w) => {
-                        if world.is_some() {
-                            return Err(Error::new(span, "cannot specify second world"));
+                    Opt::World(s) => {
+                        if document.is_some() {
+                            return Err(Error::new(s.span(), "cannot specify second document"));
                         }
-                        world = Some(w);
+                        document = Some(parse_doc(&s.value(), &mut world));
                     }
-                    Opt::Tracing(val) => ret.opts.tracing = val,
-                    Opt::Async(val) => ret.opts.async_ = val,
-                    Opt::TrappableErrorType(val) => ret.opts.trappable_error_type = val,
+                    Opt::Inline(s) => {
+                        if source.is_some() {
+                            return Err(Error::new(s.span(), "cannot specify second source"));
+                        }
+                        source = Some(Source::Inline(s.value()));
+                    }
+                    Opt::Tracing(val) => opts.tracing = val,
+                    Opt::Async(val) => opts.async_ = val,
+                    Opt::TrappableErrorType(val) => opts.trappable_error_type = val,
+                }
+            }
+            match (document, &source) {
+                (Some(doc), _) => doc,
+                (None, Some(Source::Inline(_))) => "macro-input".to_string(),
+                _ => {
+                    return Err(Error::new(
+                        call_site,
+                        "must specify a `world` to generate bindings for",
+                    ))
                 }
             }
         } else {
-            let s = input.parse::<syn::LitStr>()?;
-            world = Some(ret.parse(s)?);
-        }
-        ret.world = world.ok_or_else(|| {
-            Error::new(
-                call_site,
-                "must specify a `*.wit` file to generate bindings for",
-            )
-        })?;
-        Ok(ret)
+            let document = input.parse::<syn::LitStr>()?;
+            if input.parse::<Option<syn::token::In>>()?.is_some() {
+                source = Some(Source::Path(input.parse::<syn::LitStr>()?.value()));
+            }
+            parse_doc(&document.value(), &mut world)
+        };
+        let (resolve, pkg, files) =
+            parse_source(&source).map_err(|err| Error::new(call_site, format!("{err:?}")))?;
+        let doc = resolve.packages[pkg]
+            .documents
+            .get(&document)
+            .copied()
+            .ok_or_else(|| {
+                Error::new(call_site, format!("no document named `{document}` found"))
+            })?;
+
+        let world = match &world {
+            Some(name) => resolve.documents[doc]
+                .worlds
+                .get(name)
+                .copied()
+                .ok_or_else(|| Error::new(call_site, format!("no world named `{name}` found")))?,
+            None => resolve.documents[doc].default_world.ok_or_else(|| {
+                Error::new(call_site, format!("no default world found in `{document}`"))
+            })?,
+        };
+        Ok(Config {
+            opts,
+            resolve,
+            world,
+            files,
+        })
     }
 }
 
-impl Config {
-    fn parse(&mut self, path: syn::LitStr) -> Result<World> {
-        let span = path.span();
-        let path = path.value();
-        let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
-        let path = manifest_dir.join(path);
-        self.files.push(path.to_str().unwrap().to_string());
-        World::parse_file(path).map_err(|e| Error::new(span, e))
+fn parse_source(source: &Option<Source>) -> anyhow::Result<(Resolve, PackageId, Vec<PathBuf>)> {
+    let mut resolve = Resolve::default();
+    let mut files = Vec::new();
+    let root = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
+    let mut parse = |path: &Path| -> anyhow::Result<_> {
+        if path.is_dir() {
+            let (pkg, sources) = resolve.push_dir(&path)?;
+            files = sources;
+            Ok(pkg)
+        } else {
+            let pkg = UnresolvedPackage::parse_file(path)?;
+            files.extend(pkg.source_files().map(|s| s.to_owned()));
+            resolve.push(pkg, &Default::default())
+        }
+    };
+    let pkg = match source {
+        Some(Source::Inline(s)) => resolve.push(
+            UnresolvedPackage::parse("macro-input".as_ref(), &s)?,
+            &Default::default(),
+        )?,
+        Some(Source::Path(s)) => parse(&root.join(&s))?,
+        None => parse(&root.join("wit"))?,
+    };
+
+    Ok((resolve, pkg, files))
+}
+
+fn parse_doc(s: &str, world: &mut Option<String>) -> String {
+    match s.find('.') {
+        Some(pos) => {
+            *world = Some(s[pos + 1..].to_string());
+            s[..pos].to_string()
+        }
+        None => s.to_string(),
     }
 }
 
 mod kw {
-    syn::custom_keyword!(path);
     syn::custom_keyword!(inline);
+    syn::custom_keyword!(path);
     syn::custom_keyword!(tracing);
     syn::custom_keyword!(trappable_error_type);
+    syn::custom_keyword!(world);
 }
 
 enum Opt {
+    World(syn::LitStr),
     Path(syn::LitStr),
-    Inline(Span, World),
+    Inline(syn::LitStr),
     Tracing(bool),
     Async(bool),
-    TrappableErrorType(Vec<(String, String, String)>),
+    TrappableErrorType(Vec<TrappableError>),
 }
 
 impl Parse for Opt {
@@ -118,14 +187,13 @@ impl Parse for Opt {
             input.parse::<Token![:]>()?;
             Ok(Opt::Path(input.parse()?))
         } else if l.peek(kw::inline) {
-            let span = input.parse::<kw::inline>()?.span;
+            input.parse::<kw::inline>()?;
             input.parse::<Token![:]>()?;
-            let s = input.parse::<syn::LitStr>()?;
-            let world = Document::parse("<macro-input>".as_ref(), &s.value())
-                .map_err(|e| Error::new(s.span(), e))?
-                .into_world()
-                .map_err(|e| Error::new(s.span(), e))?;
-            Ok(Opt::Inline(span, world))
+            Ok(Opt::Inline(input.parse()?))
+        } else if l.peek(kw::world) {
+            input.parse::<kw::world>()?;
+            input.parse::<Token![:]>()?;
+            Ok(Opt::World(input.parse()?))
         } else if l.peek(kw::tracing) {
             input.parse::<kw::tracing>()?;
             input.parse::<Token![:]>()?;
@@ -141,7 +209,16 @@ impl Parse for Opt {
             let _lbrace = braced!(contents in input);
             let fields: Punctuated<(String, String, String), Token![,]> =
                 contents.parse_terminated(trappable_error_field_parse)?;
-            Ok(Opt::TrappableErrorType(fields.into_iter().collect()))
+            Ok(Opt::TrappableErrorType(
+                fields
+                    .into_iter()
+                    .map(|(wit_owner, wit_name, rust_name)| TrappableError {
+                        wit_owner: Some(wit_owner),
+                        wit_name,
+                        rust_name,
+                    })
+                    .collect(),
+            ))
         } else {
             Err(l.error())
         }

crates/component-macro/test-helpers/src/lib.rs

@@ -12,10 +12,10 @@ pub fn foreach(input: TokenStream) -> TokenStream {
         let f = f.unwrap().path();
         if f.extension().and_then(|s| s.to_str()) == Some("wit") {
             let name = f.file_stem().unwrap().to_str().unwrap();
-            let name = Ident::new(&name.replace("-", "_"), Span::call_site());
+            let ident = Ident::new(&name.replace("-", "_"), Span::call_site());
             let path = f.to_str().unwrap();
             result.push(quote! {
-                #input!(#name #path);
+                #input!(#ident #name #path);
             });
         }
     }

crates/component-macro/tests/codegen.rs

@@ -1,17 +1,19 @@
 macro_rules! gentest {
-    ($id:ident $path:tt) => {
+    ($id:ident $name:tt $path:tt) => {
         mod $id {
             mod normal {
-                wasmtime::component::bindgen!($path);
+                wasmtime::component::bindgen!($name in $path);
             }
             mod async_ {
                 wasmtime::component::bindgen!({
+                    world: $name,
                     path: $path,
                     async: true,
                 });
             }
             mod tracing {
                 wasmtime::component::bindgen!({
+                    world: $name,
                     path: $path,
                     tracing: true,
                 });

crates/component-macro/tests/codegen/char.wit

@@ -5,8 +5,7 @@ interface chars {
   return-char: func() -> char
 }
 
-world the-world {
-  import imports: chars
-  export exports: chars
-  default export chars
+default world the-world {
+  import imports: self.chars
+  export exports: self.chars
 }

crates/component-macro/tests/codegen/conventions.wit

@@ -32,8 +32,7 @@ interface conventions {
   %bool: func()
 }
 
-world the-world {
-  import imports: conventions
-  export exports: conventions
-  default export conventions
+default world the-world {
+  import imports: self.conventions
+  export exports: self.conventions
 }

crates/component-macro/tests/codegen/empty.wit

@@ -1 +1 @@
-world empty {}
+default world empty {}

crates/component-macro/tests/codegen/flags.wit

@@ -47,8 +47,7 @@ interface flegs {
   roundtrip-flag64: func(x: flag64) -> flag64
 }
 
-world the-flags {
-  import import-flags: flegs
-  export export-flags: flegs
-  default export flegs
+default world the-flags {
+  import import-flags: self.flegs
+  export export-flags: self.flegs
 }

crates/component-macro/tests/codegen/floats.wit

@@ -5,8 +5,7 @@ interface floats {
   float64-result: func() -> float64
 }
 
-world the-world {
-  import imports: floats
-  export exports: floats
-  default export floats
+default world the-world {
+  import imports: self.floats
+  export exports: self.floats
 }

crates/component-macro/tests/codegen/integers.wit

@@ -32,8 +32,7 @@ interface integers {
   pair-ret: func() -> tuple<s64, u8>
 }
 
-world the-world {
-  import imports: integers
-  export exports: integers
-  default export integers
+default world the-world {
+  import imports: self.integers
+  export exports: self.integers
 }

crates/component-macro/tests/codegen/lists.wit

@@ -77,8 +77,7 @@ interface lists {
   load-store-everything: func(a: load-store-all-sizes) -> load-store-all-sizes
 }
 
-world the-lists {
-  import import-lists: lists
-  export export-lists: lists
-  default export lists
+default world the-lists {
+  import import-lists: self.lists
+  export export-lists: self.lists
 }

crates/component-macro/tests/codegen/many-arguments.wit

@@ -44,8 +44,7 @@ interface manyarg {
   big-argument: func(x: big-struct)
 }
 
-world the-world {
-  import imports: manyarg
-  export exports: manyarg
-  default export manyarg
+default world the-world {
+  import imports: self.manyarg
+  export exports: self.manyarg
 }

crates/component-macro/tests/codegen/multi-return.wit

@@ -6,8 +6,7 @@ interface multi-return {
   mre: func() -> (a: u32, b: float32)
 }
 
-world the-world {
-  import imports: multi-return
-  export exports: multi-return
-  default export multi-return
+default world the-world {
+  import imports: self.multi-return
+  export exports: self.multi-return
 }

crates/component-macro/tests/codegen/records.wit

@@ -53,8 +53,7 @@ interface records {
   typedef-inout: func(e: tuple-typedef2) -> s32
 }
 
-world the-world {
-  import imports: records
-  export exports: records
-  default export records
+default world the-world {
+  import imports: self.records
+  export exports: self.records
 }

crates/component-macro/tests/codegen/simple-functions.wit

@@ -9,8 +9,7 @@ interface simple {
   f6: func(a: u32, b: u32, c: u32) -> tuple<u32, u32, u32>
 }
 
-world the-world {
-  import imports: simple
-  export exports: simple
-  default export simple
+default world the-world {
+  import imports: self.simple
+  export exports: self.simple
 }

crates/component-macro/tests/codegen/simple-lists.wit

@@ -1,13 +1,11 @@
 interface simple-lists {
   simple-list1: func(l: list<u32>)
   simple-list2: func() -> list<u32>
-  // TODO: reenable this when smw implements this
-  // simple-list3: func(a: list<u32>, b: list<u32>) -> tuple<list<u32>, list<u32>>
+  simple-list3: func(a: list<u32>, b: list<u32>) -> tuple<list<u32>, list<u32>>
   simple-list4: func(l: list<list<u32>>) -> list<list<u32>>
 }
 
-world my-world {
-  import imports: simple-lists
-  export exports: simple-lists
-  default export simple-lists
+default world my-world {
+  import imports: self.simple-lists
+  export exports: self.simple-lists
 }

crates/component-macro/tests/codegen/small-anonymous.wit

@@ -7,8 +7,7 @@ interface anon {
   option-test: func() -> result<option<string>, error>
 }
 
-world the-world {
-  import imports: anon
-  export exports: anon
-  default export anon
+default world the-world {
+  import imports: self.anon
+  export exports: self.anon
 }

crates/component-macro/tests/codegen/smoke-default.wit

@@ -1,5 +1,3 @@
-world the-world {
-  default export interface {
-    y: func()
-  }
+default world the-world {
+  export y: func()
 }

crates/component-macro/tests/codegen/smoke-export.wit

@@ -1,4 +1,4 @@
-world the-world {
+default world the-world {
   export the-name: interface {
     y: func()
   }

crates/component-macro/tests/codegen/smoke.wit

@@ -1,4 +1,4 @@
-world the-world {
+default world the-world {
   import imports: interface {
     y: func()
   }

crates/component-macro/tests/codegen/strings.wit

@@ -4,8 +4,7 @@ interface strings {
   c: func(a: string, b: string) -> string
 }
 
-world the-world {
-  import imports: strings
-  export exports: strings
-  default export strings
+default world the-world {
+  import imports: self.strings
+  export exports: self.strings
 }

crates/component-macro/tests/codegen/unions.wit

@@ -58,8 +58,7 @@ interface unions {
   identify-distinguishable-num: func(num: distinguishable-num) -> u8
 }
 
-world the-unions {
-  import import-unions: unions
-  export export-unions: unions
-  default export unions
+default world the-unions {
+  import import-unions: self.unions
+  export export-unions: self.unions
 }

crates/component-macro/tests/codegen/use-paths.wit

@@ -0,0 +1,27 @@
+interface a {
+  record foo {}
+
+  a: func() -> foo
+}
+
+interface b {
+  use self.a.{foo}
+
+  a: func() -> foo
+}
+
+interface c {
+  use self.b.{foo}
+
+  a: func() -> foo
+}
+
+default world d {
+  import a: self.a
+  import b: self.b
+  import d: interface {
+    use self.c.{foo}
+
+    b: func() -> foo
+  }
+}

crates/component-macro/tests/codegen/variants.wit

@@ -139,8 +139,7 @@ interface variants {
   return-named-result: func() -> (a: result<u8, my-errno>)
 }
 
-world my-world {
-  import imports: variants
-  export exports: variants
-  default export variants
+default world my-world {
+  import imports: self.variants
+  export exports: self.variants
 }