diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b6eaed06d..ed195c3d2d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -558,3 +558,19 @@ jobs: - run: | cargo install --root ${{ runner.tool_cache }}/cargo-audit --version ${{ env.CARGO_AUDIT_VERSION }} cargo-audit cargo audit + + verify-publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - run: rustup update stable && rustup default stable + - run: | + cd ${{ runner.tool_cache }} + curl -L https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz | tar xzf - + echo "::add-path::`pwd`/sccache-0.2.13-x86_64-unknown-linux-musl" + echo ::set-env name=RUSTC_WRAPPER::sccache + - run: | + rustc scripts/publish.rs + ./publish verify diff --git a/.gitignore b/.gitignore index 18a3a0200c..41bec43a18 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ tags target .z3-trace foo +.cargo +publish +vendor diff --git a/crates/wiggle/test-helpers/Cargo.toml b/crates/wiggle/test-helpers/Cargo.toml index cc407c1678..3effeb7061 100644 --- a/crates/wiggle/test-helpers/Cargo.toml +++ b/crates/wiggle/test-helpers/Cargo.toml @@ -9,6 +9,7 @@ categories = ["wasm"] keywords = ["webassembly", "wasm"] repository = "https://github.com/bytecodealliance/wasmtime" include = ["src/**/*", "LICENSE"] +publish = false [dependencies] proptest = "0.9" diff --git a/crates/wiggle/wasmtime/macro/Cargo.toml b/crates/wiggle/wasmtime/macro/Cargo.toml index dee7417d1d..6a7c4ac521 100644 --- a/crates/wiggle/wasmtime/macro/Cargo.toml +++ b/crates/wiggle/wasmtime/macro/Cargo.toml @@ -18,7 +18,7 @@ test = false witx = { path = "../../../wasi-common/WASI/tools/witx", version = "0.8.5" } wiggle-generate = { path = "../../generate", version = "0.19.0" } quote = "1.0" -syn = { version = "1.0", features = ["full"] } +syn = { version = "1.0", features = ["full", "extra-traits"] } proc-macro2 = "1.0" [badges] diff --git a/docs/contributing-release-process.md b/docs/contributing-release-process.md index 0bf3e5cf39..fad3eaa02e 100644 --- a/docs/contributing-release-process.md +++ b/docs/contributing-release-process.md @@ -11,10 +11,11 @@ whimsical currently, or on request from others) then the following steps need to be executed to make the release: 1. `git pull` - make sure you've got the latest changes -1. Update the version numbers in `Cargo.toml` for all crates - * Edit `scripts/bump-wasmtime-version.sh`, notable the `version` variable - * Run the script - * Commit the changes +1. Run `rustc scripts/publish.rs` +1. Run `./publish bump` + * Review and commit the changes + * Note that this bumps all cranelift/wasmtime versions as a major version bump + at this time. See the `bump_version` function in `publish.rs` to tweak this. 1. Make sure `RELEASES.md` is up-to-date, and fill it out if it doesn't have an entry yet for the current release. 1. Send this version update as a PR to the `wasmtime` repository, wait for a merge @@ -22,6 +23,11 @@ be executed to make the release: 1. Push the tag to the repository * This will trigger the release CI which will create all release artifacts and publish them to GitHub releases. -1. Run `scripts/publish-wasmtime.sh` to publish all crates to crates.io +1. Run `./publish publish` + * This will fail on some crates, but that's expected. + * Keep running this script until all crates are published. Note that crates.io + won't let you publish something twice so rerunning is only for crates which + need the index to be udpated and if it hasn't yet. It's recommended to wait + a bit between runs of the script. And that's it, then you've done a Wasmtime release. diff --git a/scripts/bump-cranelift-version.sh b/scripts/bump-cranelift-version.sh deleted file mode 100755 index 0751dc9278..0000000000 --- a/scripts/bump-cranelift-version.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# This is a convenience script for maintainers changing a cranelift -# dependencies versions. To use, bump the version number below, run the -# script. - -topdir=$(dirname "$0")/.. -cd "$topdir" - -# All the cranelift-* crates have the same version number -version="0.66.0" - -# Update all of the Cargo.toml files. -echo "Updating crate versions to $version" -for toml in cranelift/Cargo.toml cranelift/*/Cargo.toml cranelift/*/*/Cargo.toml; do - # Update the version number of this crate to $version. - sed -i.bk -e "/^version = /s/\"[^\"]*\"/\"$version\"/" \ - "$toml" -done - -# Update the required version numbers of path dependencies. -find -name Cargo.toml \ - -not -path ./crates/wasi-common/WASI/tools/witx/Cargo.toml \ - -exec sed -i.bk \ - -e "/^cranelift/s/version = \"[^\"]*\"/version = \"$version\"/" \ - -e "/^peepmatic /s/version = \"[^\"]*\"/version = \"$version\"/" \ - {} \; - -# Update the Cargo.lock file for the new versions. -cargo update diff --git a/scripts/bump-wasmtime-version.sh b/scripts/bump-wasmtime-version.sh deleted file mode 100755 index 143dc75fa5..0000000000 --- a/scripts/bump-wasmtime-version.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# This is a convenience script for maintainers publishing a new version of -# Wasmtime to crates.io. To use, bump the version number below, run the -# script, and then run the commands that the script prints. - -topdir=$(dirname "$0")/.. -cd "$topdir" - -# All the wasmtime-* crates have the same version number -short_version="19" -version="0.$short_version.0" - -# Update the version numbers of the crates to $version. Skip crates with -# a version of "0.0.0", which are unpublished. -echo "Updating crate versions to $version" -find crates -name Cargo.toml \ - -not -path crates/wasi-common/WASI/tools/witx/Cargo.toml \ - -exec sed -i.bk -e "s/^version = \"[.*[^0.].*\"$/version = \"$version\"/" {} \; - -# Updat the top-level Cargo.toml too. -sed -i.bk -e "s/^version = \"[.*[^0.].*\"$/version = \"$version\"/" Cargo.toml - -# Update the required version numbers of path dependencies. -find -name Cargo.toml \ - -not -path ./crates/wasi-common/WASI/tools/witx/Cargo.toml \ - -exec sed -i.bk \ - -e "/^\(wasmtime\|wiggle\)/s/version = \"[^\"]*\"/version = \"$version\"/" \ - {} \; -find -name Cargo.toml \ - -not -path ./crates/wasi-common/WASI/tools/witx/Cargo.toml \ - -exec sed -i.bk \ - -e "/^\(wasi-common\|wig\|yanix\|winx\|lightbeam\) = /s/version = \"[^\"]*\"/version = \"$version\"/" \ - {} \; - -find crates -type f -print0 | xargs -0 sed -i \ - "s/wasi-common-[0-9][0-9]*/wasi-common-$short_version/" -find crates -type f -print0 | xargs -0 sed -i \ - "s/DEP_WASI_COMMON_[0-9][0-9]*/DEP_WASI_COMMON_${short_version}/" - -# Update the Cargo.lock files for the new versions. -cargo update -cd crates/test-programs/wasi-tests -cargo update -cd - >/dev/null diff --git a/scripts/publish-cranelift.sh b/scripts/publish-cranelift.sh deleted file mode 100755 index 25b2d8d089..0000000000 --- a/scripts/publish-cranelift.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# This is a convenience script for maintainers publishing a new version of -# Cranelift to crates.io. To use, first bump the version number by running the -# `scripts/bump-cranelift-version.sh` script, then run this script, and run the -# commands that it prints. -# -# Don't forget to push a git tag for this release! - -topdir=$(dirname "$0")/.. -cd "$topdir" - -# Commands needed to publish. -# -# Note that libraries need to be published in topological order. - -for crate in \ - entity \ - bforest \ - peepmatic/crates/macro \ - peepmatic/crates/automata \ - peepmatic/crates/runtime \ - peepmatic \ - codegen/shared \ - codegen/meta \ - codegen \ - frontend \ - native \ - preopt \ - reader \ - wasm \ - module \ - faerie \ - umbrella \ - simplejit \ - object -do - echo cargo publish --manifest-path "cranelift/$crate/Cargo.toml" - - # Sleep for a few seconds to allow the server to update the index. - # https://internals.rust-lang.org/t/changes-to-how-crates-io-handles-index-updates/9608 - echo sleep 30 -done - -echo git tag cranelift-v$(grep version cranelift/Cargo.toml | head -n 1 | cut -d '"' -f 2) diff --git a/scripts/publish-wasmtime.sh b/scripts/publish-wasmtime.sh deleted file mode 100755 index 207b813366..0000000000 --- a/scripts/publish-wasmtime.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# This is a convenience script for maintainers publishing a new version of -# Wasmtime to crates.io. To use, first bump the Wasmtime versions with -# `scripts/bump-wasmtime-version.sh` and Cranelift versions with -# `scripts/bump-cranelift-version.sh`, then run this script, and run the -# commands that it prints. - -topdir=$(dirname "$0")/.. -cd "$topdir" - -# Publishing Wasmtime requires publishing any local Cranelift changes. -scripts/publish-cranelift.sh - -# Commands needed to publish. -# -# Note that libraries need to be published in topological order. - -for cargo_toml in \ - crates/wasi-common/winx/Cargo.toml \ - crates/wasi-common/yanix/Cargo.toml \ - crates/wasi-common/wig/Cargo.toml \ - crates/wiggle/generate/Cargo.toml \ - crates/wiggle/macro/Cargo.toml \ - crates/wiggle/wasmtime/macro/Cargo.toml \ - crates/wiggle/wasmtime/Cargo.toml \ - crates/wiggle/Cargo.toml \ - crates/wasi-common/Cargo.toml \ - crates/lightbeam/Cargo.toml \ - crates/environ/Cargo.toml \ - crates/obj/Cargo.toml \ - crates/runtime/Cargo.toml \ - crates/profiling/Cargo.toml \ - crates/debug/Cargo.toml \ - crates/jit/Cargo.toml \ - crates/wasmtime/Cargo.toml \ - crates/wasi/Cargo.toml \ - crates/wast/Cargo.toml \ - crates/misc/rust/macro/Cargo.toml \ - crates/misc/rust/Cargo.toml \ - Cargo.toml \ -; do - version="" - case $cargo_toml in - crates/lightbeam/Cargo.toml) version=" +nightly" ;; - crates/misc/py/Cargo.toml) version=" +nightly" ;; - esac - - echo cargo$version publish --manifest-path "$cargo_toml" - - # Sleep for a few seconds to allow the server to update the index. - # https://internals.rust-lang.org/t/changes-to-how-crates-io-handles-index-updates/9608 - echo sleep 30 -done - -echo git tag v$(grep "version =" Cargo.toml | head -n 1 | cut -d '"' -f 2) diff --git a/scripts/publish.rs b/scripts/publish.rs new file mode 100644 index 0000000000..57749a9d2b --- /dev/null +++ b/scripts/publish.rs @@ -0,0 +1,323 @@ +//! Helper script to publish the wasmtime and cranelift suites of crates +//! +//! See documentation in `docs/contributing-release-process.md` for more +//! information, but in a nutshell: +//! +//! * `./publish bump` - bump crate versions in-tree +//! * `./publish verify` - verify crates can be published to crates.io +//! * `./publish publish` - actually publish crates to crates.io + +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +// note that this list must be topologically sorted by dependencies +const CRATES_TO_PUBLISH: &[&str] = &[ + // peepmatic + "peepmatic-macro", + "peepmatic-automata", + "peepmatic-runtime", + "peepmatic", + // cranelift + "cranelift-entity", + "cranelift-bforest", + "cranelift-codegen-shared", + "cranelift-codegen-meta", + "cranelift-codegen", + "cranelift-reader", + "cranelift-serde", + "cranelift-module", + "cranelift-preopt", + "cranelift-frontend", + "cranelift-wasm", + "cranelift-faerie", + "cranelift-native", + "cranelift-object", + "cranelift-interpreter", + "cranelift", + "cranelift-simplejit", + // wig/wiggle + "wig", + "wiggle-generate", + "wiggle-macro", + "wiggle", + "wasmtime-wiggle-macro", + // wasi-common bits + "winx", + "yanix", + "wasi-common", + // wasmtime + "lightbeam", + "wasmtime-environ", + "wasmtime-runtime", + "wasmtime-debug", + "wasmtime-profiling", + "wasmtime-obj", + "wasmtime-jit", + "wasmtime", + "wasmtime-wiggle", + "wasmtime-wasi", + "wasmtime-rust-macro", + "wasmtime-rust", + "wasmtime-wast", + "wasmtime-cli", +]; + +struct Crate { + manifest: PathBuf, + name: String, + version: String, + next_version: String, + publish: bool, +} + +fn main() { + let mut crates = Vec::new(); + crates.push(read_crate("./Cargo.toml".as_ref())); + find_crates("crates".as_ref(), &mut crates); + find_crates("cranelift".as_ref(), &mut crates); + + let pos = CRATES_TO_PUBLISH + .iter() + .enumerate() + .map(|(i, c)| (*c, i)) + .collect::>(); + crates.sort_by_key(|krate| pos.get(&krate.name[..])); + + match &env::args().nth(1).expect("must have one argument")[..] { + "bump" => { + for krate in crates.iter() { + bump_version(&krate, &crates); + } + // update the lock file + assert!(Command::new("cargo") + .arg("fetch") + .status() + .unwrap() + .success()); + } + + "publish" => { + for krate in crates.iter() { + publish(&krate); + } + } + + "verify" => { + verify(&crates); + } + + s => panic!("unknown command: {}", s), + } +} + +fn find_crates(dir: &Path, dst: &mut Vec) { + if dir.join("Cargo.toml").exists() { + let krate = read_crate(&dir.join("Cargo.toml")); + if !krate.publish || CRATES_TO_PUBLISH.iter().any(|c| krate.name == *c) { + dst.push(krate); + } else { + panic!("failed to find {:?} in whitelist or blacklist", krate.name); + } + } + + for entry in dir.read_dir().unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + find_crates(&entry.path(), dst); + } + } +} + +fn read_crate(manifest: &Path) -> Crate { + let mut name = None; + let mut version = None; + let mut publish = true; + for line in fs::read_to_string(manifest).unwrap().lines() { + if name.is_none() && line.starts_with("name = \"") { + name = Some( + line.replace("name = \"", "") + .replace("\"", "") + .trim() + .to_string(), + ); + } + if version.is_none() && line.starts_with("version = \"") { + version = Some( + line.replace("version = \"", "") + .replace("\"", "") + .trim() + .to_string(), + ); + } + if line.starts_with("publish = false") { + publish = false; + } + } + let name = name.unwrap(); + let version = version.unwrap(); + let next_version = if CRATES_TO_PUBLISH.contains(&&name[..]) { + bump(&version) + } else { + version.clone() + }; + if name == "witx" { + publish = false; + } + Crate { + manifest: manifest.to_path_buf(), + name, + version, + next_version, + publish, + } +} + +fn bump_version(krate: &Crate, crates: &[Crate]) { + let contents = fs::read_to_string(&krate.manifest).unwrap(); + + let mut new_manifest = String::new(); + let mut is_deps = false; + for line in contents.lines() { + let mut rewritten = false; + if !is_deps && line.starts_with("version =") { + if CRATES_TO_PUBLISH.contains(&&krate.name[..]) { + println!( + "bump `{}` {} => {}", + krate.name, krate.version, krate.next_version + ); + new_manifest.push_str(&line.replace(&krate.version, &krate.next_version)); + rewritten = true; + } + } + + is_deps = if line.starts_with("[") { + line.contains("dependencies") + } else { + is_deps + }; + + for other in crates { + if !is_deps || !line.starts_with(&format!("{} ", other.name)) { + continue; + } + if !line.contains(&other.version) { + if !line.contains("version =") { + continue; + } + panic!( + "{:?} has a dep on {} but doesn't list version {}", + krate.manifest, other.name, other.version + ); + } + rewritten = true; + new_manifest.push_str(&line.replace(&other.version, &other.next_version)); + break; + } + if !rewritten { + new_manifest.push_str(line); + } + new_manifest.push_str("\n"); + } + fs::write(&krate.manifest, new_manifest).unwrap(); +} + +/// Performs a major version bump increment on the semver version `version`. +/// +/// This function will perform a semver-major-version bump on the `version` +/// specified. This is used to calculate the next version of a crate in this +/// repository since we're currently making major version bumps for all our +/// releases. This may end up getting tweaked as we stabilize crates and start +/// doing more minor/patch releases, but for now this should do the trick. +fn bump(version: &str) -> String { + let mut iter = version.split('.').map(|s| s.parse::().unwrap()); + let major = iter.next().expect("major version"); + let minor = iter.next().expect("minor version"); + let patch = iter.next().expect("patch version"); + if major != 0 { + format!("{}.0.0", major + 1) + } else if minor != 0 { + format!("0.{}.0", minor + 1) + } else { + format!("0.0.{}", patch + 1) + } +} + +fn publish(krate: &Crate) { + if !CRATES_TO_PUBLISH.iter().any(|s| *s == krate.name) { + return; + } + let status = Command::new("cargo") + .arg("publish") + .current_dir(krate.manifest.parent().unwrap()) + .arg("--no-verify") + .status() + .expect("failed to run cargo"); + if !status.success() { + println!("FAIL: failed to publish `{}`: {}", krate.name, status); + } +} + +// Verify the current tree is publish-able to crates.io. The intention here is +// that we'll run `cargo package` on everything which verifies the build as-if +// it were published to crates.io. This requires using an incrementally-built +// directory registry generated from `cargo vendor` because the versions +// referenced from `Cargo.toml` may not exist on crates.io. +fn verify(crates: &[Crate]) { + drop(fs::remove_dir_all(".cargo")); + drop(fs::remove_dir_all("vendor")); + let vendor = Command::new("cargo") + .arg("vendor") + .stderr(Stdio::inherit()) + .output() + .unwrap(); + assert!(vendor.status.success()); + + fs::create_dir_all(".cargo").unwrap(); + fs::write(".cargo/config.toml", vendor.stdout).unwrap(); + + // Vendor witx which wasn't vendored because it's a path dependency, but + // it'll need to be in our directory registry for crates that depend on it. + let witx = crates.iter().find(|c| c.name == "witx").unwrap(); + verify_and_vendor(&witx); + + for krate in crates { + if !krate.publish { + continue; + } + verify_and_vendor(&krate); + } + + fn verify_and_vendor(krate: &Crate) { + let mut cmd = Command::new("cargo"); + cmd.arg("package") + .arg("--manifest-path") + .arg(&krate.manifest) + .env("CARGO_TARGET_DIR", "./target"); + if krate.name == "lightbeam" || krate.name == "witx" { + cmd.arg("--no-verify"); + } + let status = cmd.status().unwrap(); + assert!(status.success()); + let tar = Command::new("tar") + .arg("xf") + .arg(format!( + "../target/package/{}-{}.crate", + krate.name, krate.version + )) + .current_dir("./vendor") + .status() + .unwrap(); + assert!(tar.success()); + fs::write( + format!( + "./vendor/{}-{}/.cargo-checksum.json", + krate.name, krate.version + ), + "{\"files\":{}}", + ) + .unwrap(); + } +}