T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Verify crates are publish-able on CI (#2036)

Browse Source 
This commit updates our CI to verify that all crates are publish-able at
all times on every commit. During the 0.19.0 release we found another
case where the crates as they live in this repository weren't
publish-able, so the hope is that this no longer comes up again!

The script added in this commit also takes the time/liberty to remove
the existing bump/publish scripts and instead replace them with one Rust
script originally sourced from wasm-bindgen. The intention of this
script is that it has three modes:

* `./publish bump` - bumps version numbers which are sent as a PR to get
  reviewed (probably with a changelog as well)

* `./publish verify` - run on CI on every commit, builds every crate we
  publish as if it's being published to crates.io, notably without raw
  access to other crates in the repository.

* `./publish publish` - publishes all crates to crates.io, passing the
  `--no-verify` flag to make this a much speedier process than it is
  today.
This commit is contained in:
Alex Crichton
2020-07-17 16:19:35 -05:00
committed by GitHub
parent a7cedf3100
commit 978070c020
10 changed files with 355 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/main.yml vendored
View File

@@ -558,3 +558,19 @@ jobs:
- run: |
cargo install --root ${{ runner.tool_cache }}/cargo-audit --version ${{ env.CARGO_AUDIT_VERSION }} cargo-audit
cargo audit
verify-publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- run: rustup update stable && rustup default stable
- run: |
cd ${{ runner.tool_cache }}
curl -L https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz | tar xzf -
echo "::add-path::`pwd`/sccache-0.2.13-x86_64-unknown-linux-musl"
echo ::set-env name=RUSTC_WRAPPER::sccache
- run: |
rustc scripts/publish.rs
./publish verify

3
.gitignore vendored
View File

@@ -16,3 +16,6 @@ tags
target
.z3-trace
foo
.cargo
publish
vendor

1
crates/wiggle/test-helpers/Cargo.toml
View File

@@ -9,6 +9,7 @@ categories = ["wasm"]
keywords = ["webassembly", "wasm"]
repository = "https://github.com/bytecodealliance/wasmtime"
include = ["src/**/*", "LICENSE"]
publish = false
[dependencies]
proptest = "0.9"

2
crates/wiggle/wasmtime/macro/Cargo.toml
View File

@@ -18,7 +18,7 @@ test = false
witx = { path = "../../../wasi-common/WASI/tools/witx", version = "0.8.5" }
wiggle-generate = { path = "../../generate", version = "0.19.0" }
quote = "1.0"
syn = { version = "1.0", features = ["full"] }
syn = { version = "1.0", features = ["full", "extra-traits"] }
proc-macro2 = "1.0"
[badges]

16
docs/contributing-release-process.md
View File

@@ -11,10 +11,11 @@ whimsical currently, or on request from others) then the following steps need to
be executed to make the release:
1. `git pull` - make sure you've got the latest changes
1. Update the version numbers in `Cargo.toml` for all crates
* Edit `scripts/bump-wasmtime-version.sh`, notable the `version` variable
* Run the script
* Commit the changes
1. Run `rustc scripts/publish.rs`
1. Run `./publish bump`
* Review and commit the changes
* Note that this bumps all cranelift/wasmtime versions as a major version bump
at this time. See the `bump_version` function in `publish.rs` to tweak this.
1. Make sure `RELEASES.md` is up-to-date, and fill it out if it doesn't have an
entry yet for the current release.
1. Send this version update as a PR to the `wasmtime` repository, wait for a merge
@@ -22,6 +23,11 @@ be executed to make the release:
1. Push the tag to the repository
* This will trigger the release CI which will create all release artifacts and
publish them to GitHub releases.
1. Run `scripts/publish-wasmtime.sh` to publish all crates to crates.io
1. Run `./publish publish`
* This will fail on some crates, but that's expected.
* Keep running this script until all crates are published. Note that crates.io
won't let you publish something twice so rerunning is only for crates which
need the index to be udpated and if it hasn't yet. It's recommended to wait
a bit between runs of the script.
And that's it, then you've done a Wasmtime release.

31
scripts/bump-cranelift-version.sh
View File

@@ -1,31 +0,0 @@
#!/bin/bash
set -euo pipefail
# This is a convenience script for maintainers changing a cranelift
# dependencies versions. To use, bump the version number below, run the
# script.
topdir=$(dirname "$0")/..
cd "$topdir"
# All the cranelift-* crates have the same version number
version="0.66.0"
# Update all of the Cargo.toml files.
echo "Updating crate versions to $version"
for toml in cranelift/Cargo.toml cranelift/*/Cargo.toml cranelift/*/*/Cargo.toml; do
# Update the version number of this crate to $version.
sed -i.bk -e "/^version = /s/\"[^\"]*\"/\"$version\"/" \
"$toml"
done
# Update the required version numbers of path dependencies.
find -name Cargo.toml \
-not -path ./crates/wasi-common/WASI/tools/witx/Cargo.toml \
-exec sed -i.bk \
-e "/^cranelift/s/version = \"[^\"]*\"/version = \"$version\"/" \
-e "/^peepmatic /s/version = \"[^\"]*\"/version = \"$version\"/" \
{} \;
# Update the Cargo.lock file for the new versions.
cargo update

46
scripts/bump-wasmtime-version.sh
View File

@@ -1,46 +0,0 @@
#!/bin/bash
set -euo pipefail
# This is a convenience script for maintainers publishing a new version of
# Wasmtime to crates.io. To use, bump the version number below, run the
# script, and then run the commands that the script prints.
topdir=$(dirname "$0")/..
cd "$topdir"
# All the wasmtime-* crates have the same version number
short_version="19"
version="0.$short_version.0"
# Update the version numbers of the crates to $version. Skip crates with
# a version of "0.0.0", which are unpublished.
echo "Updating crate versions to $version"
find crates -name Cargo.toml \
-not -path crates/wasi-common/WASI/tools/witx/Cargo.toml \
-exec sed -i.bk -e "s/^version = \"[.*[^0.].*\"$/version = \"$version\"/" {} \;
# Updat the top-level Cargo.toml too.
sed -i.bk -e "s/^version = \"[.*[^0.].*\"$/version = \"$version\"/" Cargo.toml
# Update the required version numbers of path dependencies.
find -name Cargo.toml \
-not -path ./crates/wasi-common/WASI/tools/witx/Cargo.toml \
-exec sed -i.bk \
-e "/^\(wasmtime\|wiggle\)/s/version = \"[^\"]*\"/version = \"$version\"/" \
{} \;
find -name Cargo.toml \
-not -path ./crates/wasi-common/WASI/tools/witx/Cargo.toml \
-exec sed -i.bk \
-e "/^\(wasi-common\|wig\|yanix\|winx\|lightbeam\) = /s/version = \"[^\"]*\"/version = \"$version\"/" \
{} \;
find crates -type f -print0 | xargs -0 sed -i \
"s/wasi-common-[0-9][0-9]*/wasi-common-$short_version/"
find crates -type f -print0 | xargs -0 sed -i \
"s/DEP_WASI_COMMON_[0-9][0-9]*/DEP_WASI_COMMON_${short_version}/"
# Update the Cargo.lock files for the new versions.
cargo update
cd crates/test-programs/wasi-tests
cargo update
cd - >/dev/null

46
scripts/publish-cranelift.sh
View File

@@ -1,46 +0,0 @@
#!/bin/bash
set -euo pipefail
# This is a convenience script for maintainers publishing a new version of
# Cranelift to crates.io. To use, first bump the version number by running the
# `scripts/bump-cranelift-version.sh` script, then run this script, and run the
# commands that it prints.
#
# Don't forget to push a git tag for this release!
topdir=$(dirname "$0")/..
cd "$topdir"
# Commands needed to publish.
#
# Note that libraries need to be published in topological order.
for crate in \
entity \
bforest \
peepmatic/crates/macro \
peepmatic/crates/automata \
peepmatic/crates/runtime \
peepmatic \
codegen/shared \
codegen/meta \
codegen \
frontend \
native \
preopt \
reader \
wasm \
module \
faerie \
umbrella \
simplejit \
object
do
echo cargo publish --manifest-path "cranelift/$crate/Cargo.toml"
# Sleep for a few seconds to allow the server to update the index.
# https://internals.rust-lang.org/t/changes-to-how-crates-io-handles-index-updates/9608
echo sleep 30
done
echo git tag cranelift-v$(grep version cranelift/Cargo.toml | head -n 1 | cut -d '"' -f 2)

57
scripts/publish-wasmtime.sh
View File

@@ -1,57 +0,0 @@
#!/bin/bash
set -euo pipefail
# This is a convenience script for maintainers publishing a new version of
# Wasmtime to crates.io. To use, first bump the Wasmtime versions with
# `scripts/bump-wasmtime-version.sh` and Cranelift versions with
# `scripts/bump-cranelift-version.sh`, then run this script, and run the
# commands that it prints.
topdir=$(dirname "$0")/..
cd "$topdir"
# Publishing Wasmtime requires publishing any local Cranelift changes.
scripts/publish-cranelift.sh
# Commands needed to publish.
#
# Note that libraries need to be published in topological order.
for cargo_toml in \
crates/wasi-common/winx/Cargo.toml \
crates/wasi-common/yanix/Cargo.toml \
crates/wasi-common/wig/Cargo.toml \
crates/wiggle/generate/Cargo.toml \
crates/wiggle/macro/Cargo.toml \
crates/wiggle/wasmtime/macro/Cargo.toml \
crates/wiggle/wasmtime/Cargo.toml \
crates/wiggle/Cargo.toml \
crates/wasi-common/Cargo.toml \
crates/lightbeam/Cargo.toml \
crates/environ/Cargo.toml \
crates/obj/Cargo.toml \
crates/runtime/Cargo.toml \
crates/profiling/Cargo.toml \
crates/debug/Cargo.toml \
crates/jit/Cargo.toml \
crates/wasmtime/Cargo.toml \
crates/wasi/Cargo.toml \
crates/wast/Cargo.toml \
crates/misc/rust/macro/Cargo.toml \
crates/misc/rust/Cargo.toml \
Cargo.toml \
; do
version=""
case $cargo_toml in
crates/lightbeam/Cargo.toml) version=" +nightly" ;;
crates/misc/py/Cargo.toml) version=" +nightly" ;;
esac
echo cargo$version publish --manifest-path "$cargo_toml"
# Sleep for a few seconds to allow the server to update the index.
# https://internals.rust-lang.org/t/changes-to-how-crates-io-handles-index-updates/9608
echo sleep 30
done
echo git tag v$(grep "version =" Cargo.toml | head -n 1 | cut -d '"' -f 2)

323
scripts/publish.rs Normal file
View File

@@ -0,0 +1,323 @@
//! Helper script to publish the wasmtime and cranelift suites of crates
//!
//! See documentation in `docs/contributing-release-process.md` for more
//! information, but in a nutshell:
//!
//! * `./publish bump` - bump crate versions in-tree
//! * `./publish verify` - verify crates can be published to crates.io
//! * `./publish publish` - actually publish crates to crates.io
use std::collections::HashMap;
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
// note that this list must be topologically sorted by dependencies
const CRATES_TO_PUBLISH: &[&str] = &[
// peepmatic
"peepmatic-macro",
"peepmatic-automata",
"peepmatic-runtime",
"peepmatic",
// cranelift
"cranelift-entity",
"cranelift-bforest",
"cranelift-codegen-shared",
"cranelift-codegen-meta",
"cranelift-codegen",
"cranelift-reader",
"cranelift-serde",
"cranelift-module",
"cranelift-preopt",
"cranelift-frontend",
"cranelift-wasm",
"cranelift-faerie",
"cranelift-native",
"cranelift-object",
"cranelift-interpreter",
"cranelift",
"cranelift-simplejit",
// wig/wiggle
"wig",
"wiggle-generate",
"wiggle-macro",
"wiggle",
"wasmtime-wiggle-macro",
// wasi-common bits
"winx",
"yanix",
"wasi-common",
// wasmtime
"lightbeam",
"wasmtime-environ",
"wasmtime-runtime",
"wasmtime-debug",
"wasmtime-profiling",
"wasmtime-obj",
"wasmtime-jit",
"wasmtime",
"wasmtime-wiggle",
"wasmtime-wasi",
"wasmtime-rust-macro",
"wasmtime-rust",
"wasmtime-wast",
"wasmtime-cli",
];
struct Crate {
manifest: PathBuf,
name: String,
version: String,
next_version: String,
publish: bool,
}
fn main() {
let mut crates = Vec::new();
crates.push(read_crate("./Cargo.toml".as_ref()));
find_crates("crates".as_ref(), &mut crates);
find_crates("cranelift".as_ref(), &mut crates);
let pos = CRATES_TO_PUBLISH
.iter()
.enumerate()
.map(|(i, c)| (*c, i))
.collect::<HashMap<_, _>>();
crates.sort_by_key(|krate| pos.get(&krate.name[..]));
match &env::args().nth(1).expect("must have one argument")[..] {
"bump" => {
for krate in crates.iter() {
bump_version(&krate, &crates);
}
// update the lock file
assert!(Command::new("cargo")
.arg("fetch")
.status()
.unwrap()
.success());
}
"publish" => {
for krate in crates.iter() {
publish(&krate);
}
}
"verify" => {
verify(&crates);
}
s => panic!("unknown command: {}", s),
}
}
fn find_crates(dir: &Path, dst: &mut Vec<Crate>) {
if dir.join("Cargo.toml").exists() {
let krate = read_crate(&dir.join("Cargo.toml"));
if !krate.publish || CRATES_TO_PUBLISH.iter().any(|c| krate.name == *c) {
dst.push(krate);
} else {
panic!("failed to find {:?} in whitelist or blacklist", krate.name);
}
}
for entry in dir.read_dir().unwrap() {
let entry = entry.unwrap();
if entry.file_type().unwrap().is_dir() {
find_crates(&entry.path(), dst);
}
}
}
fn read_crate(manifest: &Path) -> Crate {
let mut name = None;
let mut version = None;
let mut publish = true;
for line in fs::read_to_string(manifest).unwrap().lines() {
if name.is_none() && line.starts_with("name = \"") {
name = Some(
line.replace("name = \"", "")
.replace("\"", "")
.trim()
.to_string(),
);
}
if version.is_none() && line.starts_with("version = \"") {
version = Some(
line.replace("version = \"", "")
.replace("\"", "")
.trim()
.to_string(),
);
}
if line.starts_with("publish = false") {
publish = false;
}
}
let name = name.unwrap();
let version = version.unwrap();
let next_version = if CRATES_TO_PUBLISH.contains(&&name[..]) {
bump(&version)
} else {
version.clone()
};
if name == "witx" {
publish = false;
}
Crate {
manifest: manifest.to_path_buf(),
name,
version,
next_version,
publish,
}
}
fn bump_version(krate: &Crate, crates: &[Crate]) {
let contents = fs::read_to_string(&krate.manifest).unwrap();
let mut new_manifest = String::new();
let mut is_deps = false;
for line in contents.lines() {
let mut rewritten = false;
if !is_deps && line.starts_with("version =") {
if CRATES_TO_PUBLISH.contains(&&krate.name[..]) {
println!(
"bump `{}` {} => {}",
krate.name, krate.version, krate.next_version
);
new_manifest.push_str(&line.replace(&krate.version, &krate.next_version));
rewritten = true;
}
}
is_deps = if line.starts_with("[") {
line.contains("dependencies")
} else {
is_deps
};
for other in crates {
if !is_deps || !line.starts_with(&format!("{} ", other.name)) {
continue;
}
if !line.contains(&other.version) {
if !line.contains("version =") {
continue;
}
panic!(
"{:?} has a dep on {} but doesn't list version {}",
krate.manifest, other.name, other.version
);
}
rewritten = true;
new_manifest.push_str(&line.replace(&other.version, &other.next_version));
break;
}
if !rewritten {
new_manifest.push_str(line);
}
new_manifest.push_str("\n");
}
fs::write(&krate.manifest, new_manifest).unwrap();
}
/// Performs a major version bump increment on the semver version `version`.
///
/// This function will perform a semver-major-version bump on the `version`
/// specified. This is used to calculate the next version of a crate in this
/// repository since we're currently making major version bumps for all our
/// releases. This may end up getting tweaked as we stabilize crates and start
/// doing more minor/patch releases, but for now this should do the trick.
fn bump(version: &str) -> String {
let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap());
let major = iter.next().expect("major version");
let minor = iter.next().expect("minor version");
let patch = iter.next().expect("patch version");
if major != 0 {
format!("{}.0.0", major + 1)
} else if minor != 0 {
format!("0.{}.0", minor + 1)
} else {
format!("0.0.{}", patch + 1)
}
}
fn publish(krate: &Crate) {
if !CRATES_TO_PUBLISH.iter().any(|s| *s == krate.name) {
return;
}
let status = Command::new("cargo")
.arg("publish")
.current_dir(krate.manifest.parent().unwrap())
.arg("--no-verify")
.status()
.expect("failed to run cargo");
if !status.success() {
println!("FAIL: failed to publish `{}`: {}", krate.name, status);
}
}
// Verify the current tree is publish-able to crates.io. The intention here is
// that we'll run `cargo package` on everything which verifies the build as-if
// it were published to crates.io. This requires using an incrementally-built
// directory registry generated from `cargo vendor` because the versions
// referenced from `Cargo.toml` may not exist on crates.io.
fn verify(crates: &[Crate]) {
drop(fs::remove_dir_all(".cargo"));
drop(fs::remove_dir_all("vendor"));
let vendor = Command::new("cargo")
.arg("vendor")
.stderr(Stdio::inherit())
.output()
.unwrap();
assert!(vendor.status.success());
fs::create_dir_all(".cargo").unwrap();
fs::write(".cargo/config.toml", vendor.stdout).unwrap();
// Vendor witx which wasn't vendored because it's a path dependency, but
// it'll need to be in our directory registry for crates that depend on it.
let witx = crates.iter().find(|c| c.name == "witx").unwrap();
verify_and_vendor(&witx);
for krate in crates {
if !krate.publish {
continue;
}
verify_and_vendor(&krate);
}
fn verify_and_vendor(krate: &Crate) {
let mut cmd = Command::new("cargo");
cmd.arg("package")
.arg("--manifest-path")
.arg(&krate.manifest)
.env("CARGO_TARGET_DIR", "./target");
if krate.name == "lightbeam" || krate.name == "witx" {
cmd.arg("--no-verify");
}
let status = cmd.status().unwrap();
assert!(status.success());
let tar = Command::new("tar")
.arg("xf")
.arg(format!(
"../target/package/{}-{}.crate",
krate.name, krate.version
))
.current_dir("./vendor")
.status()
.unwrap();
assert!(tar.success());
fs::write(
format!(
"./vendor/{}-{}/.cargo-checksum.json",
krate.name, krate.version
),
"{\"files\":{}}",
)
.unwrap();
}
}