Verify crates are publish-able on CI (#2036)
This commit updates our CI to verify that all crates are publish-able at all times on every commit. During the 0.19.0 release we found another case where the crates as they live in this repository weren't publish-able, so the hope is that this no longer comes up again! The script added in this commit also takes the time/liberty to remove the existing bump/publish scripts and instead replace them with one Rust script originally sourced from wasm-bindgen. The intention of this script is that it has three modes: * `./publish bump` - bumps version numbers which are sent as a PR to get reviewed (probably with a changelog as well) * `./publish verify` - run on CI on every commit, builds every crate we publish as if it's being published to crates.io, notably without raw access to other crates in the repository. * `./publish publish` - publishes all crates to crates.io, passing the `--no-verify` flag to make this a much speedier process than it is today.
This commit is contained in:
Alex Crichton
@@ -1,31 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# This is a convenience script for maintainers changing a cranelift
|
||||
# dependencies versions. To use, bump the version number below, run the
|
||||
# script.
|
||||
|
||||
topdir=$(dirname "$0")/..
|
||||
cd "$topdir"
|
||||
|
||||
# All the cranelift-* crates have the same version number
|
||||
version="0.66.0"
|
||||
|
||||
# Update all of the Cargo.toml files.
|
||||
echo "Updating crate versions to $version"
|
||||
for toml in cranelift/Cargo.toml cranelift/*/Cargo.toml cranelift/*/*/Cargo.toml; do
|
||||
# Update the version number of this crate to $version.
|
||||
sed -i.bk -e "/^version = /s/\"[^\"]*\"/\"$version\"/" \
|
||||
"$toml"
|
||||
done
|
||||
|
||||
# Update the required version numbers of path dependencies.
|
||||
find -name Cargo.toml \
|
||||
-not -path ./crates/wasi-common/WASI/tools/witx/Cargo.toml \
|
||||
-exec sed -i.bk \
|
||||
-e "/^cranelift/s/version = \"[^\"]*\"/version = \"$version\"/" \
|
||||
-e "/^peepmatic /s/version = \"[^\"]*\"/version = \"$version\"/" \
|
||||
{} \;
|
||||
|
||||
# Update the Cargo.lock file for the new versions.
|
||||
cargo update
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# This is a convenience script for maintainers publishing a new version of
|
||||
# Wasmtime to crates.io. To use, bump the version number below, run the
|
||||
# script, and then run the commands that the script prints.
|
||||
|
||||
topdir=$(dirname "$0")/..
|
||||
cd "$topdir"
|
||||
|
||||
# All the wasmtime-* crates have the same version number
|
||||
short_version="19"
|
||||
version="0.$short_version.0"
|
||||
|
||||
# Update the version numbers of the crates to $version. Skip crates with
|
||||
# a version of "0.0.0", which are unpublished.
|
||||
echo "Updating crate versions to $version"
|
||||
find crates -name Cargo.toml \
|
||||
-not -path crates/wasi-common/WASI/tools/witx/Cargo.toml \
|
||||
-exec sed -i.bk -e "s/^version = \"[.*[^0.].*\"$/version = \"$version\"/" {} \;
|
||||
|
||||
# Updat the top-level Cargo.toml too.
|
||||
sed -i.bk -e "s/^version = \"[.*[^0.].*\"$/version = \"$version\"/" Cargo.toml
|
||||
|
||||
# Update the required version numbers of path dependencies.
|
||||
find -name Cargo.toml \
|
||||
-not -path ./crates/wasi-common/WASI/tools/witx/Cargo.toml \
|
||||
-exec sed -i.bk \
|
||||
-e "/^\(wasmtime\|wiggle\)/s/version = \"[^\"]*\"/version = \"$version\"/" \
|
||||
{} \;
|
||||
find -name Cargo.toml \
|
||||
-not -path ./crates/wasi-common/WASI/tools/witx/Cargo.toml \
|
||||
-exec sed -i.bk \
|
||||
-e "/^\(wasi-common\|wig\|yanix\|winx\|lightbeam\) = /s/version = \"[^\"]*\"/version = \"$version\"/" \
|
||||
{} \;
|
||||
|
||||
find crates -type f -print0 | xargs -0 sed -i \
|
||||
"s/wasi-common-[0-9][0-9]*/wasi-common-$short_version/"
|
||||
find crates -type f -print0 | xargs -0 sed -i \
|
||||
"s/DEP_WASI_COMMON_[0-9][0-9]*/DEP_WASI_COMMON_${short_version}/"
|
||||
|
||||
# Update the Cargo.lock files for the new versions.
|
||||
cargo update
|
||||
cd crates/test-programs/wasi-tests
|
||||
cargo update
|
||||
cd - >/dev/null
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# This is a convenience script for maintainers publishing a new version of
|
||||
# Cranelift to crates.io. To use, first bump the version number by running the
|
||||
# `scripts/bump-cranelift-version.sh` script, then run this script, and run the
|
||||
# commands that it prints.
|
||||
#
|
||||
# Don't forget to push a git tag for this release!
|
||||
|
||||
topdir=$(dirname "$0")/..
|
||||
cd "$topdir"
|
||||
|
||||
# Commands needed to publish.
|
||||
#
|
||||
# Note that libraries need to be published in topological order.
|
||||
|
||||
for crate in \
|
||||
entity \
|
||||
bforest \
|
||||
peepmatic/crates/macro \
|
||||
peepmatic/crates/automata \
|
||||
peepmatic/crates/runtime \
|
||||
peepmatic \
|
||||
codegen/shared \
|
||||
codegen/meta \
|
||||
codegen \
|
||||
frontend \
|
||||
native \
|
||||
preopt \
|
||||
reader \
|
||||
wasm \
|
||||
module \
|
||||
faerie \
|
||||
umbrella \
|
||||
simplejit \
|
||||
object
|
||||
do
|
||||
echo cargo publish --manifest-path "cranelift/$crate/Cargo.toml"
|
||||
|
||||
# Sleep for a few seconds to allow the server to update the index.
|
||||
# https://internals.rust-lang.org/t/changes-to-how-crates-io-handles-index-updates/9608
|
||||
echo sleep 30
|
||||
done
|
||||
|
||||
echo git tag cranelift-v$(grep version cranelift/Cargo.toml | head -n 1 | cut -d '"' -f 2)
|
||||
@@ -1,57 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# This is a convenience script for maintainers publishing a new version of
|
||||
# Wasmtime to crates.io. To use, first bump the Wasmtime versions with
|
||||
# `scripts/bump-wasmtime-version.sh` and Cranelift versions with
|
||||
# `scripts/bump-cranelift-version.sh`, then run this script, and run the
|
||||
# commands that it prints.
|
||||
|
||||
topdir=$(dirname "$0")/..
|
||||
cd "$topdir"
|
||||
|
||||
# Publishing Wasmtime requires publishing any local Cranelift changes.
|
||||
scripts/publish-cranelift.sh
|
||||
|
||||
# Commands needed to publish.
|
||||
#
|
||||
# Note that libraries need to be published in topological order.
|
||||
|
||||
for cargo_toml in \
|
||||
crates/wasi-common/winx/Cargo.toml \
|
||||
crates/wasi-common/yanix/Cargo.toml \
|
||||
crates/wasi-common/wig/Cargo.toml \
|
||||
crates/wiggle/generate/Cargo.toml \
|
||||
crates/wiggle/macro/Cargo.toml \
|
||||
crates/wiggle/wasmtime/macro/Cargo.toml \
|
||||
crates/wiggle/wasmtime/Cargo.toml \
|
||||
crates/wiggle/Cargo.toml \
|
||||
crates/wasi-common/Cargo.toml \
|
||||
crates/lightbeam/Cargo.toml \
|
||||
crates/environ/Cargo.toml \
|
||||
crates/obj/Cargo.toml \
|
||||
crates/runtime/Cargo.toml \
|
||||
crates/profiling/Cargo.toml \
|
||||
crates/debug/Cargo.toml \
|
||||
crates/jit/Cargo.toml \
|
||||
crates/wasmtime/Cargo.toml \
|
||||
crates/wasi/Cargo.toml \
|
||||
crates/wast/Cargo.toml \
|
||||
crates/misc/rust/macro/Cargo.toml \
|
||||
crates/misc/rust/Cargo.toml \
|
||||
Cargo.toml \
|
||||
; do
|
||||
version=""
|
||||
case $cargo_toml in
|
||||
crates/lightbeam/Cargo.toml) version=" +nightly" ;;
|
||||
crates/misc/py/Cargo.toml) version=" +nightly" ;;
|
||||
esac
|
||||
|
||||
echo cargo$version publish --manifest-path "$cargo_toml"
|
||||
|
||||
# Sleep for a few seconds to allow the server to update the index.
|
||||
# https://internals.rust-lang.org/t/changes-to-how-crates-io-handles-index-updates/9608
|
||||
echo sleep 30
|
||||
done
|
||||
|
||||
echo git tag v$(grep "version =" Cargo.toml | head -n 1 | cut -d '"' -f 2)
|
||||
323
scripts/publish.rs
Normal file
323
scripts/publish.rs
Normal file
@@ -0,0 +1,323 @@
|
||||
//! Helper script to publish the wasmtime and cranelift suites of crates
|
||||
//!
|
||||
//! See documentation in `docs/contributing-release-process.md` for more
|
||||
//! information, but in a nutshell:
|
||||
//!
|
||||
//! * `./publish bump` - bump crate versions in-tree
|
||||
//! * `./publish verify` - verify crates can be published to crates.io
|
||||
//! * `./publish publish` - actually publish crates to crates.io
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
// note that this list must be topologically sorted by dependencies
|
||||
const CRATES_TO_PUBLISH: &[&str] = &[
|
||||
// peepmatic
|
||||
"peepmatic-macro",
|
||||
"peepmatic-automata",
|
||||
"peepmatic-runtime",
|
||||
"peepmatic",
|
||||
// cranelift
|
||||
"cranelift-entity",
|
||||
"cranelift-bforest",
|
||||
"cranelift-codegen-shared",
|
||||
"cranelift-codegen-meta",
|
||||
"cranelift-codegen",
|
||||
"cranelift-reader",
|
||||
"cranelift-serde",
|
||||
"cranelift-module",
|
||||
"cranelift-preopt",
|
||||
"cranelift-frontend",
|
||||
"cranelift-wasm",
|
||||
"cranelift-faerie",
|
||||
"cranelift-native",
|
||||
"cranelift-object",
|
||||
"cranelift-interpreter",
|
||||
"cranelift",
|
||||
"cranelift-simplejit",
|
||||
// wig/wiggle
|
||||
"wig",
|
||||
"wiggle-generate",
|
||||
"wiggle-macro",
|
||||
"wiggle",
|
||||
"wasmtime-wiggle-macro",
|
||||
// wasi-common bits
|
||||
"winx",
|
||||
"yanix",
|
||||
"wasi-common",
|
||||
// wasmtime
|
||||
"lightbeam",
|
||||
"wasmtime-environ",
|
||||
"wasmtime-runtime",
|
||||
"wasmtime-debug",
|
||||
"wasmtime-profiling",
|
||||
"wasmtime-obj",
|
||||
"wasmtime-jit",
|
||||
"wasmtime",
|
||||
"wasmtime-wiggle",
|
||||
"wasmtime-wasi",
|
||||
"wasmtime-rust-macro",
|
||||
"wasmtime-rust",
|
||||
"wasmtime-wast",
|
||||
"wasmtime-cli",
|
||||
];
|
||||
|
||||
struct Crate {
|
||||
manifest: PathBuf,
|
||||
name: String,
|
||||
version: String,
|
||||
next_version: String,
|
||||
publish: bool,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let mut crates = Vec::new();
|
||||
crates.push(read_crate("./Cargo.toml".as_ref()));
|
||||
find_crates("crates".as_ref(), &mut crates);
|
||||
find_crates("cranelift".as_ref(), &mut crates);
|
||||
|
||||
let pos = CRATES_TO_PUBLISH
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, c)| (*c, i))
|
||||
.collect::<HashMap<_, _>>();
|
||||
crates.sort_by_key(|krate| pos.get(&krate.name[..]));
|
||||
|
||||
match &env::args().nth(1).expect("must have one argument")[..] {
|
||||
"bump" => {
|
||||
for krate in crates.iter() {
|
||||
bump_version(&krate, &crates);
|
||||
}
|
||||
// update the lock file
|
||||
assert!(Command::new("cargo")
|
||||
.arg("fetch")
|
||||
.status()
|
||||
.unwrap()
|
||||
.success());
|
||||
}
|
||||
|
||||
"publish" => {
|
||||
for krate in crates.iter() {
|
||||
publish(&krate);
|
||||
}
|
||||
}
|
||||
|
||||
"verify" => {
|
||||
verify(&crates);
|
||||
}
|
||||
|
||||
s => panic!("unknown command: {}", s),
|
||||
}
|
||||
}
|
||||
|
||||
fn find_crates(dir: &Path, dst: &mut Vec<Crate>) {
|
||||
if dir.join("Cargo.toml").exists() {
|
||||
let krate = read_crate(&dir.join("Cargo.toml"));
|
||||
if !krate.publish || CRATES_TO_PUBLISH.iter().any(|c| krate.name == *c) {
|
||||
dst.push(krate);
|
||||
} else {
|
||||
panic!("failed to find {:?} in whitelist or blacklist", krate.name);
|
||||
}
|
||||
}
|
||||
|
||||
for entry in dir.read_dir().unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
if entry.file_type().unwrap().is_dir() {
|
||||
find_crates(&entry.path(), dst);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn read_crate(manifest: &Path) -> Crate {
|
||||
let mut name = None;
|
||||
let mut version = None;
|
||||
let mut publish = true;
|
||||
for line in fs::read_to_string(manifest).unwrap().lines() {
|
||||
if name.is_none() && line.starts_with("name = \"") {
|
||||
name = Some(
|
||||
line.replace("name = \"", "")
|
||||
.replace("\"", "")
|
||||
.trim()
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if version.is_none() && line.starts_with("version = \"") {
|
||||
version = Some(
|
||||
line.replace("version = \"", "")
|
||||
.replace("\"", "")
|
||||
.trim()
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if line.starts_with("publish = false") {
|
||||
publish = false;
|
||||
}
|
||||
}
|
||||
let name = name.unwrap();
|
||||
let version = version.unwrap();
|
||||
let next_version = if CRATES_TO_PUBLISH.contains(&&name[..]) {
|
||||
bump(&version)
|
||||
} else {
|
||||
version.clone()
|
||||
};
|
||||
if name == "witx" {
|
||||
publish = false;
|
||||
}
|
||||
Crate {
|
||||
manifest: manifest.to_path_buf(),
|
||||
name,
|
||||
version,
|
||||
next_version,
|
||||
publish,
|
||||
}
|
||||
}
|
||||
|
||||
fn bump_version(krate: &Crate, crates: &[Crate]) {
|
||||
let contents = fs::read_to_string(&krate.manifest).unwrap();
|
||||
|
||||
let mut new_manifest = String::new();
|
||||
let mut is_deps = false;
|
||||
for line in contents.lines() {
|
||||
let mut rewritten = false;
|
||||
if !is_deps && line.starts_with("version =") {
|
||||
if CRATES_TO_PUBLISH.contains(&&krate.name[..]) {
|
||||
println!(
|
||||
"bump `{}` {} => {}",
|
||||
krate.name, krate.version, krate.next_version
|
||||
);
|
||||
new_manifest.push_str(&line.replace(&krate.version, &krate.next_version));
|
||||
rewritten = true;
|
||||
}
|
||||
}
|
||||
|
||||
is_deps = if line.starts_with("[") {
|
||||
line.contains("dependencies")
|
||||
} else {
|
||||
is_deps
|
||||
};
|
||||
|
||||
for other in crates {
|
||||
if !is_deps || !line.starts_with(&format!("{} ", other.name)) {
|
||||
continue;
|
||||
}
|
||||
if !line.contains(&other.version) {
|
||||
if !line.contains("version =") {
|
||||
continue;
|
||||
}
|
||||
panic!(
|
||||
"{:?} has a dep on {} but doesn't list version {}",
|
||||
krate.manifest, other.name, other.version
|
||||
);
|
||||
}
|
||||
rewritten = true;
|
||||
new_manifest.push_str(&line.replace(&other.version, &other.next_version));
|
||||
break;
|
||||
}
|
||||
if !rewritten {
|
||||
new_manifest.push_str(line);
|
||||
}
|
||||
new_manifest.push_str("\n");
|
||||
}
|
||||
fs::write(&krate.manifest, new_manifest).unwrap();
|
||||
}
|
||||
|
||||
/// Performs a major version bump increment on the semver version `version`.
|
||||
///
|
||||
/// This function will perform a semver-major-version bump on the `version`
|
||||
/// specified. This is used to calculate the next version of a crate in this
|
||||
/// repository since we're currently making major version bumps for all our
|
||||
/// releases. This may end up getting tweaked as we stabilize crates and start
|
||||
/// doing more minor/patch releases, but for now this should do the trick.
|
||||
fn bump(version: &str) -> String {
|
||||
let mut iter = version.split('.').map(|s| s.parse::<u32>().unwrap());
|
||||
let major = iter.next().expect("major version");
|
||||
let minor = iter.next().expect("minor version");
|
||||
let patch = iter.next().expect("patch version");
|
||||
if major != 0 {
|
||||
format!("{}.0.0", major + 1)
|
||||
} else if minor != 0 {
|
||||
format!("0.{}.0", minor + 1)
|
||||
} else {
|
||||
format!("0.0.{}", patch + 1)
|
||||
}
|
||||
}
|
||||
|
||||
fn publish(krate: &Crate) {
|
||||
if !CRATES_TO_PUBLISH.iter().any(|s| *s == krate.name) {
|
||||
return;
|
||||
}
|
||||
let status = Command::new("cargo")
|
||||
.arg("publish")
|
||||
.current_dir(krate.manifest.parent().unwrap())
|
||||
.arg("--no-verify")
|
||||
.status()
|
||||
.expect("failed to run cargo");
|
||||
if !status.success() {
|
||||
println!("FAIL: failed to publish `{}`: {}", krate.name, status);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify the current tree is publish-able to crates.io. The intention here is
|
||||
// that we'll run `cargo package` on everything which verifies the build as-if
|
||||
// it were published to crates.io. This requires using an incrementally-built
|
||||
// directory registry generated from `cargo vendor` because the versions
|
||||
// referenced from `Cargo.toml` may not exist on crates.io.
|
||||
fn verify(crates: &[Crate]) {
|
||||
drop(fs::remove_dir_all(".cargo"));
|
||||
drop(fs::remove_dir_all("vendor"));
|
||||
let vendor = Command::new("cargo")
|
||||
.arg("vendor")
|
||||
.stderr(Stdio::inherit())
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(vendor.status.success());
|
||||
|
||||
fs::create_dir_all(".cargo").unwrap();
|
||||
fs::write(".cargo/config.toml", vendor.stdout).unwrap();
|
||||
|
||||
// Vendor witx which wasn't vendored because it's a path dependency, but
|
||||
// it'll need to be in our directory registry for crates that depend on it.
|
||||
let witx = crates.iter().find(|c| c.name == "witx").unwrap();
|
||||
verify_and_vendor(&witx);
|
||||
|
||||
for krate in crates {
|
||||
if !krate.publish {
|
||||
continue;
|
||||
}
|
||||
verify_and_vendor(&krate);
|
||||
}
|
||||
|
||||
fn verify_and_vendor(krate: &Crate) {
|
||||
let mut cmd = Command::new("cargo");
|
||||
cmd.arg("package")
|
||||
.arg("--manifest-path")
|
||||
.arg(&krate.manifest)
|
||||
.env("CARGO_TARGET_DIR", "./target");
|
||||
if krate.name == "lightbeam" || krate.name == "witx" {
|
||||
cmd.arg("--no-verify");
|
||||
}
|
||||
let status = cmd.status().unwrap();
|
||||
assert!(status.success());
|
||||
let tar = Command::new("tar")
|
||||
.arg("xf")
|
||||
.arg(format!(
|
||||
"../target/package/{}-{}.crate",
|
||||
krate.name, krate.version
|
||||
))
|
||||
.current_dir("./vendor")
|
||||
.status()
|
||||
.unwrap();
|
||||
assert!(tar.success());
|
||||
fs::write(
|
||||
format!(
|
||||
"./vendor/{}-{}/.cargo-checksum.json",
|
||||
krate.name, krate.version
|
||||
),
|
||||
"{\"files\":{}}",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user