ci: Remove "publish" step (#2936)
This commit removes the publish step in GitHub actions, insteading folding all functionality into the release build steps. This avoids having a separately scheduled job after all the release build jobs which ends up getting delayed for quite a long time given the current scheduling algorithm. This involves refactoring the tarball assembly scripts and refactoring the github asset upload script too. Tarball assembly now manages everything internally and does platform-specific bits where necessary. The upload script is restructured to be run in parallel (in theory) and hopefully catches various errors and tries to not stomp over everyone else's work. The main trickiness here is handling `dev`, which is less critical for correctness than than tags themselves. As a small tweak build-wise the QEMU build for cross-compiled builders is now cached unlike before where it was unconditionally built, shaving a minute or two off build time.
This commit is contained in:
Alex Crichton
8
.github/actions/github-release/Dockerfile
vendored
8
.github/actions/github-release/Dockerfile
vendored
@@ -1,8 +0,0 @@
|
||||
FROM node:slim
|
||||
|
||||
COPY . /action
|
||||
WORKDIR /action
|
||||
|
||||
RUN npm install --production
|
||||
|
||||
ENTRYPOINT ["node", "/action/main.js"]
|
||||
7
.github/actions/github-release/action.yml
vendored
7
.github/actions/github-release/action.yml
vendored
@@ -4,12 +4,9 @@ inputs:
|
||||
token:
|
||||
description: ''
|
||||
required: true
|
||||
name:
|
||||
description: ''
|
||||
required: true
|
||||
files:
|
||||
description: ''
|
||||
required: true
|
||||
runs:
|
||||
using: 'docker'
|
||||
image: 'Dockerfile'
|
||||
using: 'node12'
|
||||
main: 'main.js'
|
||||
|
||||
116
.github/actions/github-release/main.js
vendored
116
.github/actions/github-release/main.js
vendored
@@ -11,12 +11,15 @@ function sleep(milliseconds) {
|
||||
async function runOnce() {
|
||||
// Load all our inputs and env vars. Note that `getInput` reads from `INPUT_*`
|
||||
const files = core.getInput('files');
|
||||
const name = core.getInput('name');
|
||||
const token = core.getInput('token');
|
||||
const slug = process.env.GITHUB_REPOSITORY;
|
||||
const owner = slug.split('/')[0];
|
||||
const repo = slug.split('/')[1];
|
||||
const sha = process.env.GITHUB_SHA;
|
||||
let name = 'dev';
|
||||
if (process.env.GITHUB_REF.startsWith('refs/tags/v')) {
|
||||
name = process.env.GITHUB_REF.substring(10);
|
||||
}
|
||||
|
||||
core.info(`files: ${files}`);
|
||||
core.info(`name: ${name}`);
|
||||
@@ -24,53 +27,88 @@ async function runOnce() {
|
||||
|
||||
const octokit = new github.GitHub(token);
|
||||
|
||||
// Delete the previous release since we can't overwrite one. This may happen
|
||||
// due to retrying an upload or it may happen because we're doing the dev
|
||||
// release.
|
||||
const releases = await octokit.paginate("GET /repos/:owner/:repo/releases", { owner, repo });
|
||||
for (const release of releases) {
|
||||
if (release.tag_name !== name) {
|
||||
continue;
|
||||
// For the `dev` release we may need to update the tag to point to the new
|
||||
// commit on this branch. All other names should already have tags associated
|
||||
// with them.
|
||||
if (name == 'dev') {
|
||||
let tag = null;
|
||||
try {
|
||||
tag = await octokit.request("GET /repos/:owner/:repo/git/refs/tags/:name", { owner, repo, name });
|
||||
core.info(`found existing tag`);
|
||||
console.log("tag: ", JSON.stringify(tag.data, null, 2));
|
||||
} catch (e) {
|
||||
// ignore if this tag doesn't exist
|
||||
core.info(`no existing tag found`);
|
||||
}
|
||||
|
||||
if (tag === null || tag.data.object.sha !== sha) {
|
||||
core.info(`updating existing tag or creating new one`);
|
||||
// Delete the previous release for this tag, if any
|
||||
try {
|
||||
core.info(`fetching release for ${name}`);
|
||||
const release = await octokit.repos.getReleaseByTag({ owner, repo, tag: name });
|
||||
core.info(`deleting release ${release.data.id}`);
|
||||
await octokit.repos.deleteRelease({ owner, repo, release_id: release.data.id });
|
||||
} catch (e) {
|
||||
// ignore, there may not have been a release
|
||||
console.log("ERROR: ", JSON.stringify(e, null, 2));
|
||||
}
|
||||
|
||||
try {
|
||||
core.info(`updating dev tag`);
|
||||
await octokit.git.updateRef({
|
||||
owner,
|
||||
repo,
|
||||
ref: 'tags/dev',
|
||||
sha,
|
||||
force: true,
|
||||
});
|
||||
} catch (e) {
|
||||
console.log("ERROR: ", JSON.stringify(e, null, 2));
|
||||
core.info(`creating dev tag`);
|
||||
try {
|
||||
await octokit.git.createTag({
|
||||
owner,
|
||||
repo,
|
||||
tag: 'dev',
|
||||
message: 'dev release',
|
||||
object: sha,
|
||||
type: 'commit',
|
||||
});
|
||||
} catch (e) {
|
||||
// we might race with others, so assume someone else has created the
|
||||
// tag by this point.
|
||||
}
|
||||
}
|
||||
} else {
|
||||
core.info(`existing tag works`);
|
||||
}
|
||||
const release_id = release.id;
|
||||
core.info(`deleting release ${release_id}`);
|
||||
await octokit.repos.deleteRelease({ owner, repo, release_id });
|
||||
}
|
||||
|
||||
// We also need to update the `dev` tag while we're at it on the `dev` branch.
|
||||
if (name == 'dev') {
|
||||
// Try to load the release for this tag, and if it doesn't exist then make a
|
||||
// new one. We might race with other builders on creation, though, so if the
|
||||
// creation fails try again to get the release by the tag.
|
||||
let release = null;
|
||||
try {
|
||||
core.info(`fetching release`);
|
||||
release = await octokit.repos.getReleaseByTag({ owner, repo, tag: name });
|
||||
} catch (e) {
|
||||
console.log("ERROR: ", JSON.stringify(e, null, 2));
|
||||
core.info(`creating a release`);
|
||||
try {
|
||||
core.info(`updating dev tag`);
|
||||
await octokit.git.updateRef({
|
||||
owner,
|
||||
repo,
|
||||
ref: 'tags/dev',
|
||||
sha,
|
||||
force: true,
|
||||
});
|
||||
} catch (e) {
|
||||
console.log("ERROR: ", JSON.stringify(e, null, 2));
|
||||
core.info(`creating dev tag`);
|
||||
await octokit.git.createTag({
|
||||
release = await octokit.repos.createRelease({
|
||||
owner,
|
||||
repo,
|
||||
tag: 'dev',
|
||||
message: 'dev release',
|
||||
object: sha,
|
||||
type: 'commit',
|
||||
tag_name: name,
|
||||
prerelease: name === 'dev',
|
||||
});
|
||||
} catch(e) {
|
||||
console.log("ERROR: ", JSON.stringify(e, null, 2));
|
||||
core.info(`fetching one more time`);
|
||||
release = await octokit.repos.getReleaseByTag({ owner, repo, tag: name });
|
||||
}
|
||||
}
|
||||
|
||||
// Creates an official GitHub release for this `tag`, and if this is `dev`
|
||||
// then we know that from the previous block this should be a fresh release.
|
||||
core.info(`creating a release`);
|
||||
const release = await octokit.repos.createRelease({
|
||||
owner,
|
||||
repo,
|
||||
tag_name: name,
|
||||
prerelease: name === 'dev',
|
||||
});
|
||||
console.log("found release: ", JSON.stringify(release.data, null, 2));
|
||||
|
||||
// Upload all the relevant assets for this release as just general blobs.
|
||||
for (const file of glob.sync(files)) {
|
||||
|
||||
143
.github/workflows/main.yml
vendored
143
.github/workflows/main.yml
vendored
@@ -394,6 +394,8 @@ jobs:
|
||||
build:
|
||||
name: Build wasmtime
|
||||
runs-on: ${{ matrix.os }}
|
||||
env:
|
||||
QEMU_VERSION: 5.0.0
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -434,30 +436,42 @@ jobs:
|
||||
rustup target add ${{ matrix.target }}
|
||||
if: matrix.target != ''
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ runner.tool_cache }}/qemu
|
||||
key: qemu-${{ matrix.target }}-${{ env.QEMU_VERSION }}
|
||||
if: matrix.target != '' && matrix.os == 'ubuntu-latest'
|
||||
|
||||
- name: Install cross-compilation tools
|
||||
run: |
|
||||
set -ex
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ${{ matrix.gcc_package }}
|
||||
|
||||
# Configure Cargo for cross compilation and tell it how it can run
|
||||
# cross executables
|
||||
upcase=$(echo ${{ matrix.target }} | awk '{ print toupper($0) }' | sed 's/-/_/g')
|
||||
echo CARGO_TARGET_${upcase}_RUNNER=${{ runner.tool_cache }}/qemu/bin/${{ matrix.qemu }} >> $GITHUB_ENV
|
||||
echo CARGO_TARGET_${upcase}_LINKER=${{ matrix.gcc }} >> $GITHUB_ENV
|
||||
|
||||
# See comments in the source for why we enable this during QEMU
|
||||
# emulation.
|
||||
echo WASMTIME_TEST_NO_HOG_MEMORY=1 >> $GITHUB_ENV
|
||||
|
||||
# See if qemu is already in the cache
|
||||
if [ -f ${{ runner.tool_cache }}/qemu/built ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Download and build qemu from source since the most recent release is
|
||||
# way faster at arm emulation than the current version github actions'
|
||||
# ubuntu image uses. Disable as much as we can to get it to build
|
||||
# quickly.
|
||||
curl https://download.qemu.org/qemu-5.0.0.tar.xz | tar xJf -
|
||||
cd qemu-5.0.0
|
||||
./configure --target-list=${{ matrix.qemu_target }} --prefix=$HOME/qemu --disable-tools --disable-slirp --disable-fdt --disable-capstone --disable-docs
|
||||
./configure --target-list=${{ matrix.qemu_target }} --prefix=${{ runner.tool_cache}}/qemu --disable-tools --disable-slirp --disable-fdt --disable-capstone --disable-docs
|
||||
make -j$(nproc) install
|
||||
|
||||
# Configure Cargo for cross compilation and tell it how it can run
|
||||
# cross executables
|
||||
upcase=$(echo ${{ matrix.target }} | awk '{ print toupper($0) }' | sed 's/-/_/g')
|
||||
echo CARGO_TARGET_${upcase}_RUNNER=$HOME/qemu/bin/${{ matrix.qemu }} >> $GITHUB_ENV
|
||||
echo CARGO_TARGET_${upcase}_LINKER=${{ matrix.gcc }} >> $GITHUB_ENV
|
||||
|
||||
# See comments in the source for why we enable this during QEMU
|
||||
# emulation.
|
||||
echo WASMTIME_TEST_NO_HOG_MEMORY=1 >> $GITHUB_ENV
|
||||
touch ${{ runner.tool_cache }}/qemu/built
|
||||
if: matrix.target != '' && matrix.os == 'ubuntu-latest'
|
||||
|
||||
# Install wasm32-wasi target in order to build wasi-common's integration
|
||||
@@ -487,122 +501,23 @@ jobs:
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
# Postprocess the macOS dylib a bit to have a more reasonable `LC_ID_DYLIB`
|
||||
# directive than the default one that comes out of the linker when typically
|
||||
# doing `cargo build`. For more info see #984
|
||||
- run: install_name_tool -id "@rpath/libwasmtime.dylib" target/release/libwasmtime.dylib
|
||||
if: matrix.os == 'macos-latest'
|
||||
|
||||
# ... and now perform some goop to move all the relevant artifacts into
|
||||
# something that we'll upload from this action.
|
||||
|
||||
- run: mkdir dist
|
||||
|
||||
# Move binaries to dist folder
|
||||
- run: cp target/release/wasmtime dist
|
||||
if: matrix.os != 'windows-latest' && matrix.target == ''
|
||||
- run: cp target/${{ matrix.target }}/release/wasmtime dist
|
||||
if: matrix.os != 'windows-latest' && matrix.target != ''
|
||||
- run: cp target/release/wasmtime.exe dist
|
||||
if: matrix.build == 'x86_64-windows'
|
||||
- run: cp target/x86_64-pc-windows-gnu/release/wasmtime.exe dist
|
||||
if: matrix.build == 'x86_64-mingw'
|
||||
|
||||
# Move libwasmtime dylib to dist folder
|
||||
- run: cp target/release/libwasmtime.{so,a} dist
|
||||
if: matrix.os == 'ubuntu-latest' && matrix.target == ''
|
||||
- run: cp target/${{ matrix.target }}/release/libwasmtime.{so,a} dist
|
||||
if: matrix.os == 'ubuntu-latest' && matrix.target != ''
|
||||
- run: cp target/release/libwasmtime.{dylib,a} dist
|
||||
if: matrix.os == 'macos-latest'
|
||||
- run: cp target/release/wasmtime.{dll,lib,dll.lib} dist
|
||||
if: matrix.build == 'x86_64-windows'
|
||||
- run: cp target/x86_64-pc-windows-gnu/release/{wasmtime.dll,libwasmtime.a} dist
|
||||
if: matrix.build == 'x86_64-mingw'
|
||||
|
||||
# Make a Windows MSI installer if we're on Windows
|
||||
- run: |
|
||||
export WT_VERSION=`cat Cargo.toml | sed -n 's/^version = "\([^"]*\)".*/\1/p'`
|
||||
"$WIX/bin/candle" -arch x64 -out target/wasmtime.wixobj ci/wasmtime.wxs
|
||||
"$WIX/bin/light" -out dist/installer.msi target/wasmtime.wixobj -ext WixUtilExtension
|
||||
rm dist/installer.wixpdb
|
||||
if: matrix.build == 'x86_64-windows'
|
||||
|
||||
# Assemble release artifats appropriate for this platform, then upload them
|
||||
# unconditionally to this workflow's files so we have a copy of them.
|
||||
- run: ./ci/build-tarballs.sh "${{ matrix.build }}" "${{ matrix.target }}"
|
||||
- uses: actions/upload-artifact@v1
|
||||
with:
|
||||
name: bins-${{ matrix.build }}
|
||||
path: dist
|
||||
|
||||
# Consumes all published artifacts from all the previous build steps, creates
|
||||
# a bunch of tarballs for all of them, and then publishes the tarballs
|
||||
# themselves as an artifact (for inspection) and then optionally creates
|
||||
# github releases and/or tags for pushes.
|
||||
publish:
|
||||
name: Publish
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- run: rustup update stable && rustup default stable
|
||||
|
||||
- name: Download x86_64 macOS binaries
|
||||
uses: actions/download-artifact@v1
|
||||
with:
|
||||
name: bins-x86_64-macos
|
||||
- name: Download x86_64 Linux binaries
|
||||
uses: actions/download-artifact@v1
|
||||
with:
|
||||
name: bins-x86_64-linux
|
||||
- name: Download AArch64 Linux binaries
|
||||
uses: actions/download-artifact@v1
|
||||
with:
|
||||
name: bins-aarch64-linux
|
||||
- name: Download x86_64 Windows binaries
|
||||
uses: actions/download-artifact@v1
|
||||
with:
|
||||
name: bins-x86_64-windows
|
||||
- name: Download x86_64 Windows MinGW binaries
|
||||
uses: actions/download-artifact@v1
|
||||
with:
|
||||
name: bins-x86_64-mingw
|
||||
|
||||
- name: Calculate tag name
|
||||
run: |
|
||||
name=dev
|
||||
if [[ $GITHUB_REF == refs/tags/v* ]]; then
|
||||
name=${GITHUB_REF:10}
|
||||
fi
|
||||
echo ::set-output name=val::$name
|
||||
echo TAG=$name >> $GITHUB_ENV
|
||||
id: tagname
|
||||
|
||||
# Assemble all the build artifacts into tarballs and zip archives.
|
||||
- name: Assemble tarballs
|
||||
run: |
|
||||
./ci/build-tarballs.sh x86_64-linux
|
||||
./ci/build-tarballs.sh x86_64-windows .exe
|
||||
./ci/build-tarballs.sh x86_64-mingw .exe
|
||||
./ci/build-tarballs.sh x86_64-macos
|
||||
./ci/build-tarballs.sh aarch64-linux
|
||||
|
||||
# Upload all assembled tarballs as an artifact of the github action run, so
|
||||
# that way even PRs can inspect the output.
|
||||
- uses: actions/upload-artifact@v1
|
||||
with:
|
||||
name: tarballs
|
||||
path: dist
|
||||
|
||||
# ... and if this was an actual push (tag or `main`) then we publish a
|
||||
# new release. This'll automatically publish a tag release or update `dev`
|
||||
# with this `sha`
|
||||
- run: cd .github/actions/github-release && npm install --production
|
||||
- name: Publish Release
|
||||
uses: ./.github/actions/github-release
|
||||
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
|
||||
with:
|
||||
files: "dist/*"
|
||||
name: ${{ steps.tagname.outputs.val }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
cargo-audit:
|
||||
|
||||
Reference in New Issue
Block a user