Files
wasmtime/.github/workflows/main.yml
Alex Crichton 2b0649c74c ci: Remove "publish" step (#2936)
This commit removes the publish step in GitHub actions, insteading
folding all functionality into the release build steps. This avoids
having a separately scheduled job after all the release build jobs which
ends up getting delayed for quite a long time given the current
scheduling algorithm.

This involves refactoring the tarball assembly scripts and refactoring the
github asset upload script too. Tarball assembly now manages everything
internally and does platform-specific bits where necessary. The upload
script is restructured to be run in parallel (in theory) and hopefully
catches various errors and tries to not stomp over everyone else's work.
The main trickiness here is handling `dev`, which is less critical for
correctness than than tags themselves.

As a small tweak build-wise the QEMU build for cross-compiled builders
is now cached unlike before where it was unconditionally built, shaving
a minute or two off build time.
2021-05-25 12:52:41 -05:00

553 lines
19 KiB
YAML

name: CI
on:
push:
branches: [main]
tags-ignore: [dev]
pull_request:
branches: [main]
defaults:
run:
shell: bash
# Cancel any in-flight jobs for the same PR/branch so there's only one active
# at a time
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# Check Code style quickly by running `rustfmt` over all code
rustfmt:
name: Rustfmt
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: ./.github/actions/install-rust
- run: rustup component add rustfmt
- run: cargo fmt --all -- --check
# Lint dependency graph for security advisories, duplicate versions, and
# incompatible licences
cargo_deny:
name: Cargo deny
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: ./.github/actions/install-rust
- run: |
set -e
curl -L https://github.com/EmbarkStudios/cargo-deny/releases/download/0.8.5/cargo-deny-0.8.5-x86_64-unknown-linux-musl.tar.gz | tar xzf -
mv cargo-deny-*-x86_64-unknown-linux-musl/cargo-deny cargo-deny
echo `pwd` >> $GITHUB_PATH
- run: cargo deny check
doc:
name: Doc build
runs-on: ubuntu-latest
env:
CARGO_MDBOOK_VERSION: 0.4.8
RUSTDOCFLAGS: -Dbroken_intra_doc_links --cfg nightlydoc
OPENVINO_SKIP_LINKING: 1
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: ./.github/actions/install-rust
with:
toolchain: nightly-2021-04-11
# Build C API documentation
- run: sudo apt-get update && sudo apt-get install -y doxygen git
- run: cd crates/c-api && doxygen doxygen.conf
# install mdbook, build the docs, and test the docs
- uses: actions/cache@v2
with:
path: ${{ runner.tool_cache }}/mdbook
key: cargo-mdbook-bin-${{ env.CARGO_MDBOOK_VERSION }}
- run: |
echo "${{ runner.tool_cache }}/mdbook/bin" >> $GITHUB_PATH
cargo install --root ${{ runner.tool_cache }}/mdbook --version ${{ env.CARGO_MDBOOK_VERSION }} mdbook
- run: (cd docs && mdbook build)
- run: cargo build -p wasmtime
- run: (cd docs && mdbook test -L ../target/debug/deps)
# Build Rust API documentation
- run: |
cargo doc --no-deps --workspace \
--exclude wasmtime-cli \
--exclude test-programs \
--exclude cranelift-codegen-meta \
--exclude 'peepmatic*'
- run: cargo doc --package cranelift-codegen-meta --document-private-items
# Assemble the documentation, and always upload it as an artifact for
# inspection on PRs and such.
- run: |
mv docs/book gh-pages
mv crates/c-api/html gh-pages/c-api
mv target/doc gh-pages/api
tar czf gh-pages.tar.gz gh-pages
- uses: actions/upload-artifact@v2
with:
name: gh-pages
path: gh-pages.tar.gz
# If this is a push to the main branch push to the `gh-pages` using a
# deploy key. Note that a deploy key is necessary for now because otherwise
# using the default token for github actions doesn't actually trigger a page
# rebuild.
- name: Push to gh-pages
run: curl -LsSf https://git.io/fhJ8n | rustc - && (cd gh-pages && ../rust_out)
env:
GITHUB_DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
BUILD_REPOSITORY_ID: ${{ github.repository }}
BUILD_SOURCEVERSION: ${{ github.sha }}
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
# Quick checks of various feature combinations and whether things
# compile. The goal here isn't to run tests, mostly just serve as a
# double-check that Rust code compiles and is likely to work everywhere else.
checks:
name: Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: ./.github/actions/install-rust
# Check some feature combinations of the `wasmtime` crate
- run: cargo check --manifest-path crates/wasmtime/Cargo.toml --no-default-features
- run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features wat
- run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features lightbeam
- run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features jitdump
- run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features cache
- run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features async
- run: cargo check --manifest-path crates/wasmtime/Cargo.toml --features uffd
# Check some feature combinations of the `wasmtime-c-api` crate
- run: cargo check --manifest-path crates/c-api/Cargo.toml --no-default-features
- run: cargo check --manifest-path crates/c-api/Cargo.toml --features wat
- run: cargo check --manifest-path crates/c-api/Cargo.toml --features wasi
# Check a few builds of the cranelift backend
# - only x86 backend support,
# - only arm64 backend support,
# - experimental arm32 support,
# - no debug_assertions.
- run: cargo check --manifest-path=./cranelift/Cargo.toml --bin clif-util --no-default-features --features=cranelift-codegen/arm64
- run: cargo check --manifest-path=./cranelift/Cargo.toml --bin clif-util --no-default-features --features=cranelift-codegen/x86
- run: cargo check --manifest-path=./cranelift/Cargo.toml --bin clif-util --no-default-features --features=cranelift-codegen/arm32
- run: cargo check --manifest-path=./cranelift/Cargo.toml --bin clif-util
env:
CARGO_PROFILE_DEV_DEBUG_ASSERTIONS: false
# Check whether `crates/wasi-common` cross-compiles to the following targets:
# * wasm32-unknown-emscripten
# * armv7-unknown-linux-gnueabihf
- run: |
rustup target add wasm32-unknown-emscripten
rustup target add armv7-unknown-linux-gnueabihf
- run: cargo check --target wasm32-unknown-emscripten -p wasi-common
- run: cargo check --target armv7-unknown-linux-gnueabihf -p wasi-common
# Check that codegen and wasm crates typecheck on 1.43.0; this is required
# for Firefox.
- run: rustup install 1.43.0
- run: cargo +1.43.0 check -p cranelift-codegen
- run: cargo +1.43.0 check -p cranelift-wasm
fuzz_targets:
name: Fuzz Targets
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
# Note that building with fuzzers requires nightly since it uses unstable
# flags to rustc.
- uses: ./.github/actions/install-rust
with:
toolchain: nightly-2021-04-11
- run: cargo install cargo-fuzz --vers "^0.8"
- run: cargo fetch
working-directory: ./fuzz
- run: cargo fuzz build --dev
rebuild_peephole_optimizers:
name: Rebuild Peephole Optimizers
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- run: rustup update stable && rustup default stable
- name: Test `peepmatic`
run: cargo test --package 'peepmatic*'
- name: Rebuild Peepmatic-based peephole optimizers
run: |
cargo test \
--features 'enable-peepmatic rebuild-peephole-optimizers' \
peepmatic
working-directory: ./cranelift/codegen
- name: Upload rebuilt peephole optimizers
uses: actions/upload-artifact@v2
with:
name: peephole-optimizers
path: cranelift/codegen/src/preopt.serialized
- name: Check that built peephole optimizers are up to date
run: git diff --exit-code
- name: Test with Peepmatic-based peephole optimizers
run: cargo test --features 'enable-peepmatic'
working-directory: ./cranelift
# Perform all tests (debug mode) for `wasmtime`. This runs stable/beta/nightly
# channels of Rust as well as macOS/Linux/Windows.
test:
name: Test
runs-on: ${{ matrix.os }}
strategy:
matrix:
build: [stable, beta, nightly, windows, macos]
include:
- build: stable
os: ubuntu-latest
rust: stable
- build: beta
os: ubuntu-latest
rust: beta
- build: nightly
os: ubuntu-latest
rust: nightly-2021-04-11
- build: macos
os: macos-latest
rust: stable
- build: windows
os: windows-latest
rust: stable
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: ./.github/actions/install-rust
with:
toolchain: ${{ matrix.rust }}
- name: Install libclang
# Note: libclang is pre-installed on the macOS and linux images.
if: matrix.os == 'windows-latest'
run: |
curl https://releases.llvm.org/9.0.0/LLVM-9.0.0-win64.exe -o llvm-installer.exe
7z x llvm-installer.exe -oC:/llvm-binary
echo LIBCLANG_PATH=C:/llvm-binary/bin/libclang.dll >> $GITHUB_ENV
echo C:/llvm-binary/bin >> $GITHUB_PATH
- name: Query Clang Version
if: matrix.os == 'windows-latest'
run: |
where clang.exe
clang.exe --version
# Install wasm32 targets in order to build various tests throughout the
# repo
- run: rustup target add wasm32-wasi
- run: rustup target add wasm32-unknown-unknown
- run: cargo fetch --locked
- run: cargo fetch --locked --manifest-path crates/test-programs/wasi-tests/Cargo.toml
# Ensure all our examples build and execute
- run: cargo run -p run-examples
env:
RUST_BACKTRACE: 1
# Build and test all features except for lightbeam
- run: |
cargo test \
--features test-programs/test_programs \
--workspace \
--exclude '*lightbeam*' \
--exclude 'wasmtime-wasi-*' \
--exclude 'peepmatic*'
env:
RUST_BACKTRACE: 1
# Test debug (DWARF) related functionality.
- run: |
sudo apt-get update && sudo apt-get install -y gdb lldb llvm
cargo test test_debug_dwarf -- --ignored --test-threads 1
if: matrix.os == 'ubuntu-latest'
env:
RUST_BACKTRACE: 1
# Test debug (DWARF) related functionality on new backend.
- run: |
cargo test test_debug_dwarf -- --ignored --test-threads 1 --test debug::
if: matrix.os == 'ubuntu-latest'
env:
RUST_BACKTRACE: 1
# Test uffd functionality on Linux
- run: |
cargo test --features uffd -p wasmtime-runtime instance::allocator::pooling
cargo test --features uffd -p wasmtime-cli pooling_allocator
cargo test --features uffd -p wasmtime-cli wast::Cranelift
if: matrix.os == 'ubuntu-latest'
env:
RUST_BACKTRACE: 1
# Build and test lightbeam. Note that
# Lightbeam tests fail right now, but we don't want to block on that.
- run: cargo build --package lightbeam
- run: cargo test --package lightbeam
continue-on-error: true
env:
RUST_BACKTRACE: 1
# Perform all tests (debug mode) for `wasmtime` with the old x86 backend.
test_x86:
name: Test old x86 backend
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: ./.github/actions/install-rust
with:
toolchain: stable
# Install wasm32 targets in order to build various tests throughout the
# repo.
- run: rustup target add wasm32-wasi
- run: rustup target add wasm32-unknown-unknown
# Run the old x86 backend CI (we will eventually remove this).
- run: ./ci/run-old-x86-ci.sh
env:
RUST_BACKTRACE: 1
# Build and test the wasi-nn module.
test_wasi_nn:
name: Test wasi-nn module
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: ./.github/actions/install-rust
- run: rustup target add wasm32-wasi
- uses: ./.github/actions/install-openvino
- run: ./ci/run-wasi-nn-example.sh
env:
RUST_BACKTRACE: 1
# Build and test the wasi-crypto module.
test_wasi_crypto:
name: Test wasi-crypto module
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- run: rustup target add wasm32-wasi
- name: Install Rust
run: rustup update stable && rustup default stable
- run: ./ci/run-wasi-crypto-example.sh
env:
RUST_BACKTRACE: 1
bench:
name: Run benchmarks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- run: rustup target add wasm32-wasi
- name: Install Rust
run: rustup update stable && rustup default stable
- run: cargo bench
- run: cargo bench --features uffd
# Verify that cranelift's code generation is deterministic
meta_determinist_check:
name: Meta deterministic check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Install Rust
run: rustup update stable && rustup default stable
- run: cd cranelift/codegen && cargo build --features all-arch
- run: ci/ensure_deterministic_build.sh
# Perform release builds of `wasmtime` and `libwasmtime.so`. Builds on
# Windows/Mac/Linux, and artifacts are uploaded after the build is finished.
# Note that we also run tests here to test exactly what we're deploying.
build:
name: Build wasmtime
runs-on: ${{ matrix.os }}
env:
QEMU_VERSION: 5.0.0
strategy:
matrix:
include:
- build: x86_64-linux
os: ubuntu-latest
rust: stable
- build: x86_64-macos
os: macos-latest
rust: stable
- build: x86_64-windows
os: windows-latest
rust: stable
- build: x86_64-mingw
os: windows-latest
target: x86_64-pc-windows-gnu
rust: beta # needs rust-lang/rust#69351 to ride to stable
- build: aarch64-linux
os: ubuntu-latest
rust: stable
target: aarch64-unknown-linux-gnu
gcc_package: gcc-aarch64-linux-gnu
gcc: aarch64-linux-gnu-gcc
qemu: qemu-aarch64 -L /usr/aarch64-linux-gnu
qemu_target: aarch64-linux-user
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: ./.github/actions/install-rust
with:
toolchain: ${{ matrix.rust }}
- uses: ./.github/actions/binary-compatible-builds
if: matrix.target == ''
- name: Configure Cargo target
run: |
echo CARGO_BUILD_TARGET=${{ matrix.target }} >> $GITHUB_ENV
rustup target add ${{ matrix.target }}
if: matrix.target != ''
- uses: actions/cache@v2
with:
path: ${{ runner.tool_cache }}/qemu
key: qemu-${{ matrix.target }}-${{ env.QEMU_VERSION }}
if: matrix.target != '' && matrix.os == 'ubuntu-latest'
- name: Install cross-compilation tools
run: |
set -ex
sudo apt-get update
sudo apt-get install -y ${{ matrix.gcc_package }}
# Configure Cargo for cross compilation and tell it how it can run
# cross executables
upcase=$(echo ${{ matrix.target }} | awk '{ print toupper($0) }' | sed 's/-/_/g')
echo CARGO_TARGET_${upcase}_RUNNER=${{ runner.tool_cache }}/qemu/bin/${{ matrix.qemu }} >> $GITHUB_ENV
echo CARGO_TARGET_${upcase}_LINKER=${{ matrix.gcc }} >> $GITHUB_ENV
# See comments in the source for why we enable this during QEMU
# emulation.
echo WASMTIME_TEST_NO_HOG_MEMORY=1 >> $GITHUB_ENV
# See if qemu is already in the cache
if [ -f ${{ runner.tool_cache }}/qemu/built ]; then
exit 0
fi
# Download and build qemu from source since the most recent release is
# way faster at arm emulation than the current version github actions'
# ubuntu image uses. Disable as much as we can to get it to build
# quickly.
curl https://download.qemu.org/qemu-5.0.0.tar.xz | tar xJf -
cd qemu-5.0.0
./configure --target-list=${{ matrix.qemu_target }} --prefix=${{ runner.tool_cache}}/qemu --disable-tools --disable-slirp --disable-fdt --disable-capstone --disable-docs
make -j$(nproc) install
touch ${{ runner.tool_cache }}/qemu/built
if: matrix.target != '' && matrix.os == 'ubuntu-latest'
# Install wasm32-wasi target in order to build wasi-common's integration
# tests
- run: rustup target add wasm32-wasi
# Build `wasmtime` and executables
- run: $CENTOS cargo build --release --bin wasmtime
# Build `libwasmtime.so`
- run: $CENTOS cargo build --release --manifest-path crates/c-api/Cargo.toml
# Test what we just built.
#
# Ignore some optional dependencies that are used by features that aren't
# enabled by default, are tested in other, dedicated CI jobs, and which
# would increase build times here.
- run: |
$CENTOS cargo test \
--features test-programs/test_programs \
--release \
--workspace \
--exclude '*lightbeam*' \
--exclude 'wasmtime-wasi-*' \
--exclude 'peepmatic*' \
--exclude wasmtime-fuzz
env:
RUST_BACKTRACE: 1
# Assemble release artifats appropriate for this platform, then upload them
# unconditionally to this workflow's files so we have a copy of them.
- run: ./ci/build-tarballs.sh "${{ matrix.build }}" "${{ matrix.target }}"
- uses: actions/upload-artifact@v1
with:
name: bins-${{ matrix.build }}
path: dist
# ... and if this was an actual push (tag or `main`) then we publish a
# new release. This'll automatically publish a tag release or update `dev`
# with this `sha`
- run: cd .github/actions/github-release && npm install --production
- name: Publish Release
uses: ./.github/actions/github-release
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
with:
files: "dist/*"
token: ${{ secrets.GITHUB_TOKEN }}
cargo-audit:
env:
CARGO_AUDIT_VERSION: 0.11.2
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/cache@v1
with:
path: ${{ runner.tool_cache }}/cargo-audit
key: cargo-audit-bin-${{ env.CARGO_AUDIT_VERSION }}
- run: echo "${{ runner.tool_cache }}/cargo-audit/bin" >> $GITHUB_PATH
- run: |
cargo install --root ${{ runner.tool_cache }}/cargo-audit --version ${{ env.CARGO_AUDIT_VERSION }} cargo-audit
cargo audit
verify-publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- run: rustup update stable && rustup default stable
- run: |
cd ${{ runner.tool_cache }}
curl -L https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz | tar xzf -
echo "`pwd`/sccache-0.2.13-x86_64-unknown-linux-musl" >> $GITHUB_PATH
echo RUSTC_WRAPPER=sccache >> $GITHUB_ENV
- run: |
rustc scripts/publish.rs
./publish verify