wasmtime/.github/workflows/performance.yml

# This is a workflow triggered by PR or triggered manually
# Runs quick performance tests and reports the comparison against HEAD
# Test should take less than 10 minutes to run on current self-hosted devices
name: "Performance Testing"

# Controls when the action will run.
# This workflow runs when manually triggered by keywords used in the start of a review comment
# Currently that phrase is /bench_x64. /bench_aarch64 and /bench_all are TODOs.
on:
  issue_comment:
    types: [created]
  push:

# Env variables
env:
  SG_COMMIT: b4971ae
  GITHUB_CONTEXT: ${{ toJson(github) }}

jobs:
  Wasmtime_Repo_On_PR_Comment:
    name: Benchmark x64 on PR comment Wasmtime repo
    runs-on: ubuntu-latest
    if: |
      (github.event_name == 'issue_comment') &&
      (github.event.issue.pull_request.url) &&
      (contains(github.event.comment.body, '/bench_x64')) &&
      (('abrown' == github.event.comment.user.login)
        || ('afonso360' == github.event.comment.user.login)
        || ('akirilov-arm' == github.event.comment.user.login)
        || ('alexcrichton' == github.event.comment.user.login)
        || ('bbouvier' == github.event.comment.user.login)
        || ('bjorn3' == github.event.comment.user.login)
        || ('cfallin' == github.event.comment.user.login)
        || ('fitzgen' == github.event.comment.user.login)
        || ('jlb6740' == github.event.comment.user.login)
        || ('sparker-arm' == github.event.comment.user.login)
        || ('uweigand' == github.event.comment.user.login))
    steps:
      - run: echo "$GITHUB_CONTEXT"
      - run: |
          # Create and Push Branch
          git clone https://wasmtime-publish:${{secrets.PERSONAL_ACCESS_TOKEN}}@github.com/bytecodealliance/wasmtime-sightglass-benchmarking.git
          cd wasmtime-sightglass-benchmarking
          git remote add wasmtime ${{ github.event.repository.clone_url }}
          git fetch wasmtime refs/pull/*/merge:refs/remotes/wasmtime/pull/*/merge
          export issue_pr_url=${{ github.event.issue.pull_request.url }}
          export issue_commits_url=${{ github.event.issue.comments_url }}
          export issue_ref_name=$(curl -sSL $issue_pr_url | jq -r '.head.ref' | head -n 1)
          export issue_number=$(curl -sSL $issue_pr_url | jq -r '.number' | head -n 1)
          export issue_merge_commit_sha=$(curl -sSL $issue_pr_url | jq -r '.merge_commit_sha' | head -n 1)
          git submodule update --init --recursive
          git checkout wasmtime/pull/${issue_number}/merge -b pull/${issue_number}/merge/${issue_merge_commit_sha}
          git config user.name $(curl -sSL $issue_commits_url | jq -r '.[].commit.committer.name' | tail -n 1)
          git config user.email $(curl -sSL $issue_commits_url | jq -r '.[].commit.committer.email' | tail -n 1)
          git log -n 1
          git commit --allow-empty -m "${issue_commits_url}"
          git push origin --force pull/${issue_number}/merge/${issue_merge_commit_sha}
          git log -n 1

  Performance_Repo_On_Push:
    name: Benchmark x64 on push Performance repo
    runs-on: [self-hosted, linux, x64]
    if: (github.event_name == 'push') && (github.repository == 'bytecodealliance/wasmtime-sightglass-benchmarking')
    steps:
      - run: echo "$GITHUB_CONTEXT"
      - run: echo "${{ github.event.head_commit.message }}"
      - name: "Build sightglass commit '${{ env.SG_COMMIT }}'"
        run: |
          cd ../ && ls -l && rm -rf ./sightglass
          git clone https://github.com/bytecodealliance/sightglass.git && cd ./sightglass
          git checkout ${{env.SG_COMMIT}}
          cargo build --release

      - name: Checkout patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
        uses: actions/checkout@v3
        with:
          submodules: true
          path: wasmtime_commit

      - name: Get latest rust toolchain
        uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly
          override: true

      - name: Build patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
        working-directory: ./wasmtime_commit
        run: |
          cargo --version
          cargo build --release -p wasmtime-bench-api
          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_commit.so

      - name: Checkout main from bytecodealliance/wasmtime
        uses: actions/checkout@v3
        with:
          ref: 'main'
          repository: 'bytecodealliance/wasmtime'
          submodules: true
          path: wasmtime_main

      - name: Build main from bytecodealliance/wasmtime
        working-directory: ./wasmtime_main
        run: |
          cargo build --release -p wasmtime-bench-api
          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_main.so

      - name: Run performance tests
        working-directory: ../sightglass
        run: |
          cargo run -- \
          benchmark \
          --processes 1 \
          --iterations-per-process 2 \
          --engine /tmp/wasmtime_main.so \
          --engine /tmp/wasmtime_commit.so \
          --output-format csv \
          --output-file /tmp/results.csv \
          --raw \
          -- benchmarks/*/benchmark.wasm
          ./target/release/sightglass-cli summarize --input-format csv  --output-format csv -f /tmp/results.csv > /tmp/results_summarized.csv

      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.9'

      - name: Post Process Results
        run: |
          pip3 install pandas numpy
          grep -v "nanoseconds" /tmp/results_summarized.csv > /tmp/results_cycles_summarized.csv
          sed -i 's/\/tmp\/wasmtime_commit.so/patch/g' /tmp/results_cycles_summarized.csv
          sed -i 's/\/tmp\/wasmtime_main.so/main/g' /tmp/results_cycles_summarized.csv
          sed -i 's/benchmarks-next\///g'  /tmp/results_cycles_summarized.csv
          sed -i 's/\/benchmark.wasm//g'  /tmp/results_cycles_summarized.csv
          python3 -c "import pandas as pd; pp = pd.read_csv('/tmp/results_cycles_summarized.csv', \
          usecols=['arch','engine','wasm', 'phase', 'mean'], header=0); \
          pp_sorted = pp.sort_values(['wasm', 'phase', 'engine'], ascending=True); \
          pp_pct_changed=pp_sorted.groupby(['wasm','phase'])['mean'].pct_change().reset_index().rename(columns = {'mean':'pct_change'}); \
          pp_sorted.index.name = 'index'; \
          pp_sorted_merged=pp_sorted.merge(pp_pct_changed, on='index'); \
          pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \
          pp_sorted_merged=pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \
          pp_sorted_merged=pp_sorted_merged[['wasm','arch','phase','pct_change']]; \
          print(pp_sorted_merged.to_string(index=False));" >  /tmp/results_cycles_summarized_sorted2.csv
          sed -i 's/^/ /' /tmp/results_cycles_summarized_sorted2.csv
          sed -i 's/ \+/|/g' /tmp/results_cycles_summarized_sorted2.csv
          sed -i -z 's/\n/|\n/g' /tmp/results_cycles_summarized_sorted2.csv
          sed -i '2 i\ |-|-|-|-|' /tmp/results_cycles_summarized_sorted2.csv
          sed -i '/main/d' /tmp/results_cycles_summarized_sorted2.csv
          sed -i '1 i\Shows pct_change on x64 for the patch if merged compared to current head for main.\n\
          Pct_change is based on clocktick event cycles where the benchmarks are run with Sightglass. \
          A negative pct_change means clockticks are expected to be reduced for the benchmark, \
          for that phase, and by that factor, if the patch were merged (i.e. negative is good).\n' /tmp/results_cycles_summarized_sorted2.csv

      - name: Print Results
        run: cat /tmp/results_cycles_summarized_sorted2.csv

      - id: get-comment-body
        name: Create Results Body
        run: |
            body="$(cat /tmp/results_cycles_summarized_sorted2.csv)"
            body="${body//'%'/'%25'}"
            body="${body//$'\n'/'%0A'}"
            body="${body//$'\r'/'%0D'}"
            echo "body=$body" >> $GITHUB_OUTPUT

      - name: Publish Results
        run: |
          curl -X POST -H "Accept: application/vnd.github.v3+json" \
          -H "Authorization: token ${{ secrets.WASMTIME_PUBLISHING_TOKEN }}" \
          ${{ github.event.head_commit.message }} \
          -d '{"body": ${{ toJSON(steps.get-comment-body.outputs.body) }}}'