diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index b108608bb0..54308bfc56 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -13,7 +13,7 @@ on: # Env variables env: - SG_COMMIT: b4971ae + SG_COMMIT: 2ab01ac GITHUB_CONTEXT: ${{ toJson(github) }} jobs: @@ -105,60 +105,23 @@ jobs: run: | cargo run -- \ benchmark \ - --processes 1 \ - --iterations-per-process 2 \ + --processes 5 \ + --iterations-per-process 5 \ --engine /tmp/wasmtime_main.so \ --engine /tmp/wasmtime_commit.so \ - --output-format csv \ - --output-file /tmp/results.csv \ - --raw \ - -- benchmarks/*/benchmark.wasm - ./target/release/sightglass-cli summarize --input-format csv --output-format csv -f /tmp/results.csv > /tmp/results_summarized.csv - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - name: Post Process Results - run: | - pip3 install pandas numpy - grep -v "nanoseconds" /tmp/results_summarized.csv > /tmp/results_cycles_summarized.csv - sed -i 's/\/tmp\/wasmtime_commit.so/patch/g' /tmp/results_cycles_summarized.csv - sed -i 's/\/tmp\/wasmtime_main.so/main/g' /tmp/results_cycles_summarized.csv - sed -i 's/benchmarks-next\///g' /tmp/results_cycles_summarized.csv - sed -i 's/\/benchmark.wasm//g' /tmp/results_cycles_summarized.csv - python3 -c "import pandas as pd; pp = pd.read_csv('/tmp/results_cycles_summarized.csv', \ - usecols=['arch','engine','wasm', 'phase', 'mean'], header=0); \ - pp_sorted = pp.sort_values(['wasm', 'phase', 'engine'], ascending=True); \ - pp_pct_changed=pp_sorted.groupby(['wasm','phase'])['mean'].pct_change().reset_index().rename(columns = {'mean':'pct_change'}); \ - pp_sorted.index.name = 'index'; \ - pp_sorted_merged=pp_sorted.merge(pp_pct_changed, on='index'); \ - pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \ - pp_sorted_merged=pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \ - pp_sorted_merged=pp_sorted_merged[['wasm','arch','phase','pct_change']]; \ - print(pp_sorted_merged.to_string(index=False));" > /tmp/results_cycles_summarized_sorted2.csv - sed -i 's/^/ /' /tmp/results_cycles_summarized_sorted2.csv - sed -i 's/ \+/|/g' /tmp/results_cycles_summarized_sorted2.csv - sed -i -z 's/\n/|\n/g' /tmp/results_cycles_summarized_sorted2.csv - sed -i '2 i\ |-|-|-|-|' /tmp/results_cycles_summarized_sorted2.csv - sed -i '/main/d' /tmp/results_cycles_summarized_sorted2.csv - sed -i '1 i\Shows pct_change on x64 for the patch if merged compared to current head for main.\n\ - Pct_change is based on clocktick event cycles where the benchmarks are run with Sightglass. \ - A negative pct_change means clockticks are expected to be reduced for the benchmark, \ - for that phase, and by that factor, if the patch were merged (i.e. negative is good).\n' /tmp/results_cycles_summarized_sorted2.csv + --output-file /tmp/results.txt - name: Print Results - run: cat /tmp/results_cycles_summarized_sorted2.csv + run: cat /tmp/results.txt - id: get-comment-body name: Create Results Body run: | - body="$(cat /tmp/results_cycles_summarized_sorted2.csv)" + body="$(cat /tmp/results.txt)" body="${body//'%'/'%25'}" body="${body//$'\n'/'%0A'}" body="${body//$'\r'/'%0D'}" - echo "body=$body" >> $GITHUB_OUTPUT + echo "::set-output name=body::$body" - name: Publish Results run: |