Benchmarking #8
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: OpenMP PR Benchmark | |
| on: | |
| pull_request: | |
| branches: | |
| - "main" | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: openmp-pr-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| env: | |
| ASV_CONFIG: asv.openmp.conf.json | |
| MACHINE_NAME: github-actions-ubuntu | |
| jobs: | |
| benchmark_changes: | |
| name: Detect benchmark-relevant changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| required: ${{ steps.changes.outputs.required }} | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect benchmark-relevant changes | |
| id: changes | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| if git diff --quiet "$BASE_SHA" "$HEAD_SHA" -- \ | |
| src \ | |
| benchmarks \ | |
| pyproject.toml \ | |
| asv.conf.json \ | |
| asv.openmp.conf.json \ | |
| .github/workflows/asv-benchmarks.yml \ | |
| .github/workflows/asv-openmp-pr.yml \ | |
| .github/workflows/asv-openmp-publish.yml | |
| then | |
| echo "required=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "required=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| benchmark: | |
| name: Full OpenMP comparison on Linux | |
| if: needs.benchmark_changes.outputs.required == 'true' | |
| needs: benchmark_changes | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| fetch-depth: 0 | |
| - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 | |
| with: | |
| activate-environment: gstools-asv-driver | |
| python-version: "3.14" | |
| channels: conda-forge | |
| conda-remove-defaults: true | |
| - name: Install ASV driver | |
| run: | | |
| conda install -n gstools-asv-driver -y -c conda-forge pip | |
| python -m pip install --upgrade pip asv | |
| - name: Configure PR commits and ASV branches | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| printf '%s\n%s\n' "$BASE_SHA" "$HEAD_SHA" \ | |
| > "$RUNNER_TEMP/asv-pr-commits.txt" | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| config = json.loads(Path("asv.openmp.conf.json").read_text()) | |
| config["branches"] = [os.environ["BASE_SHA"], os.environ["HEAD_SHA"]] | |
| Path("asv.openmp.pr.conf.json").write_text( | |
| json.dumps(config, indent=2), | |
| encoding="utf8", | |
| ) | |
| PY | |
| echo "ASV_CONFIG=asv.openmp.pr.conf.json" >> "$GITHUB_ENV" | |
| - name: Configure stable ASV machine name | |
| run: | | |
| python benchmarks/tools/configure_asv_machine.py "$MACHINE_NAME" | |
| - name: Run full OpenMP comparison | |
| env: | |
| GSTOOLS_BENCHMARK_THREADS: "1,2,4,8" | |
| run: | | |
| asv --config "$ASV_CONFIG" run \ | |
| "HASHFILE:$RUNNER_TEMP/asv-pr-commits.txt" \ | |
| --machine "$MACHINE_NAME" \ | |
| --interleave-rounds \ | |
| --no-pull \ | |
| --bench "^benchmark_" \ | |
| --show-stderr | |
| - name: Write informational ASV comparison | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| mkdir -p .asv-openmp/artifact | |
| set +e | |
| asv --config "$ASV_CONFIG" compare \ | |
| "$BASE_SHA" \ | |
| "$HEAD_SHA" \ | |
| --machine "$MACHINE_NAME" \ | |
| --split \ | |
| | tee .asv-openmp/artifact/comparison.txt | |
| compare_status="${PIPESTATUS[0]}" | |
| set -e | |
| echo "ASV comparison exit status: $compare_status" \ | |
| >> .asv-openmp/artifact/comparison.txt | |
| - name: Build custom and native ASV reports | |
| run: | | |
| python benchmarks/tools/plot_case_backend_comparison.py \ | |
| --results-dir .asv-openmp/results \ | |
| --max-commits 50 \ | |
| --output .asv-openmp/artifact/index.html | |
| asv --config "$ASV_CONFIG" publish \ | |
| --no-pull \ | |
| --html-dir .asv-openmp/artifact/asv | |
| cp -R .asv-openmp/results .asv-openmp/artifact/results | |
| - name: Upload OpenMP benchmark reports | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: openmp-pr-${{ github.event.pull_request.number }} | |
| path: .asv-openmp/artifact | |
| if-no-files-found: warn | |
| retention-days: 30 | |
| required_gate: | |
| name: OpenMP benchmark gate | |
| if: always() | |
| needs: | |
| - benchmark_changes | |
| - benchmark | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check required benchmark result | |
| env: | |
| CHANGE_RESULT: ${{ needs.benchmark_changes.result }} | |
| BENCHMARK_REQUIRED: ${{ needs.benchmark_changes.outputs.required }} | |
| BENCHMARK_RESULT: ${{ needs.benchmark.result }} | |
| run: | | |
| if [[ "$CHANGE_RESULT" != "success" ]]; then | |
| echo "Could not determine whether the benchmark is required." | |
| exit 1 | |
| fi | |
| if [[ "$BENCHMARK_REQUIRED" == "true" \ | |
| && "$BENCHMARK_RESULT" != "success" ]]; then | |
| echo "The required OpenMP benchmark did not complete successfully." | |
| exit 1 | |
| fi | |
| if [[ "$BENCHMARK_REQUIRED" == "true" ]]; then | |
| echo "The required OpenMP benchmark completed successfully." | |
| else | |
| echo "No benchmark-relevant files changed." | |
| fi |