Benchmarking #14
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarking | |
| on: | |
| pull_request: | |
| branches: | |
| - "main" | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: openmp-pr-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| env: | |
| ASV_CONFIG: asv.openmp.conf.json | |
| MACHINE_NAME: github-actions-ubuntu | |
| jobs: | |
| benchmark_changes: | |
| name: Detect benchmark-relevant changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| required: ${{ steps.changes.outputs.required }} | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect benchmark-relevant changes | |
| id: changes | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| if git diff --quiet "$BASE_SHA" "$HEAD_SHA" -- \ | |
| src \ | |
| benchmarks \ | |
| pyproject.toml \ | |
| asv.conf.json \ | |
| asv.openmp.conf.json \ | |
| .github/workflows/asv-benchmarks.yml \ | |
| .github/workflows/asv-openmp-pr.yml \ | |
| .github/workflows/asv-openmp-publish.yml | |
| then | |
| echo "required=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "required=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| benchmark: | |
| name: Full OpenMP comparison on Linux | |
| if: needs.benchmark_changes.outputs.required == 'true' | |
| needs: benchmark_changes | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| fetch-depth: 0 | |
| - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 | |
| with: | |
| activate-environment: gstools-asv-driver | |
| python-version: "3.14" | |
| channels: conda-forge | |
| conda-remove-defaults: true | |
| - name: Install ASV driver | |
| run: | | |
| conda install -n gstools-asv-driver -y -c conda-forge pip | |
| python -m pip install --upgrade pip asv | |
| - name: Configure PR commits and ASV branches | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| printf '%s\n%s\n' "$BASE_SHA" "$HEAD_SHA" \ | |
| > "$RUNNER_TEMP/asv-pr-commits.txt" | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| config = json.loads(Path("asv.openmp.conf.json").read_text()) | |
| config["branches"] = [os.environ["BASE_SHA"], os.environ["HEAD_SHA"]] | |
| Path("asv.openmp.pr.conf.json").write_text( | |
| json.dumps(config, indent=2), | |
| encoding="utf8", | |
| ) | |
| PY | |
| echo "ASV_CONFIG=asv.openmp.pr.conf.json" >> "$GITHUB_ENV" | |
| - name: Configure stable ASV machine name | |
| run: | | |
| python benchmarks/tools/configure_asv_machine.py "$MACHINE_NAME" | |
| - name: Run full OpenMP comparison | |
| env: | |
| GSTOOLS_BENCHMARK_THREADS: "1,2,4,8" | |
| run: | | |
| asv --config "$ASV_CONFIG" run \ | |
| "HASHFILE:$RUNNER_TEMP/asv-pr-commits.txt" \ | |
| --machine "$MACHINE_NAME" \ | |
| --interleave-rounds \ | |
| --no-pull \ | |
| --bench "^benchmark_" \ | |
| --show-stderr | |
| - name: Write informational ASV comparison | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| mkdir -p .asv-openmp/artifact | |
| set +e | |
| asv --config "$ASV_CONFIG" compare \ | |
| "$BASE_SHA" \ | |
| "$HEAD_SHA" \ | |
| --machine "$MACHINE_NAME" \ | |
| --factor 1.05 \ | |
| --split \ | |
| | tee .asv-openmp/artifact/comparison.txt | |
| compare_status="${PIPESTATUS[0]}" | |
| set -e | |
| echo "ASV comparison exit status: $compare_status" \ | |
| >> .asv-openmp/artifact/comparison.txt | |
| - name: Build custom and native ASV reports | |
| run: | | |
| python benchmarks/tools/plot_case_backend_comparison.py \ | |
| --results-dir .asv-openmp/results \ | |
| --output .asv-openmp/artifact/index.html | |
| asv --config "$ASV_CONFIG" publish \ | |
| --no-pull \ | |
| --html-dir .asv-openmp/artifact/asv | |
| cp -R .asv-openmp/results .asv-openmp/artifact/results | |
| - name: Upload OpenMP benchmark reports | |
| id: upload | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: openmp-pr-${{ github.event.pull_request.number }} | |
| path: .asv-openmp/artifact | |
| if-no-files-found: warn | |
| retention-days: 30 | |
| - name: Write benchmark step summary | |
| if: always() | |
| env: | |
| ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| python - <<'PY' | |
| import os | |
| from pathlib import Path | |
| base = os.environ["BASE_SHA"][:8] | |
| head = os.environ["HEAD_SHA"][:8] | |
| artifact_url = os.environ.get("ARTIFACT_URL", "") | |
| report_link = ( | |
| f" · [Full HTML Report ↗]({artifact_url})" if artifact_url else "" | |
| ) | |
| try: | |
| comparison = Path(".asv-openmp/artifact/comparison.txt").read_text() | |
| except FileNotFoundError: | |
| comparison = "_Comparison output not available._" | |
| lines = comparison.splitlines() | |
| regressed = sum( | |
| 1 for l in lines if l.startswith("+") and l[1:2] == " " | |
| ) | |
| improved = sum( | |
| 1 for l in lines if l.startswith("-") and l[1:2] == " " | |
| ) | |
| if regressed: | |
| badge = f"⚠️ {regressed} benchmark(s) regressed" | |
| elif improved: | |
| badge = f"✅ {improved} benchmark(s) improved, none regressed" | |
| else: | |
| badge = "✅ No significant changes detected" | |
| body = ( | |
| "<!-- gstools-openmp-benchmark -->\n" | |
| "## OpenMP Benchmark Results\n\n" | |
| f"**Base:** `{base}` → **Head:** `{head}`{report_link}\n\n" | |
| f"{badge}\n\n" | |
| "<details>\n<summary>Full ASV comparison</summary>\n\n" | |
| f"```\n{comparison}\n```\n\n" | |
| "</details>\n" | |
| ) | |
| Path("/tmp/benchmark_comment.md").write_text(body) | |
| summary_env = os.environ.get("GITHUB_STEP_SUMMARY", "") | |
| if summary_env: | |
| Path(summary_env).write_text( | |
| body.replace("<!-- gstools-openmp-benchmark -->\n", "") | |
| ) | |
| PY | |
| - name: Save PR context for comment workflow | |
| if: always() | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} | |
| run: | | |
| mkdir -p .asv-openmp/pr-context | |
| printf '%s' "$PR_NUMBER" > .asv-openmp/pr-context/pr_number.txt | |
| printf '%s' "$BASE_SHA" > .asv-openmp/pr-context/base_sha.txt | |
| printf '%s' "$HEAD_SHA" > .asv-openmp/pr-context/head_sha.txt | |
| printf '%s' "$ARTIFACT_URL" > .asv-openmp/pr-context/artifact_url.txt | |
| cp .asv-openmp/artifact/comparison.txt \ | |
| .asv-openmp/pr-context/comparison.txt 2>/dev/null \ | |
| || printf '_Comparison output not available._' \ | |
| > .asv-openmp/pr-context/comparison.txt | |
| - name: Upload PR context for comment workflow | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: benchmark-pr-context | |
| path: .asv-openmp/pr-context | |
| if-no-files-found: warn | |
| retention-days: 1 | |
| required_gate: | |
| name: OpenMP benchmark gate | |
| if: always() | |
| needs: | |
| - benchmark_changes | |
| - benchmark | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check required benchmark result | |
| env: | |
| CHANGE_RESULT: ${{ needs.benchmark_changes.result }} | |
| BENCHMARK_REQUIRED: ${{ needs.benchmark_changes.outputs.required }} | |
| BENCHMARK_RESULT: ${{ needs.benchmark.result }} | |
| run: | | |
| if [[ "$CHANGE_RESULT" != "success" ]]; then | |
| echo "Could not determine whether the benchmark is required." | |
| exit 1 | |
| fi | |
| if [[ "$BENCHMARK_REQUIRED" == "true" \ | |
| && "$BENCHMARK_RESULT" != "success" ]]; then | |
| echo "The required OpenMP benchmark did not complete successfully." | |
| exit 1 | |
| fi | |
| if [[ "$BENCHMARK_REQUIRED" == "true" ]]; then | |
| echo "The required OpenMP benchmark completed successfully." | |
| else | |
| echo "No benchmark-relevant files changed." | |
| fi |