Benchmark #9
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| model: | |
| description: "Model name (e.g., nvidia/nemotron-3-super-120b-a12b:free)" | |
| required: true | |
| type: string | |
| provider: | |
| description: "Provider" | |
| required: true | |
| default: "openai" | |
| type: choice | |
| options: | |
| - openai | |
| - anthropic | |
| base_url: | |
| description: "Provider base URL (e.g., https://openrouter.ai/api/v1)" | |
| required: false | |
| type: string | |
| runs: | |
| description: "Number of paired runs (N>=4 for statistical validity)" | |
| required: false | |
| default: "4" | |
| type: string | |
| schedule: | |
| - cron: "0 6 * * 1" # Weekly Monday 6am UTC | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| # Full Docker benchmarks require a self-hosted runner with: | |
| # - Docker installed + pre-built rle-headless image | |
| # - LLM provider API key in secrets | |
| # This workflow runs smoke-test on GitHub-hosted runners. | |
| # Docker steps are commented out as a template. | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.14" | |
| - uses: astral-sh/setup-uv@v4 | |
| - run: uv pip install -e ".[dev]" --system | |
| - name: Smoke test | |
| run: python scripts/run_benchmark.py --dry-run --ticks 5 | |
| # --- Docker benchmark (uncomment when self-hosted runner is configured) --- | |
| # | |
| # - name: Start HeadlessRim container | |
| # run: docker compose -f docker/docker-compose.yml up -d | |
| # | |
| # - name: Wait for RIMAPI | |
| # run: | | |
| # for i in $(seq 1 24); do | |
| # curl -sf http://localhost:8765/api/v1/game/state && exit 0 | |
| # sleep 5 | |
| # done | |
| # exit 1 | |
| # | |
| # - name: Run benchmark | |
| # env: | |
| # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| # run: | | |
| # python scripts/run_benchmark.py \ | |
| # --docker \ | |
| # --provider ${{ inputs.provider || 'openai' }} \ | |
| # --model "${{ inputs.model || 'nvidia/nemotron-3-super-120b-a12b:free' }}" \ | |
| # ${{ inputs.base_url && format('--base-url {0}', inputs.base_url) || '' }} \ | |
| # --runs ${{ inputs.runs || '4' }} \ | |
| # --no-think \ | |
| # --output results/ci/ | |
| # | |
| # - name: Stop container | |
| # if: always() | |
| # run: docker compose -f docker/docker-compose.yml down | |
| # | |
| # - name: Upload results | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: benchmark-results | |
| # path: results/ci/ |