formula-health #189
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: formula-health | |
| # Periodic health check for fontist formulas | |
| # Runs every other day to validate formula integrity | |
| on: | |
| push: | |
| branches: | |
| - v5 | |
| paths: | |
| - '.github/workflows/formula-health.yml' | |
| - '.github/scripts/**' | |
| schedule: | |
| # Tier 1: Fast validation - Daily at 6am UTC | |
| - cron: '0 6 * * *' | |
| # Tier 2: Rotation installation - Every other day at 7am UTC | |
| - cron: '0 7 */2 * *' | |
| # Tier 2: Full scan - Weekly on Monday at 8am UTC | |
| - cron: '0 8 * * 1' | |
| workflow_dispatch: | |
| inputs: | |
| tier: | |
| description: 'Which tier to run' | |
| required: true | |
| default: 'both' | |
| type: choice | |
| options: | |
| - both | |
| - tier1-validation | |
| - tier2-installation | |
| - tier2-full-scan | |
| group: | |
| description: 'Formula group to test (for tier 2)' | |
| required: false | |
| default: '' | |
| type: choice | |
| options: | |
| - '' | |
| - sil | |
| - macos | |
| - other | |
| sample_size: | |
| description: 'Sample N formulas (for testing)' | |
| required: false | |
| default: '' | |
| type: string | |
| env: | |
| RUBY_VERSION: '3.3' | |
| jobs: | |
| # =========================================================================== | |
| # TIER 1: Fast Validation | |
| # =========================================================================== | |
| tier1-validation: | |
| name: Tier 1 - Schema & URL Validation | |
| runs-on: ubuntu-latest | |
| # Run daily at 6am UTC, or when manually triggered | |
| if: | | |
| (github.event_name == 'schedule' && | |
| github.event.schedule == '0 6 * * *') || | |
| (github.event_name == 'workflow_dispatch' && | |
| (github.event.inputs.tier == 'both' || | |
| github.event.inputs.tier == 'tier1-validation')) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ${{ env.RUBY_VERSION }} | |
| bundler-cache: true | |
| - name: Install concurrent-ruby for URL checking | |
| run: gem install concurrent-ruby | |
| - name: Schema Validation | |
| id: schema | |
| run: | | |
| mkdir -p results | |
| echo "::group::Schema Validation" | |
| ruby .github/scripts/validate_schema.rb \ | |
| --directory Formulas \ | |
| --json-output results/schema.json 2>&1 | tee schema-results.txt | |
| echo "::endgroup::" | |
| # Check for failures | |
| if grep -q "Schema validation FAILED" schema-results.txt; then | |
| echo "schema_passed=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "schema_passed=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: URL Accessibility Check (Google fonts sample) | |
| id: urls | |
| run: | | |
| echo "::group::URL Check - Google Fonts" | |
| # Check all Google fonts (they share same URL pattern) | |
| ruby .github/scripts/check_urls.rb \ | |
| --directory Formulas/google \ | |
| --timeout 15 \ | |
| --verbose \ | |
| --json-output results/urls.json 2>&1 | tee url-results.txt | |
| echo "::endgroup::" | |
| # Check for failures | |
| if grep -q "URL check FAILED" url-results.txt; then | |
| echo "urls_passed=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "urls_passed=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Upload validation results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: tier1-validation-results | |
| path: | | |
| schema-results.txt | |
| url-results.txt | |
| results/ | |
| retention-days: 30 | |
| - name: Summary | |
| run: | | |
| echo "## Tier 1 Validation Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Check | Status |" >> $GITHUB_STEP_SUMMARY | |
| echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Schema | ${{ steps.schema.outputs.schema_passed == 'true' && '✅ Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| URLs | ${{ steps.urls.outputs.urls_passed == 'true' && '✅ Passed' || '❌ Failed' }} |" >> $GITHUB_STEP_SUMMARY | |
| - name: Fail if validation failed | |
| if: steps.schema.outputs.schema_passed != 'true' || steps.urls.outputs.urls_passed != 'true' | |
| run: exit 1 | |
| # =========================================================================== | |
| # TIER 2: Full Installation Test | |
| # =========================================================================== | |
| tier2-installation: | |
| name: Tier 2 - Installation Test (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| if: | | |
| (github.event_name == 'schedule' && | |
| github.event.schedule == '0 7 */2 * *') || | |
| (github.event_name == 'workflow_dispatch' && | |
| (github.event.inputs.tier == 'both' || | |
| github.event.inputs.tier == 'tier2-installation')) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ${{ env.RUBY_VERSION }} | |
| bundler-cache: true | |
| - name: Determine rotation and platform | |
| id: config | |
| shell: bash | |
| run: | | |
| # Calculate rotation day (1 or 2) based on current day of year | |
| ROTATION_DAY=$(( $(date +%j) % 2 + 1 )) | |
| echo "rotation_day=${ROTATION_DAY}" >> $GITHUB_OUTPUT | |
| # Map OS to platform | |
| case "${{ matrix.os }}" in | |
| ubuntu-latest) PLATFORM="linux" ;; | |
| macos-latest) PLATFORM="macos" ;; | |
| windows-latest) PLATFORM="windows" ;; | |
| esac | |
| echo "platform=${PLATFORM}" >> $GITHUB_OUTPUT | |
| # Build extra args from inputs | |
| EXTRA_ARGS="" | |
| if [ -n "${{ github.event.inputs.group }}" ]; then | |
| EXTRA_ARGS="${EXTRA_ARGS} --group ${{ github.event.inputs.group }}" | |
| fi | |
| if [ -n "${{ github.event.inputs.sample_size }}" ]; then | |
| EXTRA_ARGS="${EXTRA_ARGS} --sample ${{ github.event.inputs.sample_size }}" | |
| fi | |
| echo "extra_args=${EXTRA_ARGS}" >> $GITHUB_OUTPUT | |
| echo "Rotation day: ${ROTATION_DAY}" | |
| echo "Platform: ${PLATFORM}" | |
| - name: Run installation test (Day ${{ steps.config.outputs.rotation_day }}) | |
| id: install | |
| shell: bash | |
| continue-on-error: true | |
| run: | | |
| echo "::group::Installation Test - Rotation Day ${{ steps.config.outputs.rotation_day }}" | |
| mkdir -p results | |
| # Build the command | |
| CMD="bundle exec ruby .github/scripts/install_formulas.rb" | |
| CMD="${CMD} --directory Formulas" | |
| CMD="${CMD} --rotation ${{ steps.config.outputs.rotation_day }}" | |
| CMD="${CMD} --platform ${{ steps.config.outputs.platform }}" | |
| CMD="${CMD} --continue-on-error" | |
| CMD="${CMD} ${{ steps.config.outputs.extra_args }}" | |
| CMD="${CMD} --output install-results-${{ steps.config.outputs.platform }}.yml" | |
| CMD="${CMD} --json-output results/install-${{ steps.config.outputs.platform }}.json" | |
| echo "Running: ${CMD}" | |
| ${CMD} 2>&1 | tee install-output.txt | |
| echo "::endgroup::" | |
| # Check for failures | |
| if grep -q "Installation test FAILED" install-output.txt; then | |
| echo "install_passed=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "install_passed=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Upload installation results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: tier2-install-results-${{ steps.config.outputs.platform }} | |
| path: | | |
| install-output.txt | |
| install-results-${{ steps.config.outputs.platform }}.yml | |
| results/ | |
| retention-days: 30 | |
| - name: Summary | |
| shell: bash | |
| run: | | |
| echo "## Tier 2 Installation Results (${{ steps.config.outputs.platform }})" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- Rotation Day: ${{ steps.config.outputs.rotation_day }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Platform: ${{ steps.config.outputs.platform }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- Status: ${{ steps.install.outputs.install_passed == 'true' && '✅ Passed' || '❌ Failed' }}" >> $GITHUB_STEP_SUMMARY | |
| if [ -f "install-results-${{ steps.config.outputs.platform }}.yml" ]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Statistics" >> $GITHUB_STEP_SUMMARY | |
| echo '```yaml' >> $GITHUB_STEP_SUMMARY | |
| cat "install-results-${{ steps.config.outputs.platform }}.yml" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # =========================================================================== | |
| # TIER 2 FULL SCAN: Test ALL formulas (weekly or on-demand) | |
| # Traffic: ~1.1GB on Ubuntu, ~3GB on macOS. 60-150 min per platform. | |
| # =========================================================================== | |
| tier2-full-scan: | |
| name: Full Scan (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| if: | | |
| (github.event_name == 'schedule' && | |
| github.event.schedule == '0 8 * * 1') || | |
| (github.event_name == 'workflow_dispatch' && | |
| github.event.inputs.tier == 'tier2-full-scan') | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ${{ env.RUBY_VERSION }} | |
| bundler-cache: true | |
| - name: Determine platform | |
| id: config | |
| shell: bash | |
| run: | | |
| case "${{ matrix.os }}" in | |
| ubuntu-latest) PLATFORM="linux" ;; | |
| macos-latest) PLATFORM="macos" ;; | |
| windows-latest) PLATFORM="windows" ;; | |
| esac | |
| echo "platform=${PLATFORM}" >> $GITHUB_OUTPUT | |
| echo "Platform: ${PLATFORM}" | |
| - name: Run full scan | |
| id: install | |
| shell: bash | |
| continue-on-error: true | |
| timeout-minutes: 360 | |
| run: | | |
| echo "::group::Full Formula Scan (${{ steps.config.outputs.platform }})" | |
| mkdir -p results | |
| CMD="bundle exec ruby .github/scripts/install_formulas.rb" | |
| CMD="${CMD} --directory Formulas" | |
| CMD="${CMD} --full" | |
| CMD="${CMD} --platform ${{ steps.config.outputs.platform }}" | |
| CMD="${CMD} --continue-on-error" | |
| CMD="${CMD} --output full-scan-${{ steps.config.outputs.platform }}.yml" | |
| CMD="${CMD} --json-output results/full-scan-${{ steps.config.outputs.platform }}.json" | |
| echo "Running: ${CMD}" | |
| ${CMD} 2>&1 | tee scan-output.txt | |
| echo "::endgroup::" | |
| - name: Generate failure report | |
| id: report | |
| if: always() | |
| shell: bash | |
| run: | | |
| RESULTS_FILE="full-scan-${{ steps.config.outputs.platform }}.yml" | |
| if [ ! -f "${RESULTS_FILE}" ]; then | |
| echo "has_failures=unknown" >> $GITHUB_OUTPUT | |
| echo "Results file not found (scan may have timed out)." > failure-report.txt | |
| exit 0 | |
| fi | |
| # Extract failure count from results | |
| FAILURES=$(ruby -ryaml -e " | |
| data = YAML.load_file('${RESULTS_FILE}') | |
| failures = data['failures'] || [] | |
| puts failures.size | |
| " 2>/dev/null || echo "unknown") | |
| echo "failure_count=${FAILURES}" >> $GITHUB_OUTPUT | |
| if [ "${FAILURES}" != "0" ] && [ "${FAILURES}" != "unknown" ]; then | |
| echo "has_failures=true" >> $GITHUB_OUTPUT | |
| # Build failure list for summary | |
| ruby -ryaml -e " | |
| data = YAML.load_file('${RESULTS_FILE}') | |
| failures = data['failures'] || [] | |
| successes = data['successes'] || [] | |
| skipped = data['skipped'] || [] | |
| puts \"Successes: #{successes.size}\" | |
| puts \"Failures: #{failures.size}\" | |
| puts \"Skipped: #{skipped.size}\" | |
| puts | |
| if failures.any? | |
| puts 'FAILED FORMULAS:' | |
| failures.each do |f| | |
| puts \" - #{f['name']}: #{f['error'].to_s.split('\n').first[0..120]}\" | |
| end | |
| end | |
| " 2>/dev/null | tee failure-report.txt | |
| else | |
| echo "has_failures=false" >> $GITHUB_OUTPUT | |
| echo "No failures found." > failure-report.txt | |
| fi | |
| - name: Upload full scan results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: full-scan-${{ steps.config.outputs.platform }} | |
| path: | | |
| scan-output.txt | |
| full-scan-${{ steps.config.outputs.platform }}.yml | |
| failure-report.txt | |
| results/ | |
| retention-days: 90 | |
| - name: Summary | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo "## Full Scan Results (${{ steps.config.outputs.platform }})" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f "failure-report.txt" ]; then | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat failure-report.txt >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "Scan did not complete." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # =========================================================================== | |
| # Render: Aggregate per-tier JSON results into rendered markdown reports | |
| # and a health.json consumed by the docs dashboard. | |
| # =========================================================================== | |
| render-reports: | |
| name: Render Reports | |
| runs-on: ubuntu-latest | |
| needs: [tier1-validation, tier2-installation, tier2-full-scan] | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ${{ env.RUBY_VERSION }} | |
| bundler-cache: true | |
| - name: Download all tier artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: all-results/ | |
| - name: Consolidate JSON results | |
| run: | | |
| mkdir -p results | |
| # All tier artifacts contain a results/ subdir with their JSON output. | |
| # Flatten them into a single results/ dir for the renderer. | |
| find all-results -name "*.json" -type f -print -exec cp {} results/ \; | |
| echo "Consolidated JSON files:" | |
| ls -1 results/ | |
| - name: Render reports | |
| id: render | |
| run: | | |
| mkdir -p reports | |
| ruby .github/scripts/render_report.rb \ | |
| --input-dir results/ \ | |
| --output-dir reports/ \ | |
| --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ | |
| --commit-sha "${{ github.sha }}" | |
| # Surface pass/fail for downstream jobs | |
| if ruby -rjson -e 'exit(JSON.parse(File.read("reports/health.json"))["totals"]["checks_failed"].zero? ? 0 : 1)'; then | |
| echo "all_passed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "all_passed=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Step summary | |
| if: always() | |
| run: cat reports/step-summary.md >> $GITHUB_STEP_SUMMARY | |
| - name: Upload rendered reports | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rendered-reports | |
| path: | | |
| reports/ | |
| results/ | |
| retention-days: 30 | |
| # =========================================================================== | |
| # Report: Create or update GitHub issue with full failure details | |
| # =========================================================================== | |
| report: | |
| name: Report Results | |
| runs-on: ubuntu-latest | |
| needs: [tier1-validation, tier2-installation, tier2-full-scan, render-reports] | |
| if: always() && needs.render-reports.result == 'success' && needs.render-reports.outputs.all_passed != 'true' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download rendered reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: rendered-reports | |
| path: reports/ | |
| - name: Check for repeated failures | |
| id: check | |
| run: | | |
| # Look for existing open issue with same label | |
| ISSUE_NUMBER=$(gh issue list \ | |
| --repo ${{ github.repository }} \ | |
| --state open \ | |
| --label "formula-health-failure" \ | |
| --limit 1 \ | |
| --json number \ | |
| --jq '.[0].number // empty') | |
| if [ -n "$ISSUE_NUMBER" ]; then | |
| echo "issue_number=${ISSUE_NUMBER}" >> $GITHUB_OUTPUT | |
| echo "is_existing=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "is_existing=false" >> $GITHUB_OUTPUT | |
| fi | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Create or update issue | |
| run: | | |
| # Use the pre-rendered issue body from render-reports | |
| cp reports/issue-body.md final-issue-body.md | |
| # Append artifacts pointer | |
| { | |
| echo "" | |
| echo "### Artifacts" | |
| echo "- [Download full results](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts)" | |
| echo "" | |
| echo "_This issue is automatically updated by the formula-health workflow._" | |
| echo "_It will be auto-closed when all health checks pass._" | |
| } >> final-issue-body.md | |
| if [ "${{ steps.check.outputs.is_existing }}" == "true" ]; then | |
| gh issue comment ${{ steps.check.outputs.issue_number }} \ | |
| --repo ${{ github.repository }} \ | |
| --body-file final-issue-body.md | |
| else | |
| gh issue create \ | |
| --repo ${{ github.repository }} \ | |
| --title "Formula Health Check Failed - $(date +%Y-%m-%d)" \ | |
| --body-file final-issue-body.md \ | |
| --label "formula-health-failure,automated" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Summary | |
| run: | | |
| echo "## Health Check Report" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Failures detected. A GitHub issue has been created or updated." >> $GITHUB_STEP_SUMMARY | |
| # =========================================================================== | |
| # Close issue on success | |
| # =========================================================================== | |
| close-issue-on-success: | |
| name: Close Issue on Success | |
| runs-on: ubuntu-latest | |
| needs: [tier1-validation, tier2-installation, tier2-full-scan, render-reports] | |
| if: always() && needs.render-reports.result == 'success' && needs.render-reports.outputs.all_passed == 'true' | |
| steps: | |
| - name: Close existing health check issues | |
| run: | | |
| # Find and close open health check issues | |
| gh issue list \ | |
| --repo ${{ github.repository }} \ | |
| --state open \ | |
| --label "formula-health-failure" \ | |
| --json number \ | |
| --jq '.[].number' | while read issue_number; do | |
| gh issue close $issue_number \ | |
| --repo ${{ github.repository }} \ | |
| --comment "All formula health checks are now passing. Closing this issue." | |
| done | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| continue-on-error: true |