부하 테스트 (Scheduled Load Testing) #16
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 부하 테스트 (Scheduled Load Testing) | |
| permissions: | |
| contents: write | |
| issues: write | |
| concurrency: | |
| group: load-test-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| # Daily Light: 12:00 KST (03:00 UTC) | |
| # Weekly All: Sunday 03:00 KST (Saturday 18:00 UTC) | |
| schedule: | |
| - cron: "0 3 * * *" | |
| - cron: "0 18 * * 6" | |
| # 수동 실행 지원 | |
| workflow_dispatch: | |
| inputs: | |
| scenario: | |
| description: "실행할 시나리오 선택" | |
| required: true | |
| default: "light" | |
| type: choice | |
| options: | |
| - light | |
| - medium | |
| - heavy | |
| - stress | |
| - all | |
| test_duration: | |
| description: "Stress 테스트 지속 시간 (분)" | |
| required: false | |
| default: "5" | |
| max_users: | |
| description: "Stress 테스트 최대 동시 사용자 수" | |
| required: false | |
| default: "200" | |
| env: | |
| API_HOST: https://ion-api-64076350717.us-central1.run.app | |
| PYTHON_VERSION: "3.13" | |
| jobs: | |
| load-test: | |
| name: Locust 부하 테스트 | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| defaults: | |
| run: | |
| working-directory: LLM_Unified/ion-mentoring | |
| env: | |
| SELECTED_SCENARIO: >- | |
| ${{ | |
| github.event_name == 'workflow_dispatch' && github.event.inputs.scenario || | |
| (github.event_name == 'schedule' && github.event.schedule == '0 3 * * *' && 'light') || | |
| (github.event_name == 'schedule' && github.event.schedule == '0 18 * * 6' && 'all') || | |
| 'light' | |
| }} | |
| steps: | |
| - name: 코드 체크아웃 | |
| uses: actions/checkout@v4 | |
| - name: 선택된 시나리오 출력 | |
| run: | | |
| echo "Event: ${{ github.event_name }}" | |
| echo "Schedule: ${{ github.event.schedule }}" | |
| echo "Selected Scenario: ${{ env.SELECTED_SCENARIO }}" | |
| - name: Python ${{ env.PYTHON_VERSION }} 설정 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: "pip" | |
| - name: 의존성 설치 | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-api.txt | |
| - name: 출력 디렉토리 생성 | |
| run: mkdir -p outputs | |
| - name: Light 시나리오 실행 | |
| if: env.SELECTED_SCENARIO == 'all' || env.SELECTED_SCENARIO == 'light' | |
| continue-on-error: true | |
| run: | | |
| python -m locust -f load_test.py \ | |
| --host=${{ env.API_HOST }} \ | |
| --users 10 \ | |
| --spawn-rate 0.5 \ | |
| --run-time 2m \ | |
| --headless \ | |
| --csv=outputs/load_test_light | |
| - name: Medium 시나리오 실행 | |
| if: env.SELECTED_SCENARIO == 'all' || env.SELECTED_SCENARIO == 'medium' | |
| continue-on-error: true | |
| run: | | |
| sleep 10 | |
| python -m locust -f load_test.py \ | |
| --host=${{ env.API_HOST }} \ | |
| --users 50 \ | |
| --spawn-rate 2 \ | |
| --run-time 3m \ | |
| --headless \ | |
| --csv=outputs/load_test_medium | |
| - name: Heavy 시나리오 실행 | |
| if: env.SELECTED_SCENARIO == 'all' || env.SELECTED_SCENARIO == 'heavy' | |
| continue-on-error: true | |
| run: | | |
| sleep 10 | |
| python -m locust -f load_test.py \ | |
| --host=${{ env.API_HOST }} \ | |
| --users 100 \ | |
| --spawn-rate 5 \ | |
| --run-time 5m \ | |
| --headless \ | |
| --csv=outputs/load_test_heavy | |
| - name: Stress 시나리오 실행 | |
| if: env.SELECTED_SCENARIO == 'all' || env.SELECTED_SCENARIO == 'stress' | |
| continue-on-error: true | |
| run: | | |
| # Weekly all이면 비용/교착 방지용으로 Stress만 1분 | |
| if [ "${SELECTED_SCENARIO}" = "all" ]; then | |
| STRESS_RUN_TIME="1m" | |
| else | |
| STRESS_RUN_TIME="${{ github.event.inputs.test_duration || '10' }}m" | |
| fi | |
| echo "STRESS_RUN_TIME=${STRESS_RUN_TIME}" | |
| sleep 10 | |
| python -m locust -f load_test.py \ | |
| --host=${{ env.API_HOST }} \ | |
| --users ${{ github.event.inputs.max_users || '150' }} \ | |
| --spawn-rate 5 \ | |
| --run-time "${STRESS_RUN_TIME}" \ | |
| --headless \ | |
| --csv=outputs/load_test_stress | |
| - name: Daily Report 생성 | |
| if: always() && env.SELECTED_SCENARIO == 'light' | |
| run: | | |
| python - << 'PY' | |
| import glob, csv, pathlib, os | |
| report = pathlib.Path("outputs/daily_report.md") | |
| run_url = f"https://github.com/{os.environ.get('GITHUB_REPOSITORY','')}/actions/runs/{os.environ.get('GITHUB_RUN_ID','')}" | |
| candidates = sorted(glob.glob("outputs/*light*stats.csv")) | |
| path = candidates[0] if candidates else None | |
| reqs=fails=0; rps=None; fail_rate=0.0 | |
| if path: | |
| with open(path, newline="", encoding="utf-8") as f: | |
| rows=list(csv.DictReader(f)) | |
| agg = next((r for r in rows if r.get("Name","").lower() in ("aggregated","total")), (rows[0] if rows else {})) | |
| def num(k): | |
| v=(agg.get(k) or "0").strip() | |
| try: return float(v) | |
| except: return 0.0 | |
| reqs = num("Request Count") or num("# requests") | |
| fails = num("Failure Count") or num("# failures") | |
| rps = num("Requests/s") or num("RPS") | |
| fail_rate = (fails/reqs*100.0) if reqs else 0.0 | |
| lines = ["# Daily Light Load Test (Heartbeat)", f"- Run: {run_url}", f"- Requests: {int(reqs)}", f"- Failures: {int(fails)} ({fail_rate:.2f}%)", f"- RPS: {rps:.2f}" if rps else "- RPS: (n/a)"] | |
| report.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| PY | |
| - name: Weekly Report 생성 | |
| if: always() && env.SELECTED_SCENARIO == 'all' | |
| run: | | |
| python - << 'PY' | |
| import glob, csv, os, pathlib | |
| report = pathlib.Path("outputs/weekly_report.md") | |
| run_url = f"https://github.com/{os.environ.get('GITHUB_REPOSITORY','')}/actions/runs/{os.environ.get('GITHUB_RUN_ID','')}" | |
| candidates = sorted(glob.glob("outputs/*stats.csv")) | |
| path = next((p for p in candidates if "stress" in p), candidates[-1]) if candidates else None | |
| reqs=fails=0; rps=avg=p50=p95=None; fail_rate=0.0 | |
| if path: | |
| with open(path, newline="", encoding="utf-8") as f: | |
| rows=list(csv.DictReader(f)) | |
| agg = next((r for r in rows if r.get("Name","").lower() in ("aggregated","total")), (rows[0] if rows else {})) | |
| def num(k): | |
| v=(agg.get(k) or "0").strip() | |
| try: return float(v) | |
| except: return 0.0 | |
| reqs = num("Request Count") or num("# requests"); fails = num("Failure Count") or num("# failures") | |
| rps = num("Requests/s") or num("RPS"); avg = num("Average Response Time") or num("Average") | |
| p50 = num("50%"); p95 = num("95%") | |
| fail_rate = (fails/reqs*100.0) if reqs else 0.0 | |
| lines = [f"# Weekly Load Test Report", f"- Run: {run_url}", f"- Requests: {int(reqs)}", f"- Failures: {int(fails)} ({fail_rate:.2f}%)", | |
| f"- RPS: {rps:.2f}" if rps else "- RPS: (n/a)", | |
| f"- Latency ms: avg={avg or 0:.0f}, p50={p50 or 0:.0f}, p95={p95 or 0:.0f}" if any([avg,p50,p95]) else "- Latency ms: (n/a)"] | |
| report.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| PY | |
| - name: Weekly Summary (Actions UI) | |
| if: always() && env.SELECTED_SCENARIO == 'all' | |
| run: | | |
| cat outputs/weekly_report.md >> $GITHUB_STEP_SUMMARY | |
| - name: Resolve canonical + close duplicates (load-test-thread) | |
| if: always() && env.SELECTED_SCENARIO == 'all' && github.event_name == 'schedule' | |
| id: issue | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const owner = context.repo.owner, repo = context.repo.repo, label = "load-test-thread"; | |
| const {data: issues} = await github.rest.issues.listForRepo({ owner, repo, labels: label, state: "open", per_page: 100 }); | |
| let canonical = issues.sort((a,b) => new Date(a.created_at) - new Date(b.created_at))[0]; | |
| if (!canonical) { | |
| canonical = (await github.rest.issues.create({ | |
| owner, repo, | |
| title: "Load Test Weekly Reports (Thread)", | |
| body: "# Weekly Load Test Reports\nLiving thread for weekly results.\nLabel: load-test-thread", | |
| labels: [label, "ops", "load-test"] | |
| })).data; | |
| } | |
| const dupes = issues.filter(i => i.number !== canonical.number); | |
| for (const d of dupes) { | |
| await github.rest.issues.createComment({ owner, repo, issue_number: d.number, body: `DUPLICATE → canonical thread: #${canonical.number} (${canonical.html_url})\nClosing to keep a single operational timeline.` }); | |
| await github.rest.issues.update({ owner, repo, issue_number: d.number, state: "closed" }); | |
| } | |
| core.setOutput("number", canonical.number); | |
| core.setOutput("html_url", canonical.html_url); | |
| - name: Update ops doc with canonical thread (SSOT) | |
| if: always() && env.SELECTED_SCENARIO == 'all' && github.event_name == 'schedule' | |
| run: | | |
| OPS="docs/ops/load-test.md" | |
| mkdir -p "$(dirname "$OPS")" | |
| CANON_NUM="${{ steps.issue.outputs.number }}" | |
| CANON_URL="${{ steps.issue.outputs.html_url }}" | |
| if [ ! -f "$OPS" ]; then | |
| cat > "$OPS" <<'MD' | |
| # 부하 테스트 운영 기록 (SSOT) | |
| ## Canonical Issue Thread | |
| - Thread: (unset) | |
| MD | |
| fi | |
| if grep -qE '^- Thread:' "$OPS"; then | |
| perl -0777 -i -pe "s/^- Thread:.*\$/- Thread: #$CANON_NUM ($CANON_URL)/m" "$OPS" | |
| else | |
| printf "\n## Canonical Issue Thread\n- Thread: #%s (%s)\n" "$CANON_NUM" "$CANON_URL" >> "$OPS" | |
| fi | |
| - name: Comment weekly report to canonical issue | |
| if: always() && env.SELECTED_SCENARIO == 'all' && github.event_name == 'schedule' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const owner = context.repo.owner, repo = context.repo.repo; | |
| const issue_number = Number("${{ steps.issue.outputs.number }}"); | |
| const runUrl = `https://github.com/${owner}/${repo}/actions/runs/${context.runId}`; | |
| const report = fs.existsSync('LLM_Unified/ion-mentoring/outputs/weekly_report.md') ? fs.readFileSync('LLM_Unified/ion-mentoring/outputs/weekly_report.md', 'utf8') : "(weekly_report.md missing)"; | |
| await github.rest.issues.createComment({ owner, repo, issue_number, body: `## Weekly All Summary\n- Run: ${runUrl}\n\n${report}` }); | |
| - name: Commit ops SSOT update | |
| if: always() && env.SELECTED_SCENARIO == 'all' && github.event_name == 'schedule' | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git add docs/ops/load-test.md | |
| git diff --quiet && git diff --staged --quiet || (git commit -m "Ops: pin canonical load-test issue thread (SSOT)" && git push origin main || echo "Push failed (potentially due to branch protection)") | |
| - name: 테스트 결과 요약 생성 | |
| if: always() | |
| run: | | |
| cat > outputs/test_summary.md << 'EOF' | |
| # 부하 테스트 결과 요약 | |
| **실행 시간**: $(date -u +'%Y-%m-%d %H:%M:%S UTC') | |
| **브랜치**: ${{ github.ref_name }} | |
| **커밋**: ${{ github.sha }} | |
| ## 시나리오 결과 | |
| ### Light (10 users, 2min) | |
| - CSV: [load_test_light_stats.csv](./load_test_light_stats.csv) | |
| - HTML: [load_test_light.html](./load_test_light.html) | |
| ### Medium (50 users, 3min) | |
| - CSV: [load_test_medium_stats.csv](./load_test_medium_stats.csv) | |
| - HTML: [load_test_medium.html](./load_test_medium.html) | |
| ### Heavy (100 users, 5min) | |
| - CSV: [load_test_heavy_stats.csv](./load_test_heavy_stats.csv) | |
| - HTML: [load_test_heavy.html](./load_test_heavy.html) | |
| ### Stress (${{ github.event.inputs.max_users || '200' }} users, ${{ github.event.inputs.test_duration || '10' }}min) | |
| - CSV: [load_test_stress_stats.csv](./load_test_stress_stats.csv) | |
| ## 다음 단계 | |
| 1. HTML 리포트에서 상세 지표 확인 | |
| 2. P95/P99 latency가 목표치 초과 시 성능 최적화 | |
| 3. Cloud Monitoring 대시보드에서 인스턴스 스케일링 검토 | |
| EOF | |
| - name: 불필요한 파일 정리 | |
| if: always() | |
| run: | | |
| rm -f outputs/*.html || true | |
| rm -rf outputs/load_test_*_stats_history.csv || true | |
| - name: 테스트 결과 아티팩트 업로드 | |
| if: always() | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: load-test-results-${{ github.run_number }} | |
| path: | | |
| LLM_Unified/ion-mentoring/outputs/*.csv | |
| LLM_Unified/ion-mentoring/outputs/*.md | |
| retention-days: 3 | |
| if-no-files-found: warn | |
| - name: Slack alert (Daily light failure only) | |
| if: failure() && env.SELECTED_SCENARIO == 'light' && github.event_name == 'schedule' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| run: | | |
| [ -z "${SLACK_WEBHOOK_URL}" ] && echo "Slack skip" && exit 0 | |
| python - << 'PY' | |
| import os, json, urllib.request, pathlib | |
| report = pathlib.Path("outputs/daily_report.md") | |
| content = report.read_text(encoding="utf-8") if report.exists() else "Daily survey failed." | |
| msg = f"*Daily Light FAILED*\n- Ref: {os.environ.get('GITHUB_REF_NAME','')}\n{content}" | |
| req = urllib.request.Request(os.environ["SLACK_WEBHOOK_URL"], data=json.dumps({"text": msg}).encode("utf-8"), headers={"Content-Type":"application/json"}) | |
| urllib.request.urlopen(req) | |
| PY | |
| - name: Slack weekly summary (Weekly All) | |
| if: always() && env.SELECTED_SCENARIO == 'all' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| run: | | |
| [ -z "${SLACK_WEBHOOK_URL}" ] && echo "Slack skip" && exit 0 | |
| python - << 'PY' | |
| import os, json, urllib.request, pathlib | |
| report = pathlib.Path("outputs/weekly_report.md") | |
| content = report.read_text(encoding="utf-8") if report.exists() else "Weekly survey complete." | |
| msg = f"*Weekly All Load Test Summary*\n- Ref: {os.environ.get('GITHUB_REF_NAME','')}\n{content}" | |
| req = urllib.request.Request(os.environ["SLACK_WEBHOOK_URL"], data=json.dumps({"text": msg}).encode("utf-8"), headers={"Content-Type":"application/json"}) | |
| urllib.request.urlopen(req) | |
| PY | |
| analyze: | |
| name: 결과 분석 및 리포트 | |
| runs-on: ubuntu-latest | |
| needs: load-test | |
| if: always() | |
| steps: | |
| - name: 아티팩트 존재 확인 | |
| id: check_artifacts | |
| run: | | |
| api_url="https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" | |
| token="${{ secrets.GITHUB_TOKEN }}" | |
| artifact_name="load-test-results-${{ github.run_number }}" | |
| echo "Checking for artifact: $artifact_name" | |
| response=$(curl -s -H "Authorization: Bearer $token" "$api_url") | |
| count=$(echo "$response" | python3 -c "import sys, json; print(sum(1 for a in json.load(sys.stdin).get('artifacts', []) if a['name'] == '$artifact_name'))") | |
| echo "Found $count matching artifacts" | |
| echo "found=$count" >> $GITHUB_OUTPUT | |
| - name: 아티팩트 다운로드 | |
| if: steps.check_artifacts.outputs.found != '0' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: load-test-results-${{ github.run_number }} | |
| path: results | |
| - name: 아티팩트 누락 경고 | |
| if: steps.check_artifacts.outputs.found == '0' | |
| run: | | |
| echo "::warning::부하 테스트 결과 아티팩트를 찾을 수 없습니다. 분석을 건너뜁니다." | |
| mkdir -p results | |
| echo '{}' > results/load_test_light_stats.csv # 더미 파일 생성으로 후속 단계 에러 방지 | |
| - name: 성능 지표 추출 (Python) | |
| run: | | |
| cat > analyze.py << 'EOF' | |
| import csv | |
| import json | |
| from pathlib import Path | |
| results_dir = Path("results") | |
| scenarios = ["light", "medium", "heavy", "stress"] | |
| summary = {"scenarios": {}} | |
| for scenario in scenarios: | |
| stats_file = results_dir / f"load_test_{scenario}_stats.csv" | |
| if not stats_file.exists(): | |
| continue | |
| with open(stats_file, 'r') as f: | |
| reader = csv.DictReader(f) | |
| rows = list(reader) | |
| if not rows: | |
| continue | |
| # Aggregated row (마지막 행) | |
| agg = rows[-1] | |
| summary["scenarios"][scenario] = { | |
| "total_requests": int(agg.get("Request Count", 0)), | |
| "failures": int(agg.get("Failure Count", 0)), | |
| "avg_response_time": float(agg.get("Average Response Time", 0)), | |
| "p95_response_time": float(agg.get("95%", 0)), | |
| "p99_response_time": float(agg.get("99%", 0)), | |
| "requests_per_sec": float(agg.get("Requests/s", 0)) | |
| } | |
| print(json.dumps(summary, indent=2)) | |
| with open("results/metrics.json", "w") as f: | |
| json.dump(summary, f, indent=2) | |
| EOF | |
| python analyze.py | |
| - name: 성능 리포트 업로드 | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: performance-metrics-${{ github.run_number }} | |
| path: results/metrics.json | |
| retention-days: 90 |