Soak Test #910
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Soak Test | |
| permissions: read-all | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| branch: | |
| description: 'Soak test branch' | |
| required: true | |
| default: 'main' | |
| schedule: | |
| - cron: '0 4 * * *' # 12AM EST/9PM PST | |
| jobs: | |
| pepr-build: | |
| name: controller image | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1 | |
| with: | |
| egress-policy: audit | |
| - name: clone pepr | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| repository: defenseunicorns/pepr | |
| path: pepr | |
| - name: "set env: PEPR" | |
| run: echo "PEPR=${GITHUB_WORKSPACE}/pepr" >> "$GITHUB_ENV" | |
| - name: setup node | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: 26 | |
| cache: "npm" | |
| cache-dependency-path: pepr | |
| - name: install pepr deps | |
| run: | | |
| cd "$PEPR" | |
| npm ci | |
| - name: build pepr image | |
| run: | | |
| cd "$PEPR" | |
| npm run build:image | |
| - name: tar pepr image | |
| run: | | |
| PEPR_TAR="${GITHUB_WORKSPACE}/pepr-img.tar" | |
| echo "PEPR_TAR=${PEPR_TAR}" >> "$GITHUB_ENV" | |
| docker image save --output "$PEPR_TAR" pepr:dev | |
| - name: upload image tar artifact | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: pepr-img.tar | |
| path: pepr-img.tar | |
| retention-days: 1 | |
| soak-test: | |
| name: soak-test | |
| runs-on: ubuntu-latest | |
| needs: | |
| - pepr-build | |
| env: | |
| CACHE_MISS_GROWTH_THRESHOLD: "10" | |
| RESYNC_FAILURE_THRESHOLD: "5" | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: install k3d | |
| uses: ./.github/actions/setup-k3d | |
| - name: dowload image tar artifact | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: pepr-img.tar | |
| path: ${{ github.workspace }} | |
| - name: import pepr image from tar | |
| run: | | |
| PEPR_TAR="${GITHUB_WORKSPACE}/pepr-img.tar" | |
| echo "PEPR_TAR=${PEPR_TAR}" >> "$GITHUB_ENV" | |
| docker image load --input "$PEPR_TAR" | |
| - name: Create k3d Cluster | |
| run: "k3d cluster create" | |
| shell: bash | |
| - name: Import pepr image into k3d | |
| run: "k3d image import pepr:dev -c k3s-default" | |
| shell: bash | |
| - name: Install istioctl | |
| run: | | |
| curl -L https://istio.io/downloadIstio | sh - | |
| shell: bash | |
| - name: Install default profile | |
| run: | | |
| cd istio*/bin | |
| ./istioctl install --set profile=demo -y | |
| - name: Set up Kubernetes | |
| uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v5.1.0 | |
| with: | |
| version: 'latest' | |
| - name: Deploy applications | |
| run: | | |
| kubectl apply -f hack/auditor.ci.yaml | |
| kubectl apply -f hack/soak.ci.yaml | |
| - name: Wait for 15 minutes before starting pod checks | |
| run: | | |
| sleep 10s | |
| kubectl wait --for=condition=ready -n istio-system pod -l istio=pilot | |
| kubectl wait --for=condition=ready -n istio-system pod -l app=istio-ingressgateway | |
| kubectl wait --for=condition=ready -n watch-auditor pod -l app=watch-auditor | |
| kubectl wait --for=condition=ready -n pepr-system pod -l app=pepr-soak-ci-watcher | |
| - name: setup node | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: 26 | |
| cache: "npm" | |
| - name: install deps | |
| run: npm ci | |
| - name: Run the soak test and collect metrics | |
| run: npx tsx scripts/soak-test.ts | |
| - name: Write job summary | |
| if: always() | |
| run: npx tsx scripts/soak-summary.ts logs/metrics.csv logs/informer-log.txt logs/failure-reason.txt | |
| - name: Upload metrics CSV | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: always() | |
| with: | |
| name: soak-metrics.csv | |
| path: logs/metrics.csv | |
| - name: Upload logs | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: always() | |
| with: | |
| name: soak-test-logs | |
| path: logs |