refactor(inkless): Split InitDisklessLogManager logic into separate components #2385
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2024 Aiven, Helsinki, Finland. https://aiven.io/ | |
| # Workflow for inkless | |
| name: Inkless CI | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| - 'inkless-4.*' | |
| pull_request: | |
| types: [ opened, synchronize, ready_for_review, reopened ] | |
| jobs: | |
| validate: | |
| runs-on: ubuntu-latest | |
| name: Compile and Check Java | |
| outputs: | |
| is-draft: ${{ steps.check-draft-pr.outputs.is-draft }} | |
| steps: | |
| - name: Env | |
| run: printenv | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| - name: Check for Draft PR | |
| id: check-draft-pr | |
| if: | | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.draft | |
| run: echo "is-draft=true" >> "$GITHUB_OUTPUT" | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| # Generate JOOQ classes with JDK 25 to include @SuppressWarnings("this-escape") annotations | |
| - name: Setup JDK 25 for JOOQ generation | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 25 | |
| - name: Generate JOOQ classes | |
| run: ./gradlew :storage:inkless:generateJooqClasses | |
| - name: Verify JOOQ classes are up-to-date | |
| run: | | |
| if [[ -n $(git status -s -- storage/inkless/src/main/jooq) ]]; then | |
| echo "JOOQ classes are not up-to-date with Flyway migrations:" | |
| git status -s -- storage/inkless/src/main/jooq | |
| git diff --stat -- storage/inkless/src/main/jooq | |
| echo "" | |
| echo "Please run 'make jooq' with JDK 25 and commit the changes" | |
| exit 1 | |
| else | |
| echo "JOOQ classes are up-to-date." | |
| fi | |
| shell: bash | |
| # Switch to JDK 17 for compilation and validation | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| java-version: 17 | |
| gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }} | |
| gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }} | |
| - name: Compile and validate | |
| env: | |
| SCAN_ARG: ${{ inputs.is-public-fork && '--no-scan' || '--scan' }} | |
| # Gradle flags | |
| # --build-cache: Let Gradle restore the build cache | |
| # --info: For now, we'll generate lots of logs while setting up the GH Actions | |
| # --scan: Publish the build scan. This will only work on PRs from apache/kafka and trunk | |
| # --no-scan: For public fork PRs, we won't attempt to publish the scan | |
| run: | | |
| ./gradlew --build-cache --info $SCAN_ARG check -x test -x generateJooqClasses | |
| - name: Gen docs | |
| run: make docs | |
| - name: Check for uncommitted documentation changes | |
| run: | | |
| if [[ -n $(git status -s) ]]; then | |
| echo "There are uncommitted changes after the task:" | |
| git status -s | |
| echo "Update the documentation on the proper *Config.java and commit generated docs" | |
| exit 1 | |
| else | |
| echo "No changes detected." | |
| fi | |
| shell: bash | |
| - name: Archive check reports | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: check-reports | |
| path: | | |
| **/build/**/*.html | |
| compression-level: 9 | |
| if-no-files-found: ignore | |
| - name: Annotate checkstyle errors | |
| # Avoid duplicate annotations, only run on java 21 | |
| if: failure() | |
| run: python .github/scripts/checkstyle.py | |
| env: | |
| GITHUB_WORKSPACE: ${{ github.workspace }} | |
| # - name: Annotate Rat errors | |
| # # Avoid duplicate annotations, only run on java 21 | |
| # if: failure() | |
| # run: python .github/scripts/rat.py | |
| # env: | |
| # GITHUB_WORKSPACE: ${{ github.workspace }} | |
| test: | |
| needs: validate | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 17, 25 ] # If we change these, make sure to adjust ci-complete.yml | |
| name: JUnit tests Java ${{ matrix.java }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - run: pip install -r .github/scripts/requirements.txt | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| java-version: ${{ matrix.java }} | |
| gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }} | |
| gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }} | |
| develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }} | |
| - name: Test | |
| # Gradle flags | |
| # --build-cache: Let Gradle restore the build cache | |
| # --no-scan: Don't attempt to publish the scan yet. We want to archive it first. | |
| # --continue: Keep running even if a test fails | |
| # -PcommitId Prevent the Git SHA being written into the jar files (which breaks caching) | |
| # JOOQ classes are pre-committed with JDK 25, so no regeneration needed | |
| id: junit-test | |
| env: | |
| TIMEOUT_MINUTES: 30 | |
| GRADLE_ARGS: >- | |
| --build-cache --no-scan | |
| -x generateJooqClasses | |
| -PtestLoggingEvents=started,passed,skipped,failed | |
| -PmaxParallelForks=2 | |
| -PmaxTestRetries=1 -PmaxTestRetryFailures=3 | |
| -PmaxQuarantineTestRetries=3 -PmaxQuarantineTestRetryFailures=0 | |
| -PcommitId=xxxxxxxxxxxxxxxx | |
| # Point to inkless tests | |
| run: | | |
| set +e | |
| ./.github/scripts/thread-dump.sh & | |
| # inkless tests | |
| timeout ${TIMEOUT_MINUTES}m \ | |
| ./gradlew ${GRADLE_ARGS} :storage:inkless:test :storage:inkless:integrationTest && \ | |
| ./gradlew ${GRADLE_ARGS} :metadata:test --tests "org.apache.kafka.controller.*" && \ | |
| ./gradlew ${GRADLE_ARGS} :core:test --tests "*Inkless*" --tests "*Diskless*" --tests "kafka.log.LogConfigTest" | |
| exitcode="$?" | |
| echo "exitcode=$exitcode" >> $GITHUB_OUTPUT | |
| - name: Archive JUnit HTML reports | |
| uses: actions/upload-artifact@v4 | |
| id: junit-upload-artifact | |
| with: | |
| name: junit-reports-${{ matrix.java }} | |
| path: | | |
| **/build/reports/tests/* | |
| compression-level: 9 | |
| if-no-files-found: ignore | |
| - name: Archive JUnit XML | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: junit-xml-${{ matrix.java }} | |
| path: | | |
| build/junit-xml/**/*.xml | |
| compression-level: 9 | |
| if-no-files-found: ignore | |
| - name: Archive Thread Dumps | |
| id: thread-dump-upload-artifact | |
| if: always() && ( steps.junit-test.outputs.exitcode == '124' ) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: junit-thread-dumps-${{ matrix.java }} | |
| path: | | |
| thread-dumps/* | |
| compression-level: 9 | |
| if-no-files-found: ignore | |
| # - name: Parse JUnit tests | |
| # run: python .github/scripts/junit.py --export-test-catalog ./test-catalog >> $GITHUB_STEP_SUMMARY | |
| # env: | |
| # GITHUB_WORKSPACE: ${{ github.workspace }} | |
| # JUNIT_REPORT_URL: ${{ steps.junit-upload-artifact.outputs.artifact-url }} | |
| # THREAD_DUMP_URL: ${{ steps.thread-dump-upload-artifact.outputs.artifact-url }} | |
| # GRADLE_EXIT_CODE: ${{ steps.junit-test.outputs.exitcode }} | |
| - name: Archive Test Catalog | |
| if: ${{ always() && matrix.java == '21' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-catalog | |
| path: test-catalog | |
| compression-level: 9 | |
| if-no-files-found: ignore | |
| - name: Archive Build Scan | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-scan-test-${{ matrix.java }} | |
| path: ~/.gradle/build-scan-data | |
| compression-level: 9 | |
| if-no-files-found: ignore | |
| - name: Check test results | |
| if: always() && ( steps.junit-test.outputs.exitcode != '0' ) | |
| run: exit 1 |