diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6dff0a6144..f92e18ec5a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,4 +1,6 @@ name: Continuous Integration +run-name: "Continuous Integration${{ github.event.inputs.client_libs_test_image_tag != '' && format(' using image: {0}', github.event.inputs.client_libs_test_image_tag) || '' }}" + on: push: paths-ignore: @@ -17,11 +19,16 @@ on: schedule: - cron: '0 1 * * *' # nightly build workflow_dispatch: + inputs: + client_libs_test_image_tag: + description: 'Custom client libs test image tag to use instead of redis_version' + required: false + default: '' jobs: build: - name: Build and Test - runs-on: ubuntu-latest + name: Build and Test (Redis ${{ matrix.redis_version }}) + if: github.event.inputs.client_libs_test_image_tag == '' strategy: fail-fast: false matrix: @@ -32,54 +39,17 @@ jobs: - "8.0" - "7.4" - "7.2" - env: - REDIS_ENV_WORK_DIR: ${{ github.workspace }}/work + uses: ./.github/workflows/run-tests.yml + with: + redis_version: ${{ matrix.redis_version }} + secrets: + codecov_token: ${{ secrets.CODECOV_TOKEN }} - steps: - - name: Checkout project - uses: actions/checkout@v4 - - name: Set Java up in the runner - uses: actions/setup-java@v4 - with: - java-version: '8' - distribution: 'temurin' - cache: 'maven' - - name: Setup Maven - uses: s4u/setup-maven-action@v1.8.0 - with: - java-version: 8 - - name: Install missing dependencies to container - run: | - sudo apt update - - name: Set up Docker Compose environment - run: | - mkdir -m 777 $REDIS_ENV_WORK_DIR - make start version=${{ matrix.redis_version }} - - name: Maven offline - run: | - mvn -q dependency:go-offline - continue-on-error: true - - name: Run tests - run: | - export TEST_WORK_FOLDER=$REDIS_ENV_WORK_DIR - echo $TEST_WORK_FOLDER - ls -la $TEST_WORK_FOLDER - make test-coverage - env: - JVM_OPTS: -Xmx3200m - TERM: dumb - - name: Tear down Docker Compose environment - run: | - docker compose $COMPOSE_ENV_FILES -f src/test/resources/docker-env/docker-compose.yml down - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - - name: Upload test failure reports to Codecov - uses: codecov/test-results-action@v1 - if: always() # always upload test results to include test failures - with: - fail_ci_if_error: false - files: ./target/surefire-reports/TEST*,./target/failsafe-reports/TEST* - verbose: ${{ runner.debug }} - token: ${{ secrets.CODECOV_TOKEN }} + build_using_custom_image: + name: Build and Test using custom image + if: github.event.inputs.client_libs_test_image_tag != '' + uses: ./.github/workflows/run-tests.yml + with: + client_libs_test_image_tag: ${{ github.event.inputs.client_libs_test_image_tag }} + secrets: + codecov_token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000000..d5934cb4e8 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,166 @@ +# Note: this workflow is used as a part of redis oss release and test automation in +# redis-developer/redis-oss-release-automation repo +name: Run Lettuce Tests + +on: + workflow_call: + inputs: + redis_version: + description: 'Redis version to test against' + required: false + type: string + client_libs_test_image_tag: + description: 'Custom client libs test image tag to use instead of redis_version' + required: false + type: string + default: '' + client_libs_test_image: + description: 'Custom client libs test image name to use' + required: false + type: string + default: '' + java_version: + description: 'Java version to use' + required: false + type: string + default: '8' + java_distribution: + description: 'Java distribution to use' + required: false + type: string + default: 'temurin' + # repository and ref are required for correct checkout when using workflow + # externally (e.g.: in redis-developer/redis-oss-release-automation) + repository: + description: 'Git repository to checkout' + required: false + type: string + default: '' + ref: + description: 'Git ref to checkout' + required: false + type: string + default: '' + redis_env_work_dir: + description: 'Redis env work directory' + required: false + type: string + default: '' + redis_env_conf_dir: + description: 'Redis env conf directory' + required: false + type: string + default: '' + upload_coverage: + description: 'Whether to upload coverage reports to Codecov' + required: false + type: boolean + default: true + secrets: + codecov_token: + description: 'Codecov token for uploading coverage' + required: false + +jobs: + test: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + + - name: Validate inputs and prepare environment + id: args + env: + REDIS_VERSION: ${{ inputs.redis_version }} + CLIENT_LIBS_TEST_IMAGE: ${{ inputs.client_libs_test_image }} + CLIENT_LIBS_TEST_IMAGE_TAG: ${{ inputs.client_libs_test_image_tag }} + REDIS_ENV_WORK_DIR: ${{ inputs.redis_env_work_dir }} + REDIS_ENV_CONF_DIR: ${{ inputs.redis_env_conf_dir }} + run: | + make_args=() + + if [ -n "$CLIENT_LIBS_TEST_IMAGE" ]; then + make_args+=(CLIENT_LIBS_TEST_IMAGE="$CLIENT_LIBS_TEST_IMAGE") + fi + + if [ -n "$CLIENT_LIBS_TEST_IMAGE_TAG" ]; then + make_args+=(CLIENT_LIBS_TEST_IMAGE_TAG="$CLIENT_LIBS_TEST_IMAGE_TAG") + elif [ -n "$REDIS_VERSION" ]; then + make_args+=(version="$REDIS_VERSION") + else + echo "Error: redis_version or client_libs_test_image_tag input is required" + exit 1 + fi + + echo "make_args=${make_args[*]}" | tee -a ${GITHUB_OUTPUT} + + if [ -z "$REDIS_ENV_CONF_DIR" ]; then + REDIS_ENV_CONF_DIR="${{ github.workspace }}/src/test/resources/docker-env" + fi + echo "redis_env_conf_dir=$REDIS_ENV_CONF_DIR" | tee -a ${GITHUB_OUTPUT} + + if [ -n "$REDIS_ENV_WORK_DIR" ]; then + echo "redis_env_work_dir=$REDIS_ENV_WORK_DIR" | tee -a ${GITHUB_OUTPUT} + else + REDIS_ENV_WORK_DIR=$(mktemp -du) + echo "redis_env_work_dir=$REDIS_ENV_WORK_DIR" | tee -a ${GITHUB_OUTPUT} + fi + + - name: Set up Java + uses: actions/setup-java@v4 + with: + java-version: ${{ inputs.java_version }} + distribution: ${{ inputs.java_distribution }} + cache: 'maven' + + - name: Setup Maven + uses: s4u/setup-maven-action@v1.19.0 + with: + checkout-enabled: false + java-version: ${{ inputs.java_version }} + + - name: Set up Docker Compose environment + run: | + mkdir -m 777 $REDIS_ENV_WORK_DIR + make start ${{ steps.args.outputs.make_args }} + env: + REDIS_ENV_CONF_DIR: ${{ steps.args.outputs.redis_env_conf_dir }} + REDIS_ENV_WORK_DIR: ${{ steps.args.outputs.redis_env_work_dir }} + + - name: Maven offline + run: mvn -q dependency:go-offline + continue-on-error: true + + - name: Run tests + run: | + export TEST_WORK_FOLDER=$REDIS_ENV_WORK_DIR + make test-coverage + env: + REDIS_ENV_WORK_DIR: ${{ steps.args.outputs.redis_env_work_dir }} + JVM_OPTS: -Xmx3200m + TERM: dumb + + - name: Tear down Docker Compose environment + if: always() + run: make stop + continue-on-error: true + + - name: Upload coverage reports to Codecov + if: ${{ inputs.upload_coverage }} + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.codecov_token }} + + - name: Upload test failure reports to Codecov + if: ${{ always() && inputs.upload_coverage }} + uses: codecov/test-results-action@v1 + with: + fail_ci_if_error: false + files: ./target/surefire-reports/TEST*,./target/failsafe-reports/TEST* + verbose: ${{ runner.debug }} + token: ${{ secrets.codecov_token }} + diff --git a/Makefile b/Makefile index fd46426838..5827503f1d 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ PROFILE ?= ci SUPPORTED_TEST_ENV_VERSIONS := 8.6 8.4 8.2 8.0 7.4 7.2 DEFAULT_TEST_ENV_VERSION := 8.6 REDIS_ENV_WORK_DIR := $(or ${REDIS_ENV_WORK_DIR},$(ROOT_DIR)/work) +MVN_SOCKET_ARGS := -Ddomainsocket="$(REDIS_ENV_WORK_DIR)/socket-6482" -Dsentineldomainsocket="$(REDIS_ENV_WORK_DIR)/socket-26379" start: @if [ -z "$(version)" ]; then \ @@ -13,12 +14,18 @@ start: version="$(DEFAULT_TEST_ENV_VERSION)"; \ fi; \ fi; \ - if ! echo "$(SUPPORTED_TEST_ENV_VERSIONS)" | grep -qw "$$version"; then \ + if [ -n "$$CLIENT_LIBS_TEST_IMAGE_TAG" ]; then \ + display_version="image tag $$CLIENT_LIBS_TEST_IMAGE_TAG"; \ + echo "Using $$display_version"; \ + version=""; \ + elif ! echo "$(SUPPORTED_TEST_ENV_VERSIONS)" | grep -qw "$$version"; then \ echo "Error: Invalid version '$$version'. Supported versions are: $(SUPPORTED_TEST_ENV_VERSIONS)."; \ exit 1; \ + else \ + display_version="version $$version"; \ fi; \ echo "Version: $(version)"; \ - default_env_file="src/test/resources/docker-env/.env"; \ + default_env_file="src/test/resources/docker-env/.env"; \ custom_env_file="src/test/resources/docker-env/.env.v$$version"; \ env_files="--env-file $$default_env_file"; \ if [ -f "$$custom_env_file" ]; then \ @@ -27,15 +34,15 @@ start: echo "Environment work directory: $(REDIS_ENV_WORK_DIR)"; \ rm -rf "$(REDIS_ENV_WORK_DIR)"; \ mkdir -p "$(REDIS_ENV_WORK_DIR)"; \ - docker compose $$env_files -f src/test/resources/docker-env/docker-compose.yml --parallel 1 up -d; \ - echo "Started test environment with Redis version $$version." + docker compose $$env_files -f src/test/resources/docker-env/docker-compose.yml --parallel 1 up -d --wait --quiet-pull; \ + echo "Started test environment with Redis $$display_version."; test: - mvn -DskipITs=false clean compile verify -P$(PROFILE) + mvn -DskipITs=false $(MVN_SOCKET_ARGS) clean compile verify -P$(PROFILE) test-coverage: - mvn -DskipITs=false clean compile verify jacoco:report -P$(PROFILE) + mvn -DskipITs=false $(MVN_SOCKET_ARGS) clean compile verify jacoco:report -P$(PROFILE) stop: docker compose --env-file src/test/resources/docker-env/.env -f src/test/resources/docker-env/docker-compose.yml down; \ diff --git a/src/test/resources/docker-env/docker-compose.yml b/src/test/resources/docker-env/docker-compose.yml index 10cb2aca67..848ed5e2be 100644 --- a/src/test/resources/docker-env/docker-compose.yml +++ b/src/test/resources/docker-env/docker-compose.yml @@ -1,7 +1,7 @@ x-client-libs-image: &client-libs-image - image: "redislabs/client-libs-test:${REDIS_VERSION:-8.6.0}" + image: "${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test}:${CLIENT_LIBS_TEST_IMAGE_TAG:-${REDIS_VERSION:-8.6.0}}" x-client-libs-stack-image: &client-libs-stack-image - image: "redislabs/client-libs-test:${REDIS_STACK_VERSION:-8.6.0}" + image: "${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test}:${CLIENT_LIBS_TEST_IMAGE_TAG:-${REDIS_STACK_VERSION:-${REDIS_VERSION:-8.6.0}}}" services: # Test infrastructure used for simulating network issues to test multi-db client failover