diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 000000000000..6d06484d9444 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,289 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This reusable workflow executes a single check from `hadoop-ozone/dev-support/checks/`. +# Before and after the check, it performs various steps based on workflow inputs. + +name: ci-check + +on: + workflow_call: + inputs: + # REQUIRED + script: + type: string + description: "Test script to run from hadoop-ozone/dev-support/checks, without .sh extension" + required: true + + sha: + type: string + description: "Commit SHA to test" + required: true + + # OPTIONAL (ordered alphabetically) + checkout-fetch-depth: + type: number + description: "Fetch depth for checking out the repo (default: no history)" + default: 1 + required: false + + java-version: + type: string + description: "Java version to set up (default: none)" + default: '' + required: false + + needs-maven-cache: + type: boolean + description: "Whether to restore Maven cache before run (default: yes)" + default: true + required: false + + needs-npm-cache: + type: boolean + description: "Whether to restore NPM cache before run (default: no)" + default: false + required: false + + needs-ozone-binary-tarball: + type: boolean + description: "Whether to download Ozone binary tarball created by build (default: no)" + default: false + required: false + + needs-ozone-repo: + type: boolean + description: "Whether to download Ozone jars created by build (default: no)" + default: false + required: false + + needs-ozone-source-tarball: + type: boolean + description: "Whether to download Ozone source tarball created by build (default: no)" + default: false + required: false + + pre-script: + type: string + description: "Command to execute before the test script (default: none)" + default: '' + required: false + + post-failure: + type: string + description: "Command to execute after the test script, if it failed (default: none)" + default: '' + required: false + + post-success: + type: string + description: "Command to execute after the test script, if it succeeded (default: none)" + default: '' + required: false + + ratis-args: + type: string + description: "Version overrides from custom Ratis build (default: none)" + default: '' + required: false + + runner: + type: string + description: "GitHub Actions runner to use" + default: 'ubuntu-24.04' + required: false + + script-args: + type: string + description: "Arguments for the test script, ratis-args are appended" + default: '' + required: false + + split: + type: string + description: "Name of split for matrix jobs, only used in display name" + default: '' + required: false + + timeout-minutes: + type: number + description: "Job timeout in minutes (default: 30)" + default: 30 + required: false + + with-coverage: + type: boolean + description: "The value of OZONE_WITH_COVERAGE to set" + default: true + required: false + +env: + HADOOP_IMAGE: ghcr.io/apache/hadoop + MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + OZONE_IMAGE: ghcr.io/apache/ozone + OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner + OZONE_VOLUME_OWNER: 1000 + +jobs: + check: + name: ${{ (inputs.split && format('{0} ({1})', inputs.script, inputs.split)) || inputs.script }} + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - name: Checkout project + if: ${{ !inputs.needs-ozone-source-tarball }} + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + fetch-depth: ${{ inputs.checkout-fetch-depth }} + + - name: Download Ozone source tarball + if: ${{ inputs.needs-ozone-source-tarball }} + uses: actions/download-artifact@v4 + with: + name: ozone-src + + - name: Extract source tarball + if: ${{ inputs.needs-ozone-source-tarball }} + run: | + tar --strip-components 1 -xzvf ozone*-src.tar.gz + + - name: Cache for NPM dependencies + if: ${{ inputs.needs-npm-cache }} + uses: actions/cache@v4 + with: + path: | + ~/.pnpm-store + **/node_modules + key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm- + + - name: Cache for Maven dependencies + if: ${{ inputs.needs-maven-cache }} + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + + - name: Download Ozone repo + id: download-ozone-repo + if: ${{ inputs.needs-ozone-repo }} + uses: actions/download-artifact@v4 + with: + name: ozone-repo + path: | + ~/.m2/repository/org/apache/ozone + + - name: Download Ratis repo + if: ${{ inputs.ratis-args != '' }} + uses: actions/download-artifact@v4 + with: + name: ratis-jars + path: | + ~/.m2/repository/org/apache/ratis + + - name: Download Ozone binary tarball + if: ${{ inputs.needs-ozone-binary-tarball }} + uses: actions/download-artifact@v4 + with: + name: ozone-bin + + - name: Extract binary tarball + if: ${{ inputs.needs-ozone-binary-tarball }} + run: | + mkdir -p hadoop-ozone/dist/target + tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target + rm ozone*.tar.gz + + - name: Setup java ${{ inputs.java-version }} + if: ${{ inputs.java-version }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ inputs.java-version }} + + - name: Execute pre-test steps + if: ${{ inputs.pre-script }} + run: | + ${{ inputs.pre-script }} + + - name: Execute tests + run: | + hadoop-ozone/dev-support/checks/${{ inputs.script }}.sh ${{ inputs.script-args }} ${{ inputs.ratis-args }} + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + OZONE_WITH_COVERAGE: ${{ inputs.with-coverage }} + + - name: Execute post-failure steps + if: ${{ failure() && inputs.post-failure }} + run: | + ${{ inputs.post-failure }} + + - name: Execute post-success steps + if: ${{ !failure() && inputs.post-success }} + run: | + ${{ inputs.post-success }} + + - name: Summary of failures + if: ${{ failure() }} + run: | + if [[ -s "target/${{ inputs.script }}/summary.md" ]]; then + cat target/${{ inputs.script }}/summary.md >> $GITHUB_STEP_SUMMARY + fi + hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt + + - name: Archive build results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ (inputs.split && format('{0}-{1}', inputs.script, inputs.split)) || inputs.script }} + path: target/${{ inputs.script }} + continue-on-error: true + + # The following steps are hard-coded to be run only for 'build' check, + # to avoid the need for 3 more inputs. + - name: Store binaries for tests + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ozone-bin + path: | + hadoop-ozone/dist/target/ozone-*.tar.gz + !hadoop-ozone/dist/target/ozone-*-src.tar.gz + retention-days: 1 + + - name: Store source tarball for compilation + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ozone-src + path: | + hadoop-ozone/dist/target/ozone-*-src.tar.gz + retention-days: 1 + + - name: Store Maven repo for tests + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ozone-repo + path: | + ~/.m2/repository/org/apache/ozone + retention-days: 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87193106c287..f68ae511119d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,15 +27,14 @@ on: default: '' required: false env: - FAIL_FAST: ${{ github.event_name == 'pull_request' }} + BUILD_ARGS: "-Pdist -Psrc -Dmaven.javadoc.skip=true" # Minimum required Java version for running Ozone is defined in pom.xml (javac.version). TEST_JAVA_VERSION: 21 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image + # MAVEN_ARGS and MAVEN_OPTS are duplicated in check.yml, please keep in sync MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - HADOOP_IMAGE: ghcr.io/apache/hadoop - OZONE_IMAGE: ghcr.io/apache/ozone - OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }} + jobs: build-info: runs-on: ubuntu-24.04 @@ -53,6 +52,10 @@ jobs: needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }} needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }} sha: ${{ steps.get-sha.outputs.sha }} + # `env` context cannot be used when calling reusable workflow, so we need to convert these to `outputs` + build-args: ${{ env.BUILD_ARGS }} + java-version: ${{ env.TEST_JAVA_VERSION }} + with-coverage: ${{ env.OZONE_WITH_COVERAGE }} steps: - name: "Checkout ${{ github.ref }} / ${{ github.sha }} (push)" uses: actions/checkout@v4 @@ -105,80 +108,23 @@ jobs: env: ALL_BASIC_CHECKS: "${{ steps.selective-checks.outputs.basic-checks }}" run: dev-support/ci/categorize_basic_checks.sh + build: needs: - build-info - runs-on: ubuntu-24.04 - timeout-minutes: 60 if: needs.build-info.outputs.needs-build == 'true' - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.sha }} - - name: Cache for npm dependencies - uses: actions/cache@v4 - with: - path: | - ~/.pnpm-store - **/node_modules - key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm- - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Download Ratis repo - if: ${{ inputs.ratis_args != '' }} - uses: actions/download-artifact@v4 - with: - name: ratis-jars - path: | - ~/.m2/repository/org/apache/ratis - - name: Setup java ${{ env.TEST_JAVA_VERSION }} - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: ${{ env.TEST_JAVA_VERSION }} - - name: Run a full build - run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }} - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: ${{ github.job }} - path: target/${{ github.job }} - continue-on-error: true - - name: Store binaries for tests - uses: actions/upload-artifact@v4 - with: - name: ozone-bin - path: | - hadoop-ozone/dist/target/ozone-*.tar.gz - !hadoop-ozone/dist/target/ozone-*-src.tar.gz - retention-days: 1 - - name: Store source tarball for compilation - uses: actions/upload-artifact@v4 - if: needs.build-info.outputs.needs-compile == 'true' - with: - name: ozone-src - path: hadoop-ozone/dist/target/ozone-*-src.tar.gz - retention-days: 1 - - name: Store Maven repo for tests - uses: actions/upload-artifact@v4 - with: - name: ozone-repo - path: | - ~/.m2/repository/org/apache/ozone - retention-days: 1 + uses: ./.github/workflows/check.yml + with: + java-version: ${{ needs.build-info.outputs.java-version }} + needs-npm-cache: true + ratis-args: ${{ inputs.ratis_args }} + script: build + script-args: ${{ needs.build-info.outputs.build-args }} + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 60 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + secrets: inherit + compile: needs: - build-info @@ -186,7 +132,6 @@ jobs: - basic - dependency - license - timeout-minutes: 45 if: needs.build-info.outputs.needs-compile == 'true' strategy: matrix: @@ -196,243 +141,80 @@ jobs: - java: 8 os: macos-13 fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - name: Download Ozone source tarball - uses: actions/download-artifact@v4 - with: - name: ozone-src - - name: Untar sources - run: | - tar --strip-components 1 -xzvf ozone*-src.tar.gz - - name: Workaround for HADOOP-19011 - run: | - git init - git config user.name 'Github Actions' - git config user.email 'noreply@github.com' - git commit --allow-empty -a -m 'workaround for HADOOP-19011' - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Download Ratis repo - if: ${{ inputs.ratis_args != '' }} - uses: actions/download-artifact@v4 - with: - name: ratis-jars - path: | - ~/.m2/repository/org/apache/ratis - - name: Setup java ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - name: Compile Ozone using Java ${{ matrix.java }} - run: hadoop-ozone/dev-support/checks/compile.sh -Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }} - env: - OZONE_WITH_COVERAGE: false - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: ${{ github.job }}-${{ matrix.java }} - path: target/${{ github.job }} - continue-on-error: true + uses: ./.github/workflows/check.yml + with: + java-version: ${{ matrix.java }} + needs-ozone-source-tarball: true + ratis-args: ${{ inputs.ratis_args }} + runner: ${{ matrix.os }} + script: compile + script-args: "-Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }}" + sha: ${{ needs.build-info.outputs.sha }} + split: ${{ matrix.java }} + timeout-minutes: 45 + with-coverage: false + secrets: inherit + basic: needs: - build-info - runs-on: ubuntu-24.04 - timeout-minutes: 30 if: needs.build-info.outputs.needs-basic-check == 'true' + uses: ./.github/workflows/check.yml + with: + checkout-fetch-depth: ${{ matrix.check != 'bats' && 1 || 0 }} + java-version: 8 # HDDS-10150 + needs-maven-cache: ${{ !contains('author,bats', matrix.check) }} + ratis-args: ${{ inputs.ratis_args }} + script: ${{ matrix.check }} + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 30 + secrets: inherit strategy: matrix: check: ${{ fromJson(needs.build-info.outputs.basic-checks) }} fail-fast: false - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.sha }} - if: matrix.check != 'bats' - - name: Checkout project with history - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ needs.build-info.outputs.sha }} - if: matrix.check == 'bats' - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - if: ${{ !contains('author,bats', matrix.check) }} - - name: Download Ratis repo - if: ${{ inputs.ratis_args != '' }} - uses: actions/download-artifact@v4 - with: - name: ratis-jars - path: | - ~/.m2/repository/org/apache/ratis - - name: Setup java 8 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 8 - - name: Execute tests - run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh ${{ inputs.ratis_args }} - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ matrix.check }}/summary.txt - if: ${{ failure() }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: ${{ matrix.check }} - path: target/${{ matrix.check }} - continue-on-error: true + native: needs: - build-info - basic - runs-on: ubuntu-24.04 - timeout-minutes: 150 if: needs.build-info.outputs.needs-native-check == 'true' - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.sha }} - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Download Ratis repo - if: ${{ inputs.ratis_args != '' }} - uses: actions/download-artifact@v4 - with: - name: ratis-jars - path: | - ~/.m2/repository/org/apache/ratis - - name: Setup java ${{ env.TEST_JAVA_VERSION }} - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: ${{ env.TEST_JAVA_VERSION }} - - name: Execute tests - run: hadoop-ozone/dev-support/checks/${{ github.job }}.sh ${{ inputs.ratis_args }} - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt - if: ${{ failure() }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: ${{ github.job }} - path: target/${{ github.job }} - continue-on-error: true + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + ratis-args: ${{ inputs.ratis_args }} + script: native + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 150 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + dependency: needs: - build-info - build - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.sha }} - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Download compiled Ozone binaries - uses: actions/download-artifact@v4 - with: - name: ozone-bin - - name: Untar binaries - run: | - mkdir dist - tar -C dist --strip-components 1 -xzf ozone*.tar.gz - - name: Execute tests - run: | - export OZONE_DIST_DIR=`pwd`/dist - ./hadoop-ozone/dev-support/checks/dependency.sh - - name: Archive build results - uses: actions/upload-artifact@v4 - if: always() - with: - name: dependency - path: target/dependency - continue-on-error: true + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + needs-ozone-binary-tarball: true + script: dependency + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 5 + license: needs: - build-info - build - runs-on: ubuntu-24.04 - timeout-minutes: 15 - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.sha }} - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Download Ozone repo - id: download-ozone-repo - uses: actions/download-artifact@v4 - with: - name: ozone-repo - path: | - ~/.m2/repository/org/apache/ozone - - name: Setup java ${{ env.TEST_JAVA_VERSION }} - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: ${{ env.TEST_JAVA_VERSION }} - - name: Execute tests - run: | - hadoop-ozone/dev-support/checks/${{ github.job }}.sh - - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt - if: ${{ failure() }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ github.job }} - path: target/${{ github.job }} - continue-on-error: true + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + needs-ozone-repo: true + script: license + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 15 + repro: needs: - build-info @@ -440,56 +222,19 @@ jobs: - basic - dependency - license - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - name: Checkout project - uses: actions/checkout@v4 - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Download Ozone repo - id: download-ozone-repo - uses: actions/download-artifact@v4 - with: - name: ozone-repo - path: | - ~/.m2/repository/org/apache/ozone - - name: Download Ratis repo - if: ${{ inputs.ratis_args != '' }} - uses: actions/download-artifact@v4 - with: - name: ratis-jars - path: | - ~/.m2/repository/org/apache/ratis - - name: Setup java ${{ env.TEST_JAVA_VERSION }} - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: ${{ env.TEST_JAVA_VERSION }} - - name: Execute tests - run: | - hadoop-ozone/dev-support/checks/${{ github.job }}.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }} - - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt - if: ${{ failure() }} - - name: Check artifact differences - run: | - hadoop-ozone/dev-support/checks/_diffoscope.sh - if: ${{ failure() }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ github.job }} - path: target/${{ github.job }} - continue-on-error: true + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + needs-ozone-repo: true + ratis-args: ${{ inputs.ratis_args }} + script: repro + script-args: ${{ needs.build-info.outputs.build-args }} + post-failure: hadoop-ozone/dev-support/checks/_diffoscope.sh + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 30 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + acceptance: needs: - build-info @@ -497,57 +242,24 @@ jobs: - basic - dependency - license - runs-on: ubuntu-24.04 - timeout-minutes: 150 if: needs.build-info.outputs.needs-compose-tests == 'true' + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: 11 # Hadoop may not work with newer Java + needs-ozone-binary-tarball: true + ratis-args: ${{ inputs.ratis_args }} + script: acceptance + script-args: ${{ matrix.suite }} + sha: ${{ needs.build-info.outputs.sha }} + split: ${{ matrix.suite }} + timeout-minutes: 150 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} strategy: matrix: suite: ${{ fromJson(needs.build-info.outputs.acceptance-suites) }} fail-fast: false - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.sha }} - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Download compiled Ozone binaries - uses: actions/download-artifact@v4 - with: - name: ozone-bin - - name: Untar binaries - run: | - mkdir -p hadoop-ozone/dist/target - tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target - rm ozone*.tar.gz - - name: Setup java 11 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 11 # Hadoop may not work with newer Java - - name: Execute tests - run: | - ./hadoop-ozone/dev-support/checks/acceptance.sh - env: - KEEP_IMAGE: false - OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }} - - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt - if: ${{ failure() }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: always() - with: - name: acceptance-${{ matrix.suite }} - path: target/acceptance - continue-on-error: true + kubernetes: needs: - build-info @@ -555,100 +267,38 @@ jobs: - basic - dependency - license - runs-on: ubuntu-24.04 - timeout-minutes: 60 if: needs.build-info.outputs.needs-kubernetes-tests == 'true' - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.sha }} - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Download compiled Ozone binaries - uses: actions/download-artifact@v4 - with: - name: ozone-bin - - name: Untar binaries - run: | - mkdir -p hadoop-ozone/dist/target - tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target - - name: Execute tests - run: | - ./hadoop-ozone/dev-support/checks/kubernetes.sh - - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt - if: ${{ failure() }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: always() - with: - name: kubernetes - path: target/kubernetes - continue-on-error: true + uses: ./.github/workflows/check.yml + secrets: inherit + with: + needs-ozone-binary-tarball: true + ratis-args: ${{ inputs.ratis_args }} + script: kubernetes + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 60 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + integration: needs: - build-info - basic - runs-on: ubuntu-24.04 - timeout-minutes: 150 if: needs.build-info.outputs.needs-integration-tests == 'true' + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + ratis-args: ${{ inputs.ratis_args }} + script: integration + script-args: -Ptest-${{ matrix.profile }} + sha: ${{ needs.build-info.outputs.sha }} + split: ${{ matrix.profile }} + timeout-minutes: 150 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} strategy: matrix: profile: ${{ fromJson(needs.build-info.outputs.integration-suites) }} fail-fast: false - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.sha }} - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/ozone - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Download Ratis repo - if: ${{ inputs.ratis_args != '' }} - uses: actions/download-artifact@v4 - with: - name: ratis-jars - path: | - ~/.m2/repository/org/apache/ratis - - name: Setup java ${{ env.TEST_JAVA_VERSION }} - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: ${{ env.TEST_JAVA_VERSION }} - - name: Execute tests - run: | - hadoop-ozone/dev-support/checks/integration.sh -Ptest-${{ matrix.profile }} ${{ inputs.ratis_args }} - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - - name: Summary of failures - run: | - if [[ -s "target/${{ github.job }}/summary.md" ]]; then - cat target/${{ github.job }}/summary.md >> $GITHUB_STEP_SUMMARY - fi - hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt - if: ${{ failure() }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: always() - with: - name: it-${{ matrix.profile }} - path: target/integration - continue-on-error: true + coverage: runs-on: ubuntu-24.04 timeout-minutes: 30 diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh index 4a46efe6d618..c286c9b0f2bb 100755 --- a/hadoop-ozone/dev-support/checks/acceptance.sh +++ b/hadoop-ozone/dev-support/checks/acceptance.sh @@ -22,7 +22,7 @@ cd "$DIR/../../.." || exit 1 OZONE_ROOT=$(pwd -P) : ${HADOOP_AWS_DIR:=""} -: ${OZONE_ACCEPTANCE_SUITE:=""} +: ${OZONE_ACCEPTANCE_SUITE:="${1:-}"} : ${OZONE_TEST_SELECTOR:=""} : ${OZONE_ACCEPTANCE_TEST_TYPE:="robot"} : ${OZONE_WITH_COVERAGE:="false"}