ci.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: full-ci
on:
  workflow_call:
    inputs:
      ratis_args:
        type: string
        description: Version overrides from custom Ratis build
        default: ''
        required: false
      ref:
        type: string
        description: Ozone ref (branch, tag or commit SHA)
        default: ''
        required: false
env:
  FAIL_FAST: ${{ github.event_name == 'pull_request' }}
  # Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
  TEST_JAVA_VERSION: 17 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
  HADOOP_IMAGE: ghcr.io/apache/hadoop
  OZONE_IMAGE: ghcr.io/apache/ozone
  OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
  OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}
jobs:
  build-info:
    runs-on: ubuntu-20.04
    env:
      GITHUB_CONTEXT: ${{ toJson(github) }}
    outputs:
      acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
      needs-basic-check: ${{ steps.categorize-basic-checks.outputs.needs-basic-check }}
      needs-native-check: ${{ steps.categorize-basic-checks.outputs.needs-native-check }}
      basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
      needs-build: ${{ steps.selective-checks.outputs.needs-build }}
      needs-compile: ${{ steps.selective-checks.outputs.needs-compile }}
      needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }}
      needs-dependency-check: ${{ steps.selective-checks.outputs.needs-dependency-check }}
      needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }}
      needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }}
      sha: ${{ steps.get-sha.outputs.sha }}
    steps:
      - name: "Checkout ${{ github.ref }} / ${{ github.sha }} (push)"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
        if: github.event_name  == 'push'
      - name: "Checkout ${{ github.sha }} with its parent (pull request)"
        uses: actions/checkout@v4
        with:
          ref: ${{ github.sha }}
          fetch-depth: 2
          persist-credentials: false
        if: github.event_name  == 'pull_request'
      - name: "Checkout ${{ inputs.ref }} given in workflow input (manual dispatch)"
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
        if: github.event_name  == 'workflow_dispatch'
      - name: Get SHA of ${{ inputs.ref || github.ref }}
        id: get-sha
        run: |
          if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
            sha="$(git rev-parse --verify HEAD)"
          else
            sha="${GITHUB_SHA}"
          fi
          echo "sha=$sha" >> $GITHUB_OUTPUT
      - name: Selective checks
        id: selective-checks
        env:
          PR_LABELS: "${{ toJSON(github.event.pull_request.labels.*.name) }}"
          PR_DRAFT: "${{ github.event.pull_request.draft }}"
        run: |
          if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
            # Run selective checks
            dev-support/ci/selective_ci_checks.sh "${GITHUB_SHA}"
          else
            # Run all checks
            dev-support/ci/selective_ci_checks.sh
          fi
      - name: Acceptance suites
        id: acceptance-suites
        run: dev-support/ci/acceptance_suites.sh
      - name: Categorize Basic Checks
        id: categorize-basic-checks
        env:
          ALL_BASIC_CHECKS: "${{ steps.selective-checks.outputs.basic-checks }}"
        run: dev-support/ci/categorize_basic_checks.sh
  build:
    needs:
      - build-info
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    if: needs.build-info.outputs.needs-build == 'true'
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.build-info.outputs.sha }}
      - name: Cache for npm dependencies
        uses: actions/cache@v4
        with:
          path: |
            ~/.pnpm-store
            **/node_modules
          key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-
      - name: Cache for maven dependencies
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/ozone
          key: maven-repo-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-repo-
      - name: Download Ratis repo
        if: ${{ inputs.ratis_args != '' }}
        uses: actions/download-artifact@v4
        with:
          name: ratis-jars
          path: |
            ~/.m2/repository/org/apache/ratis
      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ env.TEST_JAVA_VERSION }}
      - name: Run a full build
        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
        env:
          DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      - name: Store binaries for tests
        uses: actions/upload-artifact@v4
        with:
          name: ozone-bin
          path: |
            hadoop-ozone/dist/target/ozone-*.tar.gz
            !hadoop-ozone/dist/target/ozone-*-src.tar.gz
          retention-days: 1
      - name: Store source tarball for compilation
        uses: actions/upload-artifact@v4
        if: needs.build-info.outputs.needs-compile == 'true'
        with:
          name: ozone-src
          path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
          retention-days: 1
      - name: Store Maven repo for tests
        uses: actions/upload-artifact@v4
        with:
          name: ozone-repo
          path: |
            ~/.m2/repository/org/apache/ozone
          retention-days: 1
  compile:
    needs:
      - build-info
      - build
      - basic
      - dependency
      - license
    timeout-minutes: 45
    if: needs.build-info.outputs.needs-compile == 'true'
    strategy:
      matrix:
        java: [ 11, 17, 21 ]
        include:
          - os: ubuntu-20.04
          - java: 8
            os: macos-13
      fail-fast: false
    runs-on: ${{ matrix.os }}
    steps:
      - name: Download Ozone source tarball
        uses: actions/download-artifact@v4
        with:
          name: ozone-src
      - name: Untar sources
        run: |
          tar --strip-components 1 -xzvf ozone*-src.tar.gz
      - name: Workaround for HADOOP-19011
        run: |
          git init
          git config user.name 'Github Actions'
          git config user.email 'noreply@github.com'
          git commit --allow-empty -a -m 'workaround for HADOOP-19011'
      - name: Cache for maven dependencies
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/ozone
          key: maven-repo-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-repo-
      - name: Download Ratis repo
        if: ${{ inputs.ratis_args != '' }}
        uses: actions/download-artifact@v4
        with:
          name: ratis-jars
          path: |
            ~/.m2/repository/org/apache/ratis
      - name: Setup java ${{ matrix.java }}
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ matrix.java }}
      - name: Compile Ozone using Java ${{ matrix.java }}
        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
        env:
          OZONE_WITH_COVERAGE: false
          DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
  basic:
    needs:
      - build-info
    runs-on: ubuntu-20.04
    timeout-minutes: 30
    if: needs.build-info.outputs.needs-basic-check == 'true'
    strategy:
      matrix:
        check: ${{ fromJson(needs.build-info.outputs.basic-checks) }}
      fail-fast: false
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.build-info.outputs.sha }}
        if: matrix.check != 'bats'
      - name: Checkout project with history
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ needs.build-info.outputs.sha }}
        if: matrix.check == 'bats'
      - name: Cache for maven dependencies
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/ozone
          key: maven-repo-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-repo-
        if: ${{ !contains('author,bats,docs', matrix.check) }}
      - name: Download Ratis repo
        if: ${{ inputs.ratis_args != '' }}
        uses: actions/download-artifact@v4
        with:
          name: ratis-jars
          path: |
            ~/.m2/repository/org/apache/ratis
      - name: Setup java 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 8
      - name: Execute tests
        run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh ${{ inputs.ratis_args }}
        env:
          DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      - name: Summary of failures
        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ matrix.check }}/summary.txt
        if: ${{ failure() }}
      - name: Archive build results
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: ${{ matrix.check }}
          path: target/${{ matrix.check }}
        continue-on-error: true
  native:
    needs:
      - build-info
      - basic
    runs-on: ubuntu-20.04
    timeout-minutes: 150
    if: needs.build-info.outputs.needs-native-check == 'true'
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.build-info.outputs.sha }}
      - name: Cache for maven dependencies
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/ozone
          key: maven-repo-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-repo-
      - name: Download Ratis repo
        if: ${{ inputs.ratis_args != '' }}
        uses: actions/download-artifact@v4
        with:
          name: ratis-jars
          path: |
            ~/.m2/repository/org/apache/ratis
      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ env.TEST_JAVA_VERSION }}
      - name: Execute tests
        run: hadoop-ozone/dev-support/checks/${{ github.job }}.sh ${{ inputs.ratis_args }}
        env:
          DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      - name: Summary of failures
        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
        if: ${{ failure() }}
      - name: Archive build results
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: ${{ github.job }}
          path: target/${{ github.job }}
        continue-on-error: true
  dependency:
    needs:
      - build-info
      - build
    runs-on: ubuntu-20.04
    timeout-minutes: 5
    if: needs.build-info.outputs.needs-dependency-check == 'true'
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.build-info.outputs.sha }}
      - name: Download compiled Ozone binaries
        uses: actions/download-artifact@v4
        with:
          name: ozone-bin
      - name: Untar binaries
        run: |
          mkdir dist
          tar  -C dist --strip-components 1 -xzf ozone*.tar.gz
      - name: Execute tests
        run: |
          export OZONE_DIST_DIR=`pwd`/dist
          ./hadoop-ozone/dev-support/checks/dependency.sh
      - name: Archive build results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: dependency
          path: target/dependency
        continue-on-error: true
  license:
    needs:
      - build-info
      - build
    runs-on: ubuntu-20.04
    timeout-minutes: 15
    if: needs.build-info.outputs.needs-dependency-check == 'true'
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.build-info.outputs.sha }}
      - name: Cache for maven dependencies
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/ozone
          key: maven-repo-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-repo-
      - name: Download Ozone repo
        id: download-ozone-repo
        uses: actions/download-artifact@v4
        with:
          name: ozone-repo
          path: |
            ~/.m2/repository/org/apache/ozone
      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ env.TEST_JAVA_VERSION }}
      - name: Execute tests
        run: |
          hadoop-ozone/dev-support/checks/${{ github.job }}.sh
      - name: Summary of failures
        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
        if: ${{ failure() }}
      - name: Archive build results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: ${{ github.job }}
          path: target/${{ github.job }}
        continue-on-error: true
  repro:
    needs:
      - build-info
      - build
    runs-on: ubuntu-20.04
    timeout-minutes: 30
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
      - name: Cache for maven dependencies
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/ozone
          key: maven-repo-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-repo-
      - name: Download Ozone repo
        id: download-ozone-repo
        uses: actions/download-artifact@v4
        with:
          name: ozone-repo
          path: |
            ~/.m2/repository/org/apache/ozone
      - name: Download Ratis repo
        if: ${{ inputs.ratis_args != '' }}
        uses: actions/download-artifact@v4
        with:
          name: ratis-jars
          path: |
            ~/.m2/repository/org/apache/ratis
      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ env.TEST_JAVA_VERSION }}
      - name: Execute tests
        run: |
          hadoop-ozone/dev-support/checks/${{ github.job }}.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
      - name: Summary of failures
        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
        if: ${{ failure() }}
      - name: Install diffoscope
        run: |
          sudo apt update -q
          sudo apt install -y diffoscope
        if: ${{ failure() }}
      - name: Check artifact differences
        run: |
          hadoop-ozone/dev-support/checks/_diffoscope.sh
        if: ${{ failure() }}
      - name: Archive build results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: ${{ github.job }}
          path: target/${{ github.job }}
        continue-on-error: true
  acceptance:
    needs:
      - build-info
      - build
      - basic
      - dependency
      - license
    runs-on: ubuntu-20.04
    timeout-minutes: 150
    if: needs.build-info.outputs.needs-compose-tests == 'true'
    strategy:
      matrix:
        suite: ${{ fromJson(needs.build-info.outputs.acceptance-suites) }}
      fail-fast: false
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.build-info.outputs.sha }}
      - name: Download compiled Ozone binaries
        uses: actions/download-artifact@v4
        with:
          name: ozone-bin
      - name: Untar binaries
        run: |
          mkdir -p hadoop-ozone/dist/target
          tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
          rm ozone*.tar.gz
      - name: Execute tests
        run: |
          pushd hadoop-ozone/dist/target/ozone-*
          sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
          popd
          ./hadoop-ozone/dev-support/checks/acceptance.sh
        env:
          KEEP_IMAGE: false
          OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
          OZONE_VOLUME_OWNER: 1000
      - name: Summary of failures
        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
        if: ${{ failure() }}
      - name: Archive build results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: acceptance-${{ matrix.suite }}
          path: target/acceptance
        continue-on-error: true
  kubernetes:
    needs:
      - build-info
      - build
      - basic
      - dependency
      - license
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    if: needs.build-info.outputs.needs-kubernetes-tests == 'true'
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.build-info.outputs.sha }}
      - name: Download compiled Ozone binaries
        uses: actions/download-artifact@v4
        with:
          name: ozone-bin
      - name: Untar binaries
        run: |
          mkdir -p hadoop-ozone/dist/target
          tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
      - name: Execute tests
        run: |
          pushd hadoop-ozone/dist/target/ozone-*
          sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
          popd
          ./hadoop-ozone/dev-support/checks/kubernetes.sh
      - name: Summary of failures
        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
        if: ${{ failure() }}
      - name: Archive build results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: kubernetes
          path: target/kubernetes
        continue-on-error: true
  integration:
    needs:
      - build-info
      - basic
    runs-on: ubuntu-20.04
    timeout-minutes: 150
    if: needs.build-info.outputs.needs-integration-tests == 'true'
    strategy:
      matrix:
        profile:
          - client
          - container
          - filesystem
          - hdds
          - om
          - ozone
          - recon
          - shell
          - snapshot
          - flaky
      fail-fast: false
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.build-info.outputs.sha }}
      - name: Cache for maven dependencies
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/ozone
          key: maven-repo-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-repo-
      - name: Download Ratis repo
        if: ${{ inputs.ratis_args != '' }}
        uses: actions/download-artifact@v4
        with:
          name: ratis-jars
          path: |
            ~/.m2/repository/org/apache/ratis
      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ env.TEST_JAVA_VERSION }}
      - name: Execute tests
        run: |
          args="${{ inputs.ratis_args }}"
          if [[ "${{ matrix.profile }}" == "flaky" ]]; then
            args="$args -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600"
          fi
          if [[ "${{ matrix.profile }}" != "filesystem" ]]; then
            args="$args -DskipShade"
          fi

          hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} ${args}
        env:
          DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      - name: Summary of failures
        run: |
          if [[ -s "target/${{ github.job }}/summary.md" ]]; then
            cat target/${{ github.job }}/summary.md >> $GITHUB_STEP_SUMMARY
          fi
          hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
        if: ${{ failure() }}
      - name: Archive build results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: it-${{ matrix.profile }}
          path: target/integration
        continue-on-error: true
  coverage:
    runs-on: ubuntu-20.04
    timeout-minutes: 30
    if: github.event_name == 'push'
    needs:
      - build-info
      - acceptance
      - integration
      - native
    steps:
      - name: Checkout project
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ needs.build-info.outputs.sha }}
      - name: Cache for maven dependencies
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/ozone
          key: maven-repo-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            maven-repo-
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          path: target/artifacts
      - name: Untar binaries
        run: |
          mkdir -p hadoop-ozone/dist/target
          tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target
      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ env.TEST_JAVA_VERSION }}
      - name: Calculate combined coverage
        run: ./hadoop-ozone/dev-support/checks/coverage.sh
      - name: Upload coverage to Sonar
        run: ./hadoop-ozone/dev-support/checks/sonar.sh
        if: github.repository == 'apache/ozone'
        env:
          SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      - name: Archive build results
        uses: actions/upload-artifact@v4
        with:
          name: coverage
          path: target/coverage
        continue-on-error: true