diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..82356bafa595 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,423 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: full-ci +on: + workflow_call: +env: + FAIL_FAST: ${{ github.event_name == 'pull_request' }} + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 +jobs: + build-info: + runs-on: ubuntu-20.04 + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + outputs: + basic-checks: ${{ steps.selective-checks.outputs.basic-checks }} + needs-basic-checks: ${{ steps.selective-checks.outputs.needs-basic-checks }} + needs-build: ${{ steps.selective-checks.outputs.needs-build }} + needs-compile: ${{ steps.selective-checks.outputs.needs-compile }} + needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }} + needs-dependency-check: ${{ steps.selective-checks.outputs.needs-dependency-check }} + needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }} + needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }} + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: Fetch incoming commit ${{ github.sha }} with its parent + uses: actions/checkout@v3 + with: + ref: ${{ github.sha }} + fetch-depth: 2 + persist-credentials: false + if: github.event_name == 'pull_request' + - name: Selective checks + id: selective-checks + env: + PR_LABELS: "${{ toJSON(github.event.pull_request.labels.*.name) }}" + PR_DRAFT: "${{ github.event.pull_request.draft }}" + run: | + if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + # Run selective checks + dev-support/ci/selective_ci_checks.sh "${GITHUB_SHA}" + else + # Run all checks + dev-support/ci/selective_ci_checks.sh + fi + build: + needs: + - build-info + runs-on: ubuntu-20.04 + timeout-minutes: 30 + if: needs.build-info.outputs.needs-build == 'true' + strategy: + matrix: + java: [ 8 ] + fail-fast: false + steps: + - name: Checkout project + uses: actions/checkout@v3 + - name: Cache for npm dependencies + uses: actions/cache@v3 + with: + path: | + ~/.pnpm-store + **/node_modules + key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm- + - name: Cache for maven dependencies + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }} + restore-keys: | + maven-repo-${{ hashFiles('**/pom.xml') }} + maven-repo- + - name: Setup java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + - name: Run a full build + run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist -Psrc + - name: Store binaries for tests + uses: actions/upload-artifact@v3 + with: + name: ozone-bin + path: | + hadoop-ozone/dist/target/ozone-*.tar.gz + !hadoop-ozone/dist/target/ozone-*-src.tar.gz + retention-days: 1 + - name: Store source tarball for compilation + uses: actions/upload-artifact@v3 + with: + name: ozone-src + path: hadoop-ozone/dist/target/ozone-*-src.tar.gz + retention-days: 1 + - name: Delete temporary build artifacts before caching + run: | + #Never cache local artifacts + rm -rf ~/.m2/repository/org/apache/ozone/hdds* + rm -rf ~/.m2/repository/org/apache/ozone/ozone* + if: always() + compile: + needs: + - build-info + - build + runs-on: ubuntu-20.04 + timeout-minutes: 30 + if: needs.build-info.outputs.needs-compile == 'true' + strategy: + matrix: + java: [ 11 ] + fail-fast: false + steps: + - name: Download Ozone source tarball + uses: actions/download-artifact@v3 + with: + name: ozone-src + - name: Untar sources + run: | + tar --strip-components 1 -xzvf ozone*-src.tar.gz + - name: Cache for maven dependencies + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }} + restore-keys: | + maven-repo-${{ hashFiles('**/pom.xml') }} + maven-repo- + - name: Setup java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + - name: Compile Ozone using Java ${{ matrix.java }} + run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx -Dskip.installnpx + - name: Delete temporary build artifacts before caching + run: | + #Never cache local artifacts + rm -rf ~/.m2/repository/org/apache/ozone/hdds* + rm -rf ~/.m2/repository/org/apache/ozone/ozone* + if: always() + basic: + needs: + - build-info + runs-on: ubuntu-20.04 + timeout-minutes: 90 + if: needs.build-info.outputs.needs-basic-checks == 'true' + strategy: + matrix: + check: ${{ fromJson(needs.build-info.outputs.basic-checks) }} + fail-fast: false + steps: + - name: Checkout project + uses: actions/checkout@v3 + if: matrix.check != 'bats' + - name: Checkout project with history + uses: actions/checkout@v3 + with: + fetch-depth: 0 + if: matrix.check == 'bats' + - name: Cache for maven dependencies + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }} + restore-keys: | + maven-repo-${{ hashFiles('**/pom.xml') }}-8 + maven-repo-${{ hashFiles('**/pom.xml') }} + maven-repo- + if: ${{ !contains('author,bats,docs', matrix.check) }} + - name: Setup java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 8 + - name: Execute tests + run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh + - name: Summary of failures + run: cat target/${{ matrix.check }}/summary.txt + if: ${{ !cancelled() }} + - name: Archive build results + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: ${{ matrix.check }} + path: target/${{ matrix.check }} + continue-on-error: true + - name: Delete temporary build artifacts before caching + run: | + #Never cache local artifacts + rm -rf ~/.m2/repository/org/apache/ozone/hdds* + rm -rf ~/.m2/repository/org/apache/ozone/ozone* + if: always() + dependency: + needs: + - build-info + - build + runs-on: ubuntu-20.04 + timeout-minutes: 5 + if: needs.build-info.outputs.needs-dependency-check == 'true' + steps: + - name: Checkout project + uses: actions/checkout@v3 + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v3 + with: + name: ozone-bin + - name: Untar binaries + run: | + mkdir dist + tar -C dist --strip-components 1 -xzf ozone*.tar.gz + - name: Execute tests + run: | + export OZONE_DIST_DIR=`pwd`/dist + ./hadoop-ozone/dev-support/checks/dependency.sh + - name: Archive build results + uses: actions/upload-artifact@v3 + if: always() + with: + name: dependency + path: target/dependency + continue-on-error: true + acceptance: + needs: + - build-info + - build + runs-on: ubuntu-20.04 + timeout-minutes: 150 + if: needs.build-info.outputs.needs-compose-tests == 'true' + strategy: + matrix: + suite: + - secure + - unsecure + - compat + - HA + - MR + - misc + fail-fast: false + steps: + - name: Checkout project + uses: actions/checkout@v3 + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v3 + with: + name: ozone-bin + - name: Untar binaries + run: | + mkdir -p hadoop-ozone/dist/target + tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target + sudo chmod -R a+rwX hadoop-ozone/dist/target + - name: Execute tests + run: | + pushd hadoop-ozone/dist/target/ozone-* + sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws + popd + ./hadoop-ozone/dev-support/checks/acceptance.sh + env: + KEEP_IMAGE: false + OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }} + OZONE_WITH_COVERAGE: true + OZONE_VOLUME_OWNER: 1000 + - name: Archive build results + uses: actions/upload-artifact@v3 + if: always() + with: + name: acceptance-${{ matrix.suite }} + path: target/acceptance + continue-on-error: true + kubernetes: + needs: + - build-info + - build + runs-on: ubuntu-20.04 + timeout-minutes: 60 + if: needs.build-info.outputs.needs-kubernetes-tests == 'true' + steps: + - name: Checkout project + uses: actions/checkout@v3 + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v3 + with: + name: ozone-bin + - name: Untar binaries + run: | + mkdir -p hadoop-ozone/dist/target + tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target + - name: Execute tests + run: | + pushd hadoop-ozone/dist/target/ozone-* + sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws + popd + ./hadoop-ozone/dev-support/checks/kubernetes.sh + - name: Archive build results + uses: actions/upload-artifact@v3 + if: always() + with: + name: kubernetes + path: target/kubernetes + continue-on-error: true + integration: + needs: + - build-info + runs-on: ubuntu-20.04 + timeout-minutes: 150 + if: needs.build-info.outputs.needs-integration-tests == 'true' + strategy: + matrix: + profile: + - client + - filesystem + - hdds + - om + - ozone + - scm + - flaky + fail-fast: false + steps: + - name: Checkout project + uses: actions/checkout@v3 + - name: Cache for maven dependencies + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.profile }} + restore-keys: | + maven-repo-${{ hashFiles('**/pom.xml') }}-8 + maven-repo-${{ hashFiles('**/pom.xml') }} + maven-repo- + - name: Setup java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 8 + - name: Execute tests + run: hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} + if: matrix.profile != 'flaky' + - name: Execute flaky tests + run: hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600 + if: matrix.profile == 'flaky' + - name: Summary of failures + run: cat target/${{ github.job }}/summary.txt + if: always() + - name: Archive build results + uses: actions/upload-artifact@v3 + if: always() + with: + name: it-${{ matrix.profile }} + path: target/integration + continue-on-error: true + - name: Delete temporary build artifacts before caching + run: | + #Never cache local artifacts + rm -rf ~/.m2/repository/org/apache/ozone/hdds* + rm -rf ~/.m2/repository/org/apache/ozone/ozone* + if: always() + coverage: + runs-on: ubuntu-20.04 + timeout-minutes: 30 + if: github.repository == 'apache/ozone' && github.event_name != 'pull_request' + needs: + - acceptance + - basic + - integration + steps: + - name: Checkout project + uses: actions/checkout@v3 + - name: Cache for maven dependencies + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ github.job }} + restore-keys: | + maven-repo-${{ hashFiles('**/pom.xml') }}-8 + maven-repo-${{ hashFiles('**/pom.xml') }} + maven-repo- + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + path: target/artifacts + - name: Untar binaries + run: | + mkdir -p hadoop-ozone/dist/target + tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target + - name: Calculate combined coverage + run: ./hadoop-ozone/dev-support/checks/coverage.sh + - name: Setup java 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 11 + - name: Upload coverage to Sonar + run: ./hadoop-ozone/dev-support/checks/sonar.sh + env: + SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Archive build results + uses: actions/upload-artifact@v3 + with: + name: coverage + path: target/coverage + continue-on-error: true + - name: Delete temporary build artifacts before caching + run: | + #Never cache local artifacts + rm -rf ~/.m2/repository/org/apache/ozone/hdds* + rm -rf ~/.m2/repository/org/apache/ozone/ozone* + if: always() diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index d7cddfa4df88..c38a0b058d49 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -17,414 +17,9 @@ on: pull_request: types: [opened, ready_for_review, synchronize] push: - schedule: - - cron: 30 0,12 * * * -env: - FAIL_FAST: ${{ github.event_name == 'pull_request' }} - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 concurrency: group: ci-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - build-info: - runs-on: ubuntu-20.04 - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - outputs: - basic-checks: ${{ steps.selective-checks.outputs.basic-checks }} - needs-basic-checks: ${{ steps.selective-checks.outputs.needs-basic-checks }} - needs-build: ${{ steps.selective-checks.outputs.needs-build }} - needs-compile: ${{ steps.selective-checks.outputs.needs-compile }} - needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }} - needs-dependency-check: ${{ steps.selective-checks.outputs.needs-dependency-check }} - needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }} - needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 - with: - persist-credentials: false - - name: Fetch incoming commit ${{ github.sha }} with its parent - uses: actions/checkout@v3 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - if: github.event_name == 'pull_request' - - name: Selective checks - id: selective-checks - env: - PR_LABELS: "${{ toJSON(github.event.pull_request.labels.*.name) }}" - PR_DRAFT: "${{ github.event.pull_request.draft }}" - run: | - if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - # Run selective checks - dev-support/ci/selective_ci_checks.sh "${GITHUB_SHA}" - else - # Run all checks - dev-support/ci/selective_ci_checks.sh - fi - build: - needs: - - build-info - runs-on: ubuntu-20.04 - timeout-minutes: 30 - if: needs.build-info.outputs.needs-build == 'true' - strategy: - matrix: - java: [ 8 ] - fail-fast: false - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Cache for npm dependencies - uses: actions/cache@v3 - with: - path: | - ~/.pnpm-store - **/node_modules - key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm- - - name: Cache for maven dependencies - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }} - restore-keys: | - maven-repo-${{ hashFiles('**/pom.xml') }} - maven-repo- - - name: Setup java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - name: Run a full build - run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist -Psrc - - name: Store binaries for tests - uses: actions/upload-artifact@v3 - with: - name: ozone-bin - path: | - hadoop-ozone/dist/target/ozone-*.tar.gz - !hadoop-ozone/dist/target/ozone-*-src.tar.gz - retention-days: 1 - - name: Store source tarball for compilation - uses: actions/upload-artifact@v3 - with: - name: ozone-src - path: hadoop-ozone/dist/target/ozone-*-src.tar.gz - retention-days: 1 - - name: Delete temporary build artifacts before caching - run: | - #Never cache local artifacts - rm -rf ~/.m2/repository/org/apache/ozone/hdds* - rm -rf ~/.m2/repository/org/apache/ozone/ozone* - if: always() - compile: - needs: - - build-info - - build - runs-on: ubuntu-20.04 - timeout-minutes: 30 - if: needs.build-info.outputs.needs-compile == 'true' - strategy: - matrix: - java: [ 11 ] - fail-fast: false - steps: - - name: Download Ozone source tarball - uses: actions/download-artifact@v3 - with: - name: ozone-src - - name: Untar sources - run: | - tar --strip-components 1 -xzvf ozone*-src.tar.gz - - name: Cache for maven dependencies - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }} - restore-keys: | - maven-repo-${{ hashFiles('**/pom.xml') }} - maven-repo- - - name: Setup java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - name: Compile Ozone using Java ${{ matrix.java }} - run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx -Dskip.installnpx - - name: Delete temporary build artifacts before caching - run: | - #Never cache local artifacts - rm -rf ~/.m2/repository/org/apache/ozone/hdds* - rm -rf ~/.m2/repository/org/apache/ozone/ozone* - if: always() - basic: - needs: - - build-info - runs-on: ubuntu-20.04 - timeout-minutes: 90 - if: needs.build-info.outputs.needs-basic-checks == 'true' - strategy: - matrix: - check: ${{ fromJson(needs.build-info.outputs.basic-checks) }} - fail-fast: false - steps: - - name: Checkout project - uses: actions/checkout@v3 - if: matrix.check != 'bats' - - name: Checkout project with history - uses: actions/checkout@v3 - with: - fetch-depth: 0 - if: matrix.check == 'bats' - - name: Cache for maven dependencies - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }} - restore-keys: | - maven-repo-${{ hashFiles('**/pom.xml') }}-8 - maven-repo-${{ hashFiles('**/pom.xml') }} - maven-repo- - if: ${{ !contains('author,bats,docs', matrix.check) }} - - name: Setup java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 8 - - name: Execute tests - run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh - - name: Summary of failures - run: cat target/${{ matrix.check }}/summary.txt - if: ${{ !cancelled() }} - - name: Archive build results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: ${{ matrix.check }} - path: target/${{ matrix.check }} - continue-on-error: true - - name: Delete temporary build artifacts before caching - run: | - #Never cache local artifacts - rm -rf ~/.m2/repository/org/apache/ozone/hdds* - rm -rf ~/.m2/repository/org/apache/ozone/ozone* - if: always() - dependency: - needs: - - build-info - - build - runs-on: ubuntu-20.04 - timeout-minutes: 5 - if: needs.build-info.outputs.needs-dependency-check == 'true' - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Download compiled Ozone binaries - uses: actions/download-artifact@v3 - with: - name: ozone-bin - - name: Untar binaries - run: | - mkdir dist - tar -C dist --strip-components 1 -xzf ozone*.tar.gz - - name: Execute tests - run: | - export OZONE_DIST_DIR=`pwd`/dist - ./hadoop-ozone/dev-support/checks/dependency.sh - - name: Archive build results - uses: actions/upload-artifact@v3 - if: always() - with: - name: dependency - path: target/dependency - continue-on-error: true - acceptance: - needs: - - build-info - - build - runs-on: ubuntu-20.04 - timeout-minutes: 150 - if: needs.build-info.outputs.needs-compose-tests == 'true' - strategy: - matrix: - suite: - - secure - - unsecure - - compat - - HA - - MR - - misc - fail-fast: false - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Download compiled Ozone binaries - uses: actions/download-artifact@v3 - with: - name: ozone-bin - - name: Untar binaries - run: | - mkdir -p hadoop-ozone/dist/target - tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target - sudo chmod -R a+rwX hadoop-ozone/dist/target - - name: Execute tests - run: | - pushd hadoop-ozone/dist/target/ozone-* - sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws - popd - ./hadoop-ozone/dev-support/checks/acceptance.sh - env: - KEEP_IMAGE: false - OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }} - OZONE_WITH_COVERAGE: true - OZONE_VOLUME_OWNER: 1000 - - name: Archive build results - uses: actions/upload-artifact@v3 - if: always() - with: - name: acceptance-${{ matrix.suite }} - path: target/acceptance - continue-on-error: true - kubernetes: - needs: - - build-info - - build - runs-on: ubuntu-20.04 - timeout-minutes: 60 - if: needs.build-info.outputs.needs-kubernetes-tests == 'true' - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Download compiled Ozone binaries - uses: actions/download-artifact@v3 - with: - name: ozone-bin - - name: Untar binaries - run: | - mkdir -p hadoop-ozone/dist/target - tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target - - name: Execute tests - run: | - pushd hadoop-ozone/dist/target/ozone-* - sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws - popd - ./hadoop-ozone/dev-support/checks/kubernetes.sh - - name: Archive build results - uses: actions/upload-artifact@v3 - if: always() - with: - name: kubernetes - path: target/kubernetes - continue-on-error: true - integration: - needs: - - build-info - runs-on: ubuntu-20.04 - timeout-minutes: 150 - if: needs.build-info.outputs.needs-integration-tests == 'true' - strategy: - matrix: - profile: - - client - - filesystem - - hdds - - om - - ozone - - scm - - flaky - fail-fast: false - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Cache for maven dependencies - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.profile }} - restore-keys: | - maven-repo-${{ hashFiles('**/pom.xml') }}-8 - maven-repo-${{ hashFiles('**/pom.xml') }} - maven-repo- - - name: Setup java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 8 - - name: Execute tests - run: hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} - if: matrix.profile != 'flaky' - - name: Execute flaky tests - run: hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600 - if: matrix.profile == 'flaky' - - name: Summary of failures - run: cat target/${{ github.job }}/summary.txt - if: always() - - name: Archive build results - uses: actions/upload-artifact@v3 - if: always() - with: - name: it-${{ matrix.profile }} - path: target/integration - continue-on-error: true - - name: Delete temporary build artifacts before caching - run: | - #Never cache local artifacts - rm -rf ~/.m2/repository/org/apache/ozone/hdds* - rm -rf ~/.m2/repository/org/apache/ozone/ozone* - if: always() - coverage: - runs-on: ubuntu-20.04 - timeout-minutes: 30 - if: github.repository == 'apache/ozone' && github.event_name != 'pull_request' - needs: - - acceptance - - basic - - integration - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Cache for maven dependencies - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ github.job }} - restore-keys: | - maven-repo-${{ hashFiles('**/pom.xml') }}-8 - maven-repo-${{ hashFiles('**/pom.xml') }} - maven-repo- - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - path: target/artifacts - - name: Untar binaries - run: | - mkdir -p hadoop-ozone/dist/target - tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target - - name: Calculate combined coverage - run: ./hadoop-ozone/dev-support/checks/coverage.sh - - name: Setup java 11 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 11 - - name: Upload coverage to Sonar - run: ./hadoop-ozone/dev-support/checks/sonar.sh - env: - SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Archive build results - uses: actions/upload-artifact@v3 - with: - name: coverage - path: target/coverage - continue-on-error: true - - name: Delete temporary build artifacts before caching - run: | - #Never cache local artifacts - rm -rf ~/.m2/repository/org/apache/ozone/hdds* - rm -rf ~/.m2/repository/org/apache/ozone/ozone* - if: always() + CI: + uses: ./.github/workflows/ci.yml diff --git a/.github/workflows/scheduled_ci.yml b/.github/workflows/scheduled_ci.yml new file mode 100644 index 000000000000..b89f52a4f189 --- /dev/null +++ b/.github/workflows/scheduled_ci.yml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: scheduled-ci +on: + schedule: + - cron: 30 0,12 * * * +jobs: + CI: + uses: ./.github/workflows/ci.yml