diff --git a/.asf.yaml b/.asf.yaml index 6185dac1c4..3a2806912c 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -47,6 +47,14 @@ github: required_linear_history: true + required_status_checks: + # strict means "Require branches to be up to date before merging". + strict: false + # Contexts are the names of checks that must pass. This is the value + # of the job's `name` property if it's present. + contexts: + - "Required Checks" + features: wiki: false issues: true diff --git a/.github/workflows/check-md-link.yml b/.github/workflows/check-md-link.yml deleted file mode 100644 index e42b6a59b7..0000000000 --- a/.github/workflows/check-md-link.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. -# This workflow will find markdown links in the folder paths -# specified, and confirm the links are valid - -name: Check Markdown links - -on: - push: - branches: [ "main", "release/*" ] - pull_request: - branches: [ "main", "release/*" ] - -jobs: - markdown-link-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Setup test environment - uses: ./.github/actions/setup-test-env - - uses: tcort/github-action-markdown-link-check@e7c7a18363c842693fadde5d41a3bd3573a7a225 - with: - use-quiet-mode: 'yes' - config-file: '.github/workflows/check-md-link-config.json' - folder-path: 'regtests, .github, build-logic, polaris-core, runtime, persistence, spec, getting-started, helm' - file-path: 'CHAT_BYLAWS.md, CODE_OF_CONDUCT.md, CONTRIBUTING.md, README.md, SECURITY.md' diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml new file mode 100644 index 0000000000..ad37556eaf --- /dev/null +++ b/.github/workflows/ci-main.yml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI/main +run-name: CI/${{github.ref_name}} + +on: + push: + branches: [ "main", "release/*" ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + ci-main: + name: CI/${{ github.ref_name }} + uses: ./.github/workflows/ci.yml diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml new file mode 100644 index 0000000000..f7a39a42cc --- /dev/null +++ b/.github/workflows/ci-pr.yml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI/PR +run-name: PR#${{ github.event.number }} ${{ github.event.pull_request.title }} + +on: + pull_request: + branches: [ "main", "release/*" ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + ci-pr: + name: PR#${{ github.event.number }} ${{ github.event.pull_request.title }} + uses: ./.github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..2498dbbdea --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,386 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI + +on: + workflow_call: + # Called from ci-main.yml and ci-pr.yml, which define different concurrency groups. + +jobs: + build-checks: + name: Gradle Build Checks + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: &setup-java-vars + java-version: '21' + distribution: 'temurin' + - name: Setup test environment + uses: ./.github/actions/setup-test-env + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + - name: Run Checks + env: &gradle_env_vars + GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }} + DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }} + DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ./gradlew check sourceTarball distTar distZip publishToMavenLocal \ + -x :polaris-runtime-service:test \ + -x :polaris-admin:test \ + -x intTest \ + --continue + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + - name: Archive test results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: failure() + with: + name: upload-${{ github.job }}-artifacts + path: &test-archive-path | + **/build/test-results/** + **/build/reports/tests/** + + runtime-service-tests: + name: Runtime Service Tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: *setup-java-vars + - name: Setup test environment + uses: ./.github/actions/setup-test-env + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + - name: Run Quarkus runtime tests + env: *gradle_env_vars + run: ./gradlew :polaris-runtime-service:test --continue + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + - name: Archive test results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: failure() + with: + name: upload-${{ github.job }}-artifacts + path: *test-archive-path + + runtime-service-int-tests: + name: Runtime Service Integration Tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: *setup-java-vars + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + - name: Run Quarkus runtime tests + env: *gradle_env_vars + run: ./gradlew :polaris-runtime-service:intTest --continue + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + - name: Archive test results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: failure() + with: + name: upload-${{ github.job }}-artifacts + path: *test-archive-path + + admin-tool-tests: + name: Admin Tool Tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: *setup-java-vars + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + - name: Run Quarkus admin tests + env: *gradle_env_vars + run: ./gradlew :polaris-admin:test --continue + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + - name: Archive test results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: failure() + with: + name: upload-${{ github.job }}-artifacts + path: *test-archive-path + + integration-tests: + name: Other Integration Tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: *setup-java-vars + - name: Setup test environment + uses: ./.github/actions/setup-test-env + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + - name: Run integration tests + env: *gradle_env_vars + run: | + ./gradlew \ + intTest \ + -x :polaris-runtime-service:intTest \ + --continue + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + - name: Archive test results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: failure() + with: + name: upload-${{ github.job }}-artifacts + path: *test-archive-path + + helm-tests: + name: Helm Tests + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: *setup-java-vars + - name: Setup test environment + uses: ./.github/actions/setup-test-env + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + - name: Set up Helm + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 + with: + version: 'v3.16.0' + - name: Set up chart-testing + uses: helm/chart-testing-action@6ec842c01de15ebb84c8627d2744a0c2f2755c9f # v2.8.0 + - name: Run 'helm template' validation + run: | + cd helm/polaris + for f in values.yaml ci/*.yaml; do + echo "::group::helm template $f" + helm template --debug --namespace polaris-ns --values $f . + echo "::endgroup::" + done + - name: Run Helm unit tests + run: | + # Pin version to 1.0.2 due to https://github.com/helm-unittest/helm-unittest/issues/790 + helm plugin install https://github.com/helm-unittest/helm-unittest.git --version 1.0.2 || true + helm unittest helm/polaris + - name: Run chart-testing (lint) + run: ct lint --target-branch ${{ github.event.repository.default_branch }} --debug --charts ./helm/polaris + - name: Set up Minikube + uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # v0.0.21 + - name: Print Docker info + run: | + docker -v + minikube docker-env + - name: Image build + env: *gradle_env_vars + run: | + eval $(minikube -p minikube docker-env) + ./gradlew \ + :polaris-server:assemble \ + :polaris-server:quarkusAppPartsBuild --rerun \ + :polaris-admin:assemble \ + :polaris-admin:quarkusAppPartsBuild --rerun \ + -Dquarkus.container-image.build=true + minikube image ls + - name: Install fixtures + run: | + kubectl create namespace polaris-ns + kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures + - name: Run chart-testing (install) + run: | + ct install --target-branch ${{ github.event.repository.default_branch }} \ + --namespace polaris-ns \ + --debug --charts ./helm/polaris + + python-client: + name: Polaris Python Client Tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: *setup-java-vars + - name: Setup test environment + uses: ./.github/actions/setup-test-env + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ matrix.python-version }} + - name: Lint + run: make client-lint + - name: License Compliance Check + run: make client-license-check + - name: Generated Client Tests + run: make client-unit-test + - name: Integration Tests + run: make client-integration-test + - name: Run Polaris Client help manual + run: ./polaris --help + + regtest: + name: Regression Tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Free disk space + uses: ./.github/actions/free-disk-space + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: *setup-java-vars + - name: Setup test environment + uses: ./.github/actions/setup-test-env + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + - name: Fix permissions + run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/* + - name: Image build + env: *gradle_env_vars + run: | + ./gradlew \ + publishToMavenLocal \ + :polaris-server:assemble \ + :polaris-server:quarkusAppPartsBuild --rerun \ + -Dquarkus.container-image.build=true + - name: Regression Test + env: + AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} + run: docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest + - name: Spark Plugin Regression Test + # NOTE: the regression test runs with spark 3.5.6 and scala 2.12 in Java 17. We also have integration + # tests runs with the existing gradle.yml, which only runs on Java 21. Since spark Java compatibility + # for 3.5 is 8, 11, and 17, we should run spark client with those compatible java versions. + # TODO: add separate spark client CI and run with Java 8, 11 and 17. + run: docker compose -f plugins/spark/v3.5/regtests/docker-compose.yml up --build --exit-code-from regtest + + markdown-link-check: + name: "Markdown Link Check" + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Setup test environment + uses: ./.github/actions/setup-test-env + - name: Markdown link check + uses: tcort/github-action-markdown-link-check@e7c7a18363c842693fadde5d41a3bd3573a7a225 + with: + use-quiet-mode: 'yes' + config-file: '.github/workflows/check-md-link-config.json' + folder-path: 'regtests, .github, build-logic, polaris-core, runtime, persistence, spec, getting-started, helm' + file-path: 'CHAT_BYLAWS.md, CODE_OF_CONDUCT.md, CONTRIBUTING.md, README.md, SECURITY.md' + + site: + name: "Site Generation" + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Checkout Versioned Docs + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: "versioned-docs" + path: site/content/releases + - name: Setup test environment + uses: ./.github/actions/setup-test-env + - name: Install docker-compose + run: | + sudo curl --fail-with-body -SL \ + https://github.com/docker/compose/releases/download/v2.29.2/docker-compose-linux-x86_64 \ + -o /usr/local/bin/docker-compose + sudo chmod 755 /usr/local/bin/docker-compose + sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose + - name: Build Apache Site + run: site/bin/create-static-site.sh + + store-gradle-cache: + name: Store Gradle Cache + runs-on: ubuntu-24.04 + timeout-minutes: 30 + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: + - build-checks + - runtime-service-tests + - runtime-service-int-tests + - admin-tool-tests + - integration-tests + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: *setup-java-vars + - name: Collect partial Gradle build caches + uses: ./.github/actions/ci-incr-build-cache-prepare + with: + cache-read-only: false + + required-checks: + name: "Required Checks" + needs: + - build-checks + - runtime-service-tests + - runtime-service-int-tests + - admin-tool-tests + - integration-tests + - helm-tests + - python-client + - regtest + - markdown-link-check + - site + runs-on: ubuntu-24.04 + steps: + - run: echo "All checks passed" diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml deleted file mode 100644 index be596532c7..0000000000 --- a/.github/workflows/gradle.yml +++ /dev/null @@ -1,224 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. -# This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle - -name: Java CI with Gradle - -on: - push: - branches: [ "main", "release/*" ] - pull_request: - branches: [ "main", "release/*" ] - -env: - GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }} - DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }} - DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }} - -jobs: - - # Durations (approximates from GH CI): - # (all):test 16m - unit-tests: - name: Unit Tests - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Set up JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: &setup-java-vars - java-version: '21' - distribution: 'temurin' - - name: Setup test environment - uses: ./.github/actions/setup-test-env - - name: Prepare Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-prepare - - name: Run unit tests - env: - # publishToMavenLocal causes a GH API requests, use the token for those requests - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: | - ./gradlew check sourceTarball distTar distZip publishToMavenLocal \ - -x :polaris-runtime-service:test \ - -x :polaris-admin:test \ - -x intTest --continue - - name: Save partial Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-save - - name: Archive test results - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: failure() - with: - name: upload-unit-test-artifacts - path: &test-archive-path | - **/build/test-results/** - **/build/reports/tests/** - - # Durations (approximates from GH CI): - # - :polaris-runtime-service:test 22m - quarkus-tests: - name: Quarkus Tests - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Set up JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: *setup-java-vars - - name: Setup test environment - uses: ./.github/actions/setup-test-env - - name: Prepare Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-prepare - - name: Run Quarkus runtime tests - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: | - ./gradlew :polaris-runtime-service:test --continue - - name: Save partial Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-save - - name: Archive test results - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: failure() - with: - name: upload-quarkus-runtime-test-artifacts - path: *test-archive-path - - # Durations (approximates from GH CI): - # - :polaris-runtime-service:intTest 15m - quarkus-int-tests: - name: Quarkus Integration Tests - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Set up JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: *setup-java-vars - - name: Prepare Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-prepare - - name: Run Quarkus runtime tests - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: | - ./gradlew :polaris-runtime-service:intTest --continue - - name: Save partial Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-save - - name: Archive test results - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: failure() - with: - name: upload-quarkus-runtime-inttest-artifacts - path: *test-archive-path - - # Durations (approximates from GH CI): - # - :polaris-admin:test 6m - quarkus-admin-tests: - name: Quarkus Admin Tests - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Set up JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: *setup-java-vars - - name: Prepare Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-prepare - - name: Run Quarkus admin tests - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: | - ./gradlew :polaris-admin:test --continue - - name: Save partial Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-save - - name: Archive test results - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: failure() - with: - name: upload-quarkus-admin-test-artifacts - path: *test-archive-path - - # Durations (approximates from GH CI): - # - :polaris-spark-integration-*:intTest 6m (2x) - # - :polaris-runtime-spark-tests:intTest 4m - # - :polaris-persistence-nosql-mongodb:intTest 1m - integration-tests: - name: Integration Tests - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Set up JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: *setup-java-vars - - name: Setup test environment - uses: ./.github/actions/setup-test-env - - name: Prepare Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-prepare - - name: Run integration tests - run: | - ./gradlew \ - intTest \ - -x :polaris-runtime-service:intTest \ - --continue - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - - name: Save partial Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-save - - name: Archive test results - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - if: failure() - with: - name: upload-integration-test-artifacts - path: *test-archive-path - - store-gradle-cache: - name: Store Gradle Cache - runs-on: ubuntu-24.04 - timeout-minutes: 30 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - needs: - - unit-tests - - quarkus-tests - - quarkus-admin-tests - - quarkus-int-tests - - integration-tests - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Set up JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: *setup-java-vars - - name: Collect partial Gradle build caches - uses: ./.github/actions/ci-incr-build-cache-prepare - with: - cache-read-only: false - - # Note: the "Post Collect partial Gradle build caches"-step cleans up stale cache entries - # and archives the updated build cache. diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml deleted file mode 100644 index 93e11c6771..0000000000 --- a/.github/workflows/helm.yml +++ /dev/null @@ -1,116 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Helm tests - -on: - push: - branches: [ "main", "release/*" ] - pull_request: - branches: [ "main", "release/*" ] - -env: - GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }} - DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }} - DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }} - -jobs: - - helm-tests: - name: Helm tests - runs-on: ubuntu-latest - timeout-minutes: 60 - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - fetch-depth: 0 - - - name: Setup test environment - uses: ./.github/actions/setup-test-env - - - name: Set up JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: - java-version: '21' - distribution: 'temurin' - - - name: Set up Helm - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 - with: - version: 'v3.16.0' - - - name: Set up chart-testing - uses: helm/chart-testing-action@6ec842c01de15ebb84c8627d2744a0c2f2755c9f # v2.8.0 - - - name: Run 'helm template' validation - run: | - cd helm/polaris - for f in values.yaml ci/*.yaml; do - echo "::group::helm template $f" - helm template --debug --namespace polaris-ns --values $f . - echo "::endgroup::" - done - - - name: Run Helm unit tests - run: | - # Pin version to 1.0.2 due to https://github.com/helm-unittest/helm-unittest/issues/790 - helm plugin install https://github.com/helm-unittest/helm-unittest.git --version 1.0.2 || true - helm unittest helm/polaris - - - name: Run chart-testing (lint) - run: ct lint --target-branch ${{ github.event.repository.default_branch }} --debug --charts ./helm/polaris - - - name: Set up Minikube - uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # v0.0.21 - - - name: Print Docker info - run: | - docker -v - minikube docker-env - - - name: Image build - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: | - eval $(minikube -p minikube docker-env) - ./gradlew \ - :polaris-server:assemble \ - :polaris-server:quarkusAppPartsBuild --rerun \ - :polaris-admin:assemble \ - :polaris-admin:quarkusAppPartsBuild --rerun \ - -Dquarkus.container-image.build=true - minikube image ls - - - name: Install fixtures - run: | - kubectl create namespace polaris-ns - kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures - - - name: Run chart-testing (install) - run: | - ct install --target-branch ${{ github.event.repository.default_branch }} \ - --namespace polaris-ns \ - --debug --charts ./helm/polaris diff --git a/.github/workflows/python-client.yml b/.github/workflows/python-client.yml deleted file mode 100644 index 8267776af1..0000000000 --- a/.github/workflows/python-client.yml +++ /dev/null @@ -1,84 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. -# This workflow will build a Python project with UV and cache/restore any dependencies to improve the workflow execution time -# For more information see: https://docs.github.com/en/actions/use-cases-and-examples/building-and-testing/building-and-testing-python - -name: Python Client CI - -on: - push: - branches: [ "main", "release/*" ] - pull_request: - branches: [ "main", "release/*" ] - -env: - GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }} - DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }} - DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }} - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - - steps: - - name: Checkout Polaris project - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Set up JDK for Gradle - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: - distribution: 'temurin' - java-version: '21' - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup test environment - uses: ./.github/actions/setup-test-env - - - name: Lint - run: | - make client-lint - - - name: License Compliance Check - run: | - make client-license-check - - - name: Generated Client Tests - run: | - make client-unit-test - - - name: Integration Tests - run: | - make client-integration-test - - - name: Run Polaris Client help manual - run: | - ./polaris --help diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml deleted file mode 100644 index cdc1be84f2..0000000000 --- a/.github/workflows/regtest.yml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Regression Tests -on: - push: - branches: [ "main", "release/*" ] - pull_request: - branches: [ "main", "release/*" ] - -env: - GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }} - DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }} - DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }} - -jobs: - regtest: - - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Free disk space - uses: ./.github/actions/free-disk-space - - - name: Set up JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: - java-version: '21' - distribution: 'temurin' - - - name: Setup test environment - uses: ./.github/actions/setup-test-env - - name: Prepare Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-prepare - - - name: Fix permissions - run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/* - - - name: Image build - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: | - ./gradlew \ - :polaris-server:assemble \ - :polaris-server:quarkusAppPartsBuild --rerun \ - -Dquarkus.container-image.build=true - - - name: Regression Test - env: - AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} - run: | - docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml index 5c83f2aca1..72db31ea8e 100644 --- a/.github/workflows/site.yml +++ b/.github/workflows/site.yml @@ -17,12 +17,12 @@ # under the License. # +# Site publishing workflow, not for pull-request CI + name: "Hugo Site" on: push: branches: [ "main" ] - pull_request: - branches: [ "main" ] workflow_call: # The workflow is programmatically "protected" to be only callable from the 'apache/polaris' repo. # Beware: this workflow will be called with the `github` context from the 'versioned-docs' branch, diff --git a/.github/workflows/spark_client_regtests.yml b/.github/workflows/spark_client_regtests.yml deleted file mode 100644 index 8a0881d720..0000000000 --- a/.github/workflows/spark_client_regtests.yml +++ /dev/null @@ -1,81 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Spark Client Regression Tests -on: - push: - branches: [ "main", "release/*" ] - pull_request: - branches: [ "main", "release/*" ] - -env: - GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }} - DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }} - DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }} - -jobs: - spark-plugin-regtest: - - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Free disk space - uses: ./.github/actions/free-disk-space - - - name: Set up JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 - with: - java-version: '21' - distribution: 'temurin' - - - name: Setup test environment - uses: ./.github/actions/setup-test-env - - name: Prepare Gradle build cache - uses: ./.github/actions/ci-incr-build-cache-prepare - - - name: Fix permissions - run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/* - - - name: Project build without testing - env: - # publishToMavenLocal causes a GH API requests, use the token for those requests - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ./gradlew assemble publishToMavenLocal - - - name: Image build - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: | - ./gradlew \ - :polaris-server:assemble \ - :polaris-server:quarkusAppPartsBuild --rerun \ - -Dquarkus.container-image.build=true - - # NOTE: the regression test runs with spark 3.5.6 and scala 2.12 in Java 17. We also have integration - # tests runs with the existing gradle.yml, which only runs on Java 21. Since spark Java compatibility - # for 3.5 is 8, 11, and 17, we should run spark client with those compatible java versions. - # TODO: add separate spark client CI and run with Java 8, 11 and 17. - - name: Regression Test - run: | - docker compose -f plugins/spark/v3.5/regtests/docker-compose.yml up --build --exit-code-from regtest