diff --git a/.github/scripts/display_dependency_analysis_error_message.sh b/.github/scripts/analyze_dependencies_script.sh similarity index 92% rename from .github/scripts/display_dependency_analysis_error_message.sh rename to .github/scripts/analyze_dependencies_script.sh index 3899d5917b15..c92d90030f71 100755 --- a/.github/scripts/display_dependency_analysis_error_message.sh +++ b/.github/scripts/analyze_dependencies_script.sh @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +#!bin/bash + +${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} || { echo " The dependency analysis has found a dependency that is either: diff --git a/.github/scripts/license_checks_script.sh b/.github/scripts/license_checks_script.sh new file mode 100755 index 000000000000..410ac60375fa --- /dev/null +++ b/.github/scripts/license_checks_script.sh @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +set -e + +./.github/scripts/setup_generate_license.sh +${MVN} apache-rat:check -Prat --fail-at-end \ +-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ +-Drat.consoleOutput=true ${HADOOP_PROFILE} +# Generate dependency reports and checks they are valid. +mkdir -p target +distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 +distribution/bin/check-licenses.py licenses.yaml target/license-reports diff --git a/.github/scripts/setup_generate_license.sh b/.github/scripts/setup_generate_license.sh index 9e6c2eae68c6..71583bfb2b26 100755 --- a/.github/scripts/setup_generate_license.sh +++ b/.github/scripts/setup_generate_license.sh @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +#!/bin/bash + +set -e + sudo apt-get update && sudo apt-get install python3 -y curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3 pip3 install wheel # install wheel first explicitly diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 7f4437f0129e..a3f2d7c0e3d5 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -35,160 +35,139 @@ env: MAVEN_OPTS: -Xmx3000m jobs: - build: + static-checks: + strategy: + matrix: + java: [ 'jdk8', 'jdk11', 'jdk17' ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: setup java 8 + - name: checkout branch + uses: actions/checkout@v3 + + - name: set java version + run: | + export jdk=${{ matrix.java }} + echo "java_version=${jdk:3}" >> $GITHUB_ENV + + - name: setup ${{ matrix.java }} uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '8' + java-version: ${{ env.java_version }} cache: 'maven' - - run: | + + - name: license checks + if: ${{ matrix.java == 'jdk8' }} + run: ./.github/scripts/license_checks_script.sh + + - name: license checks for hadoop3 + if: ${{ matrix.java == 'jdk8' }} + env: + HADOOP_PROFILE: -Phadoop3 + run: ./.github/scripts/license_checks_script.sh + + - name: analyze dependencies + if: ${{ matrix.java == 'jdk8' }} + run: | + ./.github/scripts/analyze_dependencies_script.sh + + - name: analyze dependencies for hadoop3 + if: ${{ matrix.java == 'jdk8' }} + env: + HADOOP_PROFILE: -Phadoop3 + run: | + ./.github/scripts/analyze_dependencies_script.sh + + - name: packaging check + run: | + ./.github/scripts/setup_generate_license.sh + ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \ + -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C + + - name: script checks + # who watches the watchers? + if: ${{ matrix.java == 'jdk8' }} + run: ./check_test_suite_test.py + + - name: (openjdk11) strict compilation + if: ${{ matrix.java == 'jdk11' }} + # errorprone requires JDK 11 + # Strict compilation requires more than 2 GB + run: ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} + + - name: maven install + if: ${{ matrix.java == 'jdk8' }} + run: | echo 'Running Maven install...' && ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - animal_sniffer_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} animal-sniffer:check --fail-at-end + - name: animal sniffer checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} animal-sniffer:check --fail-at-end - checkstyle: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} checkstyle:checkstyle --fail-at-end + - name: checkstyle + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} checkstyle:checkstyle --fail-at-end - enforcer_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} enforcer:enforce --fail-at-end + - name: enforcer checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} enforcer:enforce --fail-at-end - forbidden_api_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end + - name: forbidden api checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end - pmd_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check + - name: pmd checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check - spotbugs_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' + - name: spotbugs checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' - license_checks: - runs-on: ubuntu-latest - needs: [build] - strategy: - matrix: - HADOOP_PROFILE: ['', '-Phadoop3'] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: | - ./.github/scripts/setup_generate_license.sh - ${MVN} apache-rat:check -Prat --fail-at-end \ - -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - -Drat.consoleOutput=true ${{ matrix.HADOOP_PROFILE }} - # Generate dependency reports and checks they are valid. - mkdir -p target - distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 - distribution/bin/check-licenses.py licenses.yaml target/license-reports - - script_checks: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - # who watches the watchers? - - run: ./check_test_suite_test.py + - name: intellij inspections + if: ${{ matrix.java == 'jdk8' }} + run: | + docker run --rm \ + -v $(pwd):/project \ + -v ~/.m2:/home/inspect/.m2 \ + ccaominh/intellij-inspect:1.0.0 \ + /project/pom.xml \ + /project/.idea/inspectionProfiles/Druid.xml \ + --levels ERROR \ + --scope JavaInspectionsScope - analyze_dependencies: - runs-on: ubuntu-latest - needs: [build] - strategy: - matrix: - HADOOP_PROFILE: [ '', '-Phadoop3' ] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 + - name: setup node + if: ${{ matrix.java == 'jdk8' }} + uses: actions/setup-node@v3 with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: |- - ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${{ matrix.HADOOP_PROFILE }} || - ./.github/scripts/display_dependency_analysis_error_message.sh + node-version: 16.17.0 - openjdk11_strict_compilation: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: setup java 11 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '11' - cache: 'maven' + - name: docs + if: ${{ matrix.java == 'jdk8' }} + run: | + (cd website && npm install) + cd website + npm run link-lint + npm run spellcheck - # errorprone requires JDK 11 - # Strict compilation requires more than 2 GB - - run: | - ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} + - name: web console + if: ${{ matrix.java == 'jdk8' }} + run: | + ${MVN} test -pl 'web-console' + cd web-console + { for i in 1 2 3; do npm run codecov && break || sleep 15; done } + + - name: web console end-to-end test + if: ${{ matrix.java == 'jdk8' }} + run: | + ./.github/scripts/setup_generate_license.sh + sudo apt-get update && sudo apt-get install python3 -y + curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3 + pip3 install wheel # install wheel first explicitly + pip3 install --upgrade pyyaml + web-console/script/druid build + web-console/script/druid start + (cd web-console && npm run test-e2e) + web-console/script/druid stop diff --git a/.travis.yml b/.travis.yml index 750d9e7ce3b6..0c7a73290b1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,120 +70,6 @@ stages: jobs: include: - - name: "animal sniffer checks" - stage: Tests - phase 1 - script: ${MVN} animal-sniffer:check --fail-at-end - - - name: "checkstyle" - script: ${MVN} checkstyle:checkstyle --fail-at-end - - - name: "enforcer checks" - script: ${MVN} enforcer:enforce --fail-at-end - - - name: "forbidden api checks" - script: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end - - - name: "pmd checks" - script: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check - - - name: "spotbugs checks" - script: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' - - - &license_checks - name: "license checks" - before_script: &setup_generate_license - - sudo apt-get update && sudo apt-get install python3 -y - - curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3 - - ./check_test_suite.py && travis_terminate 0 || echo 'Continuing setup' - - pip3 install wheel # install wheel first explicitly - - pip3 install --upgrade pyyaml - script: - - > - ${MVN} apache-rat:check -Prat --fail-at-end - -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - -Drat.consoleOutput=true ${HADOOP_PROFILE} - # Generate dependency reports and checks they are valid. When running on Travis CI, 2 cores are available - # (https://docs.travis-ci.com/user/reference/overview/#virtualisation-environment-vs-operating-system). - - mkdir -p target - - distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 - - distribution/bin/check-licenses.py licenses.yaml target/license-reports - - - <<: *license_checks - name: "license checks with Hadoop3" - env: - - HADOOP_PROFILE='-Phadoop3' - - - name: "script checks" - install: skip - # who watches the watchers? - script: ./check_test_suite_test.py - - - name: "(openjdk11) strict compilation" - install: skip - # errorprone requires JDK 11 - jdk: openjdk11 - # Strict compilation requires more than 2 GB - script: > - ./check_test_suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -DstrictCompile compile test-compile --fail-at-end - ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - - - &analyze_dependencies - name: "analyze dependencies" - script: |- - MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} || { echo " - - The dependency analysis has found a dependency that is either: - - 1) Used and undeclared: These are available as a transitive dependency but should be explicitly - added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is - shown above. - - 2) Unused and declared: These are not needed and removing them from the POM will speed up the build - and reduce the artifact size. The dependencies to remove are shown above. - - If there are false positive dependency analysis warnings, they can be suppressed: - https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies - https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html - - For more information, refer to: - https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html - - " && false; } - - - <<: *analyze_dependencies - name: "analyze hadoop 3 dependencies" - env: - - HADOOP_PROFILE='-Phadoop3' - - - name: "intellij inspections" - script: > - ./check_test_suite.py && travis_terminate 0 || docker run --rm - -v $(pwd):/project - -v ~/.m2:/home/inspect/.m2 - ccaominh/intellij-inspect:1.0.0 - /project/pom.xml - /project/.idea/inspectionProfiles/Druid.xml - --levels ERROR - --scope JavaInspectionsScope - - - &package - name: "(openjdk8) packaging check" - install: skip - before_script: *setup_generate_license - script: > - MAVEN_OPTS='-Xmx3000m' ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end - -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C - - - <<: *package - name: "(openjdk11) packaging check" - stage: Tests - phase 2 - jdk: openjdk11 - - - <<: *package - name: "(openjdk17) packaging check" - stage: Tests - phase 2 - jdk: openjdk17 - - &test_processing_module name: "(openjdk8) processing module test" stage: Tests - phase 1 @@ -366,36 +252,6 @@ jobs: stage: Tests - phase 2 jdk: openjdk17 - - name: "web console" - install: skip - stage: Tests - phase 1 - script: - - ./check_test_suite.py && travis_terminate 0 || ${MVN} test -pl 'web-console' - after_success: - - (cd web-console && travis_retry npm run codecov) # retry in case of network error - - - name: "web console end-to-end test" - stage: Tests - phase 1 - before_install: *setup_generate_license - install: web-console/script/druid build - before_script: - - ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...' - - nvm install 16.17.0 - - web-console/script/druid start - script: (cd web-console && npm run test-e2e) - after_script: web-console/script/druid stop - - - name: "docs" - stage: Tests - phase 1 - before_script: - - ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...' - - nvm install 16.17.0 - - (cd website && npm install) - script: - - cd website - - npm run link-lint - - npm run spellcheck - - name: "Build and test on ARM64 CPU architecture (1)" stage: Tests - phase 2 arch: arm64-graviton2