diff --git a/.github/scripts/display_dependency_analysis_error_message.sh b/.github/scripts/analyze_dependencies_script.sh similarity index 92% rename from .github/scripts/display_dependency_analysis_error_message.sh rename to .github/scripts/analyze_dependencies_script.sh index 3899d5917b15..c92d90030f71 100755 --- a/.github/scripts/display_dependency_analysis_error_message.sh +++ b/.github/scripts/analyze_dependencies_script.sh @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +#!bin/bash + +${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} || { echo " The dependency analysis has found a dependency that is either: diff --git a/.github/scripts/license_checks_script.sh b/.github/scripts/license_checks_script.sh new file mode 100755 index 000000000000..410ac60375fa --- /dev/null +++ b/.github/scripts/license_checks_script.sh @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +set -e + +./.github/scripts/setup_generate_license.sh +${MVN} apache-rat:check -Prat --fail-at-end \ +-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ +-Drat.consoleOutput=true ${HADOOP_PROFILE} +# Generate dependency reports and checks they are valid. +mkdir -p target +distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 +distribution/bin/check-licenses.py licenses.yaml target/license-reports diff --git a/.github/scripts/setup_generate_license.sh b/.github/scripts/setup_generate_license.sh index 9e6c2eae68c6..71583bfb2b26 100755 --- a/.github/scripts/setup_generate_license.sh +++ b/.github/scripts/setup_generate_license.sh @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +#!/bin/bash + +set -e + sudo apt-get update && sudo apt-get install python3 -y curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3 pip3 install wheel # install wheel first explicitly diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 7f4437f0129e..c6f395058603 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -35,160 +35,133 @@ env: MAVEN_OPTS: -Xmx3000m jobs: - build: + static-checks: + strategy: + matrix: + java: [ 'jdk8', 'jdk11', 'jdk17' ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: setup java 8 + - name: checkout branch + uses: actions/checkout@v3 + + - name: set java version + run: | + export jdk=${{ matrix.java }} + echo "java_version=${jdk:3}" >> $GITHUB_ENV + + - name: setup ${{ matrix.java }} uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '8' + java-version: ${{ env.java_version }} cache: 'maven' - - run: | + + - name: license checks + if: ${{ matrix.java == 'jdk8' }} + run: ./.github/scripts/license_checks_script.sh + + - name: analyze dependencies + if: ${{ matrix.java == 'jdk8' }} + run: | + ./.github/scripts/analyze_dependencies_script.sh + + - name: analyze dependencies for hadoop2 + if: ${{ matrix.java == 'jdk8' }} + env: + HADOOP_PROFILE: -Phadoop2 + run: | + ./.github/scripts/analyze_dependencies_script.sh + + - name: packaging check + run: | + ./.github/scripts/setup_generate_license.sh + ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \ + -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C + + - name: script checks + # who watches the watchers? + if: ${{ matrix.java == 'jdk8' }} + run: ./check_test_suite_test.py + + - name: (openjdk11) strict compilation + if: ${{ matrix.java == 'jdk11' }} + # errorprone requires JDK 11 + # Strict compilation requires more than 2 GB + run: ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} + + - name: maven install + if: ${{ matrix.java == 'jdk8' }} + run: | echo 'Running Maven install...' && ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - animal_sniffer_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} animal-sniffer:check --fail-at-end + - name: animal sniffer checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} animal-sniffer:check --fail-at-end - checkstyle: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} checkstyle:checkstyle --fail-at-end + - name: checkstyle + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} checkstyle:checkstyle --fail-at-end - enforcer_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} enforcer:enforce --fail-at-end + - name: enforcer checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} enforcer:enforce --fail-at-end - forbidden_api_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end + - name: forbidden api checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end - pmd_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check + - name: pmd checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check - spotbugs_checks: - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' + - name: spotbugs checks + if: ${{ matrix.java == 'jdk8' }} + run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' - license_checks: - runs-on: ubuntu-latest - needs: [build] - strategy: - matrix: - HADOOP_PROFILE: ['', '-Phadoop3'] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: | - ./.github/scripts/setup_generate_license.sh - ${MVN} apache-rat:check -Prat --fail-at-end \ - -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - -Drat.consoleOutput=true ${{ matrix.HADOOP_PROFILE }} - # Generate dependency reports and checks they are valid. - mkdir -p target - distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 - distribution/bin/check-licenses.py licenses.yaml target/license-reports - - script_checks: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - # who watches the watchers? - - run: ./check_test_suite_test.py + - name: intellij inspections + if: ${{ matrix.java == 'jdk8' }} + run: | + docker run --rm \ + -v $(pwd):/project \ + -v ~/.m2:/home/inspect/.m2 \ + ccaominh/intellij-inspect:1.0.0 \ + /project/pom.xml \ + /project/.idea/inspectionProfiles/Druid.xml \ + --levels ERROR \ + --scope JavaInspectionsScope - analyze_dependencies: - runs-on: ubuntu-latest - needs: [build] - strategy: - matrix: - HADOOP_PROFILE: [ '', '-Phadoop3' ] - steps: - - uses: actions/checkout@v3 - - name: setup java 8 - uses: actions/setup-java@v3 + - name: setup node + if: ${{ matrix.java == 'jdk8' }} + uses: actions/setup-node@v3 with: - distribution: 'zulu' - java-version: '8' - cache: 'maven' - - run: |- - ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${{ matrix.HADOOP_PROFILE }} || - ./.github/scripts/display_dependency_analysis_error_message.sh + node-version: 16.17.0 - openjdk11_strict_compilation: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: setup java 11 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '11' - cache: 'maven' + - name: docs + if: ${{ matrix.java == 'jdk8' }} + run: | + (cd website && npm install) + cd website + npm run link-lint + npm run spellcheck - # errorprone requires JDK 11 - # Strict compilation requires more than 2 GB - - run: | - ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} + - name: web console + if: ${{ matrix.java == 'jdk8' }} + run: | + ${MVN} test -pl 'web-console' + cd web-console + { for i in 1 2 3; do npm run codecov && break || sleep 15; done } + + - name: web console end-to-end test + if: ${{ matrix.java == 'jdk8' }} + run: | + ./.github/scripts/setup_generate_license.sh + sudo apt-get update && sudo apt-get install python3 -y + curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3 + pip3 install wheel # install wheel first explicitly + pip3 install --upgrade pyyaml + web-console/script/druid build + web-console/script/druid start + (cd web-console && npm run test-e2e) + web-console/script/druid stop diff --git a/.travis.yml b/.travis.yml index 750d9e7ce3b6..0c7a73290b1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,120 +70,6 @@ stages: jobs: include: - - name: "animal sniffer checks" - stage: Tests - phase 1 - script: ${MVN} animal-sniffer:check --fail-at-end - - - name: "checkstyle" - script: ${MVN} checkstyle:checkstyle --fail-at-end - - - name: "enforcer checks" - script: ${MVN} enforcer:enforce --fail-at-end - - - name: "forbidden api checks" - script: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end - - - name: "pmd checks" - script: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check - - - name: "spotbugs checks" - script: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' - - - &license_checks - name: "license checks" - before_script: &setup_generate_license - - sudo apt-get update && sudo apt-get install python3 -y - - curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3 - - ./check_test_suite.py && travis_terminate 0 || echo 'Continuing setup' - - pip3 install wheel # install wheel first explicitly - - pip3 install --upgrade pyyaml - script: - - > - ${MVN} apache-rat:check -Prat --fail-at-end - -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - -Drat.consoleOutput=true ${HADOOP_PROFILE} - # Generate dependency reports and checks they are valid. When running on Travis CI, 2 cores are available - # (https://docs.travis-ci.com/user/reference/overview/#virtualisation-environment-vs-operating-system). - - mkdir -p target - - distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 - - distribution/bin/check-licenses.py licenses.yaml target/license-reports - - - <<: *license_checks - name: "license checks with Hadoop3" - env: - - HADOOP_PROFILE='-Phadoop3' - - - name: "script checks" - install: skip - # who watches the watchers? - script: ./check_test_suite_test.py - - - name: "(openjdk11) strict compilation" - install: skip - # errorprone requires JDK 11 - jdk: openjdk11 - # Strict compilation requires more than 2 GB - script: > - ./check_test_suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -DstrictCompile compile test-compile --fail-at-end - ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - - - &analyze_dependencies - name: "analyze dependencies" - script: |- - MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} || { echo " - - The dependency analysis has found a dependency that is either: - - 1) Used and undeclared: These are available as a transitive dependency but should be explicitly - added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is - shown above. - - 2) Unused and declared: These are not needed and removing them from the POM will speed up the build - and reduce the artifact size. The dependencies to remove are shown above. - - If there are false positive dependency analysis warnings, they can be suppressed: - https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies - https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html - - For more information, refer to: - https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html - - " && false; } - - - <<: *analyze_dependencies - name: "analyze hadoop 3 dependencies" - env: - - HADOOP_PROFILE='-Phadoop3' - - - name: "intellij inspections" - script: > - ./check_test_suite.py && travis_terminate 0 || docker run --rm - -v $(pwd):/project - -v ~/.m2:/home/inspect/.m2 - ccaominh/intellij-inspect:1.0.0 - /project/pom.xml - /project/.idea/inspectionProfiles/Druid.xml - --levels ERROR - --scope JavaInspectionsScope - - - &package - name: "(openjdk8) packaging check" - install: skip - before_script: *setup_generate_license - script: > - MAVEN_OPTS='-Xmx3000m' ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end - -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C - - - <<: *package - name: "(openjdk11) packaging check" - stage: Tests - phase 2 - jdk: openjdk11 - - - <<: *package - name: "(openjdk17) packaging check" - stage: Tests - phase 2 - jdk: openjdk17 - - &test_processing_module name: "(openjdk8) processing module test" stage: Tests - phase 1 @@ -366,36 +252,6 @@ jobs: stage: Tests - phase 2 jdk: openjdk17 - - name: "web console" - install: skip - stage: Tests - phase 1 - script: - - ./check_test_suite.py && travis_terminate 0 || ${MVN} test -pl 'web-console' - after_success: - - (cd web-console && travis_retry npm run codecov) # retry in case of network error - - - name: "web console end-to-end test" - stage: Tests - phase 1 - before_install: *setup_generate_license - install: web-console/script/druid build - before_script: - - ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...' - - nvm install 16.17.0 - - web-console/script/druid start - script: (cd web-console && npm run test-e2e) - after_script: web-console/script/druid stop - - - name: "docs" - stage: Tests - phase 1 - before_script: - - ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...' - - nvm install 16.17.0 - - (cd website && npm install) - script: - - cd website - - npm run link-lint - - npm run spellcheck - - name: "Build and test on ARM64 CPU architecture (1)" stage: Tests - phase 2 arch: arm64-graviton2 diff --git a/check_test_suite.py b/check_test_suite.py index 741b14180aab..8fd739111426 100755 --- a/check_test_suite.py +++ b/check_test_suite.py @@ -22,7 +22,7 @@ # this script does some primitive examination of git diff to determine if a test suite needs to be run or not # these jobs should always be run, no matter what -always_run_jobs = ['license checks', 'license checks with Hadoop3', '(openjdk8) packaging check', '(openjdk11) packaging check'] +always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11) packaging check'] # ignore changes to these files completely since they don't impact CI, if the changes are only to these files then all # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed diff --git a/distribution/asf-release-process-guide.md b/distribution/asf-release-process-guide.md index fecbecb2c82b..710e082f6c94 100644 --- a/distribution/asf-release-process-guide.md +++ b/distribution/asf-release-process-guide.md @@ -328,6 +328,7 @@ Ensure that the GPG key fingerprint used in the `mvn install` command matches yo $ diff <(shasum -a512 apache-druid-0.17.0-bin.tar.gz | cut -d ' ' -f1) <(cat apache-druid-0.17.0-bin.tar.gz.sha512 ; echo) ... $ diff <(shasum -a512 apache-druid-0.17.0-src.tar.gz | cut -d ' ' -f1) <(cat apache-druid-0.17.0-src.tar.gz.sha512 ; echo) +... ``` ### Verify GPG signatures @@ -336,6 +337,7 @@ $ diff <(shasum -a512 apache-druid-0.17.0-src.tar.gz | cut -d ' ' -f1) <(cat apa $ gpg --verify apache-druid-0.17.0-bin.tar.gz.asc apache-druid-0.17.0-bin.tar.gz ... $ gpg --verify apache-druid-0.17.0-src.tar.gz.asc apache-druid-0.17.0-src.tar.gz +... ``` ### Commit artifacts to SVN repo diff --git a/distribution/bin/check-licenses.py b/distribution/bin/check-licenses.py index b5a2c2e933dd..ff77eeace0bf 100755 --- a/distribution/bin/check-licenses.py +++ b/distribution/bin/check-licenses.py @@ -293,6 +293,8 @@ def build_compatible_license_names(): compatible_licenses['The MIT License (MIT)'] = 'MIT License' compatible_licenses['Bouncy Castle Licence'] = 'MIT License' + compatible_licenses['The Go license'] = 'The Go license' + compatible_licenses['-'] = '-' return compatible_licenses diff --git a/distribution/pom.xml b/distribution/pom.xml index ac9e8d5ea365..cb84e00d0365 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -116,7 +116,7 @@ - dist + dist-hadoop2 false @@ -190,6 +190,7 @@ -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies + -Dhadoop2.enabled=true org.apache.druid.cli.Main tools pull-deps @@ -225,8 +226,6 @@ -c org.apache.druid.extensions:druid-multi-stage-query -c - org.apache.druid.extensions:druid-catalog - -c org.apache.druid.extensions:druid-protobuf-extensions -c org.apache.druid.extensions:mysql-metadata-storage @@ -302,7 +301,7 @@ - dist-hadoop3 + dist false @@ -373,7 +372,6 @@ -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies - -Dhadoop3.enabled=true org.apache.druid.cli.Main tools pull-deps @@ -442,6 +440,8 @@ org.apache.druid.extensions:druid-ranger-security -c org.apache.druid.extensions:druid-kubernetes-extensions + -c + org.apache.druid.extensions:druid-catalog ${druid.distribution.pulldeps.opts} diff --git a/docs/development/build.md b/docs/development/build.md index b093b2e4f99c..b1e6777ee56d 100644 --- a/docs/development/build.md +++ b/docs/development/build.md @@ -71,24 +71,6 @@ Putting these together, if you wish to build the source and binary distributions mvn clean install -Papache-release,dist,rat -DskipTests ``` -### Building hadoop 3 distribution - -By default, druid ships hadoop 2.x.x jars along with the distribution. Exact version can be found in the -main [pom](https://github.com/apache/druid/blob/master/pom.xml). To build druid with hadoop 3.x.x jars, hadoop3 profile -needs to be activated. - -To generate build with hadoop 3 dependencies, run: - -```bash -mvn clean install -Phadoop3 -``` - -To generate distribution with hadoop3 dependencies, run : - -```bash -mvn clean install -Papache-release,dist-hadoop3,rat,hadoop3 -DskipTests -``` - #### Potential issues ##### Missing `pyyaml` diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 976730516fb3..53e308fbba24 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -136,7 +136,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -161,23 +164,20 @@ hadoop3 - - hadoop3.enabled - true - + true org.apache.hadoop hadoop-client-api ${hadoop.compile.version} - provided + compile org.apache.hadoop hadoop-client-runtime ${hadoop.compile.version} - test + runtime diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 444e72ca05d9..440449fac1d1 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -279,7 +279,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -321,17 +324,14 @@ hadoop3 - - hadoop3.enabled - true - + true org.apache.hadoop hadoop-client-api ${hadoop.compile.version} - provided + compile com.sun.jersey diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index d5d078cfc3a3..2d7e80099b7a 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -142,10 +142,19 @@ com.google.code.findbugs jsr305 + org.mortbay.jetty jetty-util + + org.eclipse.jetty + jetty-webapp + + + org.eclipse.jetty + jetty-xml + org.apache.hadoop hadoop-annotations @@ -199,7 +208,7 @@ json-smart - com.sun.jersey + com.github.pjfanning jersey-json @@ -283,6 +292,26 @@ jetty-client provided + + org.eclipse.jetty + jetty-server + provided + + + org.eclipse.jetty + jetty-util + provided + + + org.eclipse.jetty + jetty-io + provided + + + org.eclipse.jetty + jetty-servlet + provided + diff --git a/extensions-core/druid-ranger-security/pom.xml b/extensions-core/druid-ranger-security/pom.xml index 0d193d4c816a..9f5b12dab9b9 100644 --- a/extensions-core/druid-ranger-security/pom.xml +++ b/extensions-core/druid-ranger-security/pom.xml @@ -172,7 +172,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -401,10 +404,7 @@ hadoop3 - - hadoop3.enabled - true - + true @@ -417,7 +417,7 @@ org.apache.hadoop hadoop-client-runtime ${hadoop.compile.version} - test + runtime diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index fec9ff4c3cd6..8496ec95e42e 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -45,6 +45,12 @@ hadoop-aws ${hadoop.compile.version} runtime + + + com.amazonaws + aws-java-sdk-bundle + + commons-io @@ -150,7 +156,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -436,10 +445,7 @@ hadoop3 - - hadoop3.enabled - true - + true @@ -460,6 +466,12 @@ ${hadoop.compile.version} test + + com.amazonaws + aws-java-sdk-s3 + ${aws.sdk.version} + runtime + log4j log4j diff --git a/extensions-core/orc-extensions/pom.xml b/extensions-core/orc-extensions/pom.xml index a5f0cf5fbbfb..e09ed49fad39 100644 --- a/extensions-core/orc-extensions/pom.xml +++ b/extensions-core/orc-extensions/pom.xml @@ -248,7 +248,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -504,10 +507,7 @@ hadoop3 - - hadoop3.enabled - true - + true @@ -525,7 +525,7 @@ org.apache.hadoop hadoop-client-runtime ${hadoop.compile.version} - test + runtime diff --git a/extensions-core/parquet-extensions/pom.xml b/extensions-core/parquet-extensions/pom.xml index 7fe0e84502cc..9cf8a0ebf65d 100644 --- a/extensions-core/parquet-extensions/pom.xml +++ b/extensions-core/parquet-extensions/pom.xml @@ -184,7 +184,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -432,23 +435,20 @@ hadoop3 - - hadoop3.enabled - true - + true org.apache.hadoop hadoop-client-api ${hadoop.compile.version} - provided + compile org.apache.hadoop hadoop-client-runtime ${hadoop.compile.version} - test + runtime diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 05e8c39590cb..39498fc223fa 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -172,7 +172,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -227,10 +230,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index e23a8c1cafdb..1d4414eac8b9 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -290,7 +290,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -334,10 +337,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/integration-tests-ex/image/docker/launch.sh b/integration-tests-ex/image/docker/launch.sh index 1f64b4e14df0..fe941466fa6b 100644 --- a/integration-tests-ex/image/docker/launch.sh +++ b/integration-tests-ex/image/docker/launch.sh @@ -85,7 +85,7 @@ fi # Assemble Java options JAVA_OPTS="$DRUID_SERVICE_JAVA_OPTS $DRUID_COMMON_JAVA_OPTS -XX:HeapDumpPath=$LOG_DIR/$INSTANCE_NAME $DEBUG_OPTS" -LOG4J_CONFIG=$SHARED_DIR/conf/log4j2.xml +LOG4J_CONFIG=$SHARED_DIR/resources/log4j2.xml if [ -f $LOG4J_CONFIG ]; then JAVA_OPTS="$JAVA_OPTS -Dlog4j.configurationFile=$LOG4J_CONFIG" fi diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 0e7d359c3b2a..f66ada482502 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -39,8 +39,8 @@ - "org.apache.hadoop:hadoop-client:${hadoop.compile.version}", "org.apache.hadoop:hadoop-azure:${hadoop.compile.version}" - org.apache.hadoop.fs.s3native.NativeS3FileSystem + "org.apache.hadoop:hadoop-client-api:${hadoop.compile.version}", "org.apache.hadoop:hadoop-client-runtime:${hadoop.compile.version}", "org.apache.hadoop:hadoop-azure:${hadoop.compile.version}" + org.apache.hadoop.fs.s3a.S3AFileSystem @@ -560,10 +560,10 @@ - hadoop3 + hadoop2 - "org.apache.hadoop:hadoop-client-api:${hadoop.compile.version}", "org.apache.hadoop:hadoop-client-runtime:${hadoop.compile.version}", "org.apache.hadoop:hadoop-azure:${hadoop.compile.version}" - org.apache.hadoop.fs.s3a.S3AFileSystem + "org.apache.hadoop:hadoop-client:${hadoop.compile.version}", "org.apache.hadoop:hadoop-azure:${hadoop.compile.version}" + org.apache.hadoop.fs.s3native.NativeS3FileSystem diff --git a/licenses.yaml b/licenses.yaml index 12b765fea32f..ffbf8def74cc 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -644,7 +644,7 @@ name: Apache Commons Configuration license_category: binary module: java-core license_name: Apache License version 2.0 -version: 2.1.1 +version: 2.8.0 libraries: - org.apache.commons: commons-configuration2 @@ -2912,22 +2912,10 @@ name: Apache Hadoop license_category: binary module: hadoop-client license_name: Apache License version 2.0 -version: 3.3.1 +version: 3.3.6 libraries: - - org.apache.hadoop: hadoop-annotations - org.apache.hadoop: hadoop-auth - - org.apache.hadoop: hadoop-client - org.apache.hadoop: hadoop-common - - org.apache.hadoop: hadoop-hdfs-client - - org.apache.hadoop: hadoop-mapreduce-client-app - - org.apache.hadoop: hadoop-mapreduce-client-common - - org.apache.hadoop: hadoop-mapreduce-client-core - - org.apache.hadoop: hadoop-mapreduce-client-jobclient - - org.apache.hadoop: hadoop-mapreduce-client-shuffle - - org.apache.hadoop: hadoop-yarn-api - - org.apache.hadoop: hadoop-yarn-client - - org.apache.hadoop: hadoop-yarn-common - - org.apache.hadoop: hadoop-yarn-server-common --- @@ -3484,6 +3472,43 @@ notices: --- +name: reload4j +license_category: binary +module: hadoop-common +license_name: Apache License version 2.0 +version: 1.2.22 +libraries: + - ch.qos.reload4j: reload4j +notices: + - reload4j: | + Apache log4j + Copyright 2010 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + +--- + +name: slf4j-reload4j +license_category: binary +module: hadoop-common +license_name: MIT License +version: 1.7.36 +libraries: + - org.slf4j: slf4j-reload4j + +--- + +name: com.github.pjfanning jersey-json +license_category: binary +module: druid-kerberos +license_name: CDDL 1.1 +version: 1.20 +libraries: + - com.github.pjfanning: jersey-json + +--- + name: Kafka Schema Registry Client version: 5.5.1 license_category: binary @@ -3785,6 +3810,16 @@ libraries: --- +name: Hadoop Client API +license_category: binary +module: extensions/druid-hdfs-storage +license_name: Apache License version 2.0 +version: 3.3.6 +libraries: + - org.apache.hadoop: hadoop-client-api + +--- + name: xmlenc license_category: binary module: extensions/druid-hdfs-storage @@ -5036,7 +5071,7 @@ name: Woodstox license_category: binary module: java-core license_name: Apache License version 2.0 -version: 5.3.0 +version: 5.4.0 libraries: - com.fasterxml.woodstox: woodstox-core @@ -5063,6 +5098,25 @@ libraries: --- +name: RE2/J +license_category: binary +module: java-core +license_name: The Go license +version: 1.1 +license_file_path: licenses/bin/re2j.GO +libraries: + - com.google.re2j: re2j + +--- +name: jakarta.activation +license_category: binary +module: extensions/druid-avro-extensions +license_name: Eclipse Distribution License 1.0 +version: 1.2.1 +libraries: + - com.sun.activation: jakarta.activation + +--- # Web console modules start name: "@babel/code-frame" diff --git a/owasp-dependency-check-suppressions.xml b/owasp-dependency-check-suppressions.xml index a23278135ebd..f7549b5b0f33 100644 --- a/owasp-dependency-check-suppressions.xml +++ b/owasp-dependency-check-suppressions.xml @@ -785,4 +785,14 @@ 1070209 CVE-2020-7774 + + + + + CVE-2022-26612 + + CVE-2023-25613 + diff --git a/pom.xml b/pom.xml index 61f73f2f5009..adb62b57e3cc 100644 --- a/pom.xml +++ b/pom.xml @@ -109,13 +109,12 @@ 3.21.7 1.3.1 1.7.36 - - 2.8.5 + 3.3.6 4.3.1 1.12.317 2.8.0 0.8.7 - 5.2.5.Final + 5.3.6.Final 4.5.13 3.5.9 @@ -2007,17 +2006,16 @@ - hadoop3 + hadoop2 - hadoop3.enabled + hadoop2.enabled true - 3.3.1 - 5.3.6.Final - 4.5.13 + 2.8.5 + 5.2.5.Final