From 75c1a0e795eee833012b96663286b6417bc7719d Mon Sep 17 00:00:00 2001 From: Ivan Zlenko <241953+ivanzlenko@users.noreply.github.com> Date: Sat, 8 Feb 2025 18:36:57 +0500 Subject: [PATCH 1/4] HDDS-12339. Enable PMD for Basic verification --- .github/ci.md | 1 + CONTRIBUTING.md | 1 + dev-support/ci/selective_ci_checks.bats | 28 ++++++------ dev-support/ci/selective_ci_checks.sh | 20 +++++++++ dev-support/pmd/pmd-ruleset.xml | 30 +++++++++++++ hadoop-ozone/dev-support/checks/pmd.sh | 45 ++++++++++++++++++++ hadoop-ozone/httpfsgateway/src/site/site.xml | 2 +- pom.xml | 12 ++++++ 8 files changed, 124 insertions(+), 15 deletions(-) create mode 100644 dev-support/pmd/pmd-ruleset.xml create mode 100755 hadoop-ozone/dev-support/checks/pmd.sh diff --git a/.github/ci.md b/.github/ci.md index 46abf29d2137..fda4983d10ae 100644 --- a/.github/ci.md +++ b/.github/ci.md @@ -44,6 +44,7 @@ Runs a subset of the following subjobs depending on what was selected by build-i - checkstyle: [Runs](../hadoop-ozone/dev-support/checks/checkstyle.sh) 'mvn checkstyle' plugin to confirm Java source abides by Ozone coding conventions - docs: [Builds](../hadoop-ozone/dev-support/checks/docs.sh) website with [Hugo](https://gohugo.io/) - findbugs: [Runs](../hadoop-ozone/dev-support/checks/findbugs.sh) spotbugs static analysis on bytecode +- pmd: [Runs](../hadoop-ozone/dev-support/checks/pmd.sh) PMD static analysis on project's source code - rat (release audit tool): [Confirms](../hadoop-ozone/dev-support/checks/rat.sh) source files include licenses diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e633b159d225..8a1eeda8cf13 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -145,6 +145,7 @@ The [`hadoop-ozone/dev-support/checks` directory](https://github.com/apache/ozon * `docs.sh`: sanity checks for [Ozone documentation](https://github.com/apache/ozone/tree/master/hadoop-hdds/docs) * `dependency.sh`: compares list of jars in build output with known list * `checkstyle.sh`: Checkstyle + * `pmd.sh`: PMD 3. moderate (around 10 minutes) * `findbugs.sh`: SpotBugs * `kubernetes.sh`: very limited set of tests run in Kubernetes environment diff --git a/dev-support/ci/selective_ci_checks.bats b/dev-support/ci/selective_ci_checks.bats index e1a9f90807f9..99085c942e01 100644 --- a/dev-support/ci/selective_ci_checks.bats +++ b/dev-support/ci/selective_ci_checks.bats @@ -102,7 +102,7 @@ load bats-assert/load.bash @test "integration and unit: java change" { run dev-support/ci/selective_ci_checks.sh 9aebf6e25 - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=false @@ -135,7 +135,7 @@ load bats-assert/load.bash @test "unit only" { run dev-support/ci/selective_ci_checks.sh 1dd1d0ba3 - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=false @@ -146,7 +146,7 @@ load bats-assert/load.bash @test "unit helper" { run dev-support/ci/selective_ci_checks.sh 88383d1d5 - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=false @@ -157,7 +157,7 @@ load bats-assert/load.bash @test "integration only" { run dev-support/ci/selective_ci_checks.sh 61396ba9f - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=false @@ -168,7 +168,7 @@ load bats-assert/load.bash @test "native only" { run dev-support/ci/selective_ci_checks.sh 5b1319a8c2 - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd","native"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=false @@ -179,7 +179,7 @@ load bats-assert/load.bash @test "native test in other module" { run dev-support/ci/selective_ci_checks.sh 822c0dee1a - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd","native"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=false @@ -212,7 +212,7 @@ load bats-assert/load.bash @test "main/java change" { run dev-support/ci/selective_ci_checks.sh 86a771dfe - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=true @@ -223,7 +223,7 @@ load bats-assert/load.bash @test "..../java change" { run dev-support/ci/selective_ci_checks.sh 01c616536 - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=true @@ -234,7 +234,7 @@ load bats-assert/load.bash @test "java and compose change" { run dev-support/ci/selective_ci_checks.sh d0f0f806e - assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd","native"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=true @@ -245,7 +245,7 @@ load bats-assert/load.bash @test "java and docs change" { run dev-support/ci/selective_ci_checks.sh 2c0adac26 - assert_output -p 'basic-checks=["rat","author","checkstyle","docs","findbugs"]' + assert_output -p 'basic-checks=["rat","author","checkstyle","docs","findbugs","pmd"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=true @@ -256,7 +256,7 @@ load bats-assert/load.bash @test "pom change" { run dev-support/ci/selective_ci_checks.sh 9129424a9 - assert_output -p 'basic-checks=["rat","checkstyle","findbugs","native"]' + assert_output -p 'basic-checks=["rat","checkstyle","findbugs","pmd","native"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=true @@ -267,7 +267,7 @@ load bats-assert/load.bash @test "CI lib change" { run dev-support/ci/selective_ci_checks.sh ceb79acaa - assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","rat"]' + assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","pmd","rat"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=true @@ -278,7 +278,7 @@ load bats-assert/load.bash @test "CI workflow change" { run dev-support/ci/selective_ci_checks.sh 90a8d7c01 - assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","rat"]' + assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","pmd","rat"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=true @@ -301,7 +301,7 @@ load bats-assert/load.bash @test "CI workflow change (ci.yaml)" { run dev-support/ci/selective_ci_checks.sh 90fd5f2adc - assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","rat"]' + assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","pmd","rat"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=true diff --git a/dev-support/ci/selective_ci_checks.sh b/dev-support/ci/selective_ci_checks.sh index a20d3cc307de..d6b280fc5eb2 100755 --- a/dev-support/ci/selective_ci_checks.sh +++ b/dev-support/ci/selective_ci_checks.sh @@ -418,6 +418,25 @@ function check_needs_findbugs() { start_end::group_end } +function check_needs_pmd() { + start_end::group_start "Check if pmd is needed" + local pattern_array=( + "^hadoop-ozone/dev-support/checks/pmd.sh" + "pom.xml" + "src/..../java" + ) + local ignore_array=( + "^hadoop-ozone/dist" + ) + filter_changed_files + + if [[ ${match_count} != "0" ]]; then + add_basic_check pmd + fi + + start_end::group_end +} + function check_needs_native() { start_end::group_start "Check if native is needed" local pattern_array=( @@ -584,6 +603,7 @@ check_needs_bats check_needs_checkstyle check_needs_docs check_needs_findbugs +check_needs_pmd check_needs_native calculate_test_types_to_run set_outputs diff --git a/dev-support/pmd/pmd-ruleset.xml b/dev-support/pmd/pmd-ruleset.xml new file mode 100644 index 000000000000..3b251b79b7d5 --- /dev/null +++ b/dev-support/pmd/pmd-ruleset.xml @@ -0,0 +1,30 @@ + + + + + + + + PMD Ruleset for Apache Ozone + + + .*/generated-sources/.* + diff --git a/hadoop-ozone/dev-support/checks/pmd.sh b/hadoop-ozone/dev-support/checks/pmd.sh new file mode 100755 index 000000000000..203163594df1 --- /dev/null +++ b/hadoop-ozone/dev-support/checks/pmd.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#checks:basic + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/pmd"} +mkdir -p "$REPORT_DIR" + +REPORT_FILE="$REPORT_DIR/summary.txt" + +MAVEN_OPTIONS='-B -fae --no-transfer-progress' + +declare -i rc + +mvn "$MAVEN_OPTIONS" test-compile pmd:check -Dpmd.failOnViolation=false -Dpmd.printFailingErrors "$@" > "${REPORT_DIR}/pmd-output.log" + +rc=$? + +cat "${REPORT_DIR}/pmd-output.log" + +find "." -name pmd-output.log -print0 \ + | xargs -0 sed -n -e '/^\[WARNING\] PMD Failure/p' \ + | tee "$REPORT_FILE" + +## generate counter +grep -c ':' "$REPORT_FILE" > "$REPORT_DIR/failures" + +ERROR_PATTERN="\[WARNING\] PMD Failure" +source "${DIR}/_post_process.sh" diff --git a/hadoop-ozone/httpfsgateway/src/site/site.xml b/hadoop-ozone/httpfsgateway/src/site/site.xml index 98b6b5a418d6..cabc85022f25 100644 --- a/hadoop-ozone/httpfsgateway/src/site/site.xml +++ b/hadoop-ozone/httpfsgateway/src/site/site.xml @@ -17,7 +17,7 @@ org.apache.maven.skins maven-stylus-skin - ${maven-stylus-skin.version} + 1.5 diff --git a/pom.xml b/pom.xml index 5c904ed6dd3d..fe1cbeb3f0b3 100644 --- a/pom.xml +++ b/pom.xml @@ -187,6 +187,7 @@ 2.0.0-SNAPSHOT 4.7.5 4.2.2 + 3.26.0 2023-01-01T00:00:00Z @@ -1754,6 +1755,17 @@ cyclonedx-maven-plugin ${cyclonedx.version} + + org.apache.maven.plugins + maven-pmd-plugin + ${pmd.version} + + + dev-support/pmd/pmd-ruleset.xml + + true + + From a95440af8e0b2bed81057b7de94ab89c6781fd96 Mon Sep 17 00:00:00 2001 From: Ivan Zlenko <241953+ivanzlenko@users.noreply.github.com> Date: Sun, 16 Feb 2025 17:09:19 +0500 Subject: [PATCH 2/4] HDDS-12339. Fix usage of MAVEN_OPTIONS --- hadoop-ozone/dev-support/checks/pmd.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/dev-support/checks/pmd.sh b/hadoop-ozone/dev-support/checks/pmd.sh index 203163594df1..5a3fb63cd634 100755 --- a/hadoop-ozone/dev-support/checks/pmd.sh +++ b/hadoop-ozone/dev-support/checks/pmd.sh @@ -28,7 +28,8 @@ MAVEN_OPTIONS='-B -fae --no-transfer-progress' declare -i rc -mvn "$MAVEN_OPTIONS" test-compile pmd:check -Dpmd.failOnViolation=false -Dpmd.printFailingErrors "$@" > "${REPORT_DIR}/pmd-output.log" +#shellcheck disable=SC2086 +mvn $MAVEN_OPTIONS test-compile pmd:check -Dpmd.failOnViolation=false -Dpmd.printFailingErrors "$@" > "${REPORT_DIR}/pmd-output.log" rc=$? From 068406b09e20f5405d0f4a8af96e3b79e21f9c31 Mon Sep 17 00:00:00 2001 From: Ivan Zlenko <241953+ivanzlenko@users.noreply.github.com> Date: Sun, 16 Feb 2025 19:05:07 +0500 Subject: [PATCH 3/4] HDDS-12339. Fix review comments --- dev-support/ci/selective_ci_checks.sh | 1 + hadoop-ozone/dev-support/checks/pmd.sh | 19 ++++++++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dev-support/ci/selective_ci_checks.sh b/dev-support/ci/selective_ci_checks.sh index d6b280fc5eb2..213d071b9119 100755 --- a/dev-support/ci/selective_ci_checks.sh +++ b/dev-support/ci/selective_ci_checks.sh @@ -424,6 +424,7 @@ function check_needs_pmd() { "^hadoop-ozone/dev-support/checks/pmd.sh" "pom.xml" "src/..../java" + "pmd-ruleset.xml" ) local ignore_array=( "^hadoop-ozone/dist" diff --git a/hadoop-ozone/dev-support/checks/pmd.sh b/hadoop-ozone/dev-support/checks/pmd.sh index 5a3fb63cd634..0a5f630145f8 100755 --- a/hadoop-ozone/dev-support/checks/pmd.sh +++ b/hadoop-ozone/dev-support/checks/pmd.sh @@ -16,6 +16,8 @@ #checks:basic +set -u -o pipefail + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../../.." || exit 1 @@ -24,23 +26,18 @@ mkdir -p "$REPORT_DIR" REPORT_FILE="$REPORT_DIR/summary.txt" -MAVEN_OPTIONS='-B -fae --no-transfer-progress' +declare -i rc + +MAVEN_OPTIONS='-B -fae --no-transfer-progress -Dpmd.failOnViolation=false -Dpmd.printFailingErrors -DskipRecon' declare -i rc #shellcheck disable=SC2086 -mvn $MAVEN_OPTIONS test-compile pmd:check -Dpmd.failOnViolation=false -Dpmd.printFailingErrors "$@" > "${REPORT_DIR}/pmd-output.log" - +mvn $MAVEN_OPTIONS test-compile pmd:check "$@" | tee "${REPORT_DIR}/output.log" rc=$? -cat "${REPORT_DIR}/pmd-output.log" - -find "." -name pmd-output.log -print0 \ - | xargs -0 sed -n -e '/^\[WARNING\] PMD Failure/p' \ - | tee "$REPORT_FILE" +grep -o "PMD Failure.*" "${REPORT_DIR}/output.log" > "$REPORT_FILE" -## generate counter -grep -c ':' "$REPORT_FILE" > "$REPORT_DIR/failures" +ERROR_PATTERN="\[ERROR\]" -ERROR_PATTERN="\[WARNING\] PMD Failure" source "${DIR}/_post_process.sh" From 0bea5e94cc02c81cb43c39aceec09531968aa2de Mon Sep 17 00:00:00 2001 From: Ivan Zlenko <241953+ivanzlenko@users.noreply.github.com> Date: Sun, 16 Feb 2025 19:07:40 +0500 Subject: [PATCH 4/4] HDDS-12339. Remove unnecessary line of code --- hadoop-ozone/dev-support/checks/pmd.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/dev-support/checks/pmd.sh b/hadoop-ozone/dev-support/checks/pmd.sh index 0a5f630145f8..1a9c9487f311 100755 --- a/hadoop-ozone/dev-support/checks/pmd.sh +++ b/hadoop-ozone/dev-support/checks/pmd.sh @@ -26,8 +26,6 @@ mkdir -p "$REPORT_DIR" REPORT_FILE="$REPORT_DIR/summary.txt" -declare -i rc - MAVEN_OPTIONS='-B -fae --no-transfer-progress -Dpmd.failOnViolation=false -Dpmd.printFailingErrors -DskipRecon' declare -i rc