HDDS-12574. Add script to find modules by test classes

dev-support/ci/find_test_class_project.bats

@@ -0,0 +1,239 @@
+#!/usr/bin/env bats
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load find_test_class_project.sh
+
+setup() {
+  # Create a temporary directory for test files
+  TEST_DIR=$(mktemp -d)
+  mkdir -p "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test"
+  mkdir -p "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test"  
+  touch "${TEST_DIR}/project1/pom.xml"
+  touch "${TEST_DIR}/project2/pom.xml"
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/TestClass1.java"
+  touch "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test/TestClass2.java"
+
+  ORIG_DIR=$(pwd)
+  cd "${TEST_DIR}"
+}
+
+teardown() {
+  cd "${ORIG_DIR}"
+  rm -rf "${TEST_DIR}"
+}
+
+# Test the find_project_paths_for_test_class function
+
+@test "find project for simple class name" {
+  result=$(find_project_paths_for_test_class "TestClass1" 2>/dev/null)
+
+  [ "$result" = "./project1" ]
+}
+
+@test "find project for class with package" {
+  result=$(find_project_paths_for_test_class "org.apache.ozone.test.TestClass2" 2>/dev/null)
+
+  [ "$result" = "./project2" ]
+}
+
+@test "find project for wildcard class" {
+  result=$(find_project_paths_for_test_class "TestClass*" 2>/dev/null)
+  expected=$(echo -e "./project1\n./project2")
+
+  [ "$result" = "$expected" ]
+}
+
+@test "no project for non-existent class" {
+  result=$(find_project_paths_for_test_class "NonExistentClass" 2>/dev/null)
+
+  [ -z "$result" ]
+}
+
+@test "skip abstract classes" {
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/AbstractTestClass.java"
+
+  result=$(find_project_paths_for_test_class "AbstractTestClass" 2>/dev/null)
+
+  [ -z "$result" ]
+}
+
+@test "empty class name returns nothing" {
+  result=$(find_project_paths_for_test_class "" 2>/dev/null)
+
+  [ -z "$result" ]
+}
+
+@test "multiple projects with same test class name" {
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/CommonTest.java"
+  touch "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test/CommonTest.java"
+
+  result=$(find_project_paths_for_test_class "CommonTest" 2>/dev/null)
+
+  expected=$(echo -e "./project1\n./project2")
+  [ "$result" = "$expected" ]
+}
+
+@test "project without pom.xml is ignored" {
+  mkdir -p "${TEST_DIR}/project3/src/test/java/org/apache/ozone/test"
+  touch "${TEST_DIR}/project3/src/test/java/org/apache/ozone/test/TestClass3.java"
+
+  result=$(find_project_paths_for_test_class "TestClass3" 2>/dev/null)
+
+  [ -z "$result" ]
+}
+
+@test "partial package name search" {
+  result=$(find_project_paths_for_test_class "ozone.test.TestClass2" 2>/dev/null)
+
+  [ "$result" = "./project2" ]
+}
+
+@test "test class in non-standard test directory" {
+  mkdir -p "${TEST_DIR}/project1/src/test/scala/org/apache/ozone/test"
+  touch "${TEST_DIR}/project1/src/test/scala/org/apache/ozone/test/ScalaTest.java"
+
+  result=$(find_project_paths_for_test_class "ScalaTest" 2>/dev/null)
+
+  [ "$result" = "./project1" ]
+}
+
+@test "case sensitivity in class name" {
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/MixedCaseTest.java"
+
+  result=$(find_project_paths_for_test_class "mixedcasetest" 2>/dev/null)
+
+  [ -z "$result" ]
+}
+
+@test "nested project structure" {
+  mkdir -p "${TEST_DIR}/parent/child/src/test/java/org/apache/ozone/test"
+  touch "${TEST_DIR}/parent/child/pom.xml"
+  touch "${TEST_DIR}/parent/child/src/test/java/org/apache/ozone/test/NestedTest.java"
+
+  result=$(find_project_paths_for_test_class "NestedTest" 2>/dev/null)
+
+  [ "$result" = "./parent/child" ]
+}
+
+@test "test class with numeric suffix" {
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/Test1.java"
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/Test2.java"
+  touch "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test/Test3.java"
+
+  result=$(find_project_paths_for_test_class "Test[1-2]" 2>/dev/null)
+
+  [ "$result" = "./project1" ]
+}
+
+@test "multiple test classes matching pattern" {
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/TestA.java"
+  touch "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test/TestB.java"
+  touch "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test/TestC.java"
+
+  result=$(find_project_paths_for_test_class "Test[A-C]" 2>/dev/null)
+
+  expected=$(echo -e "./project1\n./project2")
+  [ "$result" = "$expected" ]
+}
+
+@test "test class in multiple package levels" {
+  mkdir -p "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/deep/nested/pkg"
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/deep/nested/pkg/DeepTest.java"
+
+  result=$(find_project_paths_for_test_class "org.apache.ozone.test.deep.nested.pkg.DeepTest" 2>/dev/null)
+
+  [ "$result" = "./project1" ]
+}
+
+@test "test class with same name in different packages" {
+  mkdir -p "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/pkg1"
+  mkdir -p "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test/pkg2"
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/pkg1/SameNameTest.java"
+  touch "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test/pkg2/SameNameTest.java"
+
+  result=$(find_project_paths_for_test_class "SameNameTest" 2>/dev/null)
+
+  expected=$(echo -e "./project1\n./project2")
+  [ "$result" = "$expected" ]
+}
+
+@test "test class with package wildcard" {
+  mkdir -p "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/pkg1"
+  mkdir -p "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test/pkg2"
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/pkg1/WildcardTest.java"
+  touch "${TEST_DIR}/project2/src/test/java/org/apache/ozone/test/pkg2/WildcardTest.java"
+
+  result=$(find_project_paths_for_test_class "org.apache.ozone.test.pkg*.WildcardTest" 2>/dev/null)
+
+  expected=$(echo -e "./project1\n./project2")
+  [ "$result" = "$expected" ]
+}
+
+@test "test class with exact package match" {
+  mkdir -p "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/exact"
+  mkdir -p "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/exactmatch"
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/exact/ExactTest.java"
+  touch "${TEST_DIR}/project1/src/test/java/org/apache/ozone/test/exactmatch/ExactTest.java"
+  result=$(find_project_paths_for_test_class "org.apache.ozone.test.exact.ExactTest" 2>/dev/null)
+
+  [ "$result" = "./project1" ]
+}
+
+@test "test class with trailing whitespace" {
+  result=$(find_project_paths_for_test_class "TestClass1 " 2>/dev/null)
+
+  [ "$result" = "./project1" ]
+}
+
+@test "test class project with trailing whitespace" {
+  result=$(find_project_paths_for_test_class "apache.ozone.test.TestClass1 " 2>/dev/null)
+
+  [ "$result" = "./project1" ]
+}
+
+@test "test class with leading whitespace" {
+  result=$(find_project_paths_for_test_class " TestClass1" 2>/dev/null)
+
+  [ "$result" = "./project1" ]
+}
+
+@test "test class with partial package and wildcard" {
+  result=$(find_project_paths_for_test_class "apache.*.TestClass*" 2>/dev/null)
+
+  expected=$(echo -e "./project1\n./project2")
+  [ "$result" = "$expected" ]
+}
+
+# Test the build_maven_project_list function
+
+@test "build maven project list with empty project paths" {
+  result=$(build_maven_project_list "")
+
+  [ "$result" = "" ]
+}
+
+@test "build maven project list with one project path" {
+  result=$(build_maven_project_list "./project1")
+
+  [ "$result" = "-pl ./project1" ]
+}
+
+@test "build maven project list with multiple project paths" {
+  local project_paths=$(echo -e "./project1\n./project2")
+  result=$(build_maven_project_list "$project_paths")
+
+  [ "$result" = "-pl ./project1,./project2" ]
+}

dev-support/ci/find_test_class_project.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Function to find project paths for a given test class
+find_project_paths_for_test_class() {
+  local test_class="$1"
+
+  if [[ -z "${test_class}" ]] || [[ "${test_class}" == "Abstract"* ]]; then
+    return
+  fi
+
+  echo "Finding project for test class: ${test_class}" >&2
+
+  # Trim test_class of whitespace
+  local base_class_name=$(echo "${test_class}" | xargs)
+
+  # If the base name is empty after removing wildcards, use a reasonable default
+  if [[ -z "${base_class_name}" ]]; then
+    echo "Test class name contains only wildcards, searching in all test directories" >&2
+    return
+  fi
+
+  echo "Searching for files matching base name: ${base_class_name}" >&2
+
+  # Find all projects containing matching test classes - use a more flexible search approach
+  # First try direct filename search
+  local test_files=($(find . -path "*/src/test/*" -name "${base_class_name}.java" | sort -u))
+
+  # If no files found and the class name contains dots (package notation), try searching by path
+  if [[ ${#test_files[@]} -eq 0 && "${base_class_name}" == *"."* ]]; then
+    # Convert base class to path format
+    local test_class_path="${base_class_name//./\/}.java"
+    echo "No files found with direct name search, trying path-based search" >&2
+    echo "TEST_CLASS_PATH pattern: ${test_class_path}" >&2
+
+    # Search by path pattern
+    test_files=($(find . -path "*/src/test/*/${test_class_path%.*}*.java" | sort -u))
+  fi
+
+  echo "Found ${#test_files[@]} matching test file(s)" >&2
+
+  if [[ ${#test_files[@]} -gt 0 ]]; then
+    # Extract project paths (up to the src/test directory)
+    local project_paths=()
+    for test_file in "${test_files[@]}"; do
+      echo "TEST_FILE: ${test_file}" >&2
+      local project_path=$(dirname "${test_file}" | sed -e 's|/src/test.*||')
+      if [[ -f "${project_path}/pom.xml" ]]; then
+        echo "Found test in project: ${project_path}" >&2
+        project_paths+=("${project_path}")
+      fi
+    done
+
+    printf '%s\n' "${project_paths[@]}" | sort -u
+  else
+    echo "Could not find project for test class pattern: ${test_class}" >&2
+  fi
+}
+
+# Takes a project list which is the output of `find_project_paths_for_test_class`
+# and returns a string that can use for maven -pl option, eg. "./project1\n./project2" -> "-pl ./project1,./project2"
+build_maven_project_list() {
+  local project_paths="$1"
+  if [[ -z "${project_paths}" ]]; then
+    echo ""
+    return
+  fi
+
+  local comma_separated=$(echo "${project_paths}" | tr '\n' ',')
+  comma_separated="${comma_separated%,}"
+  echo "-pl ${comma_separated}"
+}
+
+# If option get-pl set, write the maven -pl option value to stdout
+# otherwise, write the project paths to stdout
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+  if [[ "$1" == "--get-pl" ]]; then
+    shift
+    project_paths=$(find_project_paths_for_test_class "$@")
+    build_maven_project_list "${project_paths}"
+  else
+    find_project_paths_for_test_class "$@"
+  fi
+fi

dev-support/ci/selective_ci_checks.sh

@@ -199,6 +199,7 @@ function run_all_tests_if_environment_files_changed() {
     )
     local ignore_array=(
         "^dev-support/ci/pr_title_check"
+        "^dev-support/ci/find_test_class_project"
     )
     filter_changed_files
 
@@ -478,6 +479,7 @@ function get_count_misc_files() {
     start_end::group_start "Count misc. files"
     local pattern_array=(
         "^dev-support/ci/pr_title_check"
+        "^dev-support/ci/find_test_class_project"
         "^.github"
         "^hadoop-hdds/dev-support/checkstyle"
         "^hadoop-ozone/dev-support/checks"

hadoop-ozone/dev-support/checks/bats.sh

@@ -34,6 +34,7 @@ find * \( \
     -path '*/src/test/shell/*' -name '*.bats' \
     -or -path dev-support/ci/selective_ci_checks.bats \
     -or -path dev-support/ci/pr_title_check.bats \
+    -or -path dev-support/ci/find_test_class_project.bats \
     \) -print0 \
   | xargs -0 -n1 bats --formatter tap \
   | tee -a "${REPORT_DIR}/output.log"