HDDS-12734. Enable native lib in CI checks

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ on:
         default: ''
         required: false
 env:
-  BUILD_ARGS: "-Pdist -Psrc -Dmaven.javadoc.skip=true"
+  BUILD_ARGS: "-Pdist -Psrc -Dmaven.javadoc.skip=true -Drocks_tools_native"
   # Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
   TEST_JAVA_VERSION: 21 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
   # MAVEN_ARGS and MAVEN_OPTS are duplicated in check.yml, please keep in sync
@@ -44,7 +44,6 @@ jobs:
       acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
       integration-suites: ${{ steps.integration-suites.outputs.suites }}
       needs-basic-check: ${{ steps.categorize-basic-checks.outputs.needs-basic-check }}
-      needs-native-check: ${{ steps.categorize-basic-checks.outputs.needs-native-check }}
       basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
       needs-build: ${{ steps.selective-checks.outputs.needs-build }}
       needs-compile: ${{ steps.selective-checks.outputs.needs-compile }}
@@ -174,21 +173,6 @@ jobs:
         check: ${{ fromJson(needs.build-info.outputs.basic-checks) }}
       fail-fast: false
 
-  native:
-    needs:
-      - build-info
-      - basic
-    if: needs.build-info.outputs.needs-native-check == 'true'
-    uses: ./.github/workflows/check.yml
-    secrets: inherit
-    with:
-      java-version: ${{ needs.build-info.outputs.java-version }}
-      ratis-args: ${{ inputs.ratis_args }}
-      script: native
-      sha: ${{ needs.build-info.outputs.sha }}
-      timeout-minutes: 150
-      with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
-
   dependency:
     needs:
       - build-info
@@ -289,7 +273,7 @@ jobs:
       java-version: ${{ needs.build-info.outputs.java-version }}
       ratis-args: ${{ inputs.ratis_args }}
       script: integration
-      script-args: -Ptest-${{ matrix.profile }}
+      script-args: -Ptest-${{ matrix.profile }} -Drocks_tools_native
       sha: ${{ needs.build-info.outputs.sha }}
       split: ${{ matrix.profile }}
       timeout-minutes: 150
@@ -307,7 +291,6 @@ jobs:
       - build-info
       - acceptance
       - integration
-      - native
     steps:
       - name: Checkout project
         uses: actions/checkout@v4

dev-support/ci/categorize_basic_checks.sh

@@ -30,7 +30,7 @@ if [[ -n "${SPACE_DELIMITED_ALL_CHECKS}" ]]; then
   # add framing blanks
   SPACE_DELIMITED_ALL_CHECKS=" ${SPACE_DELIMITED_ALL_CHECKS[*]} "
 
-  for check in basic native; do
+  for check in basic; do
     CHECKS=$(grep -lr "^#checks:${check}$" hadoop-ozone/dev-support/checks \
       | sort -u \
       | xargs -n1 basename \

dev-support/ci/selective_ci_checks.bats

@@ -135,7 +135,7 @@ load bats-assert/load.bash
 @test "script change including junit.sh" {
   run dev-support/ci/selective_ci_checks.sh 66093e52c6
 
-  assert_output -p 'basic-checks=["rat","bats","checkstyle","findbugs","native"]'
+  assert_output -p 'basic-checks=["rat","bats","checkstyle","findbugs"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
@@ -179,22 +179,22 @@ load bats-assert/load.bash
 @test "native only" {
   run dev-support/ci/selective_ci_checks.sh 5b1319a8c2
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd","native"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
-  assert_output -p needs-compose-tests=false
-  assert_output -p needs-integration-tests=false
-  assert_output -p needs-kubernetes-tests=false
+  assert_output -p needs-compose-tests=true
+  assert_output -p needs-integration-tests=true
+  assert_output -p needs-kubernetes-tests=true
 }
 
 @test "native test in other module" {
   run dev-support/ci/selective_ci_checks.sh 822c0dee1a
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd","native"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
-  assert_output -p needs-integration-tests=false
+  assert_output -p needs-integration-tests=true
   assert_output -p needs-kubernetes-tests=false
 }
 
@@ -245,7 +245,7 @@ load bats-assert/load.bash
 @test "java and compose change" {
   run dev-support/ci/selective_ci_checks.sh d0f0f806e
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd","native"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","pmd"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -267,7 +267,7 @@ load bats-assert/load.bash
 @test "pom change" {
   run dev-support/ci/selective_ci_checks.sh 9129424a9
 
-  assert_output -p 'basic-checks=["rat","checkstyle","findbugs","pmd","native"]'
+  assert_output -p 'basic-checks=["rat","checkstyle","findbugs","pmd"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -278,7 +278,7 @@ load bats-assert/load.bash
 @test "CI lib change" {
   run dev-support/ci/selective_ci_checks.sh ceb79acaa
 
-  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","pmd","rat"]'
+  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","pmd","rat"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -289,7 +289,7 @@ load bats-assert/load.bash
 @test "CI workflow change" {
   run dev-support/ci/selective_ci_checks.sh 90a8d7c01
 
-  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","pmd","rat"]'
+  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","pmd","rat"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -312,7 +312,7 @@ load bats-assert/load.bash
 @test "CI workflow change (ci.yaml)" {
   run dev-support/ci/selective_ci_checks.sh 90fd5f2adc
 
-  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","native","pmd","rat"]'
+  assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","pmd","rat"]'
   assert_output -p needs-build=true
   assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
@@ -411,7 +411,7 @@ load bats-assert/load.bash
 @test "properties file in resources" {
   run dev-support/ci/selective_ci_checks.sh 71b8bdd8becf72d6f7d4e7986895504b8259b3e5
 
-  assert_output -p 'basic-checks=["rat","checkstyle","native"]'
+  assert_output -p 'basic-checks=["rat","checkstyle"]'
   assert_output -p needs-build=false
   assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=false

dev-support/ci/selective_ci_checks.sh

@@ -266,9 +266,6 @@ function get_count_integration_files() {
         "src/test/java"
         "src/test/resources"
     )
-    local ignore_array=(
-        $(grep -Flr 'org.apache.ozone.test.tag.Native' hadoop-ozone/integration-test)
-    )
     filter_changed_files true
     COUNT_INTEGRATION_CHANGED_FILES=${match_count}
     readonly COUNT_INTEGRATION_CHANGED_FILES
@@ -440,40 +437,6 @@ function check_needs_pmd() {
     start_end::group_end
 }
 
-function check_needs_native() {
-    start_end::group_start "Check if native is needed"
-    local pattern_array=(
-        "^hadoop-ozone/dev-support/checks/native.sh"
-        "^hadoop-hdds/rocks-native"
-        # include tests tagged as @Native in any module
-        $(grep -Flr 'org.apache.ozone.test.tag.Native' hadoop-*/*/src/test/java)
-    )
-    filter_changed_files true
-
-    if [[ ${match_count} != "0" ]]; then
-        add_basic_check native
-    else
-        local pattern_array=(
-            "^hadoop-ozone/dev-support/checks/junit.sh"
-            # dependencies
-            "^hadoop-hdds/annotations"
-            "^hadoop-hdds/common"
-            "^hadoop-hdds/config"
-            "^hadoop-hdds/hadoop-dependency-client"
-            "^hadoop-hdds/managed-rocksdb"
-            "^hadoop-hdds/test-utils"
-            "^pom.xml"
-        )
-        filter_changed_files
-
-        if [[ ${match_count} != "0" ]]; then
-            add_basic_check native
-        fi
-    fi
-
-    start_end::group_end
-}
-
 # Counts other files which do not need to trigger any functional test
 # (i.e. no compose/integration/kubernetes)
 function get_count_misc_files() {
@@ -609,6 +572,5 @@ check_needs_checkstyle
 check_needs_docs
 check_needs_findbugs
 check_needs_pmd
-check_needs_native
 calculate_test_types_to_run
 set_outputs

hadoop-hdds/rocks-native/pom.xml

@@ -336,7 +336,6 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <groups>native</groups>
               <argLine>${maven-surefire-plugin.argLine} @{argLine} -Djava.library.path=${project.build.directory}/native/rocksdb</argLine>
             </configuration>
           </plugin>

hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java

@@ -26,4 +26,5 @@ private NativeConstants() {
 
   }
   public static final String ROCKS_TOOLS_NATIVE_LIBRARY_NAME = "ozone_rocksdb_tools";
+  public static final String ROCKS_TOOLS_NATIVE_PROPERTY = "rocks_tools_native";
 }

hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java

@@ -32,6 +32,15 @@
  */
 public class ManagedRawSSTFileReader<T> implements Closeable {
 
+  public static boolean tryLoadLibrary() {
+    try {
+      loadLibrary();
+      return true;
+    } catch (NativeLibraryNotLoadedException ignored) {
+      return false;
+    }
+  }
+
   public static boolean loadLibrary() throws NativeLibraryNotLoadedException {
     ManagedRocksObjectUtils.loadRocksDBLibrary();
     if (!NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME, Arrays.asList(

hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdds.utils;
 
 import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
 import static org.apache.hadoop.hdds.utils.NativeLibraryLoader.NATIVE_LIB_TMP_DIR;
 import static org.apache.hadoop.hdds.utils.NativeLibraryLoader.getJniLibraryFileName;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -37,7 +38,7 @@
 import java.util.stream.Stream;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
-import org.apache.ozone.test.tag.Native;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -55,7 +56,7 @@ private static Stream<String> nativeLibraryDirectoryLocations() {
     return Stream.of("", tempDir.toAbsolutePath().toString(), null);
   }
 
-  @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+  @EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true")
   @ParameterizedTest
   @MethodSource("nativeLibraryDirectoryLocations")
   public void testNativeLibraryLoader(String nativeLibraryDirectoryLocation) throws NativeLibraryNotLoadedException {

hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileIterator.java

@@ -17,7 +17,7 @@
 
 package org.apache.hadoop.hdds.utils.db.managed;
 
-import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -37,9 +37,9 @@
 import org.apache.hadoop.hdds.StringUtils;
 import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException;
 import org.apache.hadoop.hdds.utils.TestUtils;
-import org.apache.ozone.test.tag.Native;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Named;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
@@ -48,7 +48,7 @@
 /**
  * Test for ManagedRawSSTFileReaderIterator.
  */
-@Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+@EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true")
 class TestManagedRawSSTFileIterator {
 
   @TempDir
@@ -74,7 +74,7 @@ private File createSSTFileWithKeys(
     return file;
   }
 
-  private static Stream<? extends Arguments> keyValueFormatArgs() {
+  private static Stream<Arguments> keyValueFormatArgs() {
     return Stream.of(Arguments.of(Named.of("Key starting with a single quote", "'key%1$d=>"),
             Named.of("Value starting with a number ending with a single quote", "%1$dvalue'")),
         Arguments.of(Named.of("Key ending with a number", "key%1$d"),

hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java

@@ -17,7 +17,7 @@
 
 package org.apache.ozone.rocksdb.util;
 
-import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
+import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -37,13 +37,12 @@
 import java.util.stream.Stream;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hdds.StringUtils;
-import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException;
 import org.apache.hadoop.hdds.utils.TestUtils;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter;
-import org.apache.ozone.test.tag.Native;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
@@ -153,12 +152,12 @@ public void testGetKeyStream(int numberOfFiles)
     }
   }
 
-  @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)
+  @EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true")
   @ParameterizedTest
   @ValueSource(ints = {0, 1, 2, 3, 7, 10})
   public void testGetKeyStreamWithTombstone(int numberOfFiles)
-      throws RocksDBException, IOException, NativeLibraryNotLoadedException {
-    assumeTrue(ManagedRawSSTFileReader.loadLibrary());
+      throws RocksDBException, IOException {
+    assumeTrue(ManagedRawSSTFileReader.tryLoadLibrary());
     Pair<SortedMap<String, Integer>, List<String>> data =
         createDummyData(numberOfFiles);
     List<String> files = data.getRight();