From 4d60dc7d34d7fa7efd598416dd3639e88a170a25 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 6 Feb 2024 15:41:04 -0800 Subject: [PATCH 01/39] HDDS-10149. Fix native lib to not include rocksdb library as static library --- hadoop-hdds/rocks-native/pom.xml | 2 ++ hadoop-hdds/rocks-native/src/CMakeLists.txt | 2 +- .../ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index c12ddbb091bd..5b21680db615 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -354,6 +354,8 @@ + diff --git a/hadoop-hdds/rocks-native/src/CMakeLists.txt b/hadoop-hdds/rocks-native/src/CMakeLists.txt index 051660777493..42614ac4fa91 100644 --- a/hadoop-hdds/rocks-native/src/CMakeLists.txt +++ b/hadoop-hdds/rocks-native/src/CMakeLists.txt @@ -37,7 +37,7 @@ include_directories(${GENERATED_JAVAH}) if(${SST_DUMP_INCLUDE}) include_directories(${ROCKSDB_HEADERS}) set(SOURCE_FILES ${NATIVE_DIR}/SSTDumpTool.cpp ${NATIVE_DIR}/PipeInputStream.cpp ${NATIVE_DIR}/Pipe.h ${NATIVE_DIR}/Pipe.cpp ${NATIVE_DIR}/cplusplus_to_java_convert.h) - ADD_LIBRARY(rocksdb STATIC IMPORTED) + ADD_LIBRARY(rocksdb SHARED IMPORTED) set_target_properties( rocksdb PROPERTIES diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java index 5ed2f848aed8..fcc975cd18f1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java @@ -30,7 +30,6 @@ */ @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) @Timeout(300) -@Unhealthy("HDDS-10149") class TestOmSnapshotFsoWithNativeLib extends TestOmSnapshot { TestOmSnapshotFsoWithNativeLib() throws Exception { super(FILE_SYSTEM_OPTIMIZED, false, false, false); From 2dc7c0ba09c77eb21f1e9ac1e124626067859533 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 6 Feb 2024 16:12:53 -0800 Subject: [PATCH 02/39] HDDS-10149. Fix checkstyle --- .../hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java index fcc975cd18f1..06fbebb2efa2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java @@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.om.snapshot; import org.apache.ozone.test.tag.Native; -import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.Timeout; import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; From 824678d05ed709e11695732f820ed117c3de00b1 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 6 Feb 2024 18:00:45 -0800 Subject: [PATCH 03/39] HDDS-10149. Test --- .../ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java index 06fbebb2efa2..5ed2f848aed8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.snapshot; import org.apache.ozone.test.tag.Native; +import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.Timeout; import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; @@ -29,6 +30,7 @@ */ @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) @Timeout(300) +@Unhealthy("HDDS-10149") class TestOmSnapshotFsoWithNativeLib extends TestOmSnapshot { TestOmSnapshotFsoWithNativeLib() throws Exception { super(FILE_SYSTEM_OPTIMIZED, false, false, false); From f2a78693c97ac621d1126d39de588c9b532f4aba Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 13 Feb 2024 18:50:36 -0800 Subject: [PATCH 04/39] HDDS-10149. New JNI layer containing new rawSstFileReader on rocksdb tools --- .../db/managed/JniLibNamePropertyWriter.java} | 40 +- hadoop-hdds/rocks-native/pom.xml | 204 +-- hadoop-hdds/rocks-native/src/CMakeLists.txt | 33 +- .../hdds/utils/NativeLibraryLoader.java | 1 + .../db/managed/ManagedRawSSTFileReader.java | 92 ++ .../ManagedRawSSTFileReaderIterator.java | 117 ++ .../main/native/ManagedRawSSTFileReader.cpp | 65 + .../ManagedRawSSTFileReaderIterator.cpp | 90 ++ .../rocks-native/src/main/native/Pipe.h | 55 - .../src/main/native/PipeInputStream.cpp | 48 - .../src/main/native/SSTDumpTool.cpp | 48 - .../src/main/patches/rocks-native.patch | 1133 ++++++++--------- .../TestManagedRawSSTFileReaderIterator.java | 179 +++ .../managed/TestManagedSSTDumpIterator.java | 282 ---- pom.xml | 12 +- 15 files changed, 1177 insertions(+), 1222 deletions(-) rename hadoop-hdds/{rocks-native/src/main/native/Pipe.cpp => managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java} (51%) create mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java create mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java create mode 100644 hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReader.cpp create mode 100644 hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReaderIterator.cpp delete mode 100644 hadoop-hdds/rocks-native/src/main/native/Pipe.h delete mode 100644 hadoop-hdds/rocks-native/src/main/native/PipeInputStream.cpp delete mode 100644 hadoop-hdds/rocks-native/src/main/native/SSTDumpTool.cpp create mode 100644 hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java delete mode 100644 hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedSSTDumpIterator.java diff --git a/hadoop-hdds/rocks-native/src/main/native/Pipe.cpp b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java similarity index 51% rename from hadoop-hdds/rocks-native/src/main/native/Pipe.cpp rename to hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java index f1dd54438700..e405a2d884f0 100644 --- a/hadoop-hdds/rocks-native/src/main/native/Pipe.cpp +++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java @@ -6,32 +6,38 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.hadoop.hdds.utils.db.managed; -#include "Pipe.h" -#include +import org.rocksdb.util.Environment; -const int Pipe::READ_FILE_DESCRIPTOR_IDX = 0; -const int Pipe::WRITE_FILE_DESCRIPTOR_IDX = 1; +import java.io.FileWriter; +import java.io.IOException; -Pipe::Pipe() { - pipe(p); - open = true; -} +/** + * Class to write the rocksdb lib name to a file. + * This would be used to build native ozone_rocksdb_tools library. + */ +public final class JniLibNamePropertyWriter { -Pipe::~Pipe() { - ::close(p[Pipe::READ_FILE_DESCRIPTOR_IDX]); - ::close(p[Pipe::WRITE_FILE_DESCRIPTOR_IDX]); -} + private JniLibNamePropertyWriter() { + } -void Pipe::close() { - open = false; + public static void main(String[] args) { + String filePath = args[0]; + try (FileWriter writer = new FileWriter(filePath)) { + writer.write("rocksdbLibName=" + + Environment.getJniLibraryFileName("rocksdb")); + } catch (IOException e) { + e.printStackTrace(); + } + } } diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index 5b21680db615..ab862c812b12 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -55,8 +55,6 @@ 8 8 - https://sourceware.org/pub/bzip2/bzip2-${bzip2.version}.tar.gz - https://zlib.net/fossils/zlib-${zlib.version}.tar.gz @@ -113,79 +111,80 @@ - com.googlecode.maven-download-plugin - download-maven-plugin + org.codehaus.mojo + exec-maven-plugin - rocksdb source download - generate-sources - - wget - - - https://github.com/facebook/rocksdb/archive/refs/tags/v${rocksdb.version}.tar.gz - rocksdb-v${rocksdb.version}.tar.gz - ${project.build.directory}/rocksdb - - - - zlib source download - generate-sources - - wget - - - ${zlib.url} - zlib-${zlib.version}.tar.gz - ${project.build.directory}/zlib - - - - bzip2 source download - generate-sources + set-property + initialize - wget + java - ${bzip2.url} - bzip2-v${bzip2.version}.tar.gz - ${project.build.directory}/bzip2 + org.apache.hadoop.hdds.utils.db.managed.JniLibNamePropertyWriter + + ${project.build.directory}/propertyFile.txt + + + + + org.codehaus.mojo + properties-maven-plugin + - lz4 source download - generate-sources + read-property-from-file + initialize - wget + read-project-properties - https://github.com/lz4/lz4/archive/refs/tags/v${lz4.version}.tar.gz - lz4-v${lz4.version}.tar.gz - ${project.build.directory}/lz4 + + ${project.build.directory}/propertyFile.txt + + + + + org.apache.maven.plugins + maven-dependency-plugin + - snappy source download - generate-sources + unpack-dependency + initialize - wget + unpack - https://github.com/google/snappy/archive/refs/tags/${snappy.version}.tar.gz - snappy-v${snappy.version}.tar.gz - ${project.build.directory}/snappy + + + org.rocksdb + rocksdbjni + jar + false + ${project.build.directory}/rocksdbjni + + + + + + com.googlecode.maven-download-plugin + download-maven-plugin + - zstd source download + rocksdb source download generate-sources wget - https://github.com/facebook/zstd/archive/refs/tags/v${zstd.version}.tar.gz - zstd-v${zstd.version}.tar.gz - ${project.build.directory}/zstd + https://github.com/facebook/rocksdb/archive/refs/tags/v${rocksdb.version}.tar.gz + rocksdb-v${rocksdb.version}.tar.gz + ${project.build.directory}/rocksdb @@ -219,89 +218,6 @@ - - - - - - - - - - run - - - - build-zlib - process-sources - - - - - - - - - - - - run - - - - build-bzip2 - process-sources - - - - - - - - - run - - - - build-lz4 - process-sources - - - - - - - - - run - - - - build-zstd - process-sources - - - - - - - - - run - - - - build-snappy - process-sources - - - - - - - - - @@ -320,8 +236,6 @@ - - @@ -346,16 +260,12 @@ - - - - - - + + - + + @@ -425,8 +335,8 @@ ${env.JAVA_HOME}/bin/javah - org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool - org.apache.hadoop.hdds.utils.db.managed.PipeInputStream + org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader + org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReaderIterator ${project.build.directory}/native/javah @@ -481,8 +391,8 @@ ${project.build.outputDirectory}:${project.build.directory}/dependency/* -h ${project.build.directory}/native/javah - ${project.basedir}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpTool.java - ${project.basedir}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/PipeInputStream.java + ${project.basedir}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java + ${project.basedir}/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java diff --git a/hadoop-hdds/rocks-native/src/CMakeLists.txt b/hadoop-hdds/rocks-native/src/CMakeLists.txt index 42614ac4fa91..5c3a31717cf5 100644 --- a/hadoop-hdds/rocks-native/src/CMakeLists.txt +++ b/hadoop-hdds/rocks-native/src/CMakeLists.txt @@ -36,43 +36,18 @@ endif() include_directories(${GENERATED_JAVAH}) if(${SST_DUMP_INCLUDE}) include_directories(${ROCKSDB_HEADERS}) - set(SOURCE_FILES ${NATIVE_DIR}/SSTDumpTool.cpp ${NATIVE_DIR}/PipeInputStream.cpp ${NATIVE_DIR}/Pipe.h ${NATIVE_DIR}/Pipe.cpp ${NATIVE_DIR}/cplusplus_to_java_convert.h) + set(SOURCE_FILES ${NATIVE_DIR}/ManagedRawSSTFileReader.cpp ${NATIVE_DIR}/ManagedRawSSTFileReaderIterator.cpp ${NATIVE_DIR}/cplusplus_to_java_convert.h) ADD_LIBRARY(rocksdb SHARED IMPORTED) set_target_properties( rocksdb PROPERTIES - IMPORTED_LOCATION ${ROCKSDB_LIB}/librocksdb.a) + IMPORTED_LOCATION ${ROCKSDB_LIB}) ADD_LIBRARY(rocks_tools STATIC IMPORTED) set_target_properties( rocks_tools PROPERTIES - IMPORTED_LOCATION ${ROCKSDB_LIB}/librocksdb_tools.a) - ADD_LIBRARY(bz2 STATIC IMPORTED) - set_target_properties( - bz2 - PROPERTIES - IMPORTED_LOCATION ${BZIP2_LIB}/libbz2.a) - ADD_LIBRARY(zlib STATIC IMPORTED) - set_target_properties( - zlib - PROPERTIES - IMPORTED_LOCATION ${ZLIB_LIB}/libz.a) - ADD_LIBRARY(lz4 STATIC IMPORTED) - set_target_properties( - lz4 - PROPERTIES - IMPORTED_LOCATION ${LZ4_LIB}/liblz4.a) - ADD_LIBRARY(snappy STATIC IMPORTED) - set_target_properties( - snappy - PROPERTIES - IMPORTED_LOCATION ${SNAPPY_LIB}/libsnappy.a) - ADD_LIBRARY(zstd STATIC IMPORTED) - set_target_properties( - zstd - PROPERTIES - IMPORTED_LOCATION ${ZSTD_LIB}/libzstd.a) - set(linked_libraries ${linked_libraries} bz2 zlib rocks_tools rocksdb lz4 snappy zstd) + IMPORTED_LOCATION ${ROCKSDB_TOOLS_LIB}/librocksdb_tools.a) + set(linked_libraries ${linked_libraries} rocks_tools rocksdb) endif() add_library(ozone_rocksdb_tools SHARED ${SOURCE_FILES}) target_link_libraries(ozone_rocksdb_tools ${linked_libraries}) diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java index 10df236f88d4..1264078c6e01 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java @@ -126,6 +126,7 @@ public synchronized boolean loadLibrary(final String libraryName) { LOG.warn("Unable to load library: {}", libraryName, e); } this.librariesLoaded.put(libraryName, loaded); + LOG.info("Loaded Library {}", libraryName); return isLibraryLoaded(libraryName); } diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java new file mode 100644 index 000000000000..57b6e574a873 --- /dev/null +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.utils.db.managed; + +import org.apache.hadoop.hdds.utils.NativeLibraryLoader; +import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.util.function.Function; + +import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; + +/** + * JNI for RocksDB RawSSTFileReader. + */ +public class ManagedRawSSTFileReader implements Closeable { + + public static boolean loadLibrary() throws NativeLibraryNotLoadedException { + ManagedRocksObjectUtils.loadRocksDBLibrary(); + if (!NativeLibraryLoader.getInstance() + .loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)) { + throw new NativeLibraryNotLoadedException( + ROCKS_TOOLS_NATIVE_LIBRARY_NAME); + } + return true; + } + + private final ManagedOptions options; + private final String fileName; + private final int readAheadSize; + private final long nativeHandle; + private static final Logger LOG = + LoggerFactory.getLogger(ManagedRawSSTFileReader.class); + + public ManagedRawSSTFileReader(final ManagedOptions options, + final String fileName, + final int readAheadSize) { + this.options = options; + this.fileName = fileName; + this.readAheadSize = readAheadSize; + this.nativeHandle = this.newRawSSTFileReader(options.getNativeHandle(), + fileName, readAheadSize); + } + + public ManagedRawSSTFileReaderIterator newIterator( + Function transformerFunction, + ManagedSlice fromSlice, ManagedSlice toSlice) { + long fromNativeHandle = fromSlice == null ? 0 : fromSlice.getNativeHandle(); + long toNativeHandle = toSlice == null ? 0 : toSlice.getNativeHandle(); + LOG.info("Iterating SST file: {} with native lib. " + + "LowerBound: {}, UpperBound: {}", fileName, fromSlice, toSlice); + return new ManagedRawSSTFileReaderIterator<>( + newIterator(this.nativeHandle, fromSlice != null, + fromNativeHandle, toSlice != null, toNativeHandle), + transformerFunction); + } + + private native long newRawSSTFileReader(long optionsHandle, + String filePath, + int readSize); + + + private native long newIterator(long handle, + boolean hasFrom, + long fromSliceHandle, + boolean hasTo, + long toSliceHandle); + + private native void disposeInternal(long handle); + + @Override + public void close() { + disposeInternal(nativeHandle); + } +} diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java new file mode 100644 index 000000000000..a4f1b30f38e0 --- /dev/null +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.utils.db.managed; + +import com.google.common.primitives.UnsignedLong; +import org.apache.hadoop.hdds.StringUtils; +import org.apache.hadoop.util.ClosableIterator; + +import java.util.function.Function; + +/** + * Iterator for SSTFileReader which would read all entries including tombstones. + */ +public class ManagedRawSSTFileReaderIterator + implements ClosableIterator { + private final long nativeHandle; + private final Function transformer; + + ManagedRawSSTFileReaderIterator(long nativeHandle, + Function transformer) { + this.nativeHandle = nativeHandle; + this.transformer = transformer; + } + + private native boolean hasNext(long handle); + private native void next(long handle); + private native byte[] getKey(long handle); + private native byte[] getValue(long handle); + private native long getSequenceNumber(long handle); + private native int getType(long handle); + + @Override + public boolean hasNext() { + return this.hasNext(nativeHandle); + } + + @Override + public T next() { + if (!hasNext()) { + return null; + } + + KeyValue keyValue = new KeyValue(this.getKey(nativeHandle), + UnsignedLong.fromLongBits(this.getSequenceNumber(this.nativeHandle)), + this.getType(nativeHandle), + this.getValue(nativeHandle)); + this.next(nativeHandle); + return this.transformer.apply(keyValue); + } + + private native void closeInternal(long handle); + + @Override + public void close() { + this.closeInternal(this.nativeHandle); + } + + /** + * Class containing Parsed KeyValue Record from RawSstReader output. + */ + public static final class KeyValue { + + private final byte[] key; + private final UnsignedLong sequence; + private final Integer type; + private final byte[] value; + + private KeyValue(byte[] key, UnsignedLong sequence, Integer type, + byte[] value) { + this.key = key; + this.sequence = sequence; + this.type = type; + this.value = value; + } + + public byte[] getKey() { + return key; + } + + public UnsignedLong getSequence() { + return sequence; + } + + public Integer getType() { + return type; + } + + public byte[] getValue() { + return value; + } + + @Override + public String toString() { + return "KeyValue{" + + "key=" + StringUtils.bytes2String(key) + + ", sequence=" + sequence + + ", type=" + type + + ", value=" + StringUtils.bytes2String(value) + + '}'; + } + } +} diff --git a/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReader.cpp b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReader.cpp new file mode 100644 index 000000000000..992b66a7632b --- /dev/null +++ b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReader.cpp @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReader.h" +#include "rocksdb/options.h" +#include "rocksdb/raw_sst_file_reader.h" +#include "rocksdb/raw_iterator.h" +#include +#include "cplusplus_to_java_convert.h" +#include + +jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReader_newRawSSTFileReader(JNIEnv *env, jobject obj, + jlong optionsHandle, + jstring jfilename, + jint readahead_size) { + ROCKSDB_NAMESPACE::Options *options = reinterpret_cast(optionsHandle); + const char *file_path = env->GetStringUTFChars(jfilename, nullptr); + size_t readAheadSizeValue = static_cast(readahead_size); + ROCKSDB_NAMESPACE::RawSstFileReader* rawSstFileReader = + new ROCKSDB_NAMESPACE::RawSstFileReader(*options, file_path, readAheadSizeValue, true, true); + env->ReleaseStringUTFChars(jfilename, file_path); + return GET_CPLUSPLUS_POINTER(rawSstFileReader); +} + +jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReader_newIterator(JNIEnv *env, jobject obj, + jlong native_handle, + jboolean jhas_from, + jlong fromSliceHandle, + jboolean jhas_to, + jlong toSliceHandle) { + ROCKSDB_NAMESPACE::Slice* fromSlice = nullptr; + ROCKSDB_NAMESPACE::Slice* toSlice = nullptr; + ROCKSDB_NAMESPACE::RawSstFileReader* rawSstFileReader = + reinterpret_cast(native_handle); + bool has_from = static_cast(jhas_from); + bool has_to = static_cast(jhas_to); + if (has_from) { + fromSlice = reinterpret_cast(fromSliceHandle); + } + if (has_to) { + toSlice = reinterpret_cast(toSliceHandle); + } + ROCKSDB_NAMESPACE::RawIterator* iterator = rawSstFileReader->newIterator(has_from, fromSlice, has_to, toSlice); + return GET_CPLUSPLUS_POINTER(iterator); +} + +void Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReader_disposeInternal(JNIEnv *env, jobject obj, + jlong native_handle) { + delete reinterpret_cast(native_handle); +} diff --git a/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReaderIterator.cpp b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReaderIterator.cpp new file mode 100644 index 000000000000..cc132162f16a --- /dev/null +++ b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReaderIterator.cpp @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator.h" +#include "rocksdb/options.h" +#include "rocksdb/raw_iterator.h" +#include +#include "cplusplus_to_java_convert.h" +#include + +jboolean Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_hasNext(JNIEnv *env, jobject obj, + jlong native_handle) { + return static_cast(reinterpret_cast(native_handle)->has_next()); +} + +void Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_next(JNIEnv *env, jobject obj, + jlong native_handle) { + reinterpret_cast(native_handle)->next(); +} + +jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_getKey(JNIEnv *env, + jobject obj, + jlong native_handle) { + ROCKSDB_NAMESPACE::Slice slice = reinterpret_cast(native_handle)->getKey(); + jbyteArray jkey = env->NewByteArray(static_cast(slice.size())); + if (jkey == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion( + jkey, 0, static_cast(slice.size()), + const_cast(reinterpret_cast(slice.data()))); + return jkey; +} + + +jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_getValue(JNIEnv *env, + jobject obj, + jlong native_handle) { + ROCKSDB_NAMESPACE::Slice slice = reinterpret_cast(native_handle)->getValue(); + jbyteArray jkey = env->NewByteArray(static_cast(slice.size())); + if (jkey == nullptr) { + // exception thrown: OutOfMemoryError + return nullptr; + } + env->SetByteArrayRegion( + jkey, 0, static_cast(slice.size()), + const_cast(reinterpret_cast(slice.data()))); + return jkey; +} + +jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_getSequenceNumber(JNIEnv *env, + jobject obj, + jlong native_handle) { + uint64_t sequence_number = + reinterpret_cast(native_handle)->getSequenceNumber(); + jlong result; + std::memcpy(&result, &sequence_number, sizeof(jlong)); + return result; +} + + +jint Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_getType(JNIEnv *env, + jobject obj, + jlong native_handle) { + uint32_t type = reinterpret_cast(native_handle)->getType(); + return static_cast(type); +} + + +void Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_closeInternal(JNIEnv *env, + jobject obj, + jlong native_handle) { + delete reinterpret_cast(native_handle); +} \ No newline at end of file diff --git a/hadoop-hdds/rocks-native/src/main/native/Pipe.h b/hadoop-hdds/rocks-native/src/main/native/Pipe.h deleted file mode 100644 index aa75c6311cbc..000000000000 --- a/hadoop-hdds/rocks-native/src/main/native/Pipe.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ROCKS_NATIVE_PIPE_H -#define ROCKS_NATIVE_PIPE_H - -#include - -class Pipe { - public: - static const int READ_FILE_DESCRIPTOR_IDX; - static const int WRITE_FILE_DESCRIPTOR_IDX; - Pipe(); - ~Pipe(); - void close(); - int getReadFd() { - return getPipeFileDescriptorIndex(READ_FILE_DESCRIPTOR_IDX); - } - - int getWriteFd() { - return getPipeFileDescriptorIndex(WRITE_FILE_DESCRIPTOR_IDX); - } - - int getPipeFileDescriptorIndex(int idx) { - return p[idx]; - } - - bool isOpen() { - return open; - } - - - private: - int p[2]; - FILE* wr; - bool open; - -}; - -#endif //ROCKS_NATIVE_PIPE_H diff --git a/hadoop-hdds/rocks-native/src/main/native/PipeInputStream.cpp b/hadoop-hdds/rocks-native/src/main/native/PipeInputStream.cpp deleted file mode 100644 index 53f60cdd65af..000000000000 --- a/hadoop-hdds/rocks-native/src/main/native/PipeInputStream.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "Pipe.h" -#include "cplusplus_to_java_convert.h" -#include "org_apache_hadoop_hdds_utils_db_managed_PipeInputStream.h" - - -jlong Java_org_apache_hadoop_hdds_utils_db_managed_PipeInputStream_newPipe(JNIEnv *, jobject) { - Pipe *pipe = new Pipe(); - return GET_CPLUSPLUS_POINTER(pipe); -} - -jint Java_org_apache_hadoop_hdds_utils_db_managed_PipeInputStream_readInternal(JNIEnv *env, jobject object, jbyteArray jbyteArray, jint capacity, jlong nativeHandle) { - int cap_int = capacity; - Pipe *pipe = reinterpret_cast(nativeHandle); - jbyte *b = (env)->GetByteArrayElements(jbyteArray, JNI_FALSE); - cap_int = read(pipe->getReadFd(), b, cap_int); - if (cap_int == 0) { - if (!pipe->isOpen()) { - cap_int = -1; - } - } - env->ReleaseByteArrayElements(jbyteArray, b, 0); - return cap_int; -} - -void Java_org_apache_hadoop_hdds_utils_db_managed_PipeInputStream_closeInternal(JNIEnv *env, jobject object, jlong nativeHandle) { - delete reinterpret_cast(nativeHandle); -} - diff --git a/hadoop-hdds/rocks-native/src/main/native/SSTDumpTool.cpp b/hadoop-hdds/rocks-native/src/main/native/SSTDumpTool.cpp deleted file mode 100644 index 285c5906c2d8..000000000000 --- a/hadoop-hdds/rocks-native/src/main/native/SSTDumpTool.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "org_apache_hadoop_hdds_utils_db_managed_ManagedSSTDumpTool.h" -#include "rocksdb/options.h" -#include "rocksdb/sst_dump_tool.h" -#include -#include "cplusplus_to_java_convert.h" -#include "Pipe.h" -#include - -jint Java_org_apache_hadoop_hdds_utils_db_managed_ManagedSSTDumpTool_runInternal(JNIEnv *env, jobject obj, - jobjectArray argsArray, jlong optionsHandle, jlong pipeHandle) { - ROCKSDB_NAMESPACE::SSTDumpTool dumpTool; - ROCKSDB_NAMESPACE::Options options; - Pipe *pipe = reinterpret_cast(pipeHandle); - int length = env->GetArrayLength(argsArray); - char *args[length + 1]; - for (int i = 0; i < length; i++) { - jstring str_val = (jstring)env->GetObjectArrayElement(argsArray, (jsize)i); - char *utf_str = (char *)env->GetStringUTFChars(str_val, JNI_FALSE); - args[i + 1] = utf_str; - } - FILE *wr = fdopen(pipe->getWriteFd(), "w"); - int ret = dumpTool.Run(length + 1, args, options, wr); - for (int i = 1; i < length + 1; i++) { - jstring str_val = (jstring)env->GetObjectArrayElement(argsArray, (jsize)(i - 1)); - env->ReleaseStringUTFChars(str_val, args[i]); - } - fclose(wr); - pipe->close(); - return ret; -} diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index 841c2533b863..5d0cf99ac486 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -1,607 +1,560 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +commit 0d13ca6c284e96d9e055ed8ba83e339120f6a5c5 +Author: Swaminathan Balachandran +Date: Mon Feb 12 19:13:13 2024 -0800 -diff --git a/include/rocksdb/sst_dump_tool.h b/include/rocksdb/sst_dump_tool.h -index 9261ba47d..1e62b88a3 100644 ---- a/include/rocksdb/sst_dump_tool.h -+++ b/include/rocksdb/sst_dump_tool.h -@@ -11,7 +11,8 @@ namespace ROCKSDB_NAMESPACE { - - class SSTDumpTool { - public: -- int Run(int argc, char const* const* argv, Options options = Options()); -+ int Run(int argc, char const* const* argv, Options options = Options(), -+ FILE* out = stdout, FILE* err = stderr); - }; - - } // namespace ROCKSDB_NAMESPACE -diff --git a/table/sst_file_dumper.cc b/table/sst_file_dumper.cc -index eefbaaeee..734a2f0dd 100644 ---- a/table/sst_file_dumper.cc -+++ b/table/sst_file_dumper.cc -@@ -45,7 +45,7 @@ SstFileDumper::SstFileDumper(const Options& options, - Temperature file_temp, size_t readahead_size, - bool verify_checksum, bool output_hex, - bool decode_blob_index, const EnvOptions& soptions, -- bool silent) -+ bool silent, FILE* out, FILE* err) - : file_name_(file_path), - read_num_(0), - file_temp_(file_temp), -@@ -57,10 +57,13 @@ SstFileDumper::SstFileDumper(const Options& options, - ioptions_(options_), - moptions_(ColumnFamilyOptions(options_)), - read_options_(verify_checksum, false), -- internal_comparator_(BytewiseComparator()) { -+ internal_comparator_(BytewiseComparator()), -+ out_(out), -+ err_(err) -+ { - read_options_.readahead_size = readahead_size; - if (!silent_) { -- fprintf(stdout, "Process %s\n", file_path.c_str()); -+ fprintf(out_, "Process %s\n", file_path.c_str()); - } - init_result_ = GetTableReader(file_name_); - } -@@ -253,17 +256,17 @@ Status SstFileDumper::ShowAllCompressionSizes( - int32_t compress_level_from, int32_t compress_level_to, - uint32_t max_dict_bytes, uint32_t zstd_max_train_bytes, - uint64_t max_dict_buffer_bytes, bool use_zstd_dict_trainer) { -- fprintf(stdout, "Block Size: %" ROCKSDB_PRIszt "\n", block_size); -+ fprintf(out_, "Block Size: %" ROCKSDB_PRIszt "\n", block_size); - for (auto& i : compression_types) { - if (CompressionTypeSupported(i.first)) { -- fprintf(stdout, "Compression: %-24s\n", i.second); -+ fprintf(out_, "Compression: %-24s\n", i.second); - CompressionOptions compress_opt; - compress_opt.max_dict_bytes = max_dict_bytes; - compress_opt.zstd_max_train_bytes = zstd_max_train_bytes; - compress_opt.max_dict_buffer_bytes = max_dict_buffer_bytes; - compress_opt.use_zstd_dict_trainer = use_zstd_dict_trainer; - for (int32_t j = compress_level_from; j <= compress_level_to; j++) { -- fprintf(stdout, "Compression level: %d", j); -+ fprintf(out_, "Compression level: %d", j); - compress_opt.level = j; - Status s = ShowCompressionSize(block_size, i.first, compress_opt); - if (!s.ok()) { -@@ -271,7 +274,7 @@ Status SstFileDumper::ShowAllCompressionSizes( - } - } - } else { -- fprintf(stdout, "Unsupported compression type: %s.\n", i.second); -+ fprintf(err_, "Unsupported compression type: %s.\n", i.second); - } - } - return Status::OK(); -@@ -307,9 +310,9 @@ Status SstFileDumper::ShowCompressionSize( - } - - std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); -- fprintf(stdout, " Size: %10" PRIu64, file_size); -- fprintf(stdout, " Blocks: %6" PRIu64, num_data_blocks); -- fprintf(stdout, " Time Taken: %10s microsecs", -+ fprintf(out_, " Size: %10" PRIu64, file_size); -+ fprintf(out_, " Blocks: %6" PRIu64, num_data_blocks); -+ fprintf(out_, " Time Taken: %10s microsecs", - std::to_string( - std::chrono::duration_cast(end - start) - .count()) -@@ -342,11 +345,11 @@ Status SstFileDumper::ShowCompressionSize( - : ((static_cast(not_compressed_blocks) / - static_cast(num_data_blocks)) * - 100.0); -- fprintf(stdout, " Compressed: %6" PRIu64 " (%5.1f%%)", compressed_blocks, -+ fprintf(out_, " Compressed: %6" PRIu64 " (%5.1f%%)", compressed_blocks, - compressed_pcnt); -- fprintf(stdout, " Not compressed (ratio): %6" PRIu64 " (%5.1f%%)", -+ fprintf(out_, " Not compressed (ratio): %6" PRIu64 " (%5.1f%%)", - ratio_not_compressed_blocks, ratio_not_compressed_pcnt); -- fprintf(stdout, " Not compressed (abort): %6" PRIu64 " (%5.1f%%)\n", -+ fprintf(out_, " Not compressed (abort): %6" PRIu64 " (%5.1f%%)\n", - not_compressed_blocks, not_compressed_pcnt); - return Status::OK(); - } -@@ -362,7 +365,7 @@ Status SstFileDumper::ReadTableProperties(uint64_t table_magic_number, - /* memory_allocator= */ nullptr, prefetch_buffer); - if (!s.ok()) { - if (!silent_) { -- fprintf(stdout, "Not able to read table properties\n"); -+ fprintf(err_, "Not able to read table properties\n"); - } - } - return s; -@@ -410,7 +413,7 @@ Status SstFileDumper::SetTableOptionsByMagicNumber( - - options_.table_factory.reset(NewPlainTableFactory(plain_table_options)); - if (!silent_) { -- fprintf(stdout, "Sst file format: plain table\n"); -+ fprintf(out_, "Sst file format: plain table\n"); - } - } else { - char error_msg_buffer[80]; -@@ -427,15 +430,56 @@ Status SstFileDumper::SetOldTableOptions() { - assert(table_properties_ == nullptr); - options_.table_factory = std::make_shared(); - if (!silent_) { -- fprintf(stdout, "Sst file format: block-based(old version)\n"); -+ fprintf(out_, "Sst file format: block-based(old version)\n"); - } - - return Status::OK(); - } - -+void write(int value, FILE* file) { -+ char b[4]; -+ b[3] = value & 0x000000ff; -+ b[2] = (value & 0x0000ff00) >> 8; -+ b[1] = (value & 0x00ff0000) >> 16; -+ b[0] = (value & 0xff000000) >> 24; -+ std::fwrite(b, 4, 1, file); + Dummy + +diff --git a/include/rocksdb/raw_iterator.h b/include/rocksdb/raw_iterator.h +new file mode 100644 +index 000000000..ec3c05d6d +--- /dev/null ++++ b/include/rocksdb/raw_iterator.h +@@ -0,0 +1,25 @@ ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// This source code is licensed under both the GPLv2 (found in the ++// COPYING file in the root directory) and Apache 2.0 License ++// (found in the LICENSE.Apache file in the root directory). ++#pragma once ++#ifndef ROCKSDB_LITE ++ ++ ++#include "rocksdb/advanced_options.h" ++namespace ROCKSDB_NAMESPACE { ++ ++class RawIterator { ++ public: ++ virtual ~RawIterator() {} ++ virtual bool has_next() const = 0; ++ virtual Slice getKey() const = 0; ++ virtual Slice getValue() const = 0; ++ virtual uint64_t getSequenceNumber() const = 0; ++ virtual uint32_t getType() const = 0; ++ virtual void next() = 0; ++}; ++ ++} // namespace ROCKSDB_NAMESPACE ++ ++#endif // ROCKSDB_LITE +diff --git a/include/rocksdb/raw_sst_file_reader.h b/include/rocksdb/raw_sst_file_reader.h +new file mode 100644 +index 000000000..266a0c1c8 +--- /dev/null ++++ b/include/rocksdb/raw_sst_file_reader.h +@@ -0,0 +1,62 @@ ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// This source code is licensed under both the GPLv2 (found in the ++// COPYING file in the root directory) and Apache 2.0 License ++// (found in the LICENSE.Apache file in the root directory). ++#pragma once ++#ifndef ROCKSDB_LITE ++ ++#include ++#include ++ ++#include "rocksdb/raw_iterator.h" ++#include "rocksdb/advanced_options.h" ++#include "rocksdb/options.h" ++ ++ ++ ++namespace ROCKSDB_NAMESPACE { ++ ++class RawSstFileReader { ++ public: ++ ++ RawSstFileReader(const Options& options, const std::string& file_name, ++ size_t readahead_size, bool verify_checksum, ++ bool silent = false); ++ ~RawSstFileReader(); ++ ++ RawIterator* newIterator(bool has_from, Slice* from, ++ bool has_to, Slice *to); ++ Status getStatus() { return init_result_; } ++ ++ private: ++ // Get the TableReader implementation for the sst file ++ Status GetTableReader(const std::string& file_path); ++ Status ReadTableProperties(uint64_t table_magic_number, ++ uint64_t file_size); ++ ++ Status SetTableOptionsByMagicNumber(uint64_t table_magic_number); ++ Status SetOldTableOptions(); ++ ++ // Helper function to call the factory with settings specific to the ++ // factory implementation ++ Status NewTableReader(uint64_t file_size); ++ ++ std::string file_name_; ++ Temperature file_temp_; ++ ++ // less verbose in stdout/stderr ++ bool silent_; ++ ++ // options_ and internal_comparator_ will also be used in ++ // ReadSequential internally (specifically, seek-related operations) ++ Options options_; ++ ++ Status init_result_; ++ ++ struct Rep; ++ std::unique_ptr rep_; ++}; ++ ++} // namespace ROCKSDB_NAMESPACE ++ ++#endif // ROCKSDB_LITE +diff --git a/src.mk b/src.mk +index b94bc43ca..95bfff274 100644 +--- a/src.mk ++++ b/src.mk +@@ -343,6 +343,8 @@ TOOL_LIB_SOURCES = \ + tools/ldb_tool.cc \ + tools/sst_dump_tool.cc \ + utilities/blob_db/blob_dump_tool.cc \ ++ tools/raw_sst_file_reader.cc \ ++ tools/raw_sst_file_reader_iterator.cc \ + + ANALYZER_LIB_SOURCES = \ + tools/block_cache_analyzer/block_cache_trace_analyzer.cc \ +diff --git a/tools/raw_sst_file_reader.cc b/tools/raw_sst_file_reader.cc +new file mode 100644 +index 000000000..1693bd1e6 +--- /dev/null ++++ b/tools/raw_sst_file_reader.cc +@@ -0,0 +1,285 @@ ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// This source code is licensed under both the GPLv2 (found in the ++// COPYING file in the root directory) and Apache 2.0 License ++// (found in the LICENSE.Apache file in the root directory). ++// ++#ifndef ROCKSDB_LITE ++ ++#include "rocksdb/raw_sst_file_reader.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "db/blob/blob_index.h" ++#include "db/memtable.h" ++#include "db/write_batch_internal.h" ++#include "options/cf_options.h" ++#include "port/port.h" ++#include "rocksdb/db.h" ++#include "rocksdb/env.h" ++#include "rocksdb/iterator.h" ++#include "rocksdb/slice_transform.h" ++#include "rocksdb/status.h" ++#include "rocksdb/table_properties.h" ++#include "rocksdb/utilities/ldb_cmd.h" ++#include "table/block_based/block.h" ++#include "table/block_based/block_based_table_builder.h" ++#include "table/block_based/block_based_table_factory.h" ++#include "table/block_based/block_builder.h" ++#include "table/format.h" ++#include "table/meta_blocks.h" ++#include "table/plain/plain_table_factory.h" ++#include "table/table_reader.h" ++#include "tools/raw_sst_file_reader_iterator.h" ++#include "util/compression.h" ++#include "util/random.h" ++#include "db/dbformat.h" ++#include "file/writable_file_writer.h" ++#include "options/cf_options.h" ++ ++namespace ROCKSDB_NAMESPACE { ++ ++struct RawSstFileReader::Rep { ++ Options options; ++ EnvOptions soptions_; ++ ReadOptions read_options_; ++ ImmutableOptions ioptions_; ++ MutableCFOptions moptions_; ++ InternalKeyComparator internal_comparator_; ++ std::unique_ptr table_properties_; ++ std::unique_ptr table_reader_; ++ std::unique_ptr file_; ++ ++ Rep(const Options& opts, bool verify_checksum, size_t readahead_size) ++ : options(opts), ++ soptions_(EnvOptions()), ++ read_options_(verify_checksum, false), ++ ioptions_(options), ++ moptions_(ColumnFamilyOptions(options)), ++ internal_comparator_(InternalKeyComparator(BytewiseComparator())) { ++ read_options_.readahead_size = readahead_size; ++ } ++}; ++ ++RawSstFileReader::RawSstFileReader(const Options& options, ++ const std::string& file_name, ++ size_t readahead_size, ++ bool verify_checksum, ++ bool silent) :rep_(new Rep(options, ++ verify_checksum, ++ readahead_size)) { ++ file_name_ = file_name; ++ silent_ = silent; ++ options_ = options; ++ file_temp_ = Temperature::kUnknown; ++ init_result_ = GetTableReader(file_name_); +} + -+void write(const char* value, int length, FILE* file) { -+ write(length, file); -+ fwrite(value, length, 1, file); ++RawSstFileReader::~RawSstFileReader() {} ++ ++ ++ ++extern const uint64_t kBlockBasedTableMagicNumber; ++extern const uint64_t kLegacyBlockBasedTableMagicNumber; ++extern const uint64_t kPlainTableMagicNumber; ++extern const uint64_t kLegacyPlainTableMagicNumber; ++ ++Status RawSstFileReader::GetTableReader(const std::string& file_path) { ++ // Warning about 'magic_number' being uninitialized shows up only in UBsan ++ // builds. Though access is guarded by 's.ok()' checks, fix the issue to ++ // avoid any warnings. ++ uint64_t magic_number = Footer::kNullTableMagicNumber; ++ ++ // read table magic number ++ Footer footer; ++ ++ const auto& fs = options_.env->GetFileSystem(); ++ std::unique_ptr file; ++ uint64_t file_size = 0; ++ FileOptions fopts = rep_->soptions_; ++ fopts.temperature = file_temp_; ++ Status s = fs->NewRandomAccessFile(file_path, fopts, &file, nullptr); ++ if (s.ok()) { ++ s = fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr); ++ } ++ ++ // check empty file ++ // if true, skip further processing of this file ++ if (file_size == 0) { ++ return Status::Aborted(file_path, "Empty file"); ++ } ++ ++ rep_->file_.reset(new RandomAccessFileReader(std::move(file), file_path)); ++ ++ FilePrefetchBuffer prefetch_buffer( ++ 0 /* readahead_size */, 0 /* max_readahead_size */, true /* enable */, ++ false /* track_min_offset */); ++ if (s.ok()) { ++ const uint64_t kSstDumpTailPrefetchSize = 512 * 1024; ++ uint64_t prefetch_size = (file_size > kSstDumpTailPrefetchSize) ++ ? kSstDumpTailPrefetchSize ++ : file_size; ++ uint64_t prefetch_off = file_size - prefetch_size; ++ IOOptions opts; ++ s = prefetch_buffer.Prefetch(opts, rep_->file_.get(), prefetch_off, ++ static_cast(prefetch_size), ++ Env::IO_TOTAL /* rate_limiter_priority */); ++ ++ s = ReadFooterFromFile(opts, rep_->file_.get(), &prefetch_buffer, file_size, ++ &footer); ++ } ++ if (s.ok()) { ++ magic_number = footer.table_magic_number(); ++ } ++ ++ if (s.ok()) { ++ if (magic_number == kPlainTableMagicNumber || ++ magic_number == kLegacyPlainTableMagicNumber) { ++ rep_->soptions_.use_mmap_reads = true; ++ ++ fs->NewRandomAccessFile(file_path, fopts, &file, nullptr); ++ rep_->file_.reset(new RandomAccessFileReader(std::move(file), file_path)); ++ } ++ ++ s = ROCKSDB_NAMESPACE::ReadTableProperties( ++ rep_->file_.get(), file_size, magic_number, rep_->ioptions_, &(rep_->table_properties_), ++ /* memory_allocator= */ nullptr, (magic_number == kBlockBasedTableMagicNumber) ++ ? &prefetch_buffer ++ : nullptr); ++ if (!s.ok()) { ++ if (!silent_) { ++ fprintf(stderr, "Not able to read table properties\n"); ++ } ++ } ++ // For old sst format, ReadTableProperties might fail but file can be read ++ if (s.ok()) { ++ s = SetTableOptionsByMagicNumber(magic_number); ++ if (s.ok()) { ++ if (rep_->table_properties_ && !rep_->table_properties_->comparator_name.empty()) { ++ ConfigOptions config_options; ++ const Comparator* user_comparator = nullptr; ++ s = Comparator::CreateFromString(config_options, ++ rep_->table_properties_->comparator_name, ++ &user_comparator); ++ if (s.ok()) { ++ assert(user_comparator); ++ rep_->internal_comparator_ = InternalKeyComparator(user_comparator); ++ } ++ } ++ } ++ } else { ++ s = SetOldTableOptions(); ++ } ++ options_.comparator = rep_->internal_comparator_.user_comparator(); ++ } ++ ++ if (s.ok()) { ++ s = NewTableReader(file_size); ++ } ++ return s; ++} ++ ++Status RawSstFileReader::NewTableReader(uint64_t file_size) { ++ auto t_opt = ++ TableReaderOptions(rep_->ioptions_, rep_->moptions_.prefix_extractor, rep_->soptions_, ++ rep_->internal_comparator_, false /* skip_filters */, ++ false /* imortal */, true /* force_direct_prefetch */); ++ // Allow open file with global sequence number for backward compatibility. ++ t_opt.largest_seqno = kMaxSequenceNumber; ++ ++ // We need to turn off pre-fetching of index and filter nodes for ++ // BlockBasedTable ++ if (options_.table_factory->IsInstanceOf( ++ TableFactory::kBlockBasedTableName())) { ++ return options_.table_factory->NewTableReader(t_opt, std::move(rep_->file_), ++ file_size, &(rep_->table_reader_), ++ /*enable_prefetch=*/false); ++ } ++ ++ // For all other factory implementation ++ return options_.table_factory->NewTableReader(t_opt, std::move(rep_->file_), ++ file_size, &(rep_->table_reader_)); ++} ++ ++Status RawSstFileReader::SetTableOptionsByMagicNumber( ++ uint64_t table_magic_number) { ++ assert(rep_->table_properties_); ++ if (table_magic_number == kBlockBasedTableMagicNumber || ++ table_magic_number == kLegacyBlockBasedTableMagicNumber) { ++ BlockBasedTableFactory* bbtf = new BlockBasedTableFactory(); ++ // To force tail prefetching, we fake reporting two useful reads of 512KB ++ // from the tail. ++ // It needs at least two data points to warm up the stats. ++ bbtf->tail_prefetch_stats()->RecordEffectiveSize(512 * 1024); ++ bbtf->tail_prefetch_stats()->RecordEffectiveSize(512 * 1024); ++ ++ options_.table_factory.reset(bbtf); ++ if (!silent_) { ++ fprintf(stdout, "Sst file format: block-based\n"); ++ } ++ ++ auto& props = rep_->table_properties_->user_collected_properties; ++ auto pos = props.find(BlockBasedTablePropertyNames::kIndexType); ++ if (pos != props.end()) { ++ auto index_type_on_file = static_cast( ++ DecodeFixed32(pos->second.c_str())); ++ if (index_type_on_file == ++ BlockBasedTableOptions::IndexType::kHashSearch) { ++ options_.prefix_extractor.reset(NewNoopTransform()); ++ } ++ } ++ } else if (table_magic_number == kPlainTableMagicNumber || ++ table_magic_number == kLegacyPlainTableMagicNumber) { ++ options_.allow_mmap_reads = true; ++ ++ PlainTableOptions plain_table_options; ++ plain_table_options.user_key_len = kPlainTableVariableLength; ++ plain_table_options.bloom_bits_per_key = 0; ++ plain_table_options.hash_table_ratio = 0; ++ plain_table_options.index_sparseness = 1; ++ plain_table_options.huge_page_tlb_size = 0; ++ plain_table_options.encoding_type = kPlain; ++ plain_table_options.full_scan_mode = true; ++ ++ options_.table_factory.reset(NewPlainTableFactory(plain_table_options)); ++ if (!silent_) { ++ fprintf(stdout, "Sst file format: plain table\n"); ++ } ++ } else { ++ char error_msg_buffer[80]; ++ snprintf(error_msg_buffer, sizeof(error_msg_buffer) - 1, ++ "Unsupported table magic number --- %lx", ++ (long)table_magic_number); ++ return Status::InvalidArgument(error_msg_buffer); ++ } ++ ++ return Status::OK(); ++} ++ ++Status RawSstFileReader::SetOldTableOptions() { ++ assert(rep_->table_properties_ == nullptr); ++ options_.table_factory = std::make_shared(); ++ if (!silent_) { ++ fprintf(stdout, "Sst file format: block-based(old version)\n"); ++ } ++ ++ return Status::OK(); +} + -+void write(const std::string& value, FILE* file) { -+ write(value.data(), (int)value.length(), file); ++RawIterator* RawSstFileReader::newIterator( ++ bool has_from, Slice* from, bool has_to, Slice* to) { ++ InternalIterator* iter = rep_->table_reader_->NewIterator( ++ rep_->read_options_, rep_->moptions_.prefix_extractor.get(), ++ /*arena=*/nullptr, /*skip_filters=*/false, ++ TableReaderCaller::kSSTDumpTool); ++ return new RawSstFileReaderIterator(iter, has_from, from, has_to, to); ++ +} ++} // namespace ROCKSDB_NAMESPACE ++ ++#endif // ROCKSDB_LITE +diff --git a/tools/raw_sst_file_reader_iterator.cc b/tools/raw_sst_file_reader_iterator.cc +new file mode 100644 +index 000000000..b31596c4a +--- /dev/null ++++ b/tools/raw_sst_file_reader_iterator.cc +@@ -0,0 +1,94 @@ ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// This source code is licensed under both the GPLv2 (found in the ++// COPYING file in the root directory) and Apache 2.0 License ++// (found in the LICENSE.Apache file in the root directory). ++// ++#ifndef ROCKSDB_LITE ++ ++#include "tools/raw_sst_file_reader_iterator.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "db/blob/blob_index.h" ++#include "db/memtable.h" ++#include "db/write_batch_internal.h" ++#include "options/cf_options.h" ++#include "port/port.h" ++#include "rocksdb/db.h" ++#include "rocksdb/env.h" ++#include "rocksdb/iterator.h" ++#include "rocksdb/slice_transform.h" ++#include "rocksdb/status.h" ++#include "rocksdb/table_properties.h" ++#include "rocksdb/utilities/ldb_cmd.h" ++#include "table/block_based/block.h" ++#include "table/block_based/block_based_table_builder.h" ++#include "table/block_based/block_based_table_factory.h" ++#include "table/block_based/block_builder.h" ++#include "table/format.h" ++#include "table/meta_blocks.h" ++#include "table/plain/plain_table_factory.h" ++#include "table/table_reader.h" ++#include "util/compression.h" ++#include "util/random.h" ++ ++namespace ROCKSDB_NAMESPACE { + -+void write(Slice &slice, FILE* file) { -+ int size = (int)slice.size(); -+ write(slice.data(), size, file); ++RawSstFileReaderIterator::RawSstFileReaderIterator(InternalIterator* iterator, ++ bool has_from, Slice* from_key, ++ bool has_to, Slice* to_key) ++ : iter_(iterator), ++ ikey(new ParsedInternalKey()), ++ has_to(has_to), ++ to_key(to_key) { ++ if (has_from) { ++ InternalKey k; ++ k.SetMinPossibleForUserKey(*from_key); ++ iter_->Seek(k.Encode()); ++ } else { ++ iter_->SeekToFirst(); ++ } ++ initKey(); +} + -+void write(SequenceNumber sequenceNumber, FILE* file) { ++bool RawSstFileReaderIterator::has_next() const { ++ return iter_->Valid() && (!has_to || ++ BytewiseComparator()->Compare( ++ getKey(), *to_key) < 0); ++} + -+ char b[8]; -+ int idx = 7; -+ while (idx >= 0) { -+ b[idx] = sequenceNumber % 256; -+ sequenceNumber /= 256; -+ idx -= 1; ++void RawSstFileReaderIterator::initKey() { ++ if (iter_->Valid()) { ++ ParseInternalKey(iter_->key(), ikey, true /* log_err_key */); + } -+ fwrite(b, 8, 1, file); ++} ++void RawSstFileReaderIterator::next() { ++ iter_->Next(); ++ initKey(); ++ ++} ++ ++Slice RawSstFileReaderIterator::getKey() const { ++ return ikey->user_key; +} + -+void write(ParsedInternalKey &key, FILE* file) { -+ write(key.user_key, file); -+ write(key.sequence, file); -+ write(static_cast(key.type), file); ++uint64_t RawSstFileReaderIterator::getSequenceNumber() const { ++ return ikey->sequence; +} + - Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, -- bool has_from, const std::string& from_key, -- bool has_to, const std::string& to_key, -+ bool has_from, const Slice& from_key, -+ bool has_to, const Slice& to_key, - bool use_from_as_prefix) { - if (!table_reader_) { - return init_result_; -@@ -446,6 +490,7 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, - /*arena=*/nullptr, /*skip_filters=*/false, - TableReaderCaller::kSSTDumpTool); - uint64_t i = 0; -+ - if (has_from) { - InternalKey ikey; - ikey.SetMinPossibleForUserKey(from_key); -@@ -453,6 +498,7 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, - } else { - iter->SeekToFirst(); - } -+ - for (; iter->Valid(); iter->Next()) { - Slice key = iter->key(); - Slice value = iter->value(); -@@ -478,22 +524,19 @@ Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num, - - if (print_kv) { - if (!decode_blob_index_ || ikey.type != kTypeBlobIndex) { -- fprintf(stdout, "%s => %s\n", -- ikey.DebugString(true, output_hex_).c_str(), -- value.ToString(output_hex_).c_str()); -+ write(ikey, out_); -+ write(value, out_); - } else { - BlobIndex blob_index; -- - const Status s = blob_index.DecodeFrom(value); - if (!s.ok()) { -- fprintf(stderr, "%s => error decoding blob index\n", -- ikey.DebugString(true, output_hex_).c_str()); -+ write(ikey, err_); -+ write("error decoding blob index", err_); - continue; - } -- -- fprintf(stdout, "%s => %s\n", -- ikey.DebugString(true, output_hex_).c_str(), -- blob_index.DebugString(output_hex_).c_str()); -+ write(ikey, out_); -+ std::string v = blob_index.DebugString(output_hex_); -+ write(v, out_); - } - } - } -diff --git a/table/sst_file_dumper.h b/table/sst_file_dumper.h -index 7be876390..768c5b1e2 100644 ---- a/table/sst_file_dumper.h -+++ b/table/sst_file_dumper.h -@@ -22,11 +22,13 @@ class SstFileDumper { - bool verify_checksum, bool output_hex, - bool decode_blob_index, - const EnvOptions& soptions = EnvOptions(), -- bool silent = false); -+ bool silent = false, -+ FILE* out = stdout, -+ FILE* err = stderr); - - Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from, -- const std::string& from_key, bool has_to, -- const std::string& to_key, -+ const Slice& from_key, bool has_to, -+ const Slice& to_key, - bool use_from_as_prefix = false); - - Status ReadTableProperties( -@@ -94,6 +96,8 @@ class SstFileDumper { - ReadOptions read_options_; - InternalKeyComparator internal_comparator_; - std::unique_ptr table_properties_; -+ FILE* out_; -+ FILE* err_; - }; - - } // namespace ROCKSDB_NAMESPACE -diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc -index 7053366e7..8f248ddf3 100644 ---- a/tools/sst_dump_tool.cc -+++ b/tools/sst_dump_tool.cc -@@ -31,7 +31,7 @@ static const std::vector> - - namespace { - --void print_help(bool to_stderr) { -+void print_help(bool to_stderr, FILE* err_, FILE* out_) { - std::string supported_compressions; - for (CompressionType ct : GetSupportedCompressions()) { - if (!supported_compressions.empty()) { -@@ -43,7 +43,7 @@ void print_help(bool to_stderr) { - supported_compressions += str; - } - fprintf( -- to_stderr ? stderr : stdout, -+ to_stderr ? err_ : out_, - R"(sst_dump --file= [--command=check|scan|raw|recompress|identify] - --file= - Path to SST file or directory containing SST files -@@ -149,7 +149,13 @@ bool ParseIntArg(const char* arg, const std::string arg_name, - } - } // namespace - --int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { -+Slice* AssignSlicePrependedWithLength(const char* buf) { -+ long val = std::stol(buf); -+ return reinterpret_cast(val); ++uint32_t RawSstFileReaderIterator::getType() const { ++ return static_cast(ikey->type); +} + -+int SSTDumpTool::Run(int argc, char const* const* argv, Options options, -+ FILE* out, FILE* err) { - std::string env_uri, fs_uri; - const char* dir_or_file = nullptr; - uint64_t read_num = std::numeric_limits::max(); -@@ -170,8 +176,9 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - bool has_compression_level_from = false; - bool has_compression_level_to = false; - bool has_specified_compression_types = false; -- std::string from_key; -- std::string to_key; -+ bool silent = false; -+ Slice* from_key = nullptr; -+ Slice* to_key = nullptr; - std::string block_size_str; - std::string compression_level_from_str; - std::string compression_level_to_str; -@@ -197,7 +204,9 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - int64_t tmp_val; - - for (int i = 1; i < argc; i++) { -- if (strncmp(argv[i], "--env_uri=", 10) == 0) { -+ if (strncmp(argv[i], "--silent", 8) == 0) { -+ silent = true; -+ } else if (strncmp(argv[i], "--env_uri=", 10) == 0) { - env_uri = argv[i] + 10; - } else if (strncmp(argv[i], "--fs_uri=", 9) == 0) { - fs_uri = argv[i] + 9; -@@ -217,13 +226,13 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - } else if (strncmp(argv[i], "--command=", 10) == 0) { - command = argv[i] + 10; - } else if (strncmp(argv[i], "--from=", 7) == 0) { -- from_key = argv[i] + 7; -+ from_key = AssignSlicePrependedWithLength(argv[i] + 7); - has_from = true; - } else if (strncmp(argv[i], "--to=", 5) == 0) { -- to_key = argv[i] + 5; -+ to_key = AssignSlicePrependedWithLength(argv[i] + 5); - has_to = true; - } else if (strncmp(argv[i], "--prefix=", 9) == 0) { -- from_key = argv[i] + 9; -+ from_key = AssignSlicePrependedWithLength( argv[i] + 9); - use_from_as_prefix = true; - } else if (strcmp(argv[i], "--show_properties") == 0) { - show_properties = true; -@@ -273,7 +282,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - std::cerr << pik_status.getState() << "\n"; - retc = -1; - } -- fprintf(stdout, "key=%s\n", ikey.DebugString(true, true).c_str()); -+ fprintf(out, "key=%s\n", ikey.DebugString(true, true).c_str()); - return retc; - } else if (ParseIntArg(argv[i], "--compression_level_from=", - "compression_level_from must be numeric", -@@ -288,9 +297,9 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - "compression_max_dict_bytes must be numeric", - &tmp_val)) { - if (tmp_val < 0 || tmp_val > std::numeric_limits::max()) { -- fprintf(stderr, "compression_max_dict_bytes must be a uint32_t: '%s'\n", -+ fprintf(err, "compression_max_dict_bytes must be a uint32_t: '%s'\n", - argv[i]); -- print_help(/*to_stderr*/ true); -+ print_help(/*to_stderr*/ true, err, out); - return 1; - } - compression_max_dict_bytes = static_cast(tmp_val); -@@ -298,10 +307,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - "compression_zstd_max_train_bytes must be numeric", - &tmp_val)) { - if (tmp_val < 0 || tmp_val > std::numeric_limits::max()) { -- fprintf(stderr, -+ fprintf(err, - "compression_zstd_max_train_bytes must be a uint32_t: '%s'\n", - argv[i]); -- print_help(/*to_stderr*/ true); -+ print_help(/*to_stderr*/ true, err, out); - return 1; - } - compression_zstd_max_train_bytes = static_cast(tmp_val); -@@ -309,56 +318,56 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - "compression_max_dict_buffer_bytes must be numeric", - &tmp_val)) { - if (tmp_val < 0) { -- fprintf(stderr, -+ fprintf(err, - "compression_max_dict_buffer_bytes must be positive: '%s'\n", - argv[i]); -- print_help(/*to_stderr*/ true); -+ print_help(/*to_stderr*/ true, err, out); - return 1; - } - compression_max_dict_buffer_bytes = static_cast(tmp_val); - } else if (strcmp(argv[i], "--compression_use_zstd_finalize_dict") == 0) { - compression_use_zstd_finalize_dict = true; - } else if (strcmp(argv[i], "--help") == 0) { -- print_help(/*to_stderr*/ false); -+ print_help(/*to_stderr*/ false, err, out); - return 0; - } else if (strcmp(argv[i], "--version") == 0) { - printf("%s\n", GetRocksBuildInfoAsString("sst_dump").c_str()); - return 0; - } else { -- fprintf(stderr, "Unrecognized argument '%s'\n\n", argv[i]); -- print_help(/*to_stderr*/ true); -+ fprintf(err, "Unrecognized argument '%s'\n\n", argv[i]); -+ print_help(/*to_stderr*/ true, err, out); - return 1; - } - } - - if(has_compression_level_from && has_compression_level_to) { - if(!has_specified_compression_types || compression_types.size() != 1) { -- fprintf(stderr, "Specify one compression type.\n\n"); -+ fprintf(err, "Specify one compression type.\n\n"); - exit(1); - } - } else if(has_compression_level_from || has_compression_level_to) { -- fprintf(stderr, "Specify both --compression_level_from and " -+ fprintf(err, "Specify both --compression_level_from and " - "--compression_level_to.\n\n"); - exit(1); - } - - if (use_from_as_prefix && has_from) { -- fprintf(stderr, "Cannot specify --prefix and --from\n\n"); -+ fprintf(err, "Cannot specify --prefix and --from\n\n"); - exit(1); - } - - if (input_key_hex) { - if (has_from || use_from_as_prefix) { -- from_key = ROCKSDB_NAMESPACE::LDBCommand::HexToString(from_key); -+ *from_key = ROCKSDB_NAMESPACE::LDBCommand::HexToString(from_key -> ToString()); - } - if (has_to) { -- to_key = ROCKSDB_NAMESPACE::LDBCommand::HexToString(to_key); -+ *to_key = ROCKSDB_NAMESPACE::LDBCommand::HexToString(to_key->ToString()); - } - } - - if (dir_or_file == nullptr) { -- fprintf(stderr, "file or directory must be specified.\n\n"); -- print_help(/*to_stderr*/ true); -+ fprintf(err, "file or directory must be specified.\n\n"); -+ print_help(/*to_stderr*/ true, err, out); - exit(1); - } - -@@ -373,10 +382,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - Status s = Env::CreateFromUri(config_options, env_uri, fs_uri, &options.env, - &env_guard); - if (!s.ok()) { -- fprintf(stderr, "CreateEnvFromUri: %s\n", s.ToString().c_str()); -+ fprintf(err, "CreateEnvFromUri: %s\n", s.ToString().c_str()); - exit(1); -- } else { -- fprintf(stdout, "options.env is %p\n", options.env); -+ } else if (!silent){ -+ fprintf(out, "options.env is %p\n", options.env); - } - } - -@@ -390,7 +399,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - Status s = env->FileExists(dir_or_file); - // dir_or_file does not exist - if (!s.ok()) { -- fprintf(stderr, "%s%s: No such file or directory\n", s.ToString().c_str(), -+ fprintf(err, "%s%s: No such file or directory\n", s.ToString().c_str(), - dir_or_file); - return 1; - } -@@ -421,10 +430,11 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - - ROCKSDB_NAMESPACE::SstFileDumper dumper( - options, filename, Temperature::kUnknown, readahead_size, -- verify_checksum, output_hex, decode_blob_index); -+ verify_checksum, output_hex, decode_blob_index, EnvOptions(), -+ silent, out, err); - // Not a valid SST - if (!dumper.getStatus().ok()) { -- fprintf(stderr, "%s: %s\n", filename.c_str(), -+ fprintf(err, "%s: %s\n", filename.c_str(), - dumper.getStatus().ToString().c_str()); - continue; - } else { -@@ -433,10 +443,11 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - // where there is at least one valid SST - if (valid_sst_files.size() == 1) { - // from_key and to_key are only used for "check", "scan", or "" -- if (command == "check" || command == "scan" || command == "") { -- fprintf(stdout, "from [%s] to [%s]\n", -- ROCKSDB_NAMESPACE::Slice(from_key).ToString(true).c_str(), -- ROCKSDB_NAMESPACE::Slice(to_key).ToString(true).c_str()); -+ if (!silent && (command == "check" || command == "scan" || -+ command == "")) { -+ fprintf(out, "from [%s] to [%s]\n", -+ from_key->ToString(true).c_str(), -+ to_key->ToString(true).c_str()); - } - } - } -@@ -449,7 +460,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - compression_zstd_max_train_bytes, compression_max_dict_buffer_bytes, - !compression_use_zstd_finalize_dict); - if (!st.ok()) { -- fprintf(stderr, "Failed to recompress: %s\n", st.ToString().c_str()); -+ fprintf(err, "Failed to recompress: %s\n", st.ToString().c_str()); - exit(1); - } - return 0; -@@ -461,10 +472,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - - st = dumper.DumpTable(out_filename); - if (!st.ok()) { -- fprintf(stderr, "%s: %s\n", filename.c_str(), st.ToString().c_str()); -+ fprintf(err, "%s: %s\n", filename.c_str(), st.ToString().c_str()); - exit(1); - } else { -- fprintf(stdout, "raw dump written to file %s\n", &out_filename[0]); -+ fprintf(out, "raw dump written to file %s\n", &out_filename[0]); - } - continue; - } -@@ -473,10 +484,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - if (command == "" || command == "scan" || command == "check") { - st = dumper.ReadSequential( - command == "scan", read_num > 0 ? (read_num - total_read) : read_num, -- has_from || use_from_as_prefix, from_key, has_to, to_key, -+ has_from || use_from_as_prefix, *from_key, has_to, *to_key, - use_from_as_prefix); - if (!st.ok()) { -- fprintf(stderr, "%s: %s\n", filename.c_str(), -+ fprintf(err, "%s: %s\n", filename.c_str(), - st.ToString().c_str()); - } - total_read += dumper.GetReadNumber(); -@@ -488,10 +499,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - if (command == "verify") { - st = dumper.VerifyChecksum(); - if (!st.ok()) { -- fprintf(stderr, "%s is corrupted: %s\n", filename.c_str(), -+ fprintf(err, "%s is corrupted: %s\n", filename.c_str(), - st.ToString().c_str()); - } else { -- fprintf(stdout, "The file is ok\n"); -+ fprintf(out, "The file is ok\n"); - } - continue; - } -@@ -503,15 +514,15 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - table_properties_from_reader; - st = dumper.ReadTableProperties(&table_properties_from_reader); - if (!st.ok()) { -- fprintf(stderr, "%s: %s\n", filename.c_str(), st.ToString().c_str()); -- fprintf(stderr, "Try to use initial table properties\n"); -+ fprintf(err, "%s: %s\n", filename.c_str(), st.ToString().c_str()); -+ fprintf(err, "Try to use initial table properties\n"); - table_properties = dumper.GetInitTableProperties(); - } else { - table_properties = table_properties_from_reader.get(); - } - if (table_properties != nullptr) { - if (show_properties) { -- fprintf(stdout, -+ fprintf(out, - "Table Properties:\n" - "------------------------------\n" - " %s", -@@ -523,18 +534,18 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - total_index_block_size += table_properties->index_size; - total_filter_block_size += table_properties->filter_size; - if (show_properties) { -- fprintf(stdout, -+ fprintf(out, - "Raw user collected properties\n" - "------------------------------\n"); - for (const auto& kv : table_properties->user_collected_properties) { - std::string prop_name = kv.first; - std::string prop_val = Slice(kv.second).ToString(true); -- fprintf(stdout, " # %s: 0x%s\n", prop_name.c_str(), -+ fprintf(out, " # %s: 0x%s\n", prop_name.c_str(), - prop_val.c_str()); - } - } - } else { -- fprintf(stderr, "Reader unexpectedly returned null properties\n"); -+ fprintf(err, "Reader unexpectedly returned null properties\n"); - } - } - } -@@ -555,9 +566,9 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { - // Exit with an error state - if (dir) { - fprintf(stdout, "------------------------------\n"); -- fprintf(stderr, "No valid SST files found in %s\n", dir_or_file); -+ fprintf(err, "No valid SST files found in %s\n", dir_or_file); - } else { -- fprintf(stderr, "%s is not a valid SST file\n", dir_or_file); -+ fprintf(err, "%s is not a valid SST file\n", dir_or_file); - } - return 1; - } else { ++Slice RawSstFileReaderIterator::getValue() const { ++ return iter_->value(); ++} ++} // namespace ROCKSDB_NAMESPACE ++ ++#endif // ROCKSDB_LITE +diff --git a/tools/raw_sst_file_reader_iterator.h b/tools/raw_sst_file_reader_iterator.h +new file mode 100644 +index 000000000..8c3c4acf2 +--- /dev/null ++++ b/tools/raw_sst_file_reader_iterator.h +@@ -0,0 +1,45 @@ ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// This source code is licensed under both the GPLv2 (found in the ++// COPYING file in the root directory) and Apache 2.0 License ++// (found in the LICENSE.Apache file in the root directory). ++#pragma once ++#ifndef ROCKSDB_LITE ++ ++#include ++#include ++#include "file/writable_file_writer.h" ++#include "rocksdb/advanced_options.h" ++#include "rocksdb/raw_iterator.h" ++ ++namespace ROCKSDB_NAMESPACE { ++ ++class RawSstFileReaderIterator : public RawIterator { ++ public: ++ explicit RawSstFileReaderIterator(InternalIterator* iterator, ++ bool has_from, ++ Slice* from_key, ++ bool has_to, ++ Slice* to_key); ++ ++ bool has_next() const override; ++ Slice getKey() const override; ++ Slice getValue() const override; ++ uint64_t getSequenceNumber() const override; ++ uint32_t getType() const override; ++ void next() final override; ++ ++ ~RawSstFileReaderIterator(){ ++ delete iter_; ++ } ++ ++ private: ++ void initKey(); ++ InternalIterator* iter_; ++ ParsedInternalKey* ikey; ++ bool has_to; ++ Slice* to_key; ++}; ++ ++} // namespace ROCKSDB_NAMESPACE ++ ++#endif // ROCKSDB_LITE diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java new file mode 100644 index 000000000000..e8b5b04eb5f4 --- /dev/null +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.utils.db.managed; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.StringUtils; +import org.apache.hadoop.hdds.utils.NativeLibraryLoader; +import org.apache.hadoop.hdds.utils.TestUtils; +import org.apache.ozone.test.tag.Native; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Test for ManagedRawSSTFileReaderIterator. + */ +class TestManagedRawSSTFileReaderIterator { + + @TempDir + private Path tempDir; + + private File createSSTFileWithKeys( + TreeMap, String> keys) throws Exception { + File file = Files.createFile(tempDir.resolve("tmp_sst_file.sst")).toFile(); + try (ManagedEnvOptions envOptions = new ManagedEnvOptions(); + ManagedOptions managedOptions = new ManagedOptions(); + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter( + envOptions, managedOptions)) { + sstFileWriter.open(file.getAbsolutePath()); + for (Map.Entry, String> entry : keys.entrySet()) { + if (entry.getKey().getValue() == 0) { + sstFileWriter.delete(entry.getKey().getKey() + .getBytes(StandardCharsets.UTF_8)); + } else { + sstFileWriter.put(entry.getKey().getKey() + .getBytes(StandardCharsets.UTF_8), + entry.getValue().getBytes(StandardCharsets.UTF_8)); + } + } + sstFileWriter.finish(); + } + return file; + } + + private static Stream keyValueFormatArgs() { + return Stream.of( + Arguments.of( + Named.of("Key starting with a single quote", + "'key%1$d=>"), + Named.of("Value starting with a number ending with a" + + " single quote", "%1$dvalue'") + ), + Arguments.of( + Named.of("Key ending with a number", "key%1$d"), + Named.of("Value starting & ending with a number", "%1$dvalue%1$d") + ), + Arguments.of( + Named.of("Key starting with a single quote & ending" + + " with a number", "'key%1$d"), + Named.of("Value starting & ending with a number " + + "& elosed within quotes", "%1$d'value%1$d'")), + Arguments.of( + Named.of("Key starting with a single quote & ending" + + " with a number", "'key%1$d"), + Named.of("Value starting & ending with a number " + + "& elosed within quotes", "%1$d'value%1$d'") + ), + Arguments.of( + Named.of("Key ending with a number", "key%1$d"), + Named.of("Value starting & ending with a number " + + "& containing null character & new line character", + "%1$dvalue\n\0%1$d") + ), + Arguments.of( + Named.of("Key ending with a number & containing" + + " a null character", "key\0%1$d"), + Named.of("Value starting & ending with a number " + + "& elosed within quotes", "%1$dvalue\r%1$d") + ) + ); + } + + @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) + @ParameterizedTest + @MethodSource("keyValueFormatArgs") + public void testSSTDumpIteratorWithKeyFormat(String keyFormat, + String valueFormat) + throws Exception { + ManagedRocksObjectUtils.loadRocksDBLibrary(); + assumeTrue(NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); + + TreeMap, String> keys = + IntStream.range(0, 100).boxed().collect( + Collectors.toMap( + i -> Pair.of(String.format(keyFormat, i), i % 2), + i -> i % 2 == 0 ? "" : String.format(valueFormat, i), + (v1, v2) -> v2, + TreeMap::new)); + File file = createSSTFileWithKeys(keys); + + try (ManagedOptions options = new ManagedOptions(); + ManagedRawSSTFileReader + reader = new ManagedRawSSTFileReader<>(options, + file.getAbsolutePath(), 2 * 1024 * 1024)) { + List> testBounds = TestUtils.getTestingBounds( + keys.keySet().stream().collect(Collectors.toMap(Pair::getKey, + Pair::getValue, (v1, v2) -> v1, TreeMap::new))); + for (Optional keyStart : testBounds) { + for (Optional keyEnd : testBounds) { + Map, String> expectedKeys = keys.entrySet() + .stream().filter(e -> keyStart.map(s -> e.getKey().getKey() + .compareTo(s) >= 0).orElse(true)) + .filter(e -> keyEnd.map(s -> e.getKey().getKey().compareTo(s) < 0) + .orElse(true)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Optional lowerBound = keyStart + .map(s -> new ManagedSlice(StringUtils.string2Bytes(s))); + Optional upperBound = keyEnd + .map(s -> new ManagedSlice(StringUtils.string2Bytes(s))); + try (ManagedRawSSTFileReaderIterator< + ManagedRawSSTFileReaderIterator.KeyValue> iterator + = reader.newIterator(Function.identity(), + lowerBound.orElse(null), upperBound.orElse(null))) { + while (iterator.hasNext()) { + ManagedRawSSTFileReaderIterator.KeyValue r = iterator.next(); + String key = new String(r.getKey(), StandardCharsets.UTF_8); + Pair recordKey = Pair.of(key, r.getType()); + assertThat(expectedKeys).containsKey(recordKey); + assertEquals(Optional.ofNullable(expectedKeys + .get(recordKey)).orElse(""), + new String(r.getValue(), StandardCharsets.UTF_8)); + expectedKeys.remove(recordKey); + } + assertEquals(0, expectedKeys.size()); + } finally { + lowerBound.ifPresent(ManagedSlice::close); + upperBound.ifPresent(ManagedSlice::close); + } + } + } + } + } +} diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedSSTDumpIterator.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedSSTDumpIterator.java deleted file mode 100644 index d2796c19fc50..000000000000 --- a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedSSTDumpIterator.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.utils.db.managed; - -import com.google.common.primitives.Bytes; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.hdds.StringUtils; -import org.apache.hadoop.hdds.utils.NativeLibraryLoader; -import org.apache.hadoop.hdds.utils.TestUtils; -import org.apache.ozone.test.tag.Native; -import org.apache.ozone.test.tag.Unhealthy; -import org.junit.jupiter.api.Named; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assumptions.assumeTrue; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Test for ManagedSSTDumpIterator. - */ -class TestManagedSSTDumpIterator { - - @TempDir - private Path tempDir; - - private File createSSTFileWithKeys( - TreeMap, String> keys) throws Exception { - File file = Files.createFile(tempDir.resolve("tmp_sst_file.sst")).toFile(); - try (ManagedEnvOptions envOptions = new ManagedEnvOptions(); - ManagedOptions managedOptions = new ManagedOptions(); - ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter( - envOptions, managedOptions)) { - sstFileWriter.open(file.getAbsolutePath()); - for (Map.Entry, String> entry : keys.entrySet()) { - if (entry.getKey().getValue() == 0) { - sstFileWriter.delete(entry.getKey().getKey() - .getBytes(StandardCharsets.UTF_8)); - } else { - sstFileWriter.put(entry.getKey().getKey() - .getBytes(StandardCharsets.UTF_8), - entry.getValue().getBytes(StandardCharsets.UTF_8)); - } - } - sstFileWriter.finish(); - } - return file; - } - - private static Stream keyValueFormatArgs() { - return Stream.of( - Arguments.of( - Named.of("Key starting with a single quote", - "'key%1$d=>"), - Named.of("Value starting with a number ending with a" + - " single quote", "%1$dvalue'") - ), - Arguments.of( - Named.of("Key ending with a number", "key%1$d"), - Named.of("Value starting & ending with a number", "%1$dvalue%1$d") - ), - Arguments.of( - Named.of("Key starting with a single quote & ending" + - " with a number", "'key%1$d"), - Named.of("Value starting & ending with a number " + - "& elosed within quotes", "%1$d'value%1$d'")), - Arguments.of( - Named.of("Key starting with a single quote & ending" + - " with a number", "'key%1$d"), - Named.of("Value starting & ending with a number " + - "& elosed within quotes", "%1$d'value%1$d'") - ), - Arguments.of( - Named.of("Key ending with a number", "key%1$d"), - Named.of("Value starting & ending with a number " + - "& containing null character & new line character", - "%1$dvalue\n\0%1$d") - ), - Arguments.of( - Named.of("Key ending with a number & containing" + - " a null character", "key\0%1$d"), - Named.of("Value starting & ending with a number " + - "& elosed within quotes", "%1$dvalue\r%1$d") - ) - ); - } - - private static byte[] getBytes(Integer val) { - ByteBuffer destByteBuffer = ByteBuffer.allocate(4); - destByteBuffer.order(ByteOrder.BIG_ENDIAN); - destByteBuffer.putInt(val); - return destByteBuffer.array(); - } - - private static byte[] getBytes(Long val) { - ByteBuffer destByteBuffer = ByteBuffer.allocate(8); - destByteBuffer.order(ByteOrder.BIG_ENDIAN); - destByteBuffer.putLong(val); - return destByteBuffer.array(); - } - - private static byte[] getBytes(String val) { - byte[] b = new byte[val.length()]; - for (int i = 0; i < val.length(); i++) { - b[i] = (byte) val.charAt(i); - } - return b; - } - - private static Stream invalidPipeInputStreamBytes() { - return Stream.of( - Arguments.of(Named.of("Invalid 3 byte integer", - new byte[]{0, 0, 0})), - Arguments.of(Named.of("Invalid 2 byte integer", - new byte[]{0, 0})), - Arguments.of(Named.of("Invalid 1 byte integer", - new byte[]{0, 0})), - Arguments.of(Named.of("Invalid key name length", - Bytes.concat(getBytes(4), getBytes("key")))), - Arguments.of(Named.of("Invalid Unsigned Long length", - Bytes.concat(getBytes(4), getBytes("key1"), - new byte[]{0, 0}))), - Arguments.of(Named.of("Invalid Sequence number", - Bytes.concat(getBytes(4), getBytes("key1")))), - Arguments.of(Named.of("Invalid Type", - Bytes.concat(getBytes(4), getBytes("key1"), - getBytes(4L)))), - Arguments.of(Named.of("Invalid Value", - Bytes.concat(getBytes(4), getBytes("key"), - getBytes(4L), getBytes(0)))), - Arguments.of(Named.of("Invalid Value length", - Bytes.concat(getBytes(4), getBytes("key"), - getBytes(4L), getBytes(1), getBytes(6), - getBytes("val")))) - ); - } - - @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) - @ParameterizedTest - @MethodSource("keyValueFormatArgs") - @Unhealthy("HDDS-9274") - public void testSSTDumpIteratorWithKeyFormat(String keyFormat, - String valueFormat) - throws Exception { - assumeTrue(NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); - - TreeMap, String> keys = - IntStream.range(0, 100).boxed().collect( - Collectors.toMap( - i -> Pair.of(String.format(keyFormat, i), i % 2), - i -> i % 2 == 0 ? "" : String.format(valueFormat, i), - (v1, v2) -> v2, - TreeMap::new)); - File file = createSSTFileWithKeys(keys); - ExecutorService executorService = - new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(1), - new ThreadPoolExecutor.CallerRunsPolicy()); - ManagedSSTDumpTool tool = new ManagedSSTDumpTool(executorService, 8192); - List> testBounds = TestUtils.getTestingBounds( - keys.keySet().stream().collect(Collectors.toMap(Pair::getKey, - Pair::getValue, (v1, v2) -> v1, TreeMap::new))); - for (Optional keyStart : testBounds) { - for (Optional keyEnd : testBounds) { - Map, String> expectedKeys = keys.entrySet() - .stream().filter(e -> keyStart.map(s -> e.getKey().getKey() - .compareTo(s) >= 0).orElse(true)) - .filter(e -> keyEnd.map(s -> e.getKey().getKey().compareTo(s) < 0) - .orElse(true)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - Optional lowerBound = keyStart - .map(s -> new ManagedSlice(StringUtils.string2Bytes(s))); - Optional upperBound = keyEnd - .map(s -> new ManagedSlice(StringUtils.string2Bytes(s))); - try (ManagedOptions options = new ManagedOptions(); - ManagedSSTDumpIterator iterator = - new ManagedSSTDumpIterator(tool, - file.getAbsolutePath(), options, lowerBound.orElse(null), - upperBound.orElse(null)) { - @Override - protected KeyValue getTransformedValue( - Optional value) { - return value.orElse(null); - } - } - ) { - while (iterator.hasNext()) { - ManagedSSTDumpIterator.KeyValue r = iterator.next(); - String key = new String(r.getKey(), StandardCharsets.UTF_8); - Pair recordKey = Pair.of(key, r.getType()); - assertThat(expectedKeys).containsKey(recordKey); - assertEquals(Optional.ofNullable(expectedKeys - .get(recordKey)).orElse(""), - new String(r.getValue(), StandardCharsets.UTF_8)); - expectedKeys.remove(recordKey); - } - assertEquals(0, expectedKeys.size()); - } finally { - lowerBound.ifPresent(ManagedSlice::close); - upperBound.ifPresent(ManagedSlice::close); - } - } - } - executorService.shutdown(); - } - - - @ParameterizedTest - @MethodSource("invalidPipeInputStreamBytes") - public void testInvalidSSTDumpIteratorWithKeyFormat(byte[] inputBytes) - throws ExecutionException, - InterruptedException, IOException { - ByteArrayInputStream byteArrayInputStream = - new ByteArrayInputStream(inputBytes); - ManagedSSTDumpTool tool = mock(ManagedSSTDumpTool.class); - File file = Files.createFile(tempDir.resolve("tmp_file.sst")).toFile(); - Future future = mock(Future.class); - when(future.isDone()).thenReturn(false); - when(future.get()).thenReturn(0); - when(tool.run(any(Map.class), - any(ManagedOptions.class))) - .thenReturn(new ManagedSSTDumpTool.SSTDumpToolTask(future, - byteArrayInputStream)); - try (ManagedOptions options = new ManagedOptions()) { - assertThrows(IllegalStateException.class, - () -> new ManagedSSTDumpIterator( - tool, file.getAbsolutePath(), options) { - @Override - protected KeyValue getTransformedValue( - Optional value) { - return value.orElse(null); - } - }); - } - } -} diff --git a/pom.xml b/pom.xml index 37dfb139e2cc..48b3a3df6a72 100644 --- a/pom.xml +++ b/pom.xml @@ -298,11 +298,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 1.9.7 1.14.0 2.4.0 - 1.0.8 - 1.2.13 - 1.9.3 - 1.1.8 - 1.4.9 1.0.1 5.3.27 @@ -310,9 +305,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 5.1.0 + 1.0.0 - @@ -2091,6 +2086,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + + org.codehaus.mojo + properties-maven-plugin + ${properties.maven.plugin.version} + From f25a570931d93c200f39f7c20ff1a9d3740b47b4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 13 Feb 2024 18:55:31 -0800 Subject: [PATCH 05/39] HDDS-10149. Remove usage of managed sst dump tool --- .../src/main/resources/ozone-default.xml | 18 ------ .../ozone/rocksdb/util/SstFileSetReader.java | 56 ++++++++++++++----- .../rocksdb/util/TestSstFileSetReader.java | 8 +-- .../apache/hadoop/ozone/om/OMConfigKeys.java | 12 ---- .../ozone/om/snapshot/TestOmSnapshot.java | 2 +- .../om/snapshot/SnapshotDiffManager.java | 48 ++++------------ .../om/snapshot/TestSnapshotDiffManager.java | 22 +------- 7 files changed, 57 insertions(+), 109 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 094fbff16da7..5a33aa1ab45f 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -4265,15 +4265,6 @@ - - ozone.om.snapshot.sst_dumptool.pool.size - 1 - OZONE, OM - - Threadpool size for SST Dumptool which would be used for computing snapdiff when native library is enabled. - - - ozone.om.snapshot.load.native.lib true @@ -4283,15 +4274,6 @@ - - ozone.om.snapshot.sst_dumptool.buffer.size - 8KB - OZONE, OM - - Buffer size for SST Dumptool Pipe which would be used for computing snapdiff when native library is enabled. - - - ozone.om.snapshot.diff.max.allowed.keys.changed.per.job 10000000 diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java index be949cd4fbdd..96b7bb067005 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java @@ -20,12 +20,12 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReaderIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedSlice; import org.apache.hadoop.util.ClosableIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedReadOptions; -import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpIterator; -import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDBException; import org.rocksdb.SstFileReader; @@ -37,9 +37,9 @@ import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Objects; -import java.util.Optional; import java.util.Spliterator; import java.util.Spliterators; +import java.util.function.Function; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -142,8 +142,7 @@ public void close() throws UncheckedIOException { return getStreamFromIterator(itr); } - public Stream getKeyStreamWithTombstone( - ManagedSSTDumpTool sstDumpTool, String lowerBound, + public Stream getKeyStreamWithTombstone(String lowerBound, String upperBound) throws RocksDBException { final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { @@ -166,16 +165,11 @@ protected void init() { } @Override - protected ClosableIterator getKeyIteratorForFile(String file) - throws IOException { - return new ManagedSSTDumpIterator(sstDumpTool, file, - options, lowerBoundSlice, upperBoundSlice) { - @Override - protected String getTransformedValue(Optional value) { - return value.map(v -> StringUtils.bytes2String(v.getKey())) - .orElse(null); - } - }; + protected ClosableIterator getKeyIteratorForFile( + String file) { + return new ManagedRawSstFileIterator(file, + options, lowerBoundSlice, upperBoundSlice, + keyValue -> StringUtils.bytes2String(keyValue.getKey())); } @Override @@ -223,6 +217,38 @@ public String next() { } } + private static class ManagedRawSstFileIterator implements + ClosableIterator { + private ManagedRawSSTFileReader fileReader; + private ManagedRawSSTFileReaderIterator fileReaderIterator; + + ManagedRawSstFileIterator(String path, ManagedOptions options, + ManagedSlice lowerBound, ManagedSlice upperBound, + Function keyValueFunction) { + this.fileReader = new ManagedRawSSTFileReader<>(options, path, + 2 * 1024 * 1024); + this.fileReaderIterator = fileReader.newIterator(keyValueFunction, + lowerBound, upperBound); + } + + @Override + public void close() { + this.fileReaderIterator.close(); + this.fileReader.close(); + } + + @Override + public boolean hasNext() { + return fileReaderIterator.hasNext(); + } + + @Override + public String next() { + return fileReaderIterator.next(); + } + } + private abstract static class MultipleSstFileIterator implements ClosableIterator { diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java index edc491e7c8da..aa8813ccd867 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java @@ -21,11 +21,9 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.utils.NativeLibraryLoader; -import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.apache.hadoop.hdds.utils.TestUtils; import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; -import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter; import org.apache.ozone.test.tag.Native; import org.apache.ozone.test.tag.Unhealthy; @@ -166,7 +164,7 @@ public void testGetKeyStream(int numberOfFiles) @ValueSource(ints = {0, 1, 2, 3, 7, 10}) @Unhealthy("HDDS-9274") public void testGetKeyStreamWithTombstone(int numberOfFiles) - throws RocksDBException, IOException, NativeLibraryNotLoadedException { + throws RocksDBException, IOException { assumeTrue(NativeLibraryLoader.getInstance() .loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); Pair, List> data = @@ -178,8 +176,6 @@ public void testGetKeyStreamWithTombstone(int numberOfFiles) new SynchronousQueue<>(), new ThreadFactoryBuilder() .setNameFormat("snapshot-diff-manager-sst-dump-tool-TID-%d") .build(), new ThreadPoolExecutor.DiscardPolicy()); - ManagedSSTDumpTool sstDumpTool = - new ManagedSSTDumpTool(executorService, 256); // Getting every possible combination of 2 elements from the sampled keys. // Reading the sst file lying within the given bounds and // validating the keys read from the sst file. @@ -197,7 +193,7 @@ public void testGetKeyStreamWithTombstone(int numberOfFiles) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); try (Stream keyStream = new SstFileSetReader(files) - .getKeyStreamWithTombstone(sstDumpTool, lowerBound.orElse(null), + .getKeyStreamWithTombstone(lowerBound.orElse(null), upperBound.orElse(null))) { keyStream.forEach( key -> { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 5dd7579eb916..6e1326626273 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -28,21 +28,9 @@ * Ozone Manager Constants. */ public final class OMConfigKeys { - public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE = - "ozone.om.snapshot.sst_dumptool.pool.size"; - public static final int - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT = 1; - public static final String OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB = "ozone.om.snapshot.load.native.lib"; public static final boolean OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT = true; - public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE = - "ozone.om.snapshot.sst_dumptool.buffer.size"; - public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT = "8KB"; - /** * Never constructed. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 029b0813bb55..d77778ca3b27 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -226,7 +226,7 @@ private void init() throws Exception { // stop the deletion services so that keys can still be read stopKeyManager(); - preFinalizationChecks(); +// preFinalizationChecks(); finalizeOMUpgrade(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 41e990097ecd..e46295ae9d72 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -25,14 +25,13 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.apache.hadoop.hdds.utils.db.CodecRegistry; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; -import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.ozone.OFSPath; import org.apache.hadoop.ozone.OzoneConsts; @@ -89,7 +88,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; @@ -185,9 +183,7 @@ public class SnapshotDiffManager implements AutoCloseable { private final boolean diffDisableNativeLibs; - private final Optional sstDumpTool; - - private Optional sstDumpToolExecService; + private final boolean isNativeLibsLoaded; private final BiFunction generateSnapDiffJobKey = @@ -264,7 +260,7 @@ public SnapshotDiffManager(ManagedRocksDB db, createEmptySnapDiffDir(path); this.sstBackupDirForSnapDiffJobs = path.toString(); - this.sstDumpTool = initSSTDumpTool(ozoneManager.getConfiguration()); + this.isNativeLibsLoaded = initSSTDumpTool(ozoneManager.getConfiguration()); // Ideally, loadJobsOnStartUp should run only on OM node, since SnapDiff // is not HA currently and running this on all the nodes would be @@ -287,35 +283,17 @@ public PersistentMap getSnapDiffJobTable() { return snapDiffJobTable; } - private Optional initSSTDumpTool( + private boolean initSSTDumpTool( final OzoneConfiguration conf) { if (conf.getBoolean(OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) { try { - int threadPoolSize = conf.getInt( - OMConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE, - OMConfigKeys - .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT); - int bufferSize = (int) conf.getStorageSize( - OMConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE, - OMConfigKeys - .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT, - StorageUnit.BYTES); - this.sstDumpToolExecService = Optional.of(new ThreadPoolExecutor(0, - threadPoolSize, 60, TimeUnit.SECONDS, - new SynchronousQueue<>(), new ThreadFactoryBuilder() - .setNameFormat(ozoneManager.getThreadNamePrefix() + - "snapshot-diff-manager-sst-dump-tool-TID-%d") - .build(), - new ThreadPoolExecutor.DiscardPolicy())); - return Optional.of(new ManagedSSTDumpTool(sstDumpToolExecService.get(), - bufferSize)); + return ManagedRawSSTFileReader.loadLibrary(); } catch (NativeLibraryNotLoadedException e) { - this.sstDumpToolExecService.ifPresent(exec -> - closeExecutorService(exec, "SstDumpToolExecutor")); + return false; } } - return Optional.empty(); + return false; } /** @@ -1056,12 +1034,12 @@ private void getDeltaFilesAndDiffKeysToObjectIdToKeyMap( // Workaround to handle deletes if native rocksDb tool for reading // tombstone is not loaded. // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read tombstone - if (skipNativeDiff || !sstDumpTool.isPresent()) { + if (skipNativeDiff || !isNativeLibsLoaded) { deltaFiles.addAll(getSSTFileListForSnapshot(fromSnapshot, tablesToLookUp)); } addToObjectIdMap(fsTable, tsTable, deltaFiles, - !skipNativeDiff && sstDumpTool.isPresent(), + !skipNativeDiff && isNativeLibsLoaded, oldObjIdToKeyMap, newObjIdToKeyMap, objectIdToIsDirMap, oldParentIds, newParentIds, tablePrefixes); } @@ -1093,9 +1071,9 @@ void addToObjectIdMap(Table fsTable, sstFileReaderUpperBound = String.valueOf(upperBoundCharArray); } try (Stream keysToCheck = - nativeRocksToolsLoaded && sstDumpTool.isPresent() - ? sstFileReader.getKeyStreamWithTombstone(sstDumpTool.get(), - sstFileReaderLowerBound, sstFileReaderUpperBound) + nativeRocksToolsLoaded && isNativeLibsLoaded + ? sstFileReader.getKeyStreamWithTombstone( + sstFileReaderLowerBound, sstFileReaderUpperBound) : sstFileReader.getKeyStream(sstFileReaderLowerBound, sstFileReaderUpperBound)) { keysToCheck.forEach(key -> { @@ -1678,8 +1656,6 @@ public void close() { if (snapDiffExecutor != null) { closeExecutorService(snapDiffExecutor, "SnapDiffExecutor"); } - this.sstDumpToolExecService.ifPresent(exec -> - closeExecutorService(exec, "SstDumpToolExecutor")); } private void closeExecutorService(ExecutorService executorService, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index b92546c2899b..c40543824b1a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -21,12 +21,10 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.CodecRegistry; import org.apache.hadoop.hdds.utils.db.RDBStore; @@ -35,7 +33,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; -import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; @@ -134,10 +131,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT; import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER; import static org.apache.hadoop.ozone.om.OmSnapshotManager.SNAP_DIFF_JOB_TABLE_NAME; import static org.apache.hadoop.ozone.om.OmSnapshotManager.SNAP_DIFF_REPORT_TABLE_NAME; @@ -343,15 +336,6 @@ public void init() throws RocksDBException, IOException, ExecutionException { .getInt(OZONE_OM_SNAPSHOT_DIFF_THREAD_POOL_SIZE, OZONE_OM_SNAPSHOT_DIFF_THREAD_POOL_SIZE_DEFAULT)) .thenReturn(OZONE_OM_SNAPSHOT_DIFF_THREAD_POOL_SIZE_DEFAULT); - when(configuration - .getInt(OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE, - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT)) - .thenReturn(OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT); - when(configuration - .getStorageSize(OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE, - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT, - StorageUnit.BYTES)) - .thenReturn(FileUtils.ONE_KB_BI.doubleValue()); when(configuration.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) .thenReturn(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT); @@ -653,15 +637,11 @@ public void testObjectIdMapWithTombstoneEntries(boolean nativeLibraryLoaded, try (MockedConstruction mockedSSTFileReader = mockConstruction(SstFileSetReader.class, (mock, context) -> { - when(mock.getKeyStreamWithTombstone(any(), any(), any())) + when(mock.getKeyStreamWithTombstone(any(), any())) .thenReturn(keysIncludingTombstones.stream()); when(mock.getKeyStream(any(), any())) .thenReturn(keysExcludingTombstones.stream()); }); - MockedConstruction mockedSSTDumpTool = - mockConstruction(ManagedSSTDumpTool.class, - (mock, context) -> { - }) ) { Map toSnapshotTableMap = IntStream.concat(IntStream.range(0, 25), IntStream.range(50, 100)) From a11705ad282440e4c2d88664a8c0ff095633c929 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 13 Feb 2024 22:59:34 -0800 Subject: [PATCH 06/39] HDDS-10149. Fix findbugs & rat check --- .../db/managed/JniLibNamePropertyWriter.java | 9 +- .../ManagedRawSSTFileReaderIterator.java | 2 + .../db/managed/ManagedSSTDumpIterator.java | 314 ------------------ .../utils/db/managed/ManagedSSTDumpTool.java | 99 ------ .../utils/db/managed/PipeInputStream.java | 88 ----- .../src/main/patches/rocks-native.patch | 22 +- 6 files changed, 27 insertions(+), 507 deletions(-) delete mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java delete mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpTool.java delete mode 100644 hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/PipeInputStream.java diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java index e405a2d884f0..8d2daf408c80 100644 --- a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java +++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java @@ -19,8 +19,14 @@ import org.rocksdb.util.Environment; +import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; /** * Class to write the rocksdb lib name to a file. @@ -33,7 +39,8 @@ private JniLibNamePropertyWriter() { public static void main(String[] args) { String filePath = args[0]; - try (FileWriter writer = new FileWriter(filePath)) { + try (Writer writer = new OutputStreamWriter( + Files.newOutputStream(Paths.get(filePath)), StandardCharsets.UTF_8)) { writer.write("rocksdbLibName=" + Environment.getJniLibraryFileName("rocksdb")); } catch (IOException e) { diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java index a4f1b30f38e0..087ca50e5dd2 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java @@ -88,6 +88,7 @@ private KeyValue(byte[] key, UnsignedLong sequence, Integer type, this.value = value; } + @SuppressWarnings("M V EI") public byte[] getKey() { return key; } @@ -100,6 +101,7 @@ public Integer getType() { return type; } + @SuppressWarnings("M V EI") public byte[] getValue() { return value; } diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java deleted file mode 100644 index d8844eaacbcd..000000000000 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java +++ /dev/null @@ -1,314 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.hdds.utils.db.managed; - -import com.google.common.collect.Maps; -import com.google.common.primitives.UnsignedLong; -import org.apache.hadoop.hdds.StringUtils; -import org.apache.hadoop.util.ClosableIterator; -import org.eclipse.jetty.io.RuntimeIOException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Arrays; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -/** - * Iterator to Parse output of RocksDBSSTDumpTool. - */ -public abstract class ManagedSSTDumpIterator implements ClosableIterator { - - private static final Logger LOG = - LoggerFactory.getLogger(ManagedSSTDumpIterator.class); - // Since we don't have any restriction on the key & value, we are prepending - // the length of the pattern in the sst dump tool output. - // The first token in the pattern is the key. - // The second tells the sequence number of the key. - // The third token gives the type of key in the sst file. - // The fourth token - private InputStream processOutput; - private Optional currentKey; - private byte[] intBuffer; - private Optional nextKey; - - private ManagedSSTDumpTool.SSTDumpToolTask sstDumpToolTask; - private AtomicBoolean open; - private StackTraceElement[] stackTrace; - - public ManagedSSTDumpIterator(ManagedSSTDumpTool sstDumpTool, - String sstFilePath, ManagedOptions options) - throws IOException { - this(sstDumpTool, sstFilePath, options, null, null); - } - - public ManagedSSTDumpIterator(ManagedSSTDumpTool sstDumpTool, - String sstFilePath, ManagedOptions options, - ManagedSlice lowerKeyBound, - ManagedSlice upperKeyBound) - throws IOException { - File sstFile = new File(sstFilePath); - if (!sstFile.exists()) { - throw new IOException(String.format("File in path : %s doesn't exist", - sstFile.getAbsolutePath())); - } - if (!sstFile.isFile()) { - throw new IOException(String.format("Path given: %s is not a file", - sstFile.getAbsolutePath())); - } - init(sstDumpTool, sstFile, options, lowerKeyBound, upperKeyBound); - this.stackTrace = Thread.currentThread().getStackTrace(); - } - - /** - * Parses next occuring number in the stream. - * - * @return Optional of the integer empty if no integer exists - */ - private Optional getNextNumberInStream() throws IOException { - int n = processOutput.read(intBuffer, 0, 4); - if (n == 4) { - return Optional.of(ByteBuffer.wrap(intBuffer).getInt()); - } else if (n >= 0) { - throw new IllegalStateException(String.format("Integer expects " + - "4 bytes to be read from the stream, but read only %d bytes", n)); - } - return Optional.empty(); - } - - private Optional getNextByteArray() throws IOException { - Optional size = getNextNumberInStream(); - if (size.isPresent()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Allocating byte array, size: {}", size.get()); - } - byte[] b = new byte[size.get()]; - int n = processOutput.read(b); - if (n >= 0 && n != size.get()) { - throw new IllegalStateException(String.format("Integer expects " + - "4 bytes to be read from the stream, but read only %d bytes", n)); - } - return Optional.of(b); - } - return Optional.empty(); - } - - private Optional getNextUnsignedLong() throws IOException { - long val = 0; - for (int i = 0; i < 8; i++) { - val = val << 8; - int nextByte = processOutput.read(); - if (nextByte < 0) { - if (i == 0) { - return Optional.empty(); - } - throw new IllegalStateException(String.format("Long expects " + - "8 bytes to be read from the stream, but read only %d bytes", i)); - } - val += nextByte; - } - return Optional.of(UnsignedLong.fromLongBits(val)); - } - - private void init(ManagedSSTDumpTool sstDumpTool, File sstFile, - ManagedOptions options, ManagedSlice lowerKeyBound, - ManagedSlice upperKeyBound) { - Map argMap = Maps.newHashMap(); - argMap.put("file", sstFile.getAbsolutePath()); - argMap.put("silent", null); - argMap.put("command", "scan"); - // strings containing '\0' do not have the same value when encode UTF-8 on - // java which is 0. But in jni the utf-8 encoded value for '\0' - // becomes -64 -128. Thus the value becomes different. - // In order to support this, changes have been made on the rocks-tools - // to pass the address of the ManagedSlice and the jni can use the object - // of slice directly from there. - if (Objects.nonNull(lowerKeyBound)) { - argMap.put("from", String.valueOf(lowerKeyBound.getNativeHandle())); - } - if (Objects.nonNull(upperKeyBound)) { - argMap.put("to", String.valueOf(upperKeyBound.getNativeHandle())); - } - this.sstDumpToolTask = sstDumpTool.run(argMap, options); - processOutput = sstDumpToolTask.getPipedOutput(); - intBuffer = new byte[4]; - open = new AtomicBoolean(true); - currentKey = Optional.empty(); - nextKey = Optional.empty(); - next(); - } - - /** - * Throws Runtime exception in the case iterator is closed or - * the native Dumptool exited with non zero exit value. - */ - private void checkSanityOfProcess() { - if (!this.open.get()) { - throw new RuntimeException("Iterator has been closed"); - } - if (sstDumpToolTask.getFuture().isDone() && - sstDumpToolTask.exitValue() != 0) { - throw new RuntimeException("Process Terminated with non zero " + - String.format("exit value %d", sstDumpToolTask.exitValue())); - } - } - - /** - * Checks the status of the process & sees if there is another record. - * - * @return True if next exists & false otherwise - * Throws Runtime Exception in case of SST File read failure - */ - - @Override - public boolean hasNext() { - checkSanityOfProcess(); - return nextKey.isPresent(); - } - - /** - * Transforms Key to a certain value. - * - * @param value - * @return transformed Value - */ - protected abstract T getTransformedValue(Optional value); - - /** - * Returns the next record from SSTDumpTool. - * - * @return next Key - * Throws Runtime Exception incase of failure. - */ - @Override - public T next() { - checkSanityOfProcess(); - currentKey = nextKey; - nextKey = Optional.empty(); - try { - Optional key = getNextByteArray(); - if (!key.isPresent()) { - return getTransformedValue(currentKey); - } - UnsignedLong sequenceNumber = getNextUnsignedLong() - .orElseThrow(() -> new IllegalStateException( - String.format("Error while trying to read sequence number" + - " for key %s", StringUtils.bytes2String(key.get())))); - - Integer type = getNextNumberInStream() - .orElseThrow(() -> new IllegalStateException( - String.format("Error while trying to read sequence number for " + - "key %s with sequence number %s", - StringUtils.bytes2String(key.get()), - sequenceNumber.toString()))); - byte[] val = getNextByteArray().orElseThrow(() -> - new IllegalStateException( - String.format("Error while trying to read sequence number for " + - "key %s with sequence number %s of type %d", - StringUtils.bytes2String(key.get()), - sequenceNumber.toString(), type))); - nextKey = Optional.of(new KeyValue(key.get(), sequenceNumber, type, val)); - } catch (IOException e) { - // TODO [SNAPSHOT] Throw custom snapshot exception - throw new RuntimeIOException(e); - } - return getTransformedValue(currentKey); - } - - @Override - public synchronized void close() throws UncheckedIOException { - if (this.sstDumpToolTask != null) { - if (!this.sstDumpToolTask.getFuture().isDone()) { - this.sstDumpToolTask.getFuture().cancel(true); - } - try { - this.processOutput.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - open.compareAndSet(true, false); - } - - @Override - protected void finalize() throws Throwable { - if (open.get()) { - LOG.warn("{} is not closed properly." + - " StackTrace for unclosed instance: {}", - this.getClass().getName(), - Arrays.stream(stackTrace) - .map(StackTraceElement::toString).collect( - Collectors.joining("\n"))); - } - this.close(); - super.finalize(); - } - - /** - * Class containing Parsed KeyValue Record from Sst Dumptool output. - */ - public static final class KeyValue { - - private final byte[] key; - private final UnsignedLong sequence; - private final Integer type; - private final byte[] value; - - private KeyValue(byte[] key, UnsignedLong sequence, Integer type, - byte[] value) { - this.key = key; - this.sequence = sequence; - this.type = type; - this.value = value; - } - - public byte[] getKey() { - return key; - } - - public UnsignedLong getSequence() { - return sequence; - } - - public Integer getType() { - return type; - } - - public byte[] getValue() { - return value; - } - - @Override - public String toString() { - return "KeyValue{" + - "key=" + StringUtils.bytes2String(key) + - ", sequence=" + sequence + - ", type=" + type + - ", value=" + StringUtils.bytes2String(value) + - '}'; - } - } -} diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpTool.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpTool.java deleted file mode 100644 index 5d965d7398e0..000000000000 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpTool.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.utils.db.managed; - -import org.apache.hadoop.hdds.utils.NativeLibraryLoader; -import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; - -import java.io.InputStream; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; - -import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; - -/** - * JNI for RocksDB SSTDumpTool. Pipes the output to an output stream - */ -public class ManagedSSTDumpTool { - - private int bufferCapacity; - private ExecutorService executorService; - - public ManagedSSTDumpTool(ExecutorService executorService, - int bufferCapacity) - throws NativeLibraryNotLoadedException { - if (!NativeLibraryLoader.getInstance() - .loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)) { - throw new NativeLibraryNotLoadedException( - ROCKS_TOOLS_NATIVE_LIBRARY_NAME); - } - this.bufferCapacity = bufferCapacity; - this.executorService = executorService; - } - - public SSTDumpToolTask run(String[] args, ManagedOptions options) { - PipeInputStream pipeInputStream = new PipeInputStream(bufferCapacity); - return new SSTDumpToolTask(this.executorService.submit(() -> - this.runInternal(args, options.getNativeHandle(), - pipeInputStream.getNativeHandle())), pipeInputStream); - } - - public SSTDumpToolTask run(Map args, ManagedOptions options) { - return this.run(args.entrySet().stream().map(e -> "--" - + (e.getValue() == null || e.getValue().isEmpty() ? e.getKey() : - e.getKey() + "=" + e.getValue())).toArray(String[]::new), options); - } - - private native int runInternal(String[] args, long optionsHandle, - long pipeHandle); - - /** - * Class holding piped output of SST Dumptool & future of command. - */ - static class SSTDumpToolTask { - private Future future; - private InputStream pipedOutput; - - SSTDumpToolTask(Future future, InputStream pipedOutput) { - this.future = future; - this.pipedOutput = pipedOutput; - } - - public Future getFuture() { - return future; - } - - public InputStream getPipedOutput() { - return pipedOutput; - } - - public int exitValue() { - if (this.future.isDone()) { - try { - return future.get(); - } catch (InterruptedException | ExecutionException e) { - return 1; - } - } - return 0; - } - } -} diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/PipeInputStream.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/PipeInputStream.java deleted file mode 100644 index df4f613f98e2..000000000000 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/PipeInputStream.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.utils.db.managed; - -import java.io.InputStream; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * JNI for reading data from pipe. - */ -public class PipeInputStream extends InputStream { - - private byte[] byteBuffer; - private long nativeHandle; - private int numberOfBytesLeftToRead; - private int index = 0; - private int capacity; - - private AtomicBoolean cleanup; - - PipeInputStream(int capacity) { - this.byteBuffer = new byte[capacity]; - this.numberOfBytesLeftToRead = 0; - this.capacity = capacity; - this.nativeHandle = newPipe(); - this.cleanup = new AtomicBoolean(false); - } - - long getNativeHandle() { - return nativeHandle; - } - - @Override - public int read() { - if (numberOfBytesLeftToRead < 0) { - this.close(); - return -1; - } - while (numberOfBytesLeftToRead == 0) { - numberOfBytesLeftToRead = readInternal(byteBuffer, capacity, - nativeHandle); - index = 0; - if (numberOfBytesLeftToRead != 0) { - return read(); - } - } - numberOfBytesLeftToRead--; - int ret = byteBuffer[index] & 0xFF; - index += 1; - return ret; - } - - private native long newPipe(); - - private native int readInternal(byte[] buff, int numberOfBytes, - long pipeHandle); - - private native void closeInternal(long pipeHandle); - - @Override - public void close() { - if (this.cleanup.compareAndSet(false, true)) { - closeInternal(this.nativeHandle); - } - } - - @Override - protected void finalize() throws Throwable { - close(); - super.finalize(); - } -} diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index 5d0cf99ac486..7023ab2d777f 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -1,8 +1,20 @@ -commit 0d13ca6c284e96d9e055ed8ba83e339120f6a5c5 -Author: Swaminathan Balachandran -Date: Mon Feb 12 19:13:13 2024 -0800 - - Dummy +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/include/rocksdb/raw_iterator.h b/include/rocksdb/raw_iterator.h new file mode 100644 From c80234dcc24a906626235c967ab0e6233d90bbfe Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 13 Feb 2024 23:22:58 -0800 Subject: [PATCH 07/39] HDDS-10149. Fix findbugs --- .../utils/db/managed/ManagedRawSSTFileReaderIterator.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java index 087ca50e5dd2..6191befdb78c 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.util.ClosableIterator; +import java.util.Arrays; import java.util.function.Function; /** @@ -88,9 +89,8 @@ private KeyValue(byte[] key, UnsignedLong sequence, Integer type, this.value = value; } - @SuppressWarnings("M V EI") public byte[] getKey() { - return key; + return Arrays.copyOf(key, key.length); } public UnsignedLong getSequence() { @@ -101,9 +101,8 @@ public Integer getType() { return type; } - @SuppressWarnings("M V EI") public byte[] getValue() { - return value; + return Arrays.copyOf(value, value.length); } @Override From 406db5bd656ff11e9911a19ea501ad49d51f26cc Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 13 Feb 2024 23:40:30 -0800 Subject: [PATCH 08/39] HDDS-10149. Fix checkstyle --- .../hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java index 8d2daf408c80..46fbeb412a84 100644 --- a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java +++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/JniLibNamePropertyWriter.java @@ -19,8 +19,6 @@ import org.rocksdb.util.Environment; -import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; From 41d557df4a482f7736a8fe012f0cb8b6436457fa Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 09:08:11 -0800 Subject: [PATCH 09/39] HDDS-10149. Fix cpp shadow error --- .../rocks-native/src/main/patches/rocks-native.patch | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index 7023ab2d777f..f7bd8fe51409 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -421,7 +421,7 @@ index 000000000..1693bd1e6 +#endif // ROCKSDB_LITE diff --git a/tools/raw_sst_file_reader_iterator.cc b/tools/raw_sst_file_reader_iterator.cc new file mode 100644 -index 000000000..b31596c4a +index 000000000..463c484ed --- /dev/null +++ b/tools/raw_sst_file_reader_iterator.cc @@ -0,0 +1,94 @@ @@ -472,8 +472,8 @@ index 000000000..b31596c4a + bool has_to, Slice* to_key) + : iter_(iterator), + ikey(new ParsedInternalKey()), -+ has_to(has_to), -+ to_key(to_key) { ++ has_to_(has_to), ++ to_key_(to_key) { + if (has_from) { + InternalKey k; + k.SetMinPossibleForUserKey(*from_key); @@ -521,7 +521,7 @@ index 000000000..b31596c4a +#endif // ROCKSDB_LITE diff --git a/tools/raw_sst_file_reader_iterator.h b/tools/raw_sst_file_reader_iterator.h new file mode 100644 -index 000000000..8c3c4acf2 +index 000000000..79313264f --- /dev/null +++ b/tools/raw_sst_file_reader_iterator.h @@ -0,0 +1,45 @@ @@ -563,8 +563,8 @@ index 000000000..8c3c4acf2 + void initKey(); + InternalIterator* iter_; + ParsedInternalKey* ikey; -+ bool has_to; -+ Slice* to_key; ++ bool has_to_; ++ Slice* to_key_; +}; + +} // namespace ROCKSDB_NAMESPACE From f9dcb2ee329b0d33f88f45a9c1ddd90646d90acc Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 10:07:36 -0800 Subject: [PATCH 10/39] HDDS-10149. Fix native --- .../src/main/patches/rocks-native.patch | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index f7bd8fe51409..b82f3e686151 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -1,21 +1,3 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - diff --git a/include/rocksdb/raw_iterator.h b/include/rocksdb/raw_iterator.h new file mode 100644 index 000000000..ec3c05d6d @@ -421,7 +403,7 @@ index 000000000..1693bd1e6 +#endif // ROCKSDB_LITE diff --git a/tools/raw_sst_file_reader_iterator.cc b/tools/raw_sst_file_reader_iterator.cc new file mode 100644 -index 000000000..463c484ed +index 000000000..39a2b84be --- /dev/null +++ b/tools/raw_sst_file_reader_iterator.cc @@ -0,0 +1,94 @@ @@ -485,9 +467,9 @@ index 000000000..463c484ed +} + +bool RawSstFileReaderIterator::has_next() const { -+ return iter_->Valid() && (!has_to || ++ return iter_->Valid() && (!has_to_ || + BytewiseComparator()->Compare( -+ getKey(), *to_key) < 0); ++ getKey(), *to_key_) < 0); +} + +void RawSstFileReaderIterator::initKey() { From 194c8af161a86aab12482545b7076c2b234930ed Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 10:10:38 -0800 Subject: [PATCH 11/39] HDDS-10149. Fix native --- .../src/main/patches/rocks-native.patch | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index b82f3e686151..b56fcefd6933 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + diff --git a/include/rocksdb/raw_iterator.h b/include/rocksdb/raw_iterator.h new file mode 100644 index 000000000..ec3c05d6d From 09e640c59979cab5aff0174699f30a8d63d230ce Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 10:51:11 -0800 Subject: [PATCH 12/39] HDDS-10149. Fix native --- hadoop-hdds/rocks-native/pom.xml | 4 ++-- hadoop-hdds/rocks-native/src/CMakeLists.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index ab862c812b12..d18430675a33 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -264,8 +264,8 @@ - - + diff --git a/hadoop-hdds/rocks-native/src/CMakeLists.txt b/hadoop-hdds/rocks-native/src/CMakeLists.txt index 5c3a31717cf5..1f862b6ab67a 100644 --- a/hadoop-hdds/rocks-native/src/CMakeLists.txt +++ b/hadoop-hdds/rocks-native/src/CMakeLists.txt @@ -24,6 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") project(ozone_native) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_BUILD_TYPE Release) find_package(JNI REQUIRED) include_directories(${JNI_INCLUDE_DIRS}) From bbb15ad1ae133f65de5af7bbb626e6a015778ef8 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 12:07:59 -0800 Subject: [PATCH 13/39] HDDS-10149. Debug native failure --- hadoop-hdds/rocks-native/pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index d18430675a33..fa316d504131 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -263,7 +263,9 @@ - + + + From dd78c1e788ce5711aba3233a87c4b568140ea805 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 12:35:53 -0800 Subject: [PATCH 14/39] HDDS-10149. Debug native failure --- hadoop-hdds/rocks-native/pom.xml | 1 + hadoop-hdds/rocks-native/src/CMakeLists.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index fa316d504131..43c4c25e0d61 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -236,6 +236,7 @@ + diff --git a/hadoop-hdds/rocks-native/src/CMakeLists.txt b/hadoop-hdds/rocks-native/src/CMakeLists.txt index 1f862b6ab67a..5c3a31717cf5 100644 --- a/hadoop-hdds/rocks-native/src/CMakeLists.txt +++ b/hadoop-hdds/rocks-native/src/CMakeLists.txt @@ -24,7 +24,6 @@ cmake_minimum_required(VERSION 2.8) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") project(ozone_native) -set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_BUILD_TYPE Release) find_package(JNI REQUIRED) include_directories(${JNI_INCLUDE_DIRS}) From 2d2c8f6625d5f23f6a8e680e28beebc79b12b8f7 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 13:02:00 -0800 Subject: [PATCH 15/39] HDDS-10149. Fix test cases --- .../org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java index 8fc4e83e7a1d..4edce046fbb0 100644 --- a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.utils; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils; import org.apache.ozone.test.tag.Native; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -58,6 +59,7 @@ private static Stream nativeLibraryDirectoryLocations() { @MethodSource("nativeLibraryDirectoryLocations") public void testNativeLibraryLoader( String nativeLibraryDirectoryLocation) { + ManagedRocksObjectUtils.loadRocksDBLibrary(); Map libraryLoadedMap = new HashMap<>(); NativeLibraryLoader loader = new NativeLibraryLoader(libraryLoadedMap); try (MockedStatic mockedNativeLibraryLoader = @@ -68,6 +70,7 @@ public void testNativeLibraryLoader( .thenReturn(nativeLibraryDirectoryLocation); mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getInstance()) .thenReturn(loader); + assertTrue(NativeLibraryLoader.getInstance() .loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); assertTrue(NativeLibraryLoader From be7faa3063f3a14705e8156294fb605dab755d7c Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 13:02:49 -0800 Subject: [PATCH 16/39] HDDS-10149. Remove debug flag --- hadoop-hdds/rocks-native/pom.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index 43c4c25e0d61..ecc74e706b1b 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -264,9 +264,7 @@ - - - + From 5a642303034125ed5052d0b92c9f0369ce17e44c Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 13:50:40 -0800 Subject: [PATCH 17/39] HDDS-10149. Adding log --- .../src/test/resources/auditlog.properties | 76 +++++++++++++++++++ .../src/test/resources/log4j.properties | 23 ++++++ 2 files changed, 99 insertions(+) create mode 100644 hadoop-hdds/rocks-native/src/test/resources/auditlog.properties create mode 100644 hadoop-hdds/rocks-native/src/test/resources/log4j.properties diff --git a/hadoop-hdds/rocks-native/src/test/resources/auditlog.properties b/hadoop-hdds/rocks-native/src/test/resources/auditlog.properties new file mode 100644 index 000000000000..959da047fb7f --- /dev/null +++ b/hadoop-hdds/rocks-native/src/test/resources/auditlog.properties @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +#

+# http://www.apache.org/licenses/LICENSE-2.0 +#

+# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS,WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +name=PropertiesConfig + +# Checks for config change periodically and reloads +monitorInterval=5 + +filter=read, write +# filter.read.onMatch = DENY avoids logging all READ events +# filter.read.onMatch = ACCEPT permits logging all READ events +# The above two settings ignore the log levels in configuration +# filter.read.onMatch = NEUTRAL permits logging of only those READ events +# which are attempted at log level equal or greater than log level specified +# in the configuration +filter.read.type = MarkerFilter +filter.read.marker = READ +filter.read.onMatch = NEUTRAL +filter.read.onMismatch = NEUTRAL + +# filter.write.onMatch = DENY avoids logging all WRITE events +# filter.write.onMatch = ACCEPT permits logging all WRITE events +# The above two settings ignore the log levels in configuration +# filter.write.onMatch = NEUTRAL permits logging of only those WRITE events +# which are attempted at log level equal or greater than log level specified +# in the configuration +filter.write.type = MarkerFilter +filter.write.marker = WRITE +filter.write.onMatch = NEUTRAL +filter.write.onMismatch = NEUTRAL + +# Log Levels are organized from most specific to least: +# OFF (most specific, no logging) +# FATAL (most specific, little data) +# ERROR +# WARN +# INFO +# DEBUG +# TRACE (least specific, a lot of data) +# ALL (least specific, all data) + +appenders = console, audit +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %-5level | %c{1} | %msg%n + +appender.audit.type = File +appender.audit.name = AUDITLOG +appender.audit.fileName=audit.log +appender.audit.layout.type=PatternLayout +appender.audit.layout.pattern= %-5level | %c{1} | %C | %msg%n + +loggers=audit +logger.audit.type=AsyncLogger +logger.audit.name=OMAudit +logger.audit.level = INFO +logger.audit.appenderRefs = audit +logger.audit.appenderRef.file.ref = AUDITLOG + +rootLogger.level = INFO +rootLogger.appenderRefs = stdout +rootLogger.appenderRef.stdout.ref = STDOUT diff --git a/hadoop-hdds/rocks-native/src/test/resources/log4j.properties b/hadoop-hdds/rocks-native/src/test/resources/log4j.properties new file mode 100644 index 000000000000..398786689af3 --- /dev/null +++ b/hadoop-hdds/rocks-native/src/test/resources/log4j.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# log4j configuration used during build and unit tests + +log4j.rootLogger=INFO,stdout +log4j.threshold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n From f4aa7ec370c18b1333e60cc7d47a1533c4d4c791 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 14:01:33 -0800 Subject: [PATCH 18/39] HDDS-10149. Fix bug --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74c4756cfd04..8aae47264ce1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,7 +231,6 @@ jobs: unit: needs: - build-info - - basic runs-on: ubuntu-20.04 timeout-minutes: 150 if: needs.build-info.outputs.needs-unit-check == 'true' From 0e1f50dd868669cb13982af93c9be1508d8dd46e Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 14:03:32 -0800 Subject: [PATCH 19/39] HDDS-10149. Add log --- .../java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java index 1264078c6e01..31663ff329d6 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java @@ -113,7 +113,8 @@ public synchronized boolean loadLibrary(final String libraryName) { System.loadLibrary(libraryName); loaded = true; } catch (Throwable e) { - + LOG.warn("Unable to load library from java.library.path: {}", + libraryName, e); } if (!loaded) { Optional file = copyResourceFromJarToTemp(libraryName); From 209c258911505fbda7342e35de78949fe52b73e6 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 16:00:24 -0800 Subject: [PATCH 20/39] HDDS-10149. Add ssh session --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8aae47264ce1..89948edcfef7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,6 +255,8 @@ jobs: with: distribution: 'temurin' java-version: 8 + - name: Setup upterm session + uses: lhotari/action-upterm@v1 - name: Execute tests run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh continue-on-error: true From 2e085162100b6c950d6c4b9b48b0cfbf40e15961 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 22:57:48 -0800 Subject: [PATCH 21/39] Revert "HDDS-10149. Fix bug" This reverts commit f4aa7ec370c18b1333e60cc7d47a1533c4d4c791. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89948edcfef7..41802d6e5564 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,6 +231,7 @@ jobs: unit: needs: - build-info + - basic runs-on: ubuntu-20.04 timeout-minutes: 150 if: needs.build-info.outputs.needs-unit-check == 'true' From 1a8ac2ba4b3bf2f80c3b00615281bc9b2c726dc2 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 22:58:02 -0800 Subject: [PATCH 22/39] Revert "HDDS-10149. Add ssh session" This reverts commit 209c258911505fbda7342e35de78949fe52b73e6. --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41802d6e5564..74c4756cfd04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,8 +256,6 @@ jobs: with: distribution: 'temurin' java-version: 8 - - name: Setup upterm session - uses: lhotari/action-upterm@v1 - name: Execute tests run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh continue-on-error: true From b3d101e07e22b00ba275b02a92c7554dce05f0fe Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 22:58:24 -0800 Subject: [PATCH 23/39] Revert "HDDS-10149. Add log" This reverts commit 0e1f50dd868669cb13982af93c9be1508d8dd46e. --- .../java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java index 31663ff329d6..1264078c6e01 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java @@ -113,8 +113,7 @@ public synchronized boolean loadLibrary(final String libraryName) { System.loadLibrary(libraryName); loaded = true; } catch (Throwable e) { - LOG.warn("Unable to load library from java.library.path: {}", - libraryName, e); + } if (!loaded) { Optional file = copyResourceFromJarToTemp(libraryName); From 5cd071c1f76f76b3614f9e08051641f2ea2fa230 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 23:03:04 -0800 Subject: [PATCH 24/39] HDDS-10149. Fix native library compile issue for gcc>=5 --- hadoop-hdds/rocks-native/pom.xml | 2 +- hadoop-hdds/rocks-native/src/CMakeLists.txt | 1 + .../org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index ecc74e706b1b..39768de9afb9 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -236,7 +236,7 @@ - + diff --git a/hadoop-hdds/rocks-native/src/CMakeLists.txt b/hadoop-hdds/rocks-native/src/CMakeLists.txt index 5c3a31717cf5..f8bf4c0bdafb 100644 --- a/hadoop-hdds/rocks-native/src/CMakeLists.txt +++ b/hadoop-hdds/rocks-native/src/CMakeLists.txt @@ -21,6 +21,7 @@ # cmake_minimum_required(VERSION 2.8) +add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") project(ozone_native) diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java index 4edce046fbb0..b91f56111738 100644 --- a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java @@ -70,7 +70,6 @@ public void testNativeLibraryLoader( .thenReturn(nativeLibraryDirectoryLocation); mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getInstance()) .thenReturn(loader); - assertTrue(NativeLibraryLoader.getInstance() .loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); assertTrue(NativeLibraryLoader From ef4b1476637522ca9336f145c2c4bb68e695bf45 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 23:13:33 -0800 Subject: [PATCH 25/39] HDDS-10149. Mark integration test healthy --- .../org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java | 2 +- .../ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index d77778ca3b27..029b0813bb55 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -226,7 +226,7 @@ private void init() throws Exception { // stop the deletion services so that keys can still be read stopKeyManager(); -// preFinalizationChecks(); + preFinalizationChecks(); finalizeOMUpgrade(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java index 5ed2f848aed8..fcc975cd18f1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java @@ -30,7 +30,6 @@ */ @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) @Timeout(300) -@Unhealthy("HDDS-10149") class TestOmSnapshotFsoWithNativeLib extends TestOmSnapshot { TestOmSnapshotFsoWithNativeLib() throws Exception { super(FILE_SYSTEM_OPTIMIZED, false, false, false); From f324d32e0866ae708de906fe652daf850a7073de Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 Feb 2024 23:21:18 -0800 Subject: [PATCH 26/39] HDDS-10149. Fix checkstyle --- .../hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java index fcc975cd18f1..06fbebb2efa2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java @@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.om.snapshot; import org.apache.ozone.test.tag.Native; -import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.Timeout; import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; From 76a5df6face00b8e32015e04aab84b5413bfc750 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 15 Feb 2024 07:50:09 -0800 Subject: [PATCH 27/39] HDDS-10149. Fix SnapshotDiffManager --- .../apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index e46295ae9d72..2140625bc910 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -1071,7 +1071,7 @@ void addToObjectIdMap(Table fsTable, sstFileReaderUpperBound = String.valueOf(upperBoundCharArray); } try (Stream keysToCheck = - nativeRocksToolsLoaded && isNativeLibsLoaded + nativeRocksToolsLoaded ? sstFileReader.getKeyStreamWithTombstone( sstFileReaderLowerBound, sstFileReaderUpperBound) : sstFileReader.getKeyStream(sstFileReaderLowerBound, From b5d464674638be597b09fa9b29b716606d072aa9 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 15 Feb 2024 08:21:22 -0800 Subject: [PATCH 28/39] HDDS-10149. Fix TestSnapshotDiffManager::testGetSnapshotDiffReportJob --- .../hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index c40543824b1a..38f9ed45eb0d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -65,7 +65,6 @@ import org.apache.ozone.rocksdiff.DifferSnapshotInfo; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; import org.apache.ozone.rocksdiff.RocksDiffUtils; -import org.apache.ozone.test.tag.Unhealthy; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.TimeDuration; import jakarta.annotation.Nonnull; @@ -376,6 +375,7 @@ public OmSnapshot load(@Nonnull String key) { }; omSnapshotManager = mock(OmSnapshotManager.class); + when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager); when(omSnapshotManager.isSnapshotStatus( any(), any())).thenReturn(true); snapshotCache = new SnapshotCache(omSnapshotManager, loader, 10); @@ -383,6 +383,7 @@ public OmSnapshot load(@Nonnull String key) { snapshotDiffManager = new SnapshotDiffManager(db, differ, ozoneManager, snapshotCache, snapDiffJobTable, snapDiffReportTable, columnFamilyOptions, codecRegistry); + when(omSnapshotManager.getDiffCleanupServiceInterval()).thenReturn(0L); } @AfterEach @@ -1546,14 +1547,12 @@ public void testGetSnapshotDiffReportHappyCase() throws Exception { * Tests that only QUEUED jobs are submitted to the executor and rest are * short-circuited based on previous one. */ - @Unhealthy @Test public void testGetSnapshotDiffReportJob() throws Exception { for (int i = 0; i < jobStatuses.size(); i++) { uploadSnapshotDiffJobToDb(snapshotInfo, snapshotInfoList.get(i), snapDiffJobs.get(i)); } - SnapshotDiffManager spy = spy(snapshotDiffManager); doAnswer(invocation -> { From 66921e9d5cc33c412aed8e4077cc59aa0fc41f73 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 15 Feb 2024 08:44:12 -0800 Subject: [PATCH 29/39] HDDS-10149. Unmark unhealthy snapshots native tests --- .../hdds/utils/TestNativeLibraryLoader.java | 9 ++- .../TestManagedRawSSTFileReaderIterator.java | 3 - .../rocksdb/util/TestSstFileSetReader.java | 62 +++++++------------ 3 files changed, 27 insertions(+), 47 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java index b91f56111738..a8b24789dea8 100644 --- a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdds.utils; -import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; import org.apache.ozone.test.tag.Native; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -58,8 +58,8 @@ private static Stream nativeLibraryDirectoryLocations() { @ParameterizedTest @MethodSource("nativeLibraryDirectoryLocations") public void testNativeLibraryLoader( - String nativeLibraryDirectoryLocation) { - ManagedRocksObjectUtils.loadRocksDBLibrary(); + String nativeLibraryDirectoryLocation) + throws NativeLibraryNotLoadedException { Map libraryLoadedMap = new HashMap<>(); NativeLibraryLoader loader = new NativeLibraryLoader(libraryLoadedMap); try (MockedStatic mockedNativeLibraryLoader = @@ -70,8 +70,7 @@ public void testNativeLibraryLoader( .thenReturn(nativeLibraryDirectoryLocation); mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getInstance()) .thenReturn(loader); - assertTrue(NativeLibraryLoader.getInstance() - .loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); + ManagedRawSSTFileReader.loadLibrary(); assertTrue(NativeLibraryLoader .isLibraryLoaded(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); // Mocking to force copy random bytes to create a lib file to diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java index e8b5b04eb5f4..af55e8ee478a 100644 --- a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java @@ -20,7 +20,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; -import org.apache.hadoop.hdds.utils.NativeLibraryLoader; import org.apache.hadoop.hdds.utils.TestUtils; import org.apache.ozone.test.tag.Native; import org.junit.jupiter.api.Named; @@ -45,7 +44,6 @@ import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; /** * Test for ManagedRawSSTFileReaderIterator. @@ -123,7 +121,6 @@ public void testSSTDumpIteratorWithKeyFormat(String keyFormat, String valueFormat) throws Exception { ManagedRocksObjectUtils.loadRocksDBLibrary(); - assumeTrue(NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); TreeMap, String> keys = IntStream.range(0, 100).boxed().collect( diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java index aa8813ccd867..1031992f3b5d 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestSstFileSetReader.java @@ -17,16 +17,15 @@ */ package org.apache.ozone.rocksdb.util; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; -import org.apache.hadoop.hdds.utils.NativeLibraryLoader; +import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.apache.hadoop.hdds.utils.TestUtils; import org.apache.hadoop.hdds.utils.db.managed.ManagedEnvOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileWriter; import org.apache.ozone.test.tag.Native; -import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -40,10 +39,6 @@ import java.util.Optional; import java.util.SortedMap; import java.util.TreeMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -162,49 +157,38 @@ public void testGetKeyStream(int numberOfFiles) @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) @ParameterizedTest @ValueSource(ints = {0, 1, 2, 3, 7, 10}) - @Unhealthy("HDDS-9274") public void testGetKeyStreamWithTombstone(int numberOfFiles) - throws RocksDBException, IOException { - assumeTrue(NativeLibraryLoader.getInstance() - .loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); + throws RocksDBException, IOException, NativeLibraryNotLoadedException { + assumeTrue(ManagedRawSSTFileReader.loadLibrary()); Pair, List> data = createDummyData(numberOfFiles); List files = data.getRight(); SortedMap keys = data.getLeft(); - ExecutorService executorService = new ThreadPoolExecutor(0, - 2, 60, TimeUnit.SECONDS, - new SynchronousQueue<>(), new ThreadFactoryBuilder() - .setNameFormat("snapshot-diff-manager-sst-dump-tool-TID-%d") - .build(), new ThreadPoolExecutor.DiscardPolicy()); // Getting every possible combination of 2 elements from the sampled keys. // Reading the sst file lying within the given bounds and // validating the keys read from the sst file. List> bounds = TestUtils.getTestingBounds(keys); - try { - for (Optional lowerBound : bounds) { - for (Optional upperBound : bounds) { - // Calculating the expected keys which lie in the given boundary. - Map keysInBoundary = - keys.entrySet().stream().filter(entry -> lowerBound - .map(l -> entry.getKey().compareTo(l) >= 0) - .orElse(true) && - upperBound.map(u -> entry.getKey().compareTo(u) < 0) - .orElse(true)) - .collect(Collectors.toMap(Map.Entry::getKey, - Map.Entry::getValue)); - try (Stream keyStream = new SstFileSetReader(files) - .getKeyStreamWithTombstone(lowerBound.orElse(null), - upperBound.orElse(null))) { - keyStream.forEach( - key -> { - assertNotNull(keysInBoundary.remove(key)); - }); - } - assertEquals(0, keysInBoundary.size()); + for (Optional lowerBound : bounds) { + for (Optional upperBound : bounds) { + // Calculating the expected keys which lie in the given boundary. + Map keysInBoundary = + keys.entrySet().stream().filter(entry -> lowerBound + .map(l -> entry.getKey().compareTo(l) >= 0) + .orElse(true) && + upperBound.map(u -> entry.getKey().compareTo(u) < 0) + .orElse(true)) + .collect(Collectors.toMap(Map.Entry::getKey, + Map.Entry::getValue)); + try (Stream keyStream = new SstFileSetReader(files) + .getKeyStreamWithTombstone(lowerBound.orElse(null), + upperBound.orElse(null))) { + keyStream.forEach( + key -> { + assertNotNull(keysInBoundary.remove(key)); + }); } + assertEquals(0, keysInBoundary.size()); } - } finally { - executorService.shutdown(); } } } From c0752e1e55204cc0c6d75fbc3679e0372a66877a Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 15 Feb 2024 09:27:14 -0800 Subject: [PATCH 30/39] HDDS-10149. Fix checkstyle --- .../hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 8fa7ca9b4fae..543212666e4c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -21,12 +21,10 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.CodecRegistry; import org.apache.hadoop.hdds.utils.db.RDBStore; From da803397ca986edb7b8fd52bb920dab83cb237db Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 15 Feb 2024 11:00:37 -0800 Subject: [PATCH 31/39] HDDS-10149. Fix testcase --- .../utils/db/managed/TestManagedRawSSTFileReaderIterator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java index af55e8ee478a..82994243dab8 100644 --- a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java @@ -120,7 +120,7 @@ private static Stream keyValueFormatArgs() { public void testSSTDumpIteratorWithKeyFormat(String keyFormat, String valueFormat) throws Exception { - ManagedRocksObjectUtils.loadRocksDBLibrary(); + ManagedRawSSTFileReader.loadLibrary(); TreeMap, String> keys = IntStream.range(0, 100).boxed().collect( From 03f22f960971b4a9befd58f41c77f2389cd41d4f Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 21 Feb 2024 16:34:58 -0800 Subject: [PATCH 32/39] HDDS-10149. Address review comments --- hadoop-hdds/rocks-native/pom.xml | 6 +- hadoop-hdds/rocks-native/src/CMakeLists.txt | 2 +- .../hadoop/hdds/utils/NativeConstants.java | 3 +- ...or.java => ManagedRawSSTFileIterator.java} | 10 +- .../db/managed/ManagedRawSSTFileReader.java | 36 +-- ...ator.cpp => ManagedRawSSTFileIterator.cpp} | 18 +- .../main/native/ManagedRawSSTFileReader.cpp | 32 +- .../src/main/patches/rocks-native.patch | 294 ++++++++---------- .../hdds/utils/TestNativeLibraryLoader.java | 30 +- ...ava => TestManagedRawSSTFileIterator.java} | 127 +++----- .../ozone/rocksdb/util/SstFileSetReader.java | 193 ++++++------ hadoop-ozone/dev-support/checks/native.sh | 17 +- .../om/snapshot/SnapshotDiffManager.java | 21 +- 13 files changed, 339 insertions(+), 450 deletions(-) rename hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/{ManagedRawSSTFileReaderIterator.java => ManagedRawSSTFileIterator.java} (91%) rename hadoop-hdds/rocks-native/src/main/native/{ManagedRawSSTFileReaderIterator.cpp => ManagedRawSSTFileIterator.cpp} (92%) rename hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/{TestManagedRawSSTFileReaderIterator.java => TestManagedRawSSTFileIterator.java} (53%) diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index 39768de9afb9..7d1b80ed4df2 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -135,7 +135,7 @@ read-property-from-file - initialize + initialize read-project-properties @@ -153,7 +153,7 @@ unpack-dependency - initialize + initialize unpack @@ -337,7 +337,7 @@ ${env.JAVA_HOME}/bin/javah org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader - org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReaderIterator + org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileIterator ${project.build.directory}/native/javah diff --git a/hadoop-hdds/rocks-native/src/CMakeLists.txt b/hadoop-hdds/rocks-native/src/CMakeLists.txt index f8bf4c0bdafb..4639e2a8c927 100644 --- a/hadoop-hdds/rocks-native/src/CMakeLists.txt +++ b/hadoop-hdds/rocks-native/src/CMakeLists.txt @@ -37,7 +37,7 @@ endif() include_directories(${GENERATED_JAVAH}) if(${SST_DUMP_INCLUDE}) include_directories(${ROCKSDB_HEADERS}) - set(SOURCE_FILES ${NATIVE_DIR}/ManagedRawSSTFileReader.cpp ${NATIVE_DIR}/ManagedRawSSTFileReaderIterator.cpp ${NATIVE_DIR}/cplusplus_to_java_convert.h) + set(SOURCE_FILES ${NATIVE_DIR}/ManagedRawSSTFileReader.cpp ${NATIVE_DIR}/ManagedRawSSTFileIterator.cpp ${NATIVE_DIR}/cplusplus_to_java_convert.h) ADD_LIBRARY(rocksdb SHARED IMPORTED) set_target_properties( rocksdb diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java index d3121144d37a..8937f0803a18 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeConstants.java @@ -26,6 +26,5 @@ public final class NativeConstants { private NativeConstants() { } - public static final String ROCKS_TOOLS_NATIVE_LIBRARY_NAME - = "ozone_rocksdb_tools"; + public static final String ROCKS_TOOLS_NATIVE_LIBRARY_NAME = "ozone_rocksdb_tools"; } diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileIterator.java similarity index 91% rename from hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java rename to hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileIterator.java index 6191befdb78c..02125951c1fe 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReaderIterator.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileIterator.java @@ -22,18 +22,18 @@ import org.apache.hadoop.util.ClosableIterator; import java.util.Arrays; +import java.util.NoSuchElementException; import java.util.function.Function; /** * Iterator for SSTFileReader which would read all entries including tombstones. */ -public class ManagedRawSSTFileReaderIterator - implements ClosableIterator { +public class ManagedRawSSTFileIterator implements ClosableIterator { + // Native address of pointer to the object. private final long nativeHandle; private final Function transformer; - ManagedRawSSTFileReaderIterator(long nativeHandle, - Function transformer) { + ManagedRawSSTFileIterator(long nativeHandle, Function transformer) { this.nativeHandle = nativeHandle; this.transformer = transformer; } @@ -53,7 +53,7 @@ public boolean hasNext() { @Override public T next() { if (!hasNext()) { - return null; + throw new NoSuchElementException(); } KeyValue keyValue = new KeyValue(this.getKey(nativeHandle), diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java index 57b6e574a873..7c8783b43948 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRawSSTFileReader.java @@ -34,53 +34,39 @@ public class ManagedRawSSTFileReader implements Closeable { public static boolean loadLibrary() throws NativeLibraryNotLoadedException { ManagedRocksObjectUtils.loadRocksDBLibrary(); - if (!NativeLibraryLoader.getInstance() - .loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)) { - throw new NativeLibraryNotLoadedException( - ROCKS_TOOLS_NATIVE_LIBRARY_NAME); + if (!NativeLibraryLoader.getInstance().loadLibrary(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)) { + throw new NativeLibraryNotLoadedException(ROCKS_TOOLS_NATIVE_LIBRARY_NAME); } return true; } - private final ManagedOptions options; private final String fileName; - private final int readAheadSize; + // Native address of pointer to the object. private final long nativeHandle; - private static final Logger LOG = - LoggerFactory.getLogger(ManagedRawSSTFileReader.class); + private static final Logger LOG = LoggerFactory.getLogger(ManagedRawSSTFileReader.class); - public ManagedRawSSTFileReader(final ManagedOptions options, - final String fileName, - final int readAheadSize) { - this.options = options; + public ManagedRawSSTFileReader(final ManagedOptions options, final String fileName, final int readAheadSize) { this.fileName = fileName; - this.readAheadSize = readAheadSize; - this.nativeHandle = this.newRawSSTFileReader(options.getNativeHandle(), - fileName, readAheadSize); + this.nativeHandle = this.newRawSSTFileReader(options.getNativeHandle(), fileName, readAheadSize); } - public ManagedRawSSTFileReaderIterator newIterator( - Function transformerFunction, + public ManagedRawSSTFileIterator newIterator( + Function transformerFunction, ManagedSlice fromSlice, ManagedSlice toSlice) { long fromNativeHandle = fromSlice == null ? 0 : fromSlice.getNativeHandle(); long toNativeHandle = toSlice == null ? 0 : toSlice.getNativeHandle(); LOG.info("Iterating SST file: {} with native lib. " + "LowerBound: {}, UpperBound: {}", fileName, fromSlice, toSlice); - return new ManagedRawSSTFileReaderIterator<>( + return new ManagedRawSSTFileIterator<>( newIterator(this.nativeHandle, fromSlice != null, fromNativeHandle, toSlice != null, toNativeHandle), transformerFunction); } - private native long newRawSSTFileReader(long optionsHandle, - String filePath, - int readSize); + private native long newRawSSTFileReader(long optionsHandle, String filePath, int readSize); - private native long newIterator(long handle, - boolean hasFrom, - long fromSliceHandle, - boolean hasTo, + private native long newIterator(long handle, boolean hasFrom, long fromSliceHandle, boolean hasTo, long toSliceHandle); private native void disposeInternal(long handle); diff --git a/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReaderIterator.cpp b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileIterator.cpp similarity index 92% rename from hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReaderIterator.cpp rename to hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileIterator.cpp index cc132162f16a..440dc51a7610 100644 --- a/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReaderIterator.cpp +++ b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileIterator.cpp @@ -16,24 +16,24 @@ * limitations under the License. */ -#include "org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator.h" +#include "org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator.h" #include "rocksdb/options.h" #include "rocksdb/raw_iterator.h" #include #include "cplusplus_to_java_convert.h" #include -jboolean Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_hasNext(JNIEnv *env, jobject obj, +jboolean Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_hasNext(JNIEnv *env, jobject obj, jlong native_handle) { return static_cast(reinterpret_cast(native_handle)->has_next()); } -void Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_next(JNIEnv *env, jobject obj, +void Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_next(JNIEnv *env, jobject obj, jlong native_handle) { reinterpret_cast(native_handle)->next(); } -jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_getKey(JNIEnv *env, +jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_getKey(JNIEnv *env, jobject obj, jlong native_handle) { ROCKSDB_NAMESPACE::Slice slice = reinterpret_cast(native_handle)->getKey(); @@ -49,7 +49,7 @@ jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderI } -jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_getValue(JNIEnv *env, +jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_getValue(JNIEnv *env, jobject obj, jlong native_handle) { ROCKSDB_NAMESPACE::Slice slice = reinterpret_cast(native_handle)->getValue(); @@ -64,7 +64,7 @@ jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderI return jkey; } -jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_getSequenceNumber(JNIEnv *env, +jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_getSequenceNumber(JNIEnv *env, jobject obj, jlong native_handle) { uint64_t sequence_number = @@ -75,7 +75,7 @@ jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterat } -jint Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_getType(JNIEnv *env, +jint Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_getType(JNIEnv *env, jobject obj, jlong native_handle) { uint32_t type = reinterpret_cast(native_handle)->getType(); @@ -83,8 +83,8 @@ jint Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterato } -void Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReaderIterator_closeInternal(JNIEnv *env, +void Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_closeInternal(JNIEnv *env, jobject obj, jlong native_handle) { delete reinterpret_cast(native_handle); -} \ No newline at end of file +} diff --git a/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReader.cpp b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReader.cpp index 992b66a7632b..f3b8dc02639d 100644 --- a/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReader.cpp +++ b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileReader.cpp @@ -25,37 +25,37 @@ #include jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReader_newRawSSTFileReader(JNIEnv *env, jobject obj, - jlong optionsHandle, - jstring jfilename, - jint readahead_size) { - ROCKSDB_NAMESPACE::Options *options = reinterpret_cast(optionsHandle); + jlong options_handle, + jstring jfilename, + jint readahead_size) { + ROCKSDB_NAMESPACE::Options *options = reinterpret_cast(options_handle); const char *file_path = env->GetStringUTFChars(jfilename, nullptr); - size_t readAheadSizeValue = static_cast(readahead_size); - ROCKSDB_NAMESPACE::RawSstFileReader* rawSstFileReader = - new ROCKSDB_NAMESPACE::RawSstFileReader(*options, file_path, readAheadSizeValue, true, true); + size_t read_ahead_size_value = static_cast(readahead_size); + ROCKSDB_NAMESPACE::RawSstFileReader* raw_sst_file_reader = + new ROCKSDB_NAMESPACE::RawSstFileReader(*options, file_path, read_ahead_size_value, true, true); env->ReleaseStringUTFChars(jfilename, file_path); - return GET_CPLUSPLUS_POINTER(rawSstFileReader); + return GET_CPLUSPLUS_POINTER(raw_sst_file_reader); } jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileReader_newIterator(JNIEnv *env, jobject obj, jlong native_handle, jboolean jhas_from, - jlong fromSliceHandle, + jlong from_slice_handle, jboolean jhas_to, - jlong toSliceHandle) { - ROCKSDB_NAMESPACE::Slice* fromSlice = nullptr; - ROCKSDB_NAMESPACE::Slice* toSlice = nullptr; - ROCKSDB_NAMESPACE::RawSstFileReader* rawSstFileReader = + jlong to_slice_handle) { + ROCKSDB_NAMESPACE::Slice* from_slice = nullptr; + ROCKSDB_NAMESPACE::Slice* to_slice = nullptr; + ROCKSDB_NAMESPACE::RawSstFileReader* raw_sst_file_reader = reinterpret_cast(native_handle); bool has_from = static_cast(jhas_from); bool has_to = static_cast(jhas_to); if (has_from) { - fromSlice = reinterpret_cast(fromSliceHandle); + from_slice = reinterpret_cast(from_slice_handle); } if (has_to) { - toSlice = reinterpret_cast(toSliceHandle); + to_slice = reinterpret_cast(to_slice_handle); } - ROCKSDB_NAMESPACE::RawIterator* iterator = rawSstFileReader->newIterator(has_from, fromSlice, has_to, toSlice); + ROCKSDB_NAMESPACE::RawIterator* iterator = raw_sst_file_reader->newIterator(has_from, from_slice, has_to, to_slice); return GET_CPLUSPLUS_POINTER(iterator); } diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index b56fcefd6933..c4e281d16c8e 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -116,7 +116,7 @@ index 000000000..266a0c1c8 + +#endif // ROCKSDB_LITE diff --git a/src.mk b/src.mk -index b94bc43ca..95bfff274 100644 +index b94bc43ca..39080b2cf 100644 --- a/src.mk +++ b/src.mk @@ -343,6 +343,8 @@ TOOL_LIB_SOURCES = \ @@ -124,13 +124,146 @@ index b94bc43ca..95bfff274 100644 tools/sst_dump_tool.cc \ utilities/blob_db/blob_dump_tool.cc \ + tools/raw_sst_file_reader.cc \ -+ tools/raw_sst_file_reader_iterator.cc \ ++ tools/raw_sst_file_iterator.cc \ ANALYZER_LIB_SOURCES = \ tools/block_cache_analyzer/block_cache_trace_analyzer.cc \ +diff --git a/tools/raw_sst_file_iterator.cc b/tools/raw_sst_file_iterator.cc +new file mode 100644 +index 000000000..5527b84a4 +--- /dev/null ++++ b/tools/raw_sst_file_iterator.cc +@@ -0,0 +1,76 @@ ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// This source code is licensed under both the GPLv2 (found in the ++// COPYING file in the root directory) and Apache 2.0 License ++// (found in the LICENSE.Apache file in the root directory). ++// ++#ifndef ROCKSDB_LITE ++ ++ ++#include ++#include ++ ++#include "db/memtable.h" ++#include "db/write_batch_internal.h" ++#include "rocksdb/status.h" ++#include "rocksdb/utilities/ldb_cmd.h" ++#include "table/block_based/block.h" ++#include "table/block_based/block_based_table_builder.h" ++#include "table/block_based/block_based_table_factory.h" ++#include "table/meta_blocks.h" ++#include "table/plain/plain_table_factory.h" ++#include "tools/raw_sst_file_iterator.h" ++ ++namespace ROCKSDB_NAMESPACE { ++ ++RawSstFileIterator::RawSstFileIterator(InternalIterator* iterator, ++ bool has_from, Slice* from_key, ++ bool has_to, Slice* to_key) ++ : iter_(iterator), ++ ikey(new ParsedInternalKey()), ++ has_to_(has_to), ++ to_key_(to_key) { ++ if (has_from) { ++ InternalKey k; ++ k.SetMinPossibleForUserKey(*from_key); ++ iter_->Seek(k.Encode()); ++ } else { ++ iter_->SeekToFirst(); ++ } ++ initKey(); ++} ++ ++bool RawSstFileIterator::has_next() const { ++ return iter_->Valid() && (!has_to_ || ++ BytewiseComparator()->Compare( ++ getKey(), *to_key_) < 0); ++} ++ ++void RawSstFileIterator::initKey() { ++ if (iter_->Valid()) { ++ ParseInternalKey(iter_->key(), ikey, true /* log_err_key */); ++ } ++} ++void RawSstFileIterator::next() { ++ iter_->Next(); ++ initKey(); ++ ++} ++ ++Slice RawSstFileIterator::getKey() const { ++ return ikey->user_key; ++} ++ ++uint64_t RawSstFileIterator::getSequenceNumber() const { ++ return ikey->sequence; ++} ++ ++uint32_t RawSstFileIterator::getType() const { ++ return static_cast(ikey->type); ++} ++ ++Slice RawSstFileIterator::getValue() const { ++ return iter_->value(); ++} ++} // namespace ROCKSDB_NAMESPACE ++ ++#endif // ROCKSDB_LITE +diff --git a/tools/raw_sst_file_iterator.h b/tools/raw_sst_file_iterator.h +new file mode 100644 +index 000000000..ea7de6763 +--- /dev/null ++++ b/tools/raw_sst_file_iterator.h +@@ -0,0 +1,45 @@ ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// This source code is licensed under both the GPLv2 (found in the ++// COPYING file in the root directory) and Apache 2.0 License ++// (found in the LICENSE.Apache file in the root directory). ++#pragma once ++#ifndef ROCKSDB_LITE ++ ++#include ++#include ++#include "file/writable_file_writer.h" ++#include "rocksdb/advanced_options.h" ++#include "rocksdb/raw_iterator.h" ++ ++namespace ROCKSDB_NAMESPACE { ++ ++class RawSstFileIterator : public RawIterator { ++ public: ++ explicit RawSstFileIterator(InternalIterator* iterator, ++ bool has_from, ++ Slice* from_key, ++ bool has_to, ++ Slice* to_key); ++ ++ bool has_next() const override; ++ Slice getKey() const override; ++ Slice getValue() const override; ++ uint64_t getSequenceNumber() const override; ++ uint32_t getType() const override; ++ void next() final override; ++ ++ ~RawSstFileIterator(){ ++ delete iter_; ++ } ++ ++ private: ++ void initKey(); ++ InternalIterator* iter_; ++ ParsedInternalKey* ikey; ++ bool has_to_; ++ Slice* to_key_; ++}; ++ ++} // namespace ROCKSDB_NAMESPACE ++ ++#endif // ROCKSDB_LITE diff --git a/tools/raw_sst_file_reader.cc b/tools/raw_sst_file_reader.cc new file mode 100644 -index 000000000..1693bd1e6 +index 000000000..220c814d0 --- /dev/null +++ b/tools/raw_sst_file_reader.cc @@ -0,0 +1,285 @@ @@ -171,7 +304,7 @@ index 000000000..1693bd1e6 +#include "table/meta_blocks.h" +#include "table/plain/plain_table_factory.h" +#include "table/table_reader.h" -+#include "tools/raw_sst_file_reader_iterator.h" ++#include "tools/raw_sst_file_iterator.h" +#include "util/compression.h" +#include "util/random.h" +#include "db/dbformat.h" @@ -413,160 +546,9 @@ index 000000000..1693bd1e6 + rep_->read_options_, rep_->moptions_.prefix_extractor.get(), + /*arena=*/nullptr, /*skip_filters=*/false, + TableReaderCaller::kSSTDumpTool); -+ return new RawSstFileReaderIterator(iter, has_from, from, has_to, to); -+ -+} -+} // namespace ROCKSDB_NAMESPACE -+ -+#endif // ROCKSDB_LITE -diff --git a/tools/raw_sst_file_reader_iterator.cc b/tools/raw_sst_file_reader_iterator.cc -new file mode 100644 -index 000000000..39a2b84be ---- /dev/null -+++ b/tools/raw_sst_file_reader_iterator.cc -@@ -0,0 +1,94 @@ -+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -+// This source code is licensed under both the GPLv2 (found in the -+// COPYING file in the root directory) and Apache 2.0 License -+// (found in the LICENSE.Apache file in the root directory). -+// -+#ifndef ROCKSDB_LITE -+ -+#include "tools/raw_sst_file_reader_iterator.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "db/blob/blob_index.h" -+#include "db/memtable.h" -+#include "db/write_batch_internal.h" -+#include "options/cf_options.h" -+#include "port/port.h" -+#include "rocksdb/db.h" -+#include "rocksdb/env.h" -+#include "rocksdb/iterator.h" -+#include "rocksdb/slice_transform.h" -+#include "rocksdb/status.h" -+#include "rocksdb/table_properties.h" -+#include "rocksdb/utilities/ldb_cmd.h" -+#include "table/block_based/block.h" -+#include "table/block_based/block_based_table_builder.h" -+#include "table/block_based/block_based_table_factory.h" -+#include "table/block_based/block_builder.h" -+#include "table/format.h" -+#include "table/meta_blocks.h" -+#include "table/plain/plain_table_factory.h" -+#include "table/table_reader.h" -+#include "util/compression.h" -+#include "util/random.h" -+ -+namespace ROCKSDB_NAMESPACE { ++ return new RawSstFileIterator(iter, has_from, from, has_to, to); + -+RawSstFileReaderIterator::RawSstFileReaderIterator(InternalIterator* iterator, -+ bool has_from, Slice* from_key, -+ bool has_to, Slice* to_key) -+ : iter_(iterator), -+ ikey(new ParsedInternalKey()), -+ has_to_(has_to), -+ to_key_(to_key) { -+ if (has_from) { -+ InternalKey k; -+ k.SetMinPossibleForUserKey(*from_key); -+ iter_->Seek(k.Encode()); -+ } else { -+ iter_->SeekToFirst(); -+ } -+ initKey(); -+} -+ -+bool RawSstFileReaderIterator::has_next() const { -+ return iter_->Valid() && (!has_to_ || -+ BytewiseComparator()->Compare( -+ getKey(), *to_key_) < 0); -+} -+ -+void RawSstFileReaderIterator::initKey() { -+ if (iter_->Valid()) { -+ ParseInternalKey(iter_->key(), ikey, true /* log_err_key */); -+ } +} -+void RawSstFileReaderIterator::next() { -+ iter_->Next(); -+ initKey(); -+ -+} -+ -+Slice RawSstFileReaderIterator::getKey() const { -+ return ikey->user_key; -+} -+ -+uint64_t RawSstFileReaderIterator::getSequenceNumber() const { -+ return ikey->sequence; -+} -+ -+uint32_t RawSstFileReaderIterator::getType() const { -+ return static_cast(ikey->type); -+} -+ -+Slice RawSstFileReaderIterator::getValue() const { -+ return iter_->value(); -+} -+} // namespace ROCKSDB_NAMESPACE -+ -+#endif // ROCKSDB_LITE -diff --git a/tools/raw_sst_file_reader_iterator.h b/tools/raw_sst_file_reader_iterator.h -new file mode 100644 -index 000000000..79313264f ---- /dev/null -+++ b/tools/raw_sst_file_reader_iterator.h -@@ -0,0 +1,45 @@ -+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -+// This source code is licensed under both the GPLv2 (found in the -+// COPYING file in the root directory) and Apache 2.0 License -+// (found in the LICENSE.Apache file in the root directory). -+#pragma once -+#ifndef ROCKSDB_LITE -+ -+#include -+#include -+#include "file/writable_file_writer.h" -+#include "rocksdb/advanced_options.h" -+#include "rocksdb/raw_iterator.h" -+ -+namespace ROCKSDB_NAMESPACE { -+ -+class RawSstFileReaderIterator : public RawIterator { -+ public: -+ explicit RawSstFileReaderIterator(InternalIterator* iterator, -+ bool has_from, -+ Slice* from_key, -+ bool has_to, -+ Slice* to_key); -+ -+ bool has_next() const override; -+ Slice getKey() const override; -+ Slice getValue() const override; -+ uint64_t getSequenceNumber() const override; -+ uint32_t getType() const override; -+ void next() final override; -+ -+ ~RawSstFileReaderIterator(){ -+ delete iter_; -+ } -+ -+ private: -+ void initKey(); -+ InternalIterator* iter_; -+ ParsedInternalKey* ikey; -+ bool has_to_; -+ Slice* to_key_; -+}; -+ +} // namespace ROCKSDB_NAMESPACE + +#endif // ROCKSDB_LITE diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java index a8b24789dea8..8f18f8d1e423 100644 --- a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java @@ -57,37 +57,27 @@ private static Stream nativeLibraryDirectoryLocations() { @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) @ParameterizedTest @MethodSource("nativeLibraryDirectoryLocations") - public void testNativeLibraryLoader( - String nativeLibraryDirectoryLocation) - throws NativeLibraryNotLoadedException { + public void testNativeLibraryLoader(String nativeLibraryDirectoryLocation) throws NativeLibraryNotLoadedException { Map libraryLoadedMap = new HashMap<>(); NativeLibraryLoader loader = new NativeLibraryLoader(libraryLoadedMap); - try (MockedStatic mockedNativeLibraryLoader = - mockStatic(NativeLibraryLoader.class, - CALLS_REAL_METHODS)) { - mockedNativeLibraryLoader.when(() -> - NativeLibraryLoader.getSystemProperty(same(NATIVE_LIB_TMP_DIR))) + try (MockedStatic mockedNativeLibraryLoader = mockStatic(NativeLibraryLoader.class, + CALLS_REAL_METHODS)) { + mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getSystemProperty(same(NATIVE_LIB_TMP_DIR))) .thenReturn(nativeLibraryDirectoryLocation); - mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getInstance()) - .thenReturn(loader); + mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getInstance()).thenReturn(loader); ManagedRawSSTFileReader.loadLibrary(); - assertTrue(NativeLibraryLoader - .isLibraryLoaded(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); + assertTrue(NativeLibraryLoader.isLibraryLoaded(ROCKS_TOOLS_NATIVE_LIBRARY_NAME)); // Mocking to force copy random bytes to create a lib file to // nativeLibraryDirectoryLocation. But load library will fail. - mockedNativeLibraryLoader.when(() -> - NativeLibraryLoader.getResourceStream(anyString())) + mockedNativeLibraryLoader.when(() -> NativeLibraryLoader.getResourceStream(anyString())) .thenReturn(new ByteArrayInputStream(new byte[]{0, 1, 2, 3})); String dummyLibraryName = "dummy_lib"; NativeLibraryLoader.getInstance().loadLibrary(dummyLibraryName); NativeLibraryLoader.isLibraryLoaded(dummyLibraryName); // Checking if the resource with random was copied to a temp file. - File[] libPath = - new File(nativeLibraryDirectoryLocation == null ? "" : - nativeLibraryDirectoryLocation) - .getAbsoluteFile().listFiles((dir, name) -> - name.startsWith(dummyLibraryName) && - name.endsWith(NativeLibraryLoader.getLibOsSuffix())); + File[] libPath = new File(nativeLibraryDirectoryLocation == null ? "" : nativeLibraryDirectoryLocation) + .getAbsoluteFile().listFiles((dir, name) -> name.startsWith(dummyLibraryName) && + name.endsWith(NativeLibraryLoader.getLibOsSuffix())); assertNotNull(libPath); assertEquals(1, libPath.length); assertTrue(libPath[0].delete()); diff --git a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileIterator.java similarity index 53% rename from hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java rename to hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileIterator.java index 82994243dab8..00816e60d7f2 100644 --- a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileReaderIterator.java +++ b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestManagedRawSSTFileIterator.java @@ -20,8 +20,10 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; +import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.apache.hadoop.hdds.utils.TestUtils; import org.apache.ozone.test.tag.Native; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Named; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -48,7 +50,8 @@ /** * Test for ManagedRawSSTFileReaderIterator. */ -class TestManagedRawSSTFileReaderIterator { +@Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) +class TestManagedRawSSTFileIterator { @TempDir private Path tempDir; @@ -58,16 +61,13 @@ private File createSSTFileWithKeys( File file = Files.createFile(tempDir.resolve("tmp_sst_file.sst")).toFile(); try (ManagedEnvOptions envOptions = new ManagedEnvOptions(); ManagedOptions managedOptions = new ManagedOptions(); - ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter( - envOptions, managedOptions)) { + ManagedSstFileWriter sstFileWriter = new ManagedSstFileWriter(envOptions, managedOptions)) { sstFileWriter.open(file.getAbsolutePath()); for (Map.Entry, String> entry : keys.entrySet()) { if (entry.getKey().getValue() == 0) { - sstFileWriter.delete(entry.getKey().getKey() - .getBytes(StandardCharsets.UTF_8)); + sstFileWriter.delete(entry.getKey().getKey().getBytes(StandardCharsets.UTF_8)); } else { - sstFileWriter.put(entry.getKey().getKey() - .getBytes(StandardCharsets.UTF_8), + sstFileWriter.put(entry.getKey().getKey().getBytes(StandardCharsets.UTF_8), entry.getValue().getBytes(StandardCharsets.UTF_8)); } } @@ -77,91 +77,58 @@ private File createSSTFileWithKeys( } private static Stream keyValueFormatArgs() { - return Stream.of( - Arguments.of( - Named.of("Key starting with a single quote", - "'key%1$d=>"), - Named.of("Value starting with a number ending with a" + - " single quote", "%1$dvalue'") - ), - Arguments.of( - Named.of("Key ending with a number", "key%1$d"), - Named.of("Value starting & ending with a number", "%1$dvalue%1$d") - ), - Arguments.of( - Named.of("Key starting with a single quote & ending" + - " with a number", "'key%1$d"), - Named.of("Value starting & ending with a number " + - "& elosed within quotes", "%1$d'value%1$d'")), - Arguments.of( - Named.of("Key starting with a single quote & ending" + - " with a number", "'key%1$d"), - Named.of("Value starting & ending with a number " + - "& elosed within quotes", "%1$d'value%1$d'") - ), - Arguments.of( - Named.of("Key ending with a number", "key%1$d"), - Named.of("Value starting & ending with a number " + - "& containing null character & new line character", - "%1$dvalue\n\0%1$d") - ), - Arguments.of( - Named.of("Key ending with a number & containing" + - " a null character", "key\0%1$d"), - Named.of("Value starting & ending with a number " + - "& elosed within quotes", "%1$dvalue\r%1$d") - ) - ); + return Stream.of(Arguments.of(Named.of("Key starting with a single quote", "'key%1$d=>"), + Named.of("Value starting with a number ending with a single quote", "%1$dvalue'")), + Arguments.of(Named.of("Key ending with a number", "key%1$d"), + Named.of("Value starting & ending with a number", "%1$dvalue%1$d")), + Arguments.of(Named.of("Key starting with a single quote & ending with a number", "'key%1$d"), + Named.of("Value starting & ending with a number & elosed within quotes", "%1$d'value%1$d'")), + Arguments.of(Named.of("Key starting with a single quote & ending with a number", "'key%1$d"), + Named.of("Value starting & ending with a number & elosed within quotes", "%1$d'value%1$d'")), + Arguments.of(Named.of("Key ending with a number", "key%1$d"), + Named.of("Value starting & ending with a number & containing null character & new line character", + "%1$dvalue\n\0%1$d")), + Arguments.of(Named.of("Key ending with a number & containing a null character", "key\0%1$d"), + Named.of("Value starting & ending with a number & elosed within quotes", "%1$dvalue\r%1$d"))); } - @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) - @ParameterizedTest - @MethodSource("keyValueFormatArgs") - public void testSSTDumpIteratorWithKeyFormat(String keyFormat, - String valueFormat) - throws Exception { + @BeforeAll + public static void init() throws NativeLibraryNotLoadedException { ManagedRawSSTFileReader.loadLibrary(); + } - TreeMap, String> keys = - IntStream.range(0, 100).boxed().collect( - Collectors.toMap( - i -> Pair.of(String.format(keyFormat, i), i % 2), - i -> i % 2 == 0 ? "" : String.format(valueFormat, i), - (v1, v2) -> v2, - TreeMap::new)); - File file = createSSTFileWithKeys(keys); + @ParameterizedTest + @MethodSource("keyValueFormatArgs") + public void testSSTDumpIteratorWithKeyFormat(String keyFormat, String valueFormat) throws Exception { + TreeMap, String> keys = IntStream.range(0, 100).boxed().collect(Collectors.toMap( + i -> Pair.of(String.format(keyFormat, i), i % 2), + i -> i % 2 == 0 ? "" : String.format(valueFormat, i), + (v1, v2) -> v2, + TreeMap::new)); + File file = createSSTFileWithKeys(keys); try (ManagedOptions options = new ManagedOptions(); - ManagedRawSSTFileReader - reader = new ManagedRawSSTFileReader<>(options, - file.getAbsolutePath(), 2 * 1024 * 1024)) { - List> testBounds = TestUtils.getTestingBounds( - keys.keySet().stream().collect(Collectors.toMap(Pair::getKey, - Pair::getValue, (v1, v2) -> v1, TreeMap::new))); + ManagedRawSSTFileReader reader = new ManagedRawSSTFileReader<>( + options, file.getAbsolutePath(), 2 * 1024 * 1024)) { + List> testBounds = TestUtils.getTestingBounds(keys.keySet().stream() + .collect(Collectors.toMap(Pair::getKey, Pair::getValue, (v1, v2) -> v1, TreeMap::new))); for (Optional keyStart : testBounds) { for (Optional keyEnd : testBounds) { - Map, String> expectedKeys = keys.entrySet() - .stream().filter(e -> keyStart.map(s -> e.getKey().getKey() - .compareTo(s) >= 0).orElse(true)) - .filter(e -> keyEnd.map(s -> e.getKey().getKey().compareTo(s) < 0) - .orElse(true)) + Map, String> expectedKeys = keys.entrySet().stream() + .filter(e -> keyStart.map(s -> e.getKey().getKey().compareTo(s) >= 0).orElse(true)) + .filter(e -> keyEnd.map(s -> e.getKey().getKey().compareTo(s) < 0).orElse(true)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - Optional lowerBound = keyStart - .map(s -> new ManagedSlice(StringUtils.string2Bytes(s))); - Optional upperBound = keyEnd - .map(s -> new ManagedSlice(StringUtils.string2Bytes(s))); - try (ManagedRawSSTFileReaderIterator< - ManagedRawSSTFileReaderIterator.KeyValue> iterator - = reader.newIterator(Function.identity(), - lowerBound.orElse(null), upperBound.orElse(null))) { + Optional lowerBound = keyStart.map(s -> new ManagedSlice(StringUtils.string2Bytes(s))); + Optional upperBound = keyEnd.map(s -> new ManagedSlice(StringUtils.string2Bytes(s))); + try (ManagedRawSSTFileIterator iterator + = reader.newIterator(Function.identity(), lowerBound.orElse(null), upperBound.orElse(null))) { while (iterator.hasNext()) { - ManagedRawSSTFileReaderIterator.KeyValue r = iterator.next(); - String key = new String(r.getKey(), StandardCharsets.UTF_8); + ManagedRawSSTFileIterator.KeyValue r = iterator.next(); + String key = StringUtils.bytes2String(r.getKey()); Pair recordKey = Pair.of(key, r.getType()); assertThat(expectedKeys).containsKey(recordKey); - assertEquals(Optional.ofNullable(expectedKeys - .get(recordKey)).orElse(""), - new String(r.getValue(), StandardCharsets.UTF_8)); + assertEquals(Optional.ofNullable(expectedKeys.get(recordKey)).orElse(""), + StringUtils.bytes2String(r.getValue())); expectedKeys.remove(recordKey); } assertEquals(0, expectedKeys.size()); diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java index 96b7bb067005..913eeb73384a 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/SstFileSetReader.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader; -import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReaderIterator; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedSlice; import org.apache.hadoop.util.ClosableIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; @@ -91,105 +91,96 @@ public long getEstimatedTotalKeys() throws RocksDBException { } public Stream getKeyStream(String lowerBound, - String upperBound) throws RocksDBException { + String upperBound) throws RocksDBException { // TODO: [SNAPSHOT] Check if default Options and ReadOptions is enough. - final MultipleSstFileIterator itr = - new MultipleSstFileIterator(sstFiles) { - private ManagedOptions options; - private ReadOptions readOptions; - - private ManagedSlice lowerBoundSLice; - - private ManagedSlice upperBoundSlice; - - @Override - protected void init() { - this.options = new ManagedOptions(); - this.readOptions = new ManagedReadOptions(); - if (Objects.nonNull(lowerBound)) { - this.lowerBoundSLice = new ManagedSlice( - StringUtils.string2Bytes(lowerBound)); - readOptions.setIterateLowerBound(lowerBoundSLice); - } - - if (Objects.nonNull(upperBound)) { - this.upperBoundSlice = new ManagedSlice( - StringUtils.string2Bytes(upperBound)); - readOptions.setIterateUpperBound(upperBoundSlice); - } - } + final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { + private ManagedOptions options; + private ReadOptions readOptions; + + private ManagedSlice lowerBoundSLice; + + private ManagedSlice upperBoundSlice; + + @Override + protected void init() { + this.options = new ManagedOptions(); + this.readOptions = new ManagedReadOptions(); + if (Objects.nonNull(lowerBound)) { + this.lowerBoundSLice = new ManagedSlice( + StringUtils.string2Bytes(lowerBound)); + readOptions.setIterateLowerBound(lowerBoundSLice); + } - @Override - protected ClosableIterator getKeyIteratorForFile(String file) - throws RocksDBException { - return new ManagedSstFileIterator(file, options, readOptions) { - @Override - protected String getIteratorValue( - SstFileReaderIterator iterator) { - return new String(iterator.key(), UTF_8); - } - }; - } + if (Objects.nonNull(upperBound)) { + this.upperBoundSlice = new ManagedSlice( + StringUtils.string2Bytes(upperBound)); + readOptions.setIterateUpperBound(upperBoundSlice); + } + } + @Override + protected ClosableIterator getKeyIteratorForFile(String file) throws RocksDBException { + return new ManagedSstFileIterator(file, options, readOptions) { @Override - public void close() throws UncheckedIOException { - super.close(); - options.close(); - readOptions.close(); - IOUtils.closeQuietly(lowerBoundSLice, upperBoundSlice); + protected String getIteratorValue( + SstFileReaderIterator iterator) { + return new String(iterator.key(), UTF_8); } }; + } + + @Override + public void close() throws UncheckedIOException { + super.close(); + options.close(); + readOptions.close(); + IOUtils.closeQuietly(lowerBoundSLice, upperBoundSlice); + } + }; return getStreamFromIterator(itr); } - public Stream getKeyStreamWithTombstone(String lowerBound, - String upperBound) throws RocksDBException { - final MultipleSstFileIterator itr = - new MultipleSstFileIterator(sstFiles) { - //TODO: [SNAPSHOT] Check if default Options is enough. - private ManagedOptions options; - private ManagedSlice lowerBoundSlice; - private ManagedSlice upperBoundSlice; - - @Override - protected void init() { - this.options = new ManagedOptions(); - if (Objects.nonNull(lowerBound)) { - this.lowerBoundSlice = new ManagedSlice( - StringUtils.string2Bytes(lowerBound)); - } - if (Objects.nonNull(upperBound)) { - this.upperBoundSlice = new ManagedSlice( - StringUtils.string2Bytes(upperBound)); - } - } + public Stream getKeyStreamWithTombstone(String lowerBound, String upperBound) throws RocksDBException { + final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { + //TODO: [SNAPSHOT] Check if default Options is enough. + private ManagedOptions options; + private ManagedSlice lowerBoundSlice; + private ManagedSlice upperBoundSlice; + + @Override + protected void init() { + this.options = new ManagedOptions(); + if (Objects.nonNull(lowerBound)) { + this.lowerBoundSlice = new ManagedSlice( + StringUtils.string2Bytes(lowerBound)); + } + if (Objects.nonNull(upperBound)) { + this.upperBoundSlice = new ManagedSlice( + StringUtils.string2Bytes(upperBound)); + } + } - @Override - protected ClosableIterator getKeyIteratorForFile( - String file) { - return new ManagedRawSstFileIterator(file, - options, lowerBoundSlice, upperBoundSlice, - keyValue -> StringUtils.bytes2String(keyValue.getKey())); - } + @Override + protected ClosableIterator getKeyIteratorForFile(String file) { + return new ManagedRawSstFileIterator(file, options, lowerBoundSlice, upperBoundSlice, + keyValue -> StringUtils.bytes2String(keyValue.getKey())); + } - @Override - public void close() throws UncheckedIOException { - super.close(); - options.close(); - IOUtils.closeQuietly(lowerBoundSlice, upperBoundSlice); - } - }; + @Override + public void close() throws UncheckedIOException { + super.close(); + options.close(); + IOUtils.closeQuietly(lowerBoundSlice, upperBoundSlice); + } + }; return getStreamFromIterator(itr); } - private abstract static class ManagedSstFileIterator implements - ClosableIterator { - private SstFileReader fileReader; - private SstFileReaderIterator fileReaderIterator; + private abstract static class ManagedSstFileIterator implements ClosableIterator { + private final SstFileReader fileReader; + private final SstFileReaderIterator fileReaderIterator; - ManagedSstFileIterator(String path, ManagedOptions options, - ReadOptions readOptions) - throws RocksDBException { + ManagedSstFileIterator(String path, ManagedOptions options, ReadOptions readOptions) throws RocksDBException { this.fileReader = new SstFileReader(options); this.fileReader.open(path); this.fileReaderIterator = fileReader.newIterator(readOptions); @@ -217,19 +208,15 @@ public String next() { } } - private static class ManagedRawSstFileIterator implements - ClosableIterator { - private ManagedRawSSTFileReader fileReader; - private ManagedRawSSTFileReaderIterator fileReaderIterator; - - ManagedRawSstFileIterator(String path, ManagedOptions options, - ManagedSlice lowerBound, ManagedSlice upperBound, - Function keyValueFunction) { - this.fileReader = new ManagedRawSSTFileReader<>(options, path, - 2 * 1024 * 1024); - this.fileReaderIterator = fileReader.newIterator(keyValueFunction, - lowerBound, upperBound); + private static class ManagedRawSstFileIterator implements ClosableIterator { + private final ManagedRawSSTFileReader fileReader; + private final ManagedRawSSTFileIterator fileReaderIterator; + private static final int READ_AHEAD_SIZE = 2 * 1024 * 1024; + + ManagedRawSstFileIterator(String path, ManagedOptions options, ManagedSlice lowerBound, ManagedSlice upperBound, + Function keyValueFunction) { + this.fileReader = new ManagedRawSSTFileReader<>(options, path, READ_AHEAD_SIZE); + this.fileReaderIterator = fileReader.newIterator(keyValueFunction, lowerBound, upperBound); } @Override @@ -249,8 +236,7 @@ public String next() { } } - private abstract static class MultipleSstFileIterator implements - ClosableIterator { + private abstract static class MultipleSstFileIterator implements ClosableIterator { private final Iterator fileNameIterator; @@ -264,16 +250,13 @@ private MultipleSstFileIterator(Collection files) { protected abstract void init(); - protected abstract ClosableIterator getKeyIteratorForFile(String file) - throws RocksDBException, - IOException; + protected abstract ClosableIterator getKeyIteratorForFile(String file) throws RocksDBException, IOException; @Override public boolean hasNext() { try { do { - if (Objects.nonNull(currentFileIterator) && - currentFileIterator.hasNext()) { + if (Objects.nonNull(currentFileIterator) && currentFileIterator.hasNext()) { return true; } } while (moveToNextFile()); diff --git a/hadoop-ozone/dev-support/checks/native.sh b/hadoop-ozone/dev-support/checks/native.sh index 1eeca5c0f3d9..89206b3bdf2f 100755 --- a/hadoop-ozone/dev-support/checks/native.sh +++ b/hadoop-ozone/dev-support/checks/native.sh @@ -19,20 +19,5 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" CHECK=native -zlib_version=$(mvn -N help:evaluate -Dexpression=zlib.version -q -DforceStdout) -if [[ -z "${zlib_version}" ]]; then - echo "ERROR zlib.version not defined in pom.xml" - exit 1 -fi - -bzip2_version=$(mvn -N help:evaluate -Dexpression=bzip2.version -q -DforceStdout) -if [[ -z "${bzip2_version}" ]]; then - echo "ERROR bzip2.version not defined in pom.xml" - exit 1 -fi - -source "${DIR}/junit.sh" -Pnative -Drocks_tools_native \ - -Dbzip2.url="https://github.com/libarchive/bzip2/archive/refs/tags/bzip2-${bzip2_version}.tar.gz" \ - -Dzlib.url="https://github.com/madler/zlib/releases/download/v${zlib_version}/zlib-${zlib_version}.tar.gz" \ - -DexcludedGroups="unhealthy" \ +source "${DIR}/junit.sh" -Pnative -Drocks_tools_native -DexcludedGroups="unhealthy" \ "$@" diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 8518b7b4e588..ed1642ab8703 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.ozone.OFSPath; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; @@ -109,6 +108,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotActive; @@ -255,7 +256,7 @@ public SnapshotDiffManager(ManagedRocksDB db, createEmptySnapDiffDir(path); this.sstBackupDirForSnapDiffJobs = path.toString(); - this.isNativeLibsLoaded = initSSTDumpTool(ozoneManager.getConfiguration()); + this.isNativeLibsLoaded = initNativeLibraryForEfficientDiff(ozoneManager.getConfiguration()); // Ideally, loadJobsOnStartUp should run only on OM node, since SnapDiff // is not HA currently and running this on all the nodes would be @@ -278,13 +279,12 @@ public PersistentMap getSnapDiffJobTable() { return snapDiffJobTable; } - private boolean initSSTDumpTool( - final OzoneConfiguration conf) { - if (conf.getBoolean(OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, - OMConfigKeys.OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) { + private boolean initNativeLibraryForEfficientDiff(final OzoneConfiguration conf) { + if (conf.getBoolean(OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB, OZONE_OM_SNAPSHOT_LOAD_NATIVE_LIB_DEFAULT)) { try { return ManagedRawSSTFileReader.loadLibrary(); } catch (NativeLibraryNotLoadedException e) { + LOG.info("Native Library for raw sst file reading loading failed.", e); return false; } } @@ -1066,12 +1066,9 @@ void addToObjectIdMap(Table fsTable, upperBoundCharArray[upperBoundCharArray.length - 1] += 1; sstFileReaderUpperBound = String.valueOf(upperBoundCharArray); } - try (Stream keysToCheck = - nativeRocksToolsLoaded - ? sstFileReader.getKeyStreamWithTombstone( - sstFileReaderLowerBound, sstFileReaderUpperBound) - : sstFileReader.getKeyStream(sstFileReaderLowerBound, - sstFileReaderUpperBound)) { + try (Stream keysToCheck = nativeRocksToolsLoaded ? + sstFileReader.getKeyStreamWithTombstone(sstFileReaderLowerBound, sstFileReaderUpperBound) + : sstFileReader.getKeyStream(sstFileReaderLowerBound, sstFileReaderUpperBound)) { keysToCheck.forEach(key -> { try { final WithParentObjectId fromObjectId = fsTable.get(key); From 2d70537ad90c28f55d1e5988d0462a06e0681015 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 21 Feb 2024 16:43:37 -0800 Subject: [PATCH 33/39] HDDS-10149. Fix checkstyle within patch file --- .../src/main/native/cplusplus_to_java_convert.h | 2 +- .../rocks-native/src/main/patches/rocks-native.patch | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/native/cplusplus_to_java_convert.h b/hadoop-hdds/rocks-native/src/main/native/cplusplus_to_java_convert.h index efe9d4a5be24..4862ea12a1b9 100644 --- a/hadoop-hdds/rocks-native/src/main/native/cplusplus_to_java_convert.h +++ b/hadoop-hdds/rocks-native/src/main/native/cplusplus_to_java_convert.h @@ -16,7 +16,7 @@ * limitations under the License. */ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index c4e281d16c8e..f042ec820534 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -22,7 +22,7 @@ index 000000000..ec3c05d6d --- /dev/null +++ b/include/rocksdb/raw_iterator.h @@ -0,0 +1,25 @@ -+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). @@ -53,7 +53,7 @@ index 000000000..266a0c1c8 --- /dev/null +++ b/include/rocksdb/raw_sst_file_reader.h @@ -0,0 +1,62 @@ -+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). @@ -216,7 +216,7 @@ index 000000000..ea7de6763 --- /dev/null +++ b/tools/raw_sst_file_iterator.h @@ -0,0 +1,45 @@ -+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. ++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). From 4c3890b8f196c3b5f7576e75163bb8a3abbde3c5 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 21 Feb 2024 17:03:20 -0800 Subject: [PATCH 34/39] HDDS-10149. Remove unwanted classes and tools from rocks tools --- .../src/main/patches/rocks-native.patch | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index f042ec820534..46c7cf187872 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -18,7 +18,7 @@ diff --git a/include/rocksdb/raw_iterator.h b/include/rocksdb/raw_iterator.h new file mode 100644 -index 000000000..ec3c05d6d +index 000000000..1bf9b7d15 --- /dev/null +++ b/include/rocksdb/raw_iterator.h @@ -0,0 +1,25 @@ @@ -49,7 +49,7 @@ index 000000000..ec3c05d6d +#endif // ROCKSDB_LITE diff --git a/include/rocksdb/raw_sst_file_reader.h b/include/rocksdb/raw_sst_file_reader.h new file mode 100644 -index 000000000..266a0c1c8 +index 000000000..09e748208 --- /dev/null +++ b/include/rocksdb/raw_sst_file_reader.h @@ -0,0 +1,62 @@ @@ -116,18 +116,43 @@ index 000000000..266a0c1c8 + +#endif // ROCKSDB_LITE diff --git a/src.mk b/src.mk -index b94bc43ca..39080b2cf 100644 +index b94bc43ca..83f4fc90b 100644 --- a/src.mk +++ b/src.mk -@@ -343,6 +343,8 @@ TOOL_LIB_SOURCES = \ - tools/ldb_tool.cc \ - tools/sst_dump_tool.cc \ +@@ -338,11 +338,9 @@ RANGE_TREE_SOURCES =\ + utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc + + TOOL_LIB_SOURCES = \ +- tools/io_tracer_parser_tool.cc \ +- tools/ldb_cmd.cc \ +- tools/ldb_tool.cc \ +- tools/sst_dump_tool.cc \ utilities/blob_db/blob_dump_tool.cc \ + tools/raw_sst_file_reader.cc \ + tools/raw_sst_file_iterator.cc \ ANALYZER_LIB_SOURCES = \ tools/block_cache_analyzer/block_cache_trace_analyzer.cc \ +@@ -393,19 +391,7 @@ FOLLY_SOURCES = \ + $(FOLLY_DIR)/folly/synchronization/ParkingLot.cpp \ + + TOOLS_MAIN_SOURCES = \ +- db_stress_tool/db_stress.cc \ + tools/blob_dump.cc \ +- tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc \ +- tools/db_repl_stress.cc \ +- tools/db_sanity_test.cc \ +- tools/ldb.cc \ +- tools/io_tracer_parser.cc \ +- tools/sst_dump.cc \ +- tools/write_stress.cc \ +- tools/dump/rocksdb_dump.cc \ +- tools/dump/rocksdb_undump.cc \ +- tools/trace_analyzer.cc \ +- tools/io_tracer_parser_tool.cc \ + + BENCH_MAIN_SOURCES = \ + cache/cache_bench.cc \ diff --git a/tools/raw_sst_file_iterator.cc b/tools/raw_sst_file_iterator.cc new file mode 100644 index 000000000..5527b84a4 @@ -212,7 +237,7 @@ index 000000000..5527b84a4 +#endif // ROCKSDB_LITE diff --git a/tools/raw_sst_file_iterator.h b/tools/raw_sst_file_iterator.h new file mode 100644 -index 000000000..ea7de6763 +index 000000000..622d7d75e --- /dev/null +++ b/tools/raw_sst_file_iterator.h @@ -0,0 +1,45 @@ From 8a1b0bbbdfe00b75210d7887da4e5ba578413f8d Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 21 Feb 2024 17:28:50 -0800 Subject: [PATCH 35/39] HDDS-10149. Remove unwanted classes and tools from rocks tools --- hadoop-hdds/rocks-native/pom.xml | 2 +- .../src/main/patches/rocks-native.patch | 26 +++---------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml index 7d1b80ed4df2..2146da977ffd 100644 --- a/hadoop-hdds/rocks-native/pom.xml +++ b/hadoop-hdds/rocks-native/pom.xml @@ -238,7 +238,7 @@ - + diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index 46c7cf187872..a765ba4d6310 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -116,10 +116,10 @@ index 000000000..09e748208 + +#endif // ROCKSDB_LITE diff --git a/src.mk b/src.mk -index b94bc43ca..83f4fc90b 100644 +index b94bc43ca..c13e5cde6 100644 --- a/src.mk +++ b/src.mk -@@ -338,11 +338,9 @@ RANGE_TREE_SOURCES =\ +@@ -338,11 +338,8 @@ RANGE_TREE_SOURCES =\ utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc TOOL_LIB_SOURCES = \ @@ -127,32 +127,12 @@ index b94bc43ca..83f4fc90b 100644 - tools/ldb_cmd.cc \ - tools/ldb_tool.cc \ - tools/sst_dump_tool.cc \ - utilities/blob_db/blob_dump_tool.cc \ +- utilities/blob_db/blob_dump_tool.cc \ + tools/raw_sst_file_reader.cc \ + tools/raw_sst_file_iterator.cc \ ANALYZER_LIB_SOURCES = \ tools/block_cache_analyzer/block_cache_trace_analyzer.cc \ -@@ -393,19 +391,7 @@ FOLLY_SOURCES = \ - $(FOLLY_DIR)/folly/synchronization/ParkingLot.cpp \ - - TOOLS_MAIN_SOURCES = \ -- db_stress_tool/db_stress.cc \ - tools/blob_dump.cc \ -- tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc \ -- tools/db_repl_stress.cc \ -- tools/db_sanity_test.cc \ -- tools/ldb.cc \ -- tools/io_tracer_parser.cc \ -- tools/sst_dump.cc \ -- tools/write_stress.cc \ -- tools/dump/rocksdb_dump.cc \ -- tools/dump/rocksdb_undump.cc \ -- tools/trace_analyzer.cc \ -- tools/io_tracer_parser_tool.cc \ - - BENCH_MAIN_SOURCES = \ - cache/cache_bench.cc \ diff --git a/tools/raw_sst_file_iterator.cc b/tools/raw_sst_file_iterator.cc new file mode 100644 index 000000000..5527b84a4 From 1ab1bc622fec550154aac4a0f09b21f7d4c0e176 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 21 Feb 2024 20:00:41 -0800 Subject: [PATCH 36/39] HDDS-10149. Change Function definitions & imports for raw_sst_file_iterator & reader --- .../main/native/ManagedRawSSTFileIterator.cpp | 12 ++-- .../src/main/patches/rocks-native.patch | 61 ++++++++----------- 2 files changed, 31 insertions(+), 42 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileIterator.cpp b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileIterator.cpp index 440dc51a7610..1cf222528379 100644 --- a/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileIterator.cpp +++ b/hadoop-hdds/rocks-native/src/main/native/ManagedRawSSTFileIterator.cpp @@ -25,18 +25,18 @@ jboolean Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_hasNext(JNIEnv *env, jobject obj, jlong native_handle) { - return static_cast(reinterpret_cast(native_handle)->has_next()); + return static_cast(reinterpret_cast(native_handle)->Valid()); } void Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_next(JNIEnv *env, jobject obj, jlong native_handle) { - reinterpret_cast(native_handle)->next(); + reinterpret_cast(native_handle)->Next(); } jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_getKey(JNIEnv *env, jobject obj, jlong native_handle) { - ROCKSDB_NAMESPACE::Slice slice = reinterpret_cast(native_handle)->getKey(); + ROCKSDB_NAMESPACE::Slice slice = reinterpret_cast(native_handle)->key(); jbyteArray jkey = env->NewByteArray(static_cast(slice.size())); if (jkey == nullptr) { // exception thrown: OutOfMemoryError @@ -52,7 +52,7 @@ jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterato jbyteArray Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_getValue(JNIEnv *env, jobject obj, jlong native_handle) { - ROCKSDB_NAMESPACE::Slice slice = reinterpret_cast(native_handle)->getValue(); + ROCKSDB_NAMESPACE::Slice slice = reinterpret_cast(native_handle)->value(); jbyteArray jkey = env->NewByteArray(static_cast(slice.size())); if (jkey == nullptr) { // exception thrown: OutOfMemoryError @@ -68,7 +68,7 @@ jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_get jobject obj, jlong native_handle) { uint64_t sequence_number = - reinterpret_cast(native_handle)->getSequenceNumber(); + reinterpret_cast(native_handle)->sequenceNumber(); jlong result; std::memcpy(&result, &sequence_number, sizeof(jlong)); return result; @@ -78,7 +78,7 @@ jlong Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_get jint Java_org_apache_hadoop_hdds_utils_db_managed_ManagedRawSSTFileIterator_getType(JNIEnv *env, jobject obj, jlong native_handle) { - uint32_t type = reinterpret_cast(native_handle)->getType(); + uint32_t type = reinterpret_cast(native_handle)->type(); return static_cast(type); } diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index a765ba4d6310..1019af1eb85a 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -18,7 +18,7 @@ diff --git a/include/rocksdb/raw_iterator.h b/include/rocksdb/raw_iterator.h new file mode 100644 -index 000000000..1bf9b7d15 +index 000000000..21242ed15 --- /dev/null +++ b/include/rocksdb/raw_iterator.h @@ -0,0 +1,25 @@ @@ -36,12 +36,12 @@ index 000000000..1bf9b7d15 +class RawIterator { + public: + virtual ~RawIterator() {} -+ virtual bool has_next() const = 0; -+ virtual Slice getKey() const = 0; -+ virtual Slice getValue() const = 0; -+ virtual uint64_t getSequenceNumber() const = 0; -+ virtual uint32_t getType() const = 0; -+ virtual void next() = 0; ++ virtual bool Valid() const = 0; ++ virtual Slice key() const = 0; ++ virtual Slice value() const = 0; ++ virtual uint64_t sequenceNumber() const = 0; ++ virtual uint32_t type() const = 0; ++ virtual void Next() = 0; +}; + +} // namespace ROCKSDB_NAMESPACE @@ -135,7 +135,7 @@ index b94bc43ca..c13e5cde6 100644 tools/block_cache_analyzer/block_cache_trace_analyzer.cc \ diff --git a/tools/raw_sst_file_iterator.cc b/tools/raw_sst_file_iterator.cc new file mode 100644 -index 000000000..5527b84a4 +index 000000000..3051637a3 --- /dev/null +++ b/tools/raw_sst_file_iterator.cc @@ -0,0 +1,76 @@ @@ -180,10 +180,10 @@ index 000000000..5527b84a4 + initKey(); +} + -+bool RawSstFileIterator::has_next() const { ++bool RawSstFileIterator::Valid() const { + return iter_->Valid() && (!has_to_ || + BytewiseComparator()->Compare( -+ getKey(), *to_key_) < 0); ++ key(), *to_key_) < 0); +} + +void RawSstFileIterator::initKey() { @@ -191,25 +191,25 @@ index 000000000..5527b84a4 + ParseInternalKey(iter_->key(), ikey, true /* log_err_key */); + } +} -+void RawSstFileIterator::next() { ++void RawSstFileIterator::Next() { + iter_->Next(); + initKey(); + +} + -+Slice RawSstFileIterator::getKey() const { ++Slice RawSstFileIterator::key() const { + return ikey->user_key; +} + -+uint64_t RawSstFileIterator::getSequenceNumber() const { ++uint64_t RawSstFileIterator::sequenceNumber() const { + return ikey->sequence; +} + -+uint32_t RawSstFileIterator::getType() const { ++uint32_t RawSstFileIterator::type() const { + return static_cast(ikey->type); +} + -+Slice RawSstFileIterator::getValue() const { ++Slice RawSstFileIterator::value() const { + return iter_->value(); +} +} // namespace ROCKSDB_NAMESPACE @@ -217,7 +217,7 @@ index 000000000..5527b84a4 +#endif // ROCKSDB_LITE diff --git a/tools/raw_sst_file_iterator.h b/tools/raw_sst_file_iterator.h new file mode 100644 -index 000000000..622d7d75e +index 000000000..58e34b260 --- /dev/null +++ b/tools/raw_sst_file_iterator.h @@ -0,0 +1,45 @@ @@ -244,12 +244,12 @@ index 000000000..622d7d75e + bool has_to, + Slice* to_key); + -+ bool has_next() const override; -+ Slice getKey() const override; -+ Slice getValue() const override; -+ uint64_t getSequenceNumber() const override; -+ uint32_t getType() const override; -+ void next() final override; ++ bool Valid() const override; ++ Slice key() const override; ++ Slice value() const override; ++ uint64_t sequenceNumber() const override; ++ uint32_t type() const override; ++ void Next() final override; + + ~RawSstFileIterator(){ + delete iter_; @@ -268,10 +268,10 @@ index 000000000..622d7d75e +#endif // ROCKSDB_LITE diff --git a/tools/raw_sst_file_reader.cc b/tools/raw_sst_file_reader.cc new file mode 100644 -index 000000000..220c814d0 +index 000000000..0b1b55274 --- /dev/null +++ b/tools/raw_sst_file_reader.cc -@@ -0,0 +1,285 @@ +@@ -0,0 +1,274 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License @@ -283,20 +283,14 @@ index 000000000..220c814d0 + +#include +#include -+#include +#include +#include -+#include -+#include + -+#include "db/blob/blob_index.h" ++ +#include "db/memtable.h" +#include "db/write_batch_internal.h" +#include "options/cf_options.h" -+#include "port/port.h" -+#include "rocksdb/db.h" +#include "rocksdb/env.h" -+#include "rocksdb/iterator.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/status.h" +#include "rocksdb/table_properties.h" @@ -304,17 +298,12 @@ index 000000000..220c814d0 +#include "table/block_based/block.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/block_based/block_based_table_factory.h" -+#include "table/block_based/block_builder.h" +#include "table/format.h" +#include "table/meta_blocks.h" +#include "table/plain/plain_table_factory.h" +#include "table/table_reader.h" +#include "tools/raw_sst_file_iterator.h" -+#include "util/compression.h" -+#include "util/random.h" +#include "db/dbformat.h" -+#include "file/writable_file_writer.h" -+#include "options/cf_options.h" + +namespace ROCKSDB_NAMESPACE { + From a36487b87f678bd201ab1f3c3e2a7ca44e051422 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 22 Feb 2024 07:07:51 -0800 Subject: [PATCH 37/39] HDDS-10149. Address review comments --- .../java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java | 1 - pom.xml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java index 1264078c6e01..10df236f88d4 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java @@ -126,7 +126,6 @@ public synchronized boolean loadLibrary(final String libraryName) { LOG.warn("Unable to load library: {}", libraryName, e); } this.librariesLoaded.put(libraryName, loaded); - LOG.info("Loaded Library {}", libraryName); return isLibraryLoaded(libraryName); } diff --git a/pom.xml b/pom.xml index 6c3969c119f1..72c7fe65d603 100644 --- a/pom.xml +++ b/pom.xml @@ -304,7 +304,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 5.1.0 - 1.0.0 + 1.2.1 From 9b48779285d98ff28d4e93e1fa6f124b23b91c27 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 22 Feb 2024 08:53:54 -0800 Subject: [PATCH 38/39] HDDS-10149. Address review comments --- .../hadoop/ozone/om/snapshot/SnapshotDiffManager.java | 2 +- pom.xml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index ed1642ab8703..a200a36cb25d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -284,7 +284,7 @@ private boolean initNativeLibraryForEfficientDiff(final OzoneConfiguration conf) try { return ManagedRawSSTFileReader.loadLibrary(); } catch (NativeLibraryNotLoadedException e) { - LOG.info("Native Library for raw sst file reading loading failed.", e); + LOG.error("Native Library for raw sst file reading loading failed.", e); return false; } } diff --git a/pom.xml b/pom.xml index 72c7fe65d603..c585f8d940df 100644 --- a/pom.xml +++ b/pom.xml @@ -305,6 +305,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 5.1.0 1.2.1 + 3.9.6 @@ -2090,6 +2091,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.codehaus.mojo properties-maven-plugin ${properties.maven.plugin.version} + + + org.apache.maven + maven-core + ${maven.core.version} + + From c4ee8c8afe3aa26e006fb7fd6d1248b0cb98646d Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 22 Feb 2024 12:43:09 -0800 Subject: [PATCH 39/39] HDDS-10149. Address review comments --- .../rocks-native/src/main/patches/rocks-native.patch | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch index 1019af1eb85a..12dc74614a45 100644 --- a/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch +++ b/hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch @@ -268,10 +268,10 @@ index 000000000..58e34b260 +#endif // ROCKSDB_LITE diff --git a/tools/raw_sst_file_reader.cc b/tools/raw_sst_file_reader.cc new file mode 100644 -index 000000000..0b1b55274 +index 000000000..5ba8a82ee --- /dev/null +++ b/tools/raw_sst_file_reader.cc -@@ -0,0 +1,274 @@ +@@ -0,0 +1,272 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License @@ -414,11 +414,6 @@ index 000000000..0b1b55274 + /* memory_allocator= */ nullptr, (magic_number == kBlockBasedTableMagicNumber) + ? &prefetch_buffer + : nullptr); -+ if (!s.ok()) { -+ if (!silent_) { -+ fprintf(stderr, "Not able to read table properties\n"); -+ } -+ } + // For old sst format, ReadTableProperties might fail but file can be read + if (s.ok()) { + s = SetTableOptionsByMagicNumber(magic_number); @@ -436,6 +431,9 @@ index 000000000..0b1b55274 + } + } + } else { ++ if (!silent_) { ++ fprintf(stderr, "Not able to read table properties\n"); ++ } + s = SetOldTableOptions(); + } + options_.comparator = rep_->internal_comparator_.user_comparator();