From f65fc1df13dd13691b52d98f5916b1725e5aa563 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 10 Mar 2023 10:18:30 -0800 Subject: [PATCH 01/14] HDDS-8137: Snapdiff to use tombstone entries in SST file --- .../org/apache/hadoop/hdds/HddsUtils.java | 6 + .../apache/hadoop/ozone/OzoneConfigKeys.java | 15 ++ .../src/main/resources/ozone-default.xml | 9 + .../db/managed/ManagedSSTDumpIterator.java | 48 ++++- hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 8 + .../rocksdb/util/ManagedSstFileReader.java | 154 ++++++++++++--- .../apache/ozone/rocksdb/util/RdbUtil.java | 20 ++ hadoop-ozone/ozone-manager/pom.xml | 4 + .../om/snapshot/SnapshotDiffManager.java | 183 ++++++++++++++---- 9 files changed, 371 insertions(+), 76 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 344dd3e6ffcc..8f908e60ac5d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -20,6 +20,7 @@ import com.google.protobuf.ServiceException; import javax.management.ObjectName; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -30,6 +31,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -800,4 +802,8 @@ public static Map processForLogging(OzoneConfiguration conf) { } return sortedOzoneProps; } + + public interface CloseableIterator extends Iterator, Closeable { + + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index ede1b4fd8740..0c1021cbda77 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -616,6 +616,21 @@ public final class OzoneConfigKeys { OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL = "ozone.om.snapshot.compaction.dag.prune.daemon.run.interval"; + public static final String + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE = + "ozone.om.snapshot.sst_dumptool.pool.size"; + + public static final int + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT = 1; + + public static final String + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE = + "ozone.om.snapshot.sst_dumptool.buffer.size"; + + public static final String + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT = "8KB"; + + public static final long OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT = TimeUnit.HOURS.toMillis(1); diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 018ff8066767..4e63e7cc5b0a 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3656,4 +3656,13 @@ without using the optimised DAG based pruning approach + + + ozone.om.snapshot.sst_dumptool.pool.size + 1 + OZONE, OM + + Threadpool size for SST Dumptool which would be used for computing snapdiff when native library is enabled. + + diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java index 35aaeb33b0a8..179e213f3b60 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.utils.db.managed; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.eclipse.jetty.io.RuntimeIOException; @@ -25,7 +26,8 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -33,10 +35,9 @@ /** * Iterator to Parse output of RocksDBSSTDumpTool. */ -public class ManagedSSTDumpIterator implements - Iterator, AutoCloseable { - private static final String SST_DUMP_TOOL_CLASS = - "org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool"; +public abstract class ManagedSSTDumpIterator implements + HddsUtils.CloseableIterator { + private static final String PATTERN_REGEX = "'([^=>]+)' seq:([0-9]+), type:([0-9]+) => "; @@ -116,13 +117,20 @@ public boolean hasNext() { return nextKey != null; } + /** + * Transform function to transform key to a certain value + * @param value + * @return + */ + protected abstract T getTransformedValue(KeyValue value); + /** * Returns the next record from SSTDumpTool. * @return next Key * Throws Runtime Exception incase of failure. */ @Override - public KeyValue next() { + public T next() { checkSanityOfProcess(); currentKey = nextKey; nextKey = null; @@ -139,8 +147,9 @@ public KeyValue next() { if (currentKey != null) { currentKey.setValue(stdoutString.substring(0, Math.max(stdoutString.length() - 1, 0))); + return getTransformedValue(currentKey); } - return currentKey; + throw new NoSuchElementException("No more records found"); } stdoutString.append(charBuffer, 0, numberOfCharsRead); currentMatcher.reset(); @@ -157,11 +166,11 @@ public KeyValue next() { currentMatcher.group(PATTERN_KEY_GROUP_NUMBER), currentMatcher.group(PATTERN_SEQ_GROUP_NUMBER), currentMatcher.group(PATTERN_TYPE_GROUP_NUMBER)); - return currentKey; + return getTransformedValue(currentKey); } @Override - public synchronized void close() throws Exception { + public synchronized void close() throws IOException { if (this.sstDumpToolTask != null) { if (!this.sstDumpToolTask.getFuture().isDone()) { this.sstDumpToolTask.getFuture().cancel(true); @@ -222,4 +231,25 @@ public String toString() { '}'; } } + + public static void main(String[] args) throws NativeLibraryNotLoadedException, IOException { + ManagedSSTDumpTool sstDumpTool = + new ManagedSSTDumpTool(new ForkJoinPool(), 50); + try (ManagedOptions options = new ManagedOptions(); + ManagedSSTDumpIterator iterator = new ManagedSSTDumpIterator(sstDumpTool, + "/Users/sbalachandran/Documents/code/dummyrocks/rocks/000025.sst", options) { + @Override + protected KeyValue getTransformedValue(KeyValue value) { + return value; + } + }; + ) { + while (iterator.hasNext()) { + System.out.println(iterator.next()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml index f2a932b40a14..d4f4a958e939 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml +++ b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml @@ -73,6 +73,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> junit-jupiter-params test + + org.apache.ozone + hdds-rocks-native + + + org.apache.ozone + hdds-rocks-native + diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index cf8e59331dc6..67fa6e31fec8 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -18,16 +18,24 @@ package org.apache.ozone.rocksdb.util; -import org.rocksdb.Options; +import org.apache.hadoop.hdds.HddsUtils; +import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; +import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpIterator; +import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDBException; import org.rocksdb.SstFileReader; import org.rocksdb.SstFileReaderIterator; -import java.io.Closeable; +import java.io.IOException; +import java.io.InvalidObjectException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Iterator; import java.util.NoSuchElementException; +import java.util.Optional; import java.util.Spliterator; import java.util.Spliterators; import java.util.stream.Stream; @@ -46,41 +54,129 @@ public class ManagedSstFileReader { public ManagedSstFileReader(final Collection sstFiles) { this.sstFiles = sstFiles; } - public Stream getKeyStream() throws RocksDBException { - final ManagedSstFileIterator itr = new ManagedSstFileIterator(sstFiles); - final Spliterator spliterator = Spliterators - .spliteratorUnknownSize(itr, 0); - return StreamSupport.stream(spliterator, false).onClose(itr::close); + + public Stream getKeyStream() throws RocksDBException, + NativeLibraryNotLoadedException, IOException { + //TODO: [SNAPSHOT] Check if default Options and ReadOptions is enough. + ManagedOptions options = new ManagedOptions(); + ReadOptions readOptions = new ReadOptions(); + final MultipleSstFileIterator itr = + new MultipleSstFileIterator(sstFiles) { + @Override + protected HddsUtils.CloseableIterator initNewKeyIteratorForFile( + String file) throws RocksDBException { + return new ManagedSstFileIterator(file, options, + readOptions) { + @Override + protected String getIteratorValue(SstFileReaderIterator iterator) { + return new String(iterator.key(), UTF_8); + } + }; + } + + @Override + public void close() throws IOException { + super.close(); + options.close(); + readOptions.close(); + } + }; + return RdbUtil.getStreamFromIterator(itr); + } + public Stream getKeyStreamWithTombstone( + ManagedSSTDumpTool sstDumpTool) throws IOException, RocksDBException, + NativeLibraryNotLoadedException { + //TODO: [SNAPSHOT] Check if default Options is enough. + ManagedOptions options = new ManagedOptions(); + final MultipleSstFileIterator itr = + new MultipleSstFileIterator(sstFiles) { + @Override + protected HddsUtils.CloseableIterator initNewKeyIteratorForFile( + String file) throws NativeLibraryNotLoadedException, + IOException { + return new ManagedSSTDumpIterator(sstDumpTool, file, + options) { + @Override + protected String getTransformedValue(KeyValue value) { + return value.getKey(); + } + }; + } + + @Override + public void close() throws IOException { + super.close(); + options.close(); + } + }; + return RdbUtil.getStreamFromIterator(itr); + } + + private abstract static class ManagedSstFileIterator implements + HddsUtils.CloseableIterator { + private SstFileReader fileReader; + private SstFileReaderIterator fileReaderIterator; + + public ManagedSstFileIterator(String path, ManagedOptions options, + ReadOptions readOptions) + throws RocksDBException { + this.fileReader = new SstFileReader(options); + this.fileReader.open(path); + this.fileReaderIterator = fileReader.newIterator(readOptions); + fileReaderIterator.seekToFirst(); + } + + @Override + public void close() throws IOException { + this.fileReaderIterator.close(); + this.fileReader.close(); + } + + @Override + public boolean hasNext() { + return fileReaderIterator.isValid(); + } + + protected abstract T getIteratorValue(SstFileReaderIterator iterator); + + @Override + public T next() { + T value = getIteratorValue(fileReaderIterator); + fileReaderIterator.next(); + return value; + } } - private static final class ManagedSstFileIterator implements - Iterator, Closeable { + private abstract static class MultipleSstFileIterator implements + HddsUtils.CloseableIterator { private final Iterator fileNameIterator; - private final Options options; - private final ReadOptions readOptions; + private String currentFile; private SstFileReader currentFileReader; - private SstFileReaderIterator currentFileIterator; + private HddsUtils.CloseableIterator currentFileIterator; - private ManagedSstFileIterator(Collection files) - throws RocksDBException { - // TODO: Check if default Options and ReadOptions is enough. - this.options = new Options(); - this.readOptions = new ReadOptions(); + private MultipleSstFileIterator(Collection files) + throws IOException, RocksDBException, + NativeLibraryNotLoadedException { this.fileNameIterator = files.iterator(); moveToNextFile(); } + protected abstract HddsUtils.CloseableIterator initNewKeyIteratorForFile( + String file) throws RocksDBException, + NativeLibraryNotLoadedException, IOException; + @Override public boolean hasNext() { try { do { - if (currentFileIterator.isValid()) { + if (currentFileIterator.hasNext()) { return true; } } while (moveToNextFile()); - } catch (RocksDBException e) { + } catch (IOException | RocksDBException | + NativeLibraryNotLoadedException e) { // TODO: This exception has to be handled by the caller. // We have to do better exception handling. throw new RuntimeException(e); @@ -89,37 +185,33 @@ public boolean hasNext() { } @Override - public String next() { + public T next() { if (hasNext()) { - final String value = new String(currentFileIterator.key(), UTF_8); - currentFileIterator.next(); - return value; + return currentFileIterator.next(); +// final String value = new String(currentFileIterator.key(), UTF_8); } throw new NoSuchElementException("No more keys"); } @Override - public void close() { + public void close() throws IOException { closeCurrentFile(); } - private boolean moveToNextFile() throws RocksDBException { + private boolean moveToNextFile() throws IOException, RocksDBException, + NativeLibraryNotLoadedException { if (fileNameIterator.hasNext()) { closeCurrentFile(); currentFile = fileNameIterator.next(); - currentFileReader = new SstFileReader(options); - currentFileReader.open(currentFile); - currentFileIterator = currentFileReader.newIterator(readOptions); - currentFileIterator.seekToFirst(); + this.currentFileIterator = initNewKeyIteratorForFile(currentFile); return true; } return false; } - private void closeCurrentFile() { + private void closeCurrentFile() throws IOException { if (currentFile != null) { currentFileIterator.close(); - currentFileReader.close(); currentFile = null; } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java index 172209c584b5..a7a018d407b5 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java @@ -20,17 +20,24 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; +import org.apache.hadoop.hdds.HddsUtils; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.Spliterator; +import java.util.Spliterators; import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; /** * Temporary class to test snapshot diff functionality. @@ -59,4 +66,17 @@ public static Set getSSTFilesForComparison(final String dbLocation, } } + public static Stream getStreamFromIterator( + HddsUtils.CloseableIterator itr) { + final Spliterator spliterator = Spliterators + .spliteratorUnknownSize(itr, 0); + return StreamSupport.stream(spliterator, false).onClose(() -> { + try { + itr.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + } diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 974565206d92..a2b8b284d05d 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -254,6 +254,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.junit.jupiter junit-jupiter-params + + org.apache.ozone + hdds-rocks-native + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 60e7ea7fd0d5..4f85a7094ed1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -30,6 +30,12 @@ import java.util.Map; import java.util.Set; import java.util.UUID; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.utils.NativeConstants; +import org.apache.hadoop.hdds.utils.NativeLibraryLoader; +import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -41,6 +47,7 @@ import org.apache.hadoop.hdds.utils.db.CodecRegistry; import org.apache.hadoop.hdds.utils.db.IntegerCodec; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; @@ -73,6 +80,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getSnapshotInfo; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE; @@ -124,6 +145,9 @@ public class SnapshotDiffManager implements AutoCloseable { private final PersistentMap snapDiffJobTable; private final ExecutorService executorService; + private boolean isNativeRocksToolsLoaded; + + private ManagedSSTDumpTool sstDumpTool; public SnapshotDiffManager(ManagedRocksDB db, RocksDBCheckpointDiffer differ, @@ -170,6 +194,37 @@ public SnapshotDiffManager(ManagedRocksDB db, // TODO: [SNAPSHOT] Load jobs only if it is leader node. // It could a event-triggered form OM when node is leader and up. this.loadJobsOnStartUp(); + isNativeRocksToolsLoaded = NativeLibraryLoader.getInstance() + .loadLibrary(NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME); + if (isNativeRocksToolsLoaded) { + try { + initSSTDumpTool(configuration); + isNativeRocksToolsLoaded = true; + } catch (NativeLibraryNotLoadedException e) { + LOG.error("Unable to load SSTDumpTool ", e); + isNativeRocksToolsLoaded = false; + } + } + } + + private void initSSTDumpTool(OzoneConfiguration configuration) + throws NativeLibraryNotLoadedException { + int threadPoolSize = configuration.getInt( + OzoneConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE, + OzoneConfigKeys + .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT); + int bufferSize = (int)configuration.getStorageSize( + OzoneConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE, + OzoneConfigKeys + .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT, + StorageUnit.BYTES); + ExecutorService executorService = new ThreadPoolExecutor(0, + threadPoolSize, 60, TimeUnit.SECONDS, + new SynchronousQueue<>(), new ThreadFactoryBuilder() + .setNameFormat("snapshot-diff-manager-sst-dump-tool-TID-%d") + .build(), + new ThreadPoolExecutor.DiscardPolicy()); + sstDumpTool = new ManagedSSTDumpTool(executorService, bufferSize); } private Map getTablePrefixes( @@ -209,6 +264,13 @@ private DifferSnapshotInfo getDSIFromSI(SnapshotInfo snapshotInfo, getTablePrefixes(snapshotOMMM, volumeName, bucketName)); } + private Set getSSTFileListForSnapshot(OmSnapshot snapshot, + List tablesToLookUp) throws RocksDBException { + return RdbUtil.getSSTFilesForComparison(snapshot + .getMetadataManager().getStore().getDbLocation() + .getPath(), tablesToLookUp); + } + @SuppressWarnings("parameternumber") public SnapshotDiffResponse getSnapshotDiffReport( final String volume, @@ -466,36 +528,90 @@ private void generateSnapshotDiffReport(final String jobKey, Map tablePrefixes = getTablePrefixes(toSnapshot.getMetadataManager(), volume, bucket); - - final Set deltaFilesForKeyOrFileTable = - getDeltaFiles(fromSnapshot, toSnapshot, - Collections.singletonList(fsKeyTable.getName()), fsInfo, tsInfo, + List tablesToLookUp = + Collections.singletonList(fsKeyTable.getName()); + final Set deltaFilesForKeyOrFileTable = getDeltaFiles( + fromSnapshot, toSnapshot, tablesToLookUp, fsInfo, tsInfo, useFullDiff, tablePrefixes); - addToObjectIdMap(fsKeyTable, - tsKeyTable, - deltaFilesForKeyOrFileTable, - objectIdToKeyNameMapForFromSnapshot, - objectIdToKeyNameMapForToSnapshot, - objectIDsToCheckMap, - tablePrefixes); + // Workaround to handle deletes if native rockstools for reading + // tombstone is not loaded. + // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read to + // read tombstone + if (!isNativeRocksToolsLoaded) { + deltaFilesForKeyOrFileTable.addAll(getSSTFileListForSnapshot( + fromSnapshot, tablesToLookUp)); + } + try { + addToObjectIdMap(fsKeyTable, + tsKeyTable, + deltaFilesForKeyOrFileTable, + objectIdToKeyNameMapForFromSnapshot, + objectIdToKeyNameMapForToSnapshot, + objectIDsToCheckMap, + tablePrefixes, isNativeRocksToolsLoaded); + } catch (NativeLibraryNotLoadedException e) { + // Workaround to handle deletes if use of native rockstools for reading + // tombstone fails. + // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read to + // read tombstone + deltaFilesForKeyOrFileTable.addAll(getSSTFileListForSnapshot( + fromSnapshot, tablesToLookUp)); + try { + addToObjectIdMap(fsKeyTable, + tsKeyTable, + deltaFilesForKeyOrFileTable, + objectIdToKeyNameMapForFromSnapshot, + objectIdToKeyNameMapForToSnapshot, + objectIDsToCheckMap, + tablePrefixes, false); + } catch (NativeLibraryNotLoadedException ex) { + //This code should never be never executed. + throw new RuntimeException(ex); + } + } if (bucketLayout.isFileSystemOptimized()) { final Table fsDirTable = fromSnapshot.getMetadataManager().getDirectoryTable(); final Table tsDirTable = toSnapshot.getMetadataManager().getDirectoryTable(); + tablesToLookUp = Collections.singletonList(fsDirTable.getName()); final Set deltaFilesForDirTable = - getDeltaFiles(fromSnapshot, toSnapshot, - Collections.singletonList(fsDirTable.getName()), fsInfo, tsInfo, - useFullDiff, tablePrefixes); - addToObjectIdMap(fsDirTable, - tsDirTable, - deltaFilesForDirTable, - objectIdToKeyNameMapForFromSnapshot, - objectIdToKeyNameMapForToSnapshot, - objectIDsToCheckMap, - tablePrefixes); + getDeltaFiles(fromSnapshot, toSnapshot, tablesToLookUp, fsInfo, + tsInfo, useFullDiff, tablePrefixes); + if (!isNativeRocksToolsLoaded) { + deltaFilesForDirTable.addAll(getSSTFileListForSnapshot( + fromSnapshot, tablesToLookUp)); + } + try { + addToObjectIdMap(fsDirTable, + tsDirTable, + deltaFilesForDirTable, + objectIdToKeyNameMapForFromSnapshot, + objectIdToKeyNameMapForToSnapshot, + objectIDsToCheckMap, + tablePrefixes, isNativeRocksToolsLoaded); + } catch (NativeLibraryNotLoadedException e) { + try { + // Workaround to handle deletes if use of native rockstools for + // reading tombstone fails. + // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read to + // read tombstone + deltaFilesForDirTable.addAll(getSSTFileListForSnapshot( + fromSnapshot, tablesToLookUp)); + addToObjectIdMap(fsDirTable, + tsDirTable, + deltaFilesForDirTable, + objectIdToKeyNameMapForFromSnapshot, + objectIdToKeyNameMapForToSnapshot, + objectIDsToCheckMap, + tablePrefixes, false); + } catch (NativeLibraryNotLoadedException ex) { + //This code should never be never executed. + throw new RuntimeException(ex); + } + } } generateDiffReport(jobId, @@ -521,8 +637,9 @@ private void addToObjectIdMap(Table fsTable, PersistentMap oldObjIdToKeyMap, PersistentMap newObjIdToKeyMap, PersistentSet objectIDsToCheck, - Map tablePrefixes) - throws IOException { + Map tablePrefixes, + boolean isNativeRocksToolsLoaded) + throws IOException, NativeLibraryNotLoadedException { if (deltaFiles.isEmpty()) { return; @@ -533,6 +650,13 @@ private void addToObjectIdMap(Table fsTable, try (Stream keysToCheck = new ManagedSstFileReader(deltaFiles) .getKeyStream()) { + if (deltaFiles.isEmpty()) { + return; + } + ManagedSstFileReader sstFileReader = new ManagedSstFileReader(deltaFiles); + try (Stream keysToCheck = isNativeRocksToolsLoaded + ? sstFileReader.getKeyStreamWithTombstone(sstDumpTool) + : sstFileReader.getKeyStream()) { keysToCheck.forEach(key -> { try { final WithObjectID oldKey = fsTable.get(key); @@ -602,19 +726,6 @@ private Set getDeltaFiles(OmSnapshot fromSnapshot, List sstDiffList = differ.getSSTDiffListWithFullPath(toDSI, fromDSI); deltaFiles.addAll(sstDiffList); - - // TODO: [SNAPSHOT] Remove the workaround below when the SnapDiff logic - // can read tombstones in SST files. - // Workaround: Append "From DB" SST files to the deltaFiles list so that - // the current SnapDiff logic correctly handles deleted keys. - if (!deltaFiles.isEmpty()) { - Set fromSnapshotFiles = RdbUtil.getSSTFilesForComparison( - fromSnapshot.getMetadataManager() - .getStore().getDbLocation().getPath(), - tablesToLookUp); - deltaFiles.addAll(fromSnapshotFiles); - } - // End of Workaround } if (useFullDiff || deltaFiles.isEmpty()) { From cff1971a2976c3b7fc30e179d383bd77e49b2c9a Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 10 Mar 2023 10:58:15 -0800 Subject: [PATCH 02/14] HDDS-8137: Fix checkstyle issue --- .../org/apache/hadoop/hdds/HddsUtils.java | 4 + .../db/managed/ManagedSSTDumpIterator.java | 24 +----- .../rocksdb/util/ManagedSstFileReader.java | 76 +++++++++---------- .../om/snapshot/SnapshotDiffManager.java | 40 +++++----- 4 files changed, 59 insertions(+), 85 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 8f908e60ac5d..9ab3176dcca5 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -803,6 +803,10 @@ public static Map processForLogging(OzoneConfiguration conf) { return sortedOzoneProps; } + /** + * Interface for Implementing CloseableIterators. + * @param Generic Parameter for Iterating values of type 'T' + */ public interface CloseableIterator extends Iterator, Closeable { } diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java index 179e213f3b60..94a37be81312 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java @@ -27,7 +27,6 @@ import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.NoSuchElementException; -import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -118,7 +117,7 @@ public boolean hasNext() { } /** - * Transform function to transform key to a certain value + * Transform function to transform key to a certain value. * @param value * @return */ @@ -231,25 +230,4 @@ public String toString() { '}'; } } - - public static void main(String[] args) throws NativeLibraryNotLoadedException, IOException { - ManagedSSTDumpTool sstDumpTool = - new ManagedSSTDumpTool(new ForkJoinPool(), 50); - try (ManagedOptions options = new ManagedOptions(); - ManagedSSTDumpIterator iterator = new ManagedSSTDumpIterator(sstDumpTool, - "/Users/sbalachandran/Documents/code/dummyrocks/rocks/000025.sst", options) { - @Override - protected KeyValue getTransformedValue(KeyValue value) { - return value; - } - }; - ) { - while (iterator.hasNext()) { - System.out.println(iterator.next()); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - - } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index 67fa6e31fec8..364a960475ca 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -29,17 +29,10 @@ import org.rocksdb.SstFileReaderIterator; import java.io.IOException; -import java.io.InvalidObjectException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Iterator; import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.Spliterator; -import java.util.Spliterators; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import static java.nio.charset.StandardCharsets.UTF_8; @@ -62,25 +55,26 @@ public Stream getKeyStream() throws RocksDBException, ReadOptions readOptions = new ReadOptions(); final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { - @Override - protected HddsUtils.CloseableIterator initNewKeyIteratorForFile( - String file) throws RocksDBException { - return new ManagedSstFileIterator(file, options, - readOptions) { @Override - protected String getIteratorValue(SstFileReaderIterator iterator) { - return new String(iterator.key(), UTF_8); + protected HddsUtils.CloseableIterator + initNewKeyIteratorForFile(String file) throws RocksDBException { + return new ManagedSstFileIterator(file, options, + readOptions) { + @Override + protected String getIteratorValue( + SstFileReaderIterator iterator) { + return new String(iterator.key(), UTF_8); + } + }; } - }; - } - @Override - public void close() throws IOException { - super.close(); - options.close(); - readOptions.close(); - } - }; + @Override + public void close() throws IOException { + super.close(); + options.close(); + readOptions.close(); + } + }; return RdbUtil.getStreamFromIterator(itr); } public Stream getKeyStreamWithTombstone( @@ -90,25 +84,25 @@ public Stream getKeyStreamWithTombstone( ManagedOptions options = new ManagedOptions(); final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { - @Override - protected HddsUtils.CloseableIterator initNewKeyIteratorForFile( - String file) throws NativeLibraryNotLoadedException, - IOException { - return new ManagedSSTDumpIterator(sstDumpTool, file, - options) { @Override - protected String getTransformedValue(KeyValue value) { - return value.getKey(); + protected HddsUtils.CloseableIterator + initNewKeyIteratorForFile(String file) + throws NativeLibraryNotLoadedException, IOException { + return new ManagedSSTDumpIterator(sstDumpTool, file, + options) { + @Override + protected String getTransformedValue(KeyValue value) { + return value.getKey(); + } + }; } - }; - } - @Override - public void close() throws IOException { - super.close(); - options.close(); - } - }; + @Override + public void close() throws IOException { + super.close(); + options.close(); + } + }; return RdbUtil.getStreamFromIterator(itr); } @@ -117,8 +111,8 @@ private abstract static class ManagedSstFileIterator implements private SstFileReader fileReader; private SstFileReaderIterator fileReaderIterator; - public ManagedSstFileIterator(String path, ManagedOptions options, - ReadOptions readOptions) + ManagedSstFileIterator(String path, ManagedOptions options, + ReadOptions readOptions) throws RocksDBException { this.fileReader = new SstFileReader(options); this.fileReader.open(path); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 4f85a7094ed1..bebf132e05f4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -32,6 +32,7 @@ import java.util.UUID; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.NativeConstants; import org.apache.hadoop.hdds.utils.NativeLibraryLoader; @@ -207,13 +208,13 @@ public SnapshotDiffManager(ManagedRocksDB db, } } - private void initSSTDumpTool(OzoneConfiguration configuration) + private void initSSTDumpTool(OzoneConfiguration conf) throws NativeLibraryNotLoadedException { - int threadPoolSize = configuration.getInt( + int threadPoolSize = conf.getInt( OzoneConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE, OzoneConfigKeys .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT); - int bufferSize = (int)configuration.getStorageSize( + int bufferSize = (int)conf.getStorageSize( OzoneConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE, OzoneConfigKeys .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT, @@ -545,11 +546,11 @@ private void generateSnapshotDiffReport(final String jobKey, try { addToObjectIdMap(fsKeyTable, tsKeyTable, - deltaFilesForKeyOrFileTable, + Pair.of(isNativeRocksToolsLoaded, deltaFilesForKeyOrFileTable), objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot, objectIDsToCheckMap, - tablePrefixes, isNativeRocksToolsLoaded); + tablePrefixes); } catch (NativeLibraryNotLoadedException e) { // Workaround to handle deletes if use of native rockstools for reading // tombstone fails. @@ -560,11 +561,11 @@ private void generateSnapshotDiffReport(final String jobKey, try { addToObjectIdMap(fsKeyTable, tsKeyTable, - deltaFilesForKeyOrFileTable, + Pair.of(false, deltaFilesForKeyOrFileTable), objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot, objectIDsToCheckMap, - tablePrefixes, false); + tablePrefixes); } catch (NativeLibraryNotLoadedException ex) { //This code should never be never executed. throw new RuntimeException(ex); @@ -587,11 +588,11 @@ private void generateSnapshotDiffReport(final String jobKey, try { addToObjectIdMap(fsDirTable, tsDirTable, - deltaFilesForDirTable, + Pair.of(isNativeRocksToolsLoaded, deltaFilesForDirTable), objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot, objectIDsToCheckMap, - tablePrefixes, isNativeRocksToolsLoaded); + tablePrefixes); } catch (NativeLibraryNotLoadedException e) { try { // Workaround to handle deletes if use of native rockstools for @@ -602,11 +603,11 @@ private void generateSnapshotDiffReport(final String jobKey, fromSnapshot, tablesToLookUp)); addToObjectIdMap(fsDirTable, tsDirTable, - deltaFilesForDirTable, + Pair.of(false, deltaFilesForDirTable), objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot, objectIDsToCheckMap, - tablePrefixes, false); + tablePrefixes); } catch (NativeLibraryNotLoadedException ex) { //This code should never be never executed. throw new RuntimeException(ex); @@ -633,28 +634,25 @@ private void generateSnapshotDiffReport(final String jobKey, private void addToObjectIdMap(Table fsTable, Table tsTable, - Set deltaFiles, + Pair> + isNativeRocksToolsLoadedDeltaFilesPair, PersistentMap oldObjIdToKeyMap, PersistentMap newObjIdToKeyMap, PersistentSet objectIDsToCheck, - Map tablePrefixes, - boolean isNativeRocksToolsLoaded) + Map tablePrefixes) throws IOException, NativeLibraryNotLoadedException { - if (deltaFiles.isEmpty()) { - return; - } - + Set deltaFiles = isNativeRocksToolsLoadedDeltaFilesPair.getRight(); + boolean nativeRocksToolsLoaded = + isNativeRocksToolsLoadedDeltaFilesPair.getLeft(); boolean isDirectoryTable = fsTable.getName().equals(OmMetadataManagerImpl.DIRECTORY_TABLE); - try (Stream keysToCheck = new ManagedSstFileReader(deltaFiles) - .getKeyStream()) { if (deltaFiles.isEmpty()) { return; } ManagedSstFileReader sstFileReader = new ManagedSstFileReader(deltaFiles); - try (Stream keysToCheck = isNativeRocksToolsLoaded + try (Stream keysToCheck = nativeRocksToolsLoaded ? sstFileReader.getKeyStreamWithTombstone(sstDumpTool) : sstFileReader.getKeyStream()) { keysToCheck.forEach(key -> { From 98746139a87532d67b65c5e95ec3c3422c599a6e Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sun, 12 Mar 2023 21:46:32 -0700 Subject: [PATCH 03/14] HDDS-8137: Remove redundant dependency --- hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml index d4f4a958e939..3c3764d3865e 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml +++ b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml @@ -77,10 +77,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.ozone hdds-rocks-native - - org.apache.ozone - hdds-rocks-native - From ca045d4f57165d829797d0808b6e7a9e7b05f801 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 21 Mar 2023 08:46:16 -0700 Subject: [PATCH 04/14] HDDS-8137: Address review comments --- .../org/apache/hadoop/hdds/HddsUtils.java | 7 -- .../hadoop/hdds/utils/CloseableIterator.java | 13 ++++ .../apache/hadoop/ozone/OzoneConfigKeys.java | 8 +-- .../db/managed/ManagedSSTDumpIterator.java | 10 +-- .../rocksdb/util/ManagedSstFileReader.java | 71 +++++++++++-------- .../apache/ozone/rocksdb/util/RdbUtil.java | 20 ------ .../om/snapshot/SnapshotDiffManager.java | 12 ++-- 7 files changed, 71 insertions(+), 70 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/CloseableIterator.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 9ab3176dcca5..3feb1731b543 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -803,11 +803,4 @@ public static Map processForLogging(OzoneConfiguration conf) { return sortedOzoneProps; } - /** - * Interface for Implementing CloseableIterators. - * @param Generic Parameter for Iterating values of type 'T' - */ - public interface CloseableIterator extends Iterator, Closeable { - - } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/CloseableIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/CloseableIterator.java new file mode 100644 index 000000000000..b3dd0f24d248 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/CloseableIterator.java @@ -0,0 +1,13 @@ +package org.apache.hadoop.hdds.utils; + +import java.io.Closeable; +import java.util.Iterator; + +/** + * Interface for Implementing CloseableIterators. + * + * @param Generic Parameter for Iterating values of type 'T' + */ +public interface CloseableIterator extends Iterator, Closeable { + +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 0c1021cbda77..c680e23ad946 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -618,17 +618,17 @@ public final class OzoneConfigKeys { public static final String OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE = - "ozone.om.snapshot.sst_dumptool.pool.size"; + "ozone.om.snapshot.sst_dumptool.pool.size"; public static final int OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT = 1; public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE = - "ozone.om.snapshot.sst_dumptool.buffer.size"; + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE = + "ozone.om.snapshot.sst_dumptool.buffer.size"; public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT = "8KB"; + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT = "8KB"; public static final long diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java index 94a37be81312..322dec139f5c 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.utils.db.managed; -import org.apache.hadoop.hdds.HddsUtils; +import org.apache.hadoop.hdds.utils.CloseableIterator; import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.eclipse.jetty.io.RuntimeIOException; @@ -35,7 +35,7 @@ * Iterator to Parse output of RocksDBSSTDumpTool. */ public abstract class ManagedSSTDumpIterator implements - HddsUtils.CloseableIterator { + CloseableIterator { private static final String PATTERN_REGEX = "'([^=>]+)' seq:([0-9]+), type:([0-9]+) => "; @@ -117,9 +117,9 @@ public boolean hasNext() { } /** - * Transform function to transform key to a certain value. + * Transforms Key to a certain value. * @param value - * @return + * @return transformed Value */ protected abstract T getTransformedValue(KeyValue value); @@ -148,7 +148,7 @@ public T next() { Math.max(stdoutString.length() - 1, 0))); return getTransformedValue(currentKey); } - throw new NoSuchElementException("No more records found"); + throw new NoSuchElementException("No more elements found"); } stdoutString.append(charBuffer, 0, numberOfCharsRead); currentMatcher.reset(); diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index 364a960475ca..89a3deb5f284 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -18,9 +18,10 @@ package org.apache.ozone.rocksdb.util; -import org.apache.hadoop.hdds.HddsUtils; +import org.apache.hadoop.hdds.utils.CloseableIterator; import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; +import org.apache.hadoop.hdds.utils.db.managed.ManagedReadOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool; import org.rocksdb.ReadOptions; @@ -29,10 +30,14 @@ import org.rocksdb.SstFileReaderIterator; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Collection; import java.util.Iterator; import java.util.NoSuchElementException; +import java.util.Spliterator; +import java.util.Spliterators; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import static java.nio.charset.StandardCharsets.UTF_8; @@ -48,18 +53,30 @@ public ManagedSstFileReader(final Collection sstFiles) { this.sstFiles = sstFiles; } + public static Stream getStreamFromIterator( + CloseableIterator itr) { + final Spliterator spliterator = Spliterators + .spliteratorUnknownSize(itr, 0); + return StreamSupport.stream(spliterator, false).onClose(() -> { + try { + itr.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + public Stream getKeyStream() throws RocksDBException, NativeLibraryNotLoadedException, IOException { - //TODO: [SNAPSHOT] Check if default Options and ReadOptions is enough. + // TODO: [SNAPSHOT] Check if default Options and ReadOptions is enough. ManagedOptions options = new ManagedOptions(); - ReadOptions readOptions = new ReadOptions(); + ReadOptions readOptions = new ManagedReadOptions(); final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { @Override - protected HddsUtils.CloseableIterator - initNewKeyIteratorForFile(String file) throws RocksDBException { - return new ManagedSstFileIterator(file, options, - readOptions) { + protected CloseableIterator + getKeyIteratorForFile(String file) throws RocksDBException { + return new ManagedSstFileIterator(file, options, readOptions) { @Override protected String getIteratorValue( SstFileReaderIterator iterator) { @@ -75,18 +92,18 @@ public void close() throws IOException { readOptions.close(); } }; - return RdbUtil.getStreamFromIterator(itr); + return getStreamFromIterator(itr); } public Stream getKeyStreamWithTombstone( ManagedSSTDumpTool sstDumpTool) throws IOException, RocksDBException, NativeLibraryNotLoadedException { - //TODO: [SNAPSHOT] Check if default Options is enough. - ManagedOptions options = new ManagedOptions(); final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { + //TODO: [SNAPSHOT] Check if default Options is enough. + private ManagedOptions options = new ManagedOptions(); @Override - protected HddsUtils.CloseableIterator - initNewKeyIteratorForFile(String file) + protected CloseableIterator + getKeyIteratorForFile(String file) throws NativeLibraryNotLoadedException, IOException { return new ManagedSSTDumpIterator(sstDumpTool, file, options) { @@ -103,11 +120,11 @@ public void close() throws IOException { options.close(); } }; - return RdbUtil.getStreamFromIterator(itr); + return getStreamFromIterator(itr); } - private abstract static class ManagedSstFileIterator implements - HddsUtils.CloseableIterator { + private abstract static class ManagedSstFileIterator implements + CloseableIterator { private SstFileReader fileReader; private SstFileReaderIterator fileReaderIterator; @@ -131,24 +148,23 @@ public boolean hasNext() { return fileReaderIterator.isValid(); } - protected abstract T getIteratorValue(SstFileReaderIterator iterator); + protected abstract String getIteratorValue(SstFileReaderIterator iterator); @Override - public T next() { - T value = getIteratorValue(fileReaderIterator); + public String next() { + String value = getIteratorValue(fileReaderIterator); fileReaderIterator.next(); return value; } } private abstract static class MultipleSstFileIterator implements - HddsUtils.CloseableIterator { + CloseableIterator { private final Iterator fileNameIterator; private String currentFile; - private SstFileReader currentFileReader; - private HddsUtils.CloseableIterator currentFileIterator; + private CloseableIterator currentFileIterator; private MultipleSstFileIterator(Collection files) throws IOException, RocksDBException, @@ -157,9 +173,9 @@ private MultipleSstFileIterator(Collection files) moveToNextFile(); } - protected abstract HddsUtils.CloseableIterator initNewKeyIteratorForFile( - String file) throws RocksDBException, - NativeLibraryNotLoadedException, IOException; + protected abstract CloseableIterator getKeyIteratorForFile(String file) + throws RocksDBException, NativeLibraryNotLoadedException, + IOException; @Override public boolean hasNext() { @@ -171,7 +187,7 @@ public boolean hasNext() { } while (moveToNextFile()); } catch (IOException | RocksDBException | NativeLibraryNotLoadedException e) { - // TODO: This exception has to be handled by the caller. + // TODO: [Snapshot] This exception has to be handled by the caller. // We have to do better exception handling. throw new RuntimeException(e); } @@ -182,9 +198,8 @@ public boolean hasNext() { public T next() { if (hasNext()) { return currentFileIterator.next(); -// final String value = new String(currentFileIterator.key(), UTF_8); } - throw new NoSuchElementException("No more keys"); + throw new NoSuchElementException("No more elements found."); } @Override @@ -197,7 +212,7 @@ private boolean moveToNextFile() throws IOException, RocksDBException, if (fileNameIterator.hasNext()) { closeCurrentFile(); currentFile = fileNameIterator.next(); - this.currentFileIterator = initNewKeyIteratorForFile(currentFile); + this.currentFileIterator = getKeyIteratorForFile(currentFile); return true; } return false; diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java index a7a018d407b5..172209c584b5 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java @@ -20,24 +20,17 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; -import org.apache.hadoop.hdds.HddsUtils; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.Spliterator; -import java.util.Spliterators; import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; /** * Temporary class to test snapshot diff functionality. @@ -66,17 +59,4 @@ public static Set getSSTFilesForComparison(final String dbLocation, } } - public static Stream getStreamFromIterator( - HddsUtils.CloseableIterator itr) { - final Spliterator spliterator = Spliterators - .spliteratorUnknownSize(itr, 0); - return StreamSupport.stream(spliterator, false).onClose(() -> { - try { - itr.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - } - } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index bebf132e05f4..28be7e8840d4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -202,7 +202,7 @@ public SnapshotDiffManager(ManagedRocksDB db, initSSTDumpTool(configuration); isNativeRocksToolsLoaded = true; } catch (NativeLibraryNotLoadedException e) { - LOG.error("Unable to load SSTDumpTool ", e); + LOG.error("Unable to load SSTDumpTool", e); isNativeRocksToolsLoaded = false; } } @@ -214,7 +214,7 @@ private void initSSTDumpTool(OzoneConfiguration conf) OzoneConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE, OzoneConfigKeys .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT); - int bufferSize = (int)conf.getStorageSize( + int bufferSize = (int) conf.getStorageSize( OzoneConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE, OzoneConfigKeys .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT, @@ -567,8 +567,8 @@ private void generateSnapshotDiffReport(final String jobKey, objectIDsToCheckMap, tablePrefixes); } catch (NativeLibraryNotLoadedException ex) { - //This code should never be never executed. - throw new RuntimeException(ex); + //This code should be never executed. + throw new IllegalStateException(ex); } } @@ -609,8 +609,8 @@ private void generateSnapshotDiffReport(final String jobKey, objectIDsToCheckMap, tablePrefixes); } catch (NativeLibraryNotLoadedException ex) { - //This code should never be never executed. - throw new RuntimeException(ex); + //This code should be never executed. + throw new IllegalStateException(ex); } } } From 8b90667d8c10071e8a85311a966eb56182312e4e Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 21 Mar 2023 08:52:45 -0700 Subject: [PATCH 05/14] HDDS-8137: Fix checkstyle issue --- .../src/main/java/org/apache/hadoop/hdds/HddsUtils.java | 2 -- .../main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 ++--- .../org/apache/ozone/rocksdb/util/ManagedSstFileReader.java | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 3feb1731b543..09bbef83a7cd 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -20,7 +20,6 @@ import com.google.protobuf.ServiceException; import javax.management.ObjectName; -import java.io.Closeable; import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -31,7 +30,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index c680e23ad946..a66369d62e63 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -624,12 +624,11 @@ public final class OzoneConfigKeys { OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT = 1; public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE = + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE = "ozone.om.snapshot.sst_dumptool.buffer.size"; public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT = "8KB"; - + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT = "8KB"; public static final long OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT = diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index 89a3deb5f284..cade37ddd9e3 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -75,7 +75,7 @@ public Stream getKeyStream() throws RocksDBException, new MultipleSstFileIterator(sstFiles) { @Override protected CloseableIterator - getKeyIteratorForFile(String file) throws RocksDBException { + getKeyIteratorForFile(String file) throws RocksDBException { return new ManagedSstFileIterator(file, options, readOptions) { @Override protected String getIteratorValue( @@ -103,7 +103,7 @@ public Stream getKeyStreamWithTombstone( private ManagedOptions options = new ManagedOptions(); @Override protected CloseableIterator - getKeyIteratorForFile(String file) + getKeyIteratorForFile(String file) throws NativeLibraryNotLoadedException, IOException { return new ManagedSSTDumpIterator(sstDumpTool, file, options) { From 2bba6b0f27d0113841dd28111db34d7e787b19a4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 21 Mar 2023 10:48:03 -0700 Subject: [PATCH 06/14] HDDS-8137: Add property to ozone-default --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 4e63e7cc5b0a..f8f908b90980 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3665,4 +3665,13 @@ Threadpool size for SST Dumptool which would be used for computing snapdiff when native library is enabled. + + + ozone.om.snapshot.sst_dumptool.buffer.size + 8KB + OZONE, OM + + Buffer size for SST Dumptool Pipe which would be used for computing snapdiff when native library is enabled. + + From 9af68a8f31d3ab8af14ba6fa97eb10a4592a704e Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 21 Mar 2023 21:51:20 -0700 Subject: [PATCH 07/14] HDDS-8137: Add line --- .../java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index cade37ddd9e3..6babd936e416 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -94,6 +94,7 @@ public void close() throws IOException { }; return getStreamFromIterator(itr); } + public Stream getKeyStreamWithTombstone( ManagedSSTDumpTool sstDumpTool) throws IOException, RocksDBException, NativeLibraryNotLoadedException { From 6df18381a36fb821532e5916768f94ed3de17718 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 29 Mar 2023 13:30:35 -0700 Subject: [PATCH 08/14] HDDS-8137. Address review comments --- .../rocksdb/util/ManagedSstFileReader.java | 4 +- .../om/snapshot/SnapshotDiffManager.java | 90 ++++++------------- 2 files changed, 30 insertions(+), 64 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index 6babd936e416..ee31a6fd4428 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -69,10 +69,10 @@ public static Stream getStreamFromIterator( public Stream getKeyStream() throws RocksDBException, NativeLibraryNotLoadedException, IOException { // TODO: [SNAPSHOT] Check if default Options and ReadOptions is enough. - ManagedOptions options = new ManagedOptions(); - ReadOptions readOptions = new ManagedReadOptions(); final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { + private ManagedOptions options = new ManagedOptions(); + private ReadOptions readOptions = new ManagedReadOptions(); @Override protected CloseableIterator getKeyIteratorForFile(String file) throws RocksDBException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 28be7e8840d4..b921ebcc95f3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -81,19 +81,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getSnapshotInfo; @@ -471,7 +462,6 @@ private void generateSnapshotDiffReport(final String jobKey, ColumnFamilyHandle fromSnapshotColumnFamily = null; ColumnFamilyHandle toSnapshotColumnFamily = null; ColumnFamilyHandle objectIDsColumnFamily = null; - try { // JobId is prepended to column families name to make them unique // for request. @@ -491,30 +481,19 @@ private void generateSnapshotDiffReport(final String jobKey, // Note: Store objectId and keyName as byte array to reduce unnecessary // serialization and deserialization. final PersistentMap objectIdToKeyNameMapForFromSnapshot = - new RocksDbPersistentMap<>(db, - fromSnapshotColumnFamily, - codecRegistry, - byte[].class, - byte[].class); - + new RocksDbPersistentMap<>(db, fromSnapshotColumnFamily, + codecRegistry, byte[].class, byte[].class); // ObjectId to keyName map to keep key info for toSnapshot. final PersistentMap objectIdToKeyNameMapForToSnapshot = - new RocksDbPersistentMap<>(db, - toSnapshotColumnFamily, - codecRegistry, - byte[].class, - byte[].class); - + new RocksDbPersistentMap<>(db, toSnapshotColumnFamily, codecRegistry, + byte[].class, byte[].class); // Set of unique objectId between fromSnapshot and toSnapshot. final PersistentSet objectIDsToCheckMap = - new RocksDbPersistentSet<>(db, - objectIDsColumnFamily, - codecRegistry, + new RocksDbPersistentSet<>(db, objectIDsColumnFamily, codecRegistry, byte[].class); final BucketLayout bucketLayout = getBucketLayout(volume, bucket, fromSnapshot.getMetadataManager()); - final Table fsKeyTable = fromSnapshot.getMetadataManager().getKeyTable(bucketLayout); final Table tsKeyTable = @@ -537,37 +516,31 @@ private void generateSnapshotDiffReport(final String jobKey, // Workaround to handle deletes if native rockstools for reading // tombstone is not loaded. - // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read to - // read tombstone + // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read tombstone if (!isNativeRocksToolsLoaded) { deltaFilesForKeyOrFileTable.addAll(getSSTFileListForSnapshot( fromSnapshot, tablesToLookUp)); } try { - addToObjectIdMap(fsKeyTable, - tsKeyTable, + addToObjectIdMap(fsKeyTable, tsKeyTable, Pair.of(isNativeRocksToolsLoaded, deltaFilesForKeyOrFileTable), objectIdToKeyNameMapForFromSnapshot, - objectIdToKeyNameMapForToSnapshot, - objectIDsToCheckMap, + objectIdToKeyNameMapForToSnapshot, objectIDsToCheckMap, tablePrefixes); } catch (NativeLibraryNotLoadedException e) { // Workaround to handle deletes if use of native rockstools for reading // tombstone fails. - // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read to - // read tombstone + // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read tombstone deltaFilesForKeyOrFileTable.addAll(getSSTFileListForSnapshot( fromSnapshot, tablesToLookUp)); try { - addToObjectIdMap(fsKeyTable, - tsKeyTable, - Pair.of(false, deltaFilesForKeyOrFileTable), - objectIdToKeyNameMapForFromSnapshot, - objectIdToKeyNameMapForToSnapshot, - objectIDsToCheckMap, - tablePrefixes); + addToObjectIdMap(fsKeyTable, tsKeyTable, + Pair.of(false, deltaFilesForKeyOrFileTable), + objectIdToKeyNameMapForFromSnapshot, + objectIdToKeyNameMapForToSnapshot, objectIDsToCheckMap, + tablePrefixes); } catch (NativeLibraryNotLoadedException ex) { - //This code should be never executed. + // This code should be never executed. throw new IllegalStateException(ex); } } @@ -586,8 +559,7 @@ private void generateSnapshotDiffReport(final String jobKey, fromSnapshot, tablesToLookUp)); } try { - addToObjectIdMap(fsDirTable, - tsDirTable, + addToObjectIdMap(fsDirTable, tsDirTable, Pair.of(isNativeRocksToolsLoaded, deltaFilesForDirTable), objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot, @@ -597,26 +569,21 @@ private void generateSnapshotDiffReport(final String jobKey, try { // Workaround to handle deletes if use of native rockstools for // reading tombstone fails. - // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read to - // read tombstone + // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read tombstone deltaFilesForDirTable.addAll(getSSTFileListForSnapshot( fromSnapshot, tablesToLookUp)); - addToObjectIdMap(fsDirTable, - tsDirTable, - Pair.of(false, deltaFilesForDirTable), - objectIdToKeyNameMapForFromSnapshot, - objectIdToKeyNameMapForToSnapshot, - objectIDsToCheckMap, - tablePrefixes); + addToObjectIdMap(fsDirTable, tsDirTable, + Pair.of(false, deltaFilesForDirTable), + objectIdToKeyNameMapForFromSnapshot, + objectIdToKeyNameMapForToSnapshot, objectIDsToCheckMap, + tablePrefixes); } catch (NativeLibraryNotLoadedException ex) { - //This code should be never executed. + // This code should be never executed. throw new IllegalStateException(ex); } } } - - generateDiffReport(jobId, - objectIDsToCheckMap, + generateDiffReport(jobId, objectIDsToCheckMap, objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot); updateJobStatus(jobKey, IN_PROGRESS, DONE); @@ -643,14 +610,13 @@ private void addToObjectIdMap(Table fsTable, throws IOException, NativeLibraryNotLoadedException { Set deltaFiles = isNativeRocksToolsLoadedDeltaFilesPair.getRight(); - boolean nativeRocksToolsLoaded = - isNativeRocksToolsLoadedDeltaFilesPair.getLeft(); - boolean isDirectoryTable = - fsTable.getName().equals(OmMetadataManagerImpl.DIRECTORY_TABLE); - if (deltaFiles.isEmpty()) { return; } + boolean nativeRocksToolsLoaded = + isNativeRocksToolsLoadedDeltaFilesPair.getLeft(); + boolean isDirectoryTable = + fsTable.getName().equals(OmMetadataManagerImpl.DIRECTORY_TABLE); ManagedSstFileReader sstFileReader = new ManagedSstFileReader(deltaFiles); try (Stream keysToCheck = nativeRocksToolsLoaded ? sstFileReader.getKeyStreamWithTombstone(sstDumpTool) From 3426d682e729e8ae7333f98397a480a877de4f1b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 29 Mar 2023 14:57:04 -0700 Subject: [PATCH 09/14] HDDS-8137. Address review comments --- .../hadoop/ozone/om/snapshot/SnapshotDiffManager.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index b921ebcc95f3..976bddd5fe73 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -81,10 +81,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getSnapshotInfo; @@ -210,13 +207,13 @@ private void initSSTDumpTool(OzoneConfiguration conf) OzoneConfigKeys .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT, StorageUnit.BYTES); - ExecutorService executorService = new ThreadPoolExecutor(0, + ExecutorService execService = new ThreadPoolExecutor(0, threadPoolSize, 60, TimeUnit.SECONDS, new SynchronousQueue<>(), new ThreadFactoryBuilder() .setNameFormat("snapshot-diff-manager-sst-dump-tool-TID-%d") .build(), new ThreadPoolExecutor.DiscardPolicy()); - sstDumpTool = new ManagedSSTDumpTool(executorService, bufferSize); + sstDumpTool = new ManagedSSTDumpTool(execService, bufferSize); } private Map getTablePrefixes( @@ -257,7 +254,7 @@ private DifferSnapshotInfo getDSIFromSI(SnapshotInfo snapshotInfo, } private Set getSSTFileListForSnapshot(OmSnapshot snapshot, - List tablesToLookUp) throws RocksDBException { + List tablesToLookUp) throws RocksDBException { return RdbUtil.getSSTFilesForComparison(snapshot .getMetadataManager().getStore().getDbLocation() .getPath(), tablesToLookUp); From fbd44556857d908ee1385677f9573f322f57bf56 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 3 Apr 2023 21:06:50 -0700 Subject: [PATCH 10/14] HDDS-8137. Address review comments --- .../hadoop/hdds/utils/CloseableIterator.java | 13 ----- .../apache/hadoop/ozone/OzoneConfigKeys.java | 14 ----- .../apache/hadoop/util/ClosableIterator.java | 5 +- .../db/managed/ManagedSSTDumpIterator.java | 4 +- .../rocksdb/util/ManagedSstFileReader.java | 18 +++---- .../apache/hadoop/ozone/om/OMConfigKeys.java | 11 ++++ .../om/snapshot/RocksDbPersistentList.java | 2 + .../om/snapshot/SnapshotDiffManager.java | 51 ++++++++++--------- 8 files changed, 52 insertions(+), 66 deletions(-) delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/CloseableIterator.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/CloseableIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/CloseableIterator.java deleted file mode 100644 index b3dd0f24d248..000000000000 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/CloseableIterator.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.apache.hadoop.hdds.utils; - -import java.io.Closeable; -import java.util.Iterator; - -/** - * Interface for Implementing CloseableIterators. - * - * @param Generic Parameter for Iterating values of type 'T' - */ -public interface CloseableIterator extends Iterator, Closeable { - -} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index a66369d62e63..ede1b4fd8740 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -616,20 +616,6 @@ public final class OzoneConfigKeys { OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL = "ozone.om.snapshot.compaction.dag.prune.daemon.run.interval"; - public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE = - "ozone.om.snapshot.sst_dumptool.pool.size"; - - public static final int - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT = 1; - - public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE = - "ozone.om.snapshot.sst_dumptool.buffer.size"; - - public static final String - OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT = "8KB"; - public static final long OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT = TimeUnit.HOURS.toMillis(1); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java index c014b5e4f354..eaf11c8e79b4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java @@ -17,12 +17,11 @@ */ package org.apache.hadoop.util; +import java.io.Closeable; import java.util.Iterator; /** * An {@link Iterator} that may hold resources until it is closed. */ -public interface ClosableIterator extends Iterator, AutoCloseable { - @Override - void close(); +public interface ClosableIterator extends Iterator, Closeable { } diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java index 322dec139f5c..a6e0e656614b 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.utils.db.managed; -import org.apache.hadoop.hdds.utils.CloseableIterator; +import org.apache.hadoop.util.ClosableIterator; import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.eclipse.jetty.io.RuntimeIOException; @@ -35,7 +35,7 @@ * Iterator to Parse output of RocksDBSSTDumpTool. */ public abstract class ManagedSSTDumpIterator implements - CloseableIterator { + ClosableIterator { private static final String PATTERN_REGEX = "'([^=>]+)' seq:([0-9]+), type:([0-9]+) => "; diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index ee31a6fd4428..d1db7c0ab3a4 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -18,7 +18,7 @@ package org.apache.ozone.rocksdb.util; -import org.apache.hadoop.hdds.utils.CloseableIterator; +import org.apache.hadoop.util.ClosableIterator; import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedReadOptions; @@ -54,7 +54,7 @@ public ManagedSstFileReader(final Collection sstFiles) { } public static Stream getStreamFromIterator( - CloseableIterator itr) { + ClosableIterator itr) { final Spliterator spliterator = Spliterators .spliteratorUnknownSize(itr, 0); return StreamSupport.stream(spliterator, false).onClose(() -> { @@ -74,7 +74,7 @@ public Stream getKeyStream() throws RocksDBException, private ManagedOptions options = new ManagedOptions(); private ReadOptions readOptions = new ManagedReadOptions(); @Override - protected CloseableIterator + protected ClosableIterator getKeyIteratorForFile(String file) throws RocksDBException { return new ManagedSstFileIterator(file, options, readOptions) { @Override @@ -103,7 +103,7 @@ public Stream getKeyStreamWithTombstone( //TODO: [SNAPSHOT] Check if default Options is enough. private ManagedOptions options = new ManagedOptions(); @Override - protected CloseableIterator + protected ClosableIterator getKeyIteratorForFile(String file) throws NativeLibraryNotLoadedException, IOException { return new ManagedSSTDumpIterator(sstDumpTool, file, @@ -125,7 +125,7 @@ public void close() throws IOException { } private abstract static class ManagedSstFileIterator implements - CloseableIterator { + ClosableIterator { private SstFileReader fileReader; private SstFileReaderIterator fileReaderIterator; @@ -139,7 +139,7 @@ private abstract static class ManagedSstFileIterator implements } @Override - public void close() throws IOException { + public void close() { this.fileReaderIterator.close(); this.fileReader.close(); } @@ -160,12 +160,12 @@ public String next() { } private abstract static class MultipleSstFileIterator implements - CloseableIterator { + ClosableIterator { private final Iterator fileNameIterator; private String currentFile; - private CloseableIterator currentFileIterator; + private ClosableIterator currentFileIterator; private MultipleSstFileIterator(Collection files) throws IOException, RocksDBException, @@ -174,7 +174,7 @@ private MultipleSstFileIterator(Collection files) moveToNextFile(); } - protected abstract CloseableIterator getKeyIteratorForFile(String file) + protected abstract ClosableIterator getKeyIteratorForFile(String file) throws RocksDBException, NativeLibraryNotLoadedException, IOException; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 937835fdb774..34e17519e0a1 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -28,6 +28,17 @@ * Ozone Manager Constants. */ public final class OMConfigKeys { + public static final String + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE = + "ozone.om.snapshot.sst_dumptool.pool.size"; + public static final int + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT = 1; + public static final String + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE = + "ozone.om.snapshot.sst_dumptool.buffer.size"; + public static final String + OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT = "8KB"; + /** * Never constructed. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java index 373cb5405877..f78d5cbaa5f3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java @@ -65,6 +65,8 @@ public boolean add(E entry) { public boolean addAll(PersistentList from) { try (ClosableIterator iterator = from.iterator()) { iterator.forEachRemaining(this::add); + } catch (Exception e) { + throw new IllegalStateException(e); } return true; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 976bddd5fe73..09a828e33dd3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -33,6 +33,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.NativeConstants; import org.apache.hadoop.hdds.utils.NativeLibraryLoader; @@ -52,6 +53,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; @@ -186,34 +188,33 @@ public SnapshotDiffManager(ManagedRocksDB db, isNativeRocksToolsLoaded = NativeLibraryLoader.getInstance() .loadLibrary(NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME); if (isNativeRocksToolsLoaded) { - try { - initSSTDumpTool(configuration); - isNativeRocksToolsLoaded = true; - } catch (NativeLibraryNotLoadedException e) { - LOG.error("Unable to load SSTDumpTool", e); - isNativeRocksToolsLoaded = false; - } + isNativeRocksToolsLoaded = initSSTDumpTool( + ozoneManager.getConfiguration()); } } - private void initSSTDumpTool(OzoneConfiguration conf) - throws NativeLibraryNotLoadedException { - int threadPoolSize = conf.getInt( - OzoneConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE, - OzoneConfigKeys - .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT); - int bufferSize = (int) conf.getStorageSize( - OzoneConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE, - OzoneConfigKeys - .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT, - StorageUnit.BYTES); - ExecutorService execService = new ThreadPoolExecutor(0, - threadPoolSize, 60, TimeUnit.SECONDS, - new SynchronousQueue<>(), new ThreadFactoryBuilder() - .setNameFormat("snapshot-diff-manager-sst-dump-tool-TID-%d") - .build(), - new ThreadPoolExecutor.DiscardPolicy()); - sstDumpTool = new ManagedSSTDumpTool(execService, bufferSize); + private boolean initSSTDumpTool(OzoneConfiguration conf) { + try { + int threadPoolSize = conf.getInt( + OMConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE, + OMConfigKeys + .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_POOL_SIZE_DEFAULT); + int bufferSize = (int) conf.getStorageSize( + OMConfigKeys.OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE, + OMConfigKeys + .OZONE_OM_SNAPSHOT_SST_DUMPTOOL_EXECUTOR_BUFFER_SIZE_DEFAULT, + StorageUnit.BYTES); + ExecutorService execService = new ThreadPoolExecutor(0, + threadPoolSize, 60, TimeUnit.SECONDS, + new SynchronousQueue<>(), new ThreadFactoryBuilder() + .setNameFormat("snapshot-diff-manager-sst-dump-tool-TID-%d") + .build(), + new ThreadPoolExecutor.DiscardPolicy()); + sstDumpTool = new ManagedSSTDumpTool(execService, bufferSize); + } catch (NativeLibraryNotLoadedException e) { + return false; + } + return true; } private Map getTablePrefixes( From 8c0d72850362afa27f05421d3cb0e8c3f37144cc Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 6 Apr 2023 12:45:05 -0700 Subject: [PATCH 11/14] HDDS-8137. Address review comments --- .../org/apache/hadoop/hdds/HddsUtils.java | 1 - .../db/managed/ManagedSSTDumpIterator.java | 76 +++++++++++-------- .../rocksdb/util/ManagedSstFileReader.java | 42 +++++----- 3 files changed, 66 insertions(+), 53 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 09bbef83a7cd..344dd3e6ffcc 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -800,5 +800,4 @@ public static Map processForLogging(OzoneConfiguration conf) { } return sortedOzoneProps; } - } diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java index a6e0e656614b..b35e68d1fca2 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java @@ -20,31 +20,35 @@ import org.apache.hadoop.util.ClosableIterator; import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException; import org.eclipse.jetty.io.RuntimeIOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.NoSuchElementException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Iterator to Parse output of RocksDBSSTDumpTool. */ -public abstract class ManagedSSTDumpIterator implements - ClosableIterator { +public abstract class ManagedSSTDumpIterator implements ClosableIterator { + private static final Logger LOG = + LoggerFactory.getLogger(ManagedSSTDumpIterator.class); private static final String PATTERN_REGEX = - "'([^=>]+)' seq:([0-9]+), type:([0-9]+) => "; + "'([^=>]+)' seq:([0-9]+), type:([0-9]+) => "; public static final int PATTERN_KEY_GROUP_NUMBER = 1; public static final int PATTERN_SEQ_GROUP_NUMBER = 2; public static final int PATTERN_TYPE_GROUP_NUMBER = 3; - private static final Pattern PATTERN_MATCHER = - Pattern.compile(PATTERN_REGEX); + private static final Pattern PATTERN_MATCHER = Pattern.compile(PATTERN_REGEX); private BufferedReader processOutput; private StringBuilder stdoutString; @@ -56,32 +60,33 @@ public abstract class ManagedSSTDumpIterator implements private ManagedSSTDumpTool.SSTDumpToolTask sstDumpToolTask; private AtomicBoolean open; + private StackTraceElement[] stackTrace; public ManagedSSTDumpIterator(ManagedSSTDumpTool sstDumpTool, - String sstFilePath, - ManagedOptions options) throws IOException, - NativeLibraryNotLoadedException { + String sstFilePath, ManagedOptions options) + throws IOException, NativeLibraryNotLoadedException { File sstFile = new File(sstFilePath); if (!sstFile.exists()) { throw new IOException(String.format("File in path : %s doesn't exist", - sstFile.getAbsolutePath())); + sstFile.getAbsolutePath())); } if (!sstFile.isFile()) { throw new IOException(String.format("Path given: %s is not a file", - sstFile.getAbsolutePath())); + sstFile.getAbsolutePath())); } init(sstDumpTool, sstFile, options); + this.stackTrace = Thread.currentThread().getStackTrace(); } private void init(ManagedSSTDumpTool sstDumpTool, File sstFile, ManagedOptions options) - throws NativeLibraryNotLoadedException { - String[] args = {"--file=" + sstFile.getAbsolutePath(), - "--command=scan"}; + throws NativeLibraryNotLoadedException { + String[] args = {"--file=" + sstFile.getAbsolutePath(), "--command=scan"}; this.sstDumpToolTask = sstDumpTool.run(args, options); - processOutput = new BufferedReader(new InputStreamReader( - sstDumpToolTask.getPipedOutput(), StandardCharsets.UTF_8)); + processOutput = new BufferedReader( + new InputStreamReader(sstDumpToolTask.getPipedOutput(), + StandardCharsets.UTF_8)); stdoutString = new StringBuilder(); currentMatcher = PATTERN_MATCHER.matcher(stdoutString); charBuffer = new char[8192]; @@ -97,15 +102,16 @@ private void checkSanityOfProcess() { if (!this.open.get()) { throw new RuntimeException("Iterator has been closed"); } - if (sstDumpToolTask.getFuture().isDone() - && sstDumpToolTask.exitValue() != 0) { + if (sstDumpToolTask.getFuture().isDone() && + sstDumpToolTask.exitValue() != 0) { throw new RuntimeException("Process Terminated with non zero " + - String.format("exit value %d", sstDumpToolTask.exitValue())); + String.format("exit value %d", sstDumpToolTask.exitValue())); } } /** * Checks the status of the process & sees if there is another record. + * * @return True if next exists & false otherwise * Throws Runtime Exception in case of SST File read failure */ @@ -118,6 +124,7 @@ public boolean hasNext() { /** * Transforms Key to a certain value. + * * @param value * @return transformed Value */ @@ -125,6 +132,7 @@ public boolean hasNext() { /** * Returns the next record from SSTDumpTool. + * * @return next Key * Throws Runtime Exception incase of failure. */ @@ -136,8 +144,8 @@ public T next() { while (!currentMatcher.find()) { try { if (prevMatchEndIndex != 0) { - stdoutString = new StringBuilder(stdoutString.substring( - prevMatchEndIndex, stdoutString.length())); + stdoutString = new StringBuilder( + stdoutString.substring(prevMatchEndIndex, stdoutString.length())); prevMatchEndIndex = 0; currentMatcher = PATTERN_MATCHER.matcher(stdoutString); } @@ -145,7 +153,7 @@ public T next() { if (numberOfCharsRead < 0) { if (currentKey != null) { currentKey.setValue(stdoutString.substring(0, - Math.max(stdoutString.length() - 1, 0))); + Math.max(stdoutString.length() - 1, 0))); return getTransformedValue(currentKey); } throw new NoSuchElementException("No more elements found"); @@ -158,13 +166,12 @@ public T next() { } if (currentKey != null) { currentKey.setValue(stdoutString.substring(prevMatchEndIndex, - currentMatcher.start() - 1)); + currentMatcher.start() - 1)); } prevMatchEndIndex = currentMatcher.end(); - nextKey = new KeyValue( - currentMatcher.group(PATTERN_KEY_GROUP_NUMBER), - currentMatcher.group(PATTERN_SEQ_GROUP_NUMBER), - currentMatcher.group(PATTERN_TYPE_GROUP_NUMBER)); + nextKey = new KeyValue(currentMatcher.group(PATTERN_KEY_GROUP_NUMBER), + currentMatcher.group(PATTERN_SEQ_GROUP_NUMBER), + currentMatcher.group(PATTERN_TYPE_GROUP_NUMBER)); return getTransformedValue(currentKey); } @@ -181,7 +188,16 @@ public synchronized void close() throws IOException { @Override protected void finalize() throws Throwable { + if (open.get()) { + LOG.warn("{} is not closed properly." + + " StackTrace for unclosed instance: {}", + this.getClass().getName(), + Arrays.stream(stackTrace) + .map(StackTraceElement::toString).collect( + Collectors.joining("\n"))); + } this.close(); + super.finalize(); } /** @@ -222,12 +238,8 @@ public String getValue() { @Override public String toString() { - return "KeyValue{" + - "key='" + key + '\'' + - ", sequence=" + sequence + - ", type=" + type + - ", value='" + value + '\'' + - '}'; + return "KeyValue{" + "key='" + key + '\'' + ", sequence=" + sequence + + ", type=" + type + ", value='" + value + '\'' + '}'; } } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index d1db7c0ab3a4..880d3d0ae58e 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -54,9 +54,9 @@ public ManagedSstFileReader(final Collection sstFiles) { } public static Stream getStreamFromIterator( - ClosableIterator itr) { + ClosableIterator itr) { final Spliterator spliterator = Spliterators - .spliteratorUnknownSize(itr, 0); + .spliteratorUnknownSize(itr, 0); return StreamSupport.stream(spliterator, false).onClose(() -> { try { itr.close(); @@ -67,19 +67,20 @@ public static Stream getStreamFromIterator( } public Stream getKeyStream() throws RocksDBException, - NativeLibraryNotLoadedException, IOException { + NativeLibraryNotLoadedException, IOException { // TODO: [SNAPSHOT] Check if default Options and ReadOptions is enough. final MultipleSstFileIterator itr = - new MultipleSstFileIterator(sstFiles) { - private ManagedOptions options = new ManagedOptions(); - private ReadOptions readOptions = new ManagedReadOptions(); + new MultipleSstFileIterator(sstFiles) { + private ManagedOptions options = new ManagedOptions(); + private ReadOptions readOptions = new ManagedReadOptions(); + @Override protected ClosableIterator - getKeyIteratorForFile(String file) throws RocksDBException { + getKeyIteratorForFile(String file) throws RocksDBException { return new ManagedSstFileIterator(file, options, readOptions) { @Override protected String getIteratorValue( - SstFileReaderIterator iterator) { + SstFileReaderIterator iterator) { return new String(iterator.key(), UTF_8); } }; @@ -96,18 +97,19 @@ public void close() throws IOException { } public Stream getKeyStreamWithTombstone( - ManagedSSTDumpTool sstDumpTool) throws IOException, RocksDBException, - NativeLibraryNotLoadedException { + ManagedSSTDumpTool sstDumpTool) throws IOException, RocksDBException, + NativeLibraryNotLoadedException { final MultipleSstFileIterator itr = - new MultipleSstFileIterator(sstFiles) { + new MultipleSstFileIterator(sstFiles) { //TODO: [SNAPSHOT] Check if default Options is enough. private ManagedOptions options = new ManagedOptions(); + @Override protected ClosableIterator - getKeyIteratorForFile(String file) - throws NativeLibraryNotLoadedException, IOException { + getKeyIteratorForFile(String file) + throws NativeLibraryNotLoadedException, IOException { return new ManagedSSTDumpIterator(sstDumpTool, file, - options) { + options) { @Override protected String getTransformedValue(KeyValue value) { return value.getKey(); @@ -131,7 +133,7 @@ private abstract static class ManagedSstFileIterator implements ManagedSstFileIterator(String path, ManagedOptions options, ReadOptions readOptions) - throws RocksDBException { + throws RocksDBException { this.fileReader = new SstFileReader(options); this.fileReader.open(path); this.fileReaderIterator = fileReader.newIterator(readOptions); @@ -168,15 +170,15 @@ private abstract static class MultipleSstFileIterator implements private ClosableIterator currentFileIterator; private MultipleSstFileIterator(Collection files) - throws IOException, RocksDBException, - NativeLibraryNotLoadedException { + throws IOException, RocksDBException, + NativeLibraryNotLoadedException { this.fileNameIterator = files.iterator(); moveToNextFile(); } protected abstract ClosableIterator getKeyIteratorForFile(String file) - throws RocksDBException, NativeLibraryNotLoadedException, - IOException; + throws RocksDBException, NativeLibraryNotLoadedException, + IOException; @Override public boolean hasNext() { @@ -209,7 +211,7 @@ public void close() throws IOException { } private boolean moveToNextFile() throws IOException, RocksDBException, - NativeLibraryNotLoadedException { + NativeLibraryNotLoadedException { if (fileNameIterator.hasNext()) { closeCurrentFile(); currentFile = fileNameIterator.next(); From 6a608d0be21c6d2a3b2c4fc5e50f6145d69ab41d Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 13 Apr 2023 13:36:33 -0700 Subject: [PATCH 12/14] HDDS-8137. Fix indentation --- .../apache/hadoop/util/ClosableIterator.java | 5 +-- .../db/managed/ManagedSSTDumpIterator.java | 9 ++++-- .../rocksdb/util/ManagedSstFileReader.java | 32 +++++++++---------- .../om/snapshot/RocksDbPersistentList.java | 2 -- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java index eaf11c8e79b4..c014b5e4f354 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/ClosableIterator.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.util; -import java.io.Closeable; import java.util.Iterator; /** * An {@link Iterator} that may hold resources until it is closed. */ -public interface ClosableIterator extends Iterator, Closeable { +public interface ClosableIterator extends Iterator, AutoCloseable { + @Override + void close(); } diff --git a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java index b35e68d1fca2..a083a2b98810 100644 --- a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java +++ b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSSTDumpIterator.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.NoSuchElementException; @@ -176,12 +177,16 @@ public T next() { } @Override - public synchronized void close() throws IOException { + public synchronized void close() throws UncheckedIOException { if (this.sstDumpToolTask != null) { if (!this.sstDumpToolTask.getFuture().isDone()) { this.sstDumpToolTask.getFuture().cancel(true); } - this.processOutput.close(); + try { + this.processOutput.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } open.compareAndSet(true, false); } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index 880d3d0ae58e..75c9e95cdb56 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -53,16 +53,11 @@ public ManagedSstFileReader(final Collection sstFiles) { this.sstFiles = sstFiles; } - public static Stream getStreamFromIterator( - ClosableIterator itr) { - final Spliterator spliterator = Spliterators - .spliteratorUnknownSize(itr, 0); + public static Stream getStreamFromIterator(ClosableIterator itr) { + final Spliterator spliterator = + Spliterators.spliteratorUnknownSize(itr, 0); return StreamSupport.stream(spliterator, false).onClose(() -> { - try { - itr.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + itr.close(); }); } @@ -75,8 +70,8 @@ public Stream getKeyStream() throws RocksDBException, private ReadOptions readOptions = new ManagedReadOptions(); @Override - protected ClosableIterator - getKeyIteratorForFile(String file) throws RocksDBException { + protected ClosableIterator getKeyIteratorForFile(String file) + throws RocksDBException { return new ManagedSstFileIterator(file, options, readOptions) { @Override protected String getIteratorValue( @@ -87,7 +82,7 @@ protected String getIteratorValue( } @Override - public void close() throws IOException { + public void close() throws UncheckedIOException { super.close(); options.close(); readOptions.close(); @@ -105,8 +100,7 @@ public Stream getKeyStreamWithTombstone( private ManagedOptions options = new ManagedOptions(); @Override - protected ClosableIterator - getKeyIteratorForFile(String file) + protected ClosableIterator getKeyIteratorForFile(String file) throws NativeLibraryNotLoadedException, IOException { return new ManagedSSTDumpIterator(sstDumpTool, file, options) { @@ -118,7 +112,7 @@ protected String getTransformedValue(KeyValue value) { } @Override - public void close() throws IOException { + public void close() throws UncheckedIOException { super.close(); options.close(); } @@ -206,8 +200,12 @@ public T next() { } @Override - public void close() throws IOException { - closeCurrentFile(); + public void close() throws UncheckedIOException { + try { + closeCurrentFile(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } private boolean moveToNextFile() throws IOException, RocksDBException, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java index f78d5cbaa5f3..373cb5405877 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java @@ -65,8 +65,6 @@ public boolean add(E entry) { public boolean addAll(PersistentList from) { try (ClosableIterator iterator = from.iterator()) { iterator.forEachRemaining(this::add); - } catch (Exception e) { - throw new IllegalStateException(e); } return true; } From d40c908fa44494b5d69b8b9539564766b9c7a224 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sun, 16 Apr 2023 06:44:23 -0700 Subject: [PATCH 13/14] HDDS-8137. Fix null pointer exception --- .../ozone/rocksdb/util/ManagedSstFileReader.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index 75c9e95cdb56..e628d7a7fe78 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -72,6 +72,13 @@ public Stream getKeyStream() throws RocksDBException, @Override protected ClosableIterator getKeyIteratorForFile(String file) throws RocksDBException { + if (this.options == null) { + this.options = new ManagedOptions(); + } + + if (this.readOptions == null) { + this.readOptions = new ManagedReadOptions(); + } return new ManagedSstFileIterator(file, options, readOptions) { @Override protected String getIteratorValue( @@ -97,11 +104,14 @@ public Stream getKeyStreamWithTombstone( final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { //TODO: [SNAPSHOT] Check if default Options is enough. - private ManagedOptions options = new ManagedOptions(); + private ManagedOptions options; @Override protected ClosableIterator getKeyIteratorForFile(String file) throws NativeLibraryNotLoadedException, IOException { + if (this.options == null) { + this.options = new ManagedOptions(); + } return new ManagedSSTDumpIterator(sstDumpTool, file, options) { @Override From 5d73f1f99d6c6e37aade91980ac2a5540765a4bd Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sun, 16 Apr 2023 06:50:35 -0700 Subject: [PATCH 14/14] HDDS-8137. Add init function --- .../rocksdb/util/ManagedSstFileReader.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java index e628d7a7fe78..ce05cd9715d9 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java @@ -66,19 +66,18 @@ public Stream getKeyStream() throws RocksDBException, // TODO: [SNAPSHOT] Check if default Options and ReadOptions is enough. final MultipleSstFileIterator itr = new MultipleSstFileIterator(sstFiles) { - private ManagedOptions options = new ManagedOptions(); - private ReadOptions readOptions = new ManagedReadOptions(); + private ManagedOptions options; + private ReadOptions readOptions; + + @Override + protected void init() { + this.options = new ManagedOptions(); + this.readOptions = new ManagedReadOptions(); + } @Override protected ClosableIterator getKeyIteratorForFile(String file) throws RocksDBException { - if (this.options == null) { - this.options = new ManagedOptions(); - } - - if (this.readOptions == null) { - this.readOptions = new ManagedReadOptions(); - } return new ManagedSstFileIterator(file, options, readOptions) { @Override protected String getIteratorValue( @@ -106,12 +105,14 @@ public Stream getKeyStreamWithTombstone( //TODO: [SNAPSHOT] Check if default Options is enough. private ManagedOptions options; + @Override + protected void init() { + this.options = new ManagedOptions(); + } + @Override protected ClosableIterator getKeyIteratorForFile(String file) throws NativeLibraryNotLoadedException, IOException { - if (this.options == null) { - this.options = new ManagedOptions(); - } return new ManagedSSTDumpIterator(sstDumpTool, file, options) { @Override @@ -177,9 +178,12 @@ private MultipleSstFileIterator(Collection files) throws IOException, RocksDBException, NativeLibraryNotLoadedException { this.fileNameIterator = files.iterator(); + init(); moveToNextFile(); } + protected abstract void init(); + protected abstract ClosableIterator getKeyIteratorForFile(String file) throws RocksDBException, NativeLibraryNotLoadedException, IOException;