From 4d7e2e9c68706a996e1ef399c5b09901f91d1c9b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 29 Aug 2025 12:07:25 -0400 Subject: [PATCH 1/5] HDDS-13612. Track Sst file ranges in sst file metadata Change-Id: I822eb168408b65364bc9e009a5818d7f5dfd907f --- .../org/apache/hadoop/ozone/OzoneConsts.java | 10 +- .../compaction/log/CompactionFileInfo.java | 83 +++----- .../ozone/compaction/log/SstFileInfo.java | 94 +++++++++ .../hadoop/ozone/om/OmSnapshotLocalData.java | 172 +++++++++-------- .../ozone/om/OmSnapshotLocalDataYaml.java | 181 +++++++++++++----- .../hadoop/ozone/om/OmSnapshotManager.java | 19 +- .../ozone/om/TestOmSnapshotLocalDataYaml.java | 131 ++++++++----- .../ozone/om/TestOmSnapshotManager.java | 27 ++- 8 files changed, 463 insertions(+), 254 deletions(-) create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/SstFileInfo.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 1aab4fc28dd5..70b723b21574 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -208,10 +208,16 @@ public final class OzoneConsts { public static final String OM_SLD_VERSION = "version"; public static final String OM_SLD_CHECKSUM = "checksum"; public static final String OM_SLD_IS_SST_FILTERED = "isSSTFiltered"; - public static final String OM_SLD_UNCOMPACTED_SST_FILE_LIST = "uncompactedSSTFileList"; public static final String OM_SLD_LAST_COMPACTION_TIME = "lastCompactionTime"; public static final String OM_SLD_NEEDS_COMPACTION = "needsCompaction"; - public static final String OM_SLD_COMPACTED_SST_FILE_LIST = "compactedSSTFileList"; + public static final String OM_SLD_VERSION_SST_FILE_INFO = "versionSstFileInfos"; + public static final String OM_SLD_PREV_SNAP_ID = "previousSnapshotId"; + public static final String OM_SLD_VERSION_META_SST_FILES = "sstFiles"; + public static final String OM_SLD_VERSION_META_PREV_SNAP_VERSION = "previousSnapshotVersion"; + public static final String OM_SST_FILE_INFO_FILE_NAME = "fileName"; + public static final String OM_SST_FILE_INFO_START_KEY = "startKey"; + public static final String OM_SST_FILE_INFO_END_KEY = "endKey"; + public static final String OM_SST_FILE_INFO_COL_FAMILY = "columnFamily"; // YAML fields for .container files public static final String CONTAINER_ID = "containerID"; diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java index 46189600bb44..e44c2e8522e1 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java @@ -27,11 +27,7 @@ /** * Dao to keep SST file information in the compaction log. */ -public final class CompactionFileInfo { - private final String fileName; - private final String startKey; - private final String endKey; - private final String columnFamily; +public final class CompactionFileInfo extends SstFileInfo { private boolean pruned; @VisibleForTesting @@ -47,29 +43,10 @@ public CompactionFileInfo(String fileName, String endRange, String columnFamily, boolean pruned) { - this.fileName = fileName; - this.startKey = startRange; - this.endKey = endRange; - this.columnFamily = columnFamily; + super(fileName, startRange, endRange, columnFamily); this.pruned = pruned; } - public String getFileName() { - return fileName; - } - - public String getStartKey() { - return startKey; - } - - public String getEndKey() { - return endKey; - } - - public String getColumnFamily() { - return columnFamily; - } - public boolean isPruned() { return pruned; } @@ -81,16 +58,16 @@ public void setPruned() { public HddsProtos.CompactionFileInfoProto getProtobuf() { HddsProtos.CompactionFileInfoProto.Builder builder = HddsProtos.CompactionFileInfoProto.newBuilder() - .setFileName(fileName) + .setFileName(getFileName()) .setPruned(pruned); - if (startKey != null) { - builder = builder.setStartKey(startKey); + if (getStartKey() != null) { + builder = builder.setStartKey(getStartKey()); } - if (endKey != null) { - builder = builder.setEndKey(endKey); + if (getEndKey() != null) { + builder = builder.setEndKey(getEndKey()); } - if (columnFamily != null) { - builder = builder.setColumnFamily(columnFamily); + if (getColumnFamily() != null) { + builder = builder.setColumnFamily(getColumnFamily()); } return builder.build(); } @@ -117,8 +94,25 @@ public static CompactionFileInfo getFromProtobuf( @Override public String toString() { - return String.format("fileName: '%s', startKey: '%s', endKey: '%s'," + - " columnFamily: '%s', isPruned: '%b'", fileName, startKey, endKey, columnFamily, pruned); + return String.format("%s, isPruned: '%b'", super.toString(), pruned); + } + + @Override + public SstFileInfo copyObject() { + return new CompactionFileInfo(getFileName(), getStartKey(), getEndKey(), getColumnFamily(), pruned); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CompactionFileInfo)) { + return false; + } + return super.equals(o) && pruned == ((CompactionFileInfo)o).pruned; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), pruned); } /** @@ -180,25 +174,4 @@ public CompactionFileInfo build() { columnFamily, pruned); } } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof CompactionFileInfo)) { - return false; - } - - CompactionFileInfo that = (CompactionFileInfo) o; - return Objects.equals(fileName, that.fileName) && - Objects.equals(startKey, that.startKey) && - Objects.equals(endKey, that.endKey) && - Objects.equals(columnFamily, that.columnFamily); - } - - @Override - public int hashCode() { - return Objects.hash(fileName, startKey, endKey, columnFamily); - } } diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/SstFileInfo.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/SstFileInfo.java new file mode 100644 index 000000000000..b1887ec3d1e0 --- /dev/null +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/SstFileInfo.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ozone.compaction.log; + +import java.util.Objects; +import org.apache.hadoop.hdds.StringUtils; +import org.apache.hadoop.hdds.utils.db.CopyObject; +import org.rocksdb.LiveFileMetaData; + +/** + * Dao to keep SST file information in the compaction log. + */ +public class SstFileInfo implements CopyObject { + private final String fileName; + private final String startKey; + private final String endKey; + private final String columnFamily; + + public SstFileInfo(String fileName, String startRange, String endRange, String columnFamily) { + this.fileName = fileName; + this.startKey = startRange; + this.endKey = endRange; + this.columnFamily = columnFamily; + } + + public SstFileInfo(LiveFileMetaData fileMetaData) { + this(fileMetaData.fileName(), StringUtils.bytes2String(fileMetaData.smallestKey()), + StringUtils.bytes2String(fileMetaData.largestKey()), + StringUtils.bytes2String(fileMetaData.columnFamilyName())); + } + + public String getFileName() { + return fileName; + } + + public String getStartKey() { + return startKey; + } + + public String getEndKey() { + return endKey; + } + + public String getColumnFamily() { + return columnFamily; + } + + @Override + public String toString() { + return String.format("fileName: '%s', startKey: '%s', endKey: '%s'," + + " columnFamily: '%s'", fileName, startKey, endKey, columnFamily); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SstFileInfo)) { + return false; + } + + SstFileInfo that = (SstFileInfo) o; + return Objects.equals(fileName, that.fileName) && + Objects.equals(startKey, that.startKey) && + Objects.equals(endKey, that.endKey) && + Objects.equals(columnFamily, that.columnFamily); + } + + @Override + public int hashCode() { + return Objects.hash(fileName, startKey, endKey, columnFamily); + } + + @Override + public SstFileInfo copyObject() { + return new SstFileInfo(fileName, startKey, endKey, columnFamily); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java index 7b3c7e9de80e..dc3d902db1e2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java @@ -17,16 +17,22 @@ package org.apache.hadoop.ozone.om; -import com.google.common.collect.Maps; +import static java.util.Collections.unmodifiableList; + import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Collectors; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.hdds.utils.db.CopyObject; +import org.apache.ozone.compaction.log.SstFileInfo; +import org.rocksdb.LiveFileMetaData; import org.yaml.snakeyaml.Yaml; /** @@ -45,18 +51,17 @@ public abstract class OmSnapshotLocalData { // Whether SST is filtered private boolean isSSTFiltered; - // Map of Table to uncompacted SST file list on snapshot create - private Map> uncompactedSSTFileList; - // Time of last compaction, in epoch milliseconds private long lastCompactionTime; // Whether the snapshot needs compaction private boolean needsCompaction; - // Map of version to compacted SST file list - // Map> - private Map>> compactedSSTFileList; + private UUID previousSnapshotId; + + // Map of version to VersionMeta, using linkedHashMap since the order of the map needs to be deterministic for + // checksum computation. + private final LinkedHashMap versionSstFileInfos; public static final Charset CHARSET_ENCODING = StandardCharsets.UTF_8; private static final String DUMMY_CHECKSUM = new String(new byte[64], CHARSET_ENCODING); @@ -64,13 +69,15 @@ public abstract class OmSnapshotLocalData { /** * Creates a OmSnapshotLocalData object with default values. */ - public OmSnapshotLocalData(Map> uncompactedSSTFileList) { + public OmSnapshotLocalData(List uncompactedSSTFileList, UUID previousSnapshotId) { this.isSSTFiltered = false; - this.uncompactedSSTFileList = uncompactedSSTFileList != null ? uncompactedSSTFileList : new HashMap<>(); this.lastCompactionTime = 0L; this.needsCompaction = false; - this.compactedSSTFileList = new HashMap<>(); + this.versionSstFileInfos = new LinkedHashMap<>(); + versionSstFileInfos.put(0, + new VersionMeta(0, uncompactedSSTFileList.stream().map(SstFileInfo::new).collect(Collectors.toList()))); this.version = 0; + this.previousSnapshotId = previousSnapshotId; setChecksumTo0ByteArray(); } @@ -85,31 +92,11 @@ public OmSnapshotLocalData(OmSnapshotLocalData source) { this.needsCompaction = source.needsCompaction; this.checksum = source.checksum; this.version = source.version; - - // Deep copy for uncompactedSSTFileList - this.uncompactedSSTFileList = new HashMap<>(); - for (Map.Entry> entry : - source.uncompactedSSTFileList.entrySet()) { - this.uncompactedSSTFileList.put( - entry.getKey(), - new HashSet<>(entry.getValue())); - } + this.previousSnapshotId = source.previousSnapshotId; // Deep copy for compactedSSTFileList - this.compactedSSTFileList = new HashMap<>(); - for (Map.Entry>> versionEntry : - source.compactedSSTFileList.entrySet()) { - Map> tableMap = new HashMap<>(); - - for (Map.Entry> tableEntry : - versionEntry.getValue().entrySet()) { - tableMap.put( - tableEntry.getKey(), - new HashSet<>(tableEntry.getValue())); - } - - this.compactedSSTFileList.put(versionEntry.getKey(), tableMap); - } + this.versionSstFileInfos = new LinkedHashMap<>(); + setVersionSstFileInfos(source.versionSstFileInfos); } /** @@ -128,34 +115,6 @@ public void setSstFiltered(boolean sstFiltered) { this.isSSTFiltered = sstFiltered; } - /** - * Returns the uncompacted SST file list. - * @return Map of Table to uncompacted SST file list - */ - public Map> getUncompactedSSTFileList() { - return Collections.unmodifiableMap(this.uncompactedSSTFileList); - } - - /** - * Sets the uncompacted SST file list. - * @param uncompactedSSTFileList Map of Table to uncompacted SST file list - */ - public void setUncompactedSSTFileList( - Map> uncompactedSSTFileList) { - this.uncompactedSSTFileList.clear(); - this.uncompactedSSTFileList.putAll(uncompactedSSTFileList); - } - - /** - * Adds an entry to the uncompacted SST file list. - * @param table Table name - * @param sstFiles SST file name - */ - public void addUncompactedSSTFileList(String table, Set sstFiles) { - this.uncompactedSSTFileList.computeIfAbsent(table, k -> new HashSet<>()) - .addAll(sstFiles); - } - /** * Returns the last compaction time, in epoch milliseconds. * @return Timestamp of the last compaction @@ -192,30 +151,34 @@ public void setNeedsCompaction(boolean needsCompaction) { * Returns the compacted SST file list. * @return Map of version to compacted SST file list */ - public Map>> getCompactedSSTFileList() { - return Collections.unmodifiableMap(this.compactedSSTFileList); + public Map getVersionSstFileInfos() { + return Collections.unmodifiableMap(this.versionSstFileInfos); } /** * Sets the compacted SST file list. - * @param compactedSSTFileList Map of version to compacted SST file list + * @param versionSstFileInfos Map of version to compacted SST file list */ - public void setCompactedSSTFileList( - Map>> compactedSSTFileList) { - this.compactedSSTFileList.clear(); - this.compactedSSTFileList.putAll(compactedSSTFileList); + public void setVersionSstFileInfos(Map versionSstFileInfos) { + this.versionSstFileInfos.clear(); + this.versionSstFileInfos.putAll(versionSstFileInfos); + } + + public UUID getPreviousSnapshotId() { + return previousSnapshotId; + } + + public void setPreviousSnapshotId(UUID previousSnapshotId) { + this.previousSnapshotId = previousSnapshotId; } /** * Adds an entry to the compacted SST file list. - * @param ver Version number (TODO: to be clarified) - * @param table Table name * @param sstFiles SST file name */ - public void addCompactedSSTFileList(Integer ver, String table, Set sstFiles) { - this.compactedSSTFileList.computeIfAbsent(ver, k -> Maps.newHashMap()) - .computeIfAbsent(table, k -> new HashSet<>()) - .addAll(sstFiles); + public void addVersionSSTFileInfos(List sstFiles, int previousSnapshotVersion) { + version++; + this.versionSstFileInfos.put(version, new VersionMeta(previousSnapshotVersion, sstFiles)); } /** @@ -286,4 +249,59 @@ public int getVersion() { public void setVersion(int version) { this.version = version; } + + /** + * Represents metadata for a specific version in a snapshot. + * This class maintains the version of the previous snapshot and a list of SST (Sorted String Table) files + * associated with the current version. It provides methods for accessing this data and supports a + * copy mechanism for deep cloning. + * + * Instances of this class are immutable. The list of SST files is stored as an unmodifiable list to + * maintain immutability. + */ + public static class VersionMeta implements CopyObject { + private final int previousSnapshotVersion; + private final List sstFiles; + + public VersionMeta(int previousSnapshotVersion, List sstFiles) { + this.previousSnapshotVersion = previousSnapshotVersion; + this.sstFiles = unmodifiableList(sstFiles); + } + + public int getPreviousSnapshotVersion() { + return previousSnapshotVersion; + } + + public List getSstFiles() { + return sstFiles; + } + + @Override + public VersionMeta copyObject() { + return new VersionMeta(previousSnapshotVersion, + sstFiles.stream().map(SstFileInfo::copyObject).collect(Collectors.toList())); + } + + @Override + public int hashCode() { + return Objects.hash(previousSnapshotVersion, sstFiles); + } + + @Override + public final boolean equals(Object o) { + if (!(o instanceof VersionMeta)) { + return false; + } + VersionMeta that = (VersionMeta) o; + return previousSnapshotVersion == that.previousSnapshotVersion && sstFiles.equals(that.sstFiles); + } + + @Override + public String toString() { + return "VersionMeta{" + + "previousSnapshotVersion=" + previousSnapshotVersion + + ", sstFiles=" + sstFiles + + '}'; + } + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java index 3ed915edfdef..b0c600abbfff 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java @@ -22,23 +22,32 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; +import java.util.Collections; +import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.UUID; import org.apache.hadoop.hdds.server.YamlUtils; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.ozone.compaction.log.SstFileInfo; +import org.rocksdb.LiveFileMetaData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.TypeDescription; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.AbstractConstruct; import org.yaml.snakeyaml.constructor.SafeConstructor; import org.yaml.snakeyaml.error.YAMLException; import org.yaml.snakeyaml.introspector.BeanAccess; +import org.yaml.snakeyaml.introspector.Property; import org.yaml.snakeyaml.introspector.PropertyUtils; import org.yaml.snakeyaml.nodes.MappingNode; import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.ScalarNode; import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Represent; import org.yaml.snakeyaml.representer.Representer; /** @@ -51,12 +60,14 @@ public final class OmSnapshotLocalDataYaml extends OmSnapshotLocalData { private static final Logger LOG = LoggerFactory.getLogger(OmSnapshotLocalDataYaml.class); public static final Tag SNAPSHOT_YAML_TAG = new Tag("OmSnapshotLocalData"); + public static final Tag SNAPSHOT_VERSION_META_TAG = new Tag("VersionMeta"); + public static final Tag SST_FILE_INFO_TAG = new Tag("SstFileInfo"); /** * Creates a new OmSnapshotLocalDataYaml with default values. */ - public OmSnapshotLocalDataYaml(Map> uncompactedSSTFileList) { - super(uncompactedSSTFileList); + public OmSnapshotLocalDataYaml(List liveFileMetaDatas, UUID previousSnapshotId) { + super(liveFileMetaDatas, previousSnapshotId); } /** @@ -91,7 +102,7 @@ public static boolean verifyChecksum(OmSnapshotLocalData snapshotData) snapshotDataCopy.setChecksum(null); // Get the YAML representation - final Yaml yaml = getYamlForSnapshotLocalData(); + final Yaml yaml = OmSnapshotLocalDataYamlProvider.getInstance(); // Compute new checksum snapshotDataCopy.computeAndSetChecksum(yaml); @@ -108,6 +119,63 @@ public static boolean verifyChecksum(OmSnapshotLocalData snapshotData) return isValid; } + /** + * Representer class to define which fields need to be stored in yaml file. + */ + private static class OmSnapshotLocalDataRepresenter extends Representer { + + OmSnapshotLocalDataRepresenter(DumperOptions options) { + super(options); + this.addClassTag(OmSnapshotLocalDataYaml.class, SNAPSHOT_YAML_TAG); + this.addClassTag(VersionMeta.class, SNAPSHOT_VERSION_META_TAG); + this.addClassTag(SstFileInfo.class, SST_FILE_INFO_TAG); + representers.put(SstFileInfo.class, new RepresentSstFileInfo()); + representers.put(VersionMeta.class, new RepresentVersionMeta()); + representers.put(UUID.class, data -> + new ScalarNode(Tag.STR, data.toString(), null, null, DumperOptions.ScalarStyle.PLAIN)); + } + + private class RepresentSstFileInfo implements Represent { + @Override + public Node representData(Object data) { + SstFileInfo info = (SstFileInfo) data; + Map map = new java.util.LinkedHashMap<>(); + map.put(OzoneConsts.OM_SST_FILE_INFO_FILE_NAME, info.getFileName()); + map.put(OzoneConsts.OM_SST_FILE_INFO_START_KEY, info.getStartKey()); + map.put(OzoneConsts.OM_SST_FILE_INFO_END_KEY, info.getEndKey()); + map.put(OzoneConsts.OM_SST_FILE_INFO_COL_FAMILY, info.getColumnFamily()); + + // Explicitly create a mapping node with the desired tag + return representMapping(SST_FILE_INFO_TAG, map, DumperOptions.FlowStyle.BLOCK); + } + } + + // New inner class for VersionMeta + private class RepresentVersionMeta implements Represent { + @Override + public Node representData(Object data) { + VersionMeta meta = (VersionMeta) data; + Map map = new java.util.LinkedHashMap<>(); + map.put(OzoneConsts.OM_SLD_VERSION_META_PREV_SNAP_VERSION, meta.getPreviousSnapshotVersion()); + map.put(OzoneConsts.OM_SLD_VERSION_META_SST_FILES, meta.getSstFiles()); + + return representMapping(SNAPSHOT_VERSION_META_TAG, map, DumperOptions.FlowStyle.BLOCK); + } + } + + + /** + * Omit properties with null value. + */ + @Override + protected NodeTuple representJavaBeanProperty( + Object bean, Property property, Object value, Tag tag) { + return value == null + ? null + : super.representJavaBeanProperty(bean, property, value, tag); + } + } + /** * Constructor class for OmSnapshotLocalData. * This is used when parsing YAML files into OmSnapshotLocalDataYaml objects. @@ -117,6 +185,37 @@ private static class SnapshotLocalDataConstructor extends SafeConstructor { super(new LoaderOptions()); //Adding our own specific constructors for tags. this.yamlConstructors.put(SNAPSHOT_YAML_TAG, new ConstructSnapshotLocalData()); + this.yamlConstructors.put(SNAPSHOT_VERSION_META_TAG, new ConstructVersionMeta()); + this.yamlConstructors.put(SST_FILE_INFO_TAG, new ConstructSstFileInfo()); + TypeDescription omDesc = new TypeDescription(OmSnapshotLocalDataYaml.class); + omDesc.putMapPropertyType(OzoneConsts.OM_SLD_VERSION_SST_FILE_INFO, Integer.class, VersionMeta.class); + this.addTypeDescription(omDesc); + TypeDescription versionMetaDesc = new TypeDescription(VersionMeta.class); + versionMetaDesc.putListPropertyType(OzoneConsts.OM_SLD_VERSION_META_SST_FILES, SstFileInfo.class); + this.addTypeDescription(versionMetaDesc); + } + + private final class ConstructSstFileInfo extends AbstractConstruct { + @Override + public Object construct(Node node) { + MappingNode mnode = (MappingNode) node; + Map nodes = constructMapping(mnode); + return new SstFileInfo((String) nodes.get(OzoneConsts.OM_SST_FILE_INFO_FILE_NAME), + (String) nodes.get(OzoneConsts.OM_SST_FILE_INFO_START_KEY), + (String) nodes.get(OzoneConsts.OM_SST_FILE_INFO_END_KEY), + (String) nodes.get(OzoneConsts.OM_SST_FILE_INFO_COL_FAMILY)); + } + } + + private final class ConstructVersionMeta extends AbstractConstruct { + + @Override + public Object construct(Node node) { + MappingNode mnode = (MappingNode) node; + Map nodes = constructMapping(mnode); + return new VersionMeta((Integer) nodes.get(OzoneConsts.OM_SLD_VERSION_META_PREV_SNAP_VERSION), + (List) nodes.get(OzoneConsts.OM_SLD_VERSION_META_SST_FILES)); + } } private final class ConstructSnapshotLocalData extends AbstractConstruct { @@ -125,10 +224,8 @@ private final class ConstructSnapshotLocalData extends AbstractConstruct { public Object construct(Node node) { MappingNode mnode = (MappingNode) node; Map nodes = constructMapping(mnode); - - Map> uncompactedSSTFileList = - (Map>) nodes.get(OzoneConsts.OM_SLD_UNCOMPACTED_SST_FILE_LIST); - OmSnapshotLocalDataYaml snapshotLocalData = new OmSnapshotLocalDataYaml(uncompactedSSTFileList); + UUID prevSnapId = UUID.fromString((String) nodes.get(OzoneConsts.OM_SLD_PREV_SNAP_ID)); + OmSnapshotLocalDataYaml snapshotLocalData = new OmSnapshotLocalDataYaml(Collections.emptyList(), prevSnapId); // Set version from YAML Integer version = (Integer) nodes.get(OzoneConsts.OM_SLD_VERSION); @@ -149,11 +246,10 @@ public Object construct(Node node) { snapshotLocalData.setLastCompactionTime(lastCompactionTime); snapshotLocalData.setNeedsCompaction((Boolean) nodes.getOrDefault(OzoneConsts.OM_SLD_NEEDS_COMPACTION, false)); - - Map>> compactedSSTFileList = - (Map>>) nodes.get(OzoneConsts.OM_SLD_COMPACTED_SST_FILE_LIST); - if (compactedSSTFileList != null) { - snapshotLocalData.setCompactedSSTFileList(compactedSSTFileList); + Map versionMetaMap = + (Map) nodes.get(OzoneConsts.OM_SLD_VERSION_SST_FILE_INFO); + if (versionMetaMap != null) { + snapshotLocalData.setVersionSstFileInfos(versionMetaMap); } String checksum = (String) nodes.get(OzoneConsts.OM_SLD_CHECKSUM); @@ -172,7 +268,7 @@ public Object construct(Node node) { * @return YAML string representation */ public String getYaml() { - final Yaml yaml = getYamlForSnapshotLocalData(); + final Yaml yaml = OmSnapshotLocalDataYamlProvider.getInstance(); return yaml.dump(this); } @@ -183,7 +279,7 @@ public String getYaml() { */ public void writeToYaml(File yamlFile) throws IOException { // Create Yaml - final Yaml yaml = getYamlForSnapshotLocalData(); + final Yaml yaml = OmSnapshotLocalDataYamlProvider.getInstance(); // Compute Checksum and update SnapshotData computeAndSetChecksum(yaml); // Write the SnapshotData with checksum to Yaml file. @@ -203,45 +299,14 @@ public static OmSnapshotLocalDataYaml getFromYamlFile(File yamlFile) throws IOEx } } - /** - * Returns a Yaml representation of the snapshot properties. - * @return Yaml representation of snapshot properties - */ - public static Yaml getYamlForSnapshotLocalData() { - PropertyUtils propertyUtils = new PropertyUtils(); - propertyUtils.setBeanAccess(BeanAccess.FIELD); - propertyUtils.setAllowReadOnlyProperties(true); - - DumperOptions options = new DumperOptions(); - Representer representer = new Representer(options); - representer.setPropertyUtils(propertyUtils); - representer.addClassTag(OmSnapshotLocalDataYaml.class, SNAPSHOT_YAML_TAG); - - SafeConstructor snapshotDataConstructor = new SnapshotLocalDataConstructor(); - return new Yaml(snapshotDataConstructor, representer); - } - /** * Read the YAML content InputStream, and return OmSnapshotLocalDataYaml instance. * @throws IOException */ public static OmSnapshotLocalDataYaml getFromYamlStream(InputStream input) throws IOException { OmSnapshotLocalDataYaml dataYaml; - - PropertyUtils propertyUtils = new PropertyUtils(); - propertyUtils.setBeanAccess(BeanAccess.FIELD); - propertyUtils.setAllowReadOnlyProperties(true); - - DumperOptions options = new DumperOptions(); - Representer representer = new Representer(options); - representer.setPropertyUtils(propertyUtils); - - SafeConstructor snapshotDataConstructor = new SnapshotLocalDataConstructor(); - - Yaml yaml = new Yaml(snapshotDataConstructor, representer); - try { - dataYaml = yaml.load(input); + dataYaml = OmSnapshotLocalDataYamlProvider.getInstance().load(input); } catch (YAMLException ex) { // Unchecked exception. Convert to IOException throw new IOException(ex); @@ -256,4 +321,26 @@ public static OmSnapshotLocalDataYaml getFromYamlStream(InputStream input) throw return dataYaml; } + + private static class OmSnapshotLocalDataYamlProvider { + private static volatile Yaml instance = getInstance(); + + public static Yaml getInstance() { + if (instance == null) { + synchronized (OmSnapshotLocalDataYamlProvider.class) { + if (instance == null) { + PropertyUtils propertyUtils = new PropertyUtils(); + propertyUtils.setBeanAccess(BeanAccess.FIELD); + propertyUtils.setAllowReadOnlyProperties(true); + DumperOptions options = new DumperOptions(); + Representer representer = new OmSnapshotLocalDataRepresenter(options); + representer.setPropertyUtils(propertyUtils); + SafeConstructor snapshotDataConstructor = new SnapshotLocalDataConstructor(); + instance = new Yaml(snapshotDataConstructor, representer); + } + } + } + return instance; + } + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 91e5abb217a5..c945574e8f7e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -66,10 +66,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; @@ -632,17 +630,11 @@ private static void deleteKeysFromDelKeyTableInSnapshotScope( * @param store AOS or snapshot DB for uncompacted or compacted snapshot respectively. * @return a Map of (table, set of SST files corresponding to the table) */ - private static Map> getSnapshotSSTFileList(RDBStore store) + private static List getSnapshotSSTFileList(RDBStore store) throws IOException { - Map> sstFileList = new HashMap<>(); - List liveFileMetaDataList = store.getDb().getLiveFilesMetaData(); - liveFileMetaDataList.forEach(lfm -> { - String cfName = StringUtils.bytes2String(lfm.columnFamilyName()); - if (COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT.contains(cfName)) { - sstFileList.computeIfAbsent(cfName, k -> new HashSet<>()).add(lfm.fileName()); - } - }); - return sstFileList; + return store.getDb().getLiveFilesMetaData().stream() + .filter(lfm -> COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT.contains(StringUtils.bytes2String(lfm.columnFamilyName()))) + .collect(Collectors.toList()); } /** @@ -656,7 +648,8 @@ public static void createNewOmSnapshotLocalDataFile( throws IOException { Path snapshotLocalDataPath = Paths.get(getSnapshotLocalPropertyYamlPath(omMetadataManager, snapshotInfo)); Files.deleteIfExists(snapshotLocalDataPath); - OmSnapshotLocalDataYaml snapshotLocalDataYaml = new OmSnapshotLocalDataYaml(getSnapshotSSTFileList(store)); + OmSnapshotLocalDataYaml snapshotLocalDataYaml = new OmSnapshotLocalDataYaml(getSnapshotSSTFileList(store), + snapshotInfo.getPathPreviousSnapshotId()); snapshotLocalDataYaml.writeToYaml(snapshotLocalDataPath.toFile()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java index 4ab4996961ef..d1826f923def 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java @@ -17,28 +17,38 @@ package org.apache.hadoop.ozone.om; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.apache.hadoop.hdds.StringUtils.string2Bytes; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; import java.time.Instant; import java.util.Collections; -import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.util.UUID; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OmSnapshotLocalData.VersionMeta; +import org.apache.ozone.compaction.log.SstFileInfo; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.rocksdb.LiveFileMetaData; /** * This class tests creating and reading snapshot data YAML files. @@ -59,17 +69,28 @@ public void cleanup() { FileUtil.fullyDelete(new File(testRoot)); } + private LiveFileMetaData createLiveFileMetaData(String fileName, String table, String smallestKey, + String largestKey) { + LiveFileMetaData lfm = mock(LiveFileMetaData.class); + when(lfm.columnFamilyName()).thenReturn(string2Bytes(table)); + when(lfm.fileName()).thenReturn(fileName); + when(lfm.smallestKey()).thenReturn(StringUtils.string2Bytes(smallestKey)); + when(lfm.largestKey()).thenReturn(StringUtils.string2Bytes(largestKey)); + return lfm; + } + /** * Creates a snapshot local data YAML file. */ - private File writeToYaml(String snapshotName) throws IOException { + private Pair writeToYaml(String snapshotName) throws IOException { String yamlFilePath = snapshotName + ".yaml"; - + UUID previousSnapshotId = UUID.randomUUID(); // Create snapshot data with uncompacted SST files - Map> uncompactedSSTFileList = new HashMap<>(); - uncompactedSSTFileList.put("table1", Stream.of("sst1", "sst2").collect(Collectors.toSet())); - uncompactedSSTFileList.put("table2", Stream.of("sst3").collect(Collectors.toSet())); - OmSnapshotLocalDataYaml dataYaml = new OmSnapshotLocalDataYaml(uncompactedSSTFileList); + List uncompactedSSTFileList = asList( + createLiveFileMetaData("sst1", "table1", "k1", "k2"), + createLiveFileMetaData("sst2", "table1", "k3", "k4"), + createLiveFileMetaData("sst3", "table2", "k4", "k5")); + OmSnapshotLocalDataYaml dataYaml = new OmSnapshotLocalDataYaml(uncompactedSSTFileList, previousSnapshotId); // Set version dataYaml.setVersion(42); @@ -83,9 +104,12 @@ private File writeToYaml(String snapshotName) throws IOException { dataYaml.setNeedsCompaction(true); // Add some compacted SST files - dataYaml.addCompactedSSTFileList(1, "table1", Collections.singleton("compacted-sst1")); - dataYaml.addCompactedSSTFileList(1, "table2", Collections.singleton("compacted-sst2")); - dataYaml.addCompactedSSTFileList(2, "table1", Collections.singleton("compacted-sst3")); + dataYaml.addVersionSSTFileInfos(ImmutableList.of( + new SstFileInfo("compacted-sst1", "k1", "k2", "table1"), + new SstFileInfo("compacted-sst2", "k3", "k4", "table2")), + 1); + dataYaml.addVersionSSTFileInfos(Collections.singletonList( + new SstFileInfo("compacted-sst3", "k4", "k5", "table1")), 3); File yamlFile = new File(testRoot, yamlFilePath); @@ -95,46 +119,53 @@ private File writeToYaml(String snapshotName) throws IOException { // Check YAML file exists assertTrue(yamlFile.exists()); - return yamlFile; + return Pair.of(yamlFile, previousSnapshotId); } @Test public void testWriteToYaml() throws IOException { - File yamlFile = writeToYaml("snapshot1"); + Pair yamlFilePrevIdPair = writeToYaml("snapshot1"); + File yamlFile = yamlFilePrevIdPair.getLeft(); + UUID prevSnapId = yamlFilePrevIdPair.getRight(); // Read from YAML file OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); // Verify fields - assertEquals(42, snapshotData.getVersion()); + assertEquals(44, snapshotData.getVersion()); assertTrue(snapshotData.getSstFiltered()); - Map> uncompactedFiles = snapshotData.getUncompactedSSTFileList(); - assertEquals(2, uncompactedFiles.size()); - assertEquals(2, uncompactedFiles.get("table1").size()); - assertEquals(1, uncompactedFiles.get("table2").size()); - assertTrue(uncompactedFiles.get("table1").contains("sst1")); - assertTrue(uncompactedFiles.get("table1").contains("sst2")); - assertTrue(uncompactedFiles.get("table2").contains("sst3")); - + VersionMeta uncompactedFiles = snapshotData.getVersionSstFileInfos().get(0); + assertEquals(new VersionMeta(0, + ImmutableList.of(new SstFileInfo("sst1", "k1", "k2", "table1"), + new SstFileInfo("sst2", "k3", "k4", "table1"), + new SstFileInfo("sst3", "k4", "k5", "table2"))), uncompactedFiles); assertEquals(NOW.toEpochMilli(), snapshotData.getLastCompactionTime()); assertTrue(snapshotData.getNeedsCompaction()); - Map>> compactedFiles = snapshotData.getCompactedSSTFileList(); - assertEquals(2, compactedFiles.size()); - assertTrue(compactedFiles.containsKey(1)); - assertTrue(compactedFiles.containsKey(2)); - assertEquals(2, compactedFiles.get(1).size()); - assertEquals(1, compactedFiles.get(2).size()); - assertTrue(compactedFiles.get(1).get("table1").contains("compacted-sst1")); - assertTrue(compactedFiles.get(1).get("table2").contains("compacted-sst2")); - assertTrue(compactedFiles.get(2).get("table1").contains("compacted-sst3")); + Map compactedFiles = snapshotData.getVersionSstFileInfos(); + assertEquals(3, compactedFiles.size()); + assertTrue(compactedFiles.containsKey(43)); + assertTrue(compactedFiles.containsKey(44)); + assertEquals(2, compactedFiles.get(43).getSstFiles().size()); + assertEquals(1, compactedFiles.get(44).getSstFiles().size()); + assertEquals(prevSnapId, snapshotData.getPreviousSnapshotId()); + assertEquals(ImmutableMap.of( + 0, new VersionMeta(0, + ImmutableList.of(new SstFileInfo("sst1", "k1", "k2", "table1"), + new SstFileInfo("sst2", "k3", "k4", "table1"), + new SstFileInfo("sst3", "k4", "k5", "table2"))), + 43, new VersionMeta(1, + ImmutableList.of(new SstFileInfo("compacted-sst1", "k1", "k2", "table1"), + new SstFileInfo("compacted-sst2", "k3", "k4", "table2"))), + 44, new VersionMeta(3, + ImmutableList.of(new SstFileInfo("compacted-sst3", "k4", "k5", "table1")))), compactedFiles); } @Test public void testUpdateSnapshotDataFile() throws IOException { - File yamlFile = writeToYaml("snapshot2"); - + Pair yamlFilePrevIdPair = writeToYaml("snapshot2"); + File yamlFile = yamlFilePrevIdPair.getLeft(); // Read from YAML file OmSnapshotLocalDataYaml dataYaml = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); @@ -142,8 +173,8 @@ public void testUpdateSnapshotDataFile() throws IOException { // Update snapshot data dataYaml.setSstFiltered(false); dataYaml.setNeedsCompaction(false); - dataYaml.addUncompactedSSTFileList("table3", Collections.singleton("sst4")); - dataYaml.addCompactedSSTFileList(3, "table3", Collections.singleton("compacted-sst4")); + dataYaml.addVersionSSTFileInfos( + singletonList(new SstFileInfo("compacted-sst4", "k5", "k6", "table3")), 5); // Write updated data back to file dataYaml.writeToYaml(yamlFile); @@ -155,16 +186,11 @@ public void testUpdateSnapshotDataFile() throws IOException { assertThat(dataYaml.getSstFiltered()).isFalse(); assertThat(dataYaml.getNeedsCompaction()).isFalse(); - Map> uncompactedFiles = dataYaml.getUncompactedSSTFileList(); - assertEquals(3, uncompactedFiles.size()); - assertTrue(uncompactedFiles.containsKey("table3")); - assertTrue(uncompactedFiles.get("table3").contains("sst4")); - - Map>> compactedFiles = dataYaml.getCompactedSSTFileList(); - assertEquals(3, compactedFiles.size()); - assertTrue(compactedFiles.containsKey(3)); - assertTrue(compactedFiles.get(3).containsKey("table3")); - assertTrue(compactedFiles.get(3).get("table3").contains("compacted-sst4")); + Map compactedFiles = dataYaml.getVersionSstFileInfos(); + assertEquals(4, compactedFiles.size()); + assertTrue(compactedFiles.containsKey(45)); + assertEquals(new VersionMeta(5, ImmutableList.of(new SstFileInfo("compacted-sst4", "k5", "k6", "table3"))), + compactedFiles.get(45)); } @Test @@ -180,8 +206,8 @@ public void testEmptyFile() throws IOException { @Test public void testChecksum() throws IOException { - File yamlFile = writeToYaml("snapshot3"); - + Pair yamlFilePrevIdPair = writeToYaml("snapshot3"); + File yamlFile = yamlFilePrevIdPair.getLeft(); // Read from YAML file OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); @@ -196,17 +222,16 @@ public void testChecksum() throws IOException { @Test public void testYamlContainsAllFields() throws IOException { - File yamlFile = writeToYaml("snapshot4"); - + Pair yamlFilePrevIdPair = writeToYaml("snapshot4"); + File yamlFile = yamlFilePrevIdPair.getLeft(); String content = FileUtils.readFileToString(yamlFile, Charset.defaultCharset()); // Verify the YAML content contains all expected fields assertThat(content).contains(OzoneConsts.OM_SLD_VERSION); assertThat(content).contains(OzoneConsts.OM_SLD_CHECKSUM); assertThat(content).contains(OzoneConsts.OM_SLD_IS_SST_FILTERED); - assertThat(content).contains(OzoneConsts.OM_SLD_UNCOMPACTED_SST_FILE_LIST); assertThat(content).contains(OzoneConsts.OM_SLD_LAST_COMPACTION_TIME); assertThat(content).contains(OzoneConsts.OM_SLD_NEEDS_COMPACTION); - assertThat(content).contains(OzoneConsts.OM_SLD_COMPACTED_SST_FILE_LIST); + assertThat(content).contains(OzoneConsts.OM_SLD_VERSION_SST_FILE_INFO); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index f2006deac31e..5d9248a3131e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om; import static org.apache.commons.io.file.PathUtils.copyDirectory; +import static org.apache.hadoop.hdds.StringUtils.string2Bytes; import static org.apache.hadoop.hdds.utils.HAUtils.getExistingFiles; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; @@ -46,6 +47,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; @@ -61,6 +63,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; @@ -80,6 +83,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils; import org.apache.hadoop.util.Time; +import org.apache.ozone.compaction.log.SstFileInfo; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.junit.jupiter.api.AfterAll; @@ -268,6 +272,8 @@ private LiveFileMetaData createMockLiveFileMetadata(String cfname, String fileNa LiveFileMetaData lfm = mock(LiveFileMetaData.class); when(lfm.columnFamilyName()).thenReturn(cfname.getBytes(StandardCharsets.UTF_8)); when(lfm.fileName()).thenReturn(fileName); + when(lfm.smallestKey()).thenReturn(string2Bytes("k1")); + when(lfm.largestKey()).thenReturn(string2Bytes("k2")); return lfm; } @@ -275,13 +281,20 @@ private LiveFileMetaData createMockLiveFileMetadata(String cfname, String fileNa public void testCreateNewSnapshotLocalYaml() throws IOException { SnapshotInfo snapshotInfo = createSnapshotInfo("vol1", "buck1"); - Map> expUncompactedSSTFileList = new HashMap<>(); - expUncompactedSSTFileList.put(KEY_TABLE, Stream.of("kt1.sst", "kt2.sst").collect(Collectors.toSet())); - expUncompactedSSTFileList.put(FILE_TABLE, Stream.of("ft1.sst", "ft2.sst").collect(Collectors.toSet())); - expUncompactedSSTFileList.put(DIRECTORY_TABLE, Stream.of("dt1.sst", "dt2.sst").collect(Collectors.toSet())); + Map> expUncompactedSSTFileList = new TreeMap<>(); + OmSnapshotLocalData.VersionMeta uncompactedVersionMeta = new OmSnapshotLocalData.VersionMeta(0, + ImmutableList.of(new SstFileInfo("dt1.sst", "k1", "k2", DIRECTORY_TABLE), + new SstFileInfo("dt2.sst", "k1", "k2", DIRECTORY_TABLE), + new SstFileInfo("ft1.sst", "k1", "k2", FILE_TABLE), + new SstFileInfo("ft2.sst", "k1", "k2", FILE_TABLE), + new SstFileInfo("kt1.sst", "k1", "k2", KEY_TABLE), + new SstFileInfo("kt2.sst", "k1", "k2", KEY_TABLE))); + expUncompactedSSTFileList.put(KEY_TABLE, Stream.of("kt1.sst", "kt2.sst").collect(Collectors.toList())); + expUncompactedSSTFileList.put(FILE_TABLE, Stream.of("ft1.sst", "ft2.sst").collect(Collectors.toList())); + expUncompactedSSTFileList.put(DIRECTORY_TABLE, Stream.of("dt1.sst", "dt2.sst").collect(Collectors.toList())); List mockedLiveFiles = new ArrayList<>(); - for (Map.Entry> entry : expUncompactedSSTFileList.entrySet()) { + for (Map.Entry> entry : expUncompactedSSTFileList.entrySet()) { String cfname = entry.getKey(); for (String fname : entry.getValue()) { mockedLiveFiles.add(createMockLiveFileMetadata(cfname, fname)); @@ -311,11 +324,11 @@ public void testCreateNewSnapshotLocalYaml() throws IOException { OmSnapshotLocalData localData = OmSnapshotLocalDataYaml.getFromYamlFile(snapshotYaml.toFile()); assertNotNull(localData); assertEquals(0, localData.getVersion()); - assertEquals(expUncompactedSSTFileList, localData.getUncompactedSSTFileList()); + assertEquals(uncompactedVersionMeta, localData.getVersionSstFileInfos().get(0)); assertFalse(localData.getSstFiltered()); assertEquals(0L, localData.getLastCompactionTime()); assertFalse(localData.getNeedsCompaction()); - assertTrue(localData.getCompactedSSTFileList().isEmpty()); + assertEquals(1, localData.getVersionSstFileInfos().size()); // Cleanup Files.delete(snapshotYaml); From ffcd6d420f3671842257ecb533c7cb6e15f748f2 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 29 Aug 2025 23:20:54 -0400 Subject: [PATCH 2/5] HDDS-13612. Track Sst file ranges in sst file metadata Change-Id: I2bc385065589e2ea2ffde0c09ace3ab7d4efb5ce --- hadoop-ozone/ozone-manager/pom.xml | 4 + .../ozone/om/OmSnapshotLocalDataYaml.java | 113 +++++++++++------- .../hadoop/ozone/om/OmSnapshotManager.java | 36 +++++- .../ozone/om/TestOmSnapshotLocalDataYaml.java | 35 ++++-- .../ozone/om/TestOmSnapshotManager.java | 4 +- pom.xml | 6 + 6 files changed, 137 insertions(+), 61 deletions(-) diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 6347ee2722bf..551eca63de27 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -108,6 +108,10 @@ org.apache.commons commons-lang3 + + org.apache.commons + commons-pool2 + org.apache.hadoop hadoop-auth diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java index b0c600abbfff..5d13828d2be7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java @@ -26,9 +26,13 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import org.apache.commons.pool2.BasePooledObjectFactory; +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.impl.DefaultPooledObject; import org.apache.hadoop.hdds.server.YamlUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ozone.compaction.log.SstFileInfo; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import org.rocksdb.LiveFileMetaData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,7 +88,7 @@ public OmSnapshotLocalDataYaml(OmSnapshotLocalData source) { * @return true if the checksum is valid, false otherwise * @throws IOException if there's an error computing the checksum */ - public static boolean verifyChecksum(OmSnapshotLocalData snapshotData) + public static boolean verifyChecksum(OmSnapshotManager snapshotManager, OmSnapshotLocalData snapshotData) throws IOException { Preconditions.checkNotNull(snapshotData, "snapshotData cannot be null"); @@ -102,21 +106,20 @@ public static boolean verifyChecksum(OmSnapshotLocalData snapshotData) snapshotDataCopy.setChecksum(null); // Get the YAML representation - final Yaml yaml = OmSnapshotLocalDataYamlProvider.getInstance(); + try (UncheckedAutoCloseableSupplier yaml = snapshotManager.getSnapshotLocalYaml()) { + // Compute new checksum + snapshotDataCopy.computeAndSetChecksum(yaml.get()); - // Compute new checksum - snapshotDataCopy.computeAndSetChecksum(yaml); + // Compare the stored and computed checksums + String computedChecksum = snapshotDataCopy.getChecksum(); + boolean isValid = storedChecksum.equals(computedChecksum); - // Compare the stored and computed checksums - String computedChecksum = snapshotDataCopy.getChecksum(); - boolean isValid = storedChecksum.equals(computedChecksum); - - if (!isValid) { - LOG.warn("Checksum verification failed for snapshot local data. " + - "Stored: {}, Computed: {}", storedChecksum, computedChecksum); + if (!isValid) { + LOG.warn("Checksum verification failed for snapshot local data. " + + "Stored: {}, Computed: {}", storedChecksum, computedChecksum); + } + return isValid; } - - return isValid; } /** @@ -267,9 +270,10 @@ public Object construct(Node node) { * (without triggering checksum computation or persistence). * @return YAML string representation */ - public String getYaml() { - final Yaml yaml = OmSnapshotLocalDataYamlProvider.getInstance(); - return yaml.dump(this); + public String getYaml(OmSnapshotManager snapshotManager) throws IOException { + try (UncheckedAutoCloseableSupplier yaml = snapshotManager.getSnapshotLocalYaml()) { + return yaml.get().dump(this); + } } /** @@ -277,13 +281,14 @@ public String getYaml() { * @param yamlFile The file to write to * @throws IOException If there's an error writing to the file */ - public void writeToYaml(File yamlFile) throws IOException { + public void writeToYaml(OmSnapshotManager snapshotManager, File yamlFile) throws IOException { // Create Yaml - final Yaml yaml = OmSnapshotLocalDataYamlProvider.getInstance(); - // Compute Checksum and update SnapshotData - computeAndSetChecksum(yaml); - // Write the SnapshotData with checksum to Yaml file. - YamlUtils.dump(yaml, this, yamlFile, LOG); + try (UncheckedAutoCloseableSupplier yaml = snapshotManager.getSnapshotLocalYaml()) { + // Compute Checksum and update SnapshotData + computeAndSetChecksum(yaml.get()); + // Write the SnapshotData with checksum to Yaml file. + YamlUtils.dump(yaml.get(), this, yamlFile, LOG); + } } /** @@ -292,10 +297,11 @@ public void writeToYaml(File yamlFile) throws IOException { * @return A new OmSnapshotLocalDataYaml instance * @throws IOException If there's an error reading the file */ - public static OmSnapshotLocalDataYaml getFromYamlFile(File yamlFile) throws IOException { + public static OmSnapshotLocalDataYaml getFromYamlFile(OmSnapshotManager snapshotManager, File yamlFile) + throws IOException { Preconditions.checkNotNull(yamlFile, "yamlFile cannot be null"); try (InputStream inputFileStream = Files.newInputStream(yamlFile.toPath())) { - return getFromYamlStream(inputFileStream); + return getFromYamlStream(snapshotManager, inputFileStream); } } @@ -303,10 +309,11 @@ public static OmSnapshotLocalDataYaml getFromYamlFile(File yamlFile) throws IOEx * Read the YAML content InputStream, and return OmSnapshotLocalDataYaml instance. * @throws IOException */ - public static OmSnapshotLocalDataYaml getFromYamlStream(InputStream input) throws IOException { + public static OmSnapshotLocalDataYaml getFromYamlStream(OmSnapshotManager snapshotManager, + InputStream input) throws IOException { OmSnapshotLocalDataYaml dataYaml; - try { - dataYaml = OmSnapshotLocalDataYamlProvider.getInstance().load(input); + try (UncheckedAutoCloseableSupplier yaml = snapshotManager.getSnapshotLocalYaml()) { + dataYaml = yaml.get().load(input); } catch (YAMLException ex) { // Unchecked exception. Convert to IOException throw new IOException(ex); @@ -322,25 +329,39 @@ public static OmSnapshotLocalDataYaml getFromYamlStream(InputStream input) throw return dataYaml; } - private static class OmSnapshotLocalDataYamlProvider { - private static volatile Yaml instance = getInstance(); - - public static Yaml getInstance() { - if (instance == null) { - synchronized (OmSnapshotLocalDataYamlProvider.class) { - if (instance == null) { - PropertyUtils propertyUtils = new PropertyUtils(); - propertyUtils.setBeanAccess(BeanAccess.FIELD); - propertyUtils.setAllowReadOnlyProperties(true); - DumperOptions options = new DumperOptions(); - Representer representer = new OmSnapshotLocalDataRepresenter(options); - representer.setPropertyUtils(propertyUtils); - SafeConstructor snapshotDataConstructor = new SnapshotLocalDataConstructor(); - instance = new Yaml(snapshotDataConstructor, representer); - } - } - } - return instance; + /** + * Factory class for constructing and pooling instances of the Yaml object. + * This class extends BasePooledObjectFactory to support object pooling, + * minimizing the expense of repeatedly creating and destroying Yaml instances. + * + * The Yaml instances created by this factory are customized to use a specific + * set of property and serialization/deserialization configurations. + * - BeanAccess is configured to access fields directly, allowing manipulation + * of private fields in objects. + * - The PropertyUtils allows read-only properties to be accessed. + * - Custom Representer and Constructor classes tailored to the OmSnapshotLocalData + * data structure are employed to customize how objects are represented in YAML. + * + * This class provides thread-safe pooling and management of Yaml instances, + * ensuring efficient resource usage in high-concurrency environments. + */ + public static class YamlFactory extends BasePooledObjectFactory { + + @Override + public Yaml create() { + PropertyUtils propertyUtils = new PropertyUtils(); + propertyUtils.setBeanAccess(BeanAccess.FIELD); + propertyUtils.setAllowReadOnlyProperties(true); + DumperOptions options = new DumperOptions(); + Representer representer = new OmSnapshotLocalDataRepresenter(options); + representer.setPropertyUtils(propertyUtils); + SafeConstructor snapshotDataConstructor = new SnapshotLocalDataConstructor(); + return new Yaml(snapshotDataConstructor, representer); + } + + @Override + public PooledObject wrap(Yaml yaml) { + return new DefaultPooledObject<>(yaml); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index c945574e8f7e..4f7913ec9440 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -76,6 +76,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; @@ -115,6 +116,7 @@ import org.rocksdb.RocksDBException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; /** * This class is used to manage/create OM snapshots. @@ -186,6 +188,7 @@ public final class OmSnapshotManager implements AutoCloseable { private final List columnFamilyDescriptors; private final List columnFamilyHandles; private final SnapshotDiffCleanupService snapshotDiffCleanupService; + private final GenericObjectPool yamlPool; private final int maxPageSize; @@ -196,7 +199,7 @@ public final class OmSnapshotManager implements AutoCloseable { private final AtomicInteger inFlightSnapshotCount = new AtomicInteger(0); public OmSnapshotManager(OzoneManager ozoneManager) { - + this.yamlPool = new GenericObjectPool<>(new OmSnapshotLocalDataYaml.YamlFactory()); boolean isFilesystemSnapshotEnabled = ozoneManager.isFilesystemSnapshotEnabled(); LOG.info("Ozone filesystem snapshot feature is {}.", @@ -501,8 +504,10 @@ public static DBCheckpoint createOmSnapshotCheckpoint( } else { dbCheckpoint = store.getSnapshot(snapshotInfo.getCheckpointDirName()); } + OmSnapshotManager omSnapshotManager = + ((OmMetadataManagerImpl) omMetadataManager).getOzoneManager().getOmSnapshotManager(); // Create the snapshot local property file. - OmSnapshotManager.createNewOmSnapshotLocalDataFile(omMetadataManager, snapshotInfo, store); + OmSnapshotManager.createNewOmSnapshotLocalDataFile(omSnapshotManager, omMetadataManager, snapshotInfo, store); // Clean up active DB's deletedTable right after checkpoint is taken, // There is no need to take any lock as of now, because transactions are flushed sequentially. @@ -643,14 +648,14 @@ private static List getSnapshotSSTFileList(RDBStore store) * @param snapshotInfo The metadata of snapshot to be created * @param store The store used to get uncompacted SST file list from. */ - public static void createNewOmSnapshotLocalDataFile( + public static void createNewOmSnapshotLocalDataFile(OmSnapshotManager snapshotManager, OMMetadataManager omMetadataManager, SnapshotInfo snapshotInfo, RDBStore store) throws IOException { Path snapshotLocalDataPath = Paths.get(getSnapshotLocalPropertyYamlPath(omMetadataManager, snapshotInfo)); Files.deleteIfExists(snapshotLocalDataPath); OmSnapshotLocalDataYaml snapshotLocalDataYaml = new OmSnapshotLocalDataYaml(getSnapshotSSTFileList(store), snapshotInfo.getPathPreviousSnapshotId()); - snapshotLocalDataYaml.writeToYaml(snapshotLocalDataPath.toFile()); + snapshotLocalDataYaml.writeToYaml(snapshotManager, snapshotLocalDataPath.toFile()); } // Get OmSnapshot if the keyName has ".snapshot" key indicator @@ -694,6 +699,26 @@ public UncheckedAutoCloseableSupplier getSnapshot( return getSnapshot(volumeName, bucketName, snapshotName, true); } + public UncheckedAutoCloseableSupplier getSnapshotLocalYaml() throws IOException { + try { + Yaml yaml = yamlPool.borrowObject(); + return new UncheckedAutoCloseableSupplier() { + + @Override + public void close() { + yamlPool.returnObject(yaml); + } + + @Override + public Yaml get() { + return yaml; + } + }; + } catch (Exception e) { + throw new IOException("Failed to get snapshot local yaml", e); + } + } + private UncheckedAutoCloseableSupplier getSnapshot( String volumeName, String bucketName, @@ -1167,6 +1192,9 @@ public void close() { if (options != null) { options.close(); } + if (yamlPool != null) { + yamlPool.close(); + } } public long getDiffCleanupServiceInterval() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java index d1826f923def..c86e5e89213d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java @@ -45,10 +45,12 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OmSnapshotLocalData.VersionMeta; import org.apache.ozone.compaction.log.SstFileInfo; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.rocksdb.LiveFileMetaData; +import org.yaml.snakeyaml.Yaml; /** * This class tests creating and reading snapshot data YAML files. @@ -56,12 +58,27 @@ public class TestOmSnapshotLocalDataYaml { private static String testRoot = new FileSystemTestHelper().getTestRootDir(); + private static OmSnapshotManager omSnapshotManager; + private static final Yaml YAML = new OmSnapshotLocalDataYaml.YamlFactory().create(); + private static final UncheckedAutoCloseableSupplier YAML_SUPPLIER = new UncheckedAutoCloseableSupplier() { + @Override + public Yaml get() { + return YAML; + } + + @Override + public void close() { + + } + }; private static final Instant NOW = Instant.now(); @BeforeEach - public void setUp() { + public void setUp() throws IOException { assertTrue(new File(testRoot).mkdirs()); + omSnapshotManager = mock(OmSnapshotManager.class); + when(omSnapshotManager.getSnapshotLocalYaml()).thenReturn(YAML_SUPPLIER); } @AfterEach @@ -114,7 +131,7 @@ private Pair writeToYaml(String snapshotName) throws IOException { File yamlFile = new File(testRoot, yamlFilePath); // Create YAML file with SnapshotData - dataYaml.writeToYaml(yamlFile); + dataYaml.writeToYaml(omSnapshotManager, yamlFile); // Check YAML file exists assertTrue(yamlFile.exists()); @@ -129,7 +146,7 @@ public void testWriteToYaml() throws IOException { UUID prevSnapId = yamlFilePrevIdPair.getRight(); // Read from YAML file - OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); + OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(omSnapshotManager, yamlFile); // Verify fields assertEquals(44, snapshotData.getVersion()); @@ -168,7 +185,7 @@ public void testUpdateSnapshotDataFile() throws IOException { File yamlFile = yamlFilePrevIdPair.getLeft(); // Read from YAML file OmSnapshotLocalDataYaml dataYaml = - OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); + OmSnapshotLocalDataYaml.getFromYamlFile(omSnapshotManager, yamlFile); // Update snapshot data dataYaml.setSstFiltered(false); @@ -177,10 +194,10 @@ public void testUpdateSnapshotDataFile() throws IOException { singletonList(new SstFileInfo("compacted-sst4", "k5", "k6", "table3")), 5); // Write updated data back to file - dataYaml.writeToYaml(yamlFile); + dataYaml.writeToYaml(omSnapshotManager, yamlFile); // Read back the updated data - dataYaml = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); + dataYaml = OmSnapshotLocalDataYaml.getFromYamlFile(omSnapshotManager, yamlFile); // Verify updated data assertThat(dataYaml.getSstFiltered()).isFalse(); @@ -199,7 +216,7 @@ public void testEmptyFile() throws IOException { assertTrue(emptyFile.createNewFile()); IOException ex = assertThrows(IOException.class, () -> - OmSnapshotLocalDataYaml.getFromYamlFile(emptyFile)); + OmSnapshotLocalDataYaml.getFromYamlFile(omSnapshotManager, emptyFile)); assertThat(ex).hasMessageContaining("Failed to load snapshot file. File is empty."); } @@ -209,7 +226,7 @@ public void testChecksum() throws IOException { Pair yamlFilePrevIdPair = writeToYaml("snapshot3"); File yamlFile = yamlFilePrevIdPair.getLeft(); // Read from YAML file - OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); + OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(omSnapshotManager, yamlFile); // Get the original checksum String originalChecksum = snapshotData.getChecksum(); @@ -217,7 +234,7 @@ public void testChecksum() throws IOException { // Verify the checksum is not null or empty assertThat(originalChecksum).isNotNull().isNotEmpty(); - assertTrue(OmSnapshotLocalDataYaml.verifyChecksum(snapshotData)); + assertTrue(OmSnapshotLocalDataYaml.verifyChecksum(omSnapshotManager, snapshotData)); } @Test diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 5d9248a3131e..1f0950354caa 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -316,12 +316,12 @@ public void testCreateNewSnapshotLocalYaml() throws IOException { assertTrue(snapshotYaml.toFile().createNewFile()); assertEquals(0, Files.size(snapshotYaml)); // Create a new YAML file for the snapshot - OmSnapshotManager.createNewOmSnapshotLocalDataFile(omMetadataManager, snapshotInfo, mockedStore); + OmSnapshotManager.createNewOmSnapshotLocalDataFile(omSnapshotManager, omMetadataManager, snapshotInfo, mockedStore); // Verify that previous file was overwritten assertTrue(Files.exists(snapshotYaml)); assertTrue(Files.size(snapshotYaml) > 0); // Verify the contents of the YAML file - OmSnapshotLocalData localData = OmSnapshotLocalDataYaml.getFromYamlFile(snapshotYaml.toFile()); + OmSnapshotLocalData localData = OmSnapshotLocalDataYaml.getFromYamlFile(omSnapshotManager, snapshotYaml.toFile()); assertNotNull(localData); assertEquals(0, localData.getVersion()); assertEquals(uncompactedVersionMeta, localData.getVersionSstFileInfos().get(0)); diff --git a/pom.xml b/pom.xml index 829648c9a05d..ddaf38f9a9d9 100644 --- a/pom.xml +++ b/pom.xml @@ -45,6 +45,7 @@ true 3.0.2 9.40 + 2.12.1 1.11.0 1.8.0 1.18.0 @@ -708,6 +709,11 @@ commons-math3 ${commons-math3.version} + + org.apache.commons + commons-pool2 + ${common-pool2.version} + org.apache.commons commons-text From e07d4508af0ef2b0f82217840f6d045b4cc3b5de Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sat, 30 Aug 2025 06:04:09 -0400 Subject: [PATCH 3/5] HDDS-13612. Fix dependency Change-Id: Icf245ff856f13ffab73beefc84b743f0de69ef58 --- hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 1 + hadoop-ozone/dist/src/main/license/jar-report.txt | 1 + .../hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java | 11 ++++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt index e4f77557495e..884c84a1669f 100644 --- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt @@ -367,6 +367,7 @@ Apache License 2.0 org.apache.commons:commons-compress org.apache.commons:commons-configuration2 org.apache.commons:commons-lang3 + org.apache.commons:commons-pool2 org.apache.commons:commons-text org.apache.curator:curator-client org.apache.curator:curator-framework diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index 30f02fa8ef15..d82ad46fdb3f 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -32,6 +32,7 @@ share/ozone/lib/commons-io.jar share/ozone/lib/commons-lang3.jar share/ozone/lib/commons-lang.jar share/ozone/lib/commons-net.jar +share/ozone/lib/commons-pool2.jar share/ozone/lib/commons-text.jar share/ozone/lib/commons-validator.jar share/ozone/lib/commons-fileupload.jar diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java index c86e5e89213d..9bcdb21bde60 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java @@ -47,6 +47,7 @@ import org.apache.ozone.compaction.log.SstFileInfo; import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.rocksdb.LiveFileMetaData; @@ -74,13 +75,17 @@ public void close() { private static final Instant NOW = Instant.now(); - @BeforeEach - public void setUp() throws IOException { - assertTrue(new File(testRoot).mkdirs()); + @BeforeAll + public static void setupClassMocks() throws IOException { omSnapshotManager = mock(OmSnapshotManager.class); when(omSnapshotManager.getSnapshotLocalYaml()).thenReturn(YAML_SUPPLIER); } + @BeforeEach + public void setUp() { + assertTrue(new File(testRoot).mkdirs()); + } + @AfterEach public void cleanup() { FileUtil.fullyDelete(new File(testRoot)); From 0bced599b01895847dce9fc3c90aa7822038a36b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sat, 30 Aug 2025 08:56:09 -0400 Subject: [PATCH 4/5] HDDS-13612. Fix Test case Change-Id: I220b088e09d90c7dd86af0f27a0bdfe8243a5ac0 --- .../TestOMSnapshotCreateResponse.java | 26 ++++++++++++++++++- .../TestOMSnapshotDeleteResponse.java | 25 +++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java index 398871927199..23ad7c192e30 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java @@ -23,6 +23,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.File; import java.io.IOException; @@ -38,6 +40,9 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -45,11 +50,13 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.util.Time; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.yaml.snakeyaml.Yaml; /** * This class tests OMSnapshotCreateResponse. @@ -69,7 +76,24 @@ public void setup() throws Exception { String fsPath = folder.getAbsolutePath(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + OmSnapshotLocalDataYaml.YamlFactory yamlFactory = new OmSnapshotLocalDataYaml.YamlFactory(); + Yaml yaml = yamlFactory.create(); + UncheckedAutoCloseableSupplier yamlSupplier = new UncheckedAutoCloseableSupplier() { + @Override + public Yaml get() { + return yaml; + } + + @Override + public void close() { + + } + }; + OzoneManager ozoneManager = mock(OzoneManager.class); + OmSnapshotManager omSnapshotManager = mock(OmSnapshotManager.class); + when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager); + when(omSnapshotManager.getSnapshotLocalYaml()).thenReturn(yamlSupplier); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager); batchOperation = omMetadataManager.getStore().initBatchOperation(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java index 92fc9c0a372d..bdcf2bc38437 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java @@ -21,6 +21,8 @@ import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.File; import java.nio.file.Path; @@ -31,7 +33,9 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml; import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteSnapshotResponse; @@ -39,10 +43,12 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.util.Time; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.yaml.snakeyaml.Yaml; /** * This class tests OMSnapshotDeleteResponse. @@ -62,7 +68,24 @@ public void setup() throws Exception { String fsPath = folder.toAbsolutePath().toString(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, fsPath); - omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + OmSnapshotLocalDataYaml.YamlFactory yamlFactory = new OmSnapshotLocalDataYaml.YamlFactory(); + Yaml yaml = yamlFactory.create(); + UncheckedAutoCloseableSupplier yamlSupplier = new UncheckedAutoCloseableSupplier() { + @Override + public Yaml get() { + return yaml; + } + + @Override + public void close() { + + } + }; + OzoneManager ozoneManager = mock(OzoneManager.class); + OmSnapshotManager omSnapshotManager = mock(OmSnapshotManager.class); + when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager); + when(omSnapshotManager.getSnapshotLocalYaml()).thenReturn(yamlSupplier); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager); batchOperation = omMetadataManager.getStore().initBatchOperation(); } From f643193a1249b4d0de38666baee2e94786bd6a4f Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 10 Sep 2025 08:09:09 -0400 Subject: [PATCH 5/5] HDDS-13612. Add javadoc Change-Id: Id4b3d6fb8b3138eef663cc35ba8690ae6c6cd5dc --- .../java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java index dc3d902db1e2..b3c6f1792f0a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java @@ -57,6 +57,7 @@ public abstract class OmSnapshotLocalData { // Whether the snapshot needs compaction private boolean needsCompaction; + // Previous snapshotId based on which the snapshot local data is built. private UUID previousSnapshotId; // Map of version to VersionMeta, using linkedHashMap since the order of the map needs to be deterministic for