diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 6bee926336b8..c87fcc4bf062 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -202,6 +202,15 @@ public final class OzoneConsts { public static final String SCM_CONTEXT_ATTRIBUTE = "ozone.scm"; + // YAML field constants for OmSnapshotLocalData (thus the OM_SLD_ prefix) YAML files + public static final String OM_SLD_VERSION = "version"; + public static final String OM_SLD_CHECKSUM = "checksum"; + public static final String OM_SLD_IS_SST_FILTERED = "isSSTFiltered"; + public static final String OM_SLD_UNCOMPACTED_SST_FILE_LIST = "uncompactedSSTFileList"; + public static final String OM_SLD_LAST_COMPACTION_TIME = "lastCompactionTime"; + public static final String OM_SLD_NEEDS_COMPACTION = "needsCompaction"; + public static final String OM_SLD_COMPACTED_SST_FILE_LIST = "compactedSSTFileList"; + // YAML fields for .container files public static final String CONTAINER_ID = "containerID"; public static final String CONTAINER_TYPE = "containerType"; diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index d2dcdb9d6578..14e818e85414 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -310,6 +310,10 @@ org.slf4j slf4j-api + + org.yaml + snakeyaml + org.apache.ozone hdds-docs diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java new file mode 100644 index 000000000000..ce127fb3a63a --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.codec.digest.DigestUtils; +import org.yaml.snakeyaml.Yaml; + +/** + * OmSnapshotLocalData is the in-memory representation of snapshot local metadata. + * Inspired by org.apache.hadoop.ozone.container.common.impl.ContainerData + */ +public abstract class OmSnapshotLocalData { + + // Version of the snapshot local data. A valid version shall be greater than 0. + private int version; + + // Checksum of the YAML representation + private String checksum; + + // Whether SST is filtered + private boolean isSSTFiltered; + + // Map of Table to uncompacted SST file list on snapshot create + private Map> uncompactedSSTFileList; + + // Time of last compaction, in epoch milliseconds + private long lastCompactionTime; + + // Whether the snapshot needs compaction + private boolean needsCompaction; + + // Map of version to compacted SST file list + // Map> + private Map>> compactedSSTFileList; + + public static final Charset CHARSET_ENCODING = StandardCharsets.UTF_8; + private static final String DUMMY_CHECKSUM = new String(new byte[64], CHARSET_ENCODING); + + /** + * Creates a OmSnapshotLocalData object with default values. + */ + public OmSnapshotLocalData() { + this.isSSTFiltered = false; + this.uncompactedSSTFileList = new HashMap<>(); + this.lastCompactionTime = 0L; + this.needsCompaction = false; + this.compactedSSTFileList = new HashMap<>(); + this.version = 0; + setChecksumTo0ByteArray(); + } + + /** + * Copy constructor to create a deep copy of OmSnapshotLocalData object. + * @param source The source OmSnapshotLocalData to copy from + */ + public OmSnapshotLocalData(OmSnapshotLocalData source) { + // Copy primitive fields directly + this.isSSTFiltered = source.isSSTFiltered; + this.lastCompactionTime = source.lastCompactionTime; + this.needsCompaction = source.needsCompaction; + this.checksum = source.checksum; + this.version = source.version; + + // Deep copy for uncompactedSSTFileList + this.uncompactedSSTFileList = new HashMap<>(); + for (Map.Entry> entry : + source.uncompactedSSTFileList.entrySet()) { + this.uncompactedSSTFileList.put( + entry.getKey(), + Lists.newArrayList(entry.getValue())); + } + + // Deep copy for compactedSSTFileList + this.compactedSSTFileList = new HashMap<>(); + for (Map.Entry>> versionEntry : + source.compactedSSTFileList.entrySet()) { + Map> tableMap = new HashMap<>(); + + for (Map.Entry> tableEntry : + versionEntry.getValue().entrySet()) { + tableMap.put( + tableEntry.getKey(), + Lists.newArrayList(tableEntry.getValue())); + } + + this.compactedSSTFileList.put(versionEntry.getKey(), tableMap); + } + } + + /** + * Returns whether SST is filtered for this snapshot. + * @return true if SST is filtered, false otherwise + */ + public boolean getSstFiltered() { + return isSSTFiltered; + } + + /** + * Sets whether SST is filtered for this snapshot. + * @param sstFiltered + */ + public void setSstFiltered(boolean sstFiltered) { + this.isSSTFiltered = sstFiltered; + } + + /** + * Returns the uncompacted SST file list. + * @return Map of Table to uncompacted SST file list + */ + public Map> getUncompactedSSTFileList() { + return Collections.unmodifiableMap(this.uncompactedSSTFileList); + } + + /** + * Sets the uncompacted SST file list. + * @param uncompactedSSTFileList Map of Table to uncompacted SST file list + */ + public void setUncompactedSSTFileList( + Map> uncompactedSSTFileList) { + this.uncompactedSSTFileList.clear(); + this.uncompactedSSTFileList.putAll(uncompactedSSTFileList); + } + + /** + * Adds an entry to the uncompacted SST file list. + * @param table Table name + * @param sstFile SST file name + */ + public void addUncompactedSSTFile(String table, String sstFile) { + this.uncompactedSSTFileList.computeIfAbsent(table, k -> Lists.newArrayList()) + .add(sstFile); + } + + /** + * Returns the last compaction time, in epoch milliseconds. + * @return Timestamp of the last compaction + */ + public long getLastCompactionTime() { + return lastCompactionTime; + } + + /** + * Sets the last compaction time, in epoch milliseconds. + * @param lastCompactionTime Timestamp of the last compaction + */ + public void setLastCompactionTime(Long lastCompactionTime) { + this.lastCompactionTime = lastCompactionTime; + } + + /** + * Returns whether the snapshot needs compaction. + * @return true if the snapshot needs compaction, false otherwise + */ + public boolean getNeedsCompaction() { + return needsCompaction; + } + + /** + * Sets whether the snapshot needs compaction. + * @param needsCompaction true if the snapshot needs compaction, false otherwise + */ + public void setNeedsCompaction(boolean needsCompaction) { + this.needsCompaction = needsCompaction; + } + + /** + * Returns the compacted SST file list. + * @return Map of version to compacted SST file list + */ + public Map>> getCompactedSSTFileList() { + return Collections.unmodifiableMap(this.compactedSSTFileList); + } + + /** + * Sets the compacted SST file list. + * @param compactedSSTFileList Map of version to compacted SST file list + */ + public void setCompactedSSTFileList( + Map>> compactedSSTFileList) { + this.compactedSSTFileList.clear(); + this.compactedSSTFileList.putAll(compactedSSTFileList); + } + + /** + * Adds an entry to the compacted SST file list. + * @param ver Version number (TODO: to be clarified) + * @param table Table name + * @param sstFile SST file name + */ + public void addCompactedSSTFile(Integer ver, String table, String sstFile) { + this.compactedSSTFileList.computeIfAbsent(ver, k -> Maps.newHashMap()) + .computeIfAbsent(table, k -> Lists.newArrayList()) + .add(sstFile); + } + + /** + * Returns the checksum of the YAML representation. + * @return checksum + */ + public String getChecksum() { + return checksum; + } + + /** + * Sets the checksum of the YAML representation. + * @param checksum checksum + */ + public void setChecksum(String checksum) { + this.checksum = checksum; + } + + /** + * Sets the checksum to a 0 byte array. + */ + public void setChecksumTo0ByteArray() { + this.checksum = DUMMY_CHECKSUM; + } + + /** + * Compute and set checksum for the snapshot data. + * @param yaml Yaml instance for serialization + * @throws IOException if checksum computation fails + */ + public void computeAndSetChecksum(Yaml yaml) throws IOException { + // Set checksum to dummy value - 0 byte array, to calculate the checksum + // of rest of the data. + setChecksumTo0ByteArray(); + + // Dump yaml data into a string to compute its checksum + String snapshotDataYamlStr = yaml.dump(this); + + this.checksum = getChecksum(snapshotDataYamlStr); + } + + /** + * Computes SHA-256 hash for a given string. + * @param data String data for which checksum needs to be calculated + * @return SHA-256 checksum as hex string + * @throws IOException If checksum calculation fails + */ + private static String getChecksum(String data) throws IOException { + try { + return DigestUtils.sha256Hex(data.getBytes(StandardCharsets.UTF_8)); + } catch (Exception ex) { + throw new IOException("Unable to calculate checksum", ex); + } + } + + /** + * Returns the version of the snapshot local data. + * @return version + */ + public int getVersion() { + return version; + } + + /** + * Sets the version of the snapshot local data. A valid version shall be greater than 0. + * @param version version + */ + public void setVersion(int version) { + this.version = version; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java new file mode 100644 index 000000000000..97b401d8cb9b --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import com.google.common.base.Preconditions; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hdds.server.YamlUtils; +import org.apache.hadoop.ozone.OzoneConsts; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.error.YAMLException; +import org.yaml.snakeyaml.introspector.BeanAccess; +import org.yaml.snakeyaml.introspector.PropertyUtils; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; + +/** + * Class for creating and reading snapshot local properties / data YAML files. + * Checksum of the YAML fields are computed and stored in the YAML file transparently to callers. + * Inspired by org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml + */ +public final class OmSnapshotLocalDataYaml extends OmSnapshotLocalData { + + private static final Logger LOG = LoggerFactory.getLogger(OmSnapshotLocalDataYaml.class); + + public static final Tag SNAPSHOT_YAML_TAG = new Tag("OmSnapshotLocalData"); + + /** + * Creates a new OmSnapshotLocalDataYaml with default values. + */ + public OmSnapshotLocalDataYaml() { + super(); + } + + /** + * Copy constructor to create a deep copy. + * @param source The source OmSnapshotLocalData to copy from + */ + public OmSnapshotLocalDataYaml(OmSnapshotLocalData source) { + super(source); + } + + /** + * Verifies the checksum of the snapshot data. + * @param snapshotData The snapshot data to verify + * @return true if the checksum is valid, false otherwise + * @throws IOException if there's an error computing the checksum + */ + public static boolean verifyChecksum(OmSnapshotLocalData snapshotData) + throws IOException { + Preconditions.checkNotNull(snapshotData, "snapshotData cannot be null"); + + // Get the stored checksum + String storedChecksum = snapshotData.getChecksum(); + if (storedChecksum == null) { + LOG.warn("No checksum found in snapshot data for verification"); + return false; + } + + // Create a copy of the snapshot data for computing checksum + OmSnapshotLocalDataYaml snapshotDataCopy = new OmSnapshotLocalDataYaml(snapshotData); + + // Clear the existing checksum in the copy + snapshotDataCopy.setChecksum(null); + + // Get the YAML representation + final Yaml yaml = getYamlForSnapshotLocalData(); + + // Compute new checksum + snapshotDataCopy.computeAndSetChecksum(yaml); + + // Compare the stored and computed checksums + String computedChecksum = snapshotDataCopy.getChecksum(); + boolean isValid = storedChecksum.equals(computedChecksum); + + if (!isValid) { + LOG.warn("Checksum verification failed for snapshot local data. " + + "Stored: {}, Computed: {}", storedChecksum, computedChecksum); + } + + return isValid; + } + + /** + * Constructor class for OmSnapshotLocalData. + * This is used when parsing YAML files into OmSnapshotLocalDataYaml objects. + */ + private static class SnapshotLocalDataConstructor extends SafeConstructor { + SnapshotLocalDataConstructor() { + super(new LoaderOptions()); + //Adding our own specific constructors for tags. + this.yamlConstructors.put(SNAPSHOT_YAML_TAG, new ConstructSnapshotLocalData()); + } + + private final class ConstructSnapshotLocalData extends AbstractConstruct { + @SuppressWarnings("unchecked") + @Override + public Object construct(Node node) { + MappingNode mnode = (MappingNode) node; + Map nodes = constructMapping(mnode); + + OmSnapshotLocalDataYaml snapshotLocalData = new OmSnapshotLocalDataYaml(); + + // Set version from YAML + Integer version = (Integer) nodes.get(OzoneConsts.OM_SLD_VERSION); + // Validate version. + if (version <= 0) { + // If version is not set or invalid, log a warning, but do not throw. + LOG.warn("Invalid version ({}) detected in snapshot local data YAML. Proceed with caution.", version); + } + snapshotLocalData.setVersion(version); + + // Set other fields from parsed YAML + snapshotLocalData.setSstFiltered((Boolean) nodes.getOrDefault(OzoneConsts.OM_SLD_IS_SST_FILTERED, false)); + + Map> uncompactedSSTFileList = + (Map>) nodes.get(OzoneConsts.OM_SLD_UNCOMPACTED_SST_FILE_LIST); + if (uncompactedSSTFileList != null) { + snapshotLocalData.setUncompactedSSTFileList(uncompactedSSTFileList); + } + + snapshotLocalData.setLastCompactionTime( + (Long) nodes.getOrDefault(OzoneConsts.OM_SLD_LAST_COMPACTION_TIME, -1L)); + snapshotLocalData.setNeedsCompaction((Boolean) nodes.getOrDefault(OzoneConsts.OM_SLD_NEEDS_COMPACTION, false)); + + Map>> compactedSSTFileList = + (Map>>) nodes.get(OzoneConsts.OM_SLD_COMPACTED_SST_FILE_LIST); + if (compactedSSTFileList != null) { + snapshotLocalData.setCompactedSSTFileList(compactedSSTFileList); + } + + String checksum = (String) nodes.get(OzoneConsts.OM_SLD_CHECKSUM); + if (checksum != null) { + snapshotLocalData.setChecksum(checksum); + } + + return snapshotLocalData; + } + } + } + + /** + * Returns the YAML representation of this object as a String + * (without triggering checksum computation or persistence). + * @return YAML string representation + */ + public String getYaml() { + final Yaml yaml = getYamlForSnapshotLocalData(); + return yaml.dump(this); + } + + /** + * Computes checksum (stored in this object), and writes this object to a YAML file. + * @param yamlFile The file to write to + * @throws IOException If there's an error writing to the file + */ + public void writeToYaml(File yamlFile) throws IOException { + // Create Yaml + final Yaml yaml = getYamlForSnapshotLocalData(); + // Compute Checksum and update SnapshotData + computeAndSetChecksum(yaml); + // Write the SnapshotData with checksum to Yaml file. + YamlUtils.dump(yaml, this, yamlFile, LOG); + } + + /** + * Creates a OmSnapshotLocalDataYaml instance from a YAML file. + * @param yamlFile The YAML file to read from + * @return A new OmSnapshotLocalDataYaml instance + * @throws IOException If there's an error reading the file + */ + public static OmSnapshotLocalDataYaml getFromYamlFile(File yamlFile) throws IOException { + Preconditions.checkNotNull(yamlFile, "yamlFile cannot be null"); + try (InputStream inputFileStream = Files.newInputStream(yamlFile.toPath())) { + return getFromYamlStream(inputFileStream); + } + } + + /** + * Returns a Yaml representation of the snapshot properties. + * @return Yaml representation of snapshot properties + */ + public static Yaml getYamlForSnapshotLocalData() { + PropertyUtils propertyUtils = new PropertyUtils(); + propertyUtils.setBeanAccess(BeanAccess.FIELD); + propertyUtils.setAllowReadOnlyProperties(true); + + DumperOptions options = new DumperOptions(); + Representer representer = new Representer(options); + representer.setPropertyUtils(propertyUtils); + representer.addClassTag(OmSnapshotLocalDataYaml.class, SNAPSHOT_YAML_TAG); + + SafeConstructor snapshotDataConstructor = new SnapshotLocalDataConstructor(); + return new Yaml(snapshotDataConstructor, representer); + } + + /** + * Read the YAML content InputStream, and return OmSnapshotLocalDataYaml instance. + * @throws IOException + */ + public static OmSnapshotLocalDataYaml getFromYamlStream(InputStream input) throws IOException { + OmSnapshotLocalDataYaml dataYaml; + + PropertyUtils propertyUtils = new PropertyUtils(); + propertyUtils.setBeanAccess(BeanAccess.FIELD); + propertyUtils.setAllowReadOnlyProperties(true); + + DumperOptions options = new DumperOptions(); + Representer representer = new Representer(options); + representer.setPropertyUtils(propertyUtils); + + SafeConstructor snapshotDataConstructor = new SnapshotLocalDataConstructor(); + + Yaml yaml = new Yaml(snapshotDataConstructor, representer); + + try { + dataYaml = yaml.load(input); + } catch (YAMLException ex) { + // Unchecked exception. Convert to IOException + throw new IOException(ex); + } + + if (dataYaml == null) { + // If Yaml#load returned null, then the file is empty. This is valid yaml + // but considered an error in this case since we have lost data about + // the snapshot. + throw new IOException("Failed to load snapshot file. File is empty."); + } + + return dataYaml; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 262750e5c2f2..ec1b06758f66 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -764,6 +764,11 @@ public static String getSnapshotPath(OzoneConfiguration conf, OM_DB_NAME + snapshotInfo.getCheckpointDirName(); } + public static String getSnapshotLocalPropertyYamlPath(OzoneConfiguration conf, + SnapshotInfo snapshotInfo) { + return getSnapshotPath(conf, snapshotInfo) + ".yaml"; + } + public static boolean isSnapshotKey(String[] keyParts) { return (keyParts.length > 1) && (keyParts[0].compareTo(OM_SNAPSHOT_INDICATOR) == 0); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java new file mode 100644 index 000000000000..00fea3e36012 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.FileSystemTestHelper; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.ozone.OzoneConsts; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * This class tests creating and reading snapshot data YAML files. + */ +public class TestOmSnapshotLocalDataYaml { + + private static String testRoot = new FileSystemTestHelper().getTestRootDir(); + + private static final Instant NOW = Instant.now(); + + @BeforeEach + public void setUp() { + assertTrue(new File(testRoot).mkdirs()); + } + + @AfterEach + public void cleanup() { + FileUtil.fullyDelete(new File(testRoot)); + } + + /** + * Creates a snapshot local data YAML file. + */ + private File writeToYaml(String snapshotName) throws IOException { + String yamlFilePath = snapshotName + ".yaml"; + + OmSnapshotLocalDataYaml dataYaml = new OmSnapshotLocalDataYaml(); + + // Set version + dataYaml.setVersion(42); + // Set SST filtered flag + dataYaml.setSstFiltered(true); + + // Add some uncompacted SST files + dataYaml.addUncompactedSSTFile("table1", "sst1"); + dataYaml.addUncompactedSSTFile("table1", "sst2"); + dataYaml.addUncompactedSSTFile("table2", "sst3"); + + // Set last compaction time + dataYaml.setLastCompactionTime(NOW.toEpochMilli()); + + // Set needs compaction flag + dataYaml.setNeedsCompaction(true); + + // Add some compacted SST files + dataYaml.addCompactedSSTFile(1, "table1", "compacted-sst1"); + dataYaml.addCompactedSSTFile(1, "table2", "compacted-sst2"); + dataYaml.addCompactedSSTFile(2, "table1", "compacted-sst3"); + + File yamlFile = new File(testRoot, yamlFilePath); + + // Create YAML file with SnapshotData + dataYaml.writeToYaml(yamlFile); + + // Check YAML file exists + assertTrue(yamlFile.exists()); + + return yamlFile; + } + + @Test + public void testWriteToYaml() throws IOException { + File yamlFile = writeToYaml("snapshot1"); + + // Read from YAML file + OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); + + // Verify fields + assertEquals(42, snapshotData.getVersion()); + assertTrue(snapshotData.getSstFiltered()); + + Map> uncompactedFiles = snapshotData.getUncompactedSSTFileList(); + assertEquals(2, uncompactedFiles.size()); + assertEquals(2, uncompactedFiles.get("table1").size()); + assertEquals(1, uncompactedFiles.get("table2").size()); + assertTrue(uncompactedFiles.get("table1").contains("sst1")); + assertTrue(uncompactedFiles.get("table1").contains("sst2")); + assertTrue(uncompactedFiles.get("table2").contains("sst3")); + + assertEquals(NOW.toEpochMilli(), snapshotData.getLastCompactionTime()); + assertTrue(snapshotData.getNeedsCompaction()); + + Map>> compactedFiles = snapshotData.getCompactedSSTFileList(); + assertEquals(2, compactedFiles.size()); + assertTrue(compactedFiles.containsKey(1)); + assertTrue(compactedFiles.containsKey(2)); + assertEquals(2, compactedFiles.get(1).size()); + assertEquals(1, compactedFiles.get(2).size()); + assertTrue(compactedFiles.get(1).get("table1").contains("compacted-sst1")); + assertTrue(compactedFiles.get(1).get("table2").contains("compacted-sst2")); + assertTrue(compactedFiles.get(2).get("table1").contains("compacted-sst3")); + } + + @Test + public void testUpdateSnapshotDataFile() throws IOException { + File yamlFile = writeToYaml("snapshot2"); + + // Read from YAML file + OmSnapshotLocalDataYaml dataYaml = + OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); + + // Update snapshot data + dataYaml.setSstFiltered(false); + dataYaml.setNeedsCompaction(false); + dataYaml.addUncompactedSSTFile("table3", "sst4"); + dataYaml.addCompactedSSTFile(3, "table3", "compacted-sst4"); + + // Write updated data back to file + dataYaml.writeToYaml(yamlFile); + + // Read back the updated data + dataYaml = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); + + // Verify updated data + assertThat(dataYaml.getSstFiltered()).isFalse(); + assertThat(dataYaml.getNeedsCompaction()).isFalse(); + + Map> uncompactedFiles = dataYaml.getUncompactedSSTFileList(); + assertEquals(3, uncompactedFiles.size()); + assertTrue(uncompactedFiles.containsKey("table3")); + assertTrue(uncompactedFiles.get("table3").contains("sst4")); + + Map>> compactedFiles = dataYaml.getCompactedSSTFileList(); + assertEquals(3, compactedFiles.size()); + assertTrue(compactedFiles.containsKey(3)); + assertTrue(compactedFiles.get(3).containsKey("table3")); + assertTrue(compactedFiles.get(3).get("table3").contains("compacted-sst4")); + } + + @Test + public void testEmptyFile() throws IOException { + File emptyFile = new File(testRoot, "empty.yaml"); + assertTrue(emptyFile.createNewFile()); + + IOException ex = assertThrows(IOException.class, () -> + OmSnapshotLocalDataYaml.getFromYamlFile(emptyFile)); + + assertThat(ex).hasMessageContaining("Failed to load snapshot file. File is empty."); + } + + @Test + public void testChecksum() throws IOException { + File yamlFile = writeToYaml("snapshot3"); + + // Read from YAML file + OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile); + + // Get the original checksum + String originalChecksum = snapshotData.getChecksum(); + + // Verify the checksum is not null or empty + assertThat(originalChecksum).isNotNull().isNotEmpty(); + + assertTrue(OmSnapshotLocalDataYaml.verifyChecksum(snapshotData)); + } + + @Test + public void testYamlContainsAllFields() throws IOException { + File yamlFile = writeToYaml("snapshot4"); + + String content = FileUtils.readFileToString(yamlFile, Charset.defaultCharset()); + + // Verify the YAML content contains all expected fields + assertThat(content).contains(OzoneConsts.OM_SLD_VERSION); + assertThat(content).contains(OzoneConsts.OM_SLD_CHECKSUM); + assertThat(content).contains(OzoneConsts.OM_SLD_IS_SST_FILTERED); + assertThat(content).contains(OzoneConsts.OM_SLD_UNCOMPACTED_SST_FILE_LIST); + assertThat(content).contains(OzoneConsts.OM_SLD_LAST_COMPACTION_TIME); + assertThat(content).contains(OzoneConsts.OM_SLD_NEEDS_COMPACTION); + assertThat(content).contains(OzoneConsts.OM_SLD_COMPACTED_SST_FILE_LIST); + } +}