HDDS-13452. Prevent snapshot defrag from happening before upgrade finalization

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/MapBackedTableIterator.java

@@ -19,7 +19,7 @@
 
 import java.util.Iterator;
 import java.util.Map;
-import java.util.TreeMap;
+import java.util.NavigableMap;
 import org.apache.hadoop.hdds.utils.db.Table;
 
 /**
@@ -29,9 +29,9 @@ public class MapBackedTableIterator<V> implements Table.KeyValueIterator<String,
 
   private Iterator<Table.KeyValue<String, V>> itr;
   private final String prefix;
-  private final TreeMap<String, V> values;
+  private final NavigableMap<String, V> values;
 
-  public MapBackedTableIterator(TreeMap<String, V> values, String prefix) {
+  public MapBackedTableIterator(NavigableMap<String, V> values, String prefix) {
     this.prefix = prefix;
     this.values = values;
     this.seekToFirst();

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java

@@ -21,21 +21,22 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.NavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
 import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
 
 /**
  * InMemory Table implementation for tests.
  */
-public final class InMemoryTestTable<KEY, VALUE> implements Table<KEY, VALUE> {
-  private final Map<KEY, VALUE> map;
+public class InMemoryTestTable<KEY, VALUE> implements Table<KEY, VALUE> {
+  private final NavigableMap<KEY, VALUE> map;
 
   public InMemoryTestTable() {
     this(Collections.emptyMap());
   }
 
   public InMemoryTestTable(Map<KEY, VALUE> map) {
-    this.map = new ConcurrentHashMap<>();
+    this.map = new ConcurrentSkipListMap<>(map);
     this.map.putAll(map);
   }
 
@@ -124,4 +125,8 @@ public void dumpToFileWithPrefix(File externalFile, KEY prefix) {
   public void loadFromFile(File externalFile) {
     throw new UnsupportedOperationException();
   }
+
+  NavigableMap<KEY, VALUE> getMap() {
+    return map;
+  }
 }

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/StringInMemoryTestTable.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils.db;
+
+import org.apache.hadoop.hdds.utils.MapBackedTableIterator;
+
+/**
+ * In memory test table for String keys.
+ * @param <V> Value type.
+ */
+public class StringInMemoryTestTable<V> extends InMemoryTestTable<String, V> {
+  @Override
+  public KeyValueIterator<String, V> iterator(String prefix, KeyValueIterator.Type type) {
+    return new MapBackedTableIterator<>(getMap(), prefix);
+  }
+}

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java

@@ -46,9 +46,6 @@
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
-import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
-import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TIMEOUT;
@@ -62,7 +59,6 @@
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.RemovalListener;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import jakarta.annotation.Nonnull;
 import java.io.File;
 import java.io.IOException;
@@ -111,10 +107,10 @@
 import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
 import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
 import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer;
+import org.apache.ratis.util.function.CheckedFunction;
 import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
-import org.rocksdb.LiveFileMetaData;
 import org.rocksdb.RocksDBException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -177,7 +173,7 @@ public final class OmSnapshotManager implements AutoCloseable {
    * families before compaction.
    */
   public static final Set<String> COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT =
-      ImmutableSet.of(KEY_TABLE, DIRECTORY_TABLE, FILE_TABLE);
+      RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG;
 
   private final long diffCleanupServiceInterval;
   private final int maxOpenSstFilesInSnapshotDb;
@@ -198,12 +194,9 @@ public final class OmSnapshotManager implements AutoCloseable {
 
   public OmSnapshotManager(OzoneManager ozoneManager) throws IOException {
     OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager();
-    this.snapshotLocalDataManager = new OmSnapshotLocalDataManager(ozoneManager.getMetadataManager(),
-        omMetadataManager.getSnapshotChainManager(), ozoneManager.getConfiguration());
     boolean isFilesystemSnapshotEnabled = ozoneManager.isFilesystemSnapshotEnabled();
     LOG.info("Ozone filesystem snapshot feature is {}.",
         isFilesystemSnapshotEnabled ? "enabled" : "disabled");
-
     // Confirm that snapshot feature can be safely disabled.
     // Throw unchecked exception if that is not the case.
     if (!isFilesystemSnapshotEnabled &&
@@ -216,7 +209,6 @@ public OmSnapshotManager(OzoneManager ozoneManager) throws IOException {
           "Please set config ozone.filesystem.snapshot.enabled to true and " +
           "try to start this Ozone Manager again.");
     }
-
     this.options = new ManagedDBOptions();
     this.options.setCreateIfMissing(true);
     this.columnFamilyOptions = new ManagedColumnFamilyOptions();
@@ -231,14 +223,19 @@ public OmSnapshotManager(OzoneManager ozoneManager) throws IOException {
         OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES,
         OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT
     );
+    CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException> defaultSnapDBProvider = snapshotInfo ->
+        getSnapshotOmMetadataManager(snapshotInfo, 0, maxOpenSstFilesInSnapshotDb,
+            ozoneManager.getConfiguration());
+    this.snapshotLocalDataManager = new OmSnapshotLocalDataManager(ozoneManager.getMetadataManager(),
+        omMetadataManager.getSnapshotChainManager(), ozoneManager.getVersionManager(), defaultSnapDBProvider,
+        ozoneManager.getConfiguration());
     Preconditions.checkArgument(this.maxOpenSstFilesInSnapshotDb >= -1,
         OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES + " value should be larger than or equal to -1.");
 
     ColumnFamilyHandle snapDiffJobCf;
     ColumnFamilyHandle snapDiffReportCf;
     ColumnFamilyHandle snapDiffPurgedJobCf;
     String dbPath = getDbPath(ozoneManager.getConfiguration());
-
     try {
       // Add default CF
       columnFamilyDescriptors.add(new ColumnFamilyDescriptor(
@@ -377,6 +374,12 @@ public boolean canDisableFsSnapshot(OMMetadataManager ommm) {
     return isSnapshotInfoTableEmpty;
   }
 
+  private static OmMetadataManagerImpl getSnapshotOmMetadataManager(SnapshotInfo snapshotInfo, int version,
+      int maxOpenSstFilesInSnapshotDb, OzoneConfiguration conf) throws IOException {
+    return new OmMetadataManagerImpl(conf, snapshotInfo.getCheckpointDirName(version),
+        maxOpenSstFilesInSnapshotDb);
+  }
+
   private CacheLoader<UUID, OmSnapshot> createCacheLoader() {
     return new CacheLoader<UUID, OmSnapshot>() {
 
@@ -429,9 +432,8 @@ public OmSnapshot load(@Nonnull UUID snapshotId) throws IOException {
           }
           try (OmSnapshotLocalDataManager.ReadableOmSnapshotLocalDataMetaProvider snapshotLocalDataProvider =
                    snapshotLocalDataManager.getOmSnapshotLocalDataMeta(snapshotInfo)) {
-            snapshotMetadataManager = new OmMetadataManagerImpl(conf,
-                snapshotInfo.getCheckpointDirName(snapshotLocalDataProvider.getMeta().getVersion()),
-                maxOpenSstFilesInSnapshotDb);
+            snapshotMetadataManager = getSnapshotOmMetadataManager(snapshotInfo,
+                snapshotLocalDataProvider.getMeta().getVersion(), maxOpenSstFilesInSnapshotDb, conf);
           }
         } catch (IOException e) {
           LOG.error("Failed to retrieve snapshot: {}", snapshotTableKey, e);
@@ -596,17 +598,6 @@ public SnapshotDiffCleanupService getSnapshotDiffCleanupService() {
     return snapshotDiffCleanupService;
   }
 
-  /**
-   * Captures the list of SST files for keyTable, fileTable and directoryTable in the DB.
-   * @param store AOS or snapshot DB for not defragged or defragged snapshot respectively.
-   * @return a Map of (table, set of SST files corresponding to the table)
-   */
-  public static List<LiveFileMetaData> getSnapshotSSTFileList(RDBStore store) throws IOException {
-    return store.getDb().getLiveFilesMetaData().stream()
-        .filter(lfm -> COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT.contains(StringUtils.bytes2String(lfm.columnFamilyName())))
-        .collect(Collectors.toList());
-  }
-
   // Get OmSnapshot if the keyName has ".snapshot" key indicator
   @SuppressWarnings("unchecked")
   public UncheckedAutoCloseableSupplier<IOmMetadataReader> getActiveFsMetadataOrSnapshot(

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java

@@ -45,6 +45,7 @@
 import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
 import org.apache.hadoop.ozone.om.snapshot.MultiSnapshotLocks;
 import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
 import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -347,7 +348,8 @@ private boolean shouldRun() {
       return false;
     }
     // The service only runs if current OM node is ready
-    return running.get() && ozoneManager.isRunning();
+    return running.get() && ozoneManager.isRunning() &&
+        ozoneManager.getVersionManager().isAllowed(OMLayoutFeature.SNAPSHOT_DEFRAG);
   }
 
   public AtomicLong getSnapshotsDefraggedCount() {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java

@@ -20,6 +20,10 @@
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOCAL_DATA_MANAGER_SERVICE_INTERVAL;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_LOCAL_DATA_MANAGER_SERVICE_INTERVAL_DEFAULT;
 import static org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml.YAML_FILE_EXTENSION;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT;
+import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE;
+import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.SNAPSHOT_DEFRAG;
+import static org.apache.ozone.rocksdb.util.RdbUtil.getLiveSSTFilesForCFs;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.graph.GraphBuilder;
@@ -52,7 +56,10 @@
 import org.apache.hadoop.hdds.utils.Scheduler;
 import org.apache.hadoop.hdds.utils.TransactionInfo;
 import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
 import org.apache.hadoop.ozone.om.OmSnapshotLocalData;
 import org.apache.hadoop.ozone.om.OmSnapshotLocalData.VersionMeta;
 import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml;
@@ -63,8 +70,10 @@
 import org.apache.hadoop.ozone.om.lock.HierarchicalResourceLockManager;
 import org.apache.hadoop.ozone.om.lock.HierarchicalResourceLockManager.HierarchicalResourceLock;
 import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
 import org.apache.hadoop.ozone.util.ObjectSerializer;
 import org.apache.hadoop.ozone.util.YamlSerializer;
+import org.apache.ratis.util.function.CheckedFunction;
 import org.apache.ratis.util.function.CheckedSupplier;
 import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
 import org.rocksdb.LiveFileMetaData;
@@ -113,7 +122,8 @@ public class OmSnapshotLocalDataManager implements AutoCloseable {
   private volatile boolean closed;
 
   public OmSnapshotLocalDataManager(OMMetadataManager omMetadataManager,
-      SnapshotChainManager snapshotChainManager,
+      SnapshotChainManager snapshotChainManager, OMLayoutVersionManager omLayoutVersionManager,
+      CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException> defaultSnapProvider,
       OzoneConfiguration configuration) throws IOException {
     this.localDataGraph = GraphBuilder.directed().build();
     this.omMetadataManager = omMetadataManager;
@@ -128,7 +138,7 @@ public void computeAndSetChecksum(Yaml yaml, OmSnapshotLocalData data) throws IO
     this.versionNodeMap = new ConcurrentHashMap<>();
     this.fullLock = new ReentrantReadWriteLock();
     this.internalLock = new ReentrantReadWriteLock();
-    init(configuration, snapshotChainManager);
+    init(configuration, snapshotChainManager, omLayoutVersionManager, defaultSnapProvider);
   }
 
   @VisibleForTesting
@@ -172,7 +182,8 @@ public void createNewOmSnapshotLocalDataFile(RDBStore snapshotStore, SnapshotInf
     try (WritableOmSnapshotLocalDataProvider snapshotLocalData =
              new WritableOmSnapshotLocalDataProvider(snapshotInfo.getSnapshotId(),
                  () -> Pair.of(new OmSnapshotLocalData(snapshotInfo.getSnapshotId(),
-                         OmSnapshotManager.getSnapshotSSTFileList(snapshotStore),
+                         getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(),
+                             COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT),
                          snapshotInfo.getPathPreviousSnapshotId(), null),
                      null))) {
       snapshotLocalData.commit();
@@ -242,6 +253,32 @@ private void addSnapshotVersionMeta(UUID snapshotId, SnapshotVersionsMeta snapsh
     }
   }
 
+  private void addMissingSnapshotYamlFiles(
+      CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException> defaultSnapProvider) throws IOException {
+    try (Table.KeyValueIterator<String, SnapshotInfo> itr = omMetadataManager.getSnapshotInfoTable().iterator()) {
+      while (itr.hasNext()) {
+        SnapshotInfo snapshotInfo = itr.next().getValue();
+        UUID snapshotId = snapshotInfo.getSnapshotId();
+        File snapshotLocalDataFile = new File(getSnapshotLocalPropertyYamlPath(snapshotId));
+        // Create a yaml file for snapshots which are missing
+        if (!snapshotLocalDataFile.exists()) {
+          List<LiveFileMetaData> sstList = Collections.emptyList();
+          if (snapshotInfo.getSnapshotStatus() == SNAPSHOT_ACTIVE) {
+            try (OmMetadataManagerImpl snapshotMetadataManager = defaultSnapProvider.apply(snapshotInfo)) {
+              ManagedRocksDB snapDB = ((RDBStore)snapshotMetadataManager.getStore()).getDb().getManagedRocksDb();
+              sstList = getLiveSSTFilesForCFs(snapDB, COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT);
+            } catch (Exception e) {
+              throw new IOException(e);
+            }
+          }
+          OmSnapshotLocalData snapshotLocalData = new OmSnapshotLocalData(snapshotId, sstList,
+              snapshotInfo.getPathPreviousSnapshotId(), null);
+          snapshotLocalDataSerializer.save(snapshotLocalDataFile, snapshotLocalData);
+        }
+      }
+    }
+  }
+
   void addVersionNodeWithDependents(OmSnapshotLocalData snapshotLocalData) throws IOException {
     if (versionNodeMap.containsKey(snapshotLocalData.getSnapshotId())) {
       return;
@@ -297,12 +334,18 @@ Map<UUID, Integer> getSnapshotToBeCheckedForOrphans() {
     return snapshotToBeCheckedForOrphans;
   }
 
-  private void init(OzoneConfiguration configuration, SnapshotChainManager chainManager) throws IOException {
+  private void init(OzoneConfiguration configuration, SnapshotChainManager chainManager,
+      OMLayoutVersionManager layoutVersionManager,
+      CheckedFunction<SnapshotInfo, OmMetadataManagerImpl, IOException> defaultSnapProvider) throws IOException {
     this.locks = omMetadataManager.getHierarchicalLockManager();
     this.snapshotToBeCheckedForOrphans = new ConcurrentHashMap<>();
     RDBStore store = (RDBStore) omMetadataManager.getStore();
     String checkpointPrefix = store.getDbLocation().getName();
     File snapshotDir = new File(store.getSnapshotsParentDir());
+    boolean upgradeNeeded = !layoutVersionManager.isAllowed(SNAPSHOT_DEFRAG);
+    if (upgradeNeeded) {
+      addMissingSnapshotYamlFiles(defaultSnapProvider);
+    }
     File[] localDataFiles = snapshotDir.listFiles(
         (dir, name) -> name.startsWith(checkpointPrefix) && name.endsWith(YAML_FILE_EXTENSION));
     if (localDataFiles == null) {
@@ -826,7 +869,8 @@ private SnapshotVersionsMeta validateModification(OmSnapshotLocalData snapshotLo
     }
 
     public void addSnapshotVersion(RDBStore snapshotStore) throws IOException {
-      List<LiveFileMetaData> sstFiles = OmSnapshotManager.getSnapshotSSTFileList(snapshotStore);
+      List<LiveFileMetaData> sstFiles = getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(),
+          COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT);
       OmSnapshotLocalData previousSnapshotLocalData = getPreviousSnapshotLocalData();
       this.getSnapshotLocalData().addVersionSSTFileInfos(sstFiles, previousSnapshotLocalData == null ? 0 :
           previousSnapshotLocalData.getVersion());

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java

@@ -45,7 +45,8 @@ public enum OMLayoutFeature implements LayoutFeature {
 
   QUOTA(6, "Ozone quota re-calculate"),
   HBASE_SUPPORT(7, "Full support of hsync, lease recovery and listOpenFiles APIs for HBase"),
-  DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric key");
+  DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric key"),
+  SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot");
 
   ///////////////////////////////  /////////////////////////////
   //    Example OM Layout Feature with Actions