HDDS-13022. Split up exclusive size tracking for key and directory cleanup in SnapshotInfo

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.client;
 
+import java.util.Objects;
 import java.util.UUID;
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus;
@@ -194,8 +195,31 @@ public static OzoneSnapshot fromSnapshotInfo(SnapshotInfo snapshotInfo) {
         snapshotInfo.getCheckpointDir(),
         snapshotInfo.getReferencedSize(),
         snapshotInfo.getReferencedReplicatedSize(),
-        snapshotInfo.getExclusiveSize(),
-        snapshotInfo.getExclusiveReplicatedSize()
+        snapshotInfo.getExclusiveSize() + snapshotInfo.getExclusiveSizeDeltaFromDirDeepCleaning(),
+        snapshotInfo.getExclusiveReplicatedSize() + snapshotInfo.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning()
     );
   }
+
+  @Override
+  public final boolean equals(Object o) {
+    if (!(o instanceof OzoneSnapshot)) {
+      return false;
+    }
+
+    OzoneSnapshot that = (OzoneSnapshot) o;
+    return creationTime == that.creationTime && referencedSize == that.referencedSize &&
+        referencedReplicatedSize == that.referencedReplicatedSize && exclusiveSize == that.exclusiveSize &&
+        exclusiveReplicatedSize == that.exclusiveReplicatedSize &&
+        Objects.equals(volumeName, that.volumeName) && Objects.equals(bucketName, that.bucketName) &&
+        Objects.equals(name, that.name) && snapshotStatus == that.snapshotStatus &&
+        Objects.equals(snapshotId, that.snapshotId) &&
+        Objects.equals(snapshotPath, that.snapshotPath) &&
+        Objects.equals(checkpointDir, that.checkpointDir);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(volumeName, bucketName, name, creationTime, snapshotStatus, snapshotId, snapshotPath,
+        checkpointDir, referencedSize, referencedReplicatedSize, exclusiveSize, exclusiveReplicatedSize);
+  }
 }

hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneSnapshot.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.client;
+
+import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.when;
+
+import java.util.UUID;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test class for OzoneSnapshot class.
+ */
+public class TestOzoneSnapshot {
+
+  private SnapshotInfo getMockedSnapshotInfo(UUID snapshotId) {
+    SnapshotInfo snapshotInfo = Mockito.mock(SnapshotInfo.class);
+    when(snapshotInfo.getVolumeName()).thenReturn("volume");
+    when(snapshotInfo.getBucketName()).thenReturn("bucket");
+    when(snapshotInfo.getName()).thenReturn("snap");
+    when(snapshotInfo.getCreationTime()).thenReturn(1000L);
+    when(snapshotInfo.getSnapshotStatus()).thenReturn(SNAPSHOT_ACTIVE);
+    when(snapshotInfo.getSnapshotId()).thenReturn(snapshotId);
+    when(snapshotInfo.getSnapshotPath()).thenReturn("volume/bucket");
+    when(snapshotInfo.getCheckpointDir()).thenReturn("checkpointDir");
+    when(snapshotInfo.getReferencedSize()).thenReturn(1000L);
+    when(snapshotInfo.getReferencedReplicatedSize()).thenReturn(3000L);
+    when(snapshotInfo.getExclusiveSize()).thenReturn(4000L);
+    when(snapshotInfo.getExclusiveReplicatedSize()).thenReturn(12000L);
+    when(snapshotInfo.getExclusiveSizeDeltaFromDirDeepCleaning()).thenReturn(2000L);
+    when(snapshotInfo.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning()).thenReturn(6000L);
+    return snapshotInfo;
+  }
+
+  @Test
+  public void testOzoneSnapshotFromSnapshotInfo() {
+    UUID snapshotId = UUID.randomUUID();
+    SnapshotInfo snapshotInfo = getMockedSnapshotInfo(snapshotId);
+    OzoneSnapshot ozoneSnapshot = OzoneSnapshot.fromSnapshotInfo(snapshotInfo);
+    OzoneSnapshot expectedOzoneSnapshot = new OzoneSnapshot(
+        "volume", "bucket", "snap", 1000L, SNAPSHOT_ACTIVE, snapshotId,
+        "volume/bucket", "checkpointDir", 1000L, 3000L, 6000L, 18000L);
+    assertEquals(expectedOzoneSnapshot, ozoneSnapshot);
+  }
+}

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java

@@ -82,6 +82,8 @@ public final class SnapshotInfo implements Auditable, CopyObject<SnapshotInfo> {
   private long referencedReplicatedSize;
   private long exclusiveSize;
   private long exclusiveReplicatedSize;
+  private long exclusiveSizeDeltaFromDirDeepCleaning;
+  private long exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
   private boolean deepCleanedDeletedDir;
   private ByteString lastTransactionInfo;
 
@@ -104,6 +106,8 @@ private SnapshotInfo(Builder b) {
     this.referencedReplicatedSize = b.referencedReplicatedSize;
     this.exclusiveSize = b.exclusiveSize;
     this.exclusiveReplicatedSize = b.exclusiveReplicatedSize;
+    this.exclusiveSizeDeltaFromDirDeepCleaning = b.exclusiveSizeDeltaFromDirDeepCleaning;
+    this.exclusiveReplicatedSizeDeltaFromDirDeepCleaning = b.exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
     this.deepCleanedDeletedDir = b.deepCleanedDeletedDir;
     this.lastTransactionInfo = b.lastTransactionInfo;
   }
@@ -233,6 +237,8 @@ public SnapshotInfo.Builder toBuilder() {
         .setReferencedReplicatedSize(referencedReplicatedSize)
         .setExclusiveSize(exclusiveSize)
         .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+        .setExclusiveSizeDeltaFromDirDeepCleaning(exclusiveSizeDeltaFromDirDeepCleaning)
+        .setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(exclusiveReplicatedSizeDeltaFromDirDeepCleaning)
         .setDeepCleanedDeletedDir(deepCleanedDeletedDir)
         .setLastTransactionInfo(lastTransactionInfo);
   }
@@ -259,6 +265,8 @@ public static class Builder {
     private long referencedReplicatedSize;
     private long exclusiveSize;
     private long exclusiveReplicatedSize;
+    private long exclusiveSizeDeltaFromDirDeepCleaning;
+    private long exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
     private boolean deepCleanedDeletedDir;
     private ByteString lastTransactionInfo;
 
@@ -374,6 +382,19 @@ public Builder setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
       return this;
     }
 
+    /** @param exclusiveSizeDeltaFromDirDeepCleaning - Snapshot exclusive size. */
+    public Builder setExclusiveSizeDeltaFromDirDeepCleaning(long exclusiveSizeDeltaFromDirDeepCleaning) {
+      this.exclusiveSizeDeltaFromDirDeepCleaning = exclusiveSizeDeltaFromDirDeepCleaning;
+      return this;
+    }
+
+    /** @param exclusiveReplicatedSizeDeltaFromDirDeepCleaning - Snapshot exclusive size w/ replication. */
+    public Builder setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(
+        long exclusiveReplicatedSizeDeltaFromDirDeepCleaning) {
+      this.exclusiveReplicatedSizeDeltaFromDirDeepCleaning = exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
+      return this;
+    }
+
     public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) {
       this.deepCleanedDeletedDir = deepCleanedDeletedDir;
       return this;
@@ -408,6 +429,8 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() {
             .setReferencedReplicatedSize(referencedReplicatedSize)
             .setExclusiveSize(exclusiveSize)
             .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+            .setExclusiveSizeDeltaFromDirDeepCleaning(exclusiveSizeDeltaFromDirDeepCleaning)
+            .setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(exclusiveReplicatedSizeDeltaFromDirDeepCleaning)
             .setDeepCleanedDeletedDir(deepCleanedDeletedDir);
 
     if (pathPreviousSnapshotId != null) {
@@ -485,6 +508,15 @@ public static SnapshotInfo getFromProtobuf(
           snapshotInfoProto.getExclusiveReplicatedSize());
     }
 
+    if (snapshotInfoProto.hasExclusiveSizeDeltaFromDirDeepCleaning()) {
+      osib.setExclusiveSizeDeltaFromDirDeepCleaning(snapshotInfoProto.getExclusiveSizeDeltaFromDirDeepCleaning());
+    }
+
+    if (snapshotInfoProto.hasExclusiveReplicatedSizeDeltaFromDirDeepCleaning()) {
+      osib.setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(
+          snapshotInfoProto.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning());
+    }
+
     if (snapshotInfoProto.hasDeepCleanedDeletedDir()) {
       osib.setDeepCleanedDeletedDir(
           snapshotInfoProto.getDeepCleanedDeletedDir());
@@ -571,10 +603,26 @@ public long getExclusiveSize() {
     return exclusiveSize;
   }
 
+  public void setExclusiveSizeDeltaFromDirDeepCleaning(long exclusiveSizeDeltaFromDirDeepCleaning) {
+    this.exclusiveSizeDeltaFromDirDeepCleaning = exclusiveSizeDeltaFromDirDeepCleaning;
+  }
+
+  public long getExclusiveSizeDeltaFromDirDeepCleaning() {
+    return exclusiveSizeDeltaFromDirDeepCleaning;
+  }
+
   public void setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
     this.exclusiveReplicatedSize = exclusiveReplicatedSize;
   }
 
+  public void setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(long exclusiveReplicatedSizeDeltaFromDirDeepCleaning) {
+    this.exclusiveReplicatedSizeDeltaFromDirDeepCleaning = exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
+  }
+
+  public long getExclusiveReplicatedSizeDeltaFromDirDeepCleaning() {
+    return exclusiveReplicatedSizeDeltaFromDirDeepCleaning;
+  }
+
   public long getExclusiveReplicatedSize() {
     return exclusiveReplicatedSize;
   }
@@ -707,7 +755,9 @@ public String toString() {
         ", referencedReplicatedSize: '" + referencedReplicatedSize + '\'' +
         ", exclusiveSize: '" + exclusiveSize + '\'' +
         ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' +
-        ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' +
+        ", exclusiveSizeDeltaFromDirDeepCleaning: '" + exclusiveSizeDeltaFromDirDeepCleaning + '\'' +
+        ", exclusiveReplicatedSizeDeltaFromDirDeepCleaning: '" + exclusiveReplicatedSizeDeltaFromDirDeepCleaning +
+        "', deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' +
         ", lastTransactionInfo: '" + lastTransactionInfo + '\'' +
         '}';
   }

hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java

@@ -67,7 +67,9 @@ private SnapshotInfo createSnapshotInfo() {
         .setReferencedSize(2000L)
         .setReferencedReplicatedSize(6000L)
         .setExclusiveSize(1000L)
+        .setExclusiveSizeDeltaFromDirDeepCleaning(2000L)
         .setExclusiveReplicatedSize(3000L)
+        .setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(6000L)
         .setDeepCleanedDeletedDir(false)
         .build();
   }
@@ -92,6 +94,8 @@ private OzoneManagerProtocolProtos.SnapshotInfo createSnapshotInfoProto() {
         .setReferencedReplicatedSize(6000L)
         .setExclusiveSize(1000L)
         .setExclusiveReplicatedSize(3000L)
+        .setExclusiveSizeDeltaFromDirDeepCleaning(2000L)
+        .setExclusiveReplicatedSizeDeltaFromDirDeepCleaning(6000L)
         .setDeepCleanedDeletedDir(false)
         .build();
   }
@@ -179,7 +183,10 @@ public void testSnapshotInfoProtoToSnapshotInfo() {
         snapshotInfoActual.getExclusiveReplicatedSize());
     assertEquals(snapshotInfoExpected.getDeepCleanedDeletedDir(),
         snapshotInfoActual.getDeepCleanedDeletedDir());
-
+    assertEquals(snapshotInfoExpected.getExclusiveSizeDeltaFromDirDeepCleaning(),
+        snapshotInfoActual.getExclusiveSizeDeltaFromDirDeepCleaning());
+    assertEquals(snapshotInfoExpected.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning(),
+        snapshotInfoActual.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning());
     assertEquals(snapshotInfoExpected, snapshotInfoActual);
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java

@@ -234,17 +234,18 @@ public void testExclusiveSizeWithDirectoryDeepClean() throws Exception {
         put("snap2", 5L);
         put("snap3", 0L);
       }};
-    Thread.sleep(500);
     try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>>
         iterator = snapshotInfoTable.iterator()) {
       while (iterator.hasNext()) {
         Table.KeyValue<String, SnapshotInfo> snapshotEntry = iterator.next();
         String snapshotName = snapshotEntry.getValue().getName();
-        assertEquals(expectedSize.get(snapshotName), snapshotEntry.getValue().
-            getExclusiveSize());
+        SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotEntry.getKey());
+        System.out.println(snapshotInfo.getName() + " " + snapshotInfo.getDeepCleanedDeletedDir());
+        assertEquals(expectedSize.get(snapshotName),
+            snapshotInfo.getExclusiveSize() + snapshotInfo.getExclusiveSizeDeltaFromDirDeepCleaning());
         // Since for the test we are using RATIS/THREE
         assertEquals(expectedSize.get(snapshotName) * 3,
-            snapshotEntry.getValue().getExclusiveReplicatedSize());
+            snapshotInfo.getExclusiveReplicatedSize() + snapshotInfo.getExclusiveReplicatedSizeDeltaFromDirDeepCleaning());
 
       }
     }

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -303,6 +303,7 @@ message OMRequest {
   optional GetObjectTaggingRequest          getObjectTaggingRequest        = 140;
   optional PutObjectTaggingRequest          putObjectTaggingRequest        = 141;
   optional DeleteObjectTaggingRequest       deleteObjectTaggingRequest     = 142;
+  repeated SetSnapshotPropertyRequest       SetSnapshotPropertyRequests    = 143;
 }
 
 message OMResponse {
@@ -892,6 +893,9 @@ message SnapshotInfo {
   // note: shared sizes can be calculated from: referenced - exclusive
   optional bool deepCleanedDeletedDir = 19;
   optional bytes lastTransactionInfo = 20;
+  optional uint64 exclusiveSizeDeltaFromDirDeepCleaning = 21;
+  // snapshot exclusive size after replication
+  optional uint64 exclusiveReplicatedSizeDeltaFromDirDeepCleaning = 22;
 }
 
 message SnapshotDiffJobProto {
@@ -2029,6 +2033,7 @@ message SetSnapshotPropertyRequest {
   optional SnapshotSize snapshotSize = 3;
   optional bool deepCleanedDeletedDir = 4;
   optional bool deepCleanedDeletedKey = 5;
+  optional SnapshotSize snapshotSizeDeltaFromDirDeepCleaning = 6;
 }
 
 // SnapshotProperty in entirely deprecated, Keeping it here for proto.lock compatibility