HDDS-13022. Split up exclusive size tracking for key and directory cleanup in SnapshotInfo

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.client;
 
+import java.util.Objects;
 import java.util.UUID;
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus;
@@ -194,8 +195,31 @@ public static OzoneSnapshot fromSnapshotInfo(SnapshotInfo snapshotInfo) {
         snapshotInfo.getCheckpointDir(),
         snapshotInfo.getReferencedSize(),
         snapshotInfo.getReferencedReplicatedSize(),
-        snapshotInfo.getExclusiveSize(),
-        snapshotInfo.getExclusiveReplicatedSize()
+        snapshotInfo.getExclusiveSize() + snapshotInfo.getExclusiveDirSize(),
+        snapshotInfo.getExclusiveReplicatedSize() + snapshotInfo.getExclusiveDirReplicatedSize()
     );
   }
+
+  @Override
+  public final boolean equals(Object o) {
+    if (!(o instanceof OzoneSnapshot)) {
+      return false;
+    }
+
+    OzoneSnapshot that = (OzoneSnapshot) o;
+    return creationTime == that.creationTime && referencedSize == that.referencedSize &&
+        referencedReplicatedSize == that.referencedReplicatedSize && exclusiveSize == that.exclusiveSize &&
+        exclusiveReplicatedSize == that.exclusiveReplicatedSize &&
+        Objects.equals(volumeName, that.volumeName) && Objects.equals(bucketName, that.bucketName) &&
+        Objects.equals(name, that.name) && snapshotStatus == that.snapshotStatus &&
+        Objects.equals(snapshotId, that.snapshotId) &&
+        Objects.equals(snapshotPath, that.snapshotPath) &&
+        Objects.equals(checkpointDir, that.checkpointDir);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(volumeName, bucketName, name, creationTime, snapshotStatus, snapshotId, snapshotPath,
+        checkpointDir, referencedSize, referencedReplicatedSize, exclusiveSize, exclusiveReplicatedSize);
+  }
 }

hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneSnapshot.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.client;
+
+import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.when;
+
+import java.util.UUID;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test class for OzoneSnapshot class.
+ */
+public class TestOzoneSnapshot {
+
+  private SnapshotInfo getMockedSnapshotInfo(UUID snapshotId) {
+    SnapshotInfo snapshotInfo = Mockito.mock(SnapshotInfo.class);
+    when(snapshotInfo.getVolumeName()).thenReturn("volume");
+    when(snapshotInfo.getBucketName()).thenReturn("bucket");
+    when(snapshotInfo.getName()).thenReturn("snap");
+    when(snapshotInfo.getCreationTime()).thenReturn(1000L);
+    when(snapshotInfo.getSnapshotStatus()).thenReturn(SNAPSHOT_ACTIVE);
+    when(snapshotInfo.getSnapshotId()).thenReturn(snapshotId);
+    when(snapshotInfo.getSnapshotPath()).thenReturn("volume/bucket");
+    when(snapshotInfo.getCheckpointDir()).thenReturn("checkpointDir");
+    when(snapshotInfo.getReferencedSize()).thenReturn(1000L);
+    when(snapshotInfo.getReferencedReplicatedSize()).thenReturn(3000L);
+    when(snapshotInfo.getExclusiveSize()).thenReturn(4000L);
+    when(snapshotInfo.getExclusiveReplicatedSize()).thenReturn(12000L);
+    when(snapshotInfo.getExclusiveDirSize()).thenReturn(2000L);
+    when(snapshotInfo.getExclusiveDirReplicatedSize()).thenReturn(6000L);
+    return snapshotInfo;
+  }
+
+  @Test
+  public void testOzoneSnapshotFromSnapshotInfo() {
+    UUID snapshotId = UUID.randomUUID();
+    SnapshotInfo snapshotInfo = getMockedSnapshotInfo(snapshotId);
+    OzoneSnapshot ozoneSnapshot = OzoneSnapshot.fromSnapshotInfo(snapshotInfo);
+    OzoneSnapshot expectedOzoneSnapshot = new OzoneSnapshot(
+        "volume", "bucket", "snap", 1000L, SNAPSHOT_ACTIVE, snapshotId,
+        "volume/bucket", "checkpointDir", 1000L, 3000L, 6000L, 18000L);
+    assertEquals(expectedOzoneSnapshot, ozoneSnapshot);
+  }
+}

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java

@@ -82,6 +82,8 @@ public final class SnapshotInfo implements Auditable, CopyObject<SnapshotInfo> {
   private long referencedReplicatedSize;
   private long exclusiveSize;
   private long exclusiveReplicatedSize;
+  private long exclusiveDirSize;
+  private long exclusiveDirReplicatedSize;
   private boolean deepCleanedDeletedDir;
   private ByteString lastTransactionInfo;
 
@@ -104,6 +106,8 @@ private SnapshotInfo(Builder b) {
     this.referencedReplicatedSize = b.referencedReplicatedSize;
     this.exclusiveSize = b.exclusiveSize;
     this.exclusiveReplicatedSize = b.exclusiveReplicatedSize;
+    this.exclusiveDirSize = b.exclusiveDirSize;
+    this.exclusiveDirReplicatedSize = b.exclusiveDirReplicatedSize;
     this.deepCleanedDeletedDir = b.deepCleanedDeletedDir;
     this.lastTransactionInfo = b.lastTransactionInfo;
   }
@@ -233,6 +237,8 @@ public SnapshotInfo.Builder toBuilder() {
         .setReferencedReplicatedSize(referencedReplicatedSize)
         .setExclusiveSize(exclusiveSize)
         .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+        .setExclusiveDirSize(exclusiveDirSize)
+        .setExclusiveDirReplicatedSize(exclusiveDirReplicatedSize)
         .setDeepCleanedDeletedDir(deepCleanedDeletedDir)
         .setLastTransactionInfo(lastTransactionInfo);
   }
@@ -259,6 +265,8 @@ public static class Builder {
     private long referencedReplicatedSize;
     private long exclusiveSize;
     private long exclusiveReplicatedSize;
+    private long exclusiveDirSize;
+    private long exclusiveDirReplicatedSize;
     private boolean deepCleanedDeletedDir;
     private ByteString lastTransactionInfo;
 
@@ -374,6 +382,18 @@ public Builder setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
       return this;
     }
 
+    /** @param exclusiveDirSize - Snapshot exclusive size. */
+    public Builder setExclusiveDirSize(long exclusiveDirSize) {
+      this.exclusiveDirSize = exclusiveDirSize;
+      return this;
+    }
+
+    /** @param exclusiveDirReplicatedSize - Snapshot exclusive size w/ replication. */
+    public Builder setExclusiveDirReplicatedSize(long exclusiveDirReplicatedSize) {
+      this.exclusiveDirReplicatedSize = exclusiveDirReplicatedSize;
+      return this;
+    }
+
     public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) {
       this.deepCleanedDeletedDir = deepCleanedDeletedDir;
       return this;
@@ -408,6 +428,8 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() {
             .setReferencedReplicatedSize(referencedReplicatedSize)
             .setExclusiveSize(exclusiveSize)
             .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+            .setExclusiveDirSize(exclusiveDirSize)
+            .setExclusiveDirReplicatedSize(exclusiveDirReplicatedSize)
             .setDeepCleanedDeletedDir(deepCleanedDeletedDir);
 
     if (pathPreviousSnapshotId != null) {
@@ -485,6 +507,14 @@ public static SnapshotInfo getFromProtobuf(
           snapshotInfoProto.getExclusiveReplicatedSize());
     }
 
+    if (snapshotInfoProto.hasExclusiveDirSize()) {
+      osib.setExclusiveDirSize(snapshotInfoProto.getExclusiveDirSize());
+    }
+
+    if (snapshotInfoProto.hasExclusiveDirReplicatedSize()) {
+      osib.setExclusiveDirReplicatedSize(snapshotInfoProto.getExclusiveDirReplicatedSize());
+    }
+
     if (snapshotInfoProto.hasDeepCleanedDeletedDir()) {
       osib.setDeepCleanedDeletedDir(
           snapshotInfoProto.getDeepCleanedDeletedDir());
@@ -571,10 +601,26 @@ public long getExclusiveSize() {
     return exclusiveSize;
   }
 
+  public void setExclusiveDirSize(long exclusiveDirSize) {
+    this.exclusiveDirSize = exclusiveDirSize;
+  }
+
+  public long getExclusiveDirSize() {
+    return exclusiveDirSize;
+  }
+
   public void setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
     this.exclusiveReplicatedSize = exclusiveReplicatedSize;
   }
 
+  public void setExclusiveDirReplicatedSize(long exclusiveDirReplicatedSize) {
+    this.exclusiveDirReplicatedSize = exclusiveDirReplicatedSize;
+  }
+
+  public long getExclusiveDirReplicatedSize() {
+    return exclusiveDirReplicatedSize;
+  }
+
   public long getExclusiveReplicatedSize() {
     return exclusiveReplicatedSize;
   }
@@ -707,6 +753,8 @@ public String toString() {
         ", referencedReplicatedSize: '" + referencedReplicatedSize + '\'' +
         ", exclusiveSize: '" + exclusiveSize + '\'' +
         ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' +
+        ", exclusiveDirSize: '" + exclusiveDirSize + '\'' +
+        ", exclusiveDirReplicatedSize: '" + exclusiveDirReplicatedSize + '\'' +
         ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' +
         ", lastTransactionInfo: '" + lastTransactionInfo + '\'' +
         '}';

hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java

@@ -67,7 +67,9 @@ private SnapshotInfo createSnapshotInfo() {
         .setReferencedSize(2000L)
         .setReferencedReplicatedSize(6000L)
         .setExclusiveSize(1000L)
+        .setExclusiveDirSize(2000L)
         .setExclusiveReplicatedSize(3000L)
+        .setExclusiveDirReplicatedSize(6000L)
         .setDeepCleanedDeletedDir(false)
         .build();
   }
@@ -92,6 +94,8 @@ private OzoneManagerProtocolProtos.SnapshotInfo createSnapshotInfoProto() {
         .setReferencedReplicatedSize(6000L)
         .setExclusiveSize(1000L)
         .setExclusiveReplicatedSize(3000L)
+        .setExclusiveDirSize(2000L)
+        .setExclusiveDirReplicatedSize(6000L)
         .setDeepCleanedDeletedDir(false)
         .build();
   }
@@ -179,7 +183,9 @@ public void testSnapshotInfoProtoToSnapshotInfo() {
         snapshotInfoActual.getExclusiveReplicatedSize());
     assertEquals(snapshotInfoExpected.getDeepCleanedDeletedDir(),
         snapshotInfoActual.getDeepCleanedDeletedDir());
-
+    assertEquals(snapshotInfoExpected.getExclusiveDirSize(), snapshotInfoActual.getExclusiveDirSize());
+    assertEquals(snapshotInfoExpected.getExclusiveDirReplicatedSize(),
+        snapshotInfoActual.getExclusiveDirReplicatedSize());
     assertEquals(snapshotInfoExpected, snapshotInfoActual);
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java

@@ -234,17 +234,18 @@ public void testExclusiveSizeWithDirectoryDeepClean() throws Exception {
         put("snap2", 5L);
         put("snap3", 0L);
       }};
-    Thread.sleep(500);
     try (TableIterator<String, ? extends Table.KeyValue<String, SnapshotInfo>>
         iterator = snapshotInfoTable.iterator()) {
       while (iterator.hasNext()) {
         Table.KeyValue<String, SnapshotInfo> snapshotEntry = iterator.next();
         String snapshotName = snapshotEntry.getValue().getName();
-        assertEquals(expectedSize.get(snapshotName), snapshotEntry.getValue().
-            getExclusiveSize());
+        SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotEntry.getKey());
+        System.out.println(snapshotInfo.getName() + " " + snapshotInfo.getDeepCleanedDeletedDir());
+        assertEquals(expectedSize.get(snapshotName),
+            snapshotInfo.getExclusiveSize() + snapshotInfo.getExclusiveDirSize());
         // Since for the test we are using RATIS/THREE
         assertEquals(expectedSize.get(snapshotName) * 3,
-            snapshotEntry.getValue().getExclusiveReplicatedSize());
+            snapshotInfo.getExclusiveReplicatedSize() + snapshotInfo.getExclusiveDirReplicatedSize());
 
       }
     }

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -303,6 +303,7 @@ message OMRequest {
   optional GetObjectTaggingRequest          getObjectTaggingRequest        = 140;
   optional PutObjectTaggingRequest          putObjectTaggingRequest        = 141;
   optional DeleteObjectTaggingRequest       deleteObjectTaggingRequest     = 142;
+  repeated SetSnapshotPropertyRequest       SetSnapshotPropertyRequests    = 143;
 }
 
 message OMResponse {
@@ -892,6 +893,9 @@ message SnapshotInfo {
   // note: shared sizes can be calculated from: referenced - exclusive
   optional bool deepCleanedDeletedDir = 19;
   optional bytes lastTransactionInfo = 20;
+  optional uint64 exclusiveDirSize = 21;
+  // snapshot exclusive size after replication
+  optional uint64 exclusiveDirReplicatedSize = 22;
 }
 
 message SnapshotDiffJobProto {
@@ -2029,6 +2033,7 @@ message SetSnapshotPropertyRequest {
   optional SnapshotSize snapshotSize = 3;
   optional bool deepCleanedDeletedDir = 4;
   optional bool deepCleanedDeletedKey = 5;
+  optional SnapshotSize snapshotDirSize = 6;
 }
 
 // SnapshotProperty in entirely deprecated, Keeping it here for proto.lock compatibility