diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 7c3e1b12b330..62d78da8c6a5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -541,29 +541,26 @@ public static File createOMDir(String dirPath) { */ public static RepeatedOmKeyInfo prepareKeyForDelete(long bucketId, OmKeyInfo keyInfo, long trxnLogIndex) { - OmKeyInfo sanitizedKeyInfo = keyInfo; + OmKeyInfo.Builder builder = keyInfo.toBuilder(); // If this key is in a GDPR enforced bucket, then before moving // KeyInfo to deletedTable, remove the GDPR related metadata and // FileEncryptionInfo from KeyInfo. if (Boolean.parseBoolean( keyInfo.getMetadata().get(OzoneConsts.GDPR_FLAG)) ) { - sanitizedKeyInfo = sanitizedKeyInfo.withMetadataMutations(metadata -> { - metadata.remove(OzoneConsts.GDPR_FLAG); - metadata.remove(OzoneConsts.GDPR_ALGORITHM); - metadata.remove(OzoneConsts.GDPR_SECRET); - }); - sanitizedKeyInfo.clearFileEncryptionInfo(); + Map metadata = builder.getMetadata(); + metadata.remove(OzoneConsts.GDPR_FLAG); + metadata.remove(OzoneConsts.GDPR_ALGORITHM); + metadata.remove(OzoneConsts.GDPR_SECRET); + + builder.setFileEncryptionInfo(null); } // Set the updateID - sanitizedKeyInfo.setUpdateID(trxnLogIndex); - if (sanitizedKeyInfo != keyInfo) { - keyInfo.setUpdateID(trxnLogIndex); - } + builder.withUpdateID(trxnLogIndex); //The key doesn't exist in deletedTable, so create a new instance. - return new RepeatedOmKeyInfo(sanitizedKeyInfo, bucketId); + return new RepeatedOmKeyInfo(builder.build(), bucketId); } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java index c8c1a0381397..6695cac61353 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java @@ -509,12 +509,24 @@ public Builder setObjectID(long obId) { return this; } + @Override + public Builder withObjectID(long obId) { + super.withObjectID(obId); + return this; + } + @Override public Builder setUpdateID(long id) { super.setUpdateID(id); return this; } + @Override + public Builder withUpdateID(long newValue) { + super.withUpdateID(newValue); + return this; + } + @Override public Builder addMetadata(String key, String value) { super.addMetadata(key, value); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDirectoryInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDirectoryInfo.java index c0fc58f3537f..9f80f7ce0b34 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDirectoryInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDirectoryInfo.java @@ -153,6 +153,7 @@ public Builder addAllMetadata(Map additionalMetadata) { return this; } + @Override public OmDirectoryInfo build() { return new OmDirectoryInfo(this); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index 232f0c82fe40..3164529d19db 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -624,6 +624,18 @@ public Builder setMetadata(Map map) { return this; } + @Override + public Builder withObjectID(long obId) { + super.withObjectID(obId); + return this; + } + + @Override + public Builder withUpdateID(long newValue) { + super.withUpdateID(newValue); + return this; + } + public Builder setFileEncryptionInfo(FileEncryptionInfo feInfo) { this.encInfo = feInfo; return this; @@ -676,6 +688,12 @@ public Builder setFile(boolean isAFile) { return this; } + public Builder setTags(Map tags) { + this.tags.clear(); + addAllTags(tags); + return this; + } + public Builder addTag(String key, String value) { tags.put(key, value); return this; @@ -691,6 +709,7 @@ public Builder setExpectedDataGeneration(Long existingGeneration) { return this; } + @Override public OmKeyInfo build() { return new OmKeyInfo(this); } @@ -942,6 +961,7 @@ public int hashCode() { /** * Return a new copy of the object. */ + @Override public Builder toBuilder() { return new Builder(this); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java index 7bdf31485cbc..19045aa86d2f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java @@ -222,6 +222,10 @@ public ReplicationConfig getReplicationConfig() { return replicationConfig; } + public Builder toBuilder() { + return new Builder(this); + } + /** * Builder of OmMultipartKeyInfo. */ @@ -236,6 +240,18 @@ public Builder() { this.partKeyInfoList = new TreeMap<>(); } + public Builder(OmMultipartKeyInfo multipartKeyInfo) { + super(multipartKeyInfo); + this.uploadID = multipartKeyInfo.uploadID; + this.creationTime = multipartKeyInfo.creationTime; + this.replicationConfig = multipartKeyInfo.replicationConfig; + this.partKeyInfoList = new TreeMap<>(); + for (PartKeyInfo partKeyInfo : multipartKeyInfo.partKeyInfoMap) { + this.partKeyInfoList.put(partKeyInfo.getPartNumber(), partKeyInfo); + } + this.parentID = multipartKeyInfo.parentID; + } + public Builder setUploadID(String uploadId) { this.uploadID = uploadId; return this; @@ -277,6 +293,12 @@ public Builder setUpdateID(long id) { return this; } + @Override + public Builder withUpdateID(long newValue) { + super.withUpdateID(newValue); + return this; + } + public Builder setParentID(long parentObjId) { this.parentID = parentObjId; return this; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java index bb5086d45216..056abc0e4d5e 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java @@ -225,6 +225,10 @@ public long getUsedNamespace() { return usedNamespace; } + public Builder toBuilder() { + return new Builder(this); + } + /** * Returns new builder class that builds a OmVolumeArgs. * @@ -297,6 +301,18 @@ public Builder setUpdateID(long id) { return this; } + @Override + public Builder withObjectID(long obId) { + super.withObjectID(obId); + return this; + } + + @Override + public Builder withUpdateID(long newValue) { + super.withUpdateID(newValue); + return this; + } + /** * Constructs a builder. */ @@ -310,6 +326,20 @@ private Builder(List acls) { quotaInNamespace = OzoneConsts.QUOTA_RESET; } + private Builder(OmVolumeArgs omVolumeArgs) { + super(omVolumeArgs); + this.acls = omVolumeArgs.acls; + this.adminName = omVolumeArgs.adminName; + this.ownerName = omVolumeArgs.ownerName; + this.volume = omVolumeArgs.volume; + this.creationTime = omVolumeArgs.creationTime; + this.modificationTime = omVolumeArgs.modificationTime; + this.quotaInBytes = omVolumeArgs.quotaInBytes; + this.quotaInNamespace = omVolumeArgs.quotaInNamespace; + this.usedNamespace = omVolumeArgs.usedNamespace; + this.refCount = omVolumeArgs.refCount; + } + public Builder setAdminName(String admin) { this.adminName = admin; return this; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java index 956fa5753a92..b2fcd9152bdd 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java @@ -81,7 +81,7 @@ public Builder setMetadata(Map map) { return this; } - protected Map getMetadata() { + public Map getMetadata() { return metadata; } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java index eae9c3f325aa..81fee5e50f1b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java @@ -19,16 +19,21 @@ import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; +import net.jcip.annotations.Immutable; + /** * Mixin class to handle ObjectID and UpdateID. */ +@Immutable public abstract class WithObjectID extends WithMetadata { - private long objectID; - private long updateID; + private final long objectID; + private final long updateID; protected WithObjectID() { super(); + objectID = 0; + updateID = 0; } protected WithObjectID(Builder b) { @@ -59,65 +64,6 @@ public final long getUpdateID() { return updateID; } - /** - * Set the Object ID. - * There is a reason why we cannot use the final here. The object - * ({@link OmVolumeArgs}/ {@link OmBucketInfo}/ {@link OmKeyInfo}) is - * deserialized from the protobuf in many places in code. We need to set - * this object ID, after it is deserialized. - * - * @param obId - long - */ - public final void setObjectID(long obId) { - if (this.objectID != 0 && obId != OBJECT_ID_RECLAIM_BLOCKS) { - throw new UnsupportedOperationException("Attempt to modify object ID " + - "which is not zero. Current Object ID is " + this.objectID); - } - this.objectID = obId; - } - - /** - * Sets the update ID. For each modification of this object, we will set - * this to a value greater than the current value. - */ - public final void setUpdateID(long newValue) { - - // Because in non-HA, we have multiple rpc handler threads and - // transactionID is generated in OzoneManagerServerSideTranslatorPB. - - // Lets take T1 -> Set Bucket Property - // T2 -> Set Bucket Acl - - // Now T2 got lock first, so updateID will be set to 2. Now when T1 gets - // executed we will hit the precondition exception. So for OM non-HA with - // out ratis we should not have this check. - - // Same can happen after OM restart also. - - // OM Start - // T1 -> Create Bucket - // T2 -> Set Bucket Property - - // OM restart - // T1 -> Set Bucket Acl - - // So when T1 is executing, Bucket will have updateID 2 which is set by T2 - // execution before restart. - - // Main reason, in non-HA transaction Index after restart starts from 0. - // And also because of this same reason we don't do replay checks in non-HA. - - final long currentValue = getUpdateID(); - if (newValue < currentValue) { - throw new IllegalArgumentException(String.format( - "Trying to set updateID to %d which is not greater than the " + - "current value of %d for %s", newValue, currentValue, - getObjectInfo())); - } - - updateID = newValue; - } - /** Hook method, customized in subclasses. */ public String getObjectInfo() { return this.toString(); @@ -148,6 +94,63 @@ public Builder setObjectID(long obId) { return this; } + /** + * Set the Object ID. + * The object ({@link OmVolumeArgs}/ {@link OmBucketInfo}/ {@link OmKeyInfo}) is + * deserialized from the protobuf in many places in code. We need to set + * this object ID, after it is deserialized. + * + * @param obId - long + */ + public Builder withObjectID(long obId) { + if (this.objectID != 0 && obId != OBJECT_ID_RECLAIM_BLOCKS) { + throw new UnsupportedOperationException("Attempt to modify object ID " + + "which is not zero. Current Object ID is " + this.objectID); + } + this.objectID = obId; + return this; + } + + /** + * Sets the update ID. For each modification of this object, we will set + * this to a value greater than the current value. + */ + public Builder withUpdateID(long newValue) { + // Because in non-HA, we have multiple rpc handler threads and + // transactionID is generated in OzoneManagerServerSideTranslatorPB. + + // Lets take T1 -> Set Bucket Property + // T2 -> Set Bucket Acl + + // Now T2 got lock first, so updateID will be set to 2. Now when T1 gets + // executed we will hit the precondition exception. So for OM non-HA with + // out ratis we should not have this check. + + // Same can happen after OM restart also. + + // OM Start + // T1 -> Create Bucket + // T2 -> Set Bucket Property + + // OM restart + // T1 -> Set Bucket Acl + + // So when T1 is executing, Bucket will have updateID 2 which is set by T2 + // execution before restart. + + // Main reason, in non-HA transaction Index after restart starts from 0. + // And also because of this same reason we don't do replay checks in non-HA. + final long currentValue = getUpdateID(); + if (newValue < currentValue) { + throw new IllegalArgumentException(String.format( + "Trying to set updateID to %d which is not greater than the " + + "current value of %d for %s", newValue, currentValue, + getObjectInfo())); + } + this.updateID = newValue; + return this; + } + /** * Sets the update ID for this Object. Update IDs are monotonically * increasing values which are updated each time there is an update. @@ -164,5 +167,10 @@ public long getObjectID() { public long getUpdateID() { return updateID; } + + /** Hook method, customized in subclasses. */ + public String getObjectInfo() { + return this.toString(); + } } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithParentObjectId.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithParentObjectId.java index 06f167bb9f04..0458ebbce543 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithParentObjectId.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithParentObjectId.java @@ -62,6 +62,14 @@ public final void setParentObjectID(long parentObjectID) { this.parentObjectID = parentObjectID; } + public Builder toBuilder() { + return new Builder(this); + } + + public static Builder newBuilder() { + return new Builder(); + } + /** Builder for {@link WithParentObjectId}. */ public static class Builder extends WithObjectID.Builder { private long parentObjectID; @@ -83,5 +91,15 @@ public Builder setParentObjectID(long parentObjectId) { protected long getParentObjectID() { return parentObjectID; } + + @Override + public Builder withObjectID(long obId) { + super.withObjectID(obId); + return this; + } + + public WithParentObjectId build() { + return new WithParentObjectId(this); + } } } diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java index fe046bac0aa3..b838c6e4e456 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java @@ -148,6 +148,12 @@ public Builder setUpdateID(long id) { return this; } + @Override + public Builder withUpdateID(long newValue) { + super.withUpdateID(newValue); + return this; + } + /** * Constructs the OmPrefixInfo. * @return instance of OmPrefixInfo. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 838ee3be86ca..0416924e033f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -253,9 +253,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } // Add objectID and updateID - omBucketInfo.setObjectID( - ozoneManager.getObjectIdFromTxId(transactionLogIndex)); - omBucketInfo.setUpdateID(transactionLogIndex); + omBucketInfo = omBucketInfo.toBuilder() + .withObjectID(ozoneManager.getObjectIdFromTxId(transactionLogIndex)) + .withUpdateID(transactionLogIndex) + .build(); addDefaultAcls(omBucketInfo, omVolumeArgs, ozoneManager); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetOwnerRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetOwnerRequest.java index 3d63a1d4b1ac..75a774a1222d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetOwnerRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetOwnerRequest.java @@ -150,14 +150,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut OmBucketInfo newOmBucketInfo = omBucketInfo.toBuilder() .setOwner(newOwner) .setModificationTime(setBucketPropertyRequest.getModificationTime()) + .withUpdateID(transactionLogIndex) .build(); LOG.debug("Updating bucket owner to {} for bucket: {} in volume: {}", newOwner, bucketName, volumeName); - // Set the updateID to current transaction log index - newOmBucketInfo.setUpdateID(transactionLogIndex); - // Update table cache. omMetadataManager.getBucketTable().addCacheEntry( new CacheKey<>(bucketKey), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAclRequest.java index 565c6e4854d8..a3a324d4ecca 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAclRequest.java @@ -106,7 +106,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } operationResult = omBucketAclOp.test(ozoneAcls, omBucketInfo); - omBucketInfo.setUpdateID(transactionLogIndex); if (operationResult) { // Update the modification time when updating ACLs of Bucket. @@ -122,7 +121,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .getModificationTime(); } omBucketInfo = omBucketInfo.toBuilder() - .setModificationTime(modificationTime).build(); + .withUpdateID(transactionLogIndex) + .setModificationTime(modificationTime) + .build(); // update cache. omMetadataManager.getBucketTable().addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java index c3effc5d291a..c1b342dc4caf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java @@ -230,8 +230,8 @@ private RecoverLeaseResponse doWork(OzoneManager ozoneManager, } openKeyInfo = openKeyInfo.toBuilder() .addMetadata(OzoneConsts.LEASE_RECOVERY, "true") + .withUpdateID(transactionLogIndex) .build(); - openKeyInfo.setUpdateID(transactionLogIndex); openKeyInfo.setModificationTime(Time.now()); // add to cache. omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 7c5660b93806..be8efa51ef94 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -236,7 +236,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut openKeyInfo.setModificationTime(keyArgs.getModificationTime()); // Set the UpdateID to current transactionLogIndex - openKeyInfo.setUpdateID(trxnLogIndex); + openKeyInfo = openKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Add to cache. omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java index 799983cbe441..dd5059be03e7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestWithFSO.java @@ -160,7 +160,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut openKeyInfo.setModificationTime(keyArgs.getModificationTime()); // Set the UpdateID to current transactionLogIndex - openKeyInfo.setUpdateID(trxnLogIndex); + openKeyInfo = openKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Add to cache. addOpenTableCacheEntry(trxnLogIndex, omMetadataManager, openKeyName, keyName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index a106903bce79..c64a85f9187c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -276,9 +276,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut openKeyToDelete = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKeyToDeleteKey); openKeyToDelete = openKeyToDelete.toBuilder() .addMetadata(OzoneConsts.OVERWRITTEN_HSYNC_KEY, "true") + .withUpdateID(trxnLogIndex) .build(); openKeyToDelete.setModificationTime(Time.now()); - openKeyToDelete.setUpdateID(trxnLogIndex); omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( dbOpenKeyToDeleteKey, openKeyToDelete, trxnLogIndex); } @@ -299,21 +299,21 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap); // Optimistic locking validation has passed. Now set the rewrite fields to null so they are // not persisted in the key table. - omKeyInfo.setExpectedDataGeneration(null); - - omKeyInfo = omKeyInfo.withMetadataMutations(metadata -> - metadata.putAll(KeyValueUtil.getFromProtobuf( - commitKeyArgs.getMetadataList()))); - omKeyInfo.setDataSize(commitKeyArgs.getDataSize()); + // Combination + // Set the UpdateID to current transactionLogIndex + omKeyInfo = omKeyInfo.toBuilder() + .setExpectedDataGeneration(null) + .addAllMetadata(KeyValueUtil.getFromProtobuf( + commitKeyArgs.getMetadataList())) + .withUpdateID(trxnLogIndex) + .setDataSize(commitKeyArgs.getDataSize()) + .build(); // Update the block length for each block, return the allocated but // uncommitted blocks List uncommitted = omKeyInfo.updateLocationInfoList(locationInfoList, false); - // Set the UpdateID to current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); - Map oldKeyVersionsToDeleteMap = null; long correctedSpace = omKeyInfo.getReplicatedSize(); // if keyToDelete isn't null, usedNamespace needn't check and diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index ffed1ebf5556..161c5ad7e69a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -204,9 +204,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omMetadataManager, dbOpenKeyToDeleteKey, keyName); openKeyToDelete = openKeyToDelete.toBuilder() .addMetadata(OzoneConsts.OVERWRITTEN_HSYNC_KEY, "true") + .withUpdateID(trxnLogIndex) .build(); openKeyToDelete.setModificationTime(Time.now()); - openKeyToDelete.setUpdateID(trxnLogIndex); OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, dbOpenKeyToDeleteKey, openKeyToDelete, keyName, fileName, trxnLogIndex); } @@ -224,17 +224,18 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } } - omKeyInfo = omKeyInfo.withMetadataMutations(metadata -> - metadata.putAll(KeyValueUtil.getFromProtobuf( - commitKeyArgs.getMetadataList()))); - omKeyInfo.setDataSize(commitKeyArgs.getDataSize()); + // Set the new metadata from the request and UpdateID to current + // transactionLogIndex + omKeyInfo = omKeyInfo.toBuilder() + .addAllMetadata(KeyValueUtil.getFromProtobuf( + commitKeyArgs.getMetadataList())) + .setDataSize(commitKeyArgs.getDataSize()) + .withUpdateID(trxnLogIndex) + .build(); List uncommitted = omKeyInfo.updateLocationInfoList(locationInfoList, false); - // Set the UpdateID to current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); - // If bucket versioning is turned on during the update, between key // creation and key commit, old versions will be just overwritten and // not kept. Bucket versioning will be effective from the first key diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index fba766504e7c..528d1a8922ac 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -147,7 +147,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } // Set the UpdateID to current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Update table cache. Put a tombstone entry omMetadataManager.getKeyTable(getBucketLayout()).addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java index e2541ef051c8..78d4652c95d1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java @@ -125,7 +125,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omKeyInfo.setKeyName(fileName); // Set the UpdateID to current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); final long volumeId = omMetadataManager.getVolumeId(volumeName); final long bucketId = omMetadataManager.getBucketId(volumeName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index f1d71d99fdfe..103b5b811002 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -176,7 +176,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut throw new OMException("Key not found " + fromKey, KEY_NOT_FOUND); } - fromKeyValue.setUpdateID(trxnLogIndex); + fromKeyValue = fromKeyValue.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); fromKeyValue.setKeyName(toKeyName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java index 98c2986b7930..85dff1255147 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java @@ -288,7 +288,9 @@ private OMClientResponse renameKey(OmKeyInfo toKeyParent, String toKeyName, String bucketKey = metadataMgr.getBucketKey( fromKeyValue.getVolumeName(), fromKeyValue.getBucketName()); - fromKeyValue.setUpdateID(trxnLogIndex); + fromKeyValue = fromKeyValue.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Set toFileName fromKeyValue.setKeyName(toKeyFileName); fromKeyValue.setFileName(toKeyFileName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 46cef44e1c57..03af4822627a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -973,31 +973,25 @@ protected OmKeyInfo prepareFileInfo( if (omBucketInfo.getIsVersionEnabled()) { newSize += dbKeyInfo.getDataSize(); } - dbKeyInfo.setDataSize(newSize); // The modification time is set in preExecute. Use the same // modification time. - dbKeyInfo.setModificationTime(keyArgs.getModificationTime()); - dbKeyInfo.setUpdateID(transactionLogIndex); - dbKeyInfo.setReplicationConfig(replicationConfig); - // Construct a new metadata map from KeyArgs by rebuilding via toBuilder. - dbKeyInfo = dbKeyInfo.toBuilder() - .setMetadata(KeyValueUtil.getFromProtobuf( - keyArgs.getMetadataList())) - .build(); - // Construct a new tags from KeyArgs // Clear the old one when the key is overwritten - dbKeyInfo.getTags().clear(); - dbKeyInfo.getTags().putAll(KeyValueUtil.getFromProtobuf( - keyArgs.getTagsList())); + final OmKeyInfo.Builder builder = dbKeyInfo.toBuilder() + .setDataSize(newSize) + .setModificationTime(keyArgs.getModificationTime()) + .setReplicationConfig(replicationConfig) + .setMetadata(KeyValueUtil.getFromProtobuf(keyArgs.getMetadataList())) + .withUpdateID(transactionLogIndex) + .setTags(KeyValueUtil.getFromProtobuf(keyArgs.getTagsList())) + .setFileEncryptionInfo(encInfo); if (keyArgs.hasExpectedDataGeneration()) { - dbKeyInfo.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); + builder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); } - dbKeyInfo.setFileEncryptionInfo(encInfo); - return dbKeyInfo; + return builder.build(); } // the key does not exist, create a new object. @@ -1164,10 +1158,11 @@ protected OmKeyInfo wrapUncommittedBlocksAsPseudoKey( } LOG.debug("Detect allocated but uncommitted blocks {} in key {}.", uncommitted, omKeyInfo.getKeyName()); - OmKeyInfo pseudoKeyInfo = omKeyInfo.copyObject(); + OmKeyInfo pseudoKeyInfo = omKeyInfo.toBuilder() + .withObjectID(OBJECT_ID_RECLAIM_BLOCKS) + .build(); // This is a special marker to indicate that SnapshotDeletingService // can reclaim this key's blocks unconditionally. - pseudoKeyInfo.setObjectID(OBJECT_ID_RECLAIM_BLOCKS); // TODO dataSize of pseudoKey is not real here List uncommittedGroups = new ArrayList<>(); // version not matters in the current logic of keyDeletingService, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequest.java index e137847eb396..ba626b5fdf16 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequest.java @@ -230,7 +230,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut operationResult = true; apply(omKeyInfo); - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder().withUpdateID(trxnLogIndex).build(); // update cache. omMetadataManager.getKeyTable(getBucketLayout()) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java index 6cc68b6f718a..714cb2f5c7ba 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java @@ -100,7 +100,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut boolean isDirectory = keyStatus.isDirectory(); operationResult = true; apply(omKeyInfo); - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder().withUpdateID(trxnLogIndex).build(); // update cache. if (isDirectory) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index f0f447091ec7..d3ce69bca445 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -316,7 +316,9 @@ protected Pair markKeysAsDeletedInCache(OzoneManager ozoneManager new CacheKey<>(omMetadataManager.getOzoneKey(volumeName, bucketName, keyName)), CacheValue.get(trxnLogIndex)); - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); quotaReleased += sumBlockLengths(omKeyInfo); emptyKeys += OmKeyInfo.isKeyEmpty(omKeyInfo) ? 1 : 0; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index ef4d64b27c9b..febdaff5843d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -192,7 +192,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut continue; } - fromKeyValue.setUpdateID(trxnLogIndex); + fromKeyValue = fromKeyValue.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); fromKeyValue.setKeyName(toKeyName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java index 3997f1a67782..5f74da0d7b71 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java @@ -196,7 +196,9 @@ protected void updateOpenKeyTableCache(OzoneManager ozoneManager, } // Set the UpdateID to current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); deletedOpenKeys.put(fullKeyName, Pair.of(bucketId, omKeyInfo)); // Update openKeyTable cache. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java index 5d53ce7db09a..77ad9f452ccc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java @@ -91,36 +91,43 @@ protected OzoneFileStatus getOzoneKeyStatus( protected Pair markKeysAsDeletedInCache( OzoneManager ozoneManager, long trxnLogIndex, List omKeyInfoList, List dirList, OMMetadataManager omMetadataManager, Map openKeyInfoMap) throws IOException { - long quotaReleased = 0; + long quotaReleased = 0L; int emptyKeys = 0; // Mark all keys which can be deleted, in cache as deleted. - for (OmKeyInfo omKeyInfo : omKeyInfoList) { + for (int i = 0; i < omKeyInfoList.size(); i++) { + final OmKeyInfo omKeyInfo = omKeyInfoList.get(i); + final long volumeId = omMetadataManager.getVolumeId( omKeyInfo.getVolumeName()); final long bucketId = omMetadataManager.getBucketId( omKeyInfo.getVolumeName(), omKeyInfo.getBucketName()); final long parentId = omKeyInfo.getParentObjectID(); final String fileName = omKeyInfo.getFileName(); - omMetadataManager.getKeyTable(getBucketLayout()).addCacheEntry( - new CacheKey<>(omMetadataManager - .getOzonePathKey(volumeId, bucketId, parentId, fileName)), - CacheValue.get(trxnLogIndex)); + final String dbKey = omMetadataManager.getOzonePathKey( + volumeId, bucketId, parentId, fileName); + omMetadataManager.getKeyTable(getBucketLayout()) + .addCacheEntry(new CacheKey<>(dbKey), + CacheValue.get(trxnLogIndex)); emptyKeys += OmKeyInfo.isKeyEmpty(omKeyInfo) ? 1 : 0; - omKeyInfo.setUpdateID(trxnLogIndex); - quotaReleased += sumBlockLengths(omKeyInfo); + final OmKeyInfo updatedOmKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); + quotaReleased += sumBlockLengths(updatedOmKeyInfo); + omKeyInfoList.set(i, updatedOmKeyInfo); // If omKeyInfo has hsync metadata, delete its corresponding open key as well - String hsyncClientId = omKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); + final String hsyncClientId = updatedOmKeyInfo.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID); if (hsyncClientId != null) { - Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); - String dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId, parentId, fileName, hsyncClientId); - OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); + final Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + final String dbOpenKey = omMetadataManager.getOpenFileName( + volumeId, bucketId, parentId, fileName, hsyncClientId); + final OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); if (openKeyInfo != null) { - openKeyInfo = openKeyInfo.withMetadataMutations( + final OmKeyInfo updatedOpenKeyInfo = openKeyInfo.withMetadataMutations( metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); - openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); + openKeyTable.addCacheEntry(dbOpenKey, updatedOpenKeyInfo, trxnLogIndex); // Add to the map of open keys to be deleted. - openKeyInfoMap.put(dbOpenKey, openKeyInfo); + openKeyInfoMap.put(dbOpenKey, updatedOpenKeyInfo); } else { LOG.warn("Potentially inconsistent DB state: open key not found with dbOpenKey '{}'", dbOpenKey); } @@ -128,19 +135,26 @@ protected Pair markKeysAsDeletedInCache( } // Mark directory keys. - for (OmKeyInfo omKeyInfo : dirList) { + for (int i = 0; i < dirList.size(); i++) { + final OmKeyInfo dirInfo = dirList.get(i); final long volumeId = omMetadataManager.getVolumeId( - omKeyInfo.getVolumeName()); + dirInfo.getVolumeName()); final long bucketId = omMetadataManager.getBucketId( - omKeyInfo.getVolumeName(), omKeyInfo.getBucketName()); - omMetadataManager.getDirectoryTable().addCacheEntry(new CacheKey<>( - omMetadataManager.getOzonePathKey(volumeId, bucketId, - omKeyInfo.getParentObjectID(), - omKeyInfo.getFileName())), - CacheValue.get(trxnLogIndex)); - - omKeyInfo.setUpdateID(trxnLogIndex); - quotaReleased += sumBlockLengths(omKeyInfo); + dirInfo.getVolumeName(), dirInfo.getBucketName()); + final long parentId = dirInfo.getParentObjectID(); + final String dirName = dirInfo.getFileName(); + + final String dbDirKey = omMetadataManager.getOzonePathKey( + volumeId, bucketId, parentId, dirName); + omMetadataManager.getDirectoryTable() + .addCacheEntry(new CacheKey<>(dbDirKey), + CacheValue.get(trxnLogIndex)); + + final OmKeyInfo updatedDirInfo = dirInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); + quotaReleased += sumBlockLengths(updatedDirInfo); + dirList.set(i, updatedDirInfo); } return Pair.of(quotaReleased, emptyKeys); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequest.java index 67a7f8a626b7..0e1b6834d9f9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequest.java @@ -107,7 +107,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } operationResult = apply(omKeyInfo, trxnLogIndex); - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Update the modification time when updating ACLs of Key. long modificationTime = omKeyInfo.getModificationTime(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java index f32a22b17329..0c7196164e79 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java @@ -106,7 +106,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omKeyInfo.getParentObjectID(), omKeyInfo.getFileName()); boolean isDirectory = keyStatus.isDirectory(); operationResult = apply(omKeyInfo, trxnLogIndex); - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); + // Update the modification time when updating ACLs of Key. long modificationTime = omKeyInfo.getModificationTime(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAclRequest.java index 5e1f3513564a..beebeba93ec8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/prefix/OMPrefixAclRequest.java @@ -90,7 +90,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omPrefixInfo = omMetadataManager.getPrefixTable().get(prefixPath); if (omPrefixInfo != null) { - omPrefixInfo.setUpdateID(trxnLogIndex); + omPrefixInfo = omPrefixInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); } try { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3ExpiredMultipartUploadsAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3ExpiredMultipartUploadsAbortRequest.java index 5f7d01d9a733..745a2523ea5e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3ExpiredMultipartUploadsAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3ExpiredMultipartUploadsAbortRequest.java @@ -236,7 +236,9 @@ private void updateTableCache(OzoneManager ozoneManager, } // Set the UpdateID to current transactionLogIndex - omMultipartKeyInfo.setUpdateID(trxnLogIndex); + omMultipartKeyInfo = omMultipartKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Parse the multipart upload components (e.g. volume, bucket, key) // from the multipartInfoTable db key diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 1e64edfb5be3..8f1e67855822 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -160,7 +160,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut multipartKeyInfo = omMetadataManager.getMultipartInfoTable() .get(multipartKey); - multipartKeyInfo.setUpdateID(trxnLogIndex); + multipartKeyInfo = multipartKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // When abort uploaded key, we need to subtract the PartKey length from // the volume usedBytes. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index d2c3d76e35c7..7735c1fb174a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -161,9 +161,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut openKey + "entry is not found in the openKey table", KEY_NOT_FOUND); } + // Add/Update user defined metadata. + // Set the UpdateID to current transactionLogIndex omKeyInfo = omKeyInfo.toBuilder() .addAllMetadata(KeyValueUtil.getFromProtobuf( keyArgs.getMetadataList())) + .withUpdateID(trxnLogIndex) .build(); // set the data size and location info list @@ -174,8 +177,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut .collect(Collectors.toList()), true); // Set Modification time omKeyInfo.setModificationTime(keyArgs.getModificationTime()); - // Set the UpdateID to current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); int partNumber = keyArgs.getMultipartNumber(); partName = getPartName(ozoneKey, uploadID, partNumber); @@ -206,7 +207,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut multipartKeyInfo.addPartKeyInfo(partKeyInfo.build()); // Set the UpdateID to current transactionLogIndex - multipartKeyInfo.setUpdateID(trxnLogIndex); + multipartKeyInfo = multipartKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // OldPartKeyInfo will be deleted. Its updateID will be set in // S3MultipartUploadCommitPartResponse before being added to diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index d51055fdf2f2..6c644d3f3aba 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -463,6 +463,7 @@ protected OmKeyInfo getOmKeyInfo(long trxnLogIndex, OmKeyInfo omKeyInfo = getOmKeyInfoFromKeyTable(ozoneKey, keyName, omMetadataManager); + OmKeyInfo.Builder builder = null; if (omKeyInfo == null) { // This is a newly added key, it does not have any versions. OmKeyLocationInfoGroup keyLocationInfoGroup = new @@ -473,8 +474,7 @@ protected OmKeyInfo getOmKeyInfo(long trxnLogIndex, keyName, omMetadataManager); // A newly created key, this is the first version. - OmKeyInfo.Builder builder = - new OmKeyInfo.Builder().setVolumeName(volumeName) + builder = new OmKeyInfo.Builder().setVolumeName(volumeName) .setBucketName(bucketName).setKeyName(dbOpenKeyInfo.getKeyName()) .setReplicationConfig(ReplicationConfig.fromProto( partKeyInfo.getType(), partKeyInfo.getFactor(), @@ -498,7 +498,6 @@ protected OmKeyInfo getOmKeyInfo(long trxnLogIndex, builder.setObjectID(dbOpenKeyInfo.getObjectID()); } updatePrefixFSOInfo(dbOpenKeyInfo, builder); - omKeyInfo = builder.build(); } else { OmKeyInfo dbOpenKeyInfo = getOmKeyInfoFromOpenKeyTable(multipartOpenKey, keyName, omMetadataManager); @@ -520,19 +519,16 @@ protected OmKeyInfo getOmKeyInfo(long trxnLogIndex, omKeyInfo.setDataSize(dataSize); omKeyInfo.setReplicationConfig(dbOpenKeyInfo.getReplicationConfig()); final String multipartHash = multipartUploadedKeyHash(partKeyInfoMap); - OmKeyInfo.Builder omKeyInfoBuilder = omKeyInfo.toBuilder(); + builder = omKeyInfo.toBuilder(); if (dbOpenKeyInfo.getMetadata() != null) { - omKeyInfoBuilder.setMetadata(dbOpenKeyInfo.getMetadata()); + builder.setMetadata(dbOpenKeyInfo.getMetadata()); } - omKeyInfo = omKeyInfoBuilder - .addMetadata(OzoneConsts.ETAG, multipartHash) - .build(); + builder.addMetadata(OzoneConsts.ETAG, multipartHash); if (dbOpenKeyInfo.getTags() != null) { - omKeyInfo.setTags(dbOpenKeyInfo.getTags()); + builder.setTags(dbOpenKeyInfo.getTags()); } } - omKeyInfo.setUpdateID(trxnLogIndex); - return omKeyInfo; + return builder.withUpdateID(trxnLogIndex).build(); } protected void updatePrefixFSOInfo(OmKeyInfo dbOpenKeyInfo, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3DeleteObjectTaggingRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3DeleteObjectTaggingRequest.java index 12c4ce13de5c..69656d80b06a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3DeleteObjectTaggingRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3DeleteObjectTaggingRequest.java @@ -129,7 +129,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Clear / delete the tags omKeyInfo.getTags().clear(); // Set the UpdateID to the current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Note: Key modification time is not changed because S3 last modified // time only changes when there are changes in the object content diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3DeleteObjectTaggingRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3DeleteObjectTaggingRequestWithFSO.java index b40e6c1f8a42..23922377c02e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3DeleteObjectTaggingRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3DeleteObjectTaggingRequestWithFSO.java @@ -118,7 +118,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Clear / delete the tags omKeyInfo.getTags().clear(); // Set the UpdateId to the current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Note: Key modification time is not changed because S3 last modified // time only changes when there are changes in the object content diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequest.java index 23d7a40f26de..907e7fac310f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequest.java @@ -131,7 +131,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omKeyInfo.getTags().clear(); omKeyInfo.getTags().putAll(KeyValueUtil.getFromProtobuf(keyArgs.getTagsList())); // Set the UpdateID to the current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Note: Key modification time is not changed because S3 last modified // time only changes when there are changes in the object content diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequestWithFSO.java index 05a45322c599..d1276b2af714 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tagging/S3PutObjectTaggingRequestWithFSO.java @@ -120,7 +120,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omKeyInfo.getTags().clear(); omKeyInfo.getTags().putAll(KeyValueUtil.getFromProtobuf(keyArgs.getTagsList())); // Set the UpdateId to the current transactionLogIndex - omKeyInfo.setUpdateID(trxnLogIndex); + omKeyInfo = omKeyInfo.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // Note: Key modification time is not changed because S3 last modified // time only changes when there are changes in the object content diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tenant/OMTenantCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tenant/OMTenantCreateRequest.java index 3732f074bb0c..e2fcb4f57c6f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tenant/OMTenantCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/tenant/OMTenantCreateRequest.java @@ -278,9 +278,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omVolumeArgs = OmVolumeArgs.getFromProtobuf(volumeInfo); omVolumeArgs.setQuotaInBytes(OzoneConsts.QUOTA_RESET); omVolumeArgs.setQuotaInNamespace(OzoneConsts.QUOTA_RESET); - omVolumeArgs.setObjectID( - ozoneManager.getObjectIdFromTxId(transactionLogIndex)); - omVolumeArgs.setUpdateID(transactionLogIndex); + omVolumeArgs = omVolumeArgs.toBuilder() + .withObjectID(ozoneManager.getObjectIdFromTxId(transactionLogIndex)) + .withUpdateID(transactionLogIndex) + .build(); omVolumeArgs.incRefCount(); // Remove this check when vol ref count is also used by other features diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java index 1397f8a1b9f3..e91757ee3a23 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java @@ -136,10 +136,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // when you create a volume, we set both Object ID and update ID. // The Object ID will never change, but update // ID will be set to transactionID each time we update the object. - omVolumeArgs.setObjectID( - ozoneManager.getObjectIdFromTxId(transactionLogIndex)); - omVolumeArgs.setUpdateID(transactionLogIndex); - + omVolumeArgs = omVolumeArgs.toBuilder() + .withObjectID(ozoneManager.getObjectIdFromTxId(transactionLogIndex)) + .withUpdateID(transactionLogIndex) + .build(); auditMap = omVolumeArgs.toAuditMap(); // acquire lock. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java index f9a6fa303590..a5cdcb675b59 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetOwnerRequest.java @@ -152,7 +152,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // Set owner with new owner name. omVolumeArgs.setOwnerName(newOwner); - omVolumeArgs.setUpdateID(transactionLogIndex); + omVolumeArgs = omVolumeArgs.toBuilder() + .withUpdateID(transactionLogIndex) + .build(); // Update modificationTime. omVolumeArgs.setModificationTime( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java index b4279eac2d4d..f4171b757c3f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeSetQuotaRequest.java @@ -136,7 +136,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omVolumeArgs.setQuotaInNamespace(omVolumeArgs.getQuotaInNamespace()); } - omVolumeArgs.setUpdateID(transactionLogIndex); + omVolumeArgs = omVolumeArgs.toBuilder() + .withUpdateID(transactionLogIndex) + .build(); omVolumeArgs.setModificationTime( setVolumePropertyRequest.getModificationTime()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java index 88d786cdd204..e12532c7a8ae 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAclRequest.java @@ -108,7 +108,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } omVolumeArgs.setModificationTime(modificationTime); - omVolumeArgs.setUpdateID(trxnLogIndex); + omVolumeArgs = omVolumeArgs.toBuilder() + .withUpdateID(trxnLogIndex) + .build(); // update cache. omMetadataManager.getVolumeTable().addCacheEntry( diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/om/PrefixParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/om/PrefixParser.java index 6bc4e35ae5fe..a55cc798f5f4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/om/PrefixParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/om/PrefixParser.java @@ -134,8 +134,9 @@ public void parse(String vol, String buck, String db, final long volumeObjectId = metadataManager.getVolumeId( info.getVolumeName()); long lastObjectId = info.getObjectID(); - WithParentObjectId objectBucketId = new WithParentObjectId(); - objectBucketId.setObjectID(lastObjectId); + WithParentObjectId objectBucketId = WithParentObjectId.newBuilder() + .withObjectID(lastObjectId) + .build(); dumpInfo(Types.BUCKET, effectivePath, objectBucketId, metadataManager.getBucketKey(vol, buck));