diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index f3c08b252b1f..e29aeac1e12a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -343,6 +343,7 @@ private OzoneConsts() { public static final String BUCKET_LAYOUT = "bucketLayout"; public static final String TENANT = "tenant"; public static final String USER_PREFIX = "userPrefix"; + public static final String REWRITE_GENERATION = "rewriteGeneration"; // For multi-tenancy public static final String TENANT_ID_USERNAME_DELIMITER = "$"; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 8d153a948c4e..012f029f51cc 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -470,6 +470,28 @@ public OzoneOutputStream createKey(String key, long size, .createKey(volumeName, name, key, size, replicationConfig, keyMetadata); } + /** + * This API allows to atomically update an existing key. The key read before invoking this API + * should remain unchanged for this key to be written. This is controlled by the generation + * field in the existing Key param. If the key is replaced or updated the generation will change. If the + * generation has changed since the existing Key was read, either the initial key create will fail, + * or the key will fail to commit after the data has been written as the checks are carried out + * both at key open and commit time. + * + * @param keyName Existing key to rewrite. This must exist in the bucket. + * @param size The size of the new key + * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create + * and commit time. + * @param replicationConfig The replication configuration for the key to be rewritten. + * @param metadata custom key value metadata + * @return OzoneOutputStream to which the data has to be written. + * @throws IOException + */ + public OzoneOutputStream rewriteKey(String keyName, long size, long existingKeyGeneration, + ReplicationConfig replicationConfig, Map metadata) throws IOException { + return proxy.rewriteKey(volumeName, name, keyName, size, existingKeyGeneration, replicationConfig, metadata); + } + /** * Creates a new key in the bucket, with default replication type RATIS and * with replication factor THREE. @@ -1784,8 +1806,7 @@ private void addKeyPrefixInfoToResultList(String keyPrefix, keyInfo.getDataSize(), keyInfo.getCreationTime(), keyInfo.getModificationTime(), keyInfo.getReplicationConfig(), - keyInfo.isFile(), - keyInfo.getOwnerName()); + keyInfo.isFile(), keyInfo.getOwnerName()); keysResultList.add(ozoneKey); } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 3663f6f65466..2d32a7272061 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -70,10 +70,6 @@ public class OzoneKey { */ private final boolean isFile; - /** - * Constructs OzoneKey from OmKeyInfo. - * - */ @SuppressWarnings("parameternumber") public OzoneKey(String volumeName, String bucketName, String keyName, long size, long creationTime, @@ -201,6 +197,10 @@ public boolean isFile() { return isFile; } + /** + * Constructs OzoneKey from OmKeyInfo. + * + */ public static OzoneKey fromKeyInfo(OmKeyInfo keyInfo) { return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 6b44fa1dca74..cd2978fce189 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -42,6 +42,12 @@ public class OzoneKeyDetails extends OzoneKey { private final CheckedSupplier contentSupplier; + /** + * The generation of an existing key. This can be used with atomic commits, to + * ensure the key has not changed since the key details were read. + */ + private final Long generation; + /** * Constructs OzoneKeyDetails from OmKeyInfo. */ @@ -53,12 +59,30 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, Map metadata, FileEncryptionInfo feInfo, CheckedSupplier contentSupplier, - boolean isFile, String owner) { + boolean isFile, String owner, Long generation) { super(volumeName, bucketName, keyName, size, creationTime, modificationTime, replicationConfig, metadata, isFile, owner); this.ozoneKeyLocations = ozoneKeyLocations; this.feInfo = feInfo; this.contentSupplier = contentSupplier; + this.generation = generation; + } + + /** + * Constructs OzoneKeyDetails from OmKeyInfo. + */ + @SuppressWarnings("parameternumber") + public OzoneKeyDetails(String volumeName, String bucketName, String keyName, + long size, long creationTime, long modificationTime, + List ozoneKeyLocations, + ReplicationConfig replicationConfig, + Map metadata, + FileEncryptionInfo feInfo, + CheckedSupplier contentSupplier, + boolean isFile, String owner) { + this(volumeName, bucketName, keyName, size, creationTime, + modificationTime, ozoneKeyLocations, replicationConfig, metadata, feInfo, contentSupplier, + isFile, owner, null); } /** @@ -72,6 +96,10 @@ public FileEncryptionInfo getFileEncryptionInfo() { return feInfo; } + public Long getGeneration() { + return generation; + } + /** * Get OzoneInputStream to read the content of the key. * @return OzoneInputStream diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 912a3138c478..0725b4f25349 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -353,6 +353,29 @@ OzoneOutputStream createKey(String volumeName, String bucketName, Map metadata) throws IOException; + /** + * This API allows to atomically update an existing key. The key read before invoking this API + * should remain unchanged for this key to be written. This is controlled by the generation + * field in the existing Key param. If the key is replaced or updated the generation will change. If the + * generation has changed since the existing Key was read, either the initial key create will fail, + * or the key will fail to commit after the data has been written as the checks are carried out + * both at key open and commit time. + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Existing key to rewrite. This must exist in the bucket. + * @param size The size of the new key + * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create + * and commit time. + * @param replicationConfig The replication configuration for the key to be rewritten. + * @param metadata custom key value metadata + * @return {@link OzoneOutputStream} + * @throws IOException + */ + OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyName, + long size, long existingKeyGeneration, ReplicationConfig replicationConfig, + Map metadata) throws IOException; + /** * Writes a key in an existing bucket. * @param volumeName Name of the Volume diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 2f97f2f3ccc6..000a19415010 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1387,26 +1387,7 @@ public OzoneOutputStream createKey( ReplicationConfig replicationConfig, Map metadata) throws IOException { - verifyVolumeName(volumeName); - verifyBucketName(bucketName); - if (checkKeyNameEnabled) { - HddsClientUtils.verifyKeyName(keyName); - } - HddsClientUtils.checkNotNull(keyName); - if (omVersion - .compareTo(OzoneManagerVersion.ERASURE_CODED_STORAGE_SUPPORT) < 0) { - if (replicationConfig != null && - replicationConfig.getReplicationType() - == HddsProtos.ReplicationType.EC) { - throw new IOException("Can not set the replication of the key to" - + " Erasure Coded replication, as OzoneManager does not support" - + " Erasure Coded replication."); - } - } - - if (replicationConfig != null) { - replicationConfigValidator.validate(replicationConfig); - } + createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); String ownerName = getRealUserInfo().getShortUserName(); OmKeyArgs.Builder builder = new OmKeyArgs.Builder() @@ -1431,6 +1412,59 @@ public OzoneOutputStream createKey( return createOutputStream(openKey); } + @Override + public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyName, + long size, long existingKeyGeneration, ReplicationConfig replicationConfig, + Map metadata) throws IOException { + createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); + String ownerName = getRealUserInfo().getShortUserName(); + + OmKeyArgs.Builder builder = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(size) + .setReplicationConfig(replicationConfig) + .addAllMetadataGdpr(metadata) + .setLatestVersionLocation(getLatestVersionLocation) + .setOwnerName(ownerName) + .setRewriteGeneration(existingKeyGeneration); + + OpenKeySession openKey = ozoneManagerClient.openKey(builder.build()); + // For bucket with layout OBJECT_STORE, when create an empty file (size=0), + // OM will set DataSize to OzoneConfigKeys#OZONE_SCM_BLOCK_SIZE, + // which will cause S3G's atomic write length check to fail, + // so reset size to 0 here. + if (isS3GRequest.get() && size == 0) { + openKey.getKeyInfo().setDataSize(0); + } + return createOutputStream(openKey); + } + + private void createKeyPreChecks(String volumeName, String bucketName, String keyName, + ReplicationConfig replicationConfig) throws IOException { + verifyVolumeName(volumeName); + verifyBucketName(bucketName); + if (checkKeyNameEnabled) { + HddsClientUtils.verifyKeyName(keyName); + } + HddsClientUtils.checkNotNull(keyName); + if (omVersion + .compareTo(OzoneManagerVersion.ERASURE_CODED_STORAGE_SUPPORT) < 0) { + if (replicationConfig != null && + replicationConfig.getReplicationType() + == HddsProtos.ReplicationType.EC) { + throw new IOException("Can not set the replication of the key to" + + " Erasure Coded replication, as OzoneManager does not support" + + " Erasure Coded replication."); + } + } + + if (replicationConfig != null) { + replicationConfigValidator.validate(replicationConfig); + } + } + @Override public OzoneDataStreamOutput createStreamKey( String volumeName, String bucketName, String keyName, long size, @@ -1722,7 +1756,8 @@ private OzoneKeyDetails getOzoneKeyDetails(OmKeyInfo keyInfo) { keyInfo.getModificationTime(), ozoneKeyLocations, keyInfo.getReplicationConfig(), keyInfo.getMetadata(), keyInfo.getFileEncryptionInfo(), - () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile(), keyInfo.getOwnerName()); + () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile(), keyInfo.getOwnerName(), + keyInfo.getGeneration()); } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java index e8ad2564f392..35417c0b3269 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java @@ -53,6 +53,13 @@ public final class OmKeyArgs implements Auditable { private final boolean recursive; private final boolean headOp; private final boolean forceUpdateContainerCacheFromSCM; + // RewriteGeneration, when used in key creation indicates that a + // key with the same keyName should exist with the given generation. + // For a key commit to succeed, the original key should still be present with the + // generation unchanged. + // This allows a key to be created an committed atomically if the original has not + // been modified. + private Long rewriteGeneration = null; private OmKeyArgs(Builder b) { this.volumeName = b.volumeName; @@ -72,6 +79,7 @@ private OmKeyArgs(Builder b) { this.headOp = b.headOp; this.forceUpdateContainerCacheFromSCM = b.forceUpdateContainerCacheFromSCM; this.ownerName = b.ownerName; + this.rewriteGeneration = b.rewriteGeneration; } public boolean getIsMultipartKey() { @@ -150,6 +158,10 @@ public boolean isForceUpdateContainerCacheFromSCM() { return forceUpdateContainerCacheFromSCM; } + public Long getRewriteGeneration() { + return rewriteGeneration; + } + @Override public Map toAuditMap() { Map auditMap = new LinkedHashMap<>(); @@ -173,7 +185,7 @@ public void addLocationInfo(OmKeyLocationInfo locationInfo) { } public OmKeyArgs.Builder toBuilder() { - return new OmKeyArgs.Builder() + OmKeyArgs.Builder builder = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(keyName) @@ -190,11 +202,16 @@ public OmKeyArgs.Builder toBuilder() { .setLatestVersionLocation(latestVersionLocation) .setAcls(acls) .setForceUpdateContainerCacheFromSCM(forceUpdateContainerCacheFromSCM); + + if (rewriteGeneration != null) { + builder.setRewriteGeneration(rewriteGeneration); + } + return builder; } @Nonnull public KeyArgs toProtobuf() { - return KeyArgs.newBuilder() + KeyArgs.Builder builder = KeyArgs.newBuilder() .setVolumeName(getVolumeName()) .setBucketName(getBucketName()) .setKeyName(getKeyName()) @@ -203,8 +220,11 @@ public KeyArgs toProtobuf() { .setLatestVersionLocation(getLatestVersionLocation()) .setHeadOp(isHeadOp()) .setForceUpdateContainerCacheFromSCM( - isForceUpdateContainerCacheFromSCM()) - .build(); + isForceUpdateContainerCacheFromSCM()); + if (rewriteGeneration != null) { + builder.setRewriteGeneration(rewriteGeneration); + } + return builder.build(); } /** @@ -228,6 +248,7 @@ public static class Builder { private boolean recursive; private boolean headOp; private boolean forceUpdateContainerCacheFromSCM; + private Long rewriteGeneration = null; public Builder setVolumeName(String volume) { this.volumeName = volume; @@ -327,6 +348,11 @@ public Builder setForceUpdateContainerCacheFromSCM(boolean value) { return this; } + public Builder setRewriteGeneration(long generation) { + this.rewriteGeneration = generation; + return this; + } + public OmKeyArgs build() { return new OmKeyArgs(this); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index bf31be67c59c..84a75d09e5b0 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -102,6 +102,14 @@ public static Codec getCodec(boolean ignorePipeline) { */ private final CopyOnWriteArrayList acls; + // rewriteGeneration, when used in key creation indicates that a + // key with the same keyName should exist with the given generation. + // For a key commit to succeed, the original key should still be present with the + // generation unchanged. + // This allows a key to be created an committed atomically if the original has not + // been modified. + private Long rewriteGeneration = null; + private OmKeyInfo(Builder b) { super(b); this.volumeName = b.volumeName; @@ -118,6 +126,7 @@ private OmKeyInfo(Builder b) { this.fileName = b.fileName; this.isFile = b.isFile; this.ownerName = b.ownerName; + this.rewriteGeneration = b.rewriteGeneration; } public String getVolumeName() { @@ -160,10 +169,26 @@ public String getFileName() { return fileName; } + public void setRewriteGeneration(Long generation) { + this.rewriteGeneration = generation; + } + + public Long getRewriteGeneration() { + return rewriteGeneration; + } + public String getOwnerName() { return ownerName; } + /** + * Returns the generation of the object. Note this is currently the same as updateID for a key. + * @return long + */ + public long getGeneration() { + return getUpdateID(); + } + public synchronized OmKeyLocationInfoGroup getLatestVersionLocations() { return keyLocationVersions.size() == 0 ? null : keyLocationVersions.get(keyLocationVersions.size() - 1); @@ -435,6 +460,7 @@ public static class Builder extends WithParentObjectId.Builder { private FileChecksum fileChecksum; private boolean isFile; + private Long rewriteGeneration = null; public Builder() { } @@ -563,6 +589,11 @@ public Builder setFile(boolean isAFile) { return this; } + public Builder setRewriteGeneration(Long existingGeneration) { + this.rewriteGeneration = existingGeneration; + return this; + } + public OmKeyInfo build() { return new OmKeyInfo(this); } @@ -667,6 +698,9 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, kb.setFileEncryptionInfo(OMPBHelper.convert(encInfo)); } kb.setIsFile(isFile); + if (rewriteGeneration != null) { + kb.setRewriteGeneration(rewriteGeneration); + } if (ownerName != null) { kb.setOwnerName(ownerName); } @@ -716,6 +750,9 @@ public static OmKeyInfo getFromProtobuf(KeyInfo keyInfo) throws IOException { if (keyInfo.hasIsFile()) { builder.setFile(keyInfo.getIsFile()); } + if (keyInfo.hasRewriteGeneration()) { + builder.setRewriteGeneration(keyInfo.getRewriteGeneration()); + } if (keyInfo.hasOwnerName()) { builder.setOwnerName(keyInfo.getOwnerName()); @@ -830,6 +867,9 @@ public OmKeyInfo copyObject() { if (fileChecksum != null) { builder.setFileChecksum(fileChecksum); } + if (rewriteGeneration != null) { + builder.setRewriteGeneration(rewriteGeneration); + } return builder.build(); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 61dd3f56602e..a62898411d16 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -726,6 +726,10 @@ public OpenKeySession openKey(OmKeyArgs args) throws IOException { keyArgs.setSortDatanodes(args.getSortDatanodes()); + if (args.getRewriteGeneration() != null) { + keyArgs.setRewriteGeneration(args.getRewriteGeneration()); + } + req.setKeyArgs(keyArgs.build()); OMRequest omRequest = createOMRequest(Type.CreateKey) diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java index 4aead0cd8bcb..852d49871c49 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java @@ -67,6 +67,7 @@ public void protobufConversion() throws IOException { assertFalse(key.isHsync()); key.getMetadata().put(OzoneConsts.HSYNC_CLIENT_ID, "clientid"); assertTrue(key.isHsync()); + assertEquals(5678L, key.getRewriteGeneration()); } @Test @@ -123,6 +124,7 @@ private OmKeyInfo createOmKeyInfo(ReplicationConfig replicationConfig) { .setReplicationConfig(replicationConfig) .addMetadata("key1", "value1") .addMetadata("key2", "value2") + .setRewriteGeneration(5678L) .build(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 172867903c7a..e3b0c4a7a717 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -1097,6 +1097,56 @@ public void testPutKey() throws IOException { } } + @Test + public void testRewriteKey() throws IOException { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + String value = "sample value"; + String rewriteValue = "rewrite value"; + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + + // TODO - only works on object store layout for now. + BucketArgs args = BucketArgs.newBuilder() + .setBucketLayout(BucketLayout.OBJECT_STORE) + .build(); + + volume.createBucket(bucketName, args); + OzoneBucket bucket = volume.getBucket(bucketName); + + String keyName = UUID.randomUUID().toString(); + try (OzoneOutputStream out = bucket.createKey(keyName, value.getBytes(UTF_8).length, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), new HashMap<>())) { + out.write(value.getBytes(UTF_8)); + } + OzoneKeyDetails keyDetails = bucket.getKey(keyName); + + try (OzoneOutputStream out = bucket.rewriteKey(keyDetails.getName(), keyDetails.getDataSize(), + keyDetails.getGeneration(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + keyDetails.getMetadata())) { + out.write(rewriteValue.getBytes(UTF_8)); + } + + try (OzoneInputStream is = bucket.readKey(keyName)) { + byte[] fileContent = new byte[rewriteValue.getBytes(UTF_8).length]; + is.read(fileContent); + assertEquals(rewriteValue, new String(fileContent, UTF_8)); + } + + // Delete the key + bucket.deleteKey(keyName); + + // Now try the rewrite again, and it should fail as the originally read key is no longer there. + assertThrows(IOException.class, () -> { + try (OzoneOutputStream out = bucket.rewriteKey(keyDetails.getName(), keyDetails.getDataSize(), + keyDetails.getGeneration(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE), + keyDetails.getMetadata())) { + out.write(rewriteValue.getBytes(UTF_8)); + } + }); + } + @Test public void testCheckUsedBytesQuota() throws IOException { String volumeName = UUID.randomUUID().toString(); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 4a18f308c90f..75a2562ae715 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1026,6 +1026,13 @@ message KeyArgs { // Force OM to update container cache location from SCL optional bool forceUpdateContainerCacheFromSCM = 20; optional string ownerName = 21; + // rewriteGeneration, when used in key creation indicates that a + // key with the same keyName should exist with the given generation. + // For a key commit to succeed, the original key should still be present with the + // generation unchanged. + // This allows a key to be created an committed atomically if the original has not + // been modified. + optional uint64 rewriteGeneration = 22; } message KeyLocation { @@ -1109,6 +1116,13 @@ message KeyInfo { optional FileChecksumProto fileChecksum = 18; optional bool isFile = 19; optional string ownerName = 20; + // rewriteGeneration, when used in key creation indicates that a + // key with the same keyName should exist with the given generation. + // For a key commit to succeed, the original key should still be present with the + // generation unchanged. + // This allows a key to be created an committed atomically if the original has not + // been modified. + optional uint64 rewriteGeneration = 21; } message BasicKeyInfo { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/RequestAuditor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/RequestAuditor.java index 78e67bb8ed5c..910bb2f4316b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/RequestAuditor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/RequestAuditor.java @@ -82,6 +82,10 @@ default Map buildKeyArgsAuditMap(KeyArgs keyArgs) { auditMap.put(OzoneConsts.REPLICATION_CONFIG, ECReplicationConfig.toString(keyArgs.getEcReplicationConfig())); } + if (keyArgs.hasRewriteGeneration()) { + auditMap.put(OzoneConsts.REWRITE_GENERATION, + String.valueOf(keyArgs.getRewriteGeneration())); + } for (HddsProtos.KeyValue item : keyArgs.getMetadataList()) { if (ETAG.equals(item.getKey())) { auditMap.put(ETAG, item.getValue()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 5c3593b07839..48effbee4c2b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -240,6 +240,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn throw new OMException("Failed to " + action + " key, as " + dbOpenKey + " entry is not found in the OpenKey table", KEY_NOT_FOUND); } + + validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap); + // Optimistic locking validation has passed. Now set the rewrite fields to null so they are + // not persisted in the key table. + omKeyInfo.setRewriteGeneration(null); + omKeyInfo.getMetadata().putAll(KeyValueUtil.getFromProtobuf( commitKeyArgs.getMetadataList())); if (isHSync) { @@ -497,4 +503,22 @@ public static OMRequest disallowHsync( } return req; } + + private void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map auditMap) + throws OMException { + if (toCommit.getRewriteGeneration() != null) { + // These values are not passed in the request keyArgs, so add them into the auditMap if they are present + // in the open key entry. + auditMap.put(OzoneConsts.REWRITE_GENERATION, String.valueOf(toCommit.getRewriteGeneration())); + if (existing == null) { + throw new OMException("Atomic rewrite is not allowed for a new key", KEY_NOT_FOUND); + } + if (!toCommit.getRewriteGeneration().equals(existing.getUpdateID())) { + throw new OMException("Cannot commit as current generation (" + existing.getUpdateID() + + ") does not match with the rewrite generation (" + toCommit.getRewriteGeneration() + ")", + KEY_NOT_FOUND); + } + } + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index e9a9f007197a..4ede37e6b434 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -231,6 +231,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn keyName); OmKeyInfo dbKeyInfo = omMetadataManager.getKeyTable(getBucketLayout()) .getIfExist(dbKeyName); + validateAtomicRewrite(dbKeyInfo, keyArgs); OmBucketInfo bucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); @@ -440,4 +441,17 @@ public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( } return req; } + + private void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs) + throws OMException { + if (keyArgs.hasRewriteGeneration()) { + // If a key does not exist, or if it exists but the updateID do not match, then fail this request. + if (dbKeyInfo == null) { + throw new OMException("Key not found during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + } + if (dbKeyInfo.getUpdateID() != keyArgs.getRewriteGeneration()) { + throw new OMException("Generation mismatch during expected rewrite", OMException.ResultCodes.KEY_NOT_FOUND); + } + } + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 1addd2431bb7..a29e63466e08 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -775,11 +775,13 @@ protected OmKeyInfo prepareFileInfo( dbKeyInfo.setReplicationConfig(replicationConfig); // Construct a new metadata map from KeyArgs. - // Clear the old one when the key is overwritten. dbKeyInfo.getMetadata().clear(); dbKeyInfo.getMetadata().putAll(KeyValueUtil.getFromProtobuf( keyArgs.getMetadataList())); + if (keyArgs.hasRewriteGeneration()) { + dbKeyInfo.setRewriteGeneration(keyArgs.getRewriteGeneration()); + } dbKeyInfo.setFileEncryptionInfo(encInfo); return dbKeyInfo; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java index 9719865db196..2fc38cbd3aa9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; @@ -31,11 +32,13 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; @@ -58,9 +61,12 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .OMRequest; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -116,7 +122,7 @@ public void testValidateAndUpdateCacheWithUnknownBlockId() throws Exception { OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(OzoneManagerProtocolProtos.Status.OK, + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); // Entry should be deleted from openKey Table. @@ -183,7 +189,7 @@ public void testValidateAndUpdateCache() throws Exception { OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(OzoneManagerProtocolProtos.Status.OK, + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); // Entry should be deleted from openKey Table. @@ -218,6 +224,75 @@ public void testValidateAndUpdateCache() throws Exception { omKeyInfo.getLatestVersionLocations().getLocationList()); } + @Test + public void testAtomicRewrite() throws Exception { + if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + // TODO - does not work with in FSO for now + return; + } + + Table openKeyTable = omMetadataManager.getOpenKeyTable(getBucketLayout()); + Table closedKeyTable = omMetadataManager.getKeyTable(getBucketLayout()); + + OMRequest modifiedOmRequest = doPreExecute(createCommitKeyRequest()); + OMKeyCommitRequest omKeyCommitRequest = getOmKeyCommitRequest(modifiedOmRequest); + KeyArgs keyArgs = modifiedOmRequest.getCommitKeyRequest().getKeyArgs(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, omKeyCommitRequest.getBucketLayout()); + + // Append new blocks + List allocatedLocationList = + keyArgs.getKeyLocationsList().stream() + .map(OmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()); + + OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, keyName, replicationConfig, new OmKeyLocationInfoGroup(version, new ArrayList<>())); + omKeyInfoBuilder.setRewriteGeneration(1L); + OmKeyInfo omKeyInfo = omKeyInfoBuilder.build(); + omKeyInfo.appendNewBlocks(allocatedLocationList, false); + List acls = Collections.singletonList(OzoneAcl.parseAcl("user:foo:rw")); + omKeyInfo.addAcl(acls.get(0)); + + String openKey = getOzonePathKey() + "/" + modifiedOmRequest.getCommitKeyRequest().getClientID(); + + openKeyTable.put(openKey, omKeyInfo); + OmKeyInfo openKeyInfo = openKeyTable.get(openKey); + assertNotNull(openKeyInfo); + assertEquals(acls, openKeyInfo.getAcls()); + // At this stage, we have an openKey, with rewrite generation of 1. + // However there is no closed key entry, so the commit should fail. + OMClientResponse omClientResponse = + omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); + + // Now add the key to the key table, and try again, but with different generation + omKeyInfoBuilder.setRewriteGeneration(null); + omKeyInfoBuilder.setUpdateID(0L); + OmKeyInfo invalidKeyInfo = omKeyInfoBuilder.build(); + closedKeyTable.put(getOzonePathKey(), invalidKeyInfo); + // This should fail as the updateID ia zero and the open key has rewrite generation of 1. + omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(KEY_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); + + omKeyInfoBuilder.setUpdateID(1L); + OmKeyInfo closedKeyInfo = omKeyInfoBuilder.build(); + + closedKeyTable.delete(getOzonePathKey()); + closedKeyTable.put(getOzonePathKey(), closedKeyInfo); + + // Now the key should commit as the updateID and rewrite Generation match. + omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); + + OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey()); + assertNull(committedKey.getRewriteGeneration()); + // Generation should be changed + assertNotEquals(closedKeyInfo.getGeneration(), committedKey.getGeneration()); + assertEquals(acls, committedKey.getAcls()); + } + @Test public void testValidateAndUpdateCacheWithUncommittedBlocks() throws Exception { @@ -260,7 +335,7 @@ public void testValidateAndUpdateCacheWithUncommittedBlocks() OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(OzoneManagerProtocolProtos.Status.OK, + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); Map toDeleteKeyList @@ -385,7 +460,7 @@ private Map doKeyCommit(boolean isHSync, OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(OzoneManagerProtocolProtos.Status.OK, + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); // Key should be present in both OpenKeyTable and KeyTable with HSync commit @@ -550,7 +625,7 @@ public void testValidateAndUpdateCacheWithKeyNotFound() throws Exception { OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L); - assertEquals(OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND, + assertEquals(KEY_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); omKeyInfo = @@ -596,7 +671,7 @@ public void testValidateAndUpdateCacheOnOverwrite() throws Exception { OMClientResponse omClientResponse = omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 102L); - assertEquals(OzoneManagerProtocolProtos.Status.OK, omClientResponse.getOMResponse().getStatus()); + assertEquals(OK, omClientResponse.getOMResponse().getStatus()); // New entry should be created in key Table. omKeyInfo = omMetadataManager.getKeyTable(omKeyCommitRequest.getBucketLayout()).get(ozoneKey); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 0790e2af3b67..4c7b2aa047da 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -70,6 +70,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.addVolumeAndBucketToDB; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.createOmKeyInfo; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.NOT_A_FILE; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.assertj.core.api.Assertions.assertThat; @@ -120,9 +121,9 @@ private void preExecuteTest(boolean isMultipartKey, int partNumber, long scmBlockSize = ozoneManager.getScmBlockSize(); for (int i = 0; i <= repConfig.getRequiredNodes(); i++) { doPreExecute(createKeyRequest(isMultipartKey, partNumber, - scmBlockSize * i, repConfig)); + scmBlockSize * i, repConfig, null)); doPreExecute(createKeyRequest(isMultipartKey, partNumber, - scmBlockSize * i + 1, repConfig)); + scmBlockSize * i + 1, repConfig, null)); } } @@ -482,7 +483,7 @@ public void testOverwritingExistingMetadata( Map initialMetadata = Collections.singletonMap("initialKey", "initialValue"); OMRequest initialRequest = - createKeyRequest(false, 0, keyName, initialMetadata); + createKeyRequest(false, 0, keyName, initialMetadata, Collections.emptyList()); OMKeyCreateRequest initialOmKeyCreateRequest = new OMKeyCreateRequest(initialRequest, getBucketLayout()); OMClientResponse initialResponse = @@ -500,7 +501,7 @@ public void testOverwritingExistingMetadata( Map updatedMetadata = Collections.singletonMap("initialKey", "updatedValue"); OMRequest updatedRequest = - createKeyRequest(false, 0, keyName, updatedMetadata); + createKeyRequest(false, 0, keyName, updatedMetadata, Collections.emptyList()); OMKeyCreateRequest updatedOmKeyCreateRequest = new OMKeyCreateRequest(updatedRequest, getBucketLayout()); @@ -520,7 +521,7 @@ public void testCreationWithoutMetadataFollowedByOverwriteWithMetadata( // Create the key request without any initial metadata OMRequest createRequestWithoutMetadata = createKeyRequest(false, 0, keyName, - null); // Passing 'null' for metadata + null, Collections.emptyList()); // Passing 'null' for metadata OMKeyCreateRequest createOmKeyCreateRequest = new OMKeyCreateRequest(createRequestWithoutMetadata, getBucketLayout()); @@ -543,7 +544,7 @@ public void testCreationWithoutMetadataFollowedByOverwriteWithMetadata( // Overwrite the previously created key with new metadata OMRequest overwriteRequestWithMetadata = - createKeyRequest(false, 0, keyName, overwriteMetadata); + createKeyRequest(false, 0, keyName, overwriteMetadata, Collections.emptyList()); OMKeyCreateRequest overwriteOmKeyCreateRequest = new OMKeyCreateRequest(overwriteRequestWithMetadata, getBucketLayout()); @@ -648,7 +649,7 @@ protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber) { private OMRequest createKeyRequest(boolean isMultipartKey, int partNumber, String keyName) { - return createKeyRequest(isMultipartKey, partNumber, keyName, null); + return createKeyRequest(isMultipartKey, partNumber, keyName, null, Collections.emptyList()); } /** @@ -665,7 +666,8 @@ private OMRequest createKeyRequest(boolean isMultipartKey, int partNumber, */ protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber, String keyName, - Map metadata) { + Map metadata, + List acls) { KeyArgs.Builder keyArgs = KeyArgs.newBuilder() .setVolumeName(volumeName) .setBucketName(bucketName) @@ -676,6 +678,9 @@ protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber, .setType(replicationConfig.getReplicationType()) .setLatestVersionLocation(true); + for (OzoneAcl acl : acls) { + keyArgs.addAcls(OzoneAcl.toProtobuf(acl)); + } // Configure for multipart upload, if applicable if (isMultipartKey) { keyArgs.setDataSize(dataSize).setMultipartNumber(partNumber); @@ -701,7 +706,14 @@ protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber, private OMRequest createKeyRequest( boolean isMultipartKey, int partNumber, long keyLength, - ReplicationConfig repConfig) { + ReplicationConfig repConfig, Long rewriteGeneration) { + return createKeyRequest(isMultipartKey, partNumber, keyLength, repConfig, + rewriteGeneration, null); + } + + private OMRequest createKeyRequest( + boolean isMultipartKey, int partNumber, long keyLength, + ReplicationConfig repConfig, Long rewriteGeneration, Map metaData) { KeyArgs.Builder keyArgs = KeyArgs.newBuilder() .setVolumeName(volumeName).setBucketName(bucketName) @@ -720,8 +732,17 @@ private OMRequest createKeyRequest( if (isMultipartKey) { keyArgs.setMultipartNumber(partNumber); } + if (rewriteGeneration != null) { + keyArgs.setRewriteGeneration(rewriteGeneration); + } + if (metaData != null) { + metaData.forEach((key, value) -> keyArgs.addMetadata(KeyValue.newBuilder() + .setKey(key) + .setValue(value) + .build())); + } - OzoneManagerProtocolProtos.CreateKeyRequest createKeyRequest = + CreateKeyRequest createKeyRequest = CreateKeyRequest.newBuilder().setKeyArgs(keyArgs).build(); return OMRequest.newBuilder() @@ -903,6 +924,80 @@ public void testKeyCreateInheritParentDefaultAcls( } + @ParameterizedTest + @MethodSource("data") + public void testAtomicRewrite( + boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception { + when(ozoneManager.getOzoneLockProvider()).thenReturn( + new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + + if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + // TODO: Test is not applicable for FSO layout. + return; + } + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, omMetadataManager, + OmBucketInfo.newBuilder().setVolumeName(volumeName) + .setBucketName(bucketName) + .setBucketLayout(getBucketLayout())); + + // First, create a key with the rewrite ID - this should fail as no key exists + OMRequest omRequest = createKeyRequest(false, 0, 100, + RatisReplicationConfig.getInstance(THREE), 1L); + omRequest = doPreExecute(omRequest); + OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(omRequest); + OMClientResponse response = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 105L); + assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); + + // Now pre-create the key in the system so we can rewrite it. + Map metadata = Collections.singletonMap("metakey", "metavalue"); + Map reWriteMetadata = Collections.singletonMap("metakey", "rewriteMetavalue"); + + List acls = Collections.singletonList(OzoneAcl.parseAcl("user:foo:rw")); + OmKeyInfo createdKeyInfo = createAndCheck(keyName, metadata, acls); + // Commit openKey entry. + OMRequestTestUtils.addKeyToTable(false, false, + createdKeyInfo, 0L, 0L, omMetadataManager); + // Retrieve the committed key info + String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); + OmKeyInfo existingKeyInfo = omMetadataManager.getKeyTable(getBucketLayout()).get(ozoneKey); + List existingAcls = existingKeyInfo.getAcls(); + assertEquals(acls, existingAcls); + + // Create a request with a generation which doesn't match the current key + omRequest = createKeyRequest(false, 0, 100, + RatisReplicationConfig.getInstance(THREE), existingKeyInfo.getGeneration() + 1, reWriteMetadata); + omRequest = doPreExecute(omRequest); + omKeyCreateRequest = getOMKeyCreateRequest(omRequest); + response = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 105L); + // Still fails, as the matching key is not present. + assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus()); + + // Now create the key with the correct rewrite generation + omRequest = createKeyRequest(false, 0, 100, + RatisReplicationConfig.getInstance(THREE), existingKeyInfo.getGeneration(), reWriteMetadata); + omRequest = doPreExecute(omRequest); + omKeyCreateRequest = getOMKeyCreateRequest(omRequest); + response = omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 105L); + assertEquals(OK, response.getOMResponse().getStatus()); + + // Ensure the rewriteGeneration is persisted in the open key table + String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, + keyName, omRequest.getCreateKeyRequest().getClientID()); + OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()).get(openKey); + + assertEquals(existingKeyInfo.getGeneration(), openKeyInfo.getRewriteGeneration()); + // Creation time should remain the same on rewrite. + assertEquals(existingKeyInfo.getCreationTime(), openKeyInfo.getCreationTime()); + // Update ID should change + assertNotEquals(existingKeyInfo.getGeneration(), openKeyInfo.getGeneration()); + assertEquals(metadata, existingKeyInfo.getMetadata()); + // The metadata should not be copied from the existing key. It should be passed in the request. + assertEquals(reWriteMetadata, openKeyInfo.getMetadata()); + // Ensure the ACLS are copied over from the existing key. + assertEquals(existingAcls, openKeyInfo.getAcls()); + } + /** * Leaf file has ACCESS scope acls which inherited * from parent DEFAULT acls. @@ -959,9 +1054,13 @@ private void checkNotAFile(String keyName) throws Exception { assertEquals(NOT_A_FILE, omClientResponse.getOMResponse().getStatus()); } - private void createAndCheck(String keyName) throws Exception { - OMRequest omRequest = createKeyRequest(false, 0, keyName); + createAndCheck(keyName, Collections.emptyMap(), Collections.emptyList()); + } + + private OmKeyInfo createAndCheck(String keyName, Map metadata, List acls) + throws Exception { + OMRequest omRequest = createKeyRequest(false, 0, keyName, metadata, acls); OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(omRequest); @@ -974,10 +1073,10 @@ private void createAndCheck(String keyName) throws Exception { assertEquals(OK, omClientResponse.getOMResponse().getStatus()); - checkCreatedPaths(omKeyCreateRequest, omRequest, keyName); + return checkCreatedPaths(omKeyCreateRequest, omRequest, keyName); } - protected void checkCreatedPaths( + protected OmKeyInfo checkCreatedPaths( OMKeyCreateRequest omKeyCreateRequest, OMRequest omRequest, String keyName) throws Exception { keyName = omKeyCreateRequest.validateAndNormalizeKey(true, keyName); @@ -992,6 +1091,7 @@ protected void checkCreatedPaths( omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()) .get(openKey); assertNotNull(omKeyInfo); + return omKeyInfo; } protected long checkIntermediatePaths(Path keyPath) throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java index 2a25a9b09686..87badb281261 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestWithFSO.java @@ -117,7 +117,7 @@ protected void addToKeyTable(String keyName) throws Exception { } @Override - protected void checkCreatedPaths(OMKeyCreateRequest omKeyCreateRequest, + protected OmKeyInfo checkCreatedPaths(OMKeyCreateRequest omKeyCreateRequest, OMRequest omRequest, String keyName) throws Exception { keyName = omKeyCreateRequest.validateAndNormalizeKey(true, keyName, BucketLayout.FILE_SYSTEM_OPTIMIZED); @@ -139,6 +139,7 @@ protected void checkCreatedPaths(OMKeyCreateRequest omKeyCreateRequest, omMetadataManager.getOpenKeyTable(omKeyCreateRequest.getBucketLayout()) .get(openKey); assertNotNull(omKeyInfo); + return omKeyInfo; } @Override diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index bc562d5d9363..d7d092c0e580 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -227,6 +227,14 @@ public OzoneOutputStream createKey(String volumeName, String bucketName, .createKey(keyName, size, replicationConfig, metadata); } + @Override + public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyName, + long size, long existingKeyGeneration, ReplicationConfig replicationConfig, + Map metadata) throws IOException { + return getBucket(volumeName, bucketName) + .rewriteKey(keyName, size, existingKeyGeneration, replicationConfig, metadata); + } + @Override public OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index db74bd562c3e..0dc548867676 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -157,6 +157,38 @@ public void close() throws IOException { return new OzoneOutputStream(byteArrayOutputStream, null); } + @Override + public OzoneOutputStream rewriteKey(String keyName, long size, long existingKeyGeneration, + ReplicationConfig rConfig, Map metadata) throws IOException { + final ReplicationConfig repConfig; + if (rConfig == null) { + repConfig = getReplicationConfig(); + } else { + repConfig = rConfig; + } + ReplicationConfig finalReplicationCon = repConfig; + ByteArrayOutputStream byteArrayOutputStream = + new KeyMetadataAwareOutputStream(metadata) { + @Override + public void close() throws IOException { + keyContents.put(keyName, toByteArray()); + keyDetails.put(keyName, new OzoneKeyDetails( + getVolumeName(), + getName(), + keyName, + size, + System.currentTimeMillis(), + System.currentTimeMillis(), + new ArrayList<>(), finalReplicationCon, metadata, null, + () -> readKey(keyName), true, null, null + )); + super.close(); + } + }; + + return new OzoneOutputStream(byteArrayOutputStream, null); + } + @Override public OzoneDataStreamOutput createStreamKey(String key, long size, ReplicationConfig rConfig,