diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyRenameInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyRenameInfo.java new file mode 100644 index 000000000000..bb6001d20ca5 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyRenameInfo.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.helpers; + + +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyRenameInfo; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Argument for renamedKeyTable. Helps to store List which represents + * all the renames that happened to particular key in between snapshots. + */ +public class OmKeyRenameInfo { + private List keyNamesList; + + public OmKeyRenameInfo(List keyNamesList) { + this.keyNamesList = keyNamesList; + } + + public OmKeyRenameInfo(String keyRenameInfo) { + this.keyNamesList = new ArrayList<>(); + this.keyNamesList.add(keyRenameInfo); + } + + public void addOmKeyRenameInfo(String keyRenameInfo) { + this.keyNamesList.add(keyRenameInfo); + } + + public List getOmKeyRenameInfoList() { + return keyNamesList; + } + + public List cloneOmKeyRenameInfoList() { + return new ArrayList<>(keyNamesList); + } + + + public static OmKeyRenameInfo getFromProto(KeyRenameInfo + keyRenameInfo) throws IOException { + List list = new ArrayList<>(keyRenameInfo.getKeyNamesList()); + return new OmKeyRenameInfo.Builder().setOmKeyRenameList(list).build(); + } + + public KeyRenameInfo getProto() { + List list = new ArrayList<>(cloneOmKeyRenameInfoList()); + + KeyRenameInfo.Builder builder = KeyRenameInfo.newBuilder() + .addAllKeyNames(list); + return builder.build(); + } + + public OmKeyRenameInfo copyObject() { + return new OmKeyRenameInfo(new ArrayList<>(keyNamesList)); + } + + /** + * Builder of OmKeyRenameInfo. + */ + public static class Builder { + private List keyNamesList; + + public Builder() { } + + public OmKeyRenameInfo.Builder setOmKeyRenameList(List stringList) { + this.keyNamesList = stringList; + return this; + } + + public OmKeyRenameInfo build() { + return new OmKeyRenameInfo(keyNamesList); + } + } + +} diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index a2b2994cf9d6..33ee575d6e06 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1028,6 +1028,10 @@ message RepeatedKeyInfo { repeated KeyInfo keyInfo = 1; } +message KeyRenameInfo { + repeated string keyNames = 1; +} + message OzoneFileStatusProto { optional KeyInfo keyInfo = 2; optional uint64 blockSize = 3; diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 0229e86ddabf..285fb0245b5f 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDBTenantState; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; @@ -374,6 +375,8 @@ String getMultipartKey(String volume, String bucket, String key, String Table getSnapshotInfoTable(); + Table getRenamedKeyTable(); + /** * Gets the OM Meta table. * @return meta table reference. @@ -482,6 +485,18 @@ default String getOzonePathKey(long volumeId, long bucketId, String getOpenFileName(long volumeId, long bucketId, long parentObjectId, String fileName, long id); + + /** + * Given a volume, bucket and a objectID, return the DB key name in + * renamedKeyTable. + * + * @param volume - volume name + * @param bucket - bucket name + * @param objectID - objectID of the key + * @return DB rename key as String. + */ + String getRenameKey(String volume, String bucket, long objectID); + /** * Returns the DB key name of a multipart upload key in OM metadata store. * diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyRenameInfoCodec.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyRenameInfoCodec.java new file mode 100644 index 000000000000..dbd3b6f29445 --- /dev/null +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyRenameInfoCodec.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.codec; + +import com.google.common.base.Preconditions; +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.hadoop.hdds.utils.db.Codec; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyRenameInfo; + +import java.io.IOException; + +/** + * Codec to encode OmKeyRenameInfo as byte array. + */ +public class OmKeyRenameInfoCodec implements Codec { + @Override + public byte[] toPersistedFormat(OmKeyRenameInfo object) throws IOException { + Preconditions + .checkNotNull(object, "Null object can't be converted to byte array."); + return object.getProto().toByteArray(); + } + + @Override + public OmKeyRenameInfo fromPersistedFormat(byte[] rawData) + throws IOException { + Preconditions.checkNotNull(rawData, + "Null byte array can't converted to real object."); + try { + return OmKeyRenameInfo.getFromProto(KeyRenameInfo.parseFrom(rawData)); + } catch (InvalidProtocolBufferException ex) { + throw new IllegalArgumentException( + "Can't encode the the raw data from the byte array", ex); + } + } + + @Override + public OmKeyRenameInfo copyObject(OmKeyRenameInfo object) { + return object.copyObject(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 2a0241e3b3cd..ede9b5893a32 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -64,6 +64,7 @@ import org.apache.hadoop.ozone.om.codec.OmDBTenantStateCodec; import org.apache.hadoop.ozone.om.codec.OmVolumeArgsCodec; import org.apache.hadoop.ozone.om.codec.RepeatedOmKeyInfoCodec; +import org.apache.hadoop.ozone.om.codec.OmKeyRenameInfoCodec; import org.apache.hadoop.ozone.om.codec.S3SecretValueCodec; import org.apache.hadoop.ozone.om.codec.OmDBSnapshotInfoCodec; import org.apache.hadoop.ozone.om.codec.TokenIdentifierCodec; @@ -83,6 +84,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -193,6 +195,8 @@ public class OmMetadataManagerImpl implements OMMetadataManager, * |----------------------------------------------------------------------| * | snapshotInfoTable | /volume/bucket/snapshotName -> SnapshotInfo | * |----------------------------------------------------------------------| + * | renamedKeyTable | /volumeName/bucketName/objectID -> OmKeyRenameInfo| + * |----------------------------------------------------------------------| */ public static final String USER_TABLE = "userTable"; @@ -219,6 +223,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager, "principalToAccessIdsTable"; public static final String TENANT_STATE_TABLE = "tenantStateTable"; public static final String SNAPSHOT_INFO_TABLE = "snapshotInfoTable"; + public static final String RENAMED_KEY_TABLE = "renamedKeyTable"; static final String[] ALL_TABLES = new String[] { USER_TABLE, @@ -240,7 +245,8 @@ public class OmMetadataManagerImpl implements OMMetadataManager, TENANT_ACCESS_ID_TABLE, PRINCIPAL_TO_ACCESS_IDS_TABLE, TENANT_STATE_TABLE, - SNAPSHOT_INFO_TABLE + SNAPSHOT_INFO_TABLE, + RENAMED_KEY_TABLE }; private DBStore store; @@ -267,7 +273,9 @@ public class OmMetadataManagerImpl implements OMMetadataManager, private Table tenantAccessIdTable; private Table principalToAccessIdsTable; private Table tenantStateTable; + private Table snapshotInfoTable; + private Table renamedKeyTable; private boolean isRatisEnabled; private boolean ignorePipelineinKey; @@ -502,6 +510,7 @@ public static DBStoreBuilder addOMTablesAndCodecs(DBStoreBuilder builder) { .addTable(PRINCIPAL_TO_ACCESS_IDS_TABLE) .addTable(TENANT_STATE_TABLE) .addTable(SNAPSHOT_INFO_TABLE) + .addTable(RENAMED_KEY_TABLE) .addCodec(OzoneTokenIdentifier.class, new TokenIdentifierCodec()) .addCodec(OmKeyInfo.class, new OmKeyInfoCodec(true)) .addCodec(RepeatedOmKeyInfo.class, @@ -516,10 +525,9 @@ public static DBStoreBuilder addOMTablesAndCodecs(DBStoreBuilder builder) { .addCodec(OmDirectoryInfo.class, new OmDirectoryInfoCodec()) .addCodec(OmDBTenantState.class, new OmDBTenantStateCodec()) .addCodec(OmDBAccessIdInfo.class, new OmDBAccessIdInfoCodec()) - .addCodec(OmDBUserPrincipalInfo.class, - new OmDBUserPrincipalInfoCodec()) - .addCodec(SnapshotInfo.class, - new OmDBSnapshotInfoCodec()); + .addCodec(OmDBUserPrincipalInfo.class, new OmDBUserPrincipalInfoCodec()) + .addCodec(SnapshotInfo.class, new OmDBSnapshotInfoCodec()) + .addCodec(OmKeyRenameInfo.class, new OmKeyRenameInfoCodec()); } /** @@ -622,6 +630,11 @@ protected void initializeOmTables(boolean addCacheMetrics) String.class, SnapshotInfo.class); checkTableStatus(snapshotInfoTable, SNAPSHOT_INFO_TABLE, addCacheMetrics); + // objectID -> renamedKeys (renamed keys for key table) + renamedKeyTable = this.store.getTable(RENAMED_KEY_TABLE, + String.class, OmKeyRenameInfo.class); + checkTableStatus(renamedKeyTable, RENAMED_KEY_TABLE, addCacheMetrics); + // TODO: [SNAPSHOT] Initialize table lock for renamedKeyTable. } /** @@ -1614,6 +1627,11 @@ public Table getSnapshotInfoTable() { return snapshotInfoTable; } + @Override + public Table getRenamedKeyTable() { + return renamedKeyTable; + } + /** * Update store used by subclass. * @@ -1676,6 +1694,16 @@ public String getOpenFileName(long volumeId, long bucketId, return openKey.toString(); } + @Override + public String getRenameKey(String volumeName, String bucketName, + long objectID) { + StringBuilder renameKey = new StringBuilder(); + renameKey.append(OM_KEY_PREFIX).append(volumeName); + renameKey.append(OM_KEY_PREFIX).append(bucketName); + renameKey.append(OM_KEY_PREFIX).append(objectID); + return renameKey.toString(); + } + @Override public String getMultipartKey(long volumeId, long bucketId, long parentID, String fileName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index aa40cc87ad50..ae888594b5e5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; @@ -229,6 +230,15 @@ String.class, new StringCodec(), OmKeyInfo.class, SnapshotInfo.class, new OmDBSnapshotInfoCodec()); + public static final DBColumnFamilyDefinition + RENAMED_KEY_TABLE = + new DBColumnFamilyDefinition<>( + OmMetadataManagerImpl.RENAMED_KEY_TABLE, + String.class, // /volumeName/bucketName/objectID + new StringCodec(), + OmKeyRenameInfo.class, // list of key renames + new OmKeyRenameInfoCodec()); + @Override public String getName() { return OzoneConsts.OM_DB_NAME; @@ -248,7 +258,7 @@ public DBColumnFamilyDefinition[] getColumnFamilies() { FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE, META_TABLE, TENANT_ACCESS_ID_TABLE, PRINCIPAL_TO_ACCESS_IDS_TABLE, TENANT_STATE_TABLE, - SNAPSHOT_INFO_TABLE}; + SNAPSHOT_INFO_TABLE, RENAMED_KEY_TABLE}; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java index b5cc494f25fc..e755e75d62be 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java @@ -21,6 +21,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .OMResponse; @@ -30,11 +31,12 @@ import javax.annotation.Nonnull; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.RENAMED_KEY_TABLE; /** * Response for RenameKey request. */ -@CleanupTableInfo(cleanupTables = {KEY_TABLE}) +@CleanupTableInfo(cleanupTables = {KEY_TABLE, RENAMED_KEY_TABLE}) public class OMKeyRenameResponse extends OmKeyResponse { private String fromKeyName; @@ -73,13 +75,26 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { String volumeName = renameKeyInfo.getVolumeName(); String bucketName = renameKeyInfo.getBucketName(); + String fromDbKey = omMetadataManager + .getOzoneKey(volumeName, bucketName, fromKeyName); omMetadataManager.getKeyTable(getBucketLayout()) - .deleteWithBatch(batchOperation, - omMetadataManager.getOzoneKey(volumeName, bucketName, fromKeyName)); + .deleteWithBatch(batchOperation, fromDbKey); omMetadataManager.getKeyTable(getBucketLayout()) .putWithBatch(batchOperation, omMetadataManager.getOzoneKey(volumeName, bucketName, toKeyName), renameKeyInfo); + + String renameDbKey = omMetadataManager.getRenameKey( + renameKeyInfo.getVolumeName(), renameKeyInfo.getBucketName(), + renameKeyInfo.getObjectID()); + OmKeyRenameInfo omKeyRenameInfo = omMetadataManager.getRenamedKeyTable() + .get(renameDbKey); + if (omKeyRenameInfo == null) { + omKeyRenameInfo = new OmKeyRenameInfo(fromDbKey); + omMetadataManager.getRenamedKeyTable().putWithBatch( + batchOperation, renameDbKey, omKeyRenameInfo); + } + } public OmKeyInfo getRenameKeyInfo() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java index 4b5f6d1fdbfe..eb8f4a2b6cc2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseWithFSO.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -33,11 +34,13 @@ import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.RENAMED_KEY_TABLE; /** * Response for RenameKey request - prefix layout. */ -@CleanupTableInfo(cleanupTables = {FILE_TABLE, DIRECTORY_TABLE}) +@CleanupTableInfo(cleanupTables = {FILE_TABLE, DIRECTORY_TABLE, + RENAMED_KEY_TABLE}) public class OMKeyRenameResponseWithFSO extends OMKeyRenameResponse { private boolean isRenameDirectory; @@ -88,7 +91,21 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, .deleteWithBatch(batchOperation, getFromKeyName()); omMetadataManager.getKeyTable(getBucketLayout()) .putWithBatch(batchOperation, getToKeyName(), getRenameKeyInfo()); + + String renameDbKey = omMetadataManager.getRenameKey( + getRenameKeyInfo().getVolumeName(), + getRenameKeyInfo().getBucketName(), + getRenameKeyInfo().getObjectID()); + + OmKeyRenameInfo omKeyRenameInfo = omMetadataManager.getRenamedKeyTable() + .get(renameDbKey); + if (omKeyRenameInfo == null) { + omKeyRenameInfo = new OmKeyRenameInfo(getFromKeyName()); + omMetadataManager.getRenamedKeyTable().putWithBatch( + batchOperation, renameDbKey, omKeyRenameInfo); + } } + if (fromKeyParent != null) { addDirToDBBatch(omMetadataManager, fromKeyParent, volumeId, bucketId, batchOperation); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java index b09ff9e42166..dd69092e524b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysRenameResponse.java @@ -22,6 +22,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmRenameKeys; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -31,13 +32,14 @@ import java.util.Map; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.RENAMED_KEY_TABLE; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_RENAME; /** * Response for RenameKeys request. */ -@CleanupTableInfo(cleanupTables = {KEY_TABLE}) +@CleanupTableInfo(cleanupTables = {KEY_TABLE, RENAMED_KEY_TABLE}) public class OMKeysRenameResponse extends OMClientResponse { private OmRenameKeys omRenameKeys; @@ -77,14 +79,28 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, String fromKeyName = entry.getKey(); OmKeyInfo newKeyInfo = entry.getValue(); String toKeyName = newKeyInfo.getKeyName(); + String fromDbKey = omMetadataManager + .getOzoneKey(volumeName, bucketName, fromKeyName); omMetadataManager.getKeyTable(getBucketLayout()) - .deleteWithBatch(batchOperation, omMetadataManager - .getOzoneKey(volumeName, bucketName, fromKeyName)); + .deleteWithBatch(batchOperation, fromDbKey); omMetadataManager.getKeyTable(getBucketLayout()) .putWithBatch(batchOperation, omMetadataManager.getOzoneKey(volumeName, bucketName, toKeyName), newKeyInfo); + + + String renameDbKey = omMetadataManager.getRenameKey( + newKeyInfo.getVolumeName(), newKeyInfo.getBucketName(), + newKeyInfo.getObjectID()); + + OmKeyRenameInfo omKeyRenameInfo = omMetadataManager.getRenamedKeyTable() + .get(renameDbKey); + if (omKeyRenameInfo == null) { + omKeyRenameInfo = new OmKeyRenameInfo(fromDbKey); + omMetadataManager.getRenamedKeyTable().putWithBatch( + batchOperation, renameDbKey, omKeyRenameInfo); + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java index e5ba1c0511c4..718b67ec89f6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotCreateResponse.java @@ -19,8 +19,11 @@ package org.apache.hadoop.ozone.om.response.snapshot; import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -28,12 +31,15 @@ import javax.annotation.Nonnull; import java.io.IOException; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.RENAMED_KEY_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; /** * Response for OMSnapshotCreateRequest. */ -@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) +@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE, RENAMED_KEY_TABLE}) public class OMSnapshotCreateResponse extends OMClientResponse { public OMSnapshotCreateResponse(@Nonnull OMResponse omResponse, @@ -68,5 +74,25 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, // Add to db omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, key, snapshotInfo); + + // TODO: [SNAPSHOT] Move to createOmSnapshotCheckpoint and add table lock + // Remove all entries from renamedKeyTable + TableIterator> + iterator = omMetadataManager.getRenamedKeyTable().iterator(); + + String dbSnapshotBucketKey = omMetadataManager.getBucketKey( + snapshotInfo.getVolumeName(), snapshotInfo.getBucketName()) + + OM_KEY_PREFIX; + iterator.seek(dbSnapshotBucketKey); + + while (iterator.hasNext()) { + String renameDbKey = iterator.next().getKey(); + if (!renameDbKey.startsWith(dbSnapshotBucketKey)) { + break; + } + omMetadataManager.getRenamedKeyTable() + .deleteWithBatch(batchOperation, renameDbKey); + } + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index c029ea39e316..94e551a33526 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -23,6 +23,8 @@ import java.util.UUID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.AuditMessage; @@ -32,8 +34,10 @@ import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse; import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.ozone.test.LambdaTestUtils; import org.junit.After; @@ -67,6 +71,7 @@ public class TestOMSnapshotCreateRequest { private OzoneManager ozoneManager; private OMMetrics omMetrics; private OMMetadataManager omMetadataManager; + private BatchOperation batchOperation; private String volumeName; private String bucketName; @@ -98,6 +103,7 @@ public void setup() throws Exception { AuditLogger auditLogger = mock(AuditLogger.class); when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); + batchOperation = omMetadataManager.getStore().initBatchOperation(); volumeName = UUID.randomUUID().toString(); bucketName = UUID.randomUUID().toString(); @@ -111,6 +117,9 @@ public void setup() throws Exception { public void stop() { omMetrics.unRegister(); Mockito.framework().clearInlineMocks(); + if (batchOperation != null) { + batchOperation.close(); + } } @Test @@ -238,6 +247,50 @@ public void testValidateAndUpdateCache() throws Exception { omResponse.getStatus()); } + @Test + public void testEmptyRenamedKeyTable() throws Exception { + when(ozoneManager.isAdmin(any())).thenReturn(true); + OmKeyInfo toKeyInfo = addKey("key1"); + OmKeyInfo fromKeyInfo = addKey("key2"); + + OMResponse omResponse = + OMResponse.newBuilder().setRenameKeyResponse( + OzoneManagerProtocolProtos.RenameKeyResponse.getDefaultInstance()) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey) + .build(); + OMKeyRenameResponse omKeyRenameResponse = + new OMKeyRenameResponse(omResponse, fromKeyInfo.getKeyName(), + toKeyInfo.getKeyName(), toKeyInfo); + + Assert.assertTrue(omMetadataManager.getRenamedKeyTable().isEmpty()); + omKeyRenameResponse.addToDBBatch(omMetadataManager, batchOperation); + omMetadataManager.getStore().commitBatchOperation(batchOperation); + Assert.assertFalse(omMetadataManager.getRenamedKeyTable().isEmpty()); + + OMRequest omRequest = + OMRequestTestUtils.createSnapshotRequest( + volumeName, bucketName, snapshotName); + OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); + String key = SnapshotInfo.getTableKey(volumeName, + bucketName, snapshotName); + + Assert.assertNull(omMetadataManager.getSnapshotInfoTable().get(key)); + + //create entry + OMClientResponse omClientResponse = + omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1, + ozoneManagerDoubleBufferHelper); + omClientResponse.checkAndUpdateDB(omMetadataManager, batchOperation); + omMetadataManager.getStore().commitBatchOperation(batchOperation); + + SnapshotInfo snapshotInfo = + omMetadataManager.getSnapshotInfoTable().get(key); + Assert.assertNotNull(snapshotInfo); + Assert.assertTrue(omMetadataManager.getRenamedKeyTable().isEmpty()); + + } + @Test public void testEntryExists() throws Exception { when(ozoneManager.isAdmin(any())).thenReturn(true); @@ -290,4 +343,16 @@ static OMSnapshotCreateRequest doPreExecute( return new OMSnapshotCreateRequest(modifiedRequest); } + private OmKeyInfo addKey(String keyName) { + return OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName, + HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE, 0L); + } + + protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception { + OMRequestTestUtils.addKeyToTable(false, false, keyInfo, 0, 0L, + omMetadataManager); + return omMetadataManager.getOzoneKey(keyInfo.getVolumeName(), + keyInfo.getBucketName(), keyInfo.getKeyName()); + } + } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java index ecd80c897dc8..1e1294168c21 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponse.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.junit.Assert; import org.junit.Test; @@ -52,7 +53,7 @@ public void testAddToDBBatch() throws Exception { String toKeyName = UUID.randomUUID().toString(); OmKeyInfo toKeyInfo = getOmKeyInfo(toKeyName); - OmKeyInfo fromKeyInfo = getOmKeyInfo(keyName); + OmKeyInfo fromKeyInfo = getOmKeyInfo(toKeyInfo, keyName); String dbFromKey = addKeyToTable(fromKeyInfo); String dbToKey = getDBKeyName(toKeyInfo); @@ -63,6 +64,7 @@ public void testAddToDBBatch() throws Exception { .isExist(dbFromKey)); Assert.assertFalse(omMetadataManager.getKeyTable(getBucketLayout()) .isExist(dbToKey)); + Assert.assertTrue(omMetadataManager.getRenamedKeyTable().isEmpty()); if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) { Assert.assertFalse(omMetadataManager.getDirectoryTable() .isExist(getDBKeyName(fromKeyParent))); @@ -81,6 +83,17 @@ public void testAddToDBBatch() throws Exception { .isExist(dbFromKey)); Assert.assertTrue(omMetadataManager.getKeyTable(getBucketLayout()) .isExist(dbToKey)); + + String renameDbKey = omMetadataManager.getRenameKey( + fromKeyInfo.getVolumeName(), fromKeyInfo.getBucketName(), + fromKeyInfo.getObjectID()); + Assert.assertTrue(omMetadataManager.getRenamedKeyTable() + .isExist(renameDbKey)); + + OmKeyRenameInfo omKeyRenameInfo = + omMetadataManager.getRenamedKeyTable().get(renameDbKey); + Assert.assertTrue(omKeyRenameInfo.getOmKeyRenameInfoList() + .contains(dbFromKey)); if (getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) { Assert.assertTrue(omMetadataManager.getDirectoryTable() .isExist(getDBKeyName(fromKeyParent))); @@ -104,7 +117,7 @@ public void testAddToDBBatchWithErrorResponse() throws Exception { String toKeyName = UUID.randomUUID().toString(); OmKeyInfo toKeyInfo = getOmKeyInfo(toKeyName); - OmKeyInfo fromKeyInfo = getOmKeyInfo(keyName); + OmKeyInfo fromKeyInfo = getOmKeyInfo(toKeyInfo, keyName); String dbFromKey = addKeyToTable(fromKeyInfo); String dbToKey = getDBKeyName(toKeyInfo); @@ -145,6 +158,11 @@ protected OmKeyInfo getOmKeyInfo(String keyName) { replicationType, replicationFactor, 0L); } + protected OmKeyInfo getOmKeyInfo(OmKeyInfo toKeyInfo, + String keyName) { + return getOmKeyInfo(keyName); + } + protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception { OMRequestTestUtils.addKeyToTable(false, false, keyInfo, clientID, 0L, omMetadataManager); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java index 6dcf38df9cb1..f2f9ccaf872e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java @@ -44,6 +44,15 @@ protected OmKeyInfo getOmKeyInfo(String keyName) { bucketId + 100L, bucketId + 101L, 0L, Time.now()); } + @Override + protected OmKeyInfo getOmKeyInfo(OmKeyInfo toKeyInfo, + String keyName) { + return OMRequestTestUtils.createOmKeyInfo(toKeyInfo.getVolumeName(), + toKeyInfo.getBucketName(), keyName, replicationType, + replicationFactor, toKeyInfo.getObjectID(), + toKeyInfo.getParentObjectID(), 0L, toKeyInfo.getCreationTime()); + } + @Override protected String addKeyToTable(OmKeyInfo keyInfo) throws Exception { OMRequestTestUtils.addFileToKeyTable(false, false,