diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index b757eb94a3a..ad27b9c8b3f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -17,6 +17,8 @@ package org.apache.hadoop.ozone; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.protobuf.ServiceException; import java.io.File; import java.io.IOException; @@ -76,6 +78,18 @@ public final class OmUtils { private static final SecureRandom SRAND = new SecureRandom(); private static byte[] randomBytes = new byte[32]; + private static final long TRANSACTION_ID_SHIFT = 8; + // from the 64 bits of ObjectID (long variable), 2 bits are reserved for + // epoch and 8 bits for recursive directory creation, if required. This + // leaves 54 bits for the transaction ID. Also, the last transaction ID is + // reserved for creating S3G volume on OM start {@link + // OzoneManager#addS3GVolumeToDB()}. + public static final long EPOCH_ID_SHIFT = 62; // 64 - 2 + public static final long REVERSE_EPOCH_ID_SHIFT = 2; // 64 - EPOCH_ID_SHIFT + public static final long MAX_TRXN_ID = (long) ((1 << 54) - 2); + public static final int EPOCH_WHEN_RATIS_NOT_ENABLED = 1; + public static final int EPOCH_WHEN_RATIS_ENABLED = 2; + private OmUtils() { } @@ -360,8 +374,6 @@ public static Collection emptyAsSingletonNull(Collection } } - - /** * If a OM conf is only set with key suffixed with OM Node ID, return the * set value. @@ -524,6 +536,49 @@ public static OmKeyInfo prepareKeyForRecover(OmKeyInfo keyInfo, } } + public static int getOMEpoch(boolean isRatisEnabled) { + return isRatisEnabled ? EPOCH_WHEN_RATIS_ENABLED : + EPOCH_WHEN_RATIS_NOT_ENABLED; + } + + /** + * Get the valid base object id given the transaction id. + * @param epoch a 2 bit epoch number. The 2 most significant bits of the + * object will be set to this epoch. + * @param txId of the transaction. This value cannot exceed 2^54 - 1 as + * out of the 64 bits for a long, 2 are reserved for the epoch + * and 8 for recursive directory creation. + * @return base object id allocated against the transaction + */ + public static long getObjectIdFromTxId(long epoch, long txId) { + Preconditions.checkArgument(txId <= MAX_TRXN_ID, "TransactionID " + + "exceeds max limit of " + MAX_TRXN_ID); + return addEpochToTxId(epoch, txId); + } + + /** + * Note - This function should not be called directly. It is directly called + * only from OzoneManager#addS3GVolumeToDB() which is a one time operation + * when OM is started first time to add S3G volume. In call other cases, + * getObjectIdFromTxId() should be called to append epoch to objectID. + */ + public static long addEpochToTxId(long epoch, long txId) { + long lsb54 = txId << TRANSACTION_ID_SHIFT; + long msb2 = epoch << EPOCH_ID_SHIFT; + + return msb2 | lsb54; + } + + /** + * Given an objectId, unset the 2 most significant bits to get the + * corresponding transaction index. + */ + @VisibleForTesting + public static long getTxIdFromObjectId(long objectId) { + return ((Long.MAX_VALUE >> REVERSE_EPOCH_ID_SHIFT) & objectId) + >> TRANSACTION_ID_SHIFT; + } + /** * Verify key name is a valid name. */ diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/ExitManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/ExitManager.java index 4a83c1d8c23..46a4fd791b5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/ExitManager.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/ExitManager.java @@ -28,6 +28,6 @@ public class ExitManager { public void exitSystem(int status, String message, Throwable throwable, Logger log) { - ExitUtils.terminate(1, message, throwable, log); + ExitUtils.terminate(status, message, throwable, log); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index b7b75a4f840..41c4d0dc693 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -111,6 +112,7 @@ import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE; import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE; +import static org.apache.hadoop.ozone.OmUtils.MAX_TRXN_ID; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; @@ -254,8 +256,9 @@ public void testDefaultS3GVolumeExists() throws Exception { Assert.assertEquals(ozoneVolume.getName(), s3VolumeName); OMMetadataManager omMetadataManager = cluster.getOzoneManager().getMetadataManager(); - long transactionID = Long.MAX_VALUE -1 >> 8; - long objectID = transactionID << 8; + long transactionID = MAX_TRXN_ID + 1; + long objectID = OmUtils.addEpochToTxId(omMetadataManager.getOmEpoch(), + transactionID); OmVolumeArgs omVolumeArgs = cluster.getOzoneManager().getMetadataManager().getVolumeTable().get( omMetadataManager.getVolumeKey(s3VolumeName)); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java index 8938cfa4869..70eb8d44c6d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java @@ -26,17 +26,27 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory; +import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.commons.lang3.RandomStringUtils; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT; +import static org.apache.hadoop.ozone.OmUtils.EPOCH_ID_SHIFT; +import static org.apache.hadoop.ozone.OmUtils.EPOCH_WHEN_RATIS_NOT_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; @@ -104,7 +114,6 @@ public void testRestartOMWithVolumeOperation() throws Exception { String volumeName = "volume" + RandomStringUtils.randomNumeric(5); OzoneClient client = cluster.getClient(); - ObjectStore objectStore = client.getObjectStore(); objectStore.createVolume(volumeName); @@ -209,5 +218,94 @@ public void testRestartOMWithKeyOperation() throws Exception { ReplicationType.RATIS)); } + @Test + public void testUniqueTrxnIndexOnOMRestart() throws Exception { + // When OM is restarted, the transaction index for requests should not + // start from 0. It should incrementally increase from the last + // transaction index which was stored in DB before restart. + + String volumeName = "volume" + RandomStringUtils.randomNumeric(5); + String bucketName = "bucket" + RandomStringUtils.randomNumeric(5); + String keyName = "key" + RandomStringUtils.randomNumeric(5); + + OzoneManager om = cluster.getOzoneManager(); + OzoneClient client = cluster.getClient(); + ObjectStore objectStore = client.getObjectStore(); + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + OzoneManagerProtocolClientSideTranslatorPB omClient = + new OzoneManagerProtocolClientSideTranslatorPB( + OmTransportFactory.create(conf, ugi, null), + RandomStringUtils.randomAscii(5)); + + objectStore.createVolume(volumeName); + + // Verify that the last transactionIndex stored in DB after volume + // creation equals the transaction index corresponding to volume's + // objectID. Also, the volume transaction index should be 1 as this is + // the first transaction in this cluster. + OmVolumeArgs volumeInfo = omClient.getVolumeInfo(volumeName); + long volumeTrxnIndex = OmUtils.getTxIdFromObjectId( + volumeInfo.getObjectID()); + Assert.assertEquals(1, volumeTrxnIndex); + Assert.assertEquals(volumeTrxnIndex, om.getLastTrxnIndexForNonRatis()); + + OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); + ozoneVolume.createBucket(bucketName); + + // Verify last transactionIndex is updated after bucket creation + OmBucketInfo bucketInfo = omClient.getBucketInfo(volumeName, bucketName); + long bucketTrxnIndex = OmUtils.getTxIdFromObjectId( + bucketInfo.getObjectID()); + Assert.assertEquals(2, bucketTrxnIndex); + Assert.assertEquals(bucketTrxnIndex, om.getLastTrxnIndexForNonRatis()); + + // Restart the OM and create new object + cluster.restartOzoneManager(); + String data = "random data"; + OzoneOutputStream ozoneOutputStream = ozoneVolume.getBucket(bucketName) + .createKey(keyName, data.length(), ReplicationType.RATIS, + ReplicationFactor.ONE, new HashMap<>()); + ozoneOutputStream.write(data.getBytes(), 0, data.length()); + ozoneOutputStream.close(); + + // Verify last transactionIndex is updated after key creation and the + // transaction index after restart is incremented from the last + // transaction index before restart. + OmKeyInfo omKeyInfo = omClient.lookupKey(new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setRefreshPipeline(true).build()); + long keyTrxnIndex = OmUtils.getTxIdFromObjectId( + omKeyInfo.getObjectID()); + Assert.assertEquals(3, keyTrxnIndex); + // Key commit is a separate transaction. Hence, the last trxn index in DB + // should be 1 more than KeyTrxnIndex + Assert.assertEquals(4, om.getLastTrxnIndexForNonRatis()); + } + + @Test + public void testEpochIntegrationInObjectID() throws Exception { + // Create a volume and check the objectID has the epoch as + // EPOCH_FOR_RATIS_NOT_ENABLED in the first 2 bits. + + OzoneClient client = cluster.getClient(); + ObjectStore objectStore = client.getObjectStore(); + + String volumeName = "volume" + RandomStringUtils.randomNumeric(5); + objectStore.createVolume(volumeName); + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + OzoneManagerProtocolClientSideTranslatorPB omClient = + new OzoneManagerProtocolClientSideTranslatorPB( + OmTransportFactory.create(conf, ugi, null), + RandomStringUtils.randomAscii(5)); + + long volObjId = omClient.getVolumeInfo(volumeName).getObjectID(); + long epochInVolObjId = volObjId >> EPOCH_ID_SHIFT; + + Assert.assertEquals(EPOCH_WHEN_RATIS_NOT_ENABLED, epochInVolObjId); + } } diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index c687a4b2290..217612d8c8b 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -72,6 +72,11 @@ public interface OMMetadataManager { */ OzoneManagerLock getLock(); + /** + * Returns the epoch associated with current OM process. + */ + long getOmEpoch(); + /** * Given a volume return the corresponding DB key. * diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/ratis/OMTransactionInfo.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/ratis/OMTransactionInfo.java index c0db0db7436..1628883fca3 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/ratis/OMTransactionInfo.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/ratis/OMTransactionInfo.java @@ -34,8 +34,12 @@ */ public final class OMTransactionInfo { + // Term associated with Ratis Log index in Ratis enabled cluster. In + // non-Ratis cluster, term is set to -1. private long term; // term associated with the ratis log index. - // Transaction index corresponds to ratis log index + // Ratis Log index in Ratis enabled cluster or the unique transaction + // index {@link OzoneManagerServerSideTransalatorPB#transactionIndex} in + // non-Ratis cluster private long transactionIndex; private OMTransactionInfo(String transactionInfo) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index da7e98515a9..d7be0977cdf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.hdds.utils.db.cache.TableCacheImpl; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.codec.OMTransactionInfoCodec; @@ -162,6 +163,15 @@ public class OmMetadataManagerImpl implements OMMetadataManager { private boolean isRatisEnabled; private boolean ignorePipelineinKey; + // Epoch is used to generate the objectIDs. The most significant 2 bits of + // objectIDs is set to this epoch. For clusters before HDDS-4315 there is + // no epoch as such. But it can be safely assumed that the most significant + // 2 bits of the objectID will be 00. From HDDS-4315 onwards, the Epoch for + // non-ratis OM clusters will be binary 01 (= decimal 1) and for ratis + // enabled OM cluster will be binary 10 (= decimal 2). This epoch is added + // to ensure uniqueness of objectIDs. + private final long omEpoch; + private Map tableMap = new HashMap<>(); public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException { @@ -176,6 +186,7 @@ public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException { isRatisEnabled = conf.getBoolean( OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); + this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled); // For test purpose only ignorePipelineinKey = conf.getBoolean( "ozone.om.ignore.pipeline", Boolean.TRUE); @@ -189,6 +200,7 @@ protected OmMetadataManagerImpl() { this.lock = new OzoneManagerLock(new OzoneConfiguration()); this.openKeyExpireThresholdMS = OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT; + this.omEpoch = 0; } @Override @@ -235,7 +247,6 @@ public Table getMultipartInfoTable() { return multipartInfoTable; } - private void checkTableStatus(Table table, String name) throws IOException { String logMessage = "Unable to get a reference to %s table. Cannot " + "continue."; @@ -498,6 +509,11 @@ public org.apache.hadoop.ozone.om.lock.OzoneManagerLock getLock() { return lock; } + @Override + public long getOmEpoch() { + return omEpoch; + } + /** * Returns true if the firstArray startsWith the bytes of secondArray. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index e58af8b49ab..e1d516add03 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -139,7 +139,6 @@ import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.OMClientRequest; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.snapshot.OzoneManagerSnapshotProvider; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest; @@ -193,6 +192,7 @@ import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress; import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithFixedSleep; +import static org.apache.hadoop.ozone.OmUtils.MAX_TRXN_ID; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY; @@ -1255,7 +1255,8 @@ private RPC.Server getRpcServer(OzoneConfiguration conf) throws IOException { RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class, ProtobufRpcEngine.class); this.omServerProtocol = new OzoneManagerProtocolServerSideTranslatorPB( - this, omRatisServer, omClientProtocolMetrics, isRatisEnabled); + this, omRatisServer, omClientProtocolMetrics, isRatisEnabled, + getLastTrxnIndexForNonRatis()); BlockingService omService = newReflectiveBlockingService(omServerProtocol); @@ -1283,6 +1284,27 @@ private void initializeRatisServer() throws IOException { } } + public long getObjectIdFromTxId(long trxnId) { + return OmUtils.getObjectIdFromTxId(metadataManager.getOmEpoch(), + trxnId); + } + + @VisibleForTesting + long getLastTrxnIndexForNonRatis() throws IOException { + OMTransactionInfo omTransactionInfo = + OMTransactionInfo.readTransactionInfo(metadataManager); + // If the OMTransactionInfo does not exist in DB or if the term is not -1 + // (corresponding to non-Ratis cluster), return 0 so that new incoming + // requests can have transaction index starting from 1. + if (omTransactionInfo == null || omTransactionInfo.getTerm() != -1) { + return 0; + } + // If there exists a last transaction index in DB, the new incoming + // requests in non-Ratis cluster must have transaction index + // incrementally increasing from the stored transaction index onwards. + return omTransactionInfo.getTransactionIndex(); + } + public OMRatisSnapshotInfo getSnapshotInfo() { return omRatisSnapshotInfo; } @@ -3583,8 +3605,10 @@ private void addS3GVolumeToDB() throws IOException { "configured by S3Gateway"); } if (!metadataManager.getVolumeTable().isExist(dbVolumeKey)) { - long transactionID = (Long.MAX_VALUE - 1) >> 8; - long objectID = OMFileRequest.getObjIDFromTxId(transactionID); + // the highest transaction ID is reserved for this operation. + long transactionID = MAX_TRXN_ID + 1; + long objectID = OmUtils.addEpochToTxId(metadataManager.getOmEpoch(), + transactionID); String userName = UserGroupInformation.getCurrentUser().getShortUserName(); @@ -3657,7 +3681,6 @@ private OmVolumeArgs createS3VolumeInfo(String s3Volume, long transactionID, } return omVolumeArgs.build(); - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PrefixManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PrefixManagerImpl.java index 6461a7e86d7..81520f32d53 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PrefixManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PrefixManagerImpl.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om; import com.google.common.base.Strings; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -315,7 +316,8 @@ public OMPrefixAclOpResult addAcl(OzoneObj ozoneObj, OzoneAcl ozoneAcl, new OmPrefixInfo.Builder() .setName(ozoneObj.getPath()); if (transactionLogIndex > 0) { - prefixInfoBuilder.setObjectID(transactionLogIndex); + prefixInfoBuilder.setObjectID(OmUtils.getObjectIdFromTxId( + metadataManager.getOmEpoch(), transactionLogIndex)); prefixInfoBuilder.setUpdateID(transactionLogIndex); } prefixInfo = prefixInfoBuilder.build(); @@ -365,7 +367,8 @@ public OMPrefixAclOpResult setAcl(OzoneObj ozoneObj, List ozoneAcls, new OmPrefixInfo.Builder() .setName(ozoneObj.getPath()); if (transactionLogIndex > 0) { - prefixInfoBuilder.setObjectID(transactionLogIndex); + prefixInfoBuilder.setObjectID(OmUtils.getObjectIdFromTxId( + metadataManager.getOmEpoch(), transactionLogIndex)); prefixInfoBuilder.setUpdateID(transactionLogIndex); } prefixInfo = prefixInfoBuilder.build(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java index f1c144e7d81..3fe20b1c204 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java @@ -267,26 +267,25 @@ private void flushTransactions() { } }); - // Only when ratis is enabled commit transaction info to DB. - if (isRatisEnabled) { - flushedEpochs = - readyBuffer.stream().map(DoubleBufferEntry::getTrxLogIndex) - .sorted().collect(Collectors.toList()); - long lastRatisTransactionIndex = - flushedEpochs.get(flushedEpochs.size() - 1); - long term = indexToTerm.apply(lastRatisTransactionIndex); - - addToBatchTransactionInfoWithTrace(lastTraceId.get(), - lastRatisTransactionIndex, - (SupplierWithIOException) () -> { + // Commit transaction info to DB. + flushedEpochs = readyBuffer.stream().map( + DoubleBufferEntry::getTrxLogIndex) + .sorted().collect(Collectors.toList()); + long lastRatisTransactionIndex = flushedEpochs.get( + flushedEpochs.size() - 1); + long term = isRatisEnabled ? + indexToTerm.apply(lastRatisTransactionIndex) : -1; + + addToBatchTransactionInfoWithTrace(lastTraceId.get(), + lastRatisTransactionIndex, + (SupplierWithIOException) () -> { omMetadataManager.getTransactionInfoTable().putWithBatch( batchOperation, TRANSACTION_INFO_KEY, new OMTransactionInfo.Builder() - .setTransactionIndex(lastRatisTransactionIndex) - .setCurrentTerm(term).build()); + .setTransactionIndex(lastRatisTransactionIndex) + .setCurrentTerm(term).build()); return null; }); - } long startTime = Time.monotonicNowNanos(); flushBatchWithTrace(lastTraceId.get(), readyBuffer.size(), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index fd303e7f09a..678d04e7f53 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -31,7 +31,6 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -199,14 +198,13 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // Add objectID and updateID omBucketInfo.setObjectID( - OMFileRequest.getObjIDFromTxId(transactionLogIndex)); + ozoneManager.getObjectIdFromTxId(transactionLogIndex)); omBucketInfo.setUpdateID(transactionLogIndex, ozoneManager.isRatisEnabled()); // Add default acls from volume. addDefaultAcls(omBucketInfo, omVolumeArgs); - // Update table cache. metadataManager.getBucketTable().addCacheEntry(new CacheKey<>(bucketKey), new CacheValue<>(Optional.of(omBucketInfo), transactionLogIndex)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 7b2ab51f0c1..45952368675 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -29,7 +29,6 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; @@ -83,6 +82,11 @@ public class OMDirectoryCreateRequest extends OMKeyRequest { private static final Logger LOG = LoggerFactory.getLogger(OMDirectoryCreateRequest.class); + // The maximum number of directories which can be created through a single + // transaction (recursive directory creations) is 2^8 - 1 as only 8 + // bits are set aside for this in ObjectID. + private static final long MAX_NUM_OF_RECURSIVE_DIRS = 255; + /** * Stores the result of request execution in * OMClientRequest#validateAndUpdateCache. @@ -185,7 +189,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } else if (omDirectoryResult == DIRECTORY_EXISTS_IN_GIVENPATH || omDirectoryResult == NONE) { List missingParents = omPathInfo.getMissingParents(); - long baseObjId = OMFileRequest.getObjIDFromTxId(trxnLogIndex); + long baseObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex); List inheritAcls = omPathInfo.getAcls(); dirKeyInfo = createDirectoryKeyInfoWithACL(keyName, @@ -246,11 +250,12 @@ public static List getAllParentInfo(OzoneManager ozoneManager, OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); List missingParentInfos = new ArrayList<>(); - ImmutablePair objIdRange = OMFileRequest - .getObjIdRangeFromTxId(trxnLogIndex); - long baseObjId = objIdRange.getLeft(); - long maxObjId = objIdRange.getRight(); - long maxLevels = maxObjId - baseObjId; + // The base id is left shifted by 8 bits for creating space to + // create (2^8 - 1) object ids in every request. + // maxObjId represents the largest object id allocation possible inside + // the transaction. + long baseObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex); + long maxObjId = baseObjId + MAX_NUM_OF_RECURSIVE_DIRS; long objectCount = 1; // baseObjID is used by the leaf directory String volumeName = keyArgs.getVolumeName(); @@ -261,8 +266,8 @@ public static List getAllParentInfo(OzoneManager ozoneManager, long nextObjId = baseObjId + objectCount; if (nextObjId > maxObjId) { throw new OMException("Too many directories in path. Exceeds limit of " - + maxLevels + ". Unable to create directory: " + keyName - + " in volume/bucket: " + volumeName + "/" + bucketName, + + MAX_NUM_OF_RECURSIVE_DIRS + ". Unable to create directory: " + + keyName + " in volume/bucket: " + volumeName + "/" + bucketName, INVALID_KEY_NAME); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 9a7f31aece9..6ca3cc37460 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -260,6 +260,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omKeyInfo = prepareKeyInfo(omMetadataManager, keyArgs, dbKeyInfo, keyArgs.getDataSize(), locations, getFileEncryptionInfo(keyArgs), ozoneManager.getPrefixManager(), bucketInfo, trxnLogIndex, + ozoneManager.getObjectIdFromTxId(trxnLogIndex), ozoneManager.isRatisEnabled()); long openVersion = omKeyInfo.getLatestVersionLocations().getVersion(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index 21ffff815e0..f020f12d864 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -24,8 +24,6 @@ import java.util.List; import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneAcl; @@ -44,7 +42,6 @@ public final class OMFileRequest { private static final Logger LOG = LoggerFactory.getLogger(OMFileRequest.class); - private static final long TRANSACTION_ID_SHIFT = 8; private OMFileRequest() { } @@ -129,33 +126,6 @@ public static OMPathInfo verifyFilesInPath( return new OMPathInfo(missing, OMDirectoryResult.NONE, inheritAcls); } - /** - * Get the valid base object id given the transaction id. - * @param id of the transaction - * @return base object id allocated against the transaction - */ - public static long getObjIDFromTxId(long id) { - return id << TRANSACTION_ID_SHIFT; - } - - /** - * Generate the valid object id range for the transaction id. - * The transaction id is left shifted by 8 bits - - * creating space to create (2^8 - 1) object ids in every request. - * maxId (right element of Immutable pair) represents the largest - * object id allocation possible inside the transaction. - * @param id - * @return object id range - */ - public static ImmutablePair getObjIdRangeFromTxId(long id) { - long baseId = getObjIDFromTxId(id); - // 1 less than the baseId for the next transaction - long maxAvailableId = getObjIDFromTxId(id+1) - 1; - Preconditions.checkState(maxAvailableId >= baseId, - "max available id must be atleast equal to the base id."); - return new ImmutablePair<>(baseId, maxAvailableId); - } - /** * Class to return the results from verifyFilesInPath. * Includes the list of missing intermediate directories and diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 5ec79b5c4e7..3205fbfdbe2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -273,6 +273,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omKeyInfo = prepareKeyInfo(omMetadataManager, keyArgs, dbKeyInfo, keyArgs.getDataSize(), locations, getFileEncryptionInfo(keyArgs), ozoneManager.getPrefixManager(), bucketInfo, trxnLogIndex, + ozoneManager.getObjectIdFromTxId(trxnLogIndex), ozoneManager.isRatisEnabled()); long openVersion = omKeyInfo.getLatestVersionLocations().getVersion(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index a048533001e..58c7599bfe2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -46,7 +46,6 @@ import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.protocolPB.OMPBHelper; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; @@ -250,9 +249,7 @@ protected OmKeyInfo createKeyInfo(@Nonnull KeyArgs keyArgs, @Nullable FileEncryptionInfo encInfo, @Nonnull PrefixManager prefixManager, @Nullable OmBucketInfo omBucketInfo, - long transactionLogIndex) { - long objectID = OMFileRequest.getObjIDFromTxId(transactionLogIndex); - + long transactionLogIndex, long objectID) { return new OmKeyInfo.Builder() .setVolumeName(keyArgs.getVolumeName()) .setBucketName(keyArgs.getBucketName()) @@ -323,12 +320,14 @@ protected OmKeyInfo prepareKeyInfo( @Nullable FileEncryptionInfo encInfo, @Nonnull PrefixManager prefixManager, @Nullable OmBucketInfo omBucketInfo, - long transactionLogIndex, boolean isRatisEnabled) + long transactionLogIndex, + @Nonnull long objectID, + boolean isRatisEnabled) throws IOException { if (keyArgs.getIsMultipartKey()) { return prepareMultipartKeyInfo(omMetadataManager, keyArgs, size, locations, encInfo, prefixManager, omBucketInfo, - transactionLogIndex); + transactionLogIndex, objectID); //TODO args.getMetadata } if (dbKeyInfo != null) { @@ -350,7 +349,7 @@ protected OmKeyInfo prepareKeyInfo( // Blocks will be appended as version 0. return createKeyInfo(keyArgs, locations, keyArgs.getFactor(), keyArgs.getType(), keyArgs.getDataSize(), encInfo, prefixManager, - omBucketInfo, transactionLogIndex); + omBucketInfo, transactionLogIndex, objectID); } /** @@ -365,7 +364,8 @@ private OmKeyInfo prepareMultipartKeyInfo( @Nonnull KeyArgs args, long size, @Nonnull List locations, FileEncryptionInfo encInfo, @Nonnull PrefixManager prefixManager, - @Nullable OmBucketInfo omBucketInfo, @Nonnull long transactionLogIndex) + @Nullable OmBucketInfo omBucketInfo, @Nonnull long transactionLogIndex, + @Nonnull long objectId) throws IOException { HddsProtos.ReplicationFactor factor; HddsProtos.ReplicationType type; @@ -394,7 +394,7 @@ private OmKeyInfo prepareMultipartKeyInfo( // For this upload part we don't need to check in KeyTable. As this // is not an actual key, it is a part of the key. return createKeyInfo(args, locations, factor, type, size, encInfo, - prefixManager, omBucketInfo, transactionLogIndex); + prefixManager, omBucketInfo, transactionLogIndex, objectId); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index 08063b6b4d9..c399cad53aa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -29,7 +29,6 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -117,7 +116,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, OmMultipartKeyInfo multipartKeyInfo = null; OmKeyInfo omKeyInfo = null; Result result = null; - long objectID = OMFileRequest.getObjIDFromTxId(transactionLogIndex); + long objectID = ozoneManager.getObjectIdFromTxId(transactionLogIndex); OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java index 9c81c36eaf2..60560cb89d4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java @@ -27,7 +27,6 @@ import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; @@ -125,7 +124,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // The Object ID will never change, but update // ID will be set to transactionID each time we update the object. omVolumeArgs.setObjectID( - OMFileRequest.getObjIDFromTxId(transactionLogIndex)); + ozoneManager.getObjectIdFromTxId(transactionLogIndex)); omVolumeArgs.setUpdateID(transactionLogIndex, ozoneManager.isRatisEnabled()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java index 73277e0dda7..57b8fac6d0d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java @@ -58,7 +58,7 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements private final boolean isRatisEnabled; private final OzoneManager ozoneManager; private final OzoneManagerDoubleBuffer ozoneManagerDoubleBuffer; - private final AtomicLong transactionIndex = new AtomicLong(0L); + private final AtomicLong transactionIndex; private final OzoneProtocolMessageDispatcher dispatcher; @@ -71,9 +71,14 @@ public OzoneManagerProtocolServerSideTranslatorPB( OzoneManager impl, OzoneManagerRatisServer ratisServer, ProtocolMessageMetrics metrics, - boolean enableRatis) { + boolean enableRatis, + long lastTransactionIndexForNonRatis) { this.ozoneManager = impl; this.isRatisEnabled = enableRatis; + // Update the transactionIndex with the last TransactionIndex read from DB. + // New requests should have transactionIndex incremented from this index + // onwards to ensure unique objectIDs. + this.transactionIndex = new AtomicLong(lastTransactionIndexForNonRatis); if (isRatisEnabled) { // In case of ratis is enabled, handler in ServerSideTransaltorPB is used diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java index 4ac1f494b00..b1bc17319dd 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeCreateRequest.java @@ -21,7 +21,6 @@ import java.util.UUID; import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; @@ -64,7 +63,7 @@ public void testValidateAndUpdateCacheWithZeroMaxUserVolumeCount() String adminName = "user1"; String ownerName = "user1"; long txLogIndex = 1; - long expectedObjId = OMFileRequest.getObjIDFromTxId(txLogIndex); + long expectedObjId = ozoneManager.getObjectIdFromTxId(txLogIndex); OMRequest originalRequest = createVolumeRequest(volumeName, adminName, ownerName); @@ -115,7 +114,7 @@ public void testValidateAndUpdateCacheSuccess() throws Exception { omVolumeCreateRequest = new OMVolumeCreateRequest(modifiedRequest); long txLogIndex = 2; - long expectedObjId = OMFileRequest.getObjIDFromTxId(txLogIndex); + long expectedObjId = ozoneManager.getObjectIdFromTxId(txLogIndex); OMClientResponse omClientResponse = omVolumeCreateRequest.validateAndUpdateCache(ozoneManager, txLogIndex,