diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index aa7a3316eb2..9c6cd9b2eff 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1890,7 +1890,7 @@ private OzoneFileStatus getOzoneFileStatusV1(String volumeName, refreshPipeline(fileKeyInfo); if (sortDatanodes) { - sortDatanodeInPipeline(fileKeyInfo, clientAddress); + sortDatanodes(clientAddress, fileKeyInfo); } return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false); } else { @@ -2430,20 +2430,19 @@ public List listStatusV1(OmKeyArgs args, boolean recursive, metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); } + List keyInfoList = new ArrayList<>(fileStatusList.size()); for (OzoneFileStatus fileStatus : fileStatusList) { if (fileStatus.isFile()) { - // refreshPipeline flag check has been removed as part of - // https://issues.apache.org/jira/browse/HDDS-3658. - // Please refer this jira for more details. - refreshPipeline(fileStatus.getKeyInfo()); - - // No need to check if a key is deleted or not here, this is handled - // when adding entries to cacheKeyMap from DB. - if (args.getSortDatanodes()) { - sortDatanodeInPipeline(fileStatus.getKeyInfo(), clientAddress); - } + keyInfoList.add(fileStatus.getKeyInfo()); } } + // refreshPipeline flag check has been removed as part of + // https://issues.apache.org/jira/browse/HDDS-3658. + // Please refer this jira for more details. + refreshPipeline(keyInfoList); + if (args.getSortDatanodes()) { + sortDatanodes(clientAddress, keyInfoList.toArray(new OmKeyInfo[0])); + } fileStatusFinalList.addAll(fileStatusList); return fileStatusFinalList; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java index d8452b3e285..a14fae5e877 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java @@ -184,11 +184,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omVolumeArgs = getVolumeInfo(omMetadataManager, volumeName); omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); - // check volume quota + // check bucket and volume quota long preAllocatedSpace = newLocationList.size() * ozoneManager.getScmBlockSize() * omFileInfo.getFactor().getNumber(); - checkVolumeQuotaInBytes(omVolumeArgs, preAllocatedSpace); + checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace); // Add to cache entry can be done outside of lock for this openKey. // Even if bucket gets deleted, when commitKey we shall identify if @@ -203,9 +203,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Optional.absent(), Optional.of(missingParentInfos), trxnLogIndex); - // update usedBytes atomically. - omVolumeArgs.getUsedBytes().add(preAllocatedSpace); - omBucketInfo.getUsedBytes().add(preAllocatedSpace); + omBucketInfo.incrUsedBytes(preAllocatedSpace); // Prepare response. Sets user given full key name in the 'keyName' // attribute in response object. @@ -216,7 +214,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, .setCmdType(Type.CreateFile); omClientResponse = new OMFileCreateResponseV1(omResponse.build(), omFileInfo, missingParentInfos, clientID, omVolumeArgs, - omBucketInfo); + omBucketInfo.copyObject()); result = Result.SUCCESS; } catch (IOException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java index e985a9aed7e..ffabcd74726 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java @@ -169,11 +169,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // be subtracted. long correctedSpace = omKeyInfo.getDataSize() * factor - locationInfoList.size() * scmBlockSize * factor; - omVolumeArgs.getUsedBytes().add(correctedSpace); - omBucketInfo.getUsedBytes().add(correctedSpace); + omBucketInfo.incrUsedBytes(correctedSpace); omClientResponse = new OMKeyCommitResponseV1(omResponse.build(), - omKeyInfo, dbFileKey, dbOpenFileKey, omVolumeArgs, omBucketInfo); + omKeyInfo, dbFileKey, dbOpenFileKey, omVolumeArgs, + omBucketInfo.copyObject()); result = Result.SUCCESS; } catch (IOException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestV1.java index cbc58790c38..7a887c97834 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestV1.java @@ -148,10 +148,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omVolumeArgs = getVolumeInfo(omMetadataManager, volumeName); omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); + // TODO: HDDS-4565: consider all the sub-paths if the path is a dir. long quotaReleased = sumBlockLengths(omKeyInfo); - // update usedBytes atomically. - omVolumeArgs.getUsedBytes().add(-quotaReleased); - omBucketInfo.getUsedBytes().add(-quotaReleased); + omBucketInfo.incrUsedBytes(-quotaReleased); // No need to add cache entries to delete table. As delete table will // be used by DeleteKeyService only, not used for any client response @@ -161,7 +160,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omClientResponse = new OMKeyDeleteResponseV1(omResponse .setDeleteKeyResponse(DeleteKeyResponse.newBuilder()).build(), omKeyInfo, ozoneManager.isRatisEnabled(), omVolumeArgs, - omBucketInfo, keyStatus.isDirectory()); + omBucketInfo.copyObject(), keyStatus.isDirectory()); result = Result.SUCCESS; } catch (IOException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseV1.java index a168d8fb983..f84fa25945b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMFileCreateResponseV1.java @@ -78,10 +78,6 @@ public void addToDBBatch(OMMetadataManager omMetadataMgr, OMFileRequest.addToOpenFileTable(omMetadataMgr, batchOp, getOmKeyInfo(), getOpenKeySessionID()); - // update volume usedBytes. - omMetadataMgr.getVolumeTable().putWithBatch(batchOp, - omMetadataMgr.getVolumeKey(getOmVolumeArgs().getVolume()), - getOmVolumeArgs()); // update bucket usedBytes. omMetadataMgr.getBucketTable().putWithBatch(batchOp, omMetadataMgr.getBucketKey(getOmVolumeArgs().getVolume(), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseV1.java index bff55545ba4..26eda2dbf93 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyCommitResponseV1.java @@ -69,10 +69,6 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, OMFileRequest.addToFileTable(omMetadataManager, batchOperation, getOmKeyInfo()); - // update volume usedBytes. - omMetadataManager.getVolumeTable().putWithBatch(batchOperation, - omMetadataManager.getVolumeKey(getOmVolumeArgs().getVolume()), - getOmVolumeArgs()); // update bucket usedBytes. omMetadataManager.getBucketTable().putWithBatch(batchOperation, omMetadataManager.getBucketKey(getOmVolumeArgs().getVolume(), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseV1.java index 858532a730b..6119aabc165 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseV1.java @@ -76,10 +76,6 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, ozoneDbKey, getOmKeyInfo()); } - // update volume usedBytes. - omMetadataManager.getVolumeTable().putWithBatch(batchOperation, - omMetadataManager.getVolumeKey(getOmVolumeArgs().getVolume()), - getOmVolumeArgs()); // update bucket usedBytes. omMetadataManager.getBucketTable().putWithBatch(batchOperation, omMetadataManager.getBucketKey(getOmVolumeArgs().getVolume(),