Skip to content

HDDS-13510. Add latency metrics for key creation operation #8896

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Tejaskriya merged 4 commits into from
Sep 15, 2025
Merged

HDDS-13510. Add latency metrics for key creation operation #8896

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d11e8b5
HDDS-13510. Add latency metrics for key creation operation
Aug 4, 2025
edc6ddc
Fixed test failure
Aug 4, 2025
3af7e94
Updated vriable names
Aug 6, 2025
40ce9b2
Updated create key latency capture
Aug 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions ...op-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,21 @@ public class OMPerformanceMetrics {
@Metric(about = "Latency of each iteration of OpenKeyCleanupService in ms")
private MutableGaugeLong openKeyCleanupServiceLatencyMs;

@Metric(about = "ResolveBucketLink and ACL check latency for createKey in nanoseconds")
private MutableRate createKeyResolveBucketAndAclCheckLatencyNs;

@Metric(about = "check quota for createKey in nanoseconds")
private MutableRate createKeyQuotaCheckLatencyNs;

@Metric(about = "Block allocation latency for createKey in nanoseconds")
private MutableRate createKeyAllocateBlockLatencyNs;

@Metric(about = "createKeyFailure latency in nanoseconds")
private MutableRate createKeyFailureLatencyNs;

@Metric(about = "creteKeySuccess latency in nanoseconds")
private MutableRate createKeySuccessLatencyNs;

public static OMPerformanceMetrics register() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME,
Expand Down Expand Up @@ -291,6 +306,26 @@ public void setDeleteKeysAclCheckLatencyNs(long latencyInNs) {
public MutableRate getDeleteKeyResolveBucketAndAclCheckLatencyNs() {
return deleteKeyResolveBucketAndAclCheckLatencyNs;
}

public MutableRate getCreateKeyResolveBucketAndAclCheckLatencyNs() {
return createKeyResolveBucketAndAclCheckLatencyNs;
}

public void addCreateKeyQuotaCheckLatencyNs(long latencyInNs) {
createKeyQuotaCheckLatencyNs.add(latencyInNs);
}

public MutableRate getCreateKeyAllocateBlockLatencyNs() {
return createKeyAllocateBlockLatencyNs;
}

public void setCreateKeyFailureLatencyNs(long latencyInNs) {
createKeyFailureLatencyNs.add(latencyInNs);
Comment thread
sreejasahithi marked this conversation as resolved.
Outdated
}

public void setCreateKeySuccessLatencyNs(long latencyInNs) {
createKeySuccessLatencyNs.add(latencyInNs);
Comment thread
sreejasahithi marked this conversation as resolved.
Outdated
}

public void addListKeysReadFromRocksDbLatencyNs(long latencyInNs) {
listKeysReadFromRocksDbLatencyNs.add(latencyInNs);
Expand Down
37 changes: 26 additions & 11 deletions ...zone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE;
import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.DIRECTORY_EXISTS;
import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.FILE_EXISTS_IN_GIVENPATH;
import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs;

import com.google.common.base.Preconditions;
import java.io.IOException;
Expand All @@ -38,6 +39,7 @@
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
import org.apache.hadoop.ozone.om.OzoneConfigUtil;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
Expand Down Expand Up @@ -89,6 +91,8 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {

KeyArgs keyArgs = createKeyRequest.getKeyArgs();

final OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics();

if (keyArgs.hasExpectedDataGeneration()) {
ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY);
}
Expand Down Expand Up @@ -141,15 +145,16 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
// till leader is identified.
UserInfo userInfo = getUserInfo();
List<OmKeyLocationInfo> omKeyLocationInfoList =
allocateBlock(ozoneManager.getScmClient(),
ozoneManager.getBlockTokenSecretManager(), repConfig,
new ExcludeList(), requestedSize, scmBlockSize,
ozoneManager.getPreallocateBlocksMax(),
ozoneManager.isGrpcBlockTokenEnabled(),
ozoneManager.getOMServiceId(),
ozoneManager.getMetrics(),
keyArgs.getSortDatanodes(),
userInfo);
captureLatencyNs(perfMetrics.getCreateKeyAllocateBlockLatencyNs(),
() -> allocateBlock(ozoneManager.getScmClient(),
ozoneManager.getBlockTokenSecretManager(), repConfig,
new ExcludeList(), requestedSize, scmBlockSize,
ozoneManager.getPreallocateBlocksMax(),
ozoneManager.isGrpcBlockTokenEnabled(),
ozoneManager.getOMServiceId(),
ozoneManager.getMetrics(),
keyArgs.getSortDatanodes(),
userInfo));

newKeyArgs = keyArgs.toBuilder().setModificationTime(Time.now())
.setType(type).setFactor(factor)
Expand All @@ -171,9 +176,11 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
generateRequiredEncryptionInfo(keyArgs, newKeyArgs, ozoneManager);
}

KeyArgs.Builder finalNewKeyArgs = newKeyArgs;
KeyArgs resolvedKeyArgs =
resolveBucketAndCheckKeyAcls(newKeyArgs.build(), ozoneManager,
IAccessAuthorizer.ACLType.CREATE);
captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(),
() -> resolveBucketAndCheckKeyAcls(finalNewKeyArgs.build(), ozoneManager,
IAccessAuthorizer.ACLType.CREATE));
newCreateKeyRequest =
createKeyRequest.toBuilder().setKeyArgs(resolvedKeyArgs)
.setClientID(UniqueId.next());
Expand Down Expand Up @@ -212,6 +219,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
Result result = null;
List<OmKeyInfo> missingParentInfos = null;
int numMissingParents = 0;
final OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics();
long startNanos = Time.monotonicNowNanos();
Comment thread
sreejasahithi marked this conversation as resolved.
Outdated
try {

mergeOmLockDetails(
Expand Down Expand Up @@ -302,9 +311,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
* ozoneManager.getScmBlockSize()
* replicationConfig.getRequiredNodes();
// check bucket and volume quota
long startTime = Time.monotonicNowNanos();
Comment thread
sreejasahithi marked this conversation as resolved.
Outdated
checkBucketQuotaInBytes(omMetadataManager, bucketInfo,
preAllocatedSpace);
checkBucketQuotaInNamespace(bucketInfo, numMissingParents + 1L);
perfMetrics.addCreateKeyQuotaCheckLatencyNs(Time.monotonicNowNanos() - startTime);
bucketInfo.incrUsedNamespace(numMissingParents);

if (numMissingParents > 0) {
Expand Down Expand Up @@ -332,13 +343,17 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
omKeyInfo, missingParentInfos, clientID, bucketInfo.copyObject());

result = Result.SUCCESS;
long endNanosCreateKeySuccessLatencyNs = Time.monotonicNowNanos();
perfMetrics.setCreateKeySuccessLatencyNs(endNanosCreateKeySuccessLatencyNs - startNanos);
} catch (IOException | InvalidPathException ex) {
result = Result.FAILURE;
exception = ex;
omMetrics.incNumKeyAllocateFails();
omResponse.setCmdType(Type.CreateKey);
omClientResponse = new OMKeyCreateResponse(
createErrorOMResponse(omResponse, exception), getBucketLayout());
long endNanosCreateKeyFailureLatencyNs = Time.monotonicNowNanos();
perfMetrics.setCreateKeyFailureLatencyNs(endNanosCreateKeyFailureLatencyNs - startNanos);
} finally {
if (acquireLock) {
mergeOmLockDetails(ozoneLockStrategy
Expand Down
10 changes: 10 additions & 0 deletions ...nager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
import org.apache.hadoop.ozone.om.OzoneConfigUtil;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
Expand All @@ -50,6 +51,7 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateKeyResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.util.Time;

/**
* Handles CreateKey request layout version1.
Expand Down Expand Up @@ -91,6 +93,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
Result result;
List<OmDirectoryInfo> missingParentInfos;
int numKeysCreated = 0;
final OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics();
long startNanos = Time.monotonicNowNanos();
Comment thread
sreejasahithi marked this conversation as resolved.
Outdated
try {
mergeOmLockDetails(omMetadataManager.getLock()
.acquireWriteLock(BUCKET_LOCK, volumeName, bucketName));
Expand Down Expand Up @@ -174,9 +178,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
long preAllocatedSpace =
newLocationList.size() * ozoneManager.getScmBlockSize() * repConfig
.getRequiredNodes();
long startTime = Time.monotonicNowNanos();
Comment thread
sreejasahithi marked this conversation as resolved.
Outdated
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
preAllocatedSpace);
checkBucketQuotaInNamespace(omBucketInfo, numKeysCreated + 1L);
perfMetrics.addCreateKeyQuotaCheckLatencyNs(Time.monotonicNowNanos() - startTime);
omBucketInfo.incrUsedNamespace(numKeysCreated);

// Add to cache entry can be done outside of lock for this openKey.
Expand Down Expand Up @@ -206,13 +212,17 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
omBucketInfo.copyObject(), volumeId);

result = Result.SUCCESS;
long endNanosCreateKeySuccessLatencyNs = Time.monotonicNowNanos();

@sarvekshayr sarvekshayr Aug 25, 2025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rename endNanosCreateKeySuccessLatencyNs in both places to createKeyEndTime, capture the time in the finally block and update the latency metrics based on result.

finally {
    long createKeyEndTime = Time.monotonicNowNanos();
    long latency = createKeyEndTime - createKeyStartTime;
    if (result == Result.SUCCESS) {
        perfMetrics.addCreateKeySuccessLatencyNs(latency);
    } else {
        perfMetrics.addCreateKeyFailureLatencyNs(latency);
    }

perfMetrics.setCreateKeySuccessLatencyNs(endNanosCreateKeySuccessLatencyNs - startNanos);
} catch (IOException | InvalidPathException ex) {
result = Result.FAILURE;
exception = ex;
omMetrics.incNumKeyAllocateFails();
omResponse.setCmdType(Type.CreateKey);
omClientResponse = new OMKeyCreateResponseWithFSO(
createErrorOMResponse(omResponse, exception), getBucketLayout());
long endNanosCreateKeyFailureLatencyNs = Time.monotonicNowNanos();
perfMetrics.setCreateKeyFailureLatencyNs(endNanosCreateKeyFailureLatencyNs - startNanos);
} finally {
if (acquireLock) {
mergeOmLockDetails(omMetadataManager.getLock()
Expand Down
5 changes: 5 additions & 0 deletions ...ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
Expand Down Expand Up @@ -99,6 +100,9 @@ public class TestCleanupTableInfo {
@Mock
private OMMetrics omMetrics;

@Mock
private OMPerformanceMetrics perfMetrics;

@Mock
private OzoneManager om;

Expand Down Expand Up @@ -184,6 +188,7 @@ public void testKeyCreateRequestSetsAllTouchedTableCachesForEviction() {
when(om.getEnableFileSystemPaths()).thenReturn(true);
when(om.getOzoneLockProvider()).thenReturn(
new OzoneLockProvider(false, false));
when(om.getPerfMetrics()).thenReturn(perfMetrics);

Map<String, Integer> cacheItemCount = recordCacheItemCounts();

Expand Down