Skip to content

HDDS-8147. Introduce latency metrics for S3 Gateway operations #4383

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 14 commits into from
Apr 18, 2023
Merged

HDDS-8147. Introduce latency metrics for S3 Gateway operations #4383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
d7457f6
HDDS-8147. Introduce latency metrics for S3 Gateway operations
tanvipenumudy Mar 13, 2023
3b14fe9
fix checkstyle
tanvipenumudy Mar 13, 2023
6afad90
fix TestPermissionCheck, TestS3GatewayMetrics UT failures
tanvipenumudy Mar 13, 2023
c0a4705
handling OS3Exception in BucketEndpoint endpoints with RuntimeException
tanvipenumudy Mar 15, 2023
b346b50
refactor capturing latency in a finally block
tanvipenumudy Mar 16, 2023
e86b609
Merge upstream master changes
tanvipenumudy Mar 31, 2023
c2d75d3
fix TestS3GatewayMetrics.testGetAclFailure post upstream/master merge
tanvipenumudy Mar 31, 2023
e298884
Resolve merge conflicts
tanvipenumudy Apr 11, 2023
d602d09
Resolve acceptance test failures - Head Bucket not existent
tanvipenumudy Apr 11, 2023
312c5f1
Remove separate metrics source for S3GatewayLatencyMetrics
tanvipenumudy Apr 11, 2023
7b9a689
Add separate latency metrics for successful and failed S3G operations
tanvipenumudy Apr 12, 2023
7de1ffd
Provide metric descriptions for the newly introduced metrics
tanvipenumudy Apr 12, 2023
bb0fec4
Simplify S3G metrics by combining operation count and latency measure…
tanvipenumudy Apr 12, 2023
1604a0a
Remove Exception-handling no longer required
tanvipenumudy Apr 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 25 additions & 17 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.apache.hadoop.ozone.s3.util.ContinueToken;
import org.apache.hadoop.ozone.s3.util.S3StorageType;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -102,6 +103,7 @@ public Response get(
@QueryParam("uploads") String uploads,
@QueryParam("acl") String aclMarker,
@Context HttpHeaders hh) throws OS3Exception, IOException {
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.GET_BUCKET;
Iterator<? extends OzoneKey> ozoneKeyIterator;
ContinueToken decodedToken =
Expand All @@ -111,7 +113,7 @@ public Response get(
if (aclMarker != null) {
s3GAction = S3GAction.GET_ACL;
S3BucketAcl result = getAcl(bucketName);
getMetrics().incGetAclSuccess();
getMetrics().updateGetAclSuccessStats(startNanos);
AUDIT.logReadSuccess(
buildAuditMessageForSuccess(s3GAction, getAuditParameters()));
return Response.ok(result, MediaType.APPLICATION_XML_TYPE).build();
Expand All @@ -130,7 +132,7 @@ public Response get(
if (startAfter == null && marker != null) {
startAfter = marker;
}

OzoneBucket bucket = getBucket(bucketName);
if (startAfter != null && continueToken != null) {
// If continuation token and start after both are provided, then we
Expand All @@ -146,14 +148,14 @@ public Response get(
} catch (OMException ex) {
AUDIT.logReadFailure(
buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex));
getMetrics().incGetBucketFailure();
getMetrics().updateGetBucketFailureStats(startNanos);
if (isAccessDenied(ex)) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
} else {
throw ex;
}
} catch (Exception ex) {
getMetrics().incGetBucketFailure();
getMetrics().updateGetBucketFailureStats(startNanos);
AUDIT.logReadFailure(
buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex));
throw ex;
Expand Down Expand Up @@ -247,7 +249,7 @@ public Response get(

AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction,
getAuditParameters()));
getMetrics().incGetBucketSuccess();
getMetrics().updateGetBucketSuccessStats(startNanos);
response.setKeyCount(
response.getCommonPrefixes().size() + response.getContents().size());
return Response.ok(response).build();
Expand All @@ -258,6 +260,7 @@ public Response put(@PathParam("bucket") String bucketName,
@QueryParam("acl") String aclMarker,
@Context HttpHeaders httpHeaders,
InputStream body) throws IOException, OS3Exception {
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.CREATE_BUCKET;

try {
Expand All @@ -271,12 +274,12 @@ public Response put(@PathParam("bucket") String bucketName,
String location = createS3Bucket(bucketName);
AUDIT.logWriteSuccess(
buildAuditMessageForSuccess(s3GAction, getAuditParameters()));
getMetrics().incCreateBucketSuccess();
getMetrics().updateCreateBucketSuccessStats(startNanos);
return Response.status(HttpStatus.SC_OK).header("Location", location)
.build();
} catch (OMException exception) {
auditWriteFailure(s3GAction, exception);
getMetrics().incCreateBucketFailure();
getMetrics().updateCreateBucketFailureStats(startNanos);
if (exception.getResult() == ResultCodes.INVALID_BUCKET_NAME) {
throw newError(S3ErrorTable.INVALID_BUCKET_NAME, bucketName, exception);
}
Expand All @@ -292,6 +295,7 @@ public Response listMultipartUploads(
@PathParam("bucket") String bucketName,
@QueryParam("prefix") String prefix)
throws OS3Exception, IOException {
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.LIST_MULTIPART_UPLOAD;

OzoneBucket bucket = getBucket(bucketName);
Expand All @@ -312,13 +316,13 @@ public Response listMultipartUploads(
)));
AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction,
getAuditParameters()));
getMetrics().incListMultipartUploadsSuccess();
getMetrics().updateListMultipartUploadsSuccessStats(startNanos);
return Response.ok(result).build();
} catch (OMException exception) {
AUDIT.logReadFailure(
buildAuditMessageForFailure(s3GAction, getAuditParameters(),
exception));
getMetrics().incListMultipartUploadsFailure();
getMetrics().updateListMultipartUploadsFailureStats(startNanos);
if (isAccessDenied(exception)) {
throw newError(S3ErrorTable.ACCESS_DENIED, prefix, exception);
}
Expand All @@ -339,12 +343,13 @@ public Response listMultipartUploads(
@HEAD
public Response head(@PathParam("bucket") String bucketName)
throws OS3Exception, IOException {
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.HEAD_BUCKET;
try {
getBucket(bucketName);
AUDIT.logReadSuccess(
buildAuditMessageForSuccess(s3GAction, getAuditParameters()));
getMetrics().incHeadBucketSuccess();
getMetrics().updateHeadBucketSuccessStats(startNanos);
return Response.ok().build();
} catch (Exception e) {
AUDIT.logReadFailure(
Expand All @@ -362,14 +367,15 @@ public Response head(@PathParam("bucket") String bucketName)
@DELETE
public Response delete(@PathParam("bucket") String bucketName)
throws IOException, OS3Exception {
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.DELETE_BUCKET;

try {
deleteS3Bucket(bucketName);
} catch (OMException ex) {
AUDIT.logWriteFailure(
buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex));
getMetrics().incDeleteBucketFailure();
getMetrics().updateDeleteBucketFailureStats(startNanos);
if (ex.getResult() == ResultCodes.BUCKET_NOT_EMPTY) {
throw newError(S3ErrorTable.BUCKET_NOT_EMPTY, bucketName, ex);
} else if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
Expand All @@ -387,7 +393,7 @@ public Response delete(@PathParam("bucket") String bucketName)

AUDIT.logWriteSuccess(buildAuditMessageForSuccess(s3GAction,
getAuditParameters()));
getMetrics().incDeleteBucketSuccess();
getMetrics().updateDeleteBucketSuccessStats(startNanos);
return Response
.status(HttpStatus.SC_NO_CONTENT)
.build();
Expand Down Expand Up @@ -454,6 +460,7 @@ public MultiDeleteResponse multiDelete(@PathParam("bucket") String bucketName,
*/
public S3BucketAcl getAcl(String bucketName)
throws OS3Exception, IOException {
long startNanos = Time.monotonicNowNanos();
S3BucketAcl result = new S3BucketAcl();
try {
OzoneBucket bucket = getBucket(bucketName);
Expand All @@ -478,7 +485,7 @@ public S3BucketAcl getAcl(String bucketName)
new S3BucketAcl.AccessControlList(grantList));
return result;
} catch (OMException ex) {
getMetrics().incGetAclFailure();
getMetrics().updateGetAclFailureStats(startNanos);
auditReadFailure(S3GAction.GET_ACL, ex);
if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex);
Expand All @@ -488,7 +495,7 @@ public S3BucketAcl getAcl(String bucketName)
throw newError(S3ErrorTable.INTERNAL_ERROR, bucketName, ex);
}
} catch (OS3Exception ex) {
getMetrics().incGetAclFailure();
getMetrics().updateGetAclFailureStats(startNanos);
throw ex;
}
}
Expand All @@ -500,6 +507,7 @@ public S3BucketAcl getAcl(String bucketName)
*/
public Response putAcl(String bucketName, HttpHeaders httpHeaders,
InputStream body) throws IOException, OS3Exception {
long startNanos = Time.monotonicNowNanos();
String grantReads = httpHeaders.getHeaderString(S3Acl.GRANT_READ);
String grantWrites = httpHeaders.getHeaderString(S3Acl.GRANT_WRITE);
String grantReadACP = httpHeaders.getHeaderString(S3Acl.GRANT_READ_CAP);
Expand Down Expand Up @@ -580,7 +588,7 @@ public Response putAcl(String bucketName, HttpHeaders httpHeaders,
volume.addAcl(acl);
}
} catch (OMException exception) {
getMetrics().incPutAclFailure();
getMetrics().updatePutAclFailureStats(startNanos);
auditWriteFailure(S3GAction.PUT_ACL, exception);
if (exception.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, exception);
Expand All @@ -589,10 +597,10 @@ public Response putAcl(String bucketName, HttpHeaders httpHeaders,
}
throw exception;
} catch (OS3Exception ex) {
getMetrics().incPutAclFailure();
getMetrics().updatePutAclFailureStats(startNanos);
throw ex;
}
getMetrics().incPutAclSuccess();
getMetrics().updatePutAclSuccessStats(startNanos);
return Response.status(HttpStatus.SC_OK).build();
}

Expand Down
4 changes: 3 additions & 1 deletion hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@

import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
import org.apache.hadoop.ozone.s3.util.AuditUtils;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -159,10 +160,11 @@ protected OzoneVolume getVolume() throws IOException {
*/
protected String createS3Bucket(String bucketName) throws
IOException, OS3Exception {
long startNanos = Time.monotonicNowNanos();
try {
client.getObjectStore().createS3Bucket(bucketName);
} catch (OMException ex) {
getMetrics().incCreateBucketFailure();
getMetrics().updateCreateBucketFailureStats(startNanos);
if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
} else if (ex.getResult() == ResultCodes.INVALID_TOKEN) {
Expand Down
Loading