From 51898fdbc6b49cd99df5f399409696f2f4371c72 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 15 Dec 2020 16:23:34 +0800 Subject: [PATCH 01/12] HDDS-4585. Support bucket acl operation in S3 gateway. --- .../hadoop/ozone/client/OzoneBucket.java | 4 +- .../hadoop/ozone/client/OzoneVolume.java | 41 +- .../ozone/security/acl/IAccessAuthorizer.java | 1 - .../rpc/TestOzoneRpcClientAbstract.java | 4 +- .../ozone/s3/endpoint/BucketEndpoint.java | 239 +++++++++++- .../endpoint/ListMultipartUploadsResult.java | 53 +-- .../PutBucketAclRequestUnmarshaller.java | 85 +++++ .../hadoop/ozone/s3/endpoint/S3Acl.java | 350 ++++++++++++++++++ .../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 226 +++++++++++ .../hadoop/ozone/s3/endpoint/S3Owner.java | 53 +++ .../ozone/s3/exception/S3ErrorTable.java | 5 + .../hadoop/ozone/client/ObjectStoreStub.java | 2 + .../hadoop/ozone/client/OzoneBucketStub.java | 17 + .../hadoop/ozone/client/OzoneVolumeStub.java | 18 + .../ozone/s3/endpoint/TestBucketAcl.java | 324 ++++++++++++++++ .../ozone/s3/endpoint/TestBucketPut.java | 6 +- .../s3/endpoint/TestPermissionCheck.java | 53 ++- 17 files changed, 1421 insertions(+), 60 deletions(-) create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketAclRequestUnmarshaller.java create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Owner.java create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index f688a661119c..0b025b6b7609 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -386,7 +386,7 @@ public long getQuotaInNamespace() { * for the bucket. * @throws IOException */ - public boolean addAcls(OzoneAcl addAcl) throws IOException { + public boolean addAcl(OzoneAcl addAcl) throws IOException { return proxy.addAcl(ozoneObj, addAcl); } @@ -396,7 +396,7 @@ public boolean addAcls(OzoneAcl addAcl) throws IOException { * removed does not exist for the bucket. * @throws IOException */ - public boolean removeAcls(OzoneAcl removeAcl) throws IOException { + public boolean removeAcl(OzoneAcl removeAcl) throws IOException { return proxy.removeAcl(ozoneObj, removeAcl); } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java index 369500f81f02..992142fcab90 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -35,6 +36,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import org.apache.hadoop.ozone.security.acl.OzoneObj; +import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import static org.apache.hadoop.ozone.OzoneConsts.QUOTA_RESET; @@ -89,6 +92,8 @@ public class OzoneVolume extends WithMetadata { private int listCacheSize; + private OzoneObj ozoneObj; + /** * Constructs OzoneVolume instance. * @param conf Configuration object. @@ -122,6 +127,10 @@ public OzoneVolume(ConfigurationSource conf, ClientProtocol proxy, modificationTime = Instant.ofEpochSecond( this.creationTime.getEpochSecond(), this.creationTime.getNano()); } + this.ozoneObj = OzoneObjInfo.Builder.newBuilder() + .setVolumeName(name) + .setResType(OzoneObj.ResourceType.VOLUME) + .setStoreType(OzoneObj.StoreType.OZONE).build(); } /** @@ -261,7 +270,37 @@ public Instant getModificationTime() { * @return aclMap */ public List getAcls() { - return acls; + return (ArrayList)((ArrayList)acls).clone(); + } + + /** + * Adds ACLs to the volume. + * @param addAcl ACL to be added + * @return true - if acl is successfully added, false if acl already exists + * for the bucket. + * @throws IOException + */ + public boolean addAcl(OzoneAcl addAcl) throws IOException { + boolean added = proxy.addAcl(ozoneObj, addAcl); + if (added) { + acls.add(addAcl); + } + return added; + } + + /** + * Remove acl for Ozone object. Return true if acl is removed successfully + * else false. + * @param acl Ozone acl to be removed. + * + * @throws IOException if there is error. + * */ + public boolean removeAcl(OzoneAcl acl) throws IOException { + boolean removed = proxy.removeAcl(ozoneObj, acl); + if (removed) { + acls.remove(acl); + } + return removed; } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/IAccessAuthorizer.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/IAccessAuthorizer.java index 6e4581e2c330..3523618d66a5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/IAccessAuthorizer.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/IAccessAuthorizer.java @@ -145,7 +145,6 @@ public static String getAclString(ACLType acl) { throw new IllegalArgumentException("ACL right is not recognized"); } } - } /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 277359cfbb3a..840310eb67d4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -649,7 +649,7 @@ public void testAddBucketAcl() acls.add(new OzoneAcl(USER, "test", ACLType.ALL, ACCESS)); OzoneBucket bucket = volume.getBucket(bucketName); for (OzoneAcl acl : acls) { - assertTrue(bucket.addAcls(acl)); + assertTrue(bucket.addAcl(acl)); } OzoneBucket newBucket = volume.getBucket(bucketName); Assert.assertEquals(bucketName, newBucket.getName()); @@ -672,7 +672,7 @@ public void testRemoveBucketAcl() volume.createBucket(bucketName, builder.build()); OzoneBucket bucket = volume.getBucket(bucketName); for (OzoneAcl acl : acls) { - assertTrue(bucket.removeAcls(acl)); + assertTrue(bucket.removeAcl(acl)); } OzoneBucket newBucket = volume.getBucket(bucketName); Assert.assertEquals(bucketName, newBucket.getName()); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index b8bed64cc830..b25f73d5f77b 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.s3.endpoint; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -33,12 +34,19 @@ import javax.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.HashSet; import java.util.Iterator; +import java.util.List; +import java.util.Set; import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; +import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; @@ -49,10 +57,15 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.ozone.s3.util.ContinueToken; import org.apache.hadoop.ozone.s3.util.S3StorageType; +import org.apache.hadoop.ozone.s3.endpoint.S3BucketAcl.Grant; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.lang3.StringUtils; + +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE; + +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -210,9 +223,13 @@ public Response list( } @PUT - public Response put(@PathParam("bucket") String bucketName, @Context - HttpHeaders httpHeaders) throws IOException, OS3Exception { - + public Response put(@PathParam("bucket") String bucketName, + @QueryParam("acl") String aclMarker, + @Context HttpHeaders httpHeaders, + InputStream body) throws IOException, OS3Exception { + if (aclMarker != null) { + return putAcl(bucketName, httpHeaders, body); + } try { String location = createS3Bucket(bucketName); LOG.info("Location is {}", location); @@ -353,6 +370,222 @@ public MultiDeleteResponse multiDelete(@PathParam("bucket") String bucketName, return result; } + /** + * Implement acl get. + *

+ * see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html + */ + @GET + @Produces(MediaType.APPLICATION_XML) + public S3BucketAcl get( + @PathParam("bucket") String bucketName, + @QueryParam("acl") String aclMarker, + @Context HttpServletRequest servletRequest) + throws OS3Exception, IOException { + if (aclMarker != null) { + S3BucketAcl result = new S3BucketAcl(); + try { + OzoneBucket bucket = getBucket(bucketName); + OzoneVolume volume = getVolume(); + S3Owner owner = new S3Owner(volume.getOwner(), volume.getOwner()); + result.setOwner(owner); + // Use set to remove ACLs with different scopes(ACCESS and DEFAULT) + Set grantSet = new HashSet<>(); + // Return ACL list + for (OzoneAcl acl : bucket.getAcls()) { + List grants = S3Acl.OzoneNativeACLToS3ACL(acl); + grantSet.addAll(grants); + } + ArrayList grantList = new ArrayList<>(); + grantList.addAll(grantSet); + result.setAclList( + new S3BucketAcl.AccessControlList(grantList)); + return result; + } catch (OMException ex) { + if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) { + throw S3ErrorTable.newError(S3ErrorTable + .NO_SUCH_BUCKET, bucketName); + } else if (ex.getResult() == ResultCodes.PERMISSION_DENIED) { + throw S3ErrorTable.newError(S3ErrorTable + .ACCESS_DENIED, bucketName); + } else { + LOG.error("Failed to get acl of Bucket " + bucketName, ex); + throw S3ErrorTable.newError(S3ErrorTable.INTERNAL_ERROR, bucketName); + } + } + } + throw S3ErrorTable.newError(NOT_IMPLEMENTED, + servletRequest.getRequestURI()); + } + + /** + * Implement acl put. + *

+ * see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html + */ + public Response putAcl(String bucketName, HttpHeaders httpHeaders, + InputStream body) throws IOException, OS3Exception { + String grantReads = httpHeaders.getHeaderString(S3Acl.grantRead); + String grantWrites = httpHeaders.getHeaderString(S3Acl.grantWrite); + String grantReadACP = httpHeaders.getHeaderString(S3Acl.grantReadACP); + String grantWriteACP = httpHeaders.getHeaderString(S3Acl.grantWriteACP); + String grantFull = httpHeaders.getHeaderString(S3Acl.grantFullControl); + + try { + OzoneBucket bucket = getBucket(bucketName); + OzoneVolume volume = getVolume(); + + List ozoneAclListOnBucket = new ArrayList<>(); + List ozoneAclListOnVolume = new ArrayList<>(); + + if (grantReads == null && grantWrites == null && grantReadACP == null + && grantWriteACP == null && grantFull == null) { + S3BucketAcl putBucketAclRequest = + new PutBucketAclRequestUnmarshaller().readFrom( + null, null, null, null, null, body); + // Handle grants in body + ozoneAclListOnBucket.addAll( + S3Acl.S3ACLToOzoneNativeACLOnBucket(putBucketAclRequest)); + ozoneAclListOnVolume.addAll( + S3Acl.S3ACLToOzoneNativeACLOnVolume(putBucketAclRequest)); + } else { + + // Handle grants in headers + if (grantReads != null) { + ozoneAclListOnBucket.addAll(getAndConvertAclOnBucket(grantReads, + S3Acl.ACLType.READ.getValue())); + ozoneAclListOnVolume.addAll(getAndConvertAclOnVolume(grantReads, + S3Acl.ACLType.READ.getValue())); + } + if (grantWrites != null) { + ozoneAclListOnBucket.addAll(getAndConvertAclOnBucket(grantWrites, + S3Acl.ACLType.WRITE.getValue())); + ozoneAclListOnVolume.addAll(getAndConvertAclOnVolume(grantWrites, + S3Acl.ACLType.WRITE.getValue())); + } + if (grantReadACP != null) { + ozoneAclListOnBucket.addAll(getAndConvertAclOnBucket(grantReadACP, + S3Acl.ACLType.READ_ACP.getValue())); + ozoneAclListOnVolume.addAll(getAndConvertAclOnVolume(grantReadACP, + S3Acl.ACLType.READ_ACP.getValue())); + } + if (grantWriteACP != null) { + ozoneAclListOnBucket.addAll(getAndConvertAclOnBucket(grantWriteACP, + S3Acl.ACLType.WRITE_ACP.getValue())); + ozoneAclListOnVolume.addAll(getAndConvertAclOnVolume(grantWriteACP, + S3Acl.ACLType.WRITE_ACP.getValue())); + } + if (grantFull != null) { + ozoneAclListOnBucket.addAll(getAndConvertAclOnBucket(grantFull, + S3Acl.ACLType.FULL_CONTROL.getValue())); + ozoneAclListOnVolume.addAll(getAndConvertAclOnVolume(grantFull, + S3Acl.ACLType.FULL_CONTROL.getValue())); + } + } + + List oldBucketAcls = bucket.getAcls(); + List oldVolumeAcls = + S3Acl.getVolumeAclFromBucketAcl(oldBucketAcls); + + // Add new ACLs + for (OzoneAcl addAcl : ozoneAclListOnBucket) { + bucket.addAcl(addAcl); + } + for (OzoneAcl addAcl : ozoneAclListOnVolume) { + volume.addAcl(addAcl); + } + + // A put request will reset all previous ACLs + for (OzoneAcl removeAcl : oldBucketAcls) { + bucket.removeAcl(removeAcl); + } + for (OzoneAcl removeAcl : oldVolumeAcls) { + volume.removeAcl(removeAcl); + } + } catch (OMException exception) { + LOG.error("Error in set ACL Request for bucket: {}", bucketName, + exception); + if (exception.getResult() == ResultCodes.BUCKET_NOT_FOUND) { + throw S3ErrorTable.newError(S3ErrorTable.NO_SUCH_BUCKET, + bucketName); + } else if (exception.getResult() == ResultCodes.PERMISSION_DENIED) { + throw S3ErrorTable.newError(S3ErrorTable + .ACCESS_DENIED, bucketName); + } + throw exception; + } + return Response.status(HttpStatus.SC_OK).build(); + } + + /** + * Example: x-amz-grant-write: uri="http://acs.amazonaws.com/groups/s3/LogDelivery", id="111122223333", id="555566667777". + */ + private List getAndConvertAclOnBucket(String value, + String permission) throws OS3Exception { + List ozoneAclList = new ArrayList<>(); + if (StringUtils.isEmpty(value)) { + return ozoneAclList; + } + String[] subValues = value.split(","); + for (String acl: subValues) { + String[] part = acl.split("="); + if (part.length != 2) { + throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, acl); + } + S3Acl.ACLIdentityType type = + S3Acl.ACLIdentityType.getTypeFromHeaderType(part[0]); + if (type == null || !type.isSupported()) { + LOG.warn("S3 grantee {} is null or not supported", part[0]); + throw S3ErrorTable.newError(NOT_IMPLEMENTED, part[0]); + } + // Build ACL on Bucket + BitSet aclsOnBucket = + S3Acl.getOzoneAclOnBucketFromS3Permission(permission); + OzoneAcl defaultOzoneAcl = new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, part[1], aclsOnBucket, + OzoneAcl.AclScope.DEFAULT); + OzoneAcl accessOzoneAcl = new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, part[1], aclsOnBucket, + OzoneAcl.AclScope.ACCESS); + ozoneAclList.add(defaultOzoneAcl); + ozoneAclList.add(accessOzoneAcl); + } + return ozoneAclList; + } + + private List getAndConvertAclOnVolume(String value, + String permission) throws OS3Exception { + List ozoneAclList = new ArrayList<>(); + if (StringUtils.isEmpty(value)) { + return ozoneAclList; + } + String[] subValues = value.split(","); + for (String acl: subValues) { + String[] part = acl.split("="); + if (part.length != 2) { + throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, acl); + } + S3Acl.ACLIdentityType type = + S3Acl.ACLIdentityType.getTypeFromHeaderType(part[0]); + if (type == null || !type.isSupported()) { + LOG.warn("S3 grantee {} is null or not supported", part[0]); + throw S3ErrorTable.newError(NOT_IMPLEMENTED, part[0]); + } + // Build ACL on Bucket + BitSet aclsOnVolume = + S3Acl.getOzoneAclOnVolumeFromS3Permission(permission); + OzoneAcl defaultOzoneAcl = new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, part[1], aclsOnVolume, + OzoneAcl.AclScope.DEFAULT); + OzoneAcl accessOzoneAcl = new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, part[1], aclsOnVolume, + OzoneAcl.AclScope.ACCESS); + ozoneAclList.add(defaultOzoneAcl); + ozoneAclList.add(accessOzoneAcl); + } + return ozoneAclList; + } + private void addKey(ListObjectResponse response, OzoneKey next) { KeyMetadata keyMetadata = new KeyMetadata(); keyMetadata.setKey(next.getName()); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java index 20dc9cdaac1e..46297f732aaa 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java @@ -38,9 +38,6 @@ "http://s3.amazonaws.com/doc/2006-03-01/") public class ListMultipartUploadsResult { - public static final Owner - NOT_SUPPORTED_OWNER = new Owner("NOT-SUPPORTED", "Not Supported"); - @XmlElement(name = "Bucket") private String bucket; @@ -148,10 +145,10 @@ public static class Upload { private String uploadId; @XmlElement(name = "Owner") - private Owner owner = NOT_SUPPORTED_OWNER; + private S3Owner owner = S3Owner.NOT_SUPPORTED_OWNER; @XmlElement(name = "Initiator") - private Owner initiator = NOT_SUPPORTED_OWNER; + private S3Owner initiator = S3Owner.NOT_SUPPORTED_OWNER; @XmlElement(name = "StorageClass") private String storageClass = "STANDARD"; @@ -193,21 +190,21 @@ public void setUploadId(String uploadId) { this.uploadId = uploadId; } - public Owner getOwner() { + public S3Owner getOwner() { return owner; } public void setOwner( - Owner owner) { + S3Owner owner) { this.owner = owner; } - public Owner getInitiator() { + public S3Owner getInitiator() { return initiator; } public void setInitiator( - Owner initiator) { + S3Owner initiator) { this.initiator = initiator; } @@ -227,42 +224,4 @@ public void setInitiated(Instant initiated) { this.initiated = initiated; } } - - /** - * Upload information. - */ - @XmlAccessorType(XmlAccessType.FIELD) - @XmlRootElement(name = "Owner") - public static class Owner { - - @XmlElement(name = "ID") - private String id; - - @XmlElement(name = "DisplayName") - private String displayName; - - public Owner() { - } - - public Owner(String id, String displayName) { - this.id = id; - this.displayName = displayName; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getDisplayName() { - return displayName; - } - - public void setDisplayName(String displayName) { - this.displayName = displayName; - } - } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketAclRequestUnmarshaller.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketAclRequestUnmarshaller.java new file mode 100644 index 000000000000..3ca2e47c469e --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PutBucketAclRequestUnmarshaller.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.endpoint; + +import org.xml.sax.InputSource; +import org.xml.sax.XMLReader; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.MessageBodyReader; +import javax.ws.rs.ext.Provider; +import javax.xml.XMLConstants; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.UnmarshallerHandler; +import javax.xml.parsers.SAXParserFactory; +import java.io.IOException; +import java.io.InputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; + +import static org.apache.hadoop.ozone.s3.util.S3Consts.S3_XML_NAMESPACE; + +/** + * Custom unmarshaller to read PutBucketAclRequest wo namespace. + */ +@Provider +public class PutBucketAclRequestUnmarshaller + implements MessageBodyReader { + + private final JAXBContext context; + private final XMLReader xmlReader; + + public PutBucketAclRequestUnmarshaller() { + try { + context = JAXBContext.newInstance(S3BucketAcl.class); + SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); + saxParserFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + xmlReader = saxParserFactory.newSAXParser().getXMLReader(); + } catch (Exception ex) { + throw new AssertionError("Can not instantiate " + + "PutBucketAclRequest parser", ex); + } + } + @Override + public boolean isReadable(Class aClass, Type type, + Annotation[] annotations, MediaType mediaType) { + return type.equals(S3BucketAcl.class); + } + + @Override + public S3BucketAcl readFrom( + Class aClass, Type type, + Annotation[] annotations, MediaType mediaType, + MultivaluedMap multivaluedMap, + InputStream inputStream) throws IOException, WebApplicationException { + try { + UnmarshallerHandler unmarshallerHandler = + context.createUnmarshaller().getUnmarshallerHandler(); + XmlNamespaceFilter filter = + new XmlNamespaceFilter(S3_XML_NAMESPACE); + filter.setContentHandler(unmarshallerHandler); + filter.setParent(xmlReader); + filter.parse(new InputSource(inputStream)); + return (S3BucketAcl)(unmarshallerHandler.getResult()); + } catch (Exception e) { + throw new WebApplicationException("Can't parse request body to XML.", e); + } + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java new file mode 100644 index 000000000000..e9a802b7c1b5 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java @@ -0,0 +1,350 @@ +package org.apache.hadoop.ozone.s3.endpoint; + +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.s3.endpoint.S3BucketAcl.Grant; +import org.apache.hadoop.ozone.s3.endpoint.S3BucketAcl.Grantee; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; + +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; + +public class S3Acl { + private static final Logger LOG = LoggerFactory.getLogger(S3Acl.class); + + // ACL put related headers + public static final String grantRead = "x-amz-grant-read"; + public static final String grantWrite = "x-amz-grant-write"; + public static final String grantReadACP = "x-amz-grant-read-acp"; + public static final String grantWriteACP = "x-amz-grant-write-acp"; + public static final String grantFullControl = "x-amz-grant-full-control"; + + // Not supported headers at current stage, may support it in future + public static final String cannedAclHeader = "x-amz-acl"; + + /** + * https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html + */ + enum ACLType { + // Allows grantee to list the objects in the bucket + READ("READ"), + // Allows grantee to create, overwrite, and delete any object in the bucket + WRITE("WRITE"), + // Allows grantee to write the bucket ACL + READ_ACP("READ_ACP"), + // Allows grantee to write the ACL for the applicable bucket + WRITE_ACP("WRITE_ACP"), + //Allows grantee the READ, WRITE, READ_ACP, and WRITE_ACP permissions on the bucket + FULL_CONTROL("FULL_CONTROL"); + + public String getValue() { + return value; + } + /** + * String value for this Enum. + */ + private final String value; + + /** + * @param val String type for this enum. + */ + ACLType(String val) { + value = val; + } + } + + enum ACLIdentityType { + USER("CanonicalUser", true, "id"), + GROUP("Group", false, "url"), + USER_BY_EMAIL("AmazonCustomerByEmail", false, "emailAddress"); + + public String getGranteeType() { + return granteeType; + } + + public String getHeaderType() { + return granteeInHeader; + } + + /** + * Grantee type in body XML. + */ + private final String granteeType; + + /** + * Is this type supported or not. + */ + private final boolean supported; + + /** + * Grantee type in header. + */ + private final String granteeInHeader; + + /** + * Init OzoneACLtypes enum. + * + * @param val String type for this enum. + */ + ACLIdentityType(String val, boolean support, String headerType) { + granteeType = val; + supported = support; + granteeInHeader = headerType; + } + + boolean isSupported() { + return supported; + } + + public static ACLIdentityType getTypeFromGranteeType(String typeStr) { + for(ACLIdentityType type: ACLIdentityType.values()) { + if (type.getGranteeType().equals(typeStr)) { + return type; + } + } + return null; + } + + public static ACLIdentityType getTypeFromHeaderType(String typeStr) { + for(ACLIdentityType type: ACLIdentityType.values()) { + if (type.getHeaderType().equals(typeStr)) { + return type; + } + } + return null; + } + } + + public static boolean isGranteeTypeSupported(String typeStr) { + ACLIdentityType type = ACLIdentityType.getTypeFromGranteeType(typeStr); + return type == null ? false : type.isSupported(); + } + + public static boolean isHeaderTypeSupported(String typeStr) { + ACLIdentityType type = ACLIdentityType.getTypeFromHeaderType(typeStr); + return type == null ? false : type.isSupported(); + } + + public static List OzoneNativeACLToS3ACL(OzoneAcl ozoneAcl) { + // Since currently only "CanonicalUser" is supported, which maps to Ozone + // "USER" + List grantList = new ArrayList<>(); + if (ozoneAcl.getType() != IAccessAuthorizer.ACLIdentityType.USER) { + return grantList; + } + + Grantee grantee = new Grantee(); + grantee.setDisplayName(ozoneAcl.getName()); + grantee.setId(ozoneAcl.getName()); + + List acls = ozoneAcl.getAclList(); + if (acls.contains(IAccessAuthorizer.ACLType.ALL)) { + Grant grant = new Grant(); + grant.setGrantee(grantee); + grant.setPermission(ACLType.FULL_CONTROL.toString()); + grantList.add(grant); + return grantList; + } else if (acls.contains(IAccessAuthorizer.ACLType.WRITE_ACL)) { + Grant grant = new Grant(); + grant.setGrantee(grantee); + grant.setPermission(ACLType.WRITE_ACP.toString()); + grantList.add(grant); + } else if (acls.contains(IAccessAuthorizer.ACLType.READ_ACL)) { + Grant grant = new Grant(); + grant.setGrantee(grantee); + grant.setPermission(ACLType.READ_ACP.toString()); + grantList.add(grant); + } else if (acls.contains(IAccessAuthorizer.ACLType.WRITE) && + acls.contains(IAccessAuthorizer.ACLType.DELETE) && + acls.contains(IAccessAuthorizer.ACLType.CREATE)) { + Grant grant = new Grant(); + grant.setGrantee(grantee); + grant.setPermission(ACLType.WRITE.toString()); + grantList.add(grant); + } else if (acls.contains(IAccessAuthorizer.ACLType.READ) && + acls.contains(IAccessAuthorizer.ACLType.LIST)) { + Grant grant = new Grant(); + grant.setGrantee(grantee); + grant.setPermission(ACLType.READ.toString()); + grantList.add(grant); + } else { + LOG.error("Cannot find a good mapping for Ozone ACL {} to S3", + ozoneAcl.toString()); + } + return grantList; + } + + public static List S3ACLToOzoneNativeACLOnBucket( + S3BucketAcl bucketAcl) throws OS3Exception { + List ozoneAclList = new ArrayList<>(); + List grantList = bucketAcl.getAclList().getGrantList(); + for (Grant grant : grantList) { + // Only "CanonicalUser" is supported, which maps to Ozone "USER" + ACLIdentityType identityType = ACLIdentityType.getTypeFromGranteeType( + grant.getGrantee().getXsiType()); + if (identityType != null && identityType.isSupported()) { + String permission = grant.getPermission(); + BitSet acls = getOzoneAclOnBucketFromS3Permission(permission); + OzoneAcl defaultOzoneAcl = new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, + grant.getGrantee().getId(), acls, + OzoneAcl.AclScope.DEFAULT); + OzoneAcl accessOzoneAcl = new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, + grant.getGrantee().getId(), acls, + OzoneAcl.AclScope.ACCESS); + ozoneAclList.add(defaultOzoneAcl); + ozoneAclList.add(accessOzoneAcl); + } else { + LOG.error("Grantee type {} is not supported", + grant.getGrantee().getXsiType()); + throw S3ErrorTable.newError(NOT_IMPLEMENTED, + grant.getGrantee().getXsiType()); + } + } + return ozoneAclList; + } + + public static BitSet getOzoneAclOnBucketFromS3Permission(String permission) + throws OS3Exception { + BitSet acls = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); + switch (permission) { + case "FULL_CONTROL": + acls.set(IAccessAuthorizer.ACLType.ALL.ordinal()); + break; + case "WRITE_ACP": + acls.set(IAccessAuthorizer.ACLType.WRITE_ACL.ordinal()); + break; + case "READ_ACP": + acls.set(IAccessAuthorizer.ACLType.READ_ACL.ordinal()); + break; + case "WRITE": + acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); + acls.set(IAccessAuthorizer.ACLType.DELETE.ordinal()); + acls.set(IAccessAuthorizer.ACLType.CREATE.ordinal()); + break; + case "READ": + acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); + acls.set(IAccessAuthorizer.ACLType.LIST.ordinal()); + break; + default: + LOG.error("Failed to recognize S3 permission {}", permission); + throw S3ErrorTable.newError(INVALID_ARGUMENT, permission); + } + return acls; + } + + public static List S3ACLToOzoneNativeACLOnVolume( + S3BucketAcl bucketAcl) throws OS3Exception { + List ozoneAclList = new ArrayList<>(); + List grantList = bucketAcl.getAclList().getGrantList(); + for (Grant grant : grantList) { + // Only "CanonicalUser" is supported, which maps to Ozone "USER" + ACLIdentityType identityType = ACLIdentityType.getTypeFromGranteeType( + grant.getGrantee().getXsiType()); + if (identityType != null && identityType.isSupported()) { + String permission = grant.getPermission(); + BitSet acls = getOzoneAclOnVolumeFromS3Permission(permission); + OzoneAcl defaultOzoneAcl = new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, + grant.getGrantee().getId(), acls, + OzoneAcl.AclScope.DEFAULT); + OzoneAcl accessOzoneAcl = new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, + grant.getGrantee().getId(), acls, + OzoneAcl.AclScope.ACCESS); + ozoneAclList.add(defaultOzoneAcl); + ozoneAclList.add(accessOzoneAcl); + } else { + LOG.error("Grantee type {} is not supported", + grant.getGrantee().getXsiType()); + throw S3ErrorTable.newError(NOT_IMPLEMENTED, + grant.getGrantee().getXsiType()); + } + } + return ozoneAclList; + } + + public static BitSet getOzoneAclOnVolumeFromS3Permission(String permission) + throws OS3Exception { + BitSet acls = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); + switch (permission) { + case "FULL_CONTROL": + acls.set(IAccessAuthorizer.ACLType.ALL.ordinal()); + break; + case "WRITE_ACP": + acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); + break; + case "READ_ACP": + acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); + break; + case "WRITE": + acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); + break; + case "READ": + acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); + break; + default: + LOG.error("Failed to recognize S3 permission {}", permission); + throw S3ErrorTable.newError(INVALID_ARGUMENT, permission); + } + return acls; + } + + public static List getVolumeAclFromBucketAcl(List aclList) { + List volumeAclList = new ArrayList<>(); + if (aclList == null || aclList.isEmpty()) { + return volumeAclList; + } + for (OzoneAcl bucketAcl: aclList) { + List acls = bucketAcl.getAclList(); + if (acls.contains(IAccessAuthorizer.ACLType.ALL)) { + OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), + bucketAcl.getName(),IAccessAuthorizer.ACLType.ALL, + bucketAcl.getAclScope()); + volumeAclList.add(volumeAcl); + } else if (acls.contains(IAccessAuthorizer.ACLType.WRITE_ACL)) { + OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), + bucketAcl.getName(),IAccessAuthorizer.ACLType.WRITE_ACL, + bucketAcl.getAclScope()); + volumeAclList.add(volumeAcl); + } else if (acls.contains(IAccessAuthorizer.ACLType.READ_ACL)) { + OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), + bucketAcl.getName(),IAccessAuthorizer.ACLType.READ_ACL, + bucketAcl.getAclScope()); + volumeAclList.add(volumeAcl); + } else if (acls.contains(IAccessAuthorizer.ACLType.WRITE) && + acls.contains(IAccessAuthorizer.ACLType.DELETE) && + acls.contains(IAccessAuthorizer.ACLType.CREATE)) { + BitSet aclBits = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); + aclBits.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); + aclBits.set(IAccessAuthorizer.ACLType.DELETE.ordinal()); + aclBits.set(IAccessAuthorizer.ACLType.CREATE.ordinal()); + + OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), + bucketAcl.getName(),aclBits, bucketAcl.getAclScope()); + volumeAclList.add(volumeAcl); + } else if (acls.contains(IAccessAuthorizer.ACLType.READ) && + acls.contains(IAccessAuthorizer.ACLType.LIST)) { + BitSet aclBits = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); + aclBits.set(IAccessAuthorizer.ACLType.READ.ordinal()); + aclBits.set(IAccessAuthorizer.ACLType.LIST.ordinal()); + + OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), + bucketAcl.getName(),aclBits, bucketAcl.getAclScope()); + volumeAclList.add(volumeAcl); + } else { + LOG.error("Cannot find a good mapping for Ozone ACL {} to S3", + bucketAcl.toString()); + } + } + return volumeAclList; + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java new file mode 100644 index 000000000000..69d61161438b --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.endpoint; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Bucket ACL. + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "AccessControlPolicy", + namespace = "http://s3.amazonaws.com/doc/2006-03-01/") +public class S3BucketAcl { + + @XmlElement(name = "Owner") + private S3Owner owner; + + @XmlElement(name = "AccessControlList") + private AccessControlList aclList; + + public S3Owner getOwner() { + return owner; + } + + public void setOwner(S3Owner owner) { + this.owner = owner; + } + + public AccessControlList getAclList() { + return aclList; + } + + public void setAclList(AccessControlList aclList) { + this.aclList = aclList; + } + + @Override + public String toString() { + return "GetBucketAclResponse{" + + "owner=" + owner + + ", aclList=" + aclList + + '}'; + } + + @XmlAccessorType(XmlAccessType.FIELD) + @XmlRootElement(name = "AccessControlList") + public static class AccessControlList { + + @XmlElement(name = "Grant") + private List grantList = new ArrayList<>(); + + public void addGrant(Grant grant) { + grantList.add(grant); + } + + public List getGrantList() { + return grantList; + } + + public AccessControlList(List grants) { + this.grantList = grants; + } + + public AccessControlList() { + + } + + @Override + public String toString() { + return "AccessControlList{" + + "grantList=" + grantList + + '}'; + } + } + + @XmlAccessorType(XmlAccessType.FIELD) + @XmlRootElement(name = "Grant") + public static class Grant { + + @XmlElement(name = "Grantee") + private Grantee grantee; + + @XmlElement(name = "Permission") + private String permission; + + public String getPermission() { + return permission; + } + + public void setPermission(String permission) { + this.permission = permission; + } + + public Grantee getGrantee() { + return grantee; + } + + public void setGrantee(Grantee grantee) { + this.grantee = grantee; + } + + @Override + public String toString() { + return "Grant{" + + "grantee=" + grantee + + ", permission='" + permission + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Grant grant = (Grant) o; + return Objects.equals(grantee, grant.grantee) && + Objects.equals(permission, grant.permission); + } + + @Override + public int hashCode() { + return Objects.hash(grantee, permission); + } + } + + /** + * A grantee can be an AWS account or one of the predefined Amazon S3 groups. + * https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlRootElement(name = "Grantee") + public static class Grantee { + + @XmlElement(name = "DisplayName") + private String displayName; + + @XmlElement(name = "ID") + private String id; + + @XmlAttribute(name="xsi:type") + private String xsiType = "CanonicalUser"; + + @XmlAttribute(name="xmlns:xsi") + private String xsiNs = "http://www.w3.org/2001/XMLSchema-instance"; + + public String getXsiNs() { + return xsiNs; + } + + public String getXsiType() { + return xsiType; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String name) { + this.displayName = name; + } + + public void setXsiType(String type) { + this.xsiType = type; + } + + public void setXsiNs(String ns) { + this.xsiNs = ns; + } + + @Override + public String toString() { + return "Grantee{" + + "displayName='" + displayName + '\'' + + ", id='" + id + '\'' + + ", xsiType='" + xsiType + '\'' + + ", xsiNs='" + xsiNs + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Grantee grantee = (Grantee) o; + return Objects.equals(displayName, grantee.displayName) && + Objects.equals(id, grantee.id) && + Objects.equals(xsiType, grantee.xsiType) && + Objects.equals(xsiNs, grantee.xsiNs); + } + + @Override + public int hashCode() { + return Objects.hash(displayName, id, xsiType, xsiNs); + } + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Owner.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Owner.java new file mode 100644 index 000000000000..6756db856205 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Owner.java @@ -0,0 +1,53 @@ +package org.apache.hadoop.ozone.s3.endpoint; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "Owner") +public class S3Owner { + + public static final S3Owner + NOT_SUPPORTED_OWNER = new S3Owner("NOT-SUPPORTED", "Not Supported"); + + @XmlElement(name = "DisplayName") + private String displayName; + + @XmlElement(name = "ID") + private String id; + + public S3Owner() { + + } + + public S3Owner(String id, String displayName) { + this.id = id; + this.displayName = displayName; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String name) { + this.displayName = name; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + @Override + public String toString() { + return "S3Owner{" + + "displayName='" + displayName + '\'' + + ", id='" + id + '\'' + + '}'; + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java index 7d9b55fb4165..86d9fc049cd8 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java @@ -25,6 +25,7 @@ import static java.net.HttpURLConnection.HTTP_FORBIDDEN; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_PRECON_FAILED; +import static java.net.HttpURLConnection.HTTP_NOT_IMPLEMENTED; import static java.net.HttpURLConnection.HTTP_SERVER_ERROR; import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_NOT_SATISFIABLE; @@ -114,6 +115,10 @@ private S3ErrorTable() { public static final OS3Exception PRECOND_FAILED = new OS3Exception( "PreconditionFailed", "At least one of the pre-conditions you " + "specified did not hold", HTTP_PRECON_FAILED); + + public static final OS3Exception NOT_IMPLEMENTED = new OS3Exception( + "NotImplemented", "This part of feature is not implemented yet.", + HTTP_NOT_IMPLEMENTED); /** * Create a new instance of Error. diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java index 74ebf4ee9939..af703188fe13 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java @@ -23,11 +23,13 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; +import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.util.Time; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 7861bd4aad5d..e567d0a9c2db 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts.PartInfo; @@ -57,6 +58,8 @@ public class OzoneBucketStub extends OzoneBucket { private Map> partList = new HashMap<>(); + private ArrayList aclList = new ArrayList<>(); + /** * Constructs OzoneBucket instance. * @@ -291,7 +294,21 @@ public OzoneMultipartUploadPartListParts listParts(String key, return ozoneMultipartUploadPartListParts; } + } + + @Override + public List getAcls() throws IOException { + return (List)aclList.clone(); + } + @Override + public boolean removeAcl(OzoneAcl removeAcl) throws IOException { + return aclList.remove(removeAcl); + } + + @Override + public boolean addAcl(OzoneAcl addAcl) throws IOException { + return aclList.add(addAcl); } /** diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java index f5e06035fed6..85c805469d23 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -38,6 +39,8 @@ public class OzoneVolumeStub extends OzoneVolume { private Map buckets = new HashMap<>(); + private ArrayList aclList = new ArrayList<>(); + public OzoneVolumeStub(String name, String admin, String owner, long quotaInBytes, long quotaInNamespace, long creationTime, List acls) { @@ -107,4 +110,19 @@ public void deleteBucket(String bucketName) throws IOException { throw new OMException("", OMException.ResultCodes.BUCKET_NOT_FOUND); } } + + @Override + public List getAcls() { + return (List)aclList.clone(); + } + + @Override + public boolean addAcl(OzoneAcl addAcl) throws IOException { + return aclList.add(addAcl); + } + + @Override + public boolean removeAcl(OzoneAcl acl) throws IOException { + return aclList.add(acl); + } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java new file mode 100644 index 000000000000..a09ef54403ce --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.hadoop.ozone.s3.endpoint; + +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.Response; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Map; + +import static java.net.HttpURLConnection.HTTP_FORBIDDEN; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static java.net.HttpURLConnection.HTTP_NOT_IMPLEMENTED; +import static java.net.HttpURLConnection.HTTP_OK; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + +/** + * This class tests Bucket ACL get/set request. + */ +public class TestBucketAcl { + + private final String bucketName = OzoneConsts.S3_BUCKET; + private OzoneClient client; + + private HttpServletRequest servletRequest; + private Map parameterMap; + private HttpHeaders headers; + private BucketEndpoint bucketEndpoint; + private final String aclMarker = "acl"; + + @Before + public void setup() throws IOException { + client = new OzoneClientStub(); + client.getObjectStore().createS3Bucket(bucketName); + + servletRequest = Mockito.mock(HttpServletRequest.class); + parameterMap = Mockito.mock(Map.class); + headers = Mockito.mock(HttpHeaders.class); + when(servletRequest.getParameterMap()).thenReturn(parameterMap); + + bucketEndpoint = new BucketEndpoint(); + bucketEndpoint.setClient(client); + } + + @After + public void clean() throws IOException { + if (client != null) { + client.close(); + } + } + + @Test + public void testGetAcl() throws Exception { + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + S3BucketAcl response = + bucketEndpoint.get(bucketName, aclMarker, servletRequest); + Assert.assertTrue(response.getAclList() != null); + System.out.println(response.toString()); + } + + @Test + public void testSetAclWithNotSupportedGranteeType() throws Exception { + when(headers.getHeaderString(S3Acl.grantRead)) + .thenReturn(S3Acl.ACLIdentityType.GROUP.getHeaderType() + "=root" ); + + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + try { + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, null); + } catch (Exception e) { + Assert.assertTrue(e instanceof OS3Exception && + ((OS3Exception) e).getHttpCode() == HTTP_NOT_IMPLEMENTED); + } + } + + @Test + public void testRead() throws Exception { + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantRead)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, null); + assertEquals(HTTP_OK, response.getStatus()); + S3BucketAcl getResponse = + bucketEndpoint.get(bucketName, aclMarker, servletRequest); + assertEquals(1, getResponse.getAclList().getGrantList().size()); + assertEquals(S3Acl.ACLType.READ.getValue(), + getResponse.getAclList().getGrantList().get(0).getPermission()); + } + + @Test + public void testWrite() throws Exception { + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantWrite)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, null); + assertEquals(HTTP_OK, response.getStatus()); + S3BucketAcl getResponse = + bucketEndpoint.get(bucketName, aclMarker, servletRequest); + assertEquals(1, getResponse.getAclList().getGrantList().size()); + assertEquals(S3Acl.ACLType.WRITE.getValue(), + getResponse.getAclList().getGrantList().get(0).getPermission()); + } + + @Test + public void testReadACP() throws Exception { + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantReadACP)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, null); + assertEquals(HTTP_OK, response.getStatus()); + S3BucketAcl getResponse = + bucketEndpoint.get(bucketName, aclMarker, servletRequest); + assertEquals(1, getResponse.getAclList().getGrantList().size()); + assertEquals(S3Acl.ACLType.READ_ACP.getValue(), + getResponse.getAclList().getGrantList().get(0).getPermission()); + } + + @Test + public void testWriteACP() throws Exception { + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantWriteACP)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, null); + assertEquals(HTTP_OK, response.getStatus()); + S3BucketAcl getResponse = + bucketEndpoint.get(bucketName, aclMarker, servletRequest); + assertEquals(1, getResponse.getAclList().getGrantList().size()); + assertEquals(S3Acl.ACLType.WRITE_ACP.getValue(), + getResponse.getAclList().getGrantList().get(0).getPermission()); + } + + @Test + public void testFullControl() throws Exception { + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantFullControl)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root" ); + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, null); + assertEquals(HTTP_OK, response.getStatus()); + S3BucketAcl getResponse = + bucketEndpoint.get(bucketName, aclMarker, servletRequest); + assertEquals(1, getResponse.getAclList().getGrantList().size()); + assertEquals(S3Acl.ACLType.FULL_CONTROL.getValue(), + getResponse.getAclList().getGrantList().get(0).getPermission()); + } + + @Test + public void testCombination() throws Exception { + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantRead)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + when(headers.getHeaderString(S3Acl.grantWrite)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + when(headers.getHeaderString(S3Acl.grantReadACP)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + when(headers.getHeaderString(S3Acl.grantWriteACP)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + when(headers.getHeaderString(S3Acl.grantFullControl)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root" ); + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, null); + assertEquals(HTTP_OK, response.getStatus()); + S3BucketAcl getResponse = + bucketEndpoint.get(bucketName, aclMarker, servletRequest); + assertEquals(5, getResponse.getAclList().getGrantList().size()); + } + + @Test + public void testPutClearOldAcls() throws Exception { + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantRead)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, null); + assertEquals(HTTP_OK, response.getStatus()); + S3BucketAcl getResponse = + bucketEndpoint.get(bucketName, aclMarker, servletRequest); + assertEquals(1, getResponse.getAclList().getGrantList().size()); + when(headers.getHeaderString(S3Acl.grantRead)) + .thenReturn(null); + when(headers.getHeaderString(S3Acl.grantWrite)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + response = + bucketEndpoint.put(bucketName, aclMarker, headers, null); + assertEquals(HTTP_OK, response.getStatus()); + getResponse = bucketEndpoint.get(bucketName, aclMarker, servletRequest); + assertEquals(1, getResponse.getAclList().getGrantList().size()); + assertEquals(S3Acl.ACLType.WRITE.getValue(), + getResponse.getAclList().getGrantList().get(0).getPermission()); + } + + @Test(expected = OS3Exception.class) + public void testAclInBodyWithGroupUser() throws Exception { + ByteArrayInputStream inputBody = + new ByteArrayInputStream( + ("\n" + + " \n" + + " 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID\n" + + " owner\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID\n" + + " owner\n" + + " \n" + + " FULL_CONTROL\n" + + " \n" + + " \n" + + " \n" + + " http://acs.amazonaws.com/groups/global/AllUsers\n" + + " \n" + + " READ\n" + + " \n" + + " \n" + + " \n" + + " http://acs.amazonaws.com/groups/s3/LogDelivery\n" + + " \n" + + " WRITE\n" + + " \n" + + " \n" + + " \n" + + " xyz@amazon.com\n" + + " \n" + + " WRITE_ACP\n" + + " \n" + + " \n" + + " \n" + + " f30716ab7115dcb44a5ef76e9d74b8e20567f63TestAccountCanonicalUserID\n" + + " \n" + + " READ_ACP\n" + + " \n" + + " \n" + + "\n").getBytes(UTF_8)); + + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, inputBody); + } + + @Test + public void testAclInBody() throws Exception { + ByteArrayInputStream inputBody = + new ByteArrayInputStream( + ("\n" + + " \n" + + " 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID\n" + + " owner\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID\n" + + " owner\n" + + " \n" + + " FULL_CONTROL\n" + + " \n" + + " \n" + + " \n" + + " f30716ab7115dcb44a5ef76e9d74b8e20567f63TestAccountCanonicalUserID\n" + + " \n" + + " READ_ACP\n" + + " \n" + + " \n" + + "\n").getBytes(UTF_8)); + + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + Response response = + bucketEndpoint.put(bucketName, aclMarker, headers, inputBody); + assertEquals(HTTP_OK, response.getStatus()); + S3BucketAcl getResponse = + bucketEndpoint.get(bucketName, aclMarker, servletRequest); + assertEquals(2, getResponse.getAclList().getGrantList().size()); + } + + @Test + public void testBucketNotExist() throws Exception { + when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantRead)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + try { + bucketEndpoint.get("bucket-not-exist", aclMarker, servletRequest); + } catch (Exception e) { + Assert.assertTrue(e instanceof OS3Exception && + ((OS3Exception)e).getHttpCode() == HTTP_NOT_FOUND); + } + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java index 738bb1d11447..dd155186e483 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java @@ -61,7 +61,7 @@ public void setup() throws Exception { public void testBucketFailWithAuthHeaderMissing() throws Exception { try { - bucketEndpoint.put(bucketName, null); + bucketEndpoint.put(bucketName, null, null, null); } catch (OS3Exception ex) { Assert.assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); Assert.assertEquals(MALFORMED_HEADER.getCode(), ex.getCode()); @@ -70,7 +70,7 @@ public void testBucketFailWithAuthHeaderMissing() throws Exception { @Test public void testBucketPut() throws Exception { - Response response = bucketEndpoint.put(bucketName, null); + Response response = bucketEndpoint.put(bucketName, null, null, null); assertEquals(200, response.getStatus()); assertNotNull(response.getLocation()); } @@ -78,7 +78,7 @@ public void testBucketPut() throws Exception { @Test public void testBucketFailWithInvalidHeader() throws Exception { try { - bucketEndpoint.put(bucketName, null); + bucketEndpoint.put(bucketName, null, null, null); } catch (OS3Exception ex) { Assert.assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); Assert.assertEquals(MALFORMED_HEADER.getCode(), ex.getCode()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index 1c5622e1baab..3c2590f66125 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -32,16 +32,19 @@ import org.junit.Test; import org.mockito.Mockito; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.HttpHeaders; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import static java.net.HttpURLConnection.HTTP_FORBIDDEN; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; /** * Test operation permission check result. @@ -115,7 +118,7 @@ public void testCreateBucket() throws IOException { bucketEndpoint.setClient(client); try { - bucketEndpoint.put("bucketName", null); + bucketEndpoint.put("bucketName", null, null, null); Assert.fail("Should fail"); } catch (Exception e) { Assert.assertTrue(e instanceof OS3Exception); @@ -191,6 +194,54 @@ public void testDeleteKeys() throws IOException, OS3Exception { response.getErrors().get(0).getCode().equals("PermissionDenied")); } + @Test + public void testGetAcl() throws Exception { + Mockito.when(objectStore.getVolume(anyString())).thenReturn(volume); + Mockito.when(objectStore.getS3Bucket(anyString())).thenReturn(bucket); + doThrow(exception).when(bucket).getAcls(); + + HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class); + Map parameterMap = Mockito.mock(Map.class); + HttpHeaders headers = Mockito.mock(HttpHeaders.class); + when(servletRequest.getParameterMap()).thenReturn(parameterMap); + + when(parameterMap.containsKey("acl")).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantRead)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + BucketEndpoint bucketEndpoint = new BucketEndpoint(); + bucketEndpoint.setClient(client); + try { + bucketEndpoint.get("bucketName", "acl", servletRequest); + } catch (Exception e) { + Assert.assertTrue(e instanceof OS3Exception && + ((OS3Exception)e).getHttpCode() == HTTP_FORBIDDEN); + } + } + + @Test + public void testSetAcl() throws Exception { + Mockito.when(objectStore.getVolume(anyString())).thenReturn(volume); + Mockito.when(objectStore.getS3Bucket(anyString())).thenReturn(bucket); + doThrow(exception).when(bucket).addAcl(any()); + + HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class); + Map parameterMap = Mockito.mock(Map.class); + HttpHeaders headers = Mockito.mock(HttpHeaders.class); + when(servletRequest.getParameterMap()).thenReturn(parameterMap); + + when(parameterMap.containsKey("acl")).thenReturn(true); + when(headers.getHeaderString(S3Acl.grantRead)) + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + BucketEndpoint bucketEndpoint = new BucketEndpoint(); + bucketEndpoint.setClient(client); + try { + bucketEndpoint.put("bucketName", "acl", headers, null); + } catch (Exception e) { + Assert.assertTrue(e instanceof OS3Exception && + ((OS3Exception)e).getHttpCode() == HTTP_FORBIDDEN); + } + } + /** * Object Endpoint. */ From 3756862901dd42c6342efc1d3d5456e74b6e1859 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 15 Dec 2020 20:44:03 +0800 Subject: [PATCH 02/12] fix rat and some checkstyle --- .../ozone/s3/endpoint/BucketEndpoint.java | 10 ++++--- .../hadoop/ozone/s3/endpoint/S3Acl.java | 29 +++++++++++++++---- .../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 8 +++-- .../hadoop/ozone/s3/endpoint/S3Owner.java | 19 ++++++++++++ .../hadoop/ozone/client/ObjectStoreStub.java | 2 -- .../ozone/s3/endpoint/TestBucketAcl.java | 2 -- .../s3/endpoint/TestPermissionCheck.java | 2 -- 7 files changed, 55 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index b25f73d5f77b..e1a563cc785a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -393,7 +393,7 @@ public S3BucketAcl get( Set grantSet = new HashSet<>(); // Return ACL list for (OzoneAcl acl : bucket.getAcls()) { - List grants = S3Acl.OzoneNativeACLToS3ACL(acl); + List grants = S3Acl.ozoneNativeAclToS3Acl(acl); grantSet.addAll(grants); } ArrayList grantList = new ArrayList<>(); @@ -445,9 +445,9 @@ public Response putAcl(String bucketName, HttpHeaders httpHeaders, null, null, null, null, null, body); // Handle grants in body ozoneAclListOnBucket.addAll( - S3Acl.S3ACLToOzoneNativeACLOnBucket(putBucketAclRequest)); + S3Acl.s3AclToOzoneNativeAclOnBucket(putBucketAclRequest)); ozoneAclListOnVolume.addAll( - S3Acl.S3ACLToOzoneNativeACLOnVolume(putBucketAclRequest)); + S3Acl.s3AclToOzoneNativeAclOnVolume(putBucketAclRequest)); } else { // Handle grants in headers @@ -518,7 +518,9 @@ public Response putAcl(String bucketName, HttpHeaders httpHeaders, } /** - * Example: x-amz-grant-write: uri="http://acs.amazonaws.com/groups/s3/LogDelivery", id="111122223333", id="555566667777". + * Example: x-amz-grant-write: \ + * uri="http://acs.amazonaws.com/groups/s3/LogDelivery", id="111122223333", \ + * id="555566667777". */ private List getAndConvertAclOnBucket(String value, String permission) throws OS3Exception { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java index e9a802b7c1b5..c362e99ef379 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ package org.apache.hadoop.ozone.s3.endpoint; import org.apache.hadoop.ozone.OzoneAcl; @@ -16,7 +35,7 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; -public class S3Acl { +public final class S3Acl { private static final Logger LOG = LoggerFactory.getLogger(S3Acl.class); // ACL put related headers @@ -30,7 +49,7 @@ public class S3Acl { public static final String cannedAclHeader = "x-amz-acl"; /** - * https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html + * https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html. */ enum ACLType { // Allows grantee to list the objects in the bucket @@ -132,7 +151,7 @@ public static boolean isHeaderTypeSupported(String typeStr) { return type == null ? false : type.isSupported(); } - public static List OzoneNativeACLToS3ACL(OzoneAcl ozoneAcl) { + public static List ozoneNativeAclToS3Acl(OzoneAcl ozoneAcl) { // Since currently only "CanonicalUser" is supported, which maps to Ozone // "USER" List grantList = new ArrayList<>(); @@ -181,7 +200,7 @@ public static List OzoneNativeACLToS3ACL(OzoneAcl ozoneAcl) { return grantList; } - public static List S3ACLToOzoneNativeACLOnBucket( + public static List s3AclToOzoneNativeAclOnBucket( S3BucketAcl bucketAcl) throws OS3Exception { List ozoneAclList = new ArrayList<>(); List grantList = bucketAcl.getAclList().getGrantList(); @@ -241,7 +260,7 @@ public static BitSet getOzoneAclOnBucketFromS3Permission(String permission) return acls; } - public static List S3ACLToOzoneNativeACLOnVolume( + public static List s3AclToOzoneNativeAclOnVolume( S3BucketAcl bucketAcl) throws OS3Exception { List ozoneAclList = new ArrayList<>(); List grantList = bucketAcl.getAclList().getGrantList(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java index 69d61161438b..9f61350b0d29 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java @@ -132,8 +132,12 @@ public String toString() { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } Grant grant = (Grant) o; return Objects.equals(grantee, grant.grantee) && Objects.equals(permission, grant.permission); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Owner.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Owner.java index 6756db856205..48f154e53bef 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Owner.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Owner.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ package org.apache.hadoop.ozone.s3.endpoint; import javax.xml.bind.annotation.XmlAccessType; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java index af703188fe13..74ebf4ee9939 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java @@ -23,13 +23,11 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; -import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.util.Time; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index a09ef54403ce..75565fc8aa3c 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.util.Map; -import static java.net.HttpURLConnection.HTTP_FORBIDDEN; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_NOT_IMPLEMENTED; import static java.net.HttpURLConnection.HTTP_OK; @@ -94,7 +93,6 @@ public void testGetAcl() throws Exception { public void testSetAclWithNotSupportedGranteeType() throws Exception { when(headers.getHeaderString(S3Acl.grantRead)) .thenReturn(S3Acl.ACLIdentityType.GROUP.getHeaderType() + "=root" ); - when(parameterMap.containsKey(aclMarker)).thenReturn(true); try { Response response = diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index 3c2590f66125..b7960651f0ce 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -202,7 +202,6 @@ public void testGetAcl() throws Exception { HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class); Map parameterMap = Mockito.mock(Map.class); - HttpHeaders headers = Mockito.mock(HttpHeaders.class); when(servletRequest.getParameterMap()).thenReturn(parameterMap); when(parameterMap.containsKey("acl")).thenReturn(true); @@ -226,7 +225,6 @@ public void testSetAcl() throws Exception { HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class); Map parameterMap = Mockito.mock(Map.class); - HttpHeaders headers = Mockito.mock(HttpHeaders.class); when(servletRequest.getParameterMap()).thenReturn(parameterMap); when(parameterMap.containsKey("acl")).thenReturn(true); From 6d8a8499ccc0e90a51b830bd173ee41fc98c0d85 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 22 Dec 2020 20:23:50 +0800 Subject: [PATCH 03/12] checkstyle fix --- .../hadoop/ozone/s3/endpoint/S3Acl.java | 57 ++++++++++++------- .../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 9 ++- .../ozone/s3/endpoint/TestBucketAcl.java | 6 +- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java index c362e99ef379..b31210e61cb7 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java @@ -60,7 +60,7 @@ enum ACLType { READ_ACP("READ_ACP"), // Allows grantee to write the ACL for the applicable bucket WRITE_ACP("WRITE_ACP"), - //Allows grantee the READ, WRITE, READ_ACP, and WRITE_ACP permissions on the bucket + // Allows grantee above all permissions on the bucket FULL_CONTROL("FULL_CONTROL"); public String getValue() { @@ -77,6 +77,16 @@ public String getValue() { ACLType(String val) { value = val; } + + + public static ACLType getType(String typeStr) { + for(ACLType type: ACLType.values()) { + if (type.getValue().equals(typeStr)) { + return type; + } + } + return null; + } } enum ACLIdentityType { @@ -233,23 +243,27 @@ public static List s3AclToOzoneNativeAclOnBucket( public static BitSet getOzoneAclOnBucketFromS3Permission(String permission) throws OS3Exception { + ACLType permissionType = ACLType.getType(permission); + if (permissionType == null) { + throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, permission); + } BitSet acls = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); - switch (permission) { - case "FULL_CONTROL": + switch (permissionType) { + case FULL_CONTROL: acls.set(IAccessAuthorizer.ACLType.ALL.ordinal()); break; - case "WRITE_ACP": + case WRITE_ACP: acls.set(IAccessAuthorizer.ACLType.WRITE_ACL.ordinal()); break; - case "READ_ACP": + case READ_ACP: acls.set(IAccessAuthorizer.ACLType.READ_ACL.ordinal()); break; - case "WRITE": + case WRITE: acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); acls.set(IAccessAuthorizer.ACLType.DELETE.ordinal()); acls.set(IAccessAuthorizer.ACLType.CREATE.ordinal()); break; - case "READ": + case READ: acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); acls.set(IAccessAuthorizer.ACLType.LIST.ordinal()); break; @@ -294,20 +308,24 @@ public static List s3AclToOzoneNativeAclOnVolume( public static BitSet getOzoneAclOnVolumeFromS3Permission(String permission) throws OS3Exception { BitSet acls = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); - switch (permission) { - case "FULL_CONTROL": + ACLType permissionType = ACLType.getType(permission); + if (permissionType == null) { + throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, permission); + } + switch (permissionType) { + case FULL_CONTROL: acls.set(IAccessAuthorizer.ACLType.ALL.ordinal()); break; - case "WRITE_ACP": + case WRITE_ACP: acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); break; - case "READ_ACP": + case READ_ACP: acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); break; - case "WRITE": + case WRITE: acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); break; - case "READ": + case READ: acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); break; default: @@ -317,7 +335,8 @@ public static BitSet getOzoneAclOnVolumeFromS3Permission(String permission) return acls; } - public static List getVolumeAclFromBucketAcl(List aclList) { + public static List getVolumeAclFromBucketAcl( + List aclList) { List volumeAclList = new ArrayList<>(); if (aclList == null || aclList.isEmpty()) { return volumeAclList; @@ -326,17 +345,17 @@ public static List getVolumeAclFromBucketAcl(List aclList) { List acls = bucketAcl.getAclList(); if (acls.contains(IAccessAuthorizer.ACLType.ALL)) { OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(),IAccessAuthorizer.ACLType.ALL, + bucketAcl.getName(), IAccessAuthorizer.ACLType.ALL, bucketAcl.getAclScope()); volumeAclList.add(volumeAcl); } else if (acls.contains(IAccessAuthorizer.ACLType.WRITE_ACL)) { OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(),IAccessAuthorizer.ACLType.WRITE_ACL, + bucketAcl.getName(), IAccessAuthorizer.ACLType.WRITE_ACL, bucketAcl.getAclScope()); volumeAclList.add(volumeAcl); } else if (acls.contains(IAccessAuthorizer.ACLType.READ_ACL)) { OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(),IAccessAuthorizer.ACLType.READ_ACL, + bucketAcl.getName(), IAccessAuthorizer.ACLType.READ_ACL, bucketAcl.getAclScope()); volumeAclList.add(volumeAcl); } else if (acls.contains(IAccessAuthorizer.ACLType.WRITE) && @@ -348,7 +367,7 @@ public static List getVolumeAclFromBucketAcl(List aclList) { aclBits.set(IAccessAuthorizer.ACLType.CREATE.ordinal()); OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(),aclBits, bucketAcl.getAclScope()); + bucketAcl.getName(), aclBits, bucketAcl.getAclScope()); volumeAclList.add(volumeAcl); } else if (acls.contains(IAccessAuthorizer.ACLType.READ) && acls.contains(IAccessAuthorizer.ACLType.LIST)) { @@ -357,7 +376,7 @@ public static List getVolumeAclFromBucketAcl(List aclList) { aclBits.set(IAccessAuthorizer.ACLType.LIST.ordinal()); OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(),aclBits, bucketAcl.getAclScope()); + bucketAcl.getName(), aclBits, bucketAcl.getAclScope()); volumeAclList.add(volumeAcl); } else { LOG.error("Cannot find a good mapping for Ozone ACL {} to S3", diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java index 9f61350b0d29..7a699e482744 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3BucketAcl.java @@ -213,8 +213,13 @@ public String toString() { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Grantee grantee = (Grantee) o; return Objects.equals(displayName, grantee.displayName) && Objects.equals(id, grantee.id) && diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index 75565fc8aa3c..2ea27492e262 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -92,7 +92,7 @@ public void testGetAcl() throws Exception { @Test public void testSetAclWithNotSupportedGranteeType() throws Exception { when(headers.getHeaderString(S3Acl.grantRead)) - .thenReturn(S3Acl.ACLIdentityType.GROUP.getHeaderType() + "=root" ); + .thenReturn(S3Acl.ACLIdentityType.GROUP.getHeaderType() + "=root"); when(parameterMap.containsKey(aclMarker)).thenReturn(true); try { Response response = @@ -167,7 +167,7 @@ public void testWriteACP() throws Exception { public void testFullControl() throws Exception { when(parameterMap.containsKey(aclMarker)).thenReturn(true); when(headers.getHeaderString(S3Acl.grantFullControl)) - .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root" ); + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); @@ -190,7 +190,7 @@ public void testCombination() throws Exception { when(headers.getHeaderString(S3Acl.grantWriteACP)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); when(headers.getHeaderString(S3Acl.grantFullControl)) - .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root" ); + .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); From ee06082dd626169e88a510a188625a32363f0133 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Fri, 25 Dec 2020 17:31:59 +0800 Subject: [PATCH 04/12] Fix acceptance tests --- .../ozone/s3/endpoint/BucketEndpoint.java | 75 +++++++++--------- .../ozone/s3/endpoint/TestBucketAcl.java | 36 ++++----- ...TestBucketGet.java => TestBucketList.java} | 76 +++++++++---------- .../s3/endpoint/TestPermissionCheck.java | 7 +- 4 files changed, 93 insertions(+), 101 deletions(-) rename hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/{TestBucketGet.java => TestBucketList.java} (80%) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index e1a563cc785a..81935a1f68f5 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -88,7 +88,7 @@ public class BucketEndpoint extends EndpointBase { @GET @SuppressFBWarnings @SuppressWarnings("parameternumber") - public Response list( + public Response get( @PathParam("bucket") String bucketName, @QueryParam("delimiter") String delimiter, @QueryParam("encoding-type") String encodingType, @@ -99,8 +99,14 @@ public Response list( @QueryParam("continuation-token") String continueToken, @QueryParam("start-after") String startAfter, @QueryParam("uploads") String uploads, + @QueryParam("acl") String aclMarker, @Context HttpHeaders hh) throws OS3Exception, IOException { + if (aclMarker != null) { + S3BucketAcl result = getAcl(bucketName); + return Response.ok(result, MediaType.APPLICATION_XML_TYPE).build(); + } + if (browser != null) { InputStream browserPage = getClass() .getResourceAsStream("/browser.html"); @@ -375,47 +381,38 @@ public MultiDeleteResponse multiDelete(@PathParam("bucket") String bucketName, *

* see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html */ - @GET - @Produces(MediaType.APPLICATION_XML) - public S3BucketAcl get( - @PathParam("bucket") String bucketName, - @QueryParam("acl") String aclMarker, - @Context HttpServletRequest servletRequest) + public S3BucketAcl getAcl(String bucketName) throws OS3Exception, IOException { - if (aclMarker != null) { - S3BucketAcl result = new S3BucketAcl(); - try { - OzoneBucket bucket = getBucket(bucketName); - OzoneVolume volume = getVolume(); - S3Owner owner = new S3Owner(volume.getOwner(), volume.getOwner()); - result.setOwner(owner); - // Use set to remove ACLs with different scopes(ACCESS and DEFAULT) - Set grantSet = new HashSet<>(); - // Return ACL list - for (OzoneAcl acl : bucket.getAcls()) { - List grants = S3Acl.ozoneNativeAclToS3Acl(acl); - grantSet.addAll(grants); - } - ArrayList grantList = new ArrayList<>(); - grantList.addAll(grantSet); - result.setAclList( - new S3BucketAcl.AccessControlList(grantList)); - return result; - } catch (OMException ex) { - if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) { - throw S3ErrorTable.newError(S3ErrorTable - .NO_SUCH_BUCKET, bucketName); - } else if (ex.getResult() == ResultCodes.PERMISSION_DENIED) { - throw S3ErrorTable.newError(S3ErrorTable - .ACCESS_DENIED, bucketName); - } else { - LOG.error("Failed to get acl of Bucket " + bucketName, ex); - throw S3ErrorTable.newError(S3ErrorTable.INTERNAL_ERROR, bucketName); - } + S3BucketAcl result = new S3BucketAcl(); + try { + OzoneBucket bucket = getBucket(bucketName); + OzoneVolume volume = getVolume(); + S3Owner owner = new S3Owner(volume.getOwner(), volume.getOwner()); + result.setOwner(owner); + // Use set to remove ACLs with different scopes(ACCESS and DEFAULT) + Set grantSet = new HashSet<>(); + // Return ACL list + for (OzoneAcl acl : bucket.getAcls()) { + List grants = S3Acl.ozoneNativeAclToS3Acl(acl); + grantSet.addAll(grants); + } + ArrayList grantList = new ArrayList<>(); + grantList.addAll(grantSet); + result.setAclList( + new S3BucketAcl.AccessControlList(grantList)); + return result; + } catch (OMException ex) { + if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) { + throw S3ErrorTable.newError(S3ErrorTable + .NO_SUCH_BUCKET, bucketName); + } else if (ex.getResult() == ResultCodes.PERMISSION_DENIED) { + throw S3ErrorTable.newError(S3ErrorTable + .ACCESS_DENIED, bucketName); + } else { + LOG.error("Failed to get acl of Bucket " + bucketName, ex); + throw S3ErrorTable.newError(S3ErrorTable.INTERNAL_ERROR, bucketName); } } - throw S3ErrorTable.newError(NOT_IMPLEMENTED, - servletRequest.getRequestURI()); } /** diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index 2ea27492e262..6ac1b1c8aeae 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -83,10 +83,11 @@ public void clean() throws IOException { @Test public void testGetAcl() throws Exception { when(parameterMap.containsKey(aclMarker)).thenReturn(true); - S3BucketAcl response = - bucketEndpoint.get(bucketName, aclMarker, servletRequest); - Assert.assertTrue(response.getAclList() != null); - System.out.println(response.toString()); + Response response = + bucketEndpoint.get(bucketName, null, null, null, 0, null, null, + null, null, null, aclMarker, headers); + assertEquals(HTTP_OK, response.getStatus()); + System.out.println(response.getEntity()); } @Test @@ -111,8 +112,7 @@ public void testRead() throws Exception { Response response = bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = - bucketEndpoint.get(bucketName, aclMarker, servletRequest); + S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.READ.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -126,8 +126,7 @@ public void testWrite() throws Exception { Response response = bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = - bucketEndpoint.get(bucketName, aclMarker, servletRequest); + S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.WRITE.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -142,7 +141,7 @@ public void testReadACP() throws Exception { bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = - bucketEndpoint.get(bucketName, aclMarker, servletRequest); + bucketEndpoint.getAcl(bucketName); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.READ_ACP.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -156,8 +155,7 @@ public void testWriteACP() throws Exception { Response response = bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = - bucketEndpoint.get(bucketName, aclMarker, servletRequest); + S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.WRITE_ACP.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -171,8 +169,7 @@ public void testFullControl() throws Exception { Response response = bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = - bucketEndpoint.get(bucketName, aclMarker, servletRequest); + S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.FULL_CONTROL.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -194,8 +191,7 @@ public void testCombination() throws Exception { Response response = bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = - bucketEndpoint.get(bucketName, aclMarker, servletRequest); + S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); assertEquals(5, getResponse.getAclList().getGrantList().size()); } @@ -207,8 +203,7 @@ public void testPutClearOldAcls() throws Exception { Response response = bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = - bucketEndpoint.get(bucketName, aclMarker, servletRequest); + S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); assertEquals(1, getResponse.getAclList().getGrantList().size()); when(headers.getHeaderString(S3Acl.grantRead)) .thenReturn(null); @@ -217,7 +212,7 @@ public void testPutClearOldAcls() throws Exception { response = bucketEndpoint.put(bucketName, aclMarker, headers, null); assertEquals(HTTP_OK, response.getStatus()); - getResponse = bucketEndpoint.get(bucketName, aclMarker, servletRequest); + getResponse = bucketEndpoint.getAcl(bucketName); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.WRITE.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -302,8 +297,7 @@ public void testAclInBody() throws Exception { Response response = bucketEndpoint.put(bucketName, aclMarker, headers, inputBody); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = - bucketEndpoint.get(bucketName, aclMarker, servletRequest); + S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); assertEquals(2, getResponse.getAclList().getGrantList().size()); } @@ -313,7 +307,7 @@ public void testBucketNotExist() throws Exception { when(headers.getHeaderString(S3Acl.grantRead)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); try { - bucketEndpoint.get("bucket-not-exist", aclMarker, servletRequest); + bucketEndpoint.getAcl("bucket-not-exist"); } catch (Exception e) { Assert.assertTrue(e instanceof OS3Exception && ((OS3Exception)e).getHttpCode() == HTTP_NOT_FOUND); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java similarity index 80% rename from hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java rename to hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index 9bafae5a4fce..84e7555c3e72 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -33,7 +33,7 @@ /** * Testing basic object list browsing. */ -public class TestBucketGet { +public class TestBucketList { @Test public void listRoot() throws OS3Exception, IOException { @@ -45,8 +45,8 @@ public void listRoot() throws OS3Exception, IOException { getBucket.setClient(client); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket - .list("b1", "/", null, null, 100, "", null, null, null, null, null) + (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, "", + null, null, null, null, null, null) .getEntity(); Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -69,8 +69,8 @@ public void listDir() throws OS3Exception, IOException { getBucket.setClient(client); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.list("b1", "/", null, null, 100, - "dir1", null, null, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, + "dir1", null, null, null, null, null, null).getEntity(); Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size()); Assert.assertEquals("dir1/", @@ -93,8 +93,8 @@ public void listSubDir() throws OS3Exception, IOException { ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket - .list("b1", "/", null, null, 100, "dir1/", null, null, - null, null, null) + .get("b1", "/", null, null, 100, "dir1/", null, null, + null, null, null, null) .getEntity(); Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -120,8 +120,8 @@ public void listWithPrefixAndDelimiter() throws OS3Exception, IOException { getBucket.setClient(ozoneClient); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.list("b1", "/", null, null, 100, - "dir1", null, null, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, + "dir1", null, null, null, null, null, null).getEntity(); Assert.assertEquals(3, getBucketResponse.getCommonPrefixes().size()); @@ -139,8 +139,8 @@ public void listWithPrefixAndDelimiter1() throws OS3Exception, IOException { getBucket.setClient(ozoneClient); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.list("b1", "/", null, null, 100, - "", null, null, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, + "", null, null, null, null, null, null).getEntity(); Assert.assertEquals(3, getBucketResponse.getCommonPrefixes().size()); Assert.assertEquals("file2", getBucketResponse.getContents().get(0) @@ -160,8 +160,8 @@ public void listWithPrefixAndDelimiter2() throws OS3Exception, IOException { getBucket.setClient(ozoneClient); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.list("b1", "/", null, null, 100, - "dir1bh", null, null, "dir1/dir2/file2", null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, "dir1bh", + null, null, "dir1/dir2/file2", null, null, null).getEntity(); Assert.assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -183,8 +183,8 @@ public void listWithContinuationToken() throws OS3Exception, IOException { // First time ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys, - "", null, null, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", null, null, null, maxKeys, + "", null, null, null, null, null, null).getEntity(); Assert.assertTrue(getBucketResponse.isTruncated()); Assert.assertEquals(2, getBucketResponse.getContents().size()); @@ -192,8 +192,8 @@ public void listWithContinuationToken() throws OS3Exception, IOException { // 2nd time String continueToken = getBucketResponse.getNextToken(); getBucketResponse = - (ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys, - "", null, continueToken, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", null, null, null, maxKeys, + "", null, continueToken, null, null, null, null).getEntity(); Assert.assertTrue(getBucketResponse.isTruncated()); Assert.assertEquals(2, getBucketResponse.getContents().size()); @@ -202,8 +202,8 @@ public void listWithContinuationToken() throws OS3Exception, IOException { //3rd time getBucketResponse = - (ListObjectResponse) getBucket.list("b1", null, null, null, maxKeys, - "", null, continueToken, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", null, null, null, maxKeys, + "", null, continueToken, null, null, null, null).getEntity(); Assert.assertFalse(getBucketResponse.isTruncated()); Assert.assertEquals(1, getBucketResponse.getContents().size()); @@ -234,8 +234,8 @@ public void listWithContinuationTokenDirBreak() ListObjectResponse getBucketResponse; getBucketResponse = - (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys, - "test/", null, null, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + "test/", null, null, null, null, null, null).getEntity(); Assert.assertEquals(0, getBucketResponse.getContents().size()); Assert.assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -245,9 +245,9 @@ public void listWithContinuationTokenDirBreak() getBucketResponse.getCommonPrefixes().get(1).getPrefix()); getBucketResponse = - (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys, - "test/", null, getBucketResponse.getNextToken(), null, null, null) - .getEntity(); + (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + "test/", null, getBucketResponse.getNextToken(), null, null, null, + null).getEntity(); Assert.assertEquals(1, getBucketResponse.getContents().size()); Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size()); Assert.assertEquals("test/dir3/", @@ -277,8 +277,8 @@ public void listWithContinuationToken1() throws OS3Exception, IOException { // First time ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys, - "dir", null, null, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + "dir", null, null, null, null, null, null).getEntity(); Assert.assertTrue(getBucketResponse.isTruncated()); Assert.assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -286,16 +286,16 @@ public void listWithContinuationToken1() throws OS3Exception, IOException { // 2nd time String continueToken = getBucketResponse.getNextToken(); getBucketResponse = - (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys, - "dir", null, continueToken, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + "dir", null, continueToken, null, null, null, null).getEntity(); Assert.assertTrue(getBucketResponse.isTruncated()); Assert.assertEquals(2, getBucketResponse.getCommonPrefixes().size()); //3rd time continueToken = getBucketResponse.getNextToken(); getBucketResponse = - (ListObjectResponse) getBucket.list("b1", "/", null, null, maxKeys, - "dir", null, continueToken, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, + "dir", null, continueToken, null, null, null, null).getEntity(); Assert.assertFalse(getBucketResponse.isTruncated()); Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -314,8 +314,8 @@ public void listWithContinuationTokenFail() throws IOException { getBucket.setClient(ozoneClient); try { - getBucket.list("b1", "/", null, null, 2, "dir", null, "random", null, - null, null).getEntity(); + getBucket.get("b1", "/", null, null, 2, "dir", null, "random", null, + null, null, null).getEntity(); fail("listWithContinuationTokenFail"); } catch (OS3Exception ex) { Assert.assertEquals("random", ex.getResource()); @@ -336,8 +336,8 @@ public void testStartAfter() throws IOException, OS3Exception { getBucket.setClient(ozoneClient); ListObjectResponse getBucketResponse = - (ListObjectResponse) getBucket.list("b1", null, null, null, 1000, - null, null, null, null, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", null, null, null, 1000, + null, null, null, null, null, null, null).getEntity(); Assert.assertFalse(getBucketResponse.isTruncated()); Assert.assertEquals(5, getBucketResponse.getContents().size()); @@ -347,15 +347,15 @@ public void testStartAfter() throws IOException, OS3Exception { String startAfter = "dir0/file1"; getBucketResponse = - (ListObjectResponse) getBucket.list("b1", null, null, null, - 1000, null, null, null, startAfter, null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", null, null, null, + 1000, null, null, null, startAfter, null, null, null).getEntity(); Assert.assertFalse(getBucketResponse.isTruncated()); Assert.assertEquals(4, getBucketResponse.getContents().size()); getBucketResponse = - (ListObjectResponse) getBucket.list("b1", null, null, null, - 1000, null, null, null, "random", null, null).getEntity(); + (ListObjectResponse) getBucket.get("b1", null, null, null, + 1000, null, null, null, "random", null, null, null).getEntity(); Assert.assertFalse(getBucketResponse.isTruncated()); Assert.assertEquals(0, getBucketResponse.getContents().size()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index b7960651f0ce..cf58b3ca8087 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -165,8 +165,8 @@ public void testListKey() throws IOException { bucketEndpoint.setClient(client); try { - bucketEndpoint.list("bucketName", null, null, null, 1000, - null, null, null, null, null, null); + bucketEndpoint.get("bucketName", null, null, null, 1000, + null, null, null, null, null, null, null); Assert.fail("Should fail"); } catch (Exception e) { Assert.assertTrue(e instanceof OS3Exception); @@ -210,7 +210,8 @@ public void testGetAcl() throws Exception { BucketEndpoint bucketEndpoint = new BucketEndpoint(); bucketEndpoint.setClient(client); try { - bucketEndpoint.get("bucketName", "acl", servletRequest); + bucketEndpoint.get("bucketName", null, null, null, 1000, + null, null, null, null, null, "acl", null); } catch (Exception e) { Assert.assertTrue(e instanceof OS3Exception && ((OS3Exception)e).getHttpCode() == HTTP_FORBIDDEN); From fd040632b82b84b816d8e5b17f14886dc7f80637 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Fri, 25 Dec 2020 17:41:27 +0800 Subject: [PATCH 05/12] fix checkstyle --- .../ozone/s3/endpoint/BucketEndpoint.java | 1 - .../hadoop/ozone/s3/endpoint/S3Acl.java | 78 +++++++++---------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 81935a1f68f5..2131b7d3cf03 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.ozone.s3.endpoint; -import javax.servlet.http.HttpServletRequest; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java index b31210e61cb7..433139aacdfe 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java @@ -249,27 +249,27 @@ public static BitSet getOzoneAclOnBucketFromS3Permission(String permission) } BitSet acls = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); switch (permissionType) { - case FULL_CONTROL: - acls.set(IAccessAuthorizer.ACLType.ALL.ordinal()); - break; - case WRITE_ACP: - acls.set(IAccessAuthorizer.ACLType.WRITE_ACL.ordinal()); - break; - case READ_ACP: - acls.set(IAccessAuthorizer.ACLType.READ_ACL.ordinal()); - break; - case WRITE: - acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); - acls.set(IAccessAuthorizer.ACLType.DELETE.ordinal()); - acls.set(IAccessAuthorizer.ACLType.CREATE.ordinal()); - break; - case READ: - acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); - acls.set(IAccessAuthorizer.ACLType.LIST.ordinal()); - break; - default: - LOG.error("Failed to recognize S3 permission {}", permission); - throw S3ErrorTable.newError(INVALID_ARGUMENT, permission); + case FULL_CONTROL: + acls.set(IAccessAuthorizer.ACLType.ALL.ordinal()); + break; + case WRITE_ACP: + acls.set(IAccessAuthorizer.ACLType.WRITE_ACL.ordinal()); + break; + case READ_ACP: + acls.set(IAccessAuthorizer.ACLType.READ_ACL.ordinal()); + break; + case WRITE: + acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); + acls.set(IAccessAuthorizer.ACLType.DELETE.ordinal()); + acls.set(IAccessAuthorizer.ACLType.CREATE.ordinal()); + break; + case READ: + acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); + acls.set(IAccessAuthorizer.ACLType.LIST.ordinal()); + break; + default: + LOG.error("Failed to recognize S3 permission {}", permission); + throw S3ErrorTable.newError(INVALID_ARGUMENT, permission); } return acls; } @@ -313,24 +313,24 @@ public static BitSet getOzoneAclOnVolumeFromS3Permission(String permission) throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, permission); } switch (permissionType) { - case FULL_CONTROL: - acls.set(IAccessAuthorizer.ACLType.ALL.ordinal()); - break; - case WRITE_ACP: - acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); - break; - case READ_ACP: - acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); - break; - case WRITE: - acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); - break; - case READ: - acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); - break; - default: - LOG.error("Failed to recognize S3 permission {}", permission); - throw S3ErrorTable.newError(INVALID_ARGUMENT, permission); + case FULL_CONTROL: + acls.set(IAccessAuthorizer.ACLType.ALL.ordinal()); + break; + case WRITE_ACP: + acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); + break; + case READ_ACP: + acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); + break; + case WRITE: + acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); + break; + case READ: + acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); + break; + default: + LOG.error("Failed to recognize S3 permission {}", permission); + throw S3ErrorTable.newError(INVALID_ARGUMENT, permission); } return acls; } From ac1c461ad7ce8ac2d965fda18388535f73c3b417 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Thu, 28 Jan 2021 19:28:52 +0800 Subject: [PATCH 06/12] address comments --- .../hadoop/ozone/client/OzoneBucket.java | 11 +++ .../hadoop/ozone/client/OzoneVolume.java | 19 ++++- .../ozone/s3/endpoint/BucketEndpoint.java | 77 ++++++++----------- .../hadoop/ozone/s3/endpoint/S3Acl.java | 1 - .../hadoop/ozone/client/OzoneBucketStub.java | 6 ++ .../hadoop/ozone/client/OzoneVolumeStub.java | 6 ++ 6 files changed, 73 insertions(+), 47 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 0b025b6b7609..edcaec047476 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -400,6 +400,17 @@ public boolean removeAcl(OzoneAcl removeAcl) throws IOException { return proxy.removeAcl(ozoneObj, removeAcl); } + /** + * Acls to be set for given Ozone object. This operations reset ACL for + * given object to list of ACLs provided in argument. + * @param acls List of acls. + * + * @throws IOException if there is error. + * */ + public boolean setAcl(List acls) throws IOException { + return proxy.setAcl(ozoneObj, acls); + } + /** * Sets/Changes the storage type of the bucket. * @param newStorageType Storage type to be set diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java index 992142fcab90..b3ea86e624aa 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.NoSuchElementException; +import org.apache.commons.collections.ListUtils; import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; @@ -270,7 +271,7 @@ public Instant getModificationTime() { * @return aclMap */ public List getAcls() { - return (ArrayList)((ArrayList)acls).clone(); + return ListUtils.unmodifiableList(acls); } /** @@ -303,6 +304,22 @@ public boolean removeAcl(OzoneAcl acl) throws IOException { return removed; } + /** + * Acls to be set for given Ozone object. This operations reset ACL for + * given object to list of ACLs provided in argument. + * @param acls List of acls. + * + * @throws IOException if there is error. + * */ + public boolean setAcl(List acls) throws IOException { + boolean reset = proxy.setAcl(ozoneObj, acls); + if (reset) { + acls.clear(); + acls.addAll(acls); + } + return reset; + } + /** * Returns used bucket namespace. * @return usedNamespace diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 2131b7d3cf03..d667336a7950 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -17,6 +17,30 @@ */ package org.apache.hadoop.ozone.s3.endpoint; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; +import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteRequest.DeleteObject; +import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteResponse.DeletedObject; +import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteResponse.Error; +import org.apache.hadoop.ozone.s3.endpoint.S3BucketAcl.Grant; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; +import org.apache.hadoop.ozone.s3.util.ContinueToken; +import org.apache.hadoop.ozone.s3.util.S3StorageType; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.apache.http.HttpStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -40,35 +64,9 @@ import java.util.List; import java.util.Set; -import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneKey; -import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; -import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; -import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; -import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteRequest.DeleteObject; -import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteResponse.DeletedObject; -import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteResponse.Error; -import org.apache.hadoop.ozone.s3.exception.OS3Exception; -import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; -import org.apache.hadoop.ozone.s3.util.ContinueToken; -import org.apache.hadoop.ozone.s3.util.S3StorageType; -import org.apache.hadoop.ozone.s3.endpoint.S3BucketAcl.Grant; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import org.apache.commons.lang3.StringUtils; - import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE; -import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; -import org.apache.http.HttpStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * Bucket level rest endpoints. */ @@ -386,8 +384,13 @@ public S3BucketAcl getAcl(String bucketName) try { OzoneBucket bucket = getBucket(bucketName); OzoneVolume volume = getVolume(); + // TODO: use bucket owner instead of volume owner here once bucket owner + // TODO: is supported. S3Owner owner = new S3Owner(volume.getOwner(), volume.getOwner()); result.setOwner(owner); + + // TODO: remove this duplication avoid logic when ACCESS and DEFAULT scope + // TODO: are merged. // Use set to remove ACLs with different scopes(ACCESS and DEFAULT) Set grantSet = new HashSet<>(); // Return ACL list @@ -479,25 +482,9 @@ public Response putAcl(String bucketName, HttpHeaders httpHeaders, } } - List oldBucketAcls = bucket.getAcls(); - List oldVolumeAcls = - S3Acl.getVolumeAclFromBucketAcl(oldBucketAcls); - - // Add new ACLs - for (OzoneAcl addAcl : ozoneAclListOnBucket) { - bucket.addAcl(addAcl); - } - for (OzoneAcl addAcl : ozoneAclListOnVolume) { - volume.addAcl(addAcl); - } - // A put request will reset all previous ACLs - for (OzoneAcl removeAcl : oldBucketAcls) { - bucket.removeAcl(removeAcl); - } - for (OzoneAcl removeAcl : oldVolumeAcls) { - volume.removeAcl(removeAcl); - } + bucket.setAcl(ozoneAclListOnBucket); + volume.setAcl(ozoneAclListOnVolume); } catch (OMException exception) { LOG.error("Error in set ACL Request for bucket: {}", bucketName, exception); @@ -569,7 +556,7 @@ private List getAndConvertAclOnVolume(String value, LOG.warn("S3 grantee {} is null or not supported", part[0]); throw S3ErrorTable.newError(NOT_IMPLEMENTED, part[0]); } - // Build ACL on Bucket + // Build ACL on Volume BitSet aclsOnVolume = S3Acl.getOzoneAclOnVolumeFromS3Permission(permission); OzoneAcl defaultOzoneAcl = new OzoneAcl( diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java index 433139aacdfe..b59e68695422 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java @@ -179,7 +179,6 @@ public static List ozoneNativeAclToS3Acl(OzoneAcl ozoneAcl) { grant.setGrantee(grantee); grant.setPermission(ACLType.FULL_CONTROL.toString()); grantList.add(grant); - return grantList; } else if (acls.contains(IAccessAuthorizer.ACLType.WRITE_ACL)) { Grant grant = new Grant(); grant.setGrantee(grantee); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index e567d0a9c2db..30cd412d343b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -311,6 +311,12 @@ public boolean addAcl(OzoneAcl addAcl) throws IOException { return aclList.add(addAcl); } + @Override + public boolean setAcl(List acls) throws IOException { + aclList.clear(); + return aclList.addAll(acls); + } + /** * Class used to hold part information in a upload part request. */ diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java index 85c805469d23..3abdd36cb081 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java @@ -125,4 +125,10 @@ public boolean addAcl(OzoneAcl addAcl) throws IOException { public boolean removeAcl(OzoneAcl acl) throws IOException { return aclList.add(acl); } + + @Override + public boolean setAcl(List acls) throws IOException { + aclList.clear(); + return aclList.addAll(acls); + } } From 9dba879fb8f5d05ece83691ed72e7f58e122a8a6 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Thu, 28 Jan 2021 19:41:21 +0800 Subject: [PATCH 07/12] checkstyle --- .../java/org/apache/hadoop/ozone/client/OzoneVolume.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java index b3ea86e624aa..3847b1214f64 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.time.Instant; -import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -307,15 +306,15 @@ public boolean removeAcl(OzoneAcl acl) throws IOException { /** * Acls to be set for given Ozone object. This operations reset ACL for * given object to list of ACLs provided in argument. - * @param acls List of acls. + * @param aclList List of acls. * * @throws IOException if there is error. * */ - public boolean setAcl(List acls) throws IOException { - boolean reset = proxy.setAcl(ozoneObj, acls); + public boolean setAcl(List aclList) throws IOException { + boolean reset = proxy.setAcl(ozoneObj, aclList); if (reset) { acls.clear(); - acls.addAll(acls); + acls.addAll(aclList); } return reset; } From 60933f505913345908f97e85cc99dc3821697daf Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Thu, 4 Feb 2021 20:23:40 +0800 Subject: [PATCH 08/12] findbugs fix --- .../apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index 6ac1b1c8aeae..a77b7cc039b4 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -50,14 +50,14 @@ */ public class TestBucketAcl { - private final String bucketName = OzoneConsts.S3_BUCKET; + private static final String bucketName = OzoneConsts.S3_BUCKET; private OzoneClient client; private HttpServletRequest servletRequest; private Map parameterMap; private HttpHeaders headers; private BucketEndpoint bucketEndpoint; - private final String aclMarker = "acl"; + private static final String aclMarker = "acl"; @Before public void setup() throws IOException { @@ -96,8 +96,7 @@ public void testSetAclWithNotSupportedGranteeType() throws Exception { .thenReturn(S3Acl.ACLIdentityType.GROUP.getHeaderType() + "=root"); when(parameterMap.containsKey(aclMarker)).thenReturn(true); try { - Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(bucketName, aclMarker, headers, null); } catch (Exception e) { Assert.assertTrue(e instanceof OS3Exception && ((OS3Exception) e).getHttpCode() == HTTP_NOT_IMPLEMENTED); @@ -263,8 +262,7 @@ public void testAclInBodyWithGroupUser() throws Exception { "\n").getBytes(UTF_8)); when(parameterMap.containsKey(aclMarker)).thenReturn(true); - Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, inputBody); + bucketEndpoint.put(bucketName, aclMarker, headers, inputBody); } @Test From 1fed47364e4b3aec2c8610e85ade92d1a10b9a8c Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Wed, 10 Mar 2021 17:14:50 +0800 Subject: [PATCH 09/12] address comments --- .../ozone/s3/endpoint/BucketEndpoint.java | 10 +- .../hadoop/ozone/s3/endpoint/S3Acl.java | 15 +- .../ozone/s3/endpoint/TestBucketAcl.java | 175 ++++++------------ .../s3/endpoint/TestPermissionCheck.java | 4 +- .../test/resources/groupAccessControlList.xml | 39 ++++ .../test/resources/userAccessControlList.xml | 24 +++ 6 files changed, 136 insertions(+), 131 deletions(-) create mode 100644 hadoop-ozone/s3gateway/src/test/resources/groupAccessControlList.xml create mode 100644 hadoop-ozone/s3gateway/src/test/resources/userAccessControlList.xml diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index d667336a7950..21b982cbfc66 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -424,11 +424,11 @@ public S3BucketAcl getAcl(String bucketName) */ public Response putAcl(String bucketName, HttpHeaders httpHeaders, InputStream body) throws IOException, OS3Exception { - String grantReads = httpHeaders.getHeaderString(S3Acl.grantRead); - String grantWrites = httpHeaders.getHeaderString(S3Acl.grantWrite); - String grantReadACP = httpHeaders.getHeaderString(S3Acl.grantReadACP); - String grantWriteACP = httpHeaders.getHeaderString(S3Acl.grantWriteACP); - String grantFull = httpHeaders.getHeaderString(S3Acl.grantFullControl); + String grantReads = httpHeaders.getHeaderString(S3Acl.GRANT_READ); + String grantWrites = httpHeaders.getHeaderString(S3Acl.GRANT_WRITE); + String grantReadACP = httpHeaders.getHeaderString(S3Acl.GRANT_READ_CAP); + String grantWriteACP = httpHeaders.getHeaderString(S3Acl.GRANT_WRITE_CAP); + String grantFull = httpHeaders.getHeaderString(S3Acl.GRANT_FULL_CONTROL); try { OzoneBucket bucket = getBucket(bucketName); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java index b59e68695422..65bfb6611041 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java @@ -39,14 +39,14 @@ public final class S3Acl { private static final Logger LOG = LoggerFactory.getLogger(S3Acl.class); // ACL put related headers - public static final String grantRead = "x-amz-grant-read"; - public static final String grantWrite = "x-amz-grant-write"; - public static final String grantReadACP = "x-amz-grant-read-acp"; - public static final String grantWriteACP = "x-amz-grant-write-acp"; - public static final String grantFullControl = "x-amz-grant-full-control"; + public static final String GRANT_READ = "x-amz-grant-read"; + public static final String GRANT_WRITE = "x-amz-grant-write"; + public static final String GRANT_READ_CAP = "x-amz-grant-read-acp"; + public static final String GRANT_WRITE_CAP = "x-amz-grant-write-acp"; + public static final String GRANT_FULL_CONTROL = "x-amz-grant-full-control"; // Not supported headers at current stage, may support it in future - public static final String cannedAclHeader = "x-amz-acl"; + public static final String CANNED_ACL_HEADER = "x-amz-acl"; /** * https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html. @@ -151,6 +151,9 @@ public static ACLIdentityType getTypeFromHeaderType(String typeStr) { } } + private S3Acl() { + } + public static boolean isGranteeTypeSupported(String typeStr) { ACLIdentityType type = ACLIdentityType.getTypeFromGranteeType(typeStr); return type == null ? false : type.isSupported(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index a77b7cc039b4..0273696a2984 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -36,6 +36,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.Map; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; @@ -50,19 +51,19 @@ */ public class TestBucketAcl { - private static final String bucketName = OzoneConsts.S3_BUCKET; + private static final String BUCKET_NAME = OzoneConsts.S3_BUCKET; private OzoneClient client; private HttpServletRequest servletRequest; private Map parameterMap; private HttpHeaders headers; private BucketEndpoint bucketEndpoint; - private static final String aclMarker = "acl"; + private static final String ACL_MARKER = "acl"; @Before public void setup() throws IOException { client = new OzoneClientStub(); - client.getObjectStore().createS3Bucket(bucketName); + client.getObjectStore().createS3Bucket(BUCKET_NAME); servletRequest = Mockito.mock(HttpServletRequest.class); parameterMap = Mockito.mock(Map.class); @@ -82,21 +83,21 @@ public void clean() throws IOException { @Test public void testGetAcl() throws Exception { - when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); Response response = - bucketEndpoint.get(bucketName, null, null, null, 0, null, null, - null, null, null, aclMarker, headers); + bucketEndpoint.get(BUCKET_NAME, null, null, null, 0, null, null, + null, null, null, ACL_MARKER, headers); assertEquals(HTTP_OK, response.getStatus()); System.out.println(response.getEntity()); } @Test public void testSetAclWithNotSupportedGranteeType() throws Exception { - when(headers.getHeaderString(S3Acl.grantRead)) + when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(S3Acl.ACLIdentityType.GROUP.getHeaderType() + "=root"); - when(parameterMap.containsKey(aclMarker)).thenReturn(true); + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); try { - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); } catch (Exception e) { Assert.assertTrue(e instanceof OS3Exception && ((OS3Exception) e).getHttpCode() == HTTP_NOT_IMPLEMENTED); @@ -105,13 +106,13 @@ public void testSetAclWithNotSupportedGranteeType() throws Exception { @Test public void testRead() throws Exception { - when(parameterMap.containsKey(aclMarker)).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantRead)) + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); + when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); + S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.READ.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -119,13 +120,13 @@ public void testRead() throws Exception { @Test public void testWrite() throws Exception { - when(parameterMap.containsKey(aclMarker)).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantWrite)) + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); + when(headers.getHeaderString(S3Acl.GRANT_WRITE)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); + S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.WRITE.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -133,14 +134,14 @@ public void testWrite() throws Exception { @Test public void testReadACP() throws Exception { - when(parameterMap.containsKey(aclMarker)).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantReadACP)) + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); + when(headers.getHeaderString(S3Acl.GRANT_READ_CAP)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = - bucketEndpoint.getAcl(bucketName); + bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.READ_ACP.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -148,13 +149,13 @@ public void testReadACP() throws Exception { @Test public void testWriteACP() throws Exception { - when(parameterMap.containsKey(aclMarker)).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantWriteACP)) + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); + when(headers.getHeaderString(S3Acl.GRANT_WRITE_CAP)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); + S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.WRITE_ACP.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -162,13 +163,13 @@ public void testWriteACP() throws Exception { @Test public void testFullControl() throws Exception { - when(parameterMap.containsKey(aclMarker)).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantFullControl)) + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); + when(headers.getHeaderString(S3Acl.GRANT_FULL_CONTROL)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); + S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.FULL_CONTROL.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -176,42 +177,42 @@ public void testFullControl() throws Exception { @Test public void testCombination() throws Exception { - when(parameterMap.containsKey(aclMarker)).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantRead)) + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); + when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); - when(headers.getHeaderString(S3Acl.grantWrite)) + when(headers.getHeaderString(S3Acl.GRANT_WRITE)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); - when(headers.getHeaderString(S3Acl.grantReadACP)) + when(headers.getHeaderString(S3Acl.GRANT_READ_CAP)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); - when(headers.getHeaderString(S3Acl.grantWriteACP)) + when(headers.getHeaderString(S3Acl.GRANT_WRITE_CAP)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); - when(headers.getHeaderString(S3Acl.grantFullControl)) + when(headers.getHeaderString(S3Acl.GRANT_FULL_CONTROL)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); + S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(5, getResponse.getAclList().getGrantList().size()); } @Test public void testPutClearOldAcls() throws Exception { - when(parameterMap.containsKey(aclMarker)).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantRead)) + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); + when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); + S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); - when(headers.getHeaderString(S3Acl.grantRead)) + when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(null); - when(headers.getHeaderString(S3Acl.grantWrite)) + when(headers.getHeaderString(S3Acl.GRANT_WRITE)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); response = - bucketEndpoint.put(bucketName, aclMarker, headers, null); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); - getResponse = bucketEndpoint.getAcl(bucketName); + getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.WRITE.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); @@ -219,90 +220,28 @@ public void testPutClearOldAcls() throws Exception { @Test(expected = OS3Exception.class) public void testAclInBodyWithGroupUser() throws Exception { - ByteArrayInputStream inputBody = - new ByteArrayInputStream( - ("\n" + - " \n" + - " 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID\n" + - " owner\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID\n" + - " owner\n" + - " \n" + - " FULL_CONTROL\n" + - " \n" + - " \n" + - " \n" + - " http://acs.amazonaws.com/groups/global/AllUsers\n" + - " \n" + - " READ\n" + - " \n" + - " \n" + - " \n" + - " http://acs.amazonaws.com/groups/s3/LogDelivery\n" + - " \n" + - " WRITE\n" + - " \n" + - " \n" + - " \n" + - " xyz@amazon.com\n" + - " \n" + - " WRITE_ACP\n" + - " \n" + - " \n" + - " \n" + - " f30716ab7115dcb44a5ef76e9d74b8e20567f63TestAccountCanonicalUserID\n" + - " \n" + - " READ_ACP\n" + - " \n" + - " \n" + - "\n").getBytes(UTF_8)); - - when(parameterMap.containsKey(aclMarker)).thenReturn(true); - bucketEndpoint.put(bucketName, aclMarker, headers, inputBody); + InputStream inputBody = TestBucketAcl.class.getClassLoader() + .getResourceAsStream("groupAccessControlList.xml"); + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, inputBody); } @Test public void testAclInBody() throws Exception { - ByteArrayInputStream inputBody = - new ByteArrayInputStream( - ("\n" + - " \n" + - " 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID\n" + - " owner\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID\n" + - " owner\n" + - " \n" + - " FULL_CONTROL\n" + - " \n" + - " \n" + - " \n" + - " f30716ab7115dcb44a5ef76e9d74b8e20567f63TestAccountCanonicalUserID\n" + - " \n" + - " READ_ACP\n" + - " \n" + - " \n" + - "\n").getBytes(UTF_8)); - - when(parameterMap.containsKey(aclMarker)).thenReturn(true); + InputStream inputBody = TestBucketAcl.class.getClassLoader() + .getResourceAsStream("userAccessControlList.xml"); + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); Response response = - bucketEndpoint.put(bucketName, aclMarker, headers, inputBody); + bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, inputBody); assertEquals(HTTP_OK, response.getStatus()); - S3BucketAcl getResponse = bucketEndpoint.getAcl(bucketName); + S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(2, getResponse.getAclList().getGrantList().size()); } @Test public void testBucketNotExist() throws Exception { - when(parameterMap.containsKey(aclMarker)).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantRead)) + when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); + when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); try { bucketEndpoint.getAcl("bucket-not-exist"); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index cf58b3ca8087..81b0718e6bae 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -205,7 +205,7 @@ public void testGetAcl() throws Exception { when(servletRequest.getParameterMap()).thenReturn(parameterMap); when(parameterMap.containsKey("acl")).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantRead)) + when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); BucketEndpoint bucketEndpoint = new BucketEndpoint(); bucketEndpoint.setClient(client); @@ -229,7 +229,7 @@ public void testSetAcl() throws Exception { when(servletRequest.getParameterMap()).thenReturn(parameterMap); when(parameterMap.containsKey("acl")).thenReturn(true); - when(headers.getHeaderString(S3Acl.grantRead)) + when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); BucketEndpoint bucketEndpoint = new BucketEndpoint(); bucketEndpoint.setClient(client); diff --git a/hadoop-ozone/s3gateway/src/test/resources/groupAccessControlList.xml b/hadoop-ozone/s3gateway/src/test/resources/groupAccessControlList.xml new file mode 100644 index 000000000000..83f5bddad8d6 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/resources/groupAccessControlList.xml @@ -0,0 +1,39 @@ + + + 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID + owner + + + + + 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID + owner + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + READ + + + + http://acs.amazonaws.com/groups/s3/LogDelivery + + WRITE + + + + xyz@amazon.com + + WRITE_ACP + + + + f30716ab7115dcb44a5ef76e9d74b8e20567f63TestAccountCanonicalUserID + + READ_ACP + + + \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/resources/userAccessControlList.xml b/hadoop-ozone/s3gateway/src/test/resources/userAccessControlList.xml new file mode 100644 index 000000000000..97c792f019c8 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/resources/userAccessControlList.xml @@ -0,0 +1,24 @@ + + + 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID + owner + + + + + 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID + + owner + + FULL_CONTROL + + + + f30716ab7115dcb44a5ef76e9d74b8e20567f63TestAccountCanonicalUserID + + READ_ACP + + + \ No newline at end of file From 1330a0c0b7b7f7206ec4becd5345e27163277922 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Wed, 10 Mar 2021 19:47:46 +0800 Subject: [PATCH 10/12] fix checkstyle and compile issue --- .../client/rpc/TestOzoneRpcClientAbstract.java | 8 ++++---- .../ozone/s3/endpoint/TestBucketAcl.java | 2 -- .../test/resources/groupAccessControlList.xml | 18 ++++++++++++++++++ .../test/resources/userAccessControlList.xml | 18 ++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 840310eb67d4..dfc7d797e290 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -2262,10 +2262,10 @@ public void testMultipartUploadWithACL() throws Exception { OzoneAcl acl2 = new OzoneAcl(USER, "Friday", ACLType.ALL, DEFAULT); OzoneAcl acl3 = new OzoneAcl(USER, "Jan", ACLType.ALL, ACCESS); OzoneAcl acl4 = new OzoneAcl(USER, "Feb", ACLType.ALL, ACCESS); - bucket.addAcls(acl1); - bucket.addAcls(acl2); - bucket.addAcls(acl3); - bucket.addAcls(acl4); + bucket.addAcl(acl1); + bucket.addAcl(acl2); + bucket.addAcl(acl3); + bucket.addAcl(acl4); doMultipartUpload(bucket, keyName, (byte)98); OzoneObj keyObj = OzoneObjInfo.Builder.newBuilder() diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index 0273696a2984..fb10b61541ed 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -34,7 +34,6 @@ import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Map; @@ -42,7 +41,6 @@ import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_NOT_IMPLEMENTED; import static java.net.HttpURLConnection.HTTP_OK; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.when; diff --git a/hadoop-ozone/s3gateway/src/test/resources/groupAccessControlList.xml b/hadoop-ozone/s3gateway/src/test/resources/groupAccessControlList.xml index 83f5bddad8d6..a774583bfaee 100644 --- a/hadoop-ozone/s3gateway/src/test/resources/groupAccessControlList.xml +++ b/hadoop-ozone/s3gateway/src/test/resources/groupAccessControlList.xml @@ -1,3 +1,21 @@ + + 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID diff --git a/hadoop-ozone/s3gateway/src/test/resources/userAccessControlList.xml b/hadoop-ozone/s3gateway/src/test/resources/userAccessControlList.xml index 97c792f019c8..45ce76e119d8 100644 --- a/hadoop-ozone/s3gateway/src/test/resources/userAccessControlList.xml +++ b/hadoop-ozone/s3gateway/src/test/resources/userAccessControlList.xml @@ -1,3 +1,21 @@ + + 852b113e7a2f25102679df27bb0ae12b3f85be6BucketOwnerCanonicalUserID From 1310e7b947d8d58df34d57bca8fa569b40b62cb5 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 20 Apr 2021 16:02:41 +0800 Subject: [PATCH 11/12] address comments --- .../hadoop/ozone/om/helpers/OzoneAclUtil.java | 20 ++++++ .../ozone/s3/endpoint/BucketEndpoint.java | 33 ++++++--- .../hadoop/ozone/s3/endpoint/S3Acl.java | 68 +++---------------- .../hadoop/ozone/client/OzoneVolumeStub.java | 2 +- .../ozone/s3/endpoint/TestBucketAcl.java | 15 ++++ 5 files changed, 71 insertions(+), 67 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java index 0c15d79fcf6d..4449cfd95463 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java @@ -68,6 +68,26 @@ public static List getAclList(String userName, return listOfAcls; } + /** + * Helper function to get acl list for one user/group. + * + * @param identityName + * @param type + * @param aclList + * @return list of OzoneAcls + * */ + public static List filterAclList(String identityName, + IAccessAuthorizer.ACLIdentityType type, List aclList) { + + if (aclList == null && aclList.isEmpty()) { + return new ArrayList<>(); + } + + List retList = aclList.stream().filter(acl -> acl.getType() == type + && acl.getName().equals(identityName)).collect(Collectors.toList()); + return retList; + } + /** * Check if acl right requested for given RequestContext exist * in provided acl list. diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 21b982cbfc66..72a95ceb6b77 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteRequest.DeleteObject; import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteResponse.DeletedObject; @@ -64,6 +65,7 @@ import java.util.List; import java.util.Set; +import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED; import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE; @@ -482,9 +484,28 @@ public Response putAcl(String bucketName, HttpHeaders httpHeaders, } } - // A put request will reset all previous ACLs + // A put request will reset all previous ACLs on bucket bucket.setAcl(ozoneAclListOnBucket); - volume.setAcl(ozoneAclListOnVolume); + // A put request will reset input user/group's permission on volume + List acls = bucket.getAcls(); + List aclsToRemoveOnVolume = new ArrayList<>(); + List currentAclsOnVolume = volume.getAcls(); + // Remove input user/group's permission from Volume first + if (currentAclsOnVolume.size() > 0) { + for (OzoneAcl acl : acls) { + if (acl.getAclScope() == ACCESS) { + aclsToRemoveOnVolume.addAll(OzoneAclUtil.filterAclList( + acl.getName(), acl.getType(), currentAclsOnVolume)); + } + } + for (OzoneAcl acl : aclsToRemoveOnVolume) { + volume.removeAcl(acl); + } + } + // Add new permission on Volume + for(OzoneAcl acl : ozoneAclListOnVolume) { + volume.addAcl(acl); + } } catch (OMException exception) { LOG.error("Error in set ACL Request for bucket: {}", bucketName, exception); @@ -531,7 +552,7 @@ private List getAndConvertAclOnBucket(String value, OzoneAcl.AclScope.DEFAULT); OzoneAcl accessOzoneAcl = new OzoneAcl( IAccessAuthorizer.ACLIdentityType.USER, part[1], aclsOnBucket, - OzoneAcl.AclScope.ACCESS); + ACCESS); ozoneAclList.add(defaultOzoneAcl); ozoneAclList.add(accessOzoneAcl); } @@ -559,13 +580,9 @@ private List getAndConvertAclOnVolume(String value, // Build ACL on Volume BitSet aclsOnVolume = S3Acl.getOzoneAclOnVolumeFromS3Permission(permission); - OzoneAcl defaultOzoneAcl = new OzoneAcl( - IAccessAuthorizer.ACLIdentityType.USER, part[1], aclsOnVolume, - OzoneAcl.AclScope.DEFAULT); OzoneAcl accessOzoneAcl = new OzoneAcl( IAccessAuthorizer.ACLIdentityType.USER, part[1], aclsOnVolume, - OzoneAcl.AclScope.ACCESS); - ozoneAclList.add(defaultOzoneAcl); + ACCESS); ozoneAclList.add(accessOzoneAcl); } return ozoneAclList; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java index 65bfb6611041..111f28ce9254 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3Acl.java @@ -287,15 +287,10 @@ public static List s3AclToOzoneNativeAclOnVolume( if (identityType != null && identityType.isSupported()) { String permission = grant.getPermission(); BitSet acls = getOzoneAclOnVolumeFromS3Permission(permission); - OzoneAcl defaultOzoneAcl = new OzoneAcl( - IAccessAuthorizer.ACLIdentityType.USER, - grant.getGrantee().getId(), acls, - OzoneAcl.AclScope.DEFAULT); OzoneAcl accessOzoneAcl = new OzoneAcl( IAccessAuthorizer.ACLIdentityType.USER, grant.getGrantee().getId(), acls, OzoneAcl.AclScope.ACCESS); - ozoneAclList.add(defaultOzoneAcl); ozoneAclList.add(accessOzoneAcl); } else { LOG.error("Grantee type {} is not supported", @@ -307,6 +302,7 @@ public static List s3AclToOzoneNativeAclOnVolume( return ozoneAclList; } + // User privilege on volume follows the "lest privilege" principle. public static BitSet getOzoneAclOnVolumeFromS3Permission(String permission) throws OS3Exception { BitSet acls = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); @@ -316,15 +312,22 @@ public static BitSet getOzoneAclOnVolumeFromS3Permission(String permission) } switch (permissionType) { case FULL_CONTROL: - acls.set(IAccessAuthorizer.ACLType.ALL.ordinal()); + acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); + acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); + acls.set(IAccessAuthorizer.ACLType.READ_ACL.ordinal()); + acls.set(IAccessAuthorizer.ACLType.WRITE_ACL.ordinal()); break; case WRITE_ACP: - acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); + acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); + acls.set(IAccessAuthorizer.ACLType.READ_ACL.ordinal()); + acls.set(IAccessAuthorizer.ACLType.WRITE_ACL.ordinal()); break; case READ_ACP: acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); + acls.set(IAccessAuthorizer.ACLType.READ_ACL.ordinal()); break; case WRITE: + acls.set(IAccessAuthorizer.ACLType.READ.ordinal()); acls.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); break; case READ: @@ -336,55 +339,4 @@ public static BitSet getOzoneAclOnVolumeFromS3Permission(String permission) } return acls; } - - public static List getVolumeAclFromBucketAcl( - List aclList) { - List volumeAclList = new ArrayList<>(); - if (aclList == null || aclList.isEmpty()) { - return volumeAclList; - } - for (OzoneAcl bucketAcl: aclList) { - List acls = bucketAcl.getAclList(); - if (acls.contains(IAccessAuthorizer.ACLType.ALL)) { - OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(), IAccessAuthorizer.ACLType.ALL, - bucketAcl.getAclScope()); - volumeAclList.add(volumeAcl); - } else if (acls.contains(IAccessAuthorizer.ACLType.WRITE_ACL)) { - OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(), IAccessAuthorizer.ACLType.WRITE_ACL, - bucketAcl.getAclScope()); - volumeAclList.add(volumeAcl); - } else if (acls.contains(IAccessAuthorizer.ACLType.READ_ACL)) { - OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(), IAccessAuthorizer.ACLType.READ_ACL, - bucketAcl.getAclScope()); - volumeAclList.add(volumeAcl); - } else if (acls.contains(IAccessAuthorizer.ACLType.WRITE) && - acls.contains(IAccessAuthorizer.ACLType.DELETE) && - acls.contains(IAccessAuthorizer.ACLType.CREATE)) { - BitSet aclBits = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); - aclBits.set(IAccessAuthorizer.ACLType.WRITE.ordinal()); - aclBits.set(IAccessAuthorizer.ACLType.DELETE.ordinal()); - aclBits.set(IAccessAuthorizer.ACLType.CREATE.ordinal()); - - OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(), aclBits, bucketAcl.getAclScope()); - volumeAclList.add(volumeAcl); - } else if (acls.contains(IAccessAuthorizer.ACLType.READ) && - acls.contains(IAccessAuthorizer.ACLType.LIST)) { - BitSet aclBits = new BitSet(IAccessAuthorizer.ACLType.getNoOfAcls()); - aclBits.set(IAccessAuthorizer.ACLType.READ.ordinal()); - aclBits.set(IAccessAuthorizer.ACLType.LIST.ordinal()); - - OzoneAcl volumeAcl = new OzoneAcl(bucketAcl.getType(), - bucketAcl.getName(), aclBits, bucketAcl.getAclScope()); - volumeAclList.add(volumeAcl); - } else { - LOG.error("Cannot find a good mapping for Ozone ACL {} to S3", - bucketAcl.toString()); - } - } - return volumeAclList; - } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java index 3abdd36cb081..55a7191ac38a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java @@ -123,7 +123,7 @@ public boolean addAcl(OzoneAcl addAcl) throws IOException { @Override public boolean removeAcl(OzoneAcl acl) throws IOException { - return aclList.add(acl); + return aclList.remove(acl); } @Override diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index fb10b61541ed..703ed98c4095 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -23,7 +23,9 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -198,15 +200,24 @@ public void testPutClearOldAcls() throws Exception { when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + // Put READ Response response = bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); S3BucketAcl getResponse = bucketEndpoint.getAcl(BUCKET_NAME); assertEquals(1, getResponse.getAclList().getGrantList().size()); + assertEquals(S3Acl.ACLType.READ.getValue(), + getResponse.getAclList().getGrantList().get(0).getPermission()); + OzoneVolume volume = bucketEndpoint.getVolume(); + assertEquals(1, volume.getAcls().size()); + assertEquals(IAccessAuthorizer.ACLType.READ, + volume.getAcls().get(0).getAclList().get(0)); + when(headers.getHeaderString(S3Acl.GRANT_READ)) .thenReturn(null); when(headers.getHeaderString(S3Acl.GRANT_WRITE)) .thenReturn(S3Acl.ACLIdentityType.USER.getHeaderType() + "=root"); + //Put WRITE response = bucketEndpoint.put(BUCKET_NAME, ACL_MARKER, headers, null); assertEquals(HTTP_OK, response.getStatus()); @@ -214,6 +225,10 @@ public void testPutClearOldAcls() throws Exception { assertEquals(1, getResponse.getAclList().getGrantList().size()); assertEquals(S3Acl.ACLType.WRITE.getValue(), getResponse.getAclList().getGrantList().get(0).getPermission()); + volume = bucketEndpoint.getVolume(); + assertEquals(1, volume.getAcls().size()); + assertEquals(IAccessAuthorizer.ACLType.READ, + volume.getAcls().get(0).getAclList().get(0)); } @Test(expected = OS3Exception.class) From f2125aa4a38e78d925586f57abaa0188cd633470 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Wed, 21 Apr 2021 11:10:47 +0800 Subject: [PATCH 12/12] fix findbugs issue --- .../java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java index 4449cfd95463..9901ae719913 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java @@ -79,7 +79,7 @@ public static List getAclList(String userName, public static List filterAclList(String identityName, IAccessAuthorizer.ACLIdentityType type, List aclList) { - if (aclList == null && aclList.isEmpty()) { + if (aclList == null || aclList.isEmpty()) { return new ArrayList<>(); }