HDDS-11298. Support owner field in the S3 listBuckets request

hadoop-ozone/dist/src/main/smoketest/s3/bucketlist.robot

@@ -29,8 +29,14 @@ ${BUCKET}             generated
 *** Test Cases ***
 
 List buckets
-    ${result} =         Execute AWSS3APICli     list-buckets | jq -r '.Buckets[].Name'
-                        Should contain          ${result}    ${BUCKET}
+    ${result} =             Execute AWSS3APICli   list-buckets
+    ${bucket_names} =       Execute               echo '''${result}''' | jq -r '.Buckets[].Name'
+    Should contain          ${bucket_names}       ${BUCKET}
+    ${ownerId} =            Execute               echo '''${result}''' | jq -r '.Owner.ID'
+    Should Not Be Equal     ${ownerId}            null
+    ${ownerDisplayName} =   Execute               echo '''${result}''' | jq -r '.Owner.DisplayName'
+    Should Not Be Equal     ${ownerDisplayName}   null
+
 
 Get bucket info with Ozone Shell to check the owner field
     Pass Execution If   '${SECURITY_ENABLED}' == 'false'    Skipping this check as security is not enabled

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java

@@ -32,13 +32,15 @@
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import java.net.URI;
+import java.net.URISyntaxException;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.server.http.HttpConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
 import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
 import software.amazon.awssdk.services.s3.S3Client;
 
 /**
@@ -158,4 +160,54 @@ public S3Client createS3ClientV2(boolean enablePathStyle) throws Exception {
         .forcePathStyle(enablePathStyle)
         .build();
   }
+
+  /**
+   * Creates an S3AsyncClient (AWS SDK V2) with path style access enabled.
+   *
+   * @return S3AsyncClient
+   * @throws URISyntaxException if there is an error creating the client
+   */
+  public S3AsyncClient createS3AsyncClientV2() throws Exception {
+    return createS3AsyncClientV2(true);
+  }
+
+  /**
+   * Creates an S3AsyncClient (AWS SDK V2).
+   *
+   * @param enablePathStyle whether to enable path style access
+   * @return S3AsyncClient
+   * @throws Exception if there is an error creating the client
+   */
+  public S3AsyncClient createS3AsyncClientV2(boolean enablePathStyle) throws Exception {
+    final String accessKey = "user";
+    final String secretKey = "password";
+    final Region region = Region.US_EAST_1;
+
+    final String protocol;
+    final HttpConfig.Policy webPolicy = getHttpPolicy(conf);
+    String host;
+
+    if (webPolicy.isHttpsEnabled()) {
+      // TODO: Currently HTTPS is disabled in the test, we can add HTTPS
+      // integration in the future
+      protocol = HTTPS_SCHEME;
+      host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY);
+    } else {
+      protocol = HTTP_SCHEME;
+      host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY);
+    }
+
+    String endpoint = protocol + "://" + host;
+
+    LOG.info("S3 Endpoint is {}", endpoint);
+
+    AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey);
+
+    return S3AsyncClient.builder()
+        .region(region)
+        .endpointOverride(new URI(endpoint))
+        .credentialsProvider(StaticCredentialsProvider.create(credentials))
+        .forcePathStyle(enablePathStyle)
+        .build();
+  }
 }

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java

@@ -106,6 +106,7 @@
 import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
 import org.apache.hadoop.ozone.s3.S3ClientFactory;
 import org.apache.hadoop.ozone.s3.S3GatewayService;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.ozone.test.OzoneTestBase;
 import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Test;
@@ -250,7 +251,7 @@ public void testBucketACLOperations() {
   }
 
   @Test
-  public void testListBuckets() {
+  public void testListBuckets() throws IOException {
     List<String> bucketNames = new ArrayList<>();
     for (int i = 0; i <= 5; i++) {
       String bucketName = getBucketName(String.valueOf(i));
@@ -263,6 +264,14 @@ public void testListBuckets() {
         .map(Bucket::getName).collect(Collectors.toList());
 
     assertThat(listBucketNames).containsAll(bucketNames);
+
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    String expectOwner = ugi.getShortUserName();
+
+    Owner s3AccountOwner = s3Client.getS3AccountOwner();
+
+    assertThat(s3AccountOwner.getDisplayName()).isEqualTo(expectOwner);
+    assertThat(s3AccountOwner.getId()).isEqualTo(expectOwner);
   }
 
   @Test

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java

@@ -41,13 +41,15 @@
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.s3.S3ClientFactory;
 import org.apache.hadoop.ozone.s3.S3GatewayService;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.ozone.test.OzoneTestBase;
 import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.junit.jupiter.api.io.TempDir;
 import software.amazon.awssdk.core.ResponseBytes;
 import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
 import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
@@ -57,6 +59,7 @@
 import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.services.s3.model.Tag;
 import software.amazon.awssdk.services.s3.model.Tagging;
@@ -82,6 +85,7 @@ public abstract class AbstractS3SDKV2Tests extends OzoneTestBase {
 
   private static MiniOzoneCluster cluster = null;
   private static S3Client s3Client = null;
+  private static S3AsyncClient s3AsyncClient = null;
 
   /**
    * Create a MiniOzoneCluster with S3G enabled for testing.
@@ -95,7 +99,9 @@ static void startCluster(OzoneConfiguration conf) throws Exception {
         .setNumDatanodes(5)
         .build();
     cluster.waitForClusterToBeReady();
-    s3Client = new S3ClientFactory(s3g.getConf()).createS3ClientV2();
+    S3ClientFactory s3ClientFactory = new S3ClientFactory(s3g.getConf());
+    s3Client = s3ClientFactory.createS3ClientV2();
+    s3AsyncClient = s3ClientFactory.createS3AsyncClientV2();
   }
 
   /**
@@ -195,6 +201,26 @@ public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception
     assertEquals(userMetadata, headObjectResponse.metadata());
   }
 
+  @Test
+  public void listBuckets() throws Exception {
+    final String bucketName = getBucketName();
+    final String expectedOwner = UserGroupInformation.getCurrentUser().getUserName();
+
+    s3Client.createBucket(b -> b.bucket(bucketName));
+
+    ListBucketsResponse syncResponse = s3Client.listBuckets();
+    assertEquals(1, syncResponse.buckets().size());
+    assertEquals(bucketName, syncResponse.buckets().get(0).name());
+    assertEquals(expectedOwner, syncResponse.owner().displayName());
+    assertEquals(expectedOwner, syncResponse.owner().id());
+
+    ListBucketsResponse asyncResponse = s3AsyncClient.listBuckets().join();
+    assertEquals(1, asyncResponse.buckets().size());
+    assertEquals(bucketName, asyncResponse.buckets().get(0).name());
+    assertEquals(expectedOwner, asyncResponse.owner().displayName());
+    assertEquals(expectedOwner, asyncResponse.owner().id());
+  }
+
   private String getBucketName() {
     return getBucketName(null);
   }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListBucketResponse.java

@@ -39,6 +39,9 @@ public class ListBucketResponse {
   @XmlElement(name = "Bucket")
   private List<BucketMetadata> buckets = new ArrayList<>();
 
+  @XmlElement(name = "Owner")
+  private S3Owner owner;
+
   public List<BucketMetadata> getBuckets() {
     return buckets;
   }
@@ -55,4 +58,12 @@ public void setBuckets(List<BucketMetadata> buckets) {
   public void addBucket(BucketMetadata bucket) {
     buckets.add(bucket);
   }
+
+  public S3Owner getOwner() {
+    return owner;
+  }
+
+  public void setOwner(S3Owner owner) {
+    this.owner = owner;
+  }
 }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java

@@ -24,6 +24,7 @@
 import javax.ws.rs.core.Response;
 import org.apache.hadoop.ozone.audit.S3GAction;
 import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.s3.commontypes.BucketMetadata;
 import org.apache.hadoop.ozone.s3.exception.OS3Exception;
 import org.apache.hadoop.util.Time;
@@ -61,6 +62,9 @@ public Response get()
         throw e;
       }
 
+      OzoneVolume volume = getVolume();
+      response.setOwner(new S3Owner(volume.getOwner(), volume.getOwner()));
+
       while (bucketIterator.hasNext()) {
         OzoneBucket next = bucketIterator.next();
         BucketMetadata bucketMetadata = new BucketMetadata();

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java

@@ -19,6 +19,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneClientStub;
 import org.junit.jupiter.api.BeforeEach;
@@ -32,6 +33,8 @@ public class TestRootList {
   private OzoneClient clientStub;
   private RootEndpoint rootEndpoint;
 
+  private static final String DEFAULT_VOLUME = OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT;
+
   @BeforeEach
   public void setup() throws Exception {
 
@@ -43,6 +46,7 @@ public void setup() throws Exception {
         .setClient(clientStub)
         .build();
 
+    clientStub.getObjectStore().createVolume(DEFAULT_VOLUME);
 
   }
 
@@ -60,6 +64,8 @@ public void testListBucket() throws Exception {
     }
     response = (ListBucketResponse) rootEndpoint.get().getEntity();
     assertEquals(10, response.getBucketsNum());
+    assertEquals("root", response.getOwner().getDisplayName());
+    assertEquals("root", response.getOwner().getId());
   }
 
 }