From e0994f23437c81deae20b66316b6724a044afcc5 Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Fri, 4 Sep 2020 13:35:10 -0700 Subject: [PATCH 1/3] HDDS-4211. [OFS] Better owner and group display for listing Ozone volumes and buckets Change-Id: I28a4d6dc8756c63b9ac242f283c3ed0b0f0b40c8 --- .../BasicRootedOzoneClientAdapterImpl.java | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index a4883350b588..09fc392a9f5e 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -663,12 +663,16 @@ private List listStatusVolume(String volumeStr, OFSPath ofsStartPath = new OFSPath(startPath); // list buckets in the volume OzoneVolume volume = objectStore.getVolume(volumeStr); + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser(volume.getOwner()); + String owner = ugi.getShortUserName(); + String group = getGroupName(ugi); Iterator iter = volume.listBuckets(null, ofsStartPath.getBucketName()); List res = new ArrayList<>(); while (iter.hasNext() && res.size() < numEntries) { OzoneBucket bucket = iter.next(); - res.add(getFileStatusAdapterForBucket(bucket, uri, username)); + res.add(getFileStatusAdapterForBucket(bucket, uri, owner, group)); if (recursive) { String pathStrNext = volumeStr + OZONE_URI_DELIMITER + bucket.getName(); res.addAll(listStatus(pathStrNext, recursive, startPath, @@ -944,6 +948,19 @@ private BlockLocation[] getBlockLocations(OzoneFileStatus fileStatus) { return blockLocations; } + /** + * Helper function to get the primary group name from a UGI. + * @param ugi UserGroupInformation + * @return String of the primary group name, empty String on exception. + */ + private static String getGroupName(UserGroupInformation ugi) { + try { + return ugi.getPrimaryGroupName(); + } catch (IOException ignored) { + return ""; + } + } + /** * Generate a FileStatusAdapter for a volume. * @param ozoneVolume OzoneVolume object @@ -959,11 +976,14 @@ private static FileStatusAdapter getFileStatusAdapterForVolume( ozoneVolume.getName(), pathStr); } Path path = new Path(pathStr); + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser(ozoneVolume.getOwner()); + String owner = ugi.getShortUserName(); + String group = getGroupName(ugi); return new FileStatusAdapter(0L, path, true, (short)0, 0L, ozoneVolume.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), - // TODO: Revisit owner and admin - ozoneVolume.getOwner(), ozoneVolume.getAdmin(), path, + owner, group, path, new BlockLocation[0] ); } @@ -972,24 +992,26 @@ private static FileStatusAdapter getFileStatusAdapterForVolume( * Generate a FileStatusAdapter for a bucket. * @param ozoneBucket OzoneBucket object. * @param uri Full URI to OFS root. + * @param owner Owner of the parent volume of the bucket. + * @param group Group of the parent volume of the bucket. * @return FileStatusAdapter for a bucket. */ private static FileStatusAdapter getFileStatusAdapterForBucket( - OzoneBucket ozoneBucket, URI uri, String username) { + OzoneBucket ozoneBucket, URI uri, String owner, String group) { String pathStr = uri.toString() + OZONE_URI_DELIMITER + ozoneBucket.getVolumeName() + OZONE_URI_DELIMITER + ozoneBucket.getName(); if (LOG.isDebugEnabled()) { - LOG.debug("getFileStatusAdapterForBucket: ozoneBucket={}, pathStr={}, " - + "username={}", ozoneBucket.getVolumeName() + OZONE_URI_DELIMITER - + ozoneBucket.getName(), pathStr, username); + LOG.debug("getFileStatusAdapterForBucket: ozoneBucket={}, pathStr={}", + ozoneBucket.getVolumeName() + OZONE_URI_DELIMITER + + ozoneBucket.getName(), pathStr); } Path path = new Path(pathStr); return new FileStatusAdapter(0L, path, true, (short)0, 0L, ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L, - FsPermission.getDirDefault().toShort(), // TODO: derive from ACLs later - // TODO: revisit owner and group - username, username, path, new BlockLocation[0]); + FsPermission.getDirDefault().toShort(), + // TODO: maybe derive owner and group from ACLs later + owner, group, path, new BlockLocation[0]); } /** From 245a16787cb67b31c30027c72eee7990c7a6e6ac Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Fri, 4 Sep 2020 14:19:49 -0700 Subject: [PATCH 2/3] Update test testListStatusRootAndVolumeNonRecursive. Change-Id: I5286aadd344dca405c062ed6c4942112b8754f8b --- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index 2c1e33697e47..0dd960ec8298 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -623,6 +623,11 @@ private void teardownVolumeBucketWithDir(Path bucketPath1) */ @Test public void testListStatusRootAndVolumeNonRecursive() throws Exception { + // Get owner and group of the user running this test + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + final String ownerShort = ugi.getShortUserName(); + final String group = ugi.getPrimaryGroupName(); + Path bucketPath1 = createRandomVolumeBucketWithDirs(); Path bucketPath2 = createRandomVolumeBucketWithDirs(); // listStatus("/volume/bucket") @@ -633,11 +638,17 @@ public void testListStatusRootAndVolumeNonRecursive() throws Exception { OZONE_URI_DELIMITER + new OFSPath(bucketPath1).getVolumeName()); FileStatus[] fileStatusVolume = ofs.listStatus(volume); Assert.assertEquals(1, fileStatusVolume.length); + Assert.assertEquals(ownerShort, fileStatusVolume[0].getOwner()); + Assert.assertEquals(group, fileStatusVolume[0].getGroup()); // listStatus("/") Path root = new Path(OZONE_URI_DELIMITER); FileStatus[] fileStatusRoot = ofs.listStatus(root); // Default volume "s3v" is created by OM during start up. Assert.assertEquals(2 + 1, fileStatusRoot.length); + for (FileStatus fileStatus : fileStatusRoot) { + Assert.assertEquals(ownerShort, fileStatus.getOwner()); + Assert.assertEquals(group, fileStatus.getGroup()); + } // Cleanup teardownVolumeBucketWithDir(bucketPath2); teardownVolumeBucketWithDir(bucketPath1); From 0ca182aaee8b52713cb54b50daa917284b79ec0c Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Wed, 9 Sep 2020 11:19:49 -0700 Subject: [PATCH 3/3] Remove inaccurate comment. Change-Id: If9a8f7106ad3a3eb3358e5118a4c08368588cb66 --- .../hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 09fc392a9f5e..848119d5f55c 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -1010,7 +1010,6 @@ private static FileStatusAdapter getFileStatusAdapterForBucket( return new FileStatusAdapter(0L, path, true, (short)0, 0L, ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), - // TODO: maybe derive owner and group from ACLs later owner, group, path, new BlockLocation[0]); }