apache · smengcl · May 27, 2022 · May 23, 2022 · May 23, 2022 · May 24, 2022
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -528,4 +528,16 @@ private OzoneConsts() {
       "/service/plugins/policies/service/";
 
   public static final String OZONE_TENANT_RANGER_POLICY_LABEL = "OzoneTenant";
+
+  /**
+   * The time (in ms) that AuthorizerLock try-lock operations would wait (by
+   * default, some can be overridden) before declaring timeout.
+   */
+  public static final long OZONE_TENANT_AUTHORIZER_LOCK_WAIT_MILLIS = 1000L;
+
+  /**
+   * The maximum length of accessId allowed when assigning new users to a
+   * tenant.
+   */
+  public static final int OZONE_MAXIMUM_ACCESS_ID_LENGTH = 100;
 }
diff --git a/hadoop-hdds/docs/content/feature/S3-Tenant-Commands.md b/hadoop-hdds/docs/content/feature/S3-Tenant-Commands.md
@@ -150,12 +150,13 @@ Both delegated and non-delegated tenant admin can assign and revoke **regular**
 The only difference between delegated tenant admin and non-delegated tenant admin is that delegated tenant admin can assign and revoke tenant **admins** in the tenant,
 while non-delegated tenant admin can't.
 
-Unless `--delegated=false` is specified, `ozone tenant assignadmin` assigns **delegated** tenant admins by default.
+By default, `ozone tenant assignadmin` assigns a **non-delegated** tenant admin.
+To assign a **delegated** tenant admin, specify `--delegated` or `-d`.
 
-It is possible to assign a user to be tenant admins in multiple tenants.
+It is possible to assign a user to be tenant admins in multiple tenants. Just a reminder, the user would have a different access ID under each tenant.
 
 ```shell
-ozone tenant user assignadmin <ACCESS_ID> --delegated=true --tenant=<TENANT_NAME>
+ozone tenant user assignadmin <ACCESS_ID> [-d|--delegated] --tenant=<TENANT_NAME>
 ```
 
 Example:

diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -158,6 +158,7 @@
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY;
 import static org.apache.hadoop.ozone.OzoneConsts.OLD_QUOTA_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_MAXIMUM_ACCESS_ID_LENGTH;
 
 import org.apache.logging.log4j.util.Strings;
 import org.apache.ratis.protocol.ClientId;
@@ -881,6 +882,10 @@ public S3SecretValue tenantAssignUserAccessId(
         "tenantId can't be null or empty.");
     Preconditions.checkArgument(Strings.isNotBlank(accessId),
         "accessId can't be null or empty.");
+    Preconditions.checkArgument(
+        accessId.length() <= OZONE_MAXIMUM_ACCESS_ID_LENGTH, "accessId length ("
+            + accessId.length() + ") exceeds the maximum length allowed ("
+            + OZONE_MAXIMUM_ACCESS_ID_LENGTH + ")");
     return ozoneManagerClient.tenantAssignUserAccessId(
         username, tenantId, accessId);
   }

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/OzoneTenant.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/OzoneTenant.java
@@ -28,8 +28,8 @@
  */
 public class OzoneTenant implements Tenant {
   private final String tenantId;
-  private List<String> tenantRoleNames;
-  private List<AccessPolicy> accessPolicies;
+  private final List<String> tenantRoleNames;
+  private final List<AccessPolicy> accessPolicies;
   private final AccountNameSpace accountNameSpace;
   private final BucketNameSpace bucketNameSpace;
 
@@ -85,4 +85,12 @@ public void removeTenantAccessRole(String roleName) {
   public List<String> getTenantRoles() {
     return tenantRoleNames;
   }
+
+  @Override
+  public String toString() {
+    return "OzoneTenant{" + "tenantId='" + tenantId + '\''
+        + ", tenantRoleNames=" + tenantRoleNames + ", accessPolicies="
+        + accessPolicies + ", accountNameSpace=" + accountNameSpace
+        + ", bucketNameSpace=" + bucketNameSpace + '}';
+  }
 }
diff --git a/...ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerAccessPolicy.java b/...ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/RangerAccessPolicy.java
@@ -303,4 +303,14 @@ private void updatePolicyJsonString() throws IOException {
             + "\"allowExceptions\":[]," + "\"denyPolicyItems\":[],"
             + "\"denyExceptions\":[]," + "\"service\":\"cm_ozone\"" + "}";
   }
+
+  @Override
+  public String toString() {
+    return "RangerAccessPolicy{" + "accessObject=" + accessObject
+        + ", policyMap=" + policyMap + ", roleList=" + roleList + ", policyID='"
+        + policyID + '\'' + ", policyJsonString='" + policyJsonString + '\''
+        + ", policyName='" + policyName + '\''
+        + ", lastPolicyUpdateTimeEpochMillis=" + lastPolicyUpdateTimeEpochMillis
+        + '}';
+  }
 }
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/Tenant.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/multitenant/Tenant.java
@@ -29,8 +29,8 @@
 public interface Tenant {
 
   /**
-   * A tenant is represnted by a globally unique TenantID.
-   * @return Tenant-ID.
+   * A tenant is represented by a globally unique tenant name.
+   * @return tenant name.
    */
   String getTenantId();
 

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -132,7 +132,15 @@ OZONE_LOG_DIR=/var/log/hadoop
 
 no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1
 
-OZONE-SITE.XML_ozone.om.ranger.https-address=http://ranger:6080
 OZONE-SITE.XML_ozone.om.multitenancy.enabled=true
+OZONE-SITE.XML_ozone.om.ranger.https-address=http://ranger:6080
+
 OZONE-SITE.XML_ozone.om.ranger.https.admin.api.user=admin
 OZONE-SITE.XML_ozone.om.ranger.https.admin.api.passwd=passwd
+
+# Note: ozone.om.kerberos.principal and ozone.om.kerberos.keytab.file
+# (which are required for the Multi-Tenancy Ranger Java client) are already
+# properly defined above.
+
+OZONE-SITE.XML_ozone.om.multitenancy.ranger.sync.interval=30s
+OZONE-SITE.XML_ozone.om.multitenancy.ranger.sync.timeout=10s
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/mockserverInitialization.json b/hadoop-ozone/dist/src/main/compose/ozonesecure/mockserverInitialization.json
@@ -70,5 +70,29 @@
     "httpResponse": {
       "body": "{\"startIndex\":0,\"pageSize\":200,\"totalCount\":13,\"resultSize\":13,\"sortType\":\"asc\",\"sortBy\":\"serviceId\",\"queryTimeMS\":1651104831041,\"services\":[{\"id\":7,\"guid\":\"b6cbaf6c-3911-4fa6-aeed-60dece4b111b\",\"isEnabled\":true,\"createdBy\":\"Admin\",\"updatedBy\":\"Admin\",\"createTime\":1651040438000,\"updateTime\":1651040438000,\"version\":1,\"type\":\"ozone\",\"name\":\"cm_ozone\",\"displayName\":\"cm_ozone\",\"description\":\"Ozone repo\",\"tagService\":\"cm_tag\",\"configs\":{\"setup.additional.default.policies\":\"true\",\"hadoop.security.authentication\":\"kerberos\",\"ozone.om.http-address\":\"http://localhost:9874\",\"default-policy.1.resource.key\":\"*\",\"ranger.plugin.audit.filters\":\"[ {'accessResult': 'DENIED', 'isAudited': true} ]\",\"default-policy.1.resource.volume\":\"s3v\",\"default-policy.1.resource.bucket\":\"*\",\"default-policy.1.policyItem.1.accessTypes\":\"all,create,write,read,list,delete\",\"tag.download.auth.users\":\"om\",\"default-policy.1.name\":\"S3_VOLUME_POLICY\",\"password\":\"*****\",\"policy.download.auth.users\":\"om\",\"hadoop.security.authorization\":\"true\",\"default-policy.1.policyItem.1.users\":\"hive\",\"username\":\"om\"},\"policyVersion\":5,\"policyUpdateTime\":1651040439000,\"tagVersion\":1,\"tagUpdateTime\":1651040438000}]}"
     }
+  },
+  {
+    "httpRequest": {
+      "path": "/service/plugins/services/7"
+    },
+    "httpResponse": {
+      "body": "{\"id\":7,\"guid\":\"2a83c846-31ed-4882-b987-57a4c7c28867\",\"isEnabled\":true,\"createdBy\":\"Admin\",\"updatedBy\":\"Admin\",\"createTime\":1649339219000,\"updateTime\":1649339219000,\"version\":1,\"type\":\"ozone\",\"name\":\"cm_ozone\",\"displayName\":\"cm_ozone\",\"description\":\"Ozone repo\",\"tagService\":\"cm_tag\",\"configs\":{\"setup.additional.default.policies\":\"true\",\"hadoop.security.authentication\":\"kerberos\",\"ozone.om.http-address\":\"http://localhost:9874\",\"default-policy.1.resource.key\":\"*\",\"ranger.plugin.audit.filters\":\"[ {'accessResult': 'DENIED', 'isAudited': true} ]\",\"default-policy.1.resource.volume\":\"s3v\",\"default-policy.1.resource.bucket\":\"*\",\"default-policy.1.policyItem.1.accessTypes\":\"all,create,write,read,list,delete\",\"tag.download.auth.users\":\"om\",\"default-policy.1.name\":\"S3_VOLUME_POLICY\",\"password\":\"*****\",\"policy.download.auth.users\":\"om\",\"hadoop.security.authorization\":\"true\",\"default-policy.1.policyItem.1.users\":\"hive\",\"username\":\"om\"},\"policyVersion\":744,\"policyUpdateTime\":1653427481000,\"tagVersion\":50,\"tagUpdateTime\":1653188038000}"
+    }
+  },
+  {
+    "httpRequest": {
+      "path": "/service/plugins/policies/service/7"
+    },
+    "httpResponse": {
+      "body": "{\"id\":7,\"guid\":\"2a83c846-31ed-4882-b987-57a4c7c28867\",\"isEnabled\":true,\"createdBy\":\"Admin\",\"updatedBy\":\"Admin\",\"createTime\":1649339219000,\"updateTime\":1649339219000,\"version\":1,\"type\":\"ozone\",\"name\":\"cm_ozone\",\"displayName\":\"cm_ozone\",\"description\":\"Ozone repo\",\"tagService\":\"cm_tag\",\"configs\":{\"setup.additional.default.policies\":\"true\",\"hadoop.security.authentication\":\"kerberos\",\"ozone.om.http-address\":\"http://localhost:9874\",\"default-policy.1.resource.key\":\"*\",\"ranger.plugin.audit.filters\":\"[ {'accessResult': 'DENIED', 'isAudited': true} ]\",\"default-policy.1.resource.volume\":\"s3v\",\"default-policy.1.resource.bucket\":\"*\",\"default-policy.1.policyItem.1.accessTypes\":\"all,create,write,read,list,delete\",\"tag.download.auth.users\":\"om\",\"default-policy.1.name\":\"S3_VOLUME_POLICY\",\"password\":\"*****\",\"policy.download.auth.users\":\"om\",\"hadoop.security.authorization\":\"true\",\"default-policy.1.policyItem.1.users\":\"hive\",\"username\":\"om\"},\"policyVersion\":790,\"policyUpdateTime\":1653607260000,\"tagVersion\":50,\"tagUpdateTime\":1653188038000}"
+    }
+  },
+  {
+    "httpRequest": {
+      "path": "/service/plugins/policies/444"
+    },
+    "httpResponse": {
+      "body": "{id: 444}"
+    }
   }
 ]
diff --git a/...-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.1.0-1.2.0/callback.sh b/...-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.1.0-1.2.0/callback.sh
@@ -70,7 +70,8 @@ with_old_version_downgraded() {
 
 with_new_version_finalized() {
   _check_hdds_mlvs 2
-  _check_om_mlvs 1
+  # In Ozone 1.2.0, OM has only one layout version.
+  _check_om_mlvs 0
 
   validate old1
   validate new1

diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-tenant.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-tenant.robot
@@ -97,11 +97,11 @@ Delete Tenant Failure Tenant Not Empty
 Create Tenant Failure with Regular User
     Run Keyword         Kinit test user     testuser2    testuser2.keytab
     ${rc}  ${output} =  Run And Return Rc And Output  ozone tenant create tenanttwo
-                        Should contain   ${output}         PERMISSION_DENIED User 'testuser2/[email protected]' is not an Ozone admin.
+                        Should contain   ${output}         PERMISSION_DENIED User 'testuser2/[email protected]' or 'testuser2' is not an Ozone admin
 
 SetSecret Failure with Regular User
     ${rc}  ${output} =  Run And Return Rc And Output  ozone tenant user set-secret 'tenantone$testuser' --secret=somesecret2
-                        Should contain   ${output}         Permission denied. Requested accessId
+                        Should contain   ${output}         USER_MISMATCH Requested accessId 'tenantone$testuser' doesn't belong to current user 'testuser2/[email protected]', nor does current user have Ozone or tenant administrator privilege
 
 Create Bucket 2 Success with somesecret1 via S3 API
     ${output} =         Execute          aws s3api --endpoint-url ${S3G_ENDPOINT_URL} create-bucket --bucket bucket-test2

diff --git a/...a/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantAccessAuthorizerRangerPlugin.java b/...a/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantAccessAuthorizerRangerPlugin.java
@@ -38,7 +38,6 @@
 import org.apache.hadoop.ozone.om.OMMultiTenantManager;
 import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
 import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
-import org.apache.http.auth.BasicUserPrincipal;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -105,7 +104,7 @@ public void testMultiTenantAccessAuthorizerRangerPlugin() throws Exception {
       OzoneTenantRolePrincipal userRole =
           new OzoneTenantRolePrincipal("tenant1-UserRole");
 
-      BasicUserPrincipal userPrincipal = new BasicUserPrincipal("user1Test");
+      String userPrincipal = "user1Test";
       usersIdsCreated.add(
           omm.assignUserToRole(userPrincipal, userRole.getName(), false));
       usersIdsCreated.add(
@@ -137,7 +136,7 @@ public void testMultiTenantAccessAuthorizerRangerPlugin() throws Exception {
         omm.deleteUser(id);
       }
       for (String id : groupIdsCreated) {
-        omm.deleteRole(id);
+        omm.deleteRoleById(id);
       }
     }
   }
@@ -146,7 +145,7 @@ public void testMultiTenantAccessAuthorizerRangerPlugin() throws Exception {
   @Ignore("TODO:Requires (mocked) Ranger endpoint")
   public void testMultiTenantAccessAuthorizerRangerPluginWithoutIds()
       throws Exception {
-    BasicUserPrincipal userPrincipal = null;
+    String userPrincipal = null;
     simulateOzoneSiteXmlConfig();
     final MultiTenantAccessAuthorizer omm =
         new MultiTenantAccessAuthorizerRangerPlugin();
@@ -163,7 +162,7 @@ public void testMultiTenantAccessAuthorizerRangerPluginWithoutIds()
       omm.createRole(group2Principal.getName(), group1Principal.getName());
       groupIdsCreated.add(omm.getRole(group2Principal));
 
-      userPrincipal = new BasicUserPrincipal("user1Test");
+      userPrincipal = "user1Test";
       omm.assignUserToRole(userPrincipal, group2Principal.getName(), false);
 
       AccessPolicy tenant1VolumeAccessPolicy = createVolumeAccessPolicy(
@@ -195,7 +194,7 @@ public void testMultiTenantAccessAuthorizerRangerPluginWithoutIds()
       String userId = omm.getUserId(userPrincipal);
       omm.deleteUser(userId);
       for (String id : groupIdsCreated) {
-        omm.deleteRole(id);
+        omm.deleteRoleById(id);
       }
     }
   }

diff --git a/...on-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestRangerBGSyncService.java b/...on-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestRangerBGSyncService.java
@@ -69,7 +69,6 @@
 import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.util.KerberosName;
-import org.apache.http.auth.BasicUserPrincipal;
 import org.apache.ozone.test.GenericTestUtils;
 import org.apache.ratis.protocol.RaftGroupId;
 import org.apache.ratis.protocol.RaftPeerId;
@@ -123,7 +122,7 @@ public class TestRangerBGSyncService {
   // List of role ID created in Ranger
   private final List<String> rolesCreated = new ArrayList<>();
   // List of users created in Ranger
-  private final List<BasicUserPrincipal> usersCreated = new ArrayList<>();
+  private final List<String> usersCreated = new ArrayList<>();
 
   private static OzoneConfiguration conf;
   private OzoneManager ozoneManager;
@@ -230,6 +229,8 @@ public void setUp() throws IOException {
     when(omMultiTenantManager.newDefaultBucketAccessPolicy(eq(TENANT_ID),
         Mockito.any(OzoneTenantRolePrincipal.class)))
         .thenReturn(newBucketAccessPolicy(TENANT_ID, TENANT_ID));
+    when(omMultiTenantManager.getAuthorizerLock())
+        .thenReturn(new AuthorizerLockImpl());
 
     // Raft client request handling
     OzoneManagerRatisServer omRatisServer = mock(OzoneManagerRatisServer.class);
@@ -301,7 +302,8 @@ private AccessPolicy newBucketAccessPolicy(String vol, String tenantId)
   }
 
   long initBGSync() throws IOException {
-    bgSync = new OMRangerBGSyncService(ozoneManager, auth,
+    bgSync = new OMRangerBGSyncService(ozoneManager,
+        ozoneManager.getMultiTenantManager(), auth,
         TEST_SYNC_INTERVAL_SEC, TimeUnit.SECONDS, TEST_SYNC_TIMEOUT_SEC);
     return bgSync.getLatestRangerServiceVersion();
   }
@@ -311,8 +313,6 @@ public void createRolesAndPoliciesInRanger(boolean populateDB) {
     policiesCreated.clear();
     rolesCreated.clear();
 
-    BasicUserPrincipal userAlice = new BasicUserPrincipal(USER_ALICE_SHORT);
-    BasicUserPrincipal userBob = new BasicUserPrincipal(USER_BOB_SHORT);
     // Tenant name to be used for this test
     final String tenantId = TENANT_ID;
     // volume name = bucket namespace name
@@ -338,21 +338,21 @@ public void createRolesAndPoliciesInRanger(boolean populateDB) {
                 bucketNamespacePolicyName, bucketPolicyName));
         // Access ID entry for alice
         final String aliceAccessId = OMMultiTenantManager.getDefaultAccessId(
-            tenantId, userAlice.getName());
+            tenantId, USER_ALICE_SHORT);
         omMetadataManager.getTenantAccessIdTable().put(aliceAccessId,
             new OmDBAccessIdInfo.Builder()
                 .setTenantId(tenantId)
-                .setUserPrincipal(userAlice.getName())
+                .setUserPrincipal(USER_ALICE_SHORT)
                 .setIsAdmin(false)
                 .setIsDelegatedAdmin(false)
                 .build());
         // Access ID entry for bob
         final String bobAccessId = OMMultiTenantManager.getDefaultAccessId(
-            tenantId, userBob.getName());
+            tenantId, USER_BOB_SHORT);
         omMetadataManager.getTenantAccessIdTable().put(bobAccessId,
             new OmDBAccessIdInfo.Builder()
                 .setTenantId(tenantId)
-                .setUserPrincipal(userBob.getName())
+                .setUserPrincipal(USER_BOB_SHORT)
                 .setIsAdmin(false)
                 .setIsDelegatedAdmin(false)
                 .build());
@@ -383,17 +383,17 @@ public void createRolesAndPoliciesInRanger(boolean populateDB) {
     try {
       LOG.info("Creating user in Ranger: {}", USER_ALICE_SHORT);
       auth.createUser(USER_ALICE_SHORT, "password1");
-      usersCreated.add(userAlice);
-      auth.assignUserToRole(userAlice, auth.getRole(userRole), false);
+      usersCreated.add(USER_ALICE_SHORT);
+      auth.assignUserToRole(USER_ALICE_SHORT, auth.getRole(userRole), false);
     } catch (Exception e) {
       Assert.fail(e.getMessage());
     }
 
     try {
       LOG.info("Creating user in Ranger: {}", USER_BOB_SHORT);
       auth.createUser(USER_BOB_SHORT, "password2");
-      usersCreated.add(userBob);
-      auth.assignUserToRole(userBob, auth.getRole(userRole), false);
+      usersCreated.add(USER_BOB_SHORT);
+      auth.assignUserToRole(USER_BOB_SHORT, auth.getRole(userRole), false);
     } catch (Exception e) {
       Assert.fail(e.getMessage());
     }
@@ -439,15 +439,15 @@ public void cleanupRoles() {
       final String roleName = jObj.get("name").getAsString();
       try {
         LOG.info("Deleting role: {}", roleName);
-        auth.deleteRole(roleId);
+        auth.deleteRoleById(roleId);
       } catch (Exception e) {
         LOG.error(e.getMessage());
       }
     }
   }
 
   public void cleanupUsers() {
-    for (BasicUserPrincipal user : usersCreated) {
+    for (String user : usersCreated) {
       try {
         LOG.info("Deleting user: {}", user);
         String userId = auth.getUserId(user);
@@ -604,8 +604,7 @@ public void testRecoverRangerRole() throws Exception {
     Assert.assertEquals(
         OMMultiTenantManager.getDefaultUserRoleName(TENANT_ID), userRoleName);
 
-    auth.revokeUserFromRole(
-        new BasicUserPrincipal(USER_BOB_SHORT), auth.getRole(userRoleName));
+    auth.revokeUserFromRole(USER_BOB_SHORT, auth.getRole(userRoleName));
 
     HashSet<String> userSet = new HashSet<>();
     userSet.add(USER_ALICE_SHORT);